add rpc_public_addr_subnet config option

In case `rpc_public_addr` is not set, but autodiscovery is used, this
allows filtering the list of automatically discovered IPs to a specific
subnet.

For example, if nodes should pick *their* IP inside a specific subnet,
but you don't want to explicitly write the IP down (as it's dynamic, or
you want to share configs across nodes), you can use this option.
This commit is contained in:
Florian Klink 2024-04-23 11:57:43 +03:00
parent a2c1de646b
commit a0f6bc5b7f
8 changed files with 60 additions and 9 deletions

1
Cargo.lock generated
View file

@ -1527,6 +1527,7 @@ dependencies = [
"garage_util", "garage_util",
"gethostname", "gethostname",
"hex", "hex",
"ipnet",
"itertools 0.12.1", "itertools 0.12.1",
"k8s-openapi", "k8s-openapi",
"kube", "kube",

View file

@ -34,7 +34,7 @@ args@{
ignoreLockHash, ignoreLockHash,
}: }:
let let
nixifiedLockHash = "1ccd5eb25a83962821e0e9da4ce6df31717b2b97a5b3a0c80c9e0e0759710143"; nixifiedLockHash = "fc41fb639a69d62c8c0fb3f9c227162162ebc8142c6fa5cd0599dc381dcd9ebb";
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
lockHashIgnored = if ignoreLockHash lockHashIgnored = if ignoreLockHash
@ -2219,6 +2219,7 @@ in
garage_util = (rustPackages."unknown".garage_util."1.0.0" { inherit profileName; }).out; garage_util = (rustPackages."unknown".garage_util."1.0.0" { inherit profileName; }).out;
gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out; gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
ipnet = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnet."2.9.0" { inherit profileName; }).out;
itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out; itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out;
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.21.0" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.21.0" { inherit profileName; }).out;
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.88.1" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.88.1" { inherit profileName; }).out;
@ -3016,8 +3017,8 @@ in
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"; }; src = fetchCratesIo { inherit name version; sha256 = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"; };
features = builtins.concatLists [ features = builtins.concatLists [
(lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default") [ "default" ]
(lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "std") [ "std" ]
]; ];
}); });

View file

@ -55,6 +55,7 @@ hexdump = "0.1"
hmac = "0.12" hmac = "0.12"
idna = "0.5" idna = "0.5"
itertools = "0.12" itertools = "0.12"
ipnet = "2.9.0"
lazy_static = "1.4" lazy_static = "1.4"
md-5 = "0.10" md-5 = "0.10"
mktemp = "0.5" mktemp = "0.5"

View file

@ -152,6 +152,8 @@ Check the following for your configuration files:
- Make sure `rpc_public_addr` contains the public IP address of the node you are configuring. - Make sure `rpc_public_addr` contains the public IP address of the node you are configuring.
This parameter is optional but recommended: if your nodes have trouble communicating with This parameter is optional but recommended: if your nodes have trouble communicating with
one another, consider adding it. one another, consider adding it.
Alternatively, you can also set `rpc_public_addr_subnet`, which can filter
the addresses announced to other peers to a specific subnet.
- Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key. - Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key.
You can generate such a key with `openssl rand -hex 32`. You can generate such a key with `openssl rand -hex 32`.

View file

@ -31,6 +31,9 @@ rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6"
rpc_bind_addr = "[::]:3901" rpc_bind_addr = "[::]:3901"
rpc_bind_outgoing = false rpc_bind_outgoing = false
rpc_public_addr = "[fc00:1::1]:3901" rpc_public_addr = "[fc00:1::1]:3901"
# or set rpc_public_adr_subnet to filter down autodiscovery to a subnet:
# rpc_public_addr_subnet = "2001:0db8:f00:b00:/64"
allow_world_readable_secrets = false allow_world_readable_secrets = false
@ -105,6 +108,7 @@ Top-level configuration options:
[`rpc_bind_addr`](#rpc_bind_addr), [`rpc_bind_addr`](#rpc_bind_addr),
[`rpc_bind_outgoing`](#rpc_bind_outgoing), [`rpc_bind_outgoing`](#rpc_bind_outgoing),
[`rpc_public_addr`](#rpc_public_addr), [`rpc_public_addr`](#rpc_public_addr),
[`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
[`rpc_secret`/`rpc_secret_file`](#rpc_secret). [`rpc_secret`/`rpc_secret_file`](#rpc_secret).
The `[consul_discovery]` section: The `[consul_discovery]` section:
@ -543,6 +547,14 @@ RPC calls. **This parameter is optional but recommended.** In case you have
a NAT that binds the RPC port to a port that is different on your public IP, a NAT that binds the RPC port to a port that is different on your public IP,
this field might help making it work. this field might help making it work.
#### `rpc_public_addr_subnet` {#rpc_public_addr_subnet}
In case `rpc_public_addr` is not set, but autodiscovery is used, this allows
filtering the list of automatically discovered IPs to a specific subnet.
For example, if nodes should pick *their* IP inside a specific subnet, but you
don't want to explicitly write the IP down (as it's dynamic, or you want to
share configs across nodes), you can use this option.
#### `bootstrap_peers` {#bootstrap_peers} #### `bootstrap_peers` {#bootstrap_peers}
A list of peer identifiers on which to contact other Garage peers of this cluster. A list of peer identifiers on which to contact other Garage peers of this cluster.

View file

@ -24,6 +24,7 @@ bytes.workspace = true
bytesize.workspace = true bytesize.workspace = true
gethostname.workspace = true gethostname.workspace = true
hex.workspace = true hex.workspace = true
ipnet.workspace = true
tracing.workspace = true tracing.workspace = true
rand.workspace = true rand.workspace = true
itertools.workspace = true itertools.workspace = true

View file

@ -844,12 +844,20 @@ impl NodeStatus {
} }
} }
fn get_default_ip() -> Option<IpAddr> { /// Obtain the list of currently available IP addresses on all non-loopback
/// interfaces, optionally filtering them to be inside a given IpNet.
fn get_default_ip(filter_ipnet: Option<ipnet::IpNet>) -> Option<IpAddr> {
pnet_datalink::interfaces() pnet_datalink::interfaces()
.iter() .into_iter()
.find(|e| e.is_up() && !e.is_loopback() && !e.ips.is_empty()) // filter down and loopback interfaces
.and_then(|e| e.ips.first()) .filter(|i| i.is_up() && !i.is_loopback())
.map(|a| a.ip()) // get all IPs
.flat_map(|e| e.ips)
// optionally, filter to be inside filter_ipnet
.find(|ipn| {
filter_ipnet.is_some_and(|ipnet| ipnet.contains(&ipn.ip())) || filter_ipnet.is_none()
})
.map(|ipn| ipn.ip())
} }
fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> { fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> {
@ -877,7 +885,28 @@ fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> {
} }
} }
None => { None => {
let addr = get_default_ip().map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port())); // `No rpc_public_addr` specified, try to discover one, optionally filtering by `rpc_public_addr_subnet`.
let filter_subnet: Option<ipnet::IpNet> = config
.rpc_public_addr_subnet
.as_ref()
.and_then(|filter_subnet_str| match filter_subnet_str.parse::<ipnet::IpNet>() {
Ok(filter_subnet) => {
let filter_subnet_trunc = filter_subnet.trunc();
if filter_subnet_trunc != filter_subnet {
warn!("`rpc_public_addr_subnet` changed after applying netmask, continuing with {}", filter_subnet.trunc());
}
Some(filter_subnet_trunc)
}
Err(e) => {
panic!(
"Cannot parse rpc_public_addr_subnet {} from config file: {}. Bailing out.",
filter_subnet_str, e
);
}
});
let addr = get_default_ip(filter_subnet)
.map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port()));
if let Some(a) = addr { if let Some(a) = addr {
warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a); warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a);
} }

View file

@ -85,6 +85,10 @@ pub struct Config {
/// Public IP address of this node /// Public IP address of this node
pub rpc_public_addr: Option<String>, pub rpc_public_addr: Option<String>,
/// In case `rpc_public_addr` was not set, this can filter
/// the addresses announced to other peers to a specific subnet.
pub rpc_public_addr_subnet: Option<String>,
/// Timeout for Netapp's ping messagess /// Timeout for Netapp's ping messagess
pub rpc_ping_timeout_msec: Option<u64>, pub rpc_ping_timeout_msec: Option<u64>,
/// Timeout for Netapp RPC calls /// Timeout for Netapp RPC calls