Compare commits

...

2 commits

Author SHA1 Message Date
770384cae1 Merge pull request 'add rpc_public_addr_subnet config option' (#817) from flokli/garage:rpc_public_addr_subnet into main
All checks were successful
ci/woodpecker/push/debug Pipeline was successful
Reviewed-on: #817
Reviewed-by: Alex <alex@adnab.me>
2024-06-18 12:40:07 +00:00
a0f6bc5b7f add rpc_public_addr_subnet config option
All checks were successful
ci/woodpecker/pr/debug Pipeline was successful
In case `rpc_public_addr` is not set, but autodiscovery is used, this
allows filtering the list of automatically discovered IPs to a specific
subnet.

For example, if nodes should pick *their* IP inside a specific subnet,
but you don't want to explicitly write the IP down (as it's dynamic, or
you want to share configs across nodes), you can use this option.
2024-06-05 08:41:36 +02:00
8 changed files with 60 additions and 9 deletions

1
Cargo.lock generated
View file

@ -1527,6 +1527,7 @@ dependencies = [
"garage_util", "garage_util",
"gethostname", "gethostname",
"hex", "hex",
"ipnet",
"itertools 0.12.1", "itertools 0.12.1",
"k8s-openapi", "k8s-openapi",
"kube", "kube",

View file

@ -34,7 +34,7 @@ args@{
ignoreLockHash, ignoreLockHash,
}: }:
let let
nixifiedLockHash = "1ccd5eb25a83962821e0e9da4ce6df31717b2b97a5b3a0c80c9e0e0759710143"; nixifiedLockHash = "fc41fb639a69d62c8c0fb3f9c227162162ebc8142c6fa5cd0599dc381dcd9ebb";
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
lockHashIgnored = if ignoreLockHash lockHashIgnored = if ignoreLockHash
@ -2219,6 +2219,7 @@ in
garage_util = (rustPackages."unknown".garage_util."1.0.0" { inherit profileName; }).out; garage_util = (rustPackages."unknown".garage_util."1.0.0" { inherit profileName; }).out;
gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out; gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
ipnet = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnet."2.9.0" { inherit profileName; }).out;
itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out; itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out;
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.21.0" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.21.0" { inherit profileName; }).out;
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.88.1" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.88.1" { inherit profileName; }).out;
@ -3016,8 +3017,8 @@ in
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"; }; src = fetchCratesIo { inherit name version; sha256 = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"; };
features = builtins.concatLists [ features = builtins.concatLists [
(lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default") [ "default" ]
(lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "std") [ "std" ]
]; ];
}); });

View file

@ -55,6 +55,7 @@ hexdump = "0.1"
hmac = "0.12" hmac = "0.12"
idna = "0.5" idna = "0.5"
itertools = "0.12" itertools = "0.12"
ipnet = "2.9.0"
lazy_static = "1.4" lazy_static = "1.4"
md-5 = "0.10" md-5 = "0.10"
mktemp = "0.5" mktemp = "0.5"

View file

@ -152,6 +152,8 @@ Check the following for your configuration files:
- Make sure `rpc_public_addr` contains the public IP address of the node you are configuring. - Make sure `rpc_public_addr` contains the public IP address of the node you are configuring.
This parameter is optional but recommended: if your nodes have trouble communicating with This parameter is optional but recommended: if your nodes have trouble communicating with
one another, consider adding it. one another, consider adding it.
Alternatively, you can also set `rpc_public_addr_subnet`, which can filter
the addresses announced to other peers to a specific subnet.
- Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key. - Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key.
You can generate such a key with `openssl rand -hex 32`. You can generate such a key with `openssl rand -hex 32`.

View file

@ -31,6 +31,9 @@ rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6"
rpc_bind_addr = "[::]:3901" rpc_bind_addr = "[::]:3901"
rpc_bind_outgoing = false rpc_bind_outgoing = false
rpc_public_addr = "[fc00:1::1]:3901" rpc_public_addr = "[fc00:1::1]:3901"
# or set rpc_public_adr_subnet to filter down autodiscovery to a subnet:
# rpc_public_addr_subnet = "2001:0db8:f00:b00:/64"
allow_world_readable_secrets = false allow_world_readable_secrets = false
@ -105,6 +108,7 @@ Top-level configuration options:
[`rpc_bind_addr`](#rpc_bind_addr), [`rpc_bind_addr`](#rpc_bind_addr),
[`rpc_bind_outgoing`](#rpc_bind_outgoing), [`rpc_bind_outgoing`](#rpc_bind_outgoing),
[`rpc_public_addr`](#rpc_public_addr), [`rpc_public_addr`](#rpc_public_addr),
[`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
[`rpc_secret`/`rpc_secret_file`](#rpc_secret). [`rpc_secret`/`rpc_secret_file`](#rpc_secret).
The `[consul_discovery]` section: The `[consul_discovery]` section:
@ -543,6 +547,14 @@ RPC calls. **This parameter is optional but recommended.** In case you have
a NAT that binds the RPC port to a port that is different on your public IP, a NAT that binds the RPC port to a port that is different on your public IP,
this field might help making it work. this field might help making it work.
#### `rpc_public_addr_subnet` {#rpc_public_addr_subnet}
In case `rpc_public_addr` is not set, but autodiscovery is used, this allows
filtering the list of automatically discovered IPs to a specific subnet.
For example, if nodes should pick *their* IP inside a specific subnet, but you
don't want to explicitly write the IP down (as it's dynamic, or you want to
share configs across nodes), you can use this option.
#### `bootstrap_peers` {#bootstrap_peers} #### `bootstrap_peers` {#bootstrap_peers}
A list of peer identifiers on which to contact other Garage peers of this cluster. A list of peer identifiers on which to contact other Garage peers of this cluster.

View file

@ -24,6 +24,7 @@ bytes.workspace = true
bytesize.workspace = true bytesize.workspace = true
gethostname.workspace = true gethostname.workspace = true
hex.workspace = true hex.workspace = true
ipnet.workspace = true
tracing.workspace = true tracing.workspace = true
rand.workspace = true rand.workspace = true
itertools.workspace = true itertools.workspace = true

View file

@ -844,12 +844,20 @@ impl NodeStatus {
} }
} }
fn get_default_ip() -> Option<IpAddr> { /// Obtain the list of currently available IP addresses on all non-loopback
/// interfaces, optionally filtering them to be inside a given IpNet.
fn get_default_ip(filter_ipnet: Option<ipnet::IpNet>) -> Option<IpAddr> {
pnet_datalink::interfaces() pnet_datalink::interfaces()
.iter() .into_iter()
.find(|e| e.is_up() && !e.is_loopback() && !e.ips.is_empty()) // filter down and loopback interfaces
.and_then(|e| e.ips.first()) .filter(|i| i.is_up() && !i.is_loopback())
.map(|a| a.ip()) // get all IPs
.flat_map(|e| e.ips)
// optionally, filter to be inside filter_ipnet
.find(|ipn| {
filter_ipnet.is_some_and(|ipnet| ipnet.contains(&ipn.ip())) || filter_ipnet.is_none()
})
.map(|ipn| ipn.ip())
} }
fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> { fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> {
@ -877,7 +885,28 @@ fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> {
} }
} }
None => { None => {
let addr = get_default_ip().map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port())); // `No rpc_public_addr` specified, try to discover one, optionally filtering by `rpc_public_addr_subnet`.
let filter_subnet: Option<ipnet::IpNet> = config
.rpc_public_addr_subnet
.as_ref()
.and_then(|filter_subnet_str| match filter_subnet_str.parse::<ipnet::IpNet>() {
Ok(filter_subnet) => {
let filter_subnet_trunc = filter_subnet.trunc();
if filter_subnet_trunc != filter_subnet {
warn!("`rpc_public_addr_subnet` changed after applying netmask, continuing with {}", filter_subnet.trunc());
}
Some(filter_subnet_trunc)
}
Err(e) => {
panic!(
"Cannot parse rpc_public_addr_subnet {} from config file: {}. Bailing out.",
filter_subnet_str, e
);
}
});
let addr = get_default_ip(filter_subnet)
.map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port()));
if let Some(a) = addr { if let Some(a) = addr {
warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a); warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a);
} }

View file

@ -85,6 +85,10 @@ pub struct Config {
/// Public IP address of this node /// Public IP address of this node
pub rpc_public_addr: Option<String>, pub rpc_public_addr: Option<String>,
/// In case `rpc_public_addr` was not set, this can filter
/// the addresses announced to other peers to a specific subnet.
pub rpc_public_addr_subnet: Option<String>,
/// Timeout for Netapp's ping messagess /// Timeout for Netapp's ping messagess
pub rpc_ping_timeout_msec: Option<u64>, pub rpc_ping_timeout_msec: Option<u64>,
/// Timeout for Netapp RPC calls /// Timeout for Netapp RPC calls