From a0f6bc5b7faa0d557179a7c4ed4c8d0facb9afa4 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Tue, 23 Apr 2024 11:57:43 +0300 Subject: add rpc_public_addr_subnet config option In case `rpc_public_addr` is not set, but autodiscovery is used, this allows filtering the list of automatically discovered IPs to a specific subnet. For example, if nodes should pick *their* IP inside a specific subnet, but you don't want to explicitly write the IP down (as it's dynamic, or you want to share configs across nodes), you can use this option. --- Cargo.lock | 1 + Cargo.nix | 7 ++--- Cargo.toml | 1 + doc/book/cookbook/real-world.md | 2 ++ doc/book/reference-manual/configuration.md | 12 +++++++++ src/rpc/Cargo.toml | 1 + src/rpc/system.rs | 41 +++++++++++++++++++++++++----- src/util/config.rs | 4 +++ 8 files changed, 60 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b96483dd..9cb4b57e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1527,6 +1527,7 @@ dependencies = [ "garage_util", "gethostname", "hex", + "ipnet", "itertools 0.12.1", "k8s-openapi", "kube", diff --git a/Cargo.nix b/Cargo.nix index e0189cdd..c2be3161 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -34,7 +34,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "1ccd5eb25a83962821e0e9da4ce6df31717b2b97a5b3a0c80c9e0e0759710143"; + nixifiedLockHash = "fc41fb639a69d62c8c0fb3f9c227162162ebc8142c6fa5cd0599dc381dcd9ebb"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -2219,6 +2219,7 @@ in garage_util = (rustPackages."unknown".garage_util."1.0.0" { inherit profileName; }).out; gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; + ipnet = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnet."2.9.0" { inherit profileName; }).out; itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.21.0" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.88.1" { inherit profileName; }).out; @@ -3016,8 +3017,8 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "std") + [ "default" ] + [ "std" ] ]; }); diff --git a/Cargo.toml b/Cargo.toml index b8840a91..500079f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ hexdump = "0.1" hmac = "0.12" idna = "0.5" itertools = "0.12" +ipnet = "2.9.0" lazy_static = "1.4" md-5 = "0.10" mktemp = "0.5" diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md index 7dba784d..48d2b369 100644 --- a/doc/book/cookbook/real-world.md +++ b/doc/book/cookbook/real-world.md @@ -152,6 +152,8 @@ Check the following for your configuration files: - Make sure `rpc_public_addr` contains the public IP address of the node you are configuring. This parameter is optional but recommended: if your nodes have trouble communicating with one another, consider adding it. + Alternatively, you can also set `rpc_public_addr_subnet`, which can filter + the addresses announced to other peers to a specific subnet. - Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key. You can generate such a key with `openssl rand -hex 32`. diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index 423795fe..20821a5b 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -31,6 +31,9 @@ rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6" rpc_bind_addr = "[::]:3901" rpc_bind_outgoing = false rpc_public_addr = "[fc00:1::1]:3901" +# or set rpc_public_adr_subnet to filter down autodiscovery to a subnet: +# rpc_public_addr_subnet = "2001:0db8:f00:b00:/64" + allow_world_readable_secrets = false @@ -105,6 +108,7 @@ Top-level configuration options: [`rpc_bind_addr`](#rpc_bind_addr), [`rpc_bind_outgoing`](#rpc_bind_outgoing), [`rpc_public_addr`](#rpc_public_addr), +[`rpc_public_addr_subnet`](#rpc_public_addr_subnet) [`rpc_secret`/`rpc_secret_file`](#rpc_secret). The `[consul_discovery]` section: @@ -543,6 +547,14 @@ RPC calls. **This parameter is optional but recommended.** In case you have a NAT that binds the RPC port to a port that is different on your public IP, this field might help making it work. +#### `rpc_public_addr_subnet` {#rpc_public_addr_subnet} +In case `rpc_public_addr` is not set, but autodiscovery is used, this allows +filtering the list of automatically discovered IPs to a specific subnet. + +For example, if nodes should pick *their* IP inside a specific subnet, but you +don't want to explicitly write the IP down (as it's dynamic, or you want to +share configs across nodes), you can use this option. + #### `bootstrap_peers` {#bootstrap_peers} A list of peer identifiers on which to contact other Garage peers of this cluster. diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 43d5568e..4c8cafd9 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -24,6 +24,7 @@ bytes.workspace = true bytesize.workspace = true gethostname.workspace = true hex.workspace = true +ipnet.workspace = true tracing.workspace = true rand.workspace = true itertools.workspace = true diff --git a/src/rpc/system.rs b/src/rpc/system.rs index 0e78060b..d94d4eec 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -844,12 +844,20 @@ impl NodeStatus { } } -fn get_default_ip() -> Option { +/// Obtain the list of currently available IP addresses on all non-loopback +/// interfaces, optionally filtering them to be inside a given IpNet. +fn get_default_ip(filter_ipnet: Option) -> Option { pnet_datalink::interfaces() - .iter() - .find(|e| e.is_up() && !e.is_loopback() && !e.ips.is_empty()) - .and_then(|e| e.ips.first()) - .map(|a| a.ip()) + .into_iter() + // filter down and loopback interfaces + .filter(|i| i.is_up() && !i.is_loopback()) + // get all IPs + .flat_map(|e| e.ips) + // optionally, filter to be inside filter_ipnet + .find(|ipn| { + filter_ipnet.is_some_and(|ipnet| ipnet.contains(&ipn.ip())) || filter_ipnet.is_none() + }) + .map(|ipn| ipn.ip()) } fn get_rpc_public_addr(config: &Config) -> Option { @@ -877,7 +885,28 @@ fn get_rpc_public_addr(config: &Config) -> Option { } } None => { - let addr = get_default_ip().map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port())); + // `No rpc_public_addr` specified, try to discover one, optionally filtering by `rpc_public_addr_subnet`. + let filter_subnet: Option = config + .rpc_public_addr_subnet + .as_ref() + .and_then(|filter_subnet_str| match filter_subnet_str.parse::() { + Ok(filter_subnet) => { + let filter_subnet_trunc = filter_subnet.trunc(); + if filter_subnet_trunc != filter_subnet { + warn!("`rpc_public_addr_subnet` changed after applying netmask, continuing with {}", filter_subnet.trunc()); + } + Some(filter_subnet_trunc) + } + Err(e) => { + panic!( + "Cannot parse rpc_public_addr_subnet {} from config file: {}. Bailing out.", + filter_subnet_str, e + ); + } + }); + + let addr = get_default_ip(filter_subnet) + .map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port())); if let Some(a) = addr { warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a); } diff --git a/src/util/config.rs b/src/util/config.rs index 028f8c68..59329c0b 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -85,6 +85,10 @@ pub struct Config { /// Public IP address of this node pub rpc_public_addr: Option, + /// In case `rpc_public_addr` was not set, this can filter + /// the addresses announced to other peers to a specific subnet. + pub rpc_public_addr_subnet: Option, + /// Timeout for Netapp's ping messagess pub rpc_ping_timeout_msec: Option, /// Timeout for Netapp RPC calls -- cgit v1.2.3