diff options
-rw-r--r-- | Cargo.lock | 1 | ||||
-rw-r--r-- | Cargo.nix | 7 | ||||
-rw-r--r-- | Cargo.toml | 1 | ||||
-rw-r--r-- | doc/book/cookbook/real-world.md | 2 | ||||
-rw-r--r-- | doc/book/design/related-work.md | 2 | ||||
-rw-r--r-- | doc/book/reference-manual/configuration.md | 14 | ||||
-rw-r--r-- | flake.nix | 1 | ||||
-rw-r--r-- | script/helm/garage/templates/workload.yaml | 5 | ||||
-rw-r--r-- | script/helm/garage/values.yaml | 4 | ||||
-rw-r--r-- | shell.nix | 1 | ||||
-rw-r--r-- | src/api/s3/post_object.rs | 22 | ||||
-rw-r--r-- | src/db/open.rs | 1 | ||||
-rw-r--r-- | src/garage/cli/structs.rs | 2 | ||||
-rw-r--r-- | src/model/garage.rs | 2 | ||||
-rw-r--r-- | src/rpc/Cargo.toml | 1 | ||||
-rw-r--r-- | src/rpc/system.rs | 41 | ||||
-rw-r--r-- | src/util/config.rs | 4 |
17 files changed, 83 insertions, 28 deletions
@@ -1527,6 +1527,7 @@ dependencies = [ "garage_util", "gethostname", "hex", + "ipnet", "itertools 0.12.1", "k8s-openapi", "kube", @@ -34,7 +34,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "1ccd5eb25a83962821e0e9da4ce6df31717b2b97a5b3a0c80c9e0e0759710143"; + nixifiedLockHash = "fc41fb639a69d62c8c0fb3f9c227162162ebc8142c6fa5cd0599dc381dcd9ebb"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -2219,6 +2219,7 @@ in garage_util = (rustPackages."unknown".garage_util."1.0.0" { inherit profileName; }).out; gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; + ipnet = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnet."2.9.0" { inherit profileName; }).out; itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.21.0" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.88.1" { inherit profileName; }).out; @@ -3016,8 +3017,8 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default") - (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "std") + [ "default" ] + [ "std" ] ]; }); @@ -55,6 +55,7 @@ hexdump = "0.1" hmac = "0.12" idna = "0.5" itertools = "0.12" +ipnet = "2.9.0" lazy_static = "1.4" md-5 = "0.10" mktemp = "0.5" diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md index 7dba784d..48d2b369 100644 --- a/doc/book/cookbook/real-world.md +++ b/doc/book/cookbook/real-world.md @@ -152,6 +152,8 @@ Check the following for your configuration files: - Make sure `rpc_public_addr` contains the public IP address of the node you are configuring. This parameter is optional but recommended: if your nodes have trouble communicating with one another, consider adding it. + Alternatively, you can also set `rpc_public_addr_subnet`, which can filter + the addresses announced to other peers to a specific subnet. - Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key. You can generate such a key with `openssl rand -hex 32`. diff --git a/doc/book/design/related-work.md b/doc/book/design/related-work.md index 6c1a6b12..84e66c4e 100644 --- a/doc/book/design/related-work.md +++ b/doc/book/design/related-work.md @@ -67,7 +67,7 @@ Pithos has been abandonned and should probably not used yet, in the following we Pithos was relying as a S3 proxy in front of Cassandra (and was working with Scylla DB too). From its designers' mouth, storing data in Cassandra has shown its limitations justifying the project abandonment. They built a closed-source version 2 that does not store blobs in the database (only metadata) but did not communicate further on it. -We considered there v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility. +We considered their v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility. **[Riak CS](https://docs.riak.com/riak/cs/2.1.1/index.html):** *Not written yet* diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index 423795fe..e3595784 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -31,6 +31,9 @@ rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6" rpc_bind_addr = "[::]:3901" rpc_bind_outgoing = false rpc_public_addr = "[fc00:1::1]:3901" +# or set rpc_public_adr_subnet to filter down autodiscovery to a subnet: +# rpc_public_addr_subnet = "2001:0db8:f00:b00:/64" + allow_world_readable_secrets = false @@ -105,6 +108,7 @@ Top-level configuration options: [`rpc_bind_addr`](#rpc_bind_addr), [`rpc_bind_outgoing`](#rpc_bind_outgoing), [`rpc_public_addr`](#rpc_public_addr), +[`rpc_public_addr_subnet`](#rpc_public_addr_subnet) [`rpc_secret`/`rpc_secret_file`](#rpc_secret). The `[consul_discovery]` section: @@ -295,7 +299,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows: | DB engine | `db_engine` value | Database path | | --------- | ----------------- | ------------- | -| [LMDB](https://www.lmdb.tech) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` | +| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` | | [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` | | [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` | @@ -543,6 +547,14 @@ RPC calls. **This parameter is optional but recommended.** In case you have a NAT that binds the RPC port to a port that is different on your public IP, this field might help making it work. +#### `rpc_public_addr_subnet` {#rpc_public_addr_subnet} +In case `rpc_public_addr` is not set, but autodiscovery is used, this allows +filtering the list of automatically discovered IPs to a specific subnet. + +For example, if nodes should pick *their* IP inside a specific subnet, but you +don't want to explicitly write the IP down (as it's dynamic, or you want to +share configs across nodes), you can use this option. + #### `bootstrap_peers` {#bootstrap_peers} A list of peer identifiers on which to contact other Garage peers of this cluster. @@ -76,6 +76,7 @@ # import the full shell using `nix develop .#full` full = shellWithPackages (with pkgs; [ rustfmt + rust-analyzer clang mold # ---- extra packages for dev tasks ---- diff --git a/script/helm/garage/templates/workload.yaml b/script/helm/garage/templates/workload.yaml index 340c0054..635e70e0 100644 --- a/script/helm/garage/templates/workload.yaml +++ b/script/helm/garage/templates/workload.yaml @@ -11,6 +11,7 @@ spec: {{- if eq .Values.deployment.kind "StatefulSet" }} replicas: {{ .Values.deployment.replicaCount }} serviceName: {{ include "garage.fullname" . }} + podManagementPolicy: {{ .Values.deployment.podManagementPolicy }} {{- end }} template: metadata: @@ -63,6 +64,10 @@ spec: name: web-api - containerPort: 3903 name: admin + {{- with .Values.environment }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} volumeMounts: - name: meta mountPath: /mnt/meta diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml index 56afa2b2..5c381f16 100644 --- a/script/helm/garage/values.yaml +++ b/script/helm/garage/values.yaml @@ -96,6 +96,8 @@ deployment: kind: StatefulSet # Number of StatefulSet replicas/garage nodes to start replicaCount: 3 + # If using statefulset, allow Parallel or OrderedReady (default) + podManagementPolicy: OrderedReady image: repository: dxflrs/amd64_garage @@ -214,6 +216,8 @@ tolerations: [] affinity: {} +environment: {} + monitoring: metrics: # If true, a service for monitoring is created with a prometheus.io/scrape annotation @@ -11,6 +11,7 @@ in { # --- Dev shell inherited from flake.nix --- devShell = devShells.default; + devShellFull = devShells.full; # --- Continuous integration shell --- # The shell used for all CI jobs (along with devShell) diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 2c106b3b..ff2361f1 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -71,21 +71,11 @@ pub async fn handle_post_object( } if let Ok(content) = HeaderValue::from_str(&field.text().await?) { - match name.as_str() { - "tag" => (/* tag need to be reencoded, but we don't support them yet anyway */), - "acl" => { - if params.insert("x-amz-acl", content).is_some() { - return Err(Error::bad_request("Field 'acl' provided more than once")); - } - } - _ => { - if params.insert(&name, content).is_some() { - return Err(Error::bad_request(format!( - "Field '{}' provided more than once", - name - ))); - } - } + if params.insert(&name, content).is_some() { + return Err(Error::bad_request(format!( + "Field '{}' provided more than once", + name + ))); } } }; @@ -222,6 +212,8 @@ pub async fn handle_post_object( ))); } + // if we ever start supporting ACLs, we likely want to map "acl" to x-amz-acl" somewhere + // arround here to make sure the rest of the machinery takes our acl into account. let headers = get_headers(¶ms)?; let expected_checksums = ExpectedChecksums { diff --git a/src/db/open.rs b/src/db/open.rs index b8de3cd7..ff3bc830 100644 --- a/src/db/open.rs +++ b/src/db/open.rs @@ -92,6 +92,7 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> { env_builder.map_size(map_size); env_builder.max_readers(2048); unsafe { + env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoRdAhead); env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync); if !opt.fsync { env_builder.flag(heed::flags::Flags::MdbNoSync); diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index 8380b5e2..6a9e6bfb 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -48,7 +48,7 @@ pub enum Command { #[structopt(name = "worker", version = garage_version())] Worker(WorkerOperation), - /// Low-level debug operations on data blocks + /// Low-level node-local debug operations on data blocks #[structopt(name = "block", version = garage_version())] Block(BlockOperation), diff --git a/src/model/garage.rs b/src/model/garage.rs index 273690db..363b02dd 100644 --- a/src/model/garage.rs +++ b/src/model/garage.rs @@ -141,7 +141,7 @@ impl Garage { )?) .ok() .and_then(|x| NetworkKey::from_slice(&x)) - .ok_or_message("Invalid RPC secret key")?; + .ok_or_message("Invalid RPC secret key: expected 32 bits of entropy, please check the documentation for requirements")?; let (replication_factor, consistency_mode) = parse_replication_mode(&config)?; diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 43d5568e..4c8cafd9 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -24,6 +24,7 @@ bytes.workspace = true bytesize.workspace = true gethostname.workspace = true hex.workspace = true +ipnet.workspace = true tracing.workspace = true rand.workspace = true itertools.workspace = true diff --git a/src/rpc/system.rs b/src/rpc/system.rs index 0e78060b..d94d4eec 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -844,12 +844,20 @@ impl NodeStatus { } } -fn get_default_ip() -> Option<IpAddr> { +/// Obtain the list of currently available IP addresses on all non-loopback +/// interfaces, optionally filtering them to be inside a given IpNet. +fn get_default_ip(filter_ipnet: Option<ipnet::IpNet>) -> Option<IpAddr> { pnet_datalink::interfaces() - .iter() - .find(|e| e.is_up() && !e.is_loopback() && !e.ips.is_empty()) - .and_then(|e| e.ips.first()) - .map(|a| a.ip()) + .into_iter() + // filter down and loopback interfaces + .filter(|i| i.is_up() && !i.is_loopback()) + // get all IPs + .flat_map(|e| e.ips) + // optionally, filter to be inside filter_ipnet + .find(|ipn| { + filter_ipnet.is_some_and(|ipnet| ipnet.contains(&ipn.ip())) || filter_ipnet.is_none() + }) + .map(|ipn| ipn.ip()) } fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> { @@ -877,7 +885,28 @@ fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> { } } None => { - let addr = get_default_ip().map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port())); + // `No rpc_public_addr` specified, try to discover one, optionally filtering by `rpc_public_addr_subnet`. + let filter_subnet: Option<ipnet::IpNet> = config + .rpc_public_addr_subnet + .as_ref() + .and_then(|filter_subnet_str| match filter_subnet_str.parse::<ipnet::IpNet>() { + Ok(filter_subnet) => { + let filter_subnet_trunc = filter_subnet.trunc(); + if filter_subnet_trunc != filter_subnet { + warn!("`rpc_public_addr_subnet` changed after applying netmask, continuing with {}", filter_subnet.trunc()); + } + Some(filter_subnet_trunc) + } + Err(e) => { + panic!( + "Cannot parse rpc_public_addr_subnet {} from config file: {}. Bailing out.", + filter_subnet_str, e + ); + } + }); + + let addr = get_default_ip(filter_subnet) + .map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port())); if let Some(a) = addr { warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a); } diff --git a/src/util/config.rs b/src/util/config.rs index 028f8c68..59329c0b 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -85,6 +85,10 @@ pub struct Config { /// Public IP address of this node pub rpc_public_addr: Option<String>, + /// In case `rpc_public_addr` was not set, this can filter + /// the addresses announced to other peers to a specific subnet. + pub rpc_public_addr_subnet: Option<String>, + /// Timeout for Netapp's ping messagess pub rpc_ping_timeout_msec: Option<u64>, /// Timeout for Netapp RPC calls |