aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Cargo.lock1
-rw-r--r--Cargo.nix7
-rw-r--r--Cargo.toml1
-rw-r--r--doc/book/cookbook/real-world.md2
-rw-r--r--doc/book/design/related-work.md2
-rw-r--r--doc/book/reference-manual/configuration.md14
-rw-r--r--flake.nix1
-rw-r--r--script/helm/garage/templates/workload.yaml5
-rw-r--r--script/helm/garage/values.yaml4
-rw-r--r--shell.nix1
-rw-r--r--src/api/s3/post_object.rs22
-rw-r--r--src/db/open.rs1
-rw-r--r--src/garage/cli/structs.rs2
-rw-r--r--src/model/garage.rs2
-rw-r--r--src/rpc/Cargo.toml1
-rw-r--r--src/rpc/system.rs41
-rw-r--r--src/util/config.rs4
17 files changed, 83 insertions, 28 deletions
diff --git a/Cargo.lock b/Cargo.lock
index b96483dd..9cb4b57e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1527,6 +1527,7 @@ dependencies = [
"garage_util",
"gethostname",
"hex",
+ "ipnet",
"itertools 0.12.1",
"k8s-openapi",
"kube",
diff --git a/Cargo.nix b/Cargo.nix
index e0189cdd..c2be3161 100644
--- a/Cargo.nix
+++ b/Cargo.nix
@@ -34,7 +34,7 @@ args@{
ignoreLockHash,
}:
let
- nixifiedLockHash = "1ccd5eb25a83962821e0e9da4ce6df31717b2b97a5b3a0c80c9e0e0759710143";
+ nixifiedLockHash = "fc41fb639a69d62c8c0fb3f9c227162162ebc8142c6fa5cd0599dc381dcd9ebb";
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
lockHashIgnored = if ignoreLockHash
@@ -2219,6 +2219,7 @@ in
garage_util = (rustPackages."unknown".garage_util."1.0.0" { inherit profileName; }).out;
gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
+ ipnet = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnet."2.9.0" { inherit profileName; }).out;
itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out;
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.21.0" { inherit profileName; }).out;
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "kube" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.88.1" { inherit profileName; }).out;
@@ -3016,8 +3017,8 @@ in
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"; };
features = builtins.concatLists [
- (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "default")
- (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest") "std")
+ [ "default" ]
+ [ "std" ]
];
});
diff --git a/Cargo.toml b/Cargo.toml
index b8840a91..500079f2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -55,6 +55,7 @@ hexdump = "0.1"
hmac = "0.12"
idna = "0.5"
itertools = "0.12"
+ipnet = "2.9.0"
lazy_static = "1.4"
md-5 = "0.10"
mktemp = "0.5"
diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md
index 7dba784d..48d2b369 100644
--- a/doc/book/cookbook/real-world.md
+++ b/doc/book/cookbook/real-world.md
@@ -152,6 +152,8 @@ Check the following for your configuration files:
- Make sure `rpc_public_addr` contains the public IP address of the node you are configuring.
This parameter is optional but recommended: if your nodes have trouble communicating with
one another, consider adding it.
+ Alternatively, you can also set `rpc_public_addr_subnet`, which can filter
+ the addresses announced to other peers to a specific subnet.
- Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key.
You can generate such a key with `openssl rand -hex 32`.
diff --git a/doc/book/design/related-work.md b/doc/book/design/related-work.md
index 6c1a6b12..84e66c4e 100644
--- a/doc/book/design/related-work.md
+++ b/doc/book/design/related-work.md
@@ -67,7 +67,7 @@ Pithos has been abandonned and should probably not used yet, in the following we
Pithos was relying as a S3 proxy in front of Cassandra (and was working with Scylla DB too).
From its designers' mouth, storing data in Cassandra has shown its limitations justifying the project abandonment.
They built a closed-source version 2 that does not store blobs in the database (only metadata) but did not communicate further on it.
-We considered there v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility.
+We considered their v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility.
**[Riak CS](https://docs.riak.com/riak/cs/2.1.1/index.html):**
*Not written yet*
diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md
index 423795fe..e3595784 100644
--- a/doc/book/reference-manual/configuration.md
+++ b/doc/book/reference-manual/configuration.md
@@ -31,6 +31,9 @@ rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6"
rpc_bind_addr = "[::]:3901"
rpc_bind_outgoing = false
rpc_public_addr = "[fc00:1::1]:3901"
+# or set rpc_public_adr_subnet to filter down autodiscovery to a subnet:
+# rpc_public_addr_subnet = "2001:0db8:f00:b00:/64"
+
allow_world_readable_secrets = false
@@ -105,6 +108,7 @@ Top-level configuration options:
[`rpc_bind_addr`](#rpc_bind_addr),
[`rpc_bind_outgoing`](#rpc_bind_outgoing),
[`rpc_public_addr`](#rpc_public_addr),
+[`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
[`rpc_secret`/`rpc_secret_file`](#rpc_secret).
The `[consul_discovery]` section:
@@ -295,7 +299,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
| DB engine | `db_engine` value | Database path |
| --------- | ----------------- | ------------- |
-| [LMDB](https://www.lmdb.tech) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
+| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
| [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
| [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
@@ -543,6 +547,14 @@ RPC calls. **This parameter is optional but recommended.** In case you have
a NAT that binds the RPC port to a port that is different on your public IP,
this field might help making it work.
+#### `rpc_public_addr_subnet` {#rpc_public_addr_subnet}
+In case `rpc_public_addr` is not set, but autodiscovery is used, this allows
+filtering the list of automatically discovered IPs to a specific subnet.
+
+For example, if nodes should pick *their* IP inside a specific subnet, but you
+don't want to explicitly write the IP down (as it's dynamic, or you want to
+share configs across nodes), you can use this option.
+
#### `bootstrap_peers` {#bootstrap_peers}
A list of peer identifiers on which to contact other Garage peers of this cluster.
diff --git a/flake.nix b/flake.nix
index 06c74220..f8840c5b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -76,6 +76,7 @@
# import the full shell using `nix develop .#full`
full = shellWithPackages (with pkgs; [
rustfmt
+ rust-analyzer
clang
mold
# ---- extra packages for dev tasks ----
diff --git a/script/helm/garage/templates/workload.yaml b/script/helm/garage/templates/workload.yaml
index 340c0054..635e70e0 100644
--- a/script/helm/garage/templates/workload.yaml
+++ b/script/helm/garage/templates/workload.yaml
@@ -11,6 +11,7 @@ spec:
{{- if eq .Values.deployment.kind "StatefulSet" }}
replicas: {{ .Values.deployment.replicaCount }}
serviceName: {{ include "garage.fullname" . }}
+ podManagementPolicy: {{ .Values.deployment.podManagementPolicy }}
{{- end }}
template:
metadata:
@@ -63,6 +64,10 @@ spec:
name: web-api
- containerPort: 3903
name: admin
+ {{- with .Values.environment }}
+ env:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
volumeMounts:
- name: meta
mountPath: /mnt/meta
diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml
index 56afa2b2..5c381f16 100644
--- a/script/helm/garage/values.yaml
+++ b/script/helm/garage/values.yaml
@@ -96,6 +96,8 @@ deployment:
kind: StatefulSet
# Number of StatefulSet replicas/garage nodes to start
replicaCount: 3
+ # If using statefulset, allow Parallel or OrderedReady (default)
+ podManagementPolicy: OrderedReady
image:
repository: dxflrs/amd64_garage
@@ -214,6 +216,8 @@ tolerations: []
affinity: {}
+environment: {}
+
monitoring:
metrics:
# If true, a service for monitoring is created with a prometheus.io/scrape annotation
diff --git a/shell.nix b/shell.nix
index 0fa02b66..7236fdcb 100644
--- a/shell.nix
+++ b/shell.nix
@@ -11,6 +11,7 @@ in
{
# --- Dev shell inherited from flake.nix ---
devShell = devShells.default;
+ devShellFull = devShells.full;
# --- Continuous integration shell ---
# The shell used for all CI jobs (along with devShell)
diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs
index 2c106b3b..ff2361f1 100644
--- a/src/api/s3/post_object.rs
+++ b/src/api/s3/post_object.rs
@@ -71,21 +71,11 @@ pub async fn handle_post_object(
}
if let Ok(content) = HeaderValue::from_str(&field.text().await?) {
- match name.as_str() {
- "tag" => (/* tag need to be reencoded, but we don't support them yet anyway */),
- "acl" => {
- if params.insert("x-amz-acl", content).is_some() {
- return Err(Error::bad_request("Field 'acl' provided more than once"));
- }
- }
- _ => {
- if params.insert(&name, content).is_some() {
- return Err(Error::bad_request(format!(
- "Field '{}' provided more than once",
- name
- )));
- }
- }
+ if params.insert(&name, content).is_some() {
+ return Err(Error::bad_request(format!(
+ "Field '{}' provided more than once",
+ name
+ )));
}
}
};
@@ -222,6 +212,8 @@ pub async fn handle_post_object(
)));
}
+ // if we ever start supporting ACLs, we likely want to map "acl" to x-amz-acl" somewhere
+ // arround here to make sure the rest of the machinery takes our acl into account.
let headers = get_headers(&params)?;
let expected_checksums = ExpectedChecksums {
diff --git a/src/db/open.rs b/src/db/open.rs
index b8de3cd7..ff3bc830 100644
--- a/src/db/open.rs
+++ b/src/db/open.rs
@@ -92,6 +92,7 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
env_builder.map_size(map_size);
env_builder.max_readers(2048);
unsafe {
+ env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoRdAhead);
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
if !opt.fsync {
env_builder.flag(heed::flags::Flags::MdbNoSync);
diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs
index 8380b5e2..6a9e6bfb 100644
--- a/src/garage/cli/structs.rs
+++ b/src/garage/cli/structs.rs
@@ -48,7 +48,7 @@ pub enum Command {
#[structopt(name = "worker", version = garage_version())]
Worker(WorkerOperation),
- /// Low-level debug operations on data blocks
+ /// Low-level node-local debug operations on data blocks
#[structopt(name = "block", version = garage_version())]
Block(BlockOperation),
diff --git a/src/model/garage.rs b/src/model/garage.rs
index 273690db..363b02dd 100644
--- a/src/model/garage.rs
+++ b/src/model/garage.rs
@@ -141,7 +141,7 @@ impl Garage {
)?)
.ok()
.and_then(|x| NetworkKey::from_slice(&x))
- .ok_or_message("Invalid RPC secret key")?;
+ .ok_or_message("Invalid RPC secret key: expected 32 bits of entropy, please check the documentation for requirements")?;
let (replication_factor, consistency_mode) = parse_replication_mode(&config)?;
diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml
index 43d5568e..4c8cafd9 100644
--- a/src/rpc/Cargo.toml
+++ b/src/rpc/Cargo.toml
@@ -24,6 +24,7 @@ bytes.workspace = true
bytesize.workspace = true
gethostname.workspace = true
hex.workspace = true
+ipnet.workspace = true
tracing.workspace = true
rand.workspace = true
itertools.workspace = true
diff --git a/src/rpc/system.rs b/src/rpc/system.rs
index 0e78060b..d94d4eec 100644
--- a/src/rpc/system.rs
+++ b/src/rpc/system.rs
@@ -844,12 +844,20 @@ impl NodeStatus {
}
}
-fn get_default_ip() -> Option<IpAddr> {
+/// Obtain the list of currently available IP addresses on all non-loopback
+/// interfaces, optionally filtering them to be inside a given IpNet.
+fn get_default_ip(filter_ipnet: Option<ipnet::IpNet>) -> Option<IpAddr> {
pnet_datalink::interfaces()
- .iter()
- .find(|e| e.is_up() && !e.is_loopback() && !e.ips.is_empty())
- .and_then(|e| e.ips.first())
- .map(|a| a.ip())
+ .into_iter()
+ // filter down and loopback interfaces
+ .filter(|i| i.is_up() && !i.is_loopback())
+ // get all IPs
+ .flat_map(|e| e.ips)
+ // optionally, filter to be inside filter_ipnet
+ .find(|ipn| {
+ filter_ipnet.is_some_and(|ipnet| ipnet.contains(&ipn.ip())) || filter_ipnet.is_none()
+ })
+ .map(|ipn| ipn.ip())
}
fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> {
@@ -877,7 +885,28 @@ fn get_rpc_public_addr(config: &Config) -> Option<SocketAddr> {
}
}
None => {
- let addr = get_default_ip().map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port()));
+ // `No rpc_public_addr` specified, try to discover one, optionally filtering by `rpc_public_addr_subnet`.
+ let filter_subnet: Option<ipnet::IpNet> = config
+ .rpc_public_addr_subnet
+ .as_ref()
+ .and_then(|filter_subnet_str| match filter_subnet_str.parse::<ipnet::IpNet>() {
+ Ok(filter_subnet) => {
+ let filter_subnet_trunc = filter_subnet.trunc();
+ if filter_subnet_trunc != filter_subnet {
+ warn!("`rpc_public_addr_subnet` changed after applying netmask, continuing with {}", filter_subnet.trunc());
+ }
+ Some(filter_subnet_trunc)
+ }
+ Err(e) => {
+ panic!(
+ "Cannot parse rpc_public_addr_subnet {} from config file: {}. Bailing out.",
+ filter_subnet_str, e
+ );
+ }
+ });
+
+ let addr = get_default_ip(filter_subnet)
+ .map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port()));
if let Some(a) = addr {
warn!("Using autodetected rpc_public_addr: {}. Consider specifying it explicitly in configuration file if possible.", a);
}
diff --git a/src/util/config.rs b/src/util/config.rs
index 028f8c68..59329c0b 100644
--- a/src/util/config.rs
+++ b/src/util/config.rs
@@ -85,6 +85,10 @@ pub struct Config {
/// Public IP address of this node
pub rpc_public_addr: Option<String>,
+ /// In case `rpc_public_addr` was not set, this can filter
+ /// the addresses announced to other peers to a specific subnet.
+ pub rpc_public_addr_subnet: Option<String>,
+
/// Timeout for Netapp's ping messagess
pub rpc_ping_timeout_msec: Option<u64>,
/// Timeout for Netapp RPC calls
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-19 01:28:51 +0000