Merge branch 'main' into next-0.10

21 files changed, 737 insertions, 267 deletions

diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml
index dce7ea73..02a72502 100644
--- a/src/garage/Cargo.toml
+++ b/src/garage/Cargo.toml
@@ -31,48 +31,51 @@ garage_table.workspace = true
 garage_util.workspace = true
 garage_web.workspace = true
 
-backtrace = "0.3"
-bytes = "1.0"
-bytesize = "1.2"
-timeago = { version = "0.4", default-features = false }
-parse_duration = "2.1"
-hex = "0.4"
-tracing = { version = "0.1" }
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-rand = "0.8"
-async-trait = "0.1.7"
-sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
-git-version = "0.3.4"
-
-serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
-serde_bytes = "0.11"
-structopt = { version = "0.3", default-features = false }
-toml = "0.6"
-
-futures = "0.3"
-futures-util = "0.3"
-tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
-
-netapp = "0.10"
-
-opentelemetry = { version = "0.17", features = [ "rt-tokio" ] }
-opentelemetry-prometheus = { version = "0.10", optional = true }
-opentelemetry-otlp = { version = "0.10", optional = true }
-prometheus = { version = "0.13", optional = true }
+backtrace.workspace = true
+bytes.workspace = true
+bytesize.workspace = true
+timeago.workspace = true
+parse_duration.workspace = true
+hex.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+rand.workspace = true
+async-trait.workspace = true
+sodiumoxide.workspace = true
+structopt.workspace = true
+git-version.workspace = true
+
+serde.workspace = true
+serde_bytes.workspace = true
+toml.workspace = true
+
+futures.workspace = true
+futures-util.workspace = true
+tokio.workspace = true
+
+netapp.workspace = true
+
+opentelemetry.workspace = true
+opentelemetry-prometheus = { workspace = true, optional = true }
+opentelemetry-otlp = { workspace = true, optional = true }
+prometheus = { workspace = true, optional = true }
 
 [dev-dependencies]
-aws-config = "0.55.2"
-aws-sdk-s3 = "0.28"
-chrono = "0.4"
-http = "0.2"
-hmac = "0.12"
-hyper = { version = "0.14", features = ["client", "http1", "runtime"] }
-sha2 = "0.10"
-
-static_init = "1.0"
-assert-json-diff = "2.0"
-serde_json = "1.0"
-base64 = "0.21"
+aws-config.workspace = true
+aws-sdk-s3.workspace = true
+chrono.workspace = true
+http.workspace = true
+hmac.workspace = true
+http-body-util.workspace = true
+hyper.workspace = true
+hyper-util.workspace = true
+mktemp.workspace = true
+sha2.workspace = true
+
+static_init.workspace = true
+assert-json-diff.workspace = true
+serde_json.workspace = true
+base64.workspace = true
 
 k2v-client.workspace = true
 
diff --git a/src/garage/admin/block.rs b/src/garage/admin/block.rs
index c4a45738..edeb88c0 100644
--- a/src/garage/admin/block.rs
+++ b/src/garage/admin/block.rs
@@ -25,8 +25,7 @@ impl AdminRpcHandler {
 	}
 
 	async fn handle_block_info(&self, hash: &String) -> Result<AdminRpc, Error> {
-		let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
-		let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+		let hash = self.find_block_hash_by_prefix(hash)?;
 		let refcount = self.garage.block_manager.get_block_rc(&hash)?;
 		let block_refs = self
 			.garage
@@ -189,4 +188,48 @@ impl AdminRpcHandler {
 
 		Ok(())
 	}
+
+	// ---- helper function ----
+	fn find_block_hash_by_prefix(&self, prefix: &str) -> Result<Hash, Error> {
+		if prefix.len() < 4 {
+			return Err(Error::BadRequest(
+				"Please specify at least 4 characters of the block hash".into(),
+			));
+		}
+
+		let prefix_bin =
+			hex::decode(&prefix[..prefix.len() & !1]).ok_or_bad_request("invalid hash")?;
+
+		let iter = self
+			.garage
+			.block_ref_table
+			.data
+			.store
+			.range(&prefix_bin[..]..)
+			.map_err(GarageError::from)?;
+		let mut found = None;
+		for item in iter {
+			let (k, _v) = item.map_err(GarageError::from)?;
+			let hash = Hash::try_from(&k[..32]).unwrap();
+			if &hash.as_slice()[..prefix_bin.len()] != prefix_bin {
+				break;
+			}
+			if hex::encode(hash.as_slice()).starts_with(prefix) {
+				match &found {
+					Some(x) if *x == hash => (),
+					Some(_) => {
+						return Err(Error::BadRequest(format!(
+							"Several blocks match prefix `{}`",
+							prefix
+						)));
+					}
+					None => {
+						found = Some(hash);
+					}
+				}
+			}
+		}
+
+		found.ok_or_else(|| Error::BadRequest("No matching block found".into()))
+	}
 }
diff --git a/src/garage/cli/convert_db.rs b/src/garage/cli/convert_db.rs
index 3c6ce69c..6b854ccb 100644
--- a/src/garage/cli/convert_db.rs
+++ b/src/garage/cli/convert_db.rs
@@ -14,44 +14,73 @@ pub struct ConvertDbOpt {
 	/// Input database engine (sled, lmdb or sqlite; limited by db engines
 	/// enabled in this build)
 	#[structopt(short = "a")]
-	input_engine: String,
+	input_engine: Engine,
 
 	/// Output database path
 	#[structopt(short = "o")]
 	output_path: PathBuf,
 	/// Output database engine
 	#[structopt(short = "b")]
-	output_engine: String,
+	output_engine: Engine,
+
+	#[structopt(flatten)]
+	db_open: OpenDbOpt,
+}
+
+/// Overrides for database open operation
+#[derive(StructOpt, Debug, Default)]
+pub struct OpenDbOpt {
+	#[cfg(feature = "lmdb")]
+	#[structopt(flatten)]
+	lmdb: OpenLmdbOpt,
+}
+
+/// Overrides for LMDB database open operation
+#[cfg(feature = "lmdb")]
+#[derive(StructOpt, Debug, Default)]
+pub struct OpenLmdbOpt {
+	/// LMDB map size override
+	/// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB)
+	#[cfg(feature = "lmdb")]
+	#[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)]
+	map_size: Option<bytesize::ByteSize>,
 }
 
 pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
-	let input = open_db(args.input_path, args.input_engine)?;
-	let output = open_db(args.output_path, args.output_engine)?;
+	if args.input_engine == args.output_engine {
+		return Err(Error("input and output database engine must differ".into()));
+	}
+
+	let input = open_db(args.input_path, args.input_engine, &args.db_open)?;
+	let output = open_db(args.output_path, args.output_engine, &args.db_open)?;
 	output.import(&input)?;
 	Ok(())
 }
 
-fn open_db(path: PathBuf, engine: String) -> Result<Db> {
-	match engine.as_str() {
+fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
+	match engine {
 		#[cfg(feature = "sled")]
-		"sled" => {
+		Engine::Sled => {
 			let db = sled_adapter::sled::Config::default().path(&path).open()?;
 			Ok(sled_adapter::SledDb::init(db))
 		}
 		#[cfg(feature = "sqlite")]
-		"sqlite" | "sqlite3" | "rusqlite" => {
+		Engine::Sqlite => {
 			let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
-			db.pragma_update(None, "journal_mode", &"WAL")?;
-			db.pragma_update(None, "synchronous", &"NORMAL")?;
+			db.pragma_update(None, "journal_mode", "WAL")?;
+			db.pragma_update(None, "synchronous", "NORMAL")?;
 			Ok(sqlite_adapter::SqliteDb::init(db))
 		}
 		#[cfg(feature = "lmdb")]
-		"lmdb" | "heed" => {
+		Engine::Lmdb => {
 			std::fs::create_dir_all(&path).map_err(|e| {
 				Error(format!("Unable to create LMDB data directory: {}", e).into())
 			})?;
 
-			let map_size = lmdb_adapter::recommended_map_size();
+			let map_size = match open.lmdb.map_size {
+				Some(c) => c.as_u64() as usize,
+				None => lmdb_adapter::recommended_map_size(),
+			};
 
 			let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
 			env_builder.max_dbs(100);
@@ -62,8 +91,13 @@ fn open_db(path: PathBuf, engine: String) -> Result<Db> {
 			let db = env_builder.open(&path)?;
 			Ok(lmdb_adapter::LmdbDb::init(db))
 		}
-		e => Err(Error(
-			format!("Invalid or unsupported DB engine: {}", e).into(),
+
+		// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
+		// attribute is added so that we won't have to change this match in case stop building
+		// support for one or more engines by default.
+		#[allow(unreachable_patterns)]
+		engine => Err(Error(
+			format!("Engine support not available in this build: {}", engine).into(),
 		)),
 	}
 }
diff --git a/src/garage/cli/init.rs b/src/garage/cli/init.rs
index 20813f1c..43ca5c09 100644
--- a/src/garage/cli/init.rs
+++ b/src/garage/cli/init.rs
@@ -43,7 +43,7 @@ pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
 			idstr
 		);
 		eprintln!(
-			"where <remote_node> is their own node identifier in the format: <pubkey>@<ip>:<port>"
+			"where <remote_node> is their own node identifier in the format: <full-node-id>@<ip>:<port>"
 		);
 		eprintln!();
 		eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:");
diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs
index 6bc3da22..40e47ee1 100644
--- a/src/garage/cli/structs.rs
+++ b/src/garage/cli/structs.rs
@@ -64,7 +64,8 @@ pub enum Command {
 
 #[derive(StructOpt, Debug)]
 pub enum NodeOperation {
-	/// Print identifier (public key) of this Garage node
+	/// Print the full node ID (public key) of this Garage node, and its publicly reachable IP
+	/// address and port if they are specified in config file under `rpc_public_addr`
 	#[structopt(name = "id", version = garage_version())]
 	NodeId(NodeIdOpt),
 
@@ -82,8 +83,9 @@ pub struct NodeIdOpt {
 
 #[derive(StructOpt, Debug)]
 pub struct ConnectNodeOpt {
-	/// Node public key and address, in the format:
-	/// `<public key hexadecimal>@<ip or hostname>:<port>`
+	/// Full node ID (public key) and IP address and port, in the format:
+	/// `<full node ID>@<ip or hostname>:<port>`.
+	/// You can retrieve this information on the target node using `garage node id`.
 	pub(crate) node: String,
 }
 
diff --git a/src/garage/main.rs b/src/garage/main.rs
index 66403d05..1a6a6e32 100644
--- a/src/garage/main.rs
+++ b/src/garage/main.rs
@@ -7,6 +7,7 @@ extern crate tracing;
 mod admin;
 mod cli;
 mod repair;
+mod secrets;
 mod server;
 #[cfg(feature = "telemetry-otlp")]
 mod tracing_setup;
@@ -28,7 +29,6 @@ use structopt::StructOpt;
 use netapp::util::parse_and_resolve_peer_addr;
 use netapp::NetworkKey;
 
-use garage_util::config::Config;
 use garage_util::error::*;
 
 use garage_rpc::system::*;
@@ -38,6 +38,7 @@ use garage_model::helper::error::Error as HelperError;
 
 use admin::*;
 use cli::*;
+use secrets::Secrets;
 
 #[derive(StructOpt, Debug)]
 #[structopt(
@@ -45,8 +46,7 @@ use cli::*;
 	about = "S3-compatible object store for self-hosted geo-distributed deployments"
 )]
 struct Opt {
-	/// Host to connect to for admin operations, in the format:
-	/// <public-key>@<ip>:<port>
+	/// Host to connect to for admin operations, in the format: <full-node-id>@<ip>:<port>
 	#[structopt(short = "h", long = "rpc-host", env = "GARAGE_RPC_HOST")]
 	pub rpc_host: Option<String>,
 
@@ -66,24 +66,6 @@ struct Opt {
 	cmd: Command,
 }
 
-#[derive(StructOpt, Debug)]
-pub struct Secrets {
-	/// RPC secret network key, used to replace rpc_secret in config.toml when running the
-	/// daemon or doing admin operations
-	#[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
-	pub rpc_secret: Option<String>,
-
-	/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
-	/// running the Garage daemon
-	#[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
-	pub admin_token: Option<String>,
-
-	/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
-	/// running the Garage daemon
-	#[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
-	pub metrics_token: Option<String>,
-}
-
 #[tokio::main]
 async fn main() {
 	// Initialize version and features info
@@ -192,7 +174,9 @@ async fn main() {
 }
 
 async fn cli_command(opt: Opt) -> Result<(), Error> {
-	let config = if opt.secrets.rpc_secret.is_none() || opt.rpc_host.is_none() {
+	let config = if (opt.secrets.rpc_secret.is_none() && opt.secrets.rpc_secret_file.is_none())
+		|| opt.rpc_host.is_none()
+	{
 		Some(garage_util::config::read_config(opt.config_file.clone())
 			.err_context(format!("Unable to read configuration file {}. Configuration file is needed because -h or -s is not provided on the command line.", opt.config_file.to_string_lossy()))?)
 	} else {
@@ -200,14 +184,19 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	};
 
 	// Find and parse network RPC secret
-	let net_key_hex_str = opt
-		.secrets
-		.rpc_secret
-		.as_ref()
-		.or_else(|| config.as_ref().and_then(|c| c.rpc_secret.as_ref()))
-		.ok_or("No RPC secret provided")?;
+	let mut rpc_secret = config.as_ref().and_then(|c| c.rpc_secret.clone());
+	secrets::fill_secret(
+		&mut rpc_secret,
+		&config.as_ref().and_then(|c| c.rpc_secret_file.clone()),
+		&opt.secrets.rpc_secret,
+		&opt.secrets.rpc_secret_file,
+		"rpc_secret",
+		true,
+	)?;
+
+	let net_key_hex_str = rpc_secret.ok_or("No RPC secret provided")?;
 	let network_key = NetworkKey::from_slice(
-		&hex::decode(net_key_hex_str).err_context("Invalid RPC secret key (bad hex)")?[..],
+		&hex::decode(&net_key_hex_str).err_context("Invalid RPC secret key (bad hex)")?[..],
 	)
 	.ok_or("Invalid RPC secret provided (wrong length)")?;
 
@@ -218,7 +207,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 
 	// Find and parse the address of the target host
 	let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host {
-		let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <pubkey>@<IP or hostname>:<port>.", h))?;
+		let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <full node id>@<IP or hostname>:<port>.", h))?;
 		(id, addrs[0], false)
 	} else {
 		let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir)
@@ -248,7 +237,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 				addr
 			);
 		}
-		Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
+		Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct full-length node ID (public key).")?;
 	}
 
 	let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
@@ -261,16 +250,3 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 		Ok(x) => Ok(x),
 	}
 }
-
-fn fill_secrets(mut config: Config, secrets: Secrets) -> Config {
-	if secrets.rpc_secret.is_some() {
-		config.rpc_secret = secrets.rpc_secret;
-	}
-	if secrets.admin_token.is_some() {
-		config.admin.admin_token = secrets.admin_token;
-	}
-	if secrets.metrics_token.is_some() {
-		config.admin.metrics_token = secrets.metrics_token;
-	}
-	config
-}
diff --git a/src/garage/repair/offline.rs b/src/garage/repair/offline.rs
index f4edcf03..45024e71 100644
--- a/src/garage/repair/offline.rs
+++ b/src/garage/repair/offline.rs
@@ -6,7 +6,7 @@ use garage_util::error::*;
 use garage_model::garage::Garage;
 
 use crate::cli::structs::*;
-use crate::{fill_secrets, Secrets};
+use crate::secrets::{fill_secrets, Secrets};
 
 pub async fn offline_repair(
 	config_file: PathBuf,
@@ -20,7 +20,7 @@ pub async fn offline_repair(
 	}
 
 	info!("Loading configuration...");
-	let config = fill_secrets(read_config(config_file)?, secrets);
+	let config = fill_secrets(read_config(config_file)?, secrets)?;
 
 	info!("Initializing Garage main data store...");
 	let garage = Garage::new(config)?;
diff --git a/src/garage/secrets.rs b/src/garage/secrets.rs
new file mode 100644
index 00000000..c3d704aa
--- /dev/null
+++ b/src/garage/secrets.rs
@@ -0,0 +1,320 @@
+use std::path::PathBuf;
+
+use structopt::StructOpt;
+
+use garage_util::config::Config;
+use garage_util::error::Error;
+
+/// Structure for secret values or paths that are passed as CLI arguments or environment
+/// variables, instead of in the config file.
+#[derive(StructOpt, Debug, Default, Clone)]
+pub struct Secrets {
+	/// Skip permission check on files containing secrets
+	#[cfg(unix)]
+	#[structopt(
+		long = "allow-world-readable-secrets",
+		env = "GARAGE_ALLOW_WORLD_READABLE_SECRETS"
+	)]
+	pub allow_world_readable_secrets: Option<bool>,
+
+	/// RPC secret network key, used to replace rpc_secret in config.toml when running the
+	/// daemon or doing admin operations
+	#[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
+	pub rpc_secret: Option<String>,
+
+	/// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret
+	/// when running the daemon or doing admin operations
+	#[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")]
+	pub rpc_secret_file: Option<PathBuf>,
+
+	/// Admin API authentication token, replaces admin.admin_token in config.toml when
+	/// running the Garage daemon
+	#[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
+	pub admin_token: Option<String>,
+
+	/// Admin API authentication token file path, replaces admin.admin_token in config.toml
+	/// and admin-token when running the Garage daemon
+	#[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")]
+	pub admin_token_file: Option<PathBuf>,
+
+	/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
+	/// running the Garage daemon
+	#[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
+	pub metrics_token: Option<String>,
+
+	/// Metrics API authentication token file path, replaces admin.metrics_token in config.toml
+	/// and metrics-token when running the Garage daemon
+	#[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")]
+	pub metrics_token_file: Option<PathBuf>,
+}
+
+/// Single function to fill all secrets in the Config struct from their correct source (value
+/// from config or CLI param or env variable or read from a file specified in config or CLI
+/// param or env variable)
+pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result<Config, Error> {
+	let allow_world_readable = secrets
+		.allow_world_readable_secrets
+		.unwrap_or(config.allow_world_readable_secrets);
+
+	fill_secret(
+		&mut config.rpc_secret,
+		&config.rpc_secret_file,
+		&secrets.rpc_secret,
+		&secrets.rpc_secret_file,
+		"rpc_secret",
+		allow_world_readable,
+	)?;
+
+	fill_secret(
+		&mut config.admin.admin_token,
+		&config.admin.admin_token_file,
+		&secrets.admin_token,
+		&secrets.admin_token_file,
+		"admin.admin_token",
+		allow_world_readable,
+	)?;
+	fill_secret(
+		&mut config.admin.metrics_token,
+		&config.admin.metrics_token_file,
+		&secrets.metrics_token,
+		&secrets.metrics_token_file,
+		"admin.metrics_token",
+		allow_world_readable,
+	)?;
+
+	Ok(config)
+}
+
+pub(crate) fn fill_secret(
+	config_secret: &mut Option<String>,
+	config_secret_file: &Option<PathBuf>,
+	cli_secret: &Option<String>,
+	cli_secret_file: &Option<PathBuf>,
+	name: &'static str,
+	allow_world_readable: bool,
+) -> Result<(), Error> {
+	let cli_value = match (&cli_secret, &cli_secret_file) {
+		(Some(_), Some(_)) => {
+			return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
+		}
+		(Some(secret), None) => Some(secret.to_string()),
+		(None, Some(file)) => Some(read_secret_file(file, allow_world_readable)?),
+		(None, None) => None,
+	};
+
+	if let Some(val) = cli_value {
+		if config_secret.is_some() || config_secret_file.is_some() {
+			debug!("Overriding secret `{}` using value specified using CLI argument or environnement variable.", name);
+		}
+
+		*config_secret = Some(val);
+	} else if let Some(file_path) = &config_secret_file {
+		if config_secret.is_some() {
+			return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
+		}
+
+		*config_secret = Some(read_secret_file(file_path, allow_world_readable)?);
+	}
+
+	Ok(())
+}
+
+fn read_secret_file(file_path: &PathBuf, allow_world_readable: bool) -> Result<String, Error> {
+	if !allow_world_readable {
+		#[cfg(unix)]
+		{
+			use std::os::unix::fs::MetadataExt;
+			let metadata = std::fs::metadata(file_path)?;
+			if metadata.mode() & 0o077 != 0 {
+				return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path.display(), metadata.mode()).into());
+			}
+		}
+	}
+
+	let secret_buf = std::fs::read_to_string(file_path)?;
+
+	// trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
+	//           also editors sometimes add a trailing newline
+	Ok(String::from(secret_buf.trim_end()))
+}
+
+#[cfg(test)]
+mod tests {
+	use std::fs::File;
+	use std::io::Write;
+
+	use garage_util::config::read_config;
+	use garage_util::error::Error;
+
+	use super::*;
+
+	#[test]
+	fn test_rpc_secret_file_works() -> Result<(), Error> {
+		let path_secret = mktemp::Temp::new_file()?;
+		let mut file_secret = File::create(path_secret.as_path())?;
+		writeln!(file_secret, "foo")?;
+		drop(file_secret);
+
+		let path_config = mktemp::Temp::new_file()?;
+		let mut file_config = File::create(path_config.as_path())?;
+		let path_secret_path = path_secret.as_path();
+		writeln!(
+			file_config,
+			r#"
+			metadata_dir = "/tmp/garage/meta"
+			data_dir = "/tmp/garage/data"
+			replication_mode = "3"
+			rpc_bind_addr = "[::]:3901"
+			rpc_secret_file = "{}"
+
+			[s3_api]
+			s3_region = "garage"
+			api_bind_addr = "[::]:3900"
+			"#,
+			path_secret_path.display()
+		)?;
+		drop(file_config);
+
+		// Second configuration file, same as previous one
+		// except it allows world-readable secrets.
+		let path_config_allow_world_readable = mktemp::Temp::new_file()?;
+		let mut file_config_allow_world_readable =
+			File::create(path_config_allow_world_readable.as_path())?;
+		writeln!(
+			file_config_allow_world_readable,
+			r#"
+			metadata_dir = "/tmp/garage/meta"
+			data_dir = "/tmp/garage/data"
+			replication_mode = "3"
+			rpc_bind_addr = "[::]:3901"
+			rpc_secret_file = "{}"
+			allow_world_readable_secrets = true
+
+			[s3_api]
+			s3_region = "garage"
+			api_bind_addr = "[::]:3900"
+			"#,
+			path_secret_path.display()
+		)?;
+		drop(file_config_allow_world_readable);
+
+		let config = read_config(path_config.to_path_buf())?;
+		let config = fill_secrets(config, Secrets::default())?;
+		assert_eq!("foo", config.rpc_secret.unwrap());
+
+		// ---- Check non world-readable secrets config ----
+		#[cfg(unix)]
+		{
+			let secrets_allow_world_readable = Secrets {
+				allow_world_readable_secrets: Some(true),
+				..Default::default()
+			};
+			let secrets_no_allow_world_readable = Secrets {
+				allow_world_readable_secrets: Some(false),
+				..Default::default()
+			};
+
+			use std::os::unix::fs::PermissionsExt;
+			let metadata = std::fs::metadata(path_secret_path)?;
+			let mut perm = metadata.permissions();
+			perm.set_mode(0o660);
+			std::fs::set_permissions(path_secret_path, perm)?;
+
+			// Config file that just specifies the path
+			let config = read_config(path_config.to_path_buf())?;
+			assert!(fill_secrets(config, Secrets::default()).is_err());
+
+			let config = read_config(path_config.to_path_buf())?;
+			assert!(fill_secrets(config, secrets_allow_world_readable.clone()).is_ok());
+
+			let config = read_config(path_config.to_path_buf())?;
+			assert!(fill_secrets(config, secrets_no_allow_world_readable.clone()).is_err());
+
+			// Config file that also specifies to allow world_readable_secrets
+			let config = read_config(path_config_allow_world_readable.to_path_buf())?;
+			assert!(fill_secrets(config, Secrets::default()).is_ok());
+
+			let config = read_config(path_config_allow_world_readable.to_path_buf())?;
+			assert!(fill_secrets(config, secrets_allow_world_readable).is_ok());
+
+			let config = read_config(path_config_allow_world_readable.to_path_buf())?;
+			assert!(fill_secrets(config, secrets_no_allow_world_readable).is_err());
+		}
+
+		// ---- Check alternative secrets specified on CLI ----
+
+		let path_secret2 = mktemp::Temp::new_file()?;
+		let mut file_secret2 = File::create(path_secret2.as_path())?;
+		writeln!(file_secret2, "bar")?;
+		drop(file_secret2);
+
+		let config = read_config(path_config.to_path_buf())?;
+		let config = fill_secrets(
+			config,
+			Secrets {
+				rpc_secret: Some("baz".into()),
+				..Default::default()
+			},
+		)?;
+		assert_eq!(config.rpc_secret.as_deref(), Some("baz"));
+
+		let config = read_config(path_config.to_path_buf())?;
+		let config = fill_secrets(
+			config,
+			Secrets {
+				rpc_secret_file: Some(path_secret2.clone()),
+				..Default::default()
+			},
+		)?;
+		assert_eq!(config.rpc_secret.as_deref(), Some("bar"));
+
+		let config = read_config(path_config.to_path_buf())?;
+		assert!(fill_secrets(
+			config,
+			Secrets {
+				rpc_secret: Some("baz".into()),
+				rpc_secret_file: Some(path_secret2.clone()),
+				..Default::default()
+			}
+		)
+		.is_err());
+
+		drop(path_secret);
+		drop(path_secret2);
+		drop(path_config);
+		drop(path_config_allow_world_readable);
+
+		Ok(())
+	}
+
+	#[test]
+	fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> {
+		let path_config = mktemp::Temp::new_file()?;
+		let mut file_config = File::create(path_config.as_path())?;
+		writeln!(
+			file_config,
+			r#"
+			metadata_dir = "/tmp/garage/meta"
+			data_dir = "/tmp/garage/data"
+			replication_mode = "3"
+			rpc_bind_addr = "[::]:3901"
+			rpc_secret= "dummy"
+			rpc_secret_file = "dummy"
+
+			[s3_api]
+			s3_region = "garage"
+			api_bind_addr = "[::]:3900"
+			"#
+		)?;
+		let config = read_config(path_config.to_path_buf())?;
+		assert_eq!(
+			"only one of `rpc_secret` and `rpc_secret_file` can be set",
+			fill_secrets(config, Secrets::default())
+				.unwrap_err()
+				.to_string()
+		);
+		drop(path_config);
+		drop(file_config);
+		Ok(())
+	}
+}
diff --git a/src/garage/server.rs b/src/garage/server.rs
index 3ad10b72..51b06b8e 100644
--- a/src/garage/server.rs
+++ b/src/garage/server.rs
@@ -15,9 +15,9 @@ use garage_web::WebServer;
 use garage_api::k2v::api_server::K2VApiServer;
 
 use crate::admin::*;
+use crate::secrets::{fill_secrets, Secrets};
 #[cfg(feature = "telemetry-otlp")]
 use crate::tracing_setup::*;
-use crate::{fill_secrets, Secrets};
 
 async fn wait_from(mut chan: watch::Receiver<bool>) {
 	while !*chan.borrow() {
@@ -29,12 +29,19 @@ async fn wait_from(mut chan: watch::Receiver<bool>) {
 
 pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Error> {
 	info!("Loading configuration...");
-	let config = fill_secrets(read_config(config_file)?, secrets);
+	let config = fill_secrets(read_config(config_file)?, secrets)?;
 
 	// ---- Initialize Garage internals ----
 
 	#[cfg(feature = "metrics")]
-	let metrics_exporter = opentelemetry_prometheus::exporter().init();
+	let metrics_exporter = opentelemetry_prometheus::exporter()
+		.with_default_summary_quantiles(vec![0.25, 0.5, 0.75, 0.9, 0.95, 0.99])
+		.with_default_histogram_boundaries(vec![
+			0.001, 0.0015, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.02, 0.03, 0.05, 0.07, 0.1,
+			0.15, 0.2, 0.3, 0.5, 0.7, 1., 1.5, 2., 3., 5., 7., 10., 15., 20., 30., 40., 50., 60.,
+			70., 100.,
+		])
+		.init();
 
 	info!("Initializing Garage main data store...");
 	let garage = Garage::new(config.clone())?;
@@ -81,7 +88,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
 				garage.clone(),
 				s3_bind_addr.clone(),
 				config.s3_api.s3_region.clone(),
-				wait_from(watch_cancel.clone()),
+				watch_cancel.clone(),
 			)),
 		));
 	}
@@ -96,7 +103,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
 					garage.clone(),
 					config.k2v_api.as_ref().unwrap().api_bind_addr.clone(),
 					config.s3_api.s3_region.clone(),
-					wait_from(watch_cancel.clone()),
+					watch_cancel.clone(),
 				)),
 			));
 		}
@@ -106,14 +113,10 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
 
 	if let Some(web_config) = &config.s3_web {
 		info!("Initializing web server...");
+		let web_server = WebServer::new(garage.clone(), web_config.root_domain.clone());
 		servers.push((
 			"Web",
-			tokio::spawn(WebServer::run(
-				garage.clone(),
-				web_config.bind_addr.clone(),
-				web_config.root_domain.clone(),
-				wait_from(watch_cancel.clone()),
-			)),
+			tokio::spawn(web_server.run(web_config.bind_addr.clone(), watch_cancel.clone())),
 		));
 	}
 
@@ -121,9 +124,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
 		info!("Launching Admin API server...");
 		servers.push((
 			"Admin",
-			tokio::spawn(
-				admin_server.run(admin_bind_addr.clone(), wait_from(watch_cancel.clone())),
-			),
+			tokio::spawn(admin_server.run(admin_bind_addr.clone(), watch_cancel.clone())),
 		));
 	}
 
diff --git a/src/garage/tests/common/client.rs b/src/garage/tests/common/client.rs
index ef4daa5d..ffa4cae8 100644
--- a/src/garage/tests/common/client.rs
+++ b/src/garage/tests/common/client.rs
@@ -1,3 +1,4 @@
+use aws_sdk_s3::config::BehaviorVersion;
 use aws_sdk_s3::config::Credentials;
 use aws_sdk_s3::{Client, Config};
 
@@ -11,6 +12,7 @@ pub fn build_client(key: &Key) -> Client {
 		.endpoint_url(format!("http://127.0.0.1:{}", DEFAULT_PORT))
 		.region(super::REGION)
 		.credentials_provider(credentials)
+		.behavior_version(BehaviorVersion::v2023_11_09())
 		.build();
 
 	Client::from_conf(config)
diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs
index 4133bb8b..e5f4cca1 100644
--- a/src/garage/tests/common/custom_requester.rs
+++ b/src/garage/tests/common/custom_requester.rs
@@ -5,12 +5,17 @@ use std::convert::TryFrom;
 
 use chrono::{offset::Utc, DateTime};
 use hmac::{Hmac, Mac};
-use hyper::client::HttpConnector;
-use hyper::{Body, Client, Method, Request, Response, Uri};
+use http_body_util::BodyExt;
+use http_body_util::Full as FullBody;
+use hyper::{Method, Request, Response, Uri};
+use hyper_util::client::legacy::{connect::HttpConnector, Client};
+use hyper_util::rt::TokioExecutor;
 
 use super::garage::{Instance, Key};
 use garage_api::signature;
 
+pub type Body = FullBody<hyper::body::Bytes>;
+
 /// You should ever only use this to send requests AWS sdk won't send,
 /// like to reproduce behavior of unusual implementations found to be
 /// problematic.
@@ -19,7 +24,7 @@ pub struct CustomRequester {
 	key: Key,
 	uri: Uri,
 	service: &'static str,
-	client: Client<HttpConnector>,
+	client: Client<HttpConnector, Body>,
 }
 
 impl CustomRequester {
@@ -28,7 +33,7 @@ impl CustomRequester {
 			key: key.clone(),
 			uri: instance.s3_uri(),
 			service: "s3",
-			client: Client::new(),
+			client: Client::builder(TokioExecutor::new()).build_http(),
 		}
 	}
 
@@ -37,7 +42,7 @@ impl CustomRequester {
 			key: key.clone(),
 			uri: instance.k2v_uri(),
 			service: "k2v",
-			client: Client::new(),
+			client: Client::builder(TokioExecutor::new()).build_http(),
 		}
 	}
 
@@ -139,7 +144,7 @@ impl<'a> RequestBuilder<'a> {
 		self
 	}
 
-	pub async fn send(&mut self) -> hyper::Result<Response<Body>> {
+	pub async fn send(&mut self) -> Result<Response<Body>, String> {
 		// TODO this is a bit incorrect in that path and query params should be url-encoded and
 		// aren't, but this is good enought for now.
 
@@ -200,8 +205,8 @@ impl<'a> RequestBuilder<'a> {
 		all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone());
 
 		let mut signed_headers = all_headers
-			.iter()
-			.map(|(k, _)| k.as_ref())
+			.keys()
+			.map(|k| k.as_ref())
 			.collect::<Vec<&str>>();
 		signed_headers.sort();
 		let signed_headers = signed_headers.join(";");
@@ -242,7 +247,22 @@ impl<'a> RequestBuilder<'a> {
 			.method(self.method.clone())
 			.body(Body::from(body))
 			.unwrap();
-		self.requester.client.request(request).await
+
+		let result = self
+			.requester
+			.client
+			.request(request)
+			.await
+			.map_err(|err| format!("hyper client error: {}", err))?;
+
+		let (head, body) = result.into_parts();
+		let body = Body::new(
+			body.collect()
+				.await
+				.map_err(|err| format!("hyper client error in body.collect: {}", err))?
+				.to_bytes(),
+		);
+		Ok(Response::from_parts(head, body))
 	}
 }
 
diff --git a/src/garage/tests/k2v/batch.rs b/src/garage/tests/k2v/batch.rs
index 71de91bf..39554d4d 100644
--- a/src/garage/tests/k2v/batch.rs
+++ b/src/garage/tests/k2v/batch.rs
@@ -7,6 +7,7 @@ use base64::prelude::*;
 use serde_json::json;
 
 use crate::json_body;
+use http_body_util::BodyExt;
 use hyper::{Method, StatusCode};
 
 #[tokio::test]
@@ -77,10 +78,7 @@ async fn test_batch() {
 				.unwrap()
 				.to_string(),
 		);
-		let res_body = hyper::body::to_bytes(res.into_body())
-			.await
-			.unwrap()
-			.to_vec();
+		let res_body = res.into_body().collect().await.unwrap().to_bytes();
 		assert_eq!(res_body, values.get(sk).unwrap().as_bytes());
 	}
 
diff --git a/src/garage/tests/k2v/item.rs b/src/garage/tests/k2v/item.rs
index 20add889..5a347bd9 100644
--- a/src/garage/tests/k2v/item.rs
+++ b/src/garage/tests/k2v/item.rs
@@ -7,6 +7,7 @@ use base64::prelude::*;
 use serde_json::json;
 
 use crate::json_body;
+use http_body_util::BodyExt;
 use hyper::{Method, StatusCode};
 
 #[tokio::test]
@@ -83,10 +84,7 @@ async fn test_items_and_indices() {
 			.to_str()
 			.unwrap()
 			.to_string();
-		let res_body = hyper::body::to_bytes(res.into_body())
-			.await
-			.unwrap()
-			.to_vec();
+		let res_body = res.into_body().collect().await.unwrap().to_bytes();
 		assert_eq!(res_body, content);
 
 		// ReadIndex -- now there should be some stuff
@@ -152,10 +150,7 @@ async fn test_items_and_indices() {
 			res.headers().get("content-type").unwrap().to_str().unwrap(),
 			"application/octet-stream"
 		);
-		let res_body = hyper::body::to_bytes(res.into_body())
-			.await
-			.unwrap()
-			.to_vec();
+		let res_body = res.into_body().collect().await.unwrap().to_bytes();
 		assert_eq!(res_body, content2);
 
 		// ReadIndex -- now there should be some stuff
@@ -394,10 +389,7 @@ async fn test_item_return_format() {
 		.to_str()
 		.unwrap()
 		.to_string();
-	let res_body = hyper::body::to_bytes(res.into_body())
-		.await
-		.unwrap()
-		.to_vec();
+	let res_body = res.into_body().collect().await.unwrap().to_bytes();
 	assert_eq!(res_body, single_value);
 
 	// f1: not specified
@@ -434,10 +426,7 @@ async fn test_item_return_format() {
 		res.headers().get("content-type").unwrap().to_str().unwrap(),
 		"application/octet-stream"
 	);
-	let res_body = hyper::body::to_bytes(res.into_body())
-		.await
-		.unwrap()
-		.to_vec();
+	let res_body = res.into_body().collect().await.unwrap().to_bytes();
 	assert_eq!(res_body, single_value);
 
 	// f3: json
diff --git a/src/garage/tests/k2v/poll.rs b/src/garage/tests/k2v/poll.rs
index 452317c2..277f8bc8 100644
--- a/src/garage/tests/k2v/poll.rs
+++ b/src/garage/tests/k2v/poll.rs
@@ -1,4 +1,5 @@
 use base64::prelude::*;
+use http_body_util::BodyExt;
 use hyper::{Method, StatusCode};
 use std::time::Duration;
 
@@ -47,11 +48,8 @@ async fn test_poll_item() {
 		.unwrap()
 		.to_string();
 
-	let res2_body = hyper::body::to_bytes(res2.into_body())
-		.await
-		.unwrap()
-		.to_vec();
-	assert_eq!(res2_body, b"Initial value");
+	let res2_body = res2.into_body().collect().await.unwrap().to_bytes();
+	assert_eq!(res2_body, b"Initial value"[..]);
 
 	// Start poll operation
 	let poll = {
@@ -95,11 +93,8 @@ async fn test_poll_item() {
 
 	assert_eq!(poll_res.status(), StatusCode::OK);
 
-	let poll_res_body = hyper::body::to_bytes(poll_res.into_body())
-		.await
-		.unwrap()
-		.to_vec();
-	assert_eq!(poll_res_body, b"New value");
+	let poll_res_body = poll_res.into_body().collect().await.unwrap().to_bytes();
+	assert_eq!(poll_res_body, b"New value"[..]);
 }
 
 #[tokio::test]
diff --git a/src/garage/tests/k2v/simple.rs b/src/garage/tests/k2v/simple.rs
index 465fc24d..1017330d 100644
--- a/src/garage/tests/k2v/simple.rs
+++ b/src/garage/tests/k2v/simple.rs
@@ -1,5 +1,6 @@
 use crate::common;
 
+use http_body_util::BodyExt;
 use hyper::{Method, StatusCode};
 
 #[tokio::test]
@@ -32,9 +33,6 @@ async fn test_simple() {
 		.unwrap();
 	assert_eq!(res2.status(), StatusCode::OK);
 
-	let res2_body = hyper::body::to_bytes(res2.into_body())
-		.await
-		.unwrap()
-		.to_vec();
-	assert_eq!(res2_body, b"Hello, world!");
+	let res2_body = res2.into_body().collect().await.unwrap().to_bytes();
+	assert_eq!(res2_body, b"Hello, world!"[..]);
 }
diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs
index ab92bc0a..ef370db3 100644
--- a/src/garage/tests/lib.rs
+++ b/src/garage/tests/lib.rs
@@ -11,15 +11,15 @@ mod k2v;
 #[cfg(feature = "k2v")]
 mod k2v_client;
 
-use hyper::{Body, Response};
+use http_body_util::BodyExt;
+use hyper::{body::Body, Response};
 
-pub async fn json_body(res: Response<Body>) -> serde_json::Value {
-	let res_body: serde_json::Value = serde_json::from_slice(
-		&hyper::body::to_bytes(res.into_body())
-			.await
-			.unwrap()
-			.to_vec()[..],
-	)
-	.unwrap();
+pub async fn json_body<B>(res: Response<B>) -> serde_json::Value
+where
+	B: Body,
+	<B as Body>::Error: std::fmt::Debug,
+{
+	let body = res.into_body().collect().await.unwrap().to_bytes();
+	let res_body: serde_json::Value = serde_json::from_slice(&body).unwrap();
 	res_body
 }
diff --git a/src/garage/tests/s3/list.rs b/src/garage/tests/s3/list.rs
index bb03f250..1b0c006d 100644
--- a/src/garage/tests/s3/list.rs
+++ b/src/garage/tests/s3/list.rs
@@ -613,3 +613,63 @@ async fn test_listmultipart() {
 		assert!(r.common_prefixes.is_none());
 	}
 }
+
+#[tokio::test]
+async fn test_multichar_delimiter() {
+	// Test case from dpape from issue #692 with reference results from Amazon
+
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("multichardelim");
+
+	for k in [
+		"a/", "a/b/", "a/b/c/", "a/b/c/d", "a/c/", "a/c/b/", "a/c/b/e",
+	] {
+		ctx.client
+			.put_object()
+			.bucket(&bucket)
+			.key(k)
+			.send()
+			.await
+			.unwrap();
+	}
+
+	// With delimiter /
+	{
+		let r = ctx
+			.client
+			.list_objects_v2()
+			.bucket(&bucket)
+			.delimiter("/")
+			.send()
+			.await
+			.unwrap();
+
+		assert!(r.contents.is_none());
+
+		let common_prefixes = r.common_prefixes.unwrap();
+		assert_eq!(common_prefixes.len(), 1);
+		assert_eq!(common_prefixes[0].prefix.as_deref().unwrap(), "a/");
+	}
+
+	// With delimiter b/
+	{
+		let r = ctx
+			.client
+			.list_objects_v2()
+			.bucket(&bucket)
+			.delimiter("b/")
+			.send()
+			.await
+			.unwrap();
+
+		let contents = r.contents.unwrap();
+		assert_eq!(contents.len(), 2);
+		assert_eq!(contents[0].key.as_deref().unwrap(), "a/");
+		assert_eq!(contents[1].key.as_deref().unwrap(), "a/c/");
+
+		let common_prefixes = r.common_prefixes.unwrap();
+		assert_eq!(common_prefixes.len(), 2);
+		assert_eq!(common_prefixes[0].prefix.as_deref().unwrap(), "a/b/");
+		assert_eq!(common_prefixes[1].prefix.as_deref().unwrap(), "a/c/b/");
+	}
+}
diff --git a/src/garage/tests/s3/multipart.rs b/src/garage/tests/s3/multipart.rs
index 09ae5e5b..51c9df74 100644
--- a/src/garage/tests/s3/multipart.rs
+++ b/src/garage/tests/s3/multipart.rs
@@ -154,7 +154,7 @@ async fn test_multipart_upload() {
 			.await
 			.unwrap();
 
-		assert_eq!(r.content_length, (SZ_5MB * 3) as i64);
+		assert_eq!(r.content_length.unwrap(), (SZ_5MB * 3) as i64);
 	}
 
 	{
@@ -183,7 +183,7 @@ async fn test_multipart_upload() {
 				.unwrap();
 
 			eprintln!("get_object with part_number = {}", part_number);
-			assert_eq!(o.content_length, SZ_5MB as i64);
+			assert_eq!(o.content_length.unwrap(), SZ_5MB as i64);
 			assert_bytes_eq!(o.body, data);
 		}
 	}
@@ -249,14 +249,14 @@ async fn test_uploadlistpart() {
 
 		let ps = r.parts.unwrap();
 		assert_eq!(ps.len(), 1);
-		assert_eq!(ps[0].part_number, 2);
+		assert_eq!(ps[0].part_number.unwrap(), 2);
 		let fp = &ps[0];
 		assert!(fp.last_modified.is_some());
 		assert_eq!(
 			fp.e_tag.as_ref().unwrap(),
 			"\"3366bb9dcf710d6801b5926467d02e19\""
 		);
-		assert_eq!(fp.size, SZ_5MB as i64);
+		assert_eq!(fp.size.unwrap(), SZ_5MB as i64);
 	}
 
 	let p2 = ctx
@@ -286,23 +286,23 @@ async fn test_uploadlistpart() {
 		let ps = r.parts.unwrap();
 		assert_eq!(ps.len(), 2);
 
-		assert_eq!(ps[0].part_number, 1);
+		assert_eq!(ps[0].part_number.unwrap(), 1);
 		let fp = &ps[0];
 		assert!(fp.last_modified.is_some());
 		assert_eq!(
 			fp.e_tag.as_ref().unwrap(),
 			"\"3c484266f9315485694556e6c693bfa2\""
 		);
-		assert_eq!(fp.size, SZ_5MB as i64);
+		assert_eq!(fp.size.unwrap(), SZ_5MB as i64);
 
-		assert_eq!(ps[1].part_number, 2);
+		assert_eq!(ps[1].part_number.unwrap(), 2);
 		let sp = &ps[1];
 		assert!(sp.last_modified.is_some());
 		assert_eq!(
 			sp.e_tag.as_ref().unwrap(),
 			"\"3366bb9dcf710d6801b5926467d02e19\""
 		);
-		assert_eq!(sp.size, SZ_5MB as i64);
+		assert_eq!(sp.size.unwrap(), SZ_5MB as i64);
 	}
 
 	{
@@ -320,14 +320,14 @@ async fn test_uploadlistpart() {
 
 		assert!(r.part_number_marker.is_none());
 		assert_eq!(r.next_part_number_marker.as_deref(), Some("1"));
-		assert_eq!(r.max_parts, 1_i32);
-		assert!(r.is_truncated);
+		assert_eq!(r.max_parts.unwrap(), 1_i32);
+		assert!(r.is_truncated.unwrap());
 		assert_eq!(r.key.unwrap(), "a");
 		assert_eq!(r.upload_id.unwrap().as_str(), uid.as_str());
 		let parts = r.parts.unwrap();
 		assert_eq!(parts.len(), 1);
 		let fp = &parts[0];
-		assert_eq!(fp.part_number, 1);
+		assert_eq!(fp.part_number.unwrap(), 1);
 		assert_eq!(
 			fp.e_tag.as_ref().unwrap(),
 			"\"3c484266f9315485694556e6c693bfa2\""
@@ -349,19 +349,19 @@ async fn test_uploadlistpart() {
 			r2.part_number_marker.as_ref().unwrap(),
 			r.next_part_number_marker.as_ref().unwrap()
 		);
-		assert_eq!(r2.max_parts, 1_i32);
+		assert_eq!(r2.max_parts.unwrap(), 1_i32);
 		assert_eq!(r2.key.unwrap(), "a");
 		assert_eq!(r2.upload_id.unwrap().as_str(), uid.as_str());
 		let parts = r2.parts.unwrap();
 		assert_eq!(parts.len(), 1);
 		let fp = &parts[0];
-		assert_eq!(fp.part_number, 2);
+		assert_eq!(fp.part_number.unwrap(), 2);
 		assert_eq!(
 			fp.e_tag.as_ref().unwrap(),
 			"\"3366bb9dcf710d6801b5926467d02e19\""
 		);
 		//assert!(r2.is_truncated);   // WHY? (this was the test before)
-		assert!(!r2.is_truncated);
+		assert!(!r2.is_truncated.unwrap());
 	}
 
 	let cmp = CompletedMultipartUpload::builder()
@@ -411,7 +411,7 @@ async fn test_uploadlistpart() {
 			.await
 			.unwrap();
 
-		assert_eq!(r.content_length, (SZ_5MB * 2) as i64);
+		assert_eq!(r.content_length.unwrap(), (SZ_5MB * 2) as i64);
 	}
 }
 
diff --git a/src/garage/tests/s3/objects.rs b/src/garage/tests/s3/objects.rs
index 27697d45..ad5f63f1 100644
--- a/src/garage/tests/s3/objects.rs
+++ b/src/garage/tests/s3/objects.rs
@@ -50,9 +50,9 @@ async fn test_putobject() {
 		// assert_eq!(o.version_id.unwrap(), _version);
 		assert_eq!(o.content_type.unwrap(), content_type);
 		assert!(o.last_modified.is_some());
-		assert_eq!(o.content_length, 0);
-		assert_eq!(o.parts_count, 0);
-		assert_eq!(o.tag_count, 0);
+		assert_eq!(o.content_length.unwrap(), 0);
+		assert_eq!(o.parts_count, None);
+		assert_eq!(o.tag_count, None);
 	}
 
 	{
@@ -86,9 +86,9 @@ async fn test_putobject() {
 		assert_bytes_eq!(o.body, b"hi");
 		assert_eq!(o.e_tag.unwrap(), etag);
 		assert!(o.last_modified.is_some());
-		assert_eq!(o.content_length, 2);
-		assert_eq!(o.parts_count, 0);
-		assert_eq!(o.tag_count, 0);
+		assert_eq!(o.content_length.unwrap(), 2);
+		assert_eq!(o.parts_count, None);
+		assert_eq!(o.tag_count, None);
 	}
 
 	{
@@ -119,9 +119,9 @@ async fn test_putobject() {
 		assert_bytes_eq!(o.body, b"");
 		assert_eq!(o.e_tag.unwrap(), etag);
 		assert!(o.last_modified.is_some());
-		assert_eq!(o.content_length, 0);
-		assert_eq!(o.parts_count, 0);
-		assert_eq!(o.tag_count, 0);
+		assert_eq!(o.content_length.unwrap(), 0);
+		assert_eq!(o.parts_count, None);
+		assert_eq!(o.tag_count, None);
 	}
 }
 
@@ -185,6 +185,30 @@ async fn test_getobject() {
 		assert_eq!(o.content_range.unwrap().as_str(), "bytes 57-61/62");
 		assert_bytes_eq!(o.body, &BODY[57..]);
 	}
+	{
+		let exp = aws_sdk_s3::primitives::DateTime::from_secs(10000000000);
+		let o = ctx
+			.client
+			.get_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.response_content_type("application/x-dummy-test")
+			.response_cache_control("ccdummy")
+			.response_content_disposition("cddummy")
+			.response_content_encoding("cedummy")
+			.response_content_language("cldummy")
+			.response_expires(exp)
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(o.content_type.unwrap().as_str(), "application/x-dummy-test");
+		assert_eq!(o.cache_control.unwrap().as_str(), "ccdummy");
+		assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy");
+		assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy");
+		assert_eq!(o.content_language.unwrap().as_str(), "cldummy");
+		assert_eq!(o.expires.unwrap(), exp);
+		assert_bytes_eq!(o.body, &BODY[..]);
+	}
 }
 
 #[tokio::test]
@@ -205,7 +229,7 @@ async fn test_deleteobject() {
 			.await
 			.unwrap();
 		if i > 0 {
-			to_del = to_del.objects(ObjectIdentifier::builder().key(k).build());
+			to_del = to_del.objects(ObjectIdentifier::builder().key(k).build().unwrap());
 		}
 	}
 
@@ -223,7 +247,7 @@ async fn test_deleteobject() {
 			.unwrap();
 
 		if i > 0 {
-			to_del = to_del.objects(ObjectIdentifier::builder().key(k).build());
+			to_del = to_del.objects(ObjectIdentifier::builder().key(k).build().unwrap());
 		}
 	}
 
@@ -247,7 +271,7 @@ async fn test_deleteobject() {
 		.client
 		.delete_objects()
 		.bucket(&bucket)
-		.delete(to_del.build())
+		.delete(to_del.build().unwrap())
 		.send()
 		.await
 		.unwrap();
diff --git a/src/garage/tests/s3/streaming_signature.rs b/src/garage/tests/s3/streaming_signature.rs
index b7a1acae..224b9ed5 100644
--- a/src/garage/tests/s3/streaming_signature.rs
+++ b/src/garage/tests/s3/streaming_signature.rs
@@ -57,9 +57,9 @@ async fn test_putobject_streaming() {
 		// assert_eq!(o.version_id.unwrap(), _version);
 		assert_eq!(o.content_type.unwrap(), content_type);
 		assert!(o.last_modified.is_some());
-		assert_eq!(o.content_length, 0);
-		assert_eq!(o.parts_count, 0);
-		assert_eq!(o.tag_count, 0);
+		assert_eq!(o.content_length.unwrap(), 0);
+		assert_eq!(o.parts_count, None);
+		assert_eq!(o.tag_count, None);
 	}
 
 	{
@@ -95,9 +95,9 @@ async fn test_putobject_streaming() {
 		assert_bytes_eq!(o.body, BODY);
 		assert_eq!(o.e_tag.unwrap(), etag);
 		assert!(o.last_modified.is_some());
-		assert_eq!(o.content_length, 62);
-		assert_eq!(o.parts_count, 0);
-		assert_eq!(o.tag_count, 0);
+		assert_eq!(o.content_length.unwrap(), 62);
+		assert_eq!(o.parts_count, None);
+		assert_eq!(o.tag_count, None);
 	}
 }
 
@@ -187,7 +187,7 @@ async fn test_put_website_streaming() {
 			.await
 			.unwrap();
 
-		assert_eq!(o.index_document.unwrap().suffix.unwrap(), "home.html");
-		assert_eq!(o.error_document.unwrap().key.unwrap(), "err/error.html");
+		assert_eq!(o.index_document.unwrap().suffix, "home.html");
+		assert_eq!(o.error_document.unwrap().key, "err/error.html");
 	}
 }
diff --git a/src/garage/tests/s3/website.rs b/src/garage/tests/s3/website.rs
index eeafb5fa..0cadc388 100644
--- a/src/garage/tests/s3/website.rs
+++ b/src/garage/tests/s3/website.rs
@@ -8,15 +8,18 @@ use aws_sdk_s3::{
 	types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
 };
 use http::{Request, StatusCode};
-use hyper::{
-	body::{to_bytes, Body},
-	Client,
-};
+use http_body_util::BodyExt;
+use http_body_util::Full as FullBody;
+use hyper::body::Bytes;
+use hyper_util::client::legacy::Client;
+use hyper_util::rt::TokioExecutor;
 use serde_json::json;
 
 const BODY: &[u8; 16] = b"<h1>bonjour</h1>";
 const BODY_ERR: &[u8; 6] = b"erreur";
 
+pub type Body = FullBody<Bytes>;
+
 #[tokio::test]
 async fn test_website() {
 	const BCKT_NAME: &str = "my-website";
@@ -34,14 +37,14 @@ async fn test_website() {
 		.await
 		.unwrap();
 
-	let client = Client::new();
+	let client = Client::builder(TokioExecutor::new()).build_http();
 
 	let req = || {
 		Request::builder()
 			.method("GET")
 			.uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port))
 			.header("Host", format!("{}.web.garage", BCKT_NAME))
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap()
 	};
 
@@ -49,7 +52,7 @@ async fn test_website() {
 
 	assert_eq!(resp.status(), StatusCode::NOT_FOUND);
 	assert_ne!(
-		to_bytes(resp.body_mut()).await.unwrap().as_ref(),
+		BodyExt::collect(resp.into_body()).await.unwrap().to_bytes(),
 		BODY.as_ref()
 	); /* check that we do not leak body */
 
@@ -58,10 +61,9 @@ async fn test_website() {
 			.method("GET")
 			.uri(format!(
 				"http://127.0.0.1:{0}/check?domain={1}",
-				ctx.garage.admin_port,
-				BCKT_NAME.to_string()
+				ctx.garage.admin_port, BCKT_NAME
 			))
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap()
 	};
 
@@ -87,7 +89,7 @@ async fn test_website() {
 	resp = client.request(req()).await.unwrap();
 	assert_eq!(resp.status(), StatusCode::OK);
 	assert_eq!(
-		to_bytes(resp.body_mut()).await.unwrap().as_ref(),
+		resp.into_body().collect().await.unwrap().to_bytes(),
 		BODY.as_ref()
 	);
 
@@ -103,14 +105,14 @@ async fn test_website() {
 					"http://127.0.0.1:{0}/check?domain={1}",
 					ctx.garage.admin_port, bname
 				))
-				.body(Body::empty())
+				.body(Body::new(Bytes::new()))
 				.unwrap()
 		};
 
-		let mut admin_resp = client.request(admin_req()).await.unwrap();
+		let admin_resp = client.request(admin_req()).await.unwrap();
 		assert_eq!(admin_resp.status(), StatusCode::OK);
 		assert_eq!(
-			to_bytes(admin_resp.body_mut()).await.unwrap().as_ref(),
+			admin_resp.into_body().collect().await.unwrap().to_bytes(),
 			format!("Domain '{bname}' is managed by Garage").as_bytes()
 		);
 	}
@@ -124,7 +126,7 @@ async fn test_website() {
 	resp = client.request(req()).await.unwrap();
 	assert_eq!(resp.status(), StatusCode::NOT_FOUND);
 	assert_ne!(
-		to_bytes(resp.body_mut()).await.unwrap().as_ref(),
+		resp.into_body().collect().await.unwrap().to_bytes(),
 		BODY.as_ref()
 	); /* check that we do not leak body */
 
@@ -133,10 +135,9 @@ async fn test_website() {
 			.method("GET")
 			.uri(format!(
 				"http://127.0.0.1:{0}/check?domain={1}",
-				ctx.garage.admin_port,
-				BCKT_NAME.to_string()
+				ctx.garage.admin_port, BCKT_NAME
 			))
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap()
 	};
 
@@ -181,8 +182,18 @@ async fn test_website_s3_api() {
 		.unwrap();
 
 	let conf = WebsiteConfiguration::builder()
-		.index_document(IndexDocument::builder().suffix("home.html").build())
-		.error_document(ErrorDocument::builder().key("err/error.html").build())
+		.index_document(
+			IndexDocument::builder()
+				.suffix("home.html")
+				.build()
+				.unwrap(),
+		)
+		.error_document(
+			ErrorDocument::builder()
+				.key("err/error.html")
+				.build()
+				.unwrap(),
+		)
 		.build();
 
 	ctx.client
@@ -201,9 +212,11 @@ async fn test_website_s3_api() {
 				.allowed_methods("GET")
 				.allowed_methods("PUT")
 				.allowed_origins("*")
-				.build(),
+				.build()
+				.unwrap(),
 		)
-		.build();
+		.build()
+		.unwrap();
 
 	ctx.client
 		.put_bucket_cors()
@@ -222,24 +235,21 @@ async fn test_website_s3_api() {
 			.await
 			.unwrap();
 
-		let main_rule = cors_res.cors_rules().unwrap().iter().next().unwrap();
+		let main_rule = cors_res.cors_rules().iter().next().unwrap();
 
 		assert_eq!(main_rule.id.as_ref().unwrap(), "main-rule");
 		assert_eq!(
 			main_rule.allowed_headers.as_ref().unwrap(),
 			&vec!["*".to_string()]
 		);
+		assert_eq!(&main_rule.allowed_origins, &vec!["*".to_string()]);
 		assert_eq!(
-			main_rule.allowed_origins.as_ref().unwrap(),
-			&vec!["*".to_string()]
-		);
-		assert_eq!(
-			main_rule.allowed_methods.as_ref().unwrap(),
+			&main_rule.allowed_methods,
 			&vec!["GET".to_string(), "PUT".to_string()]
 		);
 	}
 
-	let client = Client::new();
+	let client = Client::builder(TokioExecutor::new()).build_http();
 
 	// Test direct requests with CORS
 	{
@@ -248,10 +258,10 @@ async fn test_website_s3_api() {
 			.uri(format!("http://127.0.0.1:{}/site/", ctx.garage.web_port))
 			.header("Host", format!("{}.web.garage", BCKT_NAME))
 			.header("Origin", "https://example.com")
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap();
 
-		let mut resp = client.request(req).await.unwrap();
+		let resp = client.request(req).await.unwrap();
 
 		assert_eq!(resp.status(), StatusCode::OK);
 		assert_eq!(
@@ -259,7 +269,7 @@ async fn test_website_s3_api() {
 			"*"
 		);
 		assert_eq!(
-			to_bytes(resp.body_mut()).await.unwrap().as_ref(),
+			resp.into_body().collect().await.unwrap().to_bytes(),
 			BODY.as_ref()
 		);
 	}
@@ -273,14 +283,14 @@ async fn test_website_s3_api() {
 				ctx.garage.web_port
 			))
 			.header("Host", format!("{}.web.garage", BCKT_NAME))
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap();
 
-		let mut resp = client.request(req).await.unwrap();
+		let resp = client.request(req).await.unwrap();
 
 		assert_eq!(resp.status(), StatusCode::NOT_FOUND);
 		assert_eq!(
-			to_bytes(resp.body_mut()).await.unwrap().as_ref(),
+			resp.into_body().collect().await.unwrap().to_bytes(),
 			BODY_ERR.as_ref()
 		);
 	}
@@ -293,10 +303,10 @@ async fn test_website_s3_api() {
 			.header("Host", format!("{}.web.garage", BCKT_NAME))
 			.header("Origin", "https://example.com")
 			.header("Access-Control-Request-Method", "PUT")
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap();
 
-		let mut resp = client.request(req).await.unwrap();
+		let resp = client.request(req).await.unwrap();
 
 		assert_eq!(resp.status(), StatusCode::OK);
 		assert_eq!(
@@ -304,7 +314,7 @@ async fn test_website_s3_api() {
 			"*"
 		);
 		assert_ne!(
-			to_bytes(resp.body_mut()).await.unwrap().as_ref(),
+			resp.into_body().collect().await.unwrap().to_bytes(),
 			BODY.as_ref()
 		);
 	}
@@ -317,14 +327,14 @@ async fn test_website_s3_api() {
 			.header("Host", format!("{}.web.garage", BCKT_NAME))
 			.header("Origin", "https://example.com")
 			.header("Access-Control-Request-Method", "DELETE")
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap();
 
-		let mut resp = client.request(req).await.unwrap();
+		let resp = client.request(req).await.unwrap();
 
 		assert_eq!(resp.status(), StatusCode::FORBIDDEN);
 		assert_ne!(
-			to_bytes(resp.body_mut()).await.unwrap().as_ref(),
+			resp.into_body().collect().await.unwrap().to_bytes(),
 			BODY.as_ref()
 		);
 	}
@@ -358,14 +368,14 @@ async fn test_website_s3_api() {
 			.header("Host", format!("{}.web.garage", BCKT_NAME))
 			.header("Origin", "https://example.com")
 			.header("Access-Control-Request-Method", "PUT")
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap();
 
-		let mut resp = client.request(req).await.unwrap();
+		let resp = client.request(req).await.unwrap();
 
 		assert_eq!(resp.status(), StatusCode::FORBIDDEN);
 		assert_ne!(
-			to_bytes(resp.body_mut()).await.unwrap().as_ref(),
+			resp.into_body().collect().await.unwrap().to_bytes(),
 			BODY.as_ref()
 		);
 	}
@@ -384,20 +394,15 @@ async fn test_website_s3_api() {
 			.method("GET")
 			.uri(format!("http://127.0.0.1:{}/site/", ctx.garage.web_port))
 			.header("Host", format!("{}.web.garage", BCKT_NAME))
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap();
 
-		let mut resp = client.request(req).await.unwrap();
+		let resp = client.request(req).await.unwrap();
 
 		assert_eq!(resp.status(), StatusCode::NOT_FOUND);
-		assert_ne!(
-			to_bytes(resp.body_mut()).await.unwrap().as_ref(),
-			BODY_ERR.as_ref()
-		);
-		assert_ne!(
-			to_bytes(resp.body_mut()).await.unwrap().as_ref(),
-			BODY.as_ref()
-		);
+		let resp_bytes = resp.into_body().collect().await.unwrap().to_bytes();
+		assert_ne!(resp_bytes, BODY_ERR.as_ref());
+		assert_ne!(resp_bytes, BODY.as_ref());
 	}
 }
 
@@ -405,13 +410,13 @@ async fn test_website_s3_api() {
 async fn test_website_check_domain() {
 	let ctx = common::context();
 
-	let client = Client::new();
+	let client = Client::builder(TokioExecutor::new()).build_http();
 
 	let admin_req = || {
 		Request::builder()
 			.method("GET")
 			.uri(format!("http://127.0.0.1:{}/check", ctx.garage.admin_port))
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap()
 	};
 
@@ -435,7 +440,7 @@ async fn test_website_check_domain() {
 				"http://127.0.0.1:{}/check?domain=",
 				ctx.garage.admin_port
 			))
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap()
 	};
 
@@ -459,7 +464,7 @@ async fn test_website_check_domain() {
 				"http://127.0.0.1:{}/check?domain=foobar",
 				ctx.garage.admin_port
 			))
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap()
 	};
 
@@ -483,7 +488,7 @@ async fn test_website_check_domain() {
 				"http://127.0.0.1:{}/check?domain=%E2%98%B9",
 				ctx.garage.admin_port
 			))
-			.body(Body::empty())
+			.body(Body::new(Bytes::new()))
 			.unwrap()
 	};