From 4a19ee94bb9c846af1c74db8ba501b4ff625a3f6 Mon Sep 17 00:00:00 2001 From: networkException Date: Thu, 19 Oct 2023 03:31:50 +0200 Subject: garage: fix admin-token description --- src/garage/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/garage') diff --git a/src/garage/main.rs b/src/garage/main.rs index e8aee892..09e77b35 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -70,7 +70,7 @@ pub struct Secrets { #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")] pub rpc_secret: Option, - /// Metrics API authentication token, replaces admin.metrics_token in config.toml when + /// Admin API authentication token, replaces admin.admin_token in config.toml when /// running the Garage daemon #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")] pub admin_token: Option, -- cgit v1.2.3 From 8599051c492d7df22305e4c65659395d9102955c Mon Sep 17 00:00:00 2001 From: networkException Date: Thu, 19 Oct 2023 03:39:02 +0200 Subject: garage: support specifying token / secret as environment variables this patch adds support for specifying the `rpc_secret_file`, `metrics_token_file` and `admin_token_file` as environment variables. --- src/garage/main.rs | 30 +++++++++++++++++++++++++++--- src/garage/repair/offline.rs | 2 +- src/garage/server.rs | 2 +- 3 files changed, 29 insertions(+), 5 deletions(-) (limited to 'src/garage') diff --git a/src/garage/main.rs b/src/garage/main.rs index 09e77b35..ab84fa11 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -25,7 +25,7 @@ use structopt::StructOpt; use netapp::util::parse_and_resolve_peer_addr; use netapp::NetworkKey; -use garage_util::config::Config; +use garage_util::config::{read_secret_file, Config}; use garage_util::error::*; use garage_rpc::system::*; @@ -70,15 +70,30 @@ pub struct Secrets { #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")] pub rpc_secret: Option, + /// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret + /// when running the daemon or doing admin operations + #[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")] + pub rpc_secret_file: Option, + /// Admin API authentication token, replaces admin.admin_token in config.toml when /// running the Garage daemon #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")] pub admin_token: Option, + /// Admin API authentication token file path, replaces admin.admin_token in config.toml + /// and admin-token when running the Garage daemon + #[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")] + pub admin_token_file: Option, + /// Metrics API authentication token, replaces admin.metrics_token in config.toml when /// running the Garage daemon #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")] pub metrics_token: Option, + + /// Metrics API authentication token file path, replaces admin.metrics_token in config.toml + /// and metrics-token when running the Garage daemon + #[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")] + pub metrics_token_file: Option, } #[tokio::main] @@ -256,15 +271,24 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { } } -fn fill_secrets(mut config: Config, secrets: Secrets) -> Config { +fn fill_secrets(mut config: Config, secrets: Secrets) -> Result { if secrets.rpc_secret.is_some() { config.rpc_secret = secrets.rpc_secret; + } else if secrets.rpc_secret_file.is_some() { + config.rpc_secret = Some(read_secret_file(&secrets.rpc_secret_file.unwrap())?); } + if secrets.admin_token.is_some() { config.admin.admin_token = secrets.admin_token; + } else if secrets.admin_token_file.is_some() { + config.admin.admin_token = Some(read_secret_file(&secrets.admin_token_file.unwrap())?); } + if secrets.metrics_token.is_some() { config.admin.metrics_token = secrets.metrics_token; + } else if secrets.metrics_token_file.is_some() { + config.admin.metrics_token = Some(read_secret_file(&secrets.metrics_token_file.unwrap())?); } - config + + Ok(config) } diff --git a/src/garage/repair/offline.rs b/src/garage/repair/offline.rs index f4edcf03..beb48d65 100644 --- a/src/garage/repair/offline.rs +++ b/src/garage/repair/offline.rs @@ -20,7 +20,7 @@ pub async fn offline_repair( } info!("Loading configuration..."); - let config = fill_secrets(read_config(config_file)?, secrets); + let config = fill_secrets(read_config(config_file)?, secrets)?; info!("Initializing Garage main data store..."); let garage = Garage::new(config)?; diff --git a/src/garage/server.rs b/src/garage/server.rs index 3ad10b72..96ea900d 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -29,7 +29,7 @@ async fn wait_from(mut chan: watch::Receiver) { pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Error> { info!("Loading configuration..."); - let config = fill_secrets(read_config(config_file)?, secrets); + let config = fill_secrets(read_config(config_file)?, secrets)?; // ---- Initialize Garage internals ---- -- cgit v1.2.3 From 7228695ee288012103355589caa1ab5dd666b164 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 15 Jan 2024 17:18:46 +0100 Subject: config: refactor secret sourcing --- src/garage/Cargo.toml | 1 + src/garage/main.rs | 58 +-------- src/garage/repair/offline.rs | 2 +- src/garage/secrets.rs | 280 +++++++++++++++++++++++++++++++++++++++++++ src/garage/server.rs | 2 +- 5 files changed, 285 insertions(+), 58 deletions(-) create mode 100644 src/garage/secrets.rs (limited to 'src/garage') diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 35d87a3e..00975738 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -67,6 +67,7 @@ chrono = "0.4" http = "0.2" hmac = "0.12" hyper = { version = "0.14", features = ["client", "http1", "runtime"] } +mktemp = "0.5" sha2 = "0.10" static_init = "1.0" diff --git a/src/garage/main.rs b/src/garage/main.rs index a9f1ad29..d89762e4 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -7,6 +7,7 @@ extern crate tracing; mod admin; mod cli; mod repair; +mod secrets; mod server; #[cfg(feature = "telemetry-otlp")] mod tracing_setup; @@ -25,7 +26,6 @@ use structopt::StructOpt; use netapp::util::parse_and_resolve_peer_addr; use netapp::NetworkKey; -use garage_util::config::{read_secret_file, Config}; use garage_util::error::*; use garage_rpc::system::*; @@ -35,6 +35,7 @@ use garage_model::helper::error::Error as HelperError; use admin::*; use cli::*; +use secrets::Secrets; #[derive(StructOpt, Debug)] #[structopt( @@ -63,39 +64,6 @@ struct Opt { cmd: Command, } -#[derive(StructOpt, Debug)] -pub struct Secrets { - /// RPC secret network key, used to replace rpc_secret in config.toml when running the - /// daemon or doing admin operations - #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")] - pub rpc_secret: Option, - - /// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret - /// when running the daemon or doing admin operations - #[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")] - pub rpc_secret_file: Option, - - /// Admin API authentication token, replaces admin.admin_token in config.toml when - /// running the Garage daemon - #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")] - pub admin_token: Option, - - /// Admin API authentication token file path, replaces admin.admin_token in config.toml - /// and admin-token when running the Garage daemon - #[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")] - pub admin_token_file: Option, - - /// Metrics API authentication token, replaces admin.metrics_token in config.toml when - /// running the Garage daemon - #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")] - pub metrics_token: Option, - - /// Metrics API authentication token file path, replaces admin.metrics_token in config.toml - /// and metrics-token when running the Garage daemon - #[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")] - pub metrics_token_file: Option, -} - #[tokio::main] async fn main() { // Initialize version and features info @@ -273,25 +241,3 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { Ok(x) => Ok(x), } } - -fn fill_secrets(mut config: Config, secrets: Secrets) -> Result { - if secrets.rpc_secret.is_some() { - config.rpc_secret = secrets.rpc_secret; - } else if secrets.rpc_secret_file.is_some() { - config.rpc_secret = Some(read_secret_file(&secrets.rpc_secret_file.unwrap())?); - } - - if secrets.admin_token.is_some() { - config.admin.admin_token = secrets.admin_token; - } else if secrets.admin_token_file.is_some() { - config.admin.admin_token = Some(read_secret_file(&secrets.admin_token_file.unwrap())?); - } - - if secrets.metrics_token.is_some() { - config.admin.metrics_token = secrets.metrics_token; - } else if secrets.metrics_token_file.is_some() { - config.admin.metrics_token = Some(read_secret_file(&secrets.metrics_token_file.unwrap())?); - } - - Ok(config) -} diff --git a/src/garage/repair/offline.rs b/src/garage/repair/offline.rs index beb48d65..45024e71 100644 --- a/src/garage/repair/offline.rs +++ b/src/garage/repair/offline.rs @@ -6,7 +6,7 @@ use garage_util::error::*; use garage_model::garage::Garage; use crate::cli::structs::*; -use crate::{fill_secrets, Secrets}; +use crate::secrets::{fill_secrets, Secrets}; pub async fn offline_repair( config_file: PathBuf, diff --git a/src/garage/secrets.rs b/src/garage/secrets.rs new file mode 100644 index 00000000..c40c3cc9 --- /dev/null +++ b/src/garage/secrets.rs @@ -0,0 +1,280 @@ +use structopt::StructOpt; + +use garage_util::config::Config; +use garage_util::error::Error; + +/// Structure for secret values or paths that are passed as CLI arguments or environment +/// variables, instead of in the config file. +#[derive(StructOpt, Debug, Default, Clone)] +pub struct Secrets { + /// Skip permission check on files containing secrets + #[cfg(unix)] + #[structopt( + long = "allow-world-readable-secrets", + env = "GARAGE_ALLOW_WORLD_READABLE_SECRETS" + )] + pub allow_world_readable_secrets: Option, + + /// RPC secret network key, used to replace rpc_secret in config.toml when running the + /// daemon or doing admin operations + #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")] + pub rpc_secret: Option, + + /// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret + /// when running the daemon or doing admin operations + #[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")] + pub rpc_secret_file: Option, + + /// Admin API authentication token, replaces admin.admin_token in config.toml when + /// running the Garage daemon + #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")] + pub admin_token: Option, + + /// Admin API authentication token file path, replaces admin.admin_token in config.toml + /// and admin-token when running the Garage daemon + #[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")] + pub admin_token_file: Option, + + /// Metrics API authentication token, replaces admin.metrics_token in config.toml when + /// running the Garage daemon + #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")] + pub metrics_token: Option, + + /// Metrics API authentication token file path, replaces admin.metrics_token in config.toml + /// and metrics-token when running the Garage daemon + #[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")] + pub metrics_token_file: Option, +} + +/// Single function to fill all secrets in the Config struct from their correct source (value +/// from config or CLI param or env variable or read from a file specified in config or CLI +/// param or env variable) +pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result { + let allow_world_readable = secrets + .allow_world_readable_secrets + .unwrap_or(config.allow_world_readable_secrets); + + fill_secret( + &mut config.rpc_secret, + &config.rpc_secret_file, + &secrets.rpc_secret, + &secrets.rpc_secret_file, + "rpc_secret", + allow_world_readable, + )?; + + fill_secret( + &mut config.admin.admin_token, + &config.admin.admin_token_file, + &secrets.admin_token, + &secrets.admin_token_file, + "admin.admin_token", + allow_world_readable, + )?; + fill_secret( + &mut config.admin.metrics_token, + &config.admin.metrics_token_file, + &secrets.metrics_token, + &secrets.metrics_token_file, + "admin.metrics_token", + allow_world_readable, + )?; + + Ok(config) +} + +fn fill_secret( + config_secret: &mut Option, + config_secret_file: &Option, + cli_secret: &Option, + cli_secret_file: &Option, + name: &'static str, + allow_world_readable: bool, +) -> Result<(), Error> { + let cli_value = match (&cli_secret, &cli_secret_file) { + (Some(_), Some(_)) => { + return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); + } + (Some(secret), None) => Some(secret.to_string()), + (None, Some(file)) => Some(read_secret_file(file, allow_world_readable)?), + (None, None) => None, + }; + + if let Some(val) = cli_value { + if config_secret.is_some() || config_secret_file.is_some() { + debug!("Overriding secret `{}` using value specified using CLI argument or environnement variable.", name); + } + + *config_secret = Some(val); + } else if let Some(file_path) = &config_secret_file { + if config_secret.is_some() { + return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); + } + + *config_secret = Some(read_secret_file(file_path, allow_world_readable)?); + } + + Ok(()) +} + +fn read_secret_file(file_path: &String, allow_world_readable: bool) -> Result { + if !allow_world_readable { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::metadata(file_path)?; + if metadata.mode() & 0o077 != 0 { + return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into()); + } + } + } + + let secret_buf = std::fs::read_to_string(file_path)?; + + // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`. + // also editors sometimes add a trailing newline + Ok(String::from(secret_buf.trim_end())) +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::io::Write; + + use garage_util::config::read_config; + use garage_util::error::Error; + + use super::*; + + #[test] + fn test_rpc_secret_file_works() -> Result<(), Error> { + let path_secret = mktemp::Temp::new_file()?; + let mut file_secret = File::create(path_secret.as_path())?; + writeln!(file_secret, "foo")?; + drop(file_secret); + + let path_config = mktemp::Temp::new_file()?; + let mut file_config = File::create(path_config.as_path())?; + let path_secret_path = path_secret.as_path(); + writeln!( + file_config, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_mode = "3" + rpc_bind_addr = "[::]:3901" + rpc_secret_file = "{}" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "#, + path_secret_path.display() + )?; + drop(file_config); + + // Second configuration file, same as previous one + // except it allows world-readable secrets. + let path_config_allow_world_readable = mktemp::Temp::new_file()?; + let mut file_config_allow_world_readable = + File::create(path_config_allow_world_readable.as_path())?; + writeln!( + file_config_allow_world_readable, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_mode = "3" + rpc_bind_addr = "[::]:3901" + rpc_secret_file = "{}" + allow_world_readable_secrets = true + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "#, + path_secret_path.display() + )?; + drop(file_config_allow_world_readable); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets(config, Secrets::default())?; + assert_eq!("foo", config.rpc_secret.unwrap()); + + #[cfg(unix)] + { + // Check non world-readable secrets config + + let secrets_allow_world_readable = Secrets { + allow_world_readable_secrets: Some(true), + ..Default::default() + }; + let secrets_no_allow_world_readable = Secrets { + allow_world_readable_secrets: Some(false), + ..Default::default() + }; + + use std::os::unix::fs::PermissionsExt; + let metadata = std::fs::metadata(&path_secret_path)?; + let mut perm = metadata.permissions(); + perm.set_mode(0o660); + std::fs::set_permissions(&path_secret_path, perm)?; + + // Config file that just specifies the path + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, Secrets::default()).is_err()); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, secrets_allow_world_readable.clone()).is_ok()); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, secrets_no_allow_world_readable.clone()).is_err()); + + // Config file that also specifies to allow world_readable_secrets + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, Secrets::default()).is_ok()); + + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, secrets_allow_world_readable).is_ok()); + + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, secrets_no_allow_world_readable).is_err()); + } + + drop(path_secret); + drop(path_config); + drop(path_config_allow_world_readable); + + Ok(()) + } + + #[test] + fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> { + let path_config = mktemp::Temp::new_file()?; + let mut file_config = File::create(path_config.as_path())?; + writeln!( + file_config, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_mode = "3" + rpc_bind_addr = "[::]:3901" + rpc_secret= "dummy" + rpc_secret_file = "dummy" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "# + )?; + let config = read_config(path_config.to_path_buf())?; + assert_eq!( + "only one of `rpc_secret` and `rpc_secret_file` can be set", + fill_secrets(config, Secrets::default()) + .unwrap_err() + .to_string() + ); + drop(path_config); + drop(file_config); + Ok(()) + } +} diff --git a/src/garage/server.rs b/src/garage/server.rs index 96ea900d..25d4b845 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -15,9 +15,9 @@ use garage_web::WebServer; use garage_api::k2v::api_server::K2VApiServer; use crate::admin::*; +use crate::secrets::{fill_secrets, Secrets}; #[cfg(feature = "telemetry-otlp")] use crate::tracing_setup::*; -use crate::{fill_secrets, Secrets}; async fn wait_from(mut chan: watch::Receiver) { while !*chan.borrow() { -- cgit v1.2.3 From 97bae7213aa214022b68b65094c3e152826de408 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 15 Jan 2024 17:30:30 +0100 Subject: config: additional tests for secret sourcing --- src/garage/secrets.rs | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) (limited to 'src/garage') diff --git a/src/garage/secrets.rs b/src/garage/secrets.rs index c40c3cc9..e96be9e4 100644 --- a/src/garage/secrets.rs +++ b/src/garage/secrets.rs @@ -200,10 +200,9 @@ mod tests { let config = fill_secrets(config, Secrets::default())?; assert_eq!("foo", config.rpc_secret.unwrap()); + // ---- Check non world-readable secrets config ---- #[cfg(unix)] { - // Check non world-readable secrets config - let secrets_allow_world_readable = Secrets { allow_world_readable_secrets: Some(true), ..Default::default() @@ -240,7 +239,46 @@ mod tests { assert!(fill_secrets(config, secrets_no_allow_world_readable).is_err()); } + // ---- Check alternative secrets specified on CLI ---- + + let path_secret2 = mktemp::Temp::new_file()?; + let mut file_secret2 = File::create(path_secret2.as_path())?; + writeln!(file_secret2, "bar")?; + drop(file_secret2); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets( + config, + Secrets { + rpc_secret: Some("baz".into()), + ..Default::default() + }, + )?; + assert_eq!(config.rpc_secret.as_deref(), Some("baz")); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets( + config, + Secrets { + rpc_secret_file: Some(path_secret2.display().to_string()), + ..Default::default() + }, + )?; + assert_eq!(config.rpc_secret.as_deref(), Some("bar")); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets( + config, + Secrets { + rpc_secret: Some("baz".into()), + rpc_secret_file: Some(path_secret2.display().to_string()), + ..Default::default() + } + ) + .is_err()); + drop(path_secret); + drop(path_secret2); drop(path_config); drop(path_config_allow_world_readable); -- cgit v1.2.3 From f512609123fdf374839ca2a8385ddda8694d09fa Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 15 Jan 2024 17:33:35 +0100 Subject: monitoring: finer histogram boundaries in prometheus metrics (fix #531) --- src/garage/server.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'src/garage') diff --git a/src/garage/server.rs b/src/garage/server.rs index 96ea900d..b6570301 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -34,7 +34,14 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er // ---- Initialize Garage internals ---- #[cfg(feature = "metrics")] - let metrics_exporter = opentelemetry_prometheus::exporter().init(); + let metrics_exporter = opentelemetry_prometheus::exporter() + .with_default_summary_quantiles(vec![0.25, 0.5, 0.75, 0.9, 0.95, 0.99]) + .with_default_histogram_boundaries(vec![ + 0.001, 0.0015, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.02, 0.03, 0.05, 0.07, 0.1, + 0.15, 0.2, 0.3, 0.5, 0.7, 1., 1.5, 2., 3., 5., 7., 10., 15., 20., 30., 40., 50., 60., + 70., 100., + ]) + .init(); info!("Initializing Garage main data store..."); let garage = Garage::new(config.clone())?; -- cgit v1.2.3 From 50643e61bfb4ef0782e4364ed368bdfa62be7e5e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 16 Jan 2024 10:47:33 +0100 Subject: Bump version to 0.8.5 --- src/garage/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/garage') diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 00975738..88c4c731 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.8.4" +version = "0.8.5" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" -- cgit v1.2.3 From 82a29bf6e5bf3f0d616b3214db02eb063d7a15b4 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 16 Jan 2024 14:04:11 +0100 Subject: help, comments: make clear that full-length node ID = public key Generally, avoid using the "public key" terminology --- src/garage/cli/init.rs | 2 +- src/garage/cli/structs.rs | 8 +++++--- src/garage/main.rs | 7 +++---- 3 files changed, 9 insertions(+), 8 deletions(-) (limited to 'src/garage') diff --git a/src/garage/cli/init.rs b/src/garage/cli/init.rs index 20813f1c..43ca5c09 100644 --- a/src/garage/cli/init.rs +++ b/src/garage/cli/init.rs @@ -43,7 +43,7 @@ pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> { idstr ); eprintln!( - "where is their own node identifier in the format: @:" + "where is their own node identifier in the format: @:" ); eprintln!(); eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:"); diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index aba57551..be4d5bd6 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -64,7 +64,8 @@ pub enum Command { #[derive(StructOpt, Debug)] pub enum NodeOperation { - /// Print identifier (public key) of this Garage node + /// Print the full node ID (public key) of this Garage node, and its publicly reachable IP + /// address and port if they are specified in config file under `rpc_public_addr` #[structopt(name = "id", version = garage_version())] NodeId(NodeIdOpt), @@ -82,8 +83,9 @@ pub struct NodeIdOpt { #[derive(StructOpt, Debug)] pub struct ConnectNodeOpt { - /// Node public key and address, in the format: - /// `@:` + /// Full node ID (public key) and IP address and port, in the format: + /// `@:`. + /// You can retrieve this information on the target node using `garage node id`. pub(crate) node: String, } diff --git a/src/garage/main.rs b/src/garage/main.rs index 9f9eefed..5c92dae4 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -46,8 +46,7 @@ use secrets::Secrets; about = "S3-compatible object store for self-hosted geo-distributed deployments" )] struct Opt { - /// Host to connect to for admin operations, in the format: - /// @: + /// Host to connect to for admin operations, in the format: @: #[structopt(short = "h", long = "rpc-host", env = "GARAGE_RPC_HOST")] pub rpc_host: Option, @@ -201,7 +200,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { // Find and parse the address of the target host let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host { - let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is @:.", h))?; + let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is @:.", h))?; (id, addrs[0], false) } else { let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir) @@ -231,7 +230,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { addr ); } - Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?; + Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct full-length node ID (public key).")?; } let system_rpc_endpoint = netapp.endpoint::(SYSTEM_RPC_PATH.into()); -- cgit v1.2.3 From ee57dd922b9c396298473b41e4046c8d00ee77d5 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 16 Jan 2024 16:28:17 +0100 Subject: Bump version to 0.9.1 --- src/garage/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/garage') diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 4c41c2a1..35edd30c 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.9.0" +version = "0.9.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" -- cgit v1.2.3 From 8527dd87cc7a685b3ffd2054fca1a88574b93ea4 Mon Sep 17 00:00:00 2001 From: Zdenek Crha Date: Wed, 17 Jan 2024 21:20:34 +0100 Subject: convert_db: allow LMDB map size override --- src/garage/cli/convert_db.rs | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) (limited to 'src/garage') diff --git a/src/garage/cli/convert_db.rs b/src/garage/cli/convert_db.rs index 3c6ce69c..3b396c1b 100644 --- a/src/garage/cli/convert_db.rs +++ b/src/garage/cli/convert_db.rs @@ -22,16 +22,38 @@ pub struct ConvertDbOpt { /// Output database engine #[structopt(short = "b")] output_engine: String, + + #[structopt(flatten)] + output_open: OpenDbOpt, +} + +/// Open database config overrides +#[derive(StructOpt, Debug, Default)] +pub struct OpenDbOpt { + #[cfg(feature = "lmdb")] + #[structopt(flatten)] + lmdb: OpenLmdbOpt, +} + +/// Output LMDB database config overrides +#[cfg(feature = "lmdb")] +#[derive(StructOpt, Debug, Default)] +pub struct OpenLmdbOpt { + /// Output LMDB map size override + /// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB) + #[cfg(feature = "lmdb")] + #[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)] + map_size: Option, } pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> { - let input = open_db(args.input_path, args.input_engine)?; - let output = open_db(args.output_path, args.output_engine)?; + let input = open_db(args.input_path, args.input_engine, OpenDbOpt::default())?; + let output = open_db(args.output_path, args.output_engine, args.output_open)?; output.import(&input)?; Ok(()) } -fn open_db(path: PathBuf, engine: String) -> Result { +fn open_db(path: PathBuf, engine: String, open: OpenDbOpt) -> Result { match engine.as_str() { #[cfg(feature = "sled")] "sled" => { @@ -51,7 +73,10 @@ fn open_db(path: PathBuf, engine: String) -> Result { Error(format!("Unable to create LMDB data directory: {}", e).into()) })?; - let map_size = lmdb_adapter::recommended_map_size(); + let map_size = match open.lmdb.map_size { + Some(c) => c.as_u64() as usize, + None => lmdb_adapter::recommended_map_size(), + }; let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new(); env_builder.max_dbs(100); -- cgit v1.2.3 From 4b54e053dfd1bd950e86ffad9b4155a47806d8f1 Mon Sep 17 00:00:00 2001 From: Zdenek Crha Date: Thu, 18 Jan 2024 17:57:53 +0100 Subject: convert_db: prevent conversion between same input/output engine Use optional DB open overrides for both input and output database. Duplicating the same override flag for input/output would result in too many, too long flags. It would be too costly for very rare edge-case where converting between same DB engine, just with different flags. Because overrides flags for different engines are disjoint and we are preventing conversion between same input/ouput DB engine, we can have only one set. The override flag will be passed either to input or output, based on engine type it belongs to. It will never be passed to both of them and cause unwelcome surprise to user. --- src/garage/cli/convert_db.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) (limited to 'src/garage') diff --git a/src/garage/cli/convert_db.rs b/src/garage/cli/convert_db.rs index 3b396c1b..d33010f4 100644 --- a/src/garage/cli/convert_db.rs +++ b/src/garage/cli/convert_db.rs @@ -14,14 +14,14 @@ pub struct ConvertDbOpt { /// Input database engine (sled, lmdb or sqlite; limited by db engines /// enabled in this build) #[structopt(short = "a")] - input_engine: String, + input_engine: Engine, /// Output database path #[structopt(short = "o")] output_path: PathBuf, /// Output database engine #[structopt(short = "b")] - output_engine: String, + output_engine: Engine, #[structopt(flatten)] output_open: OpenDbOpt, @@ -47,28 +47,32 @@ pub struct OpenLmdbOpt { } pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> { - let input = open_db(args.input_path, args.input_engine, OpenDbOpt::default())?; - let output = open_db(args.output_path, args.output_engine, args.output_open)?; + if args.input_engine == args.output_engine { + return Err(Error("input and output database engine must differ".into())); + } + + let input = open_db(args.input_path, args.input_engine, &args.output_open)?; + let output = open_db(args.output_path, args.output_engine, &args.output_open)?; output.import(&input)?; Ok(()) } -fn open_db(path: PathBuf, engine: String, open: OpenDbOpt) -> Result { - match engine.as_str() { +fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result { + match engine { #[cfg(feature = "sled")] - "sled" => { + Engine::Sled => { let db = sled_adapter::sled::Config::default().path(&path).open()?; Ok(sled_adapter::SledDb::init(db)) } #[cfg(feature = "sqlite")] - "sqlite" | "sqlite3" | "rusqlite" => { + Engine::Sqlite => { let db = sqlite_adapter::rusqlite::Connection::open(&path)?; db.pragma_update(None, "journal_mode", &"WAL")?; db.pragma_update(None, "synchronous", &"NORMAL")?; Ok(sqlite_adapter::SqliteDb::init(db)) } #[cfg(feature = "lmdb")] - "lmdb" | "heed" => { + Engine::Lmdb => { std::fs::create_dir_all(&path).map_err(|e| { Error(format!("Unable to create LMDB data directory: {}", e).into()) })?; @@ -87,8 +91,5 @@ fn open_db(path: PathBuf, engine: String, open: OpenDbOpt) -> Result { let db = env_builder.open(&path)?; Ok(lmdb_adapter::LmdbDb::init(db)) } - e => Err(Error( - format!("Invalid or unsupported DB engine: {}", e).into(), - )), } } -- cgit v1.2.3 From 74e72fc99690e24de498e42d87fc04a123f712d5 Mon Sep 17 00:00:00 2001 From: Zdenek Crha Date: Mon, 22 Jan 2024 17:52:39 +0100 Subject: convert_db: cleanup naming and comments for open overrides --- src/garage/cli/convert_db.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/garage') diff --git a/src/garage/cli/convert_db.rs b/src/garage/cli/convert_db.rs index d33010f4..7307d1cf 100644 --- a/src/garage/cli/convert_db.rs +++ b/src/garage/cli/convert_db.rs @@ -24,10 +24,10 @@ pub struct ConvertDbOpt { output_engine: Engine, #[structopt(flatten)] - output_open: OpenDbOpt, + db_open: OpenDbOpt, } -/// Open database config overrides +/// Overrides for database open operation #[derive(StructOpt, Debug, Default)] pub struct OpenDbOpt { #[cfg(feature = "lmdb")] @@ -35,11 +35,11 @@ pub struct OpenDbOpt { lmdb: OpenLmdbOpt, } -/// Output LMDB database config overrides +/// Overrides for LMDB database open operation #[cfg(feature = "lmdb")] #[derive(StructOpt, Debug, Default)] pub struct OpenLmdbOpt { - /// Output LMDB map size override + /// LMDB map size override /// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB) #[cfg(feature = "lmdb")] #[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)] @@ -51,8 +51,8 @@ pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> { return Err(Error("input and output database engine must differ".into())); } - let input = open_db(args.input_path, args.input_engine, &args.output_open)?; - let output = open_db(args.output_path, args.output_engine, &args.output_open)?; + let input = open_db(args.input_path, args.input_engine, &args.db_open)?; + let output = open_db(args.output_path, args.output_engine, &args.db_open)?; output.import(&input)?; Ok(()) } -- cgit v1.2.3 From 0eef8a69f0006de305281dd374cc63e7a46e4b80 Mon Sep 17 00:00:00 2001 From: Zdenek Crha Date: Mon, 22 Jan 2024 20:38:14 +0100 Subject: make all garage_db::Engine variants un-conditional Having all Engine enum variants conditional causes compilation errors when *none* of the DB engine features is enabled. This is not an issue for full garage build, but affects crates that use garage_db as dependency. Change all variants to be present at all times. It solves compilation errors and also allows us to better differentiate between invalid DB engine name and engine with support not compiled in current binary. --- src/garage/cli/convert_db.rs | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src/garage') diff --git a/src/garage/cli/convert_db.rs b/src/garage/cli/convert_db.rs index 7307d1cf..044ccbb9 100644 --- a/src/garage/cli/convert_db.rs +++ b/src/garage/cli/convert_db.rs @@ -91,5 +91,13 @@ fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result { let db = env_builder.open(&path)?; Ok(lmdb_adapter::LmdbDb::init(db)) } + + // Pattern is unreachable when all supported DB engines are compiled into binary. The allow + // attribute is added so that we won't have to change this match in case stop building + // support for one or more engines by default. + #[allow(unreachable_patterns)] + engine => Err(Error( + format!("Engine support not available in this build: {}", engine).into(), + )), } } -- cgit v1.2.3 From fe1af5d98b8e6c87f39f1d94586e61bff100e7d2 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 5 Feb 2024 13:02:02 +0100 Subject: [dep-upgrade-202402] refactor dependencies: move all as workspace deps --- src/garage/Cargo.toml | 82 +++++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) (limited to 'src/garage') diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 35edd30c..1d299030 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -31,49 +31,49 @@ garage_table.workspace = true garage_util.workspace = true garage_web.workspace = true -backtrace = "0.3" -bytes = "1.0" -bytesize = "1.2" -timeago = { version = "0.4", default-features = false } -parse_duration = "2.1" -hex = "0.4" -tracing = { version = "0.1" } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -rand = "0.8" -async-trait = "0.1.7" -sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } -git-version = "0.3.4" - -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_bytes = "0.11" -structopt = { version = "0.3", default-features = false } -toml = "0.6" - -futures = "0.3" -futures-util = "0.3" -tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } - -netapp = "0.10" - -opentelemetry = { version = "0.17", features = [ "rt-tokio" ] } -opentelemetry-prometheus = { version = "0.10", optional = true } -opentelemetry-otlp = { version = "0.10", optional = true } -prometheus = { version = "0.13", optional = true } +backtrace.workspace = true +bytes.workspace = true +bytesize.workspace = true +timeago.workspace = true +parse_duration.workspace = true +hex.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +rand.workspace = true +async-trait.workspace = true +sodiumoxide.workspace = true +structopt.workspace = true +git-version.workspace = true + +serde.workspace = true +serde_bytes.workspace = true +toml.workspace = true + +futures.workspace = true +futures-util.workspace = true +tokio.workspace = true + +netapp.workspace = true + +opentelemetry.workspace = true +opentelemetry-prometheus = { workspace = true, optional = true } +opentelemetry-otlp = { workspace = true, optional = true } +prometheus = { workspace = true, optional = true } [dev-dependencies] -aws-config = "0.55.2" -aws-sdk-s3 = "0.28" -chrono = "0.4" -http = "0.2" -hmac = "0.12" -hyper = { version = "0.14", features = ["client", "http1", "runtime"] } -mktemp = "0.5" -sha2 = "0.10" - -static_init = "1.0" -assert-json-diff = "2.0" -serde_json = "1.0" -base64 = "0.21" +aws-config.workspace = true +aws-sdk-s3.workspace = true +chrono.workspace = true +http.workspace = true +hmac.workspace = true +hyper.workspace = true +mktemp.workspace = true +sha2.workspace = true + +static_init.workspace = true +assert-json-diff.workspace = true +serde_json.workspace = true +base64.workspace = true k2v-client.workspace = true -- cgit v1.2.3 From 6e4229e29c1ee3e0b7f3511f80b6108e3beb1efd Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 5 Feb 2024 14:02:45 +0100 Subject: [dep-upgrade-202402] update aws-sdk dependencies --- src/garage/tests/common/client.rs | 2 ++ src/garage/tests/s3/multipart.rs | 30 +++++++++++++++--------------- src/garage/tests/s3/objects.rs | 24 ++++++++++++------------ src/garage/tests/s3/streaming_signature.rs | 16 ++++++++-------- src/garage/tests/s3/website.rs | 29 +++++++++++++++++++---------- 5 files changed, 56 insertions(+), 45 deletions(-) (limited to 'src/garage') diff --git a/src/garage/tests/common/client.rs b/src/garage/tests/common/client.rs index ef4daa5d..ffa4cae8 100644 --- a/src/garage/tests/common/client.rs +++ b/src/garage/tests/common/client.rs @@ -1,3 +1,4 @@ +use aws_sdk_s3::config::BehaviorVersion; use aws_sdk_s3::config::Credentials; use aws_sdk_s3::{Client, Config}; @@ -11,6 +12,7 @@ pub fn build_client(key: &Key) -> Client { .endpoint_url(format!("http://127.0.0.1:{}", DEFAULT_PORT)) .region(super::REGION) .credentials_provider(credentials) + .behavior_version(BehaviorVersion::v2023_11_09()) .build(); Client::from_conf(config) diff --git a/src/garage/tests/s3/multipart.rs b/src/garage/tests/s3/multipart.rs index 09ae5e5b..51c9df74 100644 --- a/src/garage/tests/s3/multipart.rs +++ b/src/garage/tests/s3/multipart.rs @@ -154,7 +154,7 @@ async fn test_multipart_upload() { .await .unwrap(); - assert_eq!(r.content_length, (SZ_5MB * 3) as i64); + assert_eq!(r.content_length.unwrap(), (SZ_5MB * 3) as i64); } { @@ -183,7 +183,7 @@ async fn test_multipart_upload() { .unwrap(); eprintln!("get_object with part_number = {}", part_number); - assert_eq!(o.content_length, SZ_5MB as i64); + assert_eq!(o.content_length.unwrap(), SZ_5MB as i64); assert_bytes_eq!(o.body, data); } } @@ -249,14 +249,14 @@ async fn test_uploadlistpart() { let ps = r.parts.unwrap(); assert_eq!(ps.len(), 1); - assert_eq!(ps[0].part_number, 2); + assert_eq!(ps[0].part_number.unwrap(), 2); let fp = &ps[0]; assert!(fp.last_modified.is_some()); assert_eq!( fp.e_tag.as_ref().unwrap(), "\"3366bb9dcf710d6801b5926467d02e19\"" ); - assert_eq!(fp.size, SZ_5MB as i64); + assert_eq!(fp.size.unwrap(), SZ_5MB as i64); } let p2 = ctx @@ -286,23 +286,23 @@ async fn test_uploadlistpart() { let ps = r.parts.unwrap(); assert_eq!(ps.len(), 2); - assert_eq!(ps[0].part_number, 1); + assert_eq!(ps[0].part_number.unwrap(), 1); let fp = &ps[0]; assert!(fp.last_modified.is_some()); assert_eq!( fp.e_tag.as_ref().unwrap(), "\"3c484266f9315485694556e6c693bfa2\"" ); - assert_eq!(fp.size, SZ_5MB as i64); + assert_eq!(fp.size.unwrap(), SZ_5MB as i64); - assert_eq!(ps[1].part_number, 2); + assert_eq!(ps[1].part_number.unwrap(), 2); let sp = &ps[1]; assert!(sp.last_modified.is_some()); assert_eq!( sp.e_tag.as_ref().unwrap(), "\"3366bb9dcf710d6801b5926467d02e19\"" ); - assert_eq!(sp.size, SZ_5MB as i64); + assert_eq!(sp.size.unwrap(), SZ_5MB as i64); } { @@ -320,14 +320,14 @@ async fn test_uploadlistpart() { assert!(r.part_number_marker.is_none()); assert_eq!(r.next_part_number_marker.as_deref(), Some("1")); - assert_eq!(r.max_parts, 1_i32); - assert!(r.is_truncated); + assert_eq!(r.max_parts.unwrap(), 1_i32); + assert!(r.is_truncated.unwrap()); assert_eq!(r.key.unwrap(), "a"); assert_eq!(r.upload_id.unwrap().as_str(), uid.as_str()); let parts = r.parts.unwrap(); assert_eq!(parts.len(), 1); let fp = &parts[0]; - assert_eq!(fp.part_number, 1); + assert_eq!(fp.part_number.unwrap(), 1); assert_eq!( fp.e_tag.as_ref().unwrap(), "\"3c484266f9315485694556e6c693bfa2\"" @@ -349,19 +349,19 @@ async fn test_uploadlistpart() { r2.part_number_marker.as_ref().unwrap(), r.next_part_number_marker.as_ref().unwrap() ); - assert_eq!(r2.max_parts, 1_i32); + assert_eq!(r2.max_parts.unwrap(), 1_i32); assert_eq!(r2.key.unwrap(), "a"); assert_eq!(r2.upload_id.unwrap().as_str(), uid.as_str()); let parts = r2.parts.unwrap(); assert_eq!(parts.len(), 1); let fp = &parts[0]; - assert_eq!(fp.part_number, 2); + assert_eq!(fp.part_number.unwrap(), 2); assert_eq!( fp.e_tag.as_ref().unwrap(), "\"3366bb9dcf710d6801b5926467d02e19\"" ); //assert!(r2.is_truncated); // WHY? (this was the test before) - assert!(!r2.is_truncated); + assert!(!r2.is_truncated.unwrap()); } let cmp = CompletedMultipartUpload::builder() @@ -411,7 +411,7 @@ async fn test_uploadlistpart() { .await .unwrap(); - assert_eq!(r.content_length, (SZ_5MB * 2) as i64); + assert_eq!(r.content_length.unwrap(), (SZ_5MB * 2) as i64); } } diff --git a/src/garage/tests/s3/objects.rs b/src/garage/tests/s3/objects.rs index 27697d45..ca35b435 100644 --- a/src/garage/tests/s3/objects.rs +++ b/src/garage/tests/s3/objects.rs @@ -50,9 +50,9 @@ async fn test_putobject() { // assert_eq!(o.version_id.unwrap(), _version); assert_eq!(o.content_type.unwrap(), content_type); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 0); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } { @@ -86,9 +86,9 @@ async fn test_putobject() { assert_bytes_eq!(o.body, b"hi"); assert_eq!(o.e_tag.unwrap(), etag); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 2); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 2); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } { @@ -119,9 +119,9 @@ async fn test_putobject() { assert_bytes_eq!(o.body, b""); assert_eq!(o.e_tag.unwrap(), etag); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 0); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } } @@ -205,7 +205,7 @@ async fn test_deleteobject() { .await .unwrap(); if i > 0 { - to_del = to_del.objects(ObjectIdentifier::builder().key(k).build()); + to_del = to_del.objects(ObjectIdentifier::builder().key(k).build().unwrap()); } } @@ -223,7 +223,7 @@ async fn test_deleteobject() { .unwrap(); if i > 0 { - to_del = to_del.objects(ObjectIdentifier::builder().key(k).build()); + to_del = to_del.objects(ObjectIdentifier::builder().key(k).build().unwrap()); } } @@ -247,7 +247,7 @@ async fn test_deleteobject() { .client .delete_objects() .bucket(&bucket) - .delete(to_del.build()) + .delete(to_del.build().unwrap()) .send() .await .unwrap(); diff --git a/src/garage/tests/s3/streaming_signature.rs b/src/garage/tests/s3/streaming_signature.rs index b7a1acae..224b9ed5 100644 --- a/src/garage/tests/s3/streaming_signature.rs +++ b/src/garage/tests/s3/streaming_signature.rs @@ -57,9 +57,9 @@ async fn test_putobject_streaming() { // assert_eq!(o.version_id.unwrap(), _version); assert_eq!(o.content_type.unwrap(), content_type); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 0); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 0); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } { @@ -95,9 +95,9 @@ async fn test_putobject_streaming() { assert_bytes_eq!(o.body, BODY); assert_eq!(o.e_tag.unwrap(), etag); assert!(o.last_modified.is_some()); - assert_eq!(o.content_length, 62); - assert_eq!(o.parts_count, 0); - assert_eq!(o.tag_count, 0); + assert_eq!(o.content_length.unwrap(), 62); + assert_eq!(o.parts_count, None); + assert_eq!(o.tag_count, None); } } @@ -187,7 +187,7 @@ async fn test_put_website_streaming() { .await .unwrap(); - assert_eq!(o.index_document.unwrap().suffix.unwrap(), "home.html"); - assert_eq!(o.error_document.unwrap().key.unwrap(), "err/error.html"); + assert_eq!(o.index_document.unwrap().suffix, "home.html"); + assert_eq!(o.error_document.unwrap().key, "err/error.html"); } } diff --git a/src/garage/tests/s3/website.rs b/src/garage/tests/s3/website.rs index eeafb5fa..59c990f8 100644 --- a/src/garage/tests/s3/website.rs +++ b/src/garage/tests/s3/website.rs @@ -181,8 +181,18 @@ async fn test_website_s3_api() { .unwrap(); let conf = WebsiteConfiguration::builder() - .index_document(IndexDocument::builder().suffix("home.html").build()) - .error_document(ErrorDocument::builder().key("err/error.html").build()) + .index_document( + IndexDocument::builder() + .suffix("home.html") + .build() + .unwrap(), + ) + .error_document( + ErrorDocument::builder() + .key("err/error.html") + .build() + .unwrap(), + ) .build(); ctx.client @@ -201,9 +211,11 @@ async fn test_website_s3_api() { .allowed_methods("GET") .allowed_methods("PUT") .allowed_origins("*") - .build(), + .build() + .unwrap(), ) - .build(); + .build() + .unwrap(); ctx.client .put_bucket_cors() @@ -222,19 +234,16 @@ async fn test_website_s3_api() { .await .unwrap(); - let main_rule = cors_res.cors_rules().unwrap().iter().next().unwrap(); + let main_rule = cors_res.cors_rules().iter().next().unwrap(); assert_eq!(main_rule.id.as_ref().unwrap(), "main-rule"); assert_eq!( main_rule.allowed_headers.as_ref().unwrap(), &vec!["*".to_string()] ); + assert_eq!(&main_rule.allowed_origins, &vec!["*".to_string()]); assert_eq!( - main_rule.allowed_origins.as_ref().unwrap(), - &vec!["*".to_string()] - ); - assert_eq!( - main_rule.allowed_methods.as_ref().unwrap(), + &main_rule.allowed_methods, &vec!["GET".to_string(), "PUT".to_string()] ); } -- cgit v1.2.3 From 6e69a1fffc715c752a399750c1e26aa46683dbb2 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 5 Feb 2024 14:44:12 +0100 Subject: [dep-upgrade-202402] prepare migration to http/hyper 1.0 --- src/garage/tests/k2v/batch.rs | 7 ++----- src/garage/tests/k2v/item.rs | 22 +++++---------------- src/garage/tests/k2v/poll.rs | 16 +++++---------- src/garage/tests/k2v/simple.rs | 9 +++------ src/garage/tests/lib.rs | 11 +++-------- src/garage/tests/s3/website.rs | 45 +++++++++++++++++++----------------------- 6 files changed, 38 insertions(+), 72 deletions(-) (limited to 'src/garage') diff --git a/src/garage/tests/k2v/batch.rs b/src/garage/tests/k2v/batch.rs index 71de91bf..417fe7ea 100644 --- a/src/garage/tests/k2v/batch.rs +++ b/src/garage/tests/k2v/batch.rs @@ -7,7 +7,7 @@ use base64::prelude::*; use serde_json::json; use crate::json_body; -use hyper::{Method, StatusCode}; +use hyper::{body::HttpBody, Method, StatusCode}; #[tokio::test] async fn test_batch() { @@ -77,10 +77,7 @@ async fn test_batch() { .unwrap() .to_string(), ); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, values.get(sk).unwrap().as_bytes()); } diff --git a/src/garage/tests/k2v/item.rs b/src/garage/tests/k2v/item.rs index 20add889..6b653088 100644 --- a/src/garage/tests/k2v/item.rs +++ b/src/garage/tests/k2v/item.rs @@ -7,7 +7,7 @@ use base64::prelude::*; use serde_json::json; use crate::json_body; -use hyper::{Method, StatusCode}; +use hyper::{body::HttpBody, Method, StatusCode}; #[tokio::test] async fn test_items_and_indices() { @@ -83,10 +83,7 @@ async fn test_items_and_indices() { .to_str() .unwrap() .to_string(); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, content); // ReadIndex -- now there should be some stuff @@ -152,10 +149,7 @@ async fn test_items_and_indices() { res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" ); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, content2); // ReadIndex -- now there should be some stuff @@ -394,10 +388,7 @@ async fn test_item_return_format() { .to_str() .unwrap() .to_string(); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, single_value); // f1: not specified @@ -434,10 +425,7 @@ async fn test_item_return_format() { res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" ); - let res_body = hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec(); + let res_body = res.into_body().collect().await.unwrap().to_bytes(); assert_eq!(res_body, single_value); // f3: json diff --git a/src/garage/tests/k2v/poll.rs b/src/garage/tests/k2v/poll.rs index 452317c2..b75fa9c7 100644 --- a/src/garage/tests/k2v/poll.rs +++ b/src/garage/tests/k2v/poll.rs @@ -1,5 +1,5 @@ use base64::prelude::*; -use hyper::{Method, StatusCode}; +use hyper::{body::HttpBody, Method, StatusCode}; use std::time::Duration; use assert_json_diff::assert_json_eq; @@ -47,11 +47,8 @@ async fn test_poll_item() { .unwrap() .to_string(); - let res2_body = hyper::body::to_bytes(res2.into_body()) - .await - .unwrap() - .to_vec(); - assert_eq!(res2_body, b"Initial value"); + let res2_body = res2.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(res2_body, b"Initial value"[..]); // Start poll operation let poll = { @@ -95,11 +92,8 @@ async fn test_poll_item() { assert_eq!(poll_res.status(), StatusCode::OK); - let poll_res_body = hyper::body::to_bytes(poll_res.into_body()) - .await - .unwrap() - .to_vec(); - assert_eq!(poll_res_body, b"New value"); + let poll_res_body = poll_res.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(poll_res_body, b"New value"[..]); } #[tokio::test] diff --git a/src/garage/tests/k2v/simple.rs b/src/garage/tests/k2v/simple.rs index 465fc24d..a1d5008b 100644 --- a/src/garage/tests/k2v/simple.rs +++ b/src/garage/tests/k2v/simple.rs @@ -1,6 +1,6 @@ use crate::common; -use hyper::{Method, StatusCode}; +use hyper::{body::HttpBody, Method, StatusCode}; #[tokio::test] async fn test_simple() { @@ -32,9 +32,6 @@ async fn test_simple() { .unwrap(); assert_eq!(res2.status(), StatusCode::OK); - let res2_body = hyper::body::to_bytes(res2.into_body()) - .await - .unwrap() - .to_vec(); - assert_eq!(res2_body, b"Hello, world!"); + let res2_body = res2.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(res2_body, b"Hello, world!"[..]); } diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs index ab92bc0a..3cb17a2b 100644 --- a/src/garage/tests/lib.rs +++ b/src/garage/tests/lib.rs @@ -11,15 +11,10 @@ mod k2v; #[cfg(feature = "k2v")] mod k2v_client; -use hyper::{Body, Response}; +use hyper::{body::HttpBody, Body, Response}; pub async fn json_body(res: Response) -> serde_json::Value { - let res_body: serde_json::Value = serde_json::from_slice( - &hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec()[..], - ) - .unwrap(); + let body = res.into_body().collect().await.unwrap().to_bytes(); + let res_body: serde_json::Value = serde_json::from_slice(&body).unwrap(); res_body } diff --git a/src/garage/tests/s3/website.rs b/src/garage/tests/s3/website.rs index 59c990f8..94acafc7 100644 --- a/src/garage/tests/s3/website.rs +++ b/src/garage/tests/s3/website.rs @@ -9,7 +9,7 @@ use aws_sdk_s3::{ }; use http::{Request, StatusCode}; use hyper::{ - body::{to_bytes, Body}, + body::{Body, HttpBody}, Client, }; use serde_json::json; @@ -49,7 +49,7 @@ async fn test_website() { assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); /* check that we do not leak body */ @@ -87,7 +87,7 @@ async fn test_website() { resp = client.request(req()).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); @@ -107,10 +107,10 @@ async fn test_website() { .unwrap() }; - let mut admin_resp = client.request(admin_req()).await.unwrap(); + let admin_resp = client.request(admin_req()).await.unwrap(); assert_eq!(admin_resp.status(), StatusCode::OK); assert_eq!( - to_bytes(admin_resp.body_mut()).await.unwrap().as_ref(), + admin_resp.into_body().collect().await.unwrap().to_bytes(), format!("Domain '{bname}' is managed by Garage").as_bytes() ); } @@ -124,7 +124,7 @@ async fn test_website() { resp = client.request(req()).await.unwrap(); assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); /* check that we do not leak body */ @@ -260,7 +260,7 @@ async fn test_website_s3_api() { .body(Body::empty()) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); assert_eq!( @@ -268,7 +268,7 @@ async fn test_website_s3_api() { "*" ); assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -285,11 +285,11 @@ async fn test_website_s3_api() { .body(Body::empty()) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY_ERR.as_ref() ); } @@ -305,7 +305,7 @@ async fn test_website_s3_api() { .body(Body::empty()) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); assert_eq!( @@ -313,7 +313,7 @@ async fn test_website_s3_api() { "*" ); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -329,11 +329,11 @@ async fn test_website_s3_api() { .body(Body::empty()) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::FORBIDDEN); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -370,11 +370,11 @@ async fn test_website_s3_api() { .body(Body::empty()) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::FORBIDDEN); assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), + resp.into_body().collect().await.unwrap().to_bytes(), BODY.as_ref() ); } @@ -396,17 +396,12 @@ async fn test_website_s3_api() { .body(Body::empty()) .unwrap(); - let mut resp = client.request(req).await.unwrap(); + let resp = client.request(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::NOT_FOUND); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY_ERR.as_ref() - ); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); + let resp_bytes = resp.into_body().collect().await.unwrap().to_bytes(); + assert_ne!(resp_bytes, BODY_ERR.as_ref()); + assert_ne!(resp_bytes, BODY.as_ref()); } } -- cgit v1.2.3 From 81ccd4586ebdf707dfd37d4802e0cf475e11e004 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 5 Feb 2024 19:57:35 +0100 Subject: [dep-upgrade-202402] upgrade to http/hyper 1.x for tests --- src/garage/Cargo.toml | 2 ++ src/garage/tests/common/custom_requester.rs | 34 ++++++++++++++++----- src/garage/tests/k2v/batch.rs | 3 +- src/garage/tests/k2v/item.rs | 3 +- src/garage/tests/k2v/poll.rs | 3 +- src/garage/tests/k2v/simple.rs | 3 +- src/garage/tests/lib.rs | 9 ++++-- src/garage/tests/s3/website.rs | 47 +++++++++++++++-------------- 8 files changed, 69 insertions(+), 35 deletions(-) (limited to 'src/garage') diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 1d299030..41fd32a1 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -66,7 +66,9 @@ aws-sdk-s3.workspace = true chrono.workspace = true http.workspace = true hmac.workspace = true +http-body-util.workspace = true hyper.workspace = true +hyper-util.workspace = true mktemp.workspace = true sha2.workspace = true diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs index 4133bb8b..72fb1a46 100644 --- a/src/garage/tests/common/custom_requester.rs +++ b/src/garage/tests/common/custom_requester.rs @@ -5,12 +5,17 @@ use std::convert::TryFrom; use chrono::{offset::Utc, DateTime}; use hmac::{Hmac, Mac}; -use hyper::client::HttpConnector; -use hyper::{Body, Client, Method, Request, Response, Uri}; +use http_body_util::BodyExt; +use http_body_util::Full as FullBody; +use hyper::{Method, Request, Response, Uri}; +use hyper_util::client::legacy::{connect::HttpConnector, Client}; +use hyper_util::rt::TokioExecutor; use super::garage::{Instance, Key}; use garage_api::signature; +pub type Body = FullBody; + /// You should ever only use this to send requests AWS sdk won't send, /// like to reproduce behavior of unusual implementations found to be /// problematic. @@ -19,7 +24,7 @@ pub struct CustomRequester { key: Key, uri: Uri, service: &'static str, - client: Client, + client: Client, } impl CustomRequester { @@ -28,7 +33,7 @@ impl CustomRequester { key: key.clone(), uri: instance.s3_uri(), service: "s3", - client: Client::new(), + client: Client::builder(TokioExecutor::new()).build_http(), } } @@ -37,7 +42,7 @@ impl CustomRequester { key: key.clone(), uri: instance.k2v_uri(), service: "k2v", - client: Client::new(), + client: Client::builder(TokioExecutor::new()).build_http(), } } @@ -139,7 +144,7 @@ impl<'a> RequestBuilder<'a> { self } - pub async fn send(&mut self) -> hyper::Result> { + pub async fn send(&mut self) -> Result, String> { // TODO this is a bit incorrect in that path and query params should be url-encoded and // aren't, but this is good enought for now. @@ -242,7 +247,22 @@ impl<'a> RequestBuilder<'a> { .method(self.method.clone()) .body(Body::from(body)) .unwrap(); - self.requester.client.request(request).await + + let result = self + .requester + .client + .request(request) + .await + .map_err(|err| format!("hyper client error: {}", err))?; + + let (head, body) = result.into_parts(); + let body = Body::new( + body.collect() + .await + .map_err(|err| format!("hyper client error in body.collect: {}", err))? + .to_bytes(), + ); + Ok(Response::from_parts(head, body)) } } diff --git a/src/garage/tests/k2v/batch.rs b/src/garage/tests/k2v/batch.rs index 417fe7ea..39554d4d 100644 --- a/src/garage/tests/k2v/batch.rs +++ b/src/garage/tests/k2v/batch.rs @@ -7,7 +7,8 @@ use base64::prelude::*; use serde_json::json; use crate::json_body; -use hyper::{body::HttpBody, Method, StatusCode}; +use http_body_util::BodyExt; +use hyper::{Method, StatusCode}; #[tokio::test] async fn test_batch() { diff --git a/src/garage/tests/k2v/item.rs b/src/garage/tests/k2v/item.rs index 6b653088..5a347bd9 100644 --- a/src/garage/tests/k2v/item.rs +++ b/src/garage/tests/k2v/item.rs @@ -7,7 +7,8 @@ use base64::prelude::*; use serde_json::json; use crate::json_body; -use hyper::{body::HttpBody, Method, StatusCode}; +use http_body_util::BodyExt; +use hyper::{Method, StatusCode}; #[tokio::test] async fn test_items_and_indices() { diff --git a/src/garage/tests/k2v/poll.rs b/src/garage/tests/k2v/poll.rs index b75fa9c7..277f8bc8 100644 --- a/src/garage/tests/k2v/poll.rs +++ b/src/garage/tests/k2v/poll.rs @@ -1,5 +1,6 @@ use base64::prelude::*; -use hyper::{body::HttpBody, Method, StatusCode}; +use http_body_util::BodyExt; +use hyper::{Method, StatusCode}; use std::time::Duration; use assert_json_diff::assert_json_eq; diff --git a/src/garage/tests/k2v/simple.rs b/src/garage/tests/k2v/simple.rs index a1d5008b..1017330d 100644 --- a/src/garage/tests/k2v/simple.rs +++ b/src/garage/tests/k2v/simple.rs @@ -1,6 +1,7 @@ use crate::common; -use hyper::{body::HttpBody, Method, StatusCode}; +use http_body_util::BodyExt; +use hyper::{Method, StatusCode}; #[tokio::test] async fn test_simple() { diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs index 3cb17a2b..ef370db3 100644 --- a/src/garage/tests/lib.rs +++ b/src/garage/tests/lib.rs @@ -11,9 +11,14 @@ mod k2v; #[cfg(feature = "k2v")] mod k2v_client; -use hyper::{body::HttpBody, Body, Response}; +use http_body_util::BodyExt; +use hyper::{body::Body, Response}; -pub async fn json_body(res: Response) -> serde_json::Value { +pub async fn json_body(res: Response) -> serde_json::Value +where + B: Body, + ::Error: std::fmt::Debug, +{ let body = res.into_body().collect().await.unwrap().to_bytes(); let res_body: serde_json::Value = serde_json::from_slice(&body).unwrap(); res_body diff --git a/src/garage/tests/s3/website.rs b/src/garage/tests/s3/website.rs index 94acafc7..19f53fcd 100644 --- a/src/garage/tests/s3/website.rs +++ b/src/garage/tests/s3/website.rs @@ -8,15 +8,18 @@ use aws_sdk_s3::{ types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, }; use http::{Request, StatusCode}; -use hyper::{ - body::{Body, HttpBody}, - Client, -}; +use http_body_util::BodyExt; +use http_body_util::Full as FullBody; +use hyper::body::Bytes; +use hyper_util::client::legacy::Client; +use hyper_util::rt::TokioExecutor; use serde_json::json; const BODY: &[u8; 16] = b"

bonjour

"; const BODY_ERR: &[u8; 6] = b"erreur"; +pub type Body = FullBody; + #[tokio::test] async fn test_website() { const BCKT_NAME: &str = "my-website"; @@ -34,14 +37,14 @@ async fn test_website() { .await .unwrap(); - let client = Client::new(); + let client = Client::builder(TokioExecutor::new()).build_http(); let req = || { Request::builder() .method("GET") .uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port)) .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -49,7 +52,7 @@ async fn test_website() { assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_ne!( - resp.into_body().collect().await.unwrap().to_bytes(), + BodyExt::collect(resp.into_body()).await.unwrap().to_bytes(), BODY.as_ref() ); /* check that we do not leak body */ @@ -61,7 +64,7 @@ async fn test_website() { ctx.garage.admin_port, BCKT_NAME.to_string() )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -103,7 +106,7 @@ async fn test_website() { "http://127.0.0.1:{0}/check?domain={1}", ctx.garage.admin_port, bname )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -136,7 +139,7 @@ async fn test_website() { ctx.garage.admin_port, BCKT_NAME.to_string() )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -248,7 +251,7 @@ async fn test_website_s3_api() { ); } - let client = Client::new(); + let client = Client::builder(TokioExecutor::new()).build_http(); // Test direct requests with CORS { @@ -257,7 +260,7 @@ async fn test_website_s3_api() { .uri(format!("http://127.0.0.1:{}/site/", ctx.garage.web_port)) .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); let resp = client.request(req).await.unwrap(); @@ -282,7 +285,7 @@ async fn test_website_s3_api() { ctx.garage.web_port )) .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); let resp = client.request(req).await.unwrap(); @@ -302,7 +305,7 @@ async fn test_website_s3_api() { .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") .header("Access-Control-Request-Method", "PUT") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); let resp = client.request(req).await.unwrap(); @@ -326,7 +329,7 @@ async fn test_website_s3_api() { .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") .header("Access-Control-Request-Method", "DELETE") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); let resp = client.request(req).await.unwrap(); @@ -367,7 +370,7 @@ async fn test_website_s3_api() { .header("Host", format!("{}.web.garage", BCKT_NAME)) .header("Origin", "https://example.com") .header("Access-Control-Request-Method", "PUT") - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); let resp = client.request(req).await.unwrap(); @@ -393,7 +396,7 @@ async fn test_website_s3_api() { .method("GET") .uri(format!("http://127.0.0.1:{}/site/", ctx.garage.web_port)) .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap(); let resp = client.request(req).await.unwrap(); @@ -409,13 +412,13 @@ async fn test_website_s3_api() { async fn test_website_check_domain() { let ctx = common::context(); - let client = Client::new(); + let client = Client::builder(TokioExecutor::new()).build_http(); let admin_req = || { Request::builder() .method("GET") .uri(format!("http://127.0.0.1:{}/check", ctx.garage.admin_port)) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -439,7 +442,7 @@ async fn test_website_check_domain() { "http://127.0.0.1:{}/check?domain=", ctx.garage.admin_port )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -463,7 +466,7 @@ async fn test_website_check_domain() { "http://127.0.0.1:{}/check?domain=foobar", ctx.garage.admin_port )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; @@ -487,7 +490,7 @@ async fn test_website_check_domain() { "http://127.0.0.1:{}/check?domain=%E2%98%B9", ctx.garage.admin_port )) - .body(Body::empty()) + .body(Body::new(Bytes::new())) .unwrap() }; -- cgit v1.2.3 From fe48d60d2b63a9914297558bd5dbccae8d96c947 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 7 Feb 2024 14:33:07 +0100 Subject: [dep-upgrade-202402] refactor http listener code --- src/garage/server.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/garage') diff --git a/src/garage/server.rs b/src/garage/server.rs index ac76a44d..de8ac9e2 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -113,12 +113,11 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er if let Some(web_config) = &config.s3_web { info!("Initializing web server..."); + let web_server = WebServer::new(garage.clone(), web_config.root_domain.clone()); servers.push(( "Web", - tokio::spawn(WebServer::run( - garage.clone(), + tokio::spawn(web_server.run( web_config.bind_addr.clone(), - web_config.root_domain.clone(), wait_from(watch_cancel.clone()), )), )); -- cgit v1.2.3 From bcbd15da84181ad94ece7e794933a6ebd388f5bc Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 8 Feb 2024 19:02:48 +0100 Subject: [dep-upgrade-202402] cargo clippy fixes --- src/garage/cli/convert_db.rs | 4 ++-- src/garage/secrets.rs | 4 ++-- src/garage/tests/common/custom_requester.rs | 4 ++-- src/garage/tests/s3/website.rs | 6 ++---- 4 files changed, 8 insertions(+), 10 deletions(-) (limited to 'src/garage') diff --git a/src/garage/cli/convert_db.rs b/src/garage/cli/convert_db.rs index 044ccbb9..6b854ccb 100644 --- a/src/garage/cli/convert_db.rs +++ b/src/garage/cli/convert_db.rs @@ -67,8 +67,8 @@ fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result { #[cfg(feature = "sqlite")] Engine::Sqlite => { let db = sqlite_adapter::rusqlite::Connection::open(&path)?; - db.pragma_update(None, "journal_mode", &"WAL")?; - db.pragma_update(None, "synchronous", &"NORMAL")?; + db.pragma_update(None, "journal_mode", "WAL")?; + db.pragma_update(None, "synchronous", "NORMAL")?; Ok(sqlite_adapter::SqliteDb::init(db)) } #[cfg(feature = "lmdb")] diff --git a/src/garage/secrets.rs b/src/garage/secrets.rs index e96be9e4..8c89a262 100644 --- a/src/garage/secrets.rs +++ b/src/garage/secrets.rs @@ -213,10 +213,10 @@ mod tests { }; use std::os::unix::fs::PermissionsExt; - let metadata = std::fs::metadata(&path_secret_path)?; + let metadata = std::fs::metadata(path_secret_path)?; let mut perm = metadata.permissions(); perm.set_mode(0o660); - std::fs::set_permissions(&path_secret_path, perm)?; + std::fs::set_permissions(path_secret_path, perm)?; // Config file that just specifies the path let config = read_config(path_config.to_path_buf())?; diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs index 72fb1a46..e5f4cca1 100644 --- a/src/garage/tests/common/custom_requester.rs +++ b/src/garage/tests/common/custom_requester.rs @@ -205,8 +205,8 @@ impl<'a> RequestBuilder<'a> { all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone()); let mut signed_headers = all_headers - .iter() - .map(|(k, _)| k.as_ref()) + .keys() + .map(|k| k.as_ref()) .collect::>(); signed_headers.sort(); let signed_headers = signed_headers.join(";"); diff --git a/src/garage/tests/s3/website.rs b/src/garage/tests/s3/website.rs index 19f53fcd..0cadc388 100644 --- a/src/garage/tests/s3/website.rs +++ b/src/garage/tests/s3/website.rs @@ -61,8 +61,7 @@ async fn test_website() { .method("GET") .uri(format!( "http://127.0.0.1:{0}/check?domain={1}", - ctx.garage.admin_port, - BCKT_NAME.to_string() + ctx.garage.admin_port, BCKT_NAME )) .body(Body::new(Bytes::new())) .unwrap() @@ -136,8 +135,7 @@ async fn test_website() { .method("GET") .uri(format!( "http://127.0.0.1:{0}/check?domain={1}", - ctx.garage.admin_port, - BCKT_NAME.to_string() + ctx.garage.admin_port, BCKT_NAME )) .body(Body::new(Bytes::new())) .unwrap() -- cgit v1.2.3 From 5c63193d1de909cdecc501b482f5dc269a84874d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 8 Feb 2024 23:43:59 +0100 Subject: [dep-upgrade-202402] fix shutdown issue introduced when upgrading hyper --- src/garage/server.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'src/garage') diff --git a/src/garage/server.rs b/src/garage/server.rs index de8ac9e2..51b06b8e 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -88,7 +88,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er garage.clone(), s3_bind_addr.clone(), config.s3_api.s3_region.clone(), - wait_from(watch_cancel.clone()), + watch_cancel.clone(), )), )); } @@ -103,7 +103,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er garage.clone(), config.k2v_api.as_ref().unwrap().api_bind_addr.clone(), config.s3_api.s3_region.clone(), - wait_from(watch_cancel.clone()), + watch_cancel.clone(), )), )); } @@ -116,10 +116,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er let web_server = WebServer::new(garage.clone(), web_config.root_domain.clone()); servers.push(( "Web", - tokio::spawn(web_server.run( - web_config.bind_addr.clone(), - wait_from(watch_cancel.clone()), - )), + tokio::spawn(web_server.run(web_config.bind_addr.clone(), watch_cancel.clone())), )); } @@ -127,9 +124,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er info!("Launching Admin API server..."); servers.push(( "Admin", - tokio::spawn( - admin_server.run(admin_bind_addr.clone(), wait_from(watch_cancel.clone())), - ), + tokio::spawn(admin_server.run(admin_bind_addr.clone(), watch_cancel.clone())), )); } -- cgit v1.2.3 From 10bc2ead6015bf446451015ee73e902c3fd5e9a9 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 9 Feb 2024 14:15:29 +0100 Subject: [multi-char-delimiter-692] allow multi-character delimiters in List* (fix #692) --- src/garage/tests/s3/list.rs | 60 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) (limited to 'src/garage') diff --git a/src/garage/tests/s3/list.rs b/src/garage/tests/s3/list.rs index bb03f250..1b0c006d 100644 --- a/src/garage/tests/s3/list.rs +++ b/src/garage/tests/s3/list.rs @@ -613,3 +613,63 @@ async fn test_listmultipart() { assert!(r.common_prefixes.is_none()); } } + +#[tokio::test] +async fn test_multichar_delimiter() { + // Test case from dpape from issue #692 with reference results from Amazon + + let ctx = common::context(); + let bucket = ctx.create_bucket("multichardelim"); + + for k in [ + "a/", "a/b/", "a/b/c/", "a/b/c/d", "a/c/", "a/c/b/", "a/c/b/e", + ] { + ctx.client + .put_object() + .bucket(&bucket) + .key(k) + .send() + .await + .unwrap(); + } + + // With delimiter / + { + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .delimiter("/") + .send() + .await + .unwrap(); + + assert!(r.contents.is_none()); + + let common_prefixes = r.common_prefixes.unwrap(); + assert_eq!(common_prefixes.len(), 1); + assert_eq!(common_prefixes[0].prefix.as_deref().unwrap(), "a/"); + } + + // With delimiter b/ + { + let r = ctx + .client + .list_objects_v2() + .bucket(&bucket) + .delimiter("b/") + .send() + .await + .unwrap(); + + let contents = r.contents.unwrap(); + assert_eq!(contents.len(), 2); + assert_eq!(contents[0].key.as_deref().unwrap(), "a/"); + assert_eq!(contents[1].key.as_deref().unwrap(), "a/c/"); + + let common_prefixes = r.common_prefixes.unwrap(); + assert_eq!(common_prefixes.len(), 2); + assert_eq!(common_prefixes[0].prefix.as_deref().unwrap(), "a/b/"); + assert_eq!(common_prefixes[1].prefix.as_deref().unwrap(), "a/c/b/"); + } +} -- cgit v1.2.3 From 8da67b3aa2614326e1981ffbb215691583e058f2 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 9 Feb 2024 14:35:53 +0100 Subject: [block-info-short-682] `garage block info`: find blocks by prefix (fix #682) --- src/garage/admin/block.rs | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) (limited to 'src/garage') diff --git a/src/garage/admin/block.rs b/src/garage/admin/block.rs index c4a45738..edeb88c0 100644 --- a/src/garage/admin/block.rs +++ b/src/garage/admin/block.rs @@ -25,8 +25,7 @@ impl AdminRpcHandler { } async fn handle_block_info(&self, hash: &String) -> Result { - let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?; - let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?; + let hash = self.find_block_hash_by_prefix(hash)?; let refcount = self.garage.block_manager.get_block_rc(&hash)?; let block_refs = self .garage @@ -189,4 +188,48 @@ impl AdminRpcHandler { Ok(()) } + + // ---- helper function ---- + fn find_block_hash_by_prefix(&self, prefix: &str) -> Result { + if prefix.len() < 4 { + return Err(Error::BadRequest( + "Please specify at least 4 characters of the block hash".into(), + )); + } + + let prefix_bin = + hex::decode(&prefix[..prefix.len() & !1]).ok_or_bad_request("invalid hash")?; + + let iter = self + .garage + .block_ref_table + .data + .store + .range(&prefix_bin[..]..) + .map_err(GarageError::from)?; + let mut found = None; + for item in iter { + let (k, _v) = item.map_err(GarageError::from)?; + let hash = Hash::try_from(&k[..32]).unwrap(); + if &hash.as_slice()[..prefix_bin.len()] != prefix_bin { + break; + } + if hex::encode(hash.as_slice()).starts_with(prefix) { + match &found { + Some(x) if *x == hash => (), + Some(_) => { + return Err(Error::BadRequest(format!( + "Several blocks match prefix `{}`", + prefix + ))); + } + None => { + found = Some(hash); + } + } + } + } + + found.ok_or_else(|| Error::BadRequest("No matching block found".into())) + } } -- cgit v1.2.3 From 02e98e2d100a6af96369a72bc6979580424fe7df Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 9 Feb 2024 15:34:42 +0100 Subject: [header-override-650] implement header overriding in GetObject (fix #650) --- src/garage/tests/s3/objects.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'src/garage') diff --git a/src/garage/tests/s3/objects.rs b/src/garage/tests/s3/objects.rs index ca35b435..ad5f63f1 100644 --- a/src/garage/tests/s3/objects.rs +++ b/src/garage/tests/s3/objects.rs @@ -185,6 +185,30 @@ async fn test_getobject() { assert_eq!(o.content_range.unwrap().as_str(), "bytes 57-61/62"); assert_bytes_eq!(o.body, &BODY[57..]); } + { + let exp = aws_sdk_s3::primitives::DateTime::from_secs(10000000000); + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .response_content_type("application/x-dummy-test") + .response_cache_control("ccdummy") + .response_content_disposition("cddummy") + .response_content_encoding("cedummy") + .response_content_language("cldummy") + .response_expires(exp) + .send() + .await + .unwrap(); + assert_eq!(o.content_type.unwrap().as_str(), "application/x-dummy-test"); + assert_eq!(o.cache_control.unwrap().as_str(), "ccdummy"); + assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy"); + assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy"); + assert_eq!(o.content_language.unwrap().as_str(), "cldummy"); + assert_eq!(o.expires.unwrap(), exp); + assert_bytes_eq!(o.body, &BODY[..]); + } } #[tokio::test] -- cgit v1.2.3 From 25e5738568b2a021de3a79af3282b2b5feaee9e8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 12 Feb 2024 10:42:17 +0100 Subject: [fix-secrets-695] take into account rpc secret from file for cli commands (fix #695) --- src/garage/main.rs | 23 +++++++++++++++-------- src/garage/secrets.rs | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) (limited to 'src/garage') diff --git a/src/garage/main.rs b/src/garage/main.rs index 5c92dae4..1a6a6e32 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -174,7 +174,9 @@ async fn main() { } async fn cli_command(opt: Opt) -> Result<(), Error> { - let config = if opt.secrets.rpc_secret.is_none() || opt.rpc_host.is_none() { + let config = if (opt.secrets.rpc_secret.is_none() && opt.secrets.rpc_secret_file.is_none()) + || opt.rpc_host.is_none() + { Some(garage_util::config::read_config(opt.config_file.clone()) .err_context(format!("Unable to read configuration file {}. Configuration file is needed because -h or -s is not provided on the command line.", opt.config_file.to_string_lossy()))?) } else { @@ -182,14 +184,19 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { }; // Find and parse network RPC secret - let net_key_hex_str = opt - .secrets - .rpc_secret - .as_ref() - .or_else(|| config.as_ref().and_then(|c| c.rpc_secret.as_ref())) - .ok_or("No RPC secret provided")?; + let mut rpc_secret = config.as_ref().and_then(|c| c.rpc_secret.clone()); + secrets::fill_secret( + &mut rpc_secret, + &config.as_ref().and_then(|c| c.rpc_secret_file.clone()), + &opt.secrets.rpc_secret, + &opt.secrets.rpc_secret_file, + "rpc_secret", + true, + )?; + + let net_key_hex_str = rpc_secret.ok_or("No RPC secret provided")?; let network_key = NetworkKey::from_slice( - &hex::decode(net_key_hex_str).err_context("Invalid RPC secret key (bad hex)")?[..], + &hex::decode(&net_key_hex_str).err_context("Invalid RPC secret key (bad hex)")?[..], ) .ok_or("Invalid RPC secret provided (wrong length)")?; diff --git a/src/garage/secrets.rs b/src/garage/secrets.rs index 8c89a262..a2c64cef 100644 --- a/src/garage/secrets.rs +++ b/src/garage/secrets.rs @@ -83,7 +83,7 @@ pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result, config_secret_file: &Option, cli_secret: &Option, -- cgit v1.2.3 From bf283c99240787c7ffeba8ff3222283ee9fc61aa Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 12 Feb 2024 11:35:57 +0100 Subject: [fix-secrets-695] config: replace String by PathBuf for *_file --- src/garage/secrets.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'src/garage') diff --git a/src/garage/secrets.rs b/src/garage/secrets.rs index a2c64cef..c3d704aa 100644 --- a/src/garage/secrets.rs +++ b/src/garage/secrets.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use structopt::StructOpt; use garage_util::config::Config; @@ -23,7 +25,7 @@ pub struct Secrets { /// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret /// when running the daemon or doing admin operations #[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")] - pub rpc_secret_file: Option, + pub rpc_secret_file: Option, /// Admin API authentication token, replaces admin.admin_token in config.toml when /// running the Garage daemon @@ -33,7 +35,7 @@ pub struct Secrets { /// Admin API authentication token file path, replaces admin.admin_token in config.toml /// and admin-token when running the Garage daemon #[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")] - pub admin_token_file: Option, + pub admin_token_file: Option, /// Metrics API authentication token, replaces admin.metrics_token in config.toml when /// running the Garage daemon @@ -43,7 +45,7 @@ pub struct Secrets { /// Metrics API authentication token file path, replaces admin.metrics_token in config.toml /// and metrics-token when running the Garage daemon #[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")] - pub metrics_token_file: Option, + pub metrics_token_file: Option, } /// Single function to fill all secrets in the Config struct from their correct source (value @@ -85,9 +87,9 @@ pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result, - config_secret_file: &Option, + config_secret_file: &Option, cli_secret: &Option, - cli_secret_file: &Option, + cli_secret_file: &Option, name: &'static str, allow_world_readable: bool, ) -> Result<(), Error> { @@ -117,14 +119,14 @@ pub(crate) fn fill_secret( Ok(()) } -fn read_secret_file(file_path: &String, allow_world_readable: bool) -> Result { +fn read_secret_file(file_path: &PathBuf, allow_world_readable: bool) -> Result { if !allow_world_readable { #[cfg(unix)] { use std::os::unix::fs::MetadataExt; let metadata = std::fs::metadata(file_path)?; if metadata.mode() & 0o077 != 0 { - return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into()); + return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path.display(), metadata.mode()).into()); } } } @@ -260,7 +262,7 @@ mod tests { let config = fill_secrets( config, Secrets { - rpc_secret_file: Some(path_secret2.display().to_string()), + rpc_secret_file: Some(path_secret2.clone()), ..Default::default() }, )?; @@ -271,7 +273,7 @@ mod tests { config, Secrets { rpc_secret: Some("baz".into()), - rpc_secret_file: Some(path_secret2.display().to_string()), + rpc_secret_file: Some(path_secret2.clone()), ..Default::default() } ) -- cgit v1.2.3