From 13c554988623663a9416439baf4f85f6fa91e502 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 4 Jan 2023 11:47:56 +0100 Subject: Remove token_bucket.rs --- src/util/lib.rs | 1 - src/util/token_bucket.rs | 40 ---------------------------------------- 2 files changed, 41 deletions(-) delete mode 100644 src/util/token_bucket.rs (limited to 'src/util') diff --git a/src/util/lib.rs b/src/util/lib.rs index be82061f..d35ca72f 100644 --- a/src/util/lib.rs +++ b/src/util/lib.rs @@ -15,6 +15,5 @@ pub mod metrics; pub mod migrate; pub mod persister; pub mod time; -pub mod token_bucket; pub mod tranquilizer; pub mod version; diff --git a/src/util/token_bucket.rs b/src/util/token_bucket.rs deleted file mode 100644 index cc0dfa1f..00000000 --- a/src/util/token_bucket.rs +++ /dev/null @@ -1,40 +0,0 @@ -use std::time::{Duration, Instant}; - -use tokio::time::sleep; - -pub struct TokenBucket { - // Replenish rate: number of tokens per second - replenish_rate: u64, - // Current number of tokens - tokens: u64, - // Last replenish time - last_replenish: Instant, -} - -impl TokenBucket { - pub fn new(replenish_rate: u64) -> Self { - Self { - replenish_rate, - tokens: 0, - last_replenish: Instant::now(), - } - } - - pub async fn take(&mut self, tokens: u64) { - while self.tokens < tokens { - let needed = tokens - self.tokens; - let delay = (needed as f64) / (self.replenish_rate as f64); - sleep(Duration::from_secs_f64(delay)).await; - self.replenish(); - } - self.tokens -= tokens; - } - - pub fn replenish(&mut self) { - let now = Instant::now(); - let new_tokens = - ((now - self.last_replenish).as_secs_f64() * (self.replenish_rate as f64)) as u64; - self.tokens += new_tokens; - self.last_replenish = now; - } -} -- cgit v1.2.3 From f3f27293df83986ba29fb03f8af26a2177518e20 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 4 Jan 2023 13:07:13 +0100 Subject: Uniform framework for bg variable management --- src/util/background/mod.rs | 1 + src/util/background/vars.rs | 107 ++++++++++++++++++++++++++++++++++++++++++++ src/util/persister.rs | 34 ++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/util/background/vars.rs (limited to 'src/util') diff --git a/src/util/background/mod.rs b/src/util/background/mod.rs index 41b48e93..607cd7a3 100644 --- a/src/util/background/mod.rs +++ b/src/util/background/mod.rs @@ -1,5 +1,6 @@ //! Job runner for futures and async functions +pub mod vars; pub mod worker; use std::collections::HashMap; diff --git a/src/util/background/vars.rs b/src/util/background/vars.rs new file mode 100644 index 00000000..fe54268e --- /dev/null +++ b/src/util/background/vars.rs @@ -0,0 +1,107 @@ +use std::collections::HashMap; +use std::str::FromStr; + +use crate::error::{Error, OkOrMessage}; +use crate::migrate::Migrate; +use crate::persister::PersisterShared; + +pub struct BgVars { + vars: HashMap<&'static str, Box>, +} + +impl BgVars { + pub fn new() -> Self { + Self { + vars: HashMap::new(), + } + } + + pub fn register_rw( + &mut self, + p: &PersisterShared, + name: &'static str, + get_fn: GF, + set_fn: SF, + ) where + V: Migrate + Default + Send + Sync, + T: FromStr + ToString + Send + Sync + 'static, + GF: Fn(&PersisterShared) -> T + Send + Sync + 'static, + SF: Fn(&PersisterShared, T) -> Result<(), Error> + Send + Sync + 'static, + { + let p1 = p.clone(); + let get_fn = move || get_fn(&p1); + + let p2 = p.clone(); + let set_fn = move |v| set_fn(&p2, v); + + self.vars.insert(name, Box::new(BgVar { get_fn, set_fn })); + } + + pub fn register_ro(&mut self, p: &PersisterShared, name: &'static str, get_fn: GF) + where + V: Migrate + Default + Send + Sync, + T: FromStr + ToString + Send + Sync + 'static, + GF: Fn(&PersisterShared) -> T + Send + Sync + 'static, + { + let p1 = p.clone(); + let get_fn = move || get_fn(&p1); + + let set_fn = move |_| Err(Error::Message(format!("Cannot set value of {}", name))); + + self.vars.insert(name, Box::new(BgVar { get_fn, set_fn })); + } + + pub fn get(&self, var: &str) -> Result { + Ok(self + .vars + .get(var) + .ok_or_message("variable does not exist")? + .get()) + } + + pub fn get_all(&self) -> Vec<(&'static str, String)> { + self.vars.iter().map(|(k, v)| (*k, v.get())).collect() + } + + pub fn set(&self, var: &str, val: &str) -> Result<(), Error> { + self.vars + .get(var) + .ok_or_message("variable does not exist")? + .set(val) + } +} + +// ---- + +trait BgVarTrait: Send + Sync + 'static { + fn get(&self) -> String; + fn set(&self, v: &str) -> Result<(), Error>; +} + +struct BgVar +where + T: FromStr + ToString + Send + Sync + 'static, + GF: Fn() -> T + Send + Sync + 'static, + SF: Fn(T) -> Result<(), Error> + Sync + Send + 'static, +{ + get_fn: GF, + set_fn: SF, +} + +impl BgVarTrait for BgVar +where + T: FromStr + ToString + Sync + Send + 'static, + GF: Fn() -> T + Sync + Send + 'static, + SF: Fn(T) -> Result<(), Error> + Sync + Send + 'static, +{ + fn get(&self) -> String { + (self.get_fn)().to_string() + } + + fn set(&self, vstr: &str) -> Result<(), Error> { + let value = vstr + .parse() + .map_err(|_| Error::Message(format!("invalid value: {}", vstr)))?; + (self.set_fn)(value) + } +} diff --git a/src/util/persister.rs b/src/util/persister.rs index 4b9adf51..5c66bbed 100644 --- a/src/util/persister.rs +++ b/src/util/persister.rs @@ -1,5 +1,6 @@ use std::io::{Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -84,3 +85,36 @@ impl Persister { Ok(()) } } + +pub struct PersisterShared(Arc<(Persister, RwLock)>); + +impl Clone for PersisterShared { + fn clone(&self) -> PersisterShared { + PersisterShared(self.0.clone()) + } +} + +impl PersisterShared { + pub fn new(base_dir: &Path, file_name: &str) -> Self { + let persister = Persister::new(base_dir, file_name); + let value = persister.load().unwrap_or_default(); + Self(Arc::new((persister, RwLock::new(value)))) + } + + pub fn get_with(&self, f: F) -> R + where + F: FnOnce(&V) -> R, + { + let value = self.0 .1.read().unwrap(); + f(&value) + } + + pub fn set_with(&self, f: F) -> Result<(), Error> + where + F: FnOnce(&mut V), + { + let mut value = self.0 .1.write().unwrap(); + f(&mut value); + self.0 .0.save(&value) + } +} -- cgit v1.2.3 From 29dbcb82780dcdb6f2a01a9da5122e70abaf93bf Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 4 Jan 2023 13:25:57 +0100 Subject: bg var operation on all nodes at once --- src/util/background/vars.rs | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/util') diff --git a/src/util/background/vars.rs b/src/util/background/vars.rs index fe54268e..7a449c95 100644 --- a/src/util/background/vars.rs +++ b/src/util/background/vars.rs @@ -71,6 +71,12 @@ impl BgVars { } } +impl Default for BgVars { + fn default() -> Self { + Self::new() + } +} + // ---- trait BgVarTrait: Send + Sync + 'static { -- cgit v1.2.3 From f2106c27336d7d03671dbbbcd1401232c2beb61f Mon Sep 17 00:00:00 2001 From: Felix Scheinost Date: Wed, 4 Jan 2023 18:28:56 +0100 Subject: Implement `rpc_secret_file` --- src/util/config.rs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) (limited to 'src/util') diff --git a/src/util/config.rs b/src/util/config.rs index 04f8375a..e1120822 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -34,7 +34,10 @@ pub struct Config { pub compression_level: Option, /// RPC secret key: 32 bytes hex encoded - pub rpc_secret: String, + pub rpc_secret: Option, + + /// Optional file where RPC secret key is read from + pub rpc_secret_file: Option, /// Address to bind for RPC pub rpc_bind_addr: SocketAddr, @@ -177,7 +180,26 @@ pub fn read_config(config_file: PathBuf) -> Result { let mut config = String::new(); file.read_to_string(&mut config)?; - Ok(toml::from_str(&config)?) + let mut parsed_config: Config = toml::from_str(&config)?; + + match (&parsed_config.rpc_secret, &parsed_config.rpc_secret_file) { + (Some(_), _) => {} + (None, Some(rpc_secret_file_path_string)) => { + let mut rpc_secret_file = std::fs::OpenOptions::new() + .read(true) + .open(rpc_secret_file_path_string)?; + let mut rpc_secret_from_file = String::new(); + rpc_secret_file.read_to_string(&mut rpc_secret_from_file)?; + // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`. + // also editors sometimes add a trailing newline + parsed_config.rpc_secret = Some(String::from(rpc_secret_from_file.trim_end())); + } + (None, None) => { + return Err("either `rpc_secret` or `rpc_secret_file` needs to be set".into()) + } + }; + + Ok(parsed_config) } fn default_compression() -> Option { -- cgit v1.2.3 From 7b62fe3f0b1dcb336f153e54e72d8587292aa01b Mon Sep 17 00:00:00 2001 From: Felix Scheinost Date: Sat, 7 Jan 2023 13:49:03 +0100 Subject: Error on both `rpc_secret` and `rpc_secret_file` --- src/util/config.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'src/util') diff --git a/src/util/config.rs b/src/util/config.rs index e1120822..5471fc41 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -34,6 +34,7 @@ pub struct Config { pub compression_level: Option, /// RPC secret key: 32 bytes hex encoded + /// Note: When using `read_config` this should never be `None` pub rpc_secret: Option, /// Optional file where RPC secret key is read from @@ -183,7 +184,12 @@ pub fn read_config(config_file: PathBuf) -> Result { let mut parsed_config: Config = toml::from_str(&config)?; match (&parsed_config.rpc_secret, &parsed_config.rpc_secret_file) { - (Some(_), _) => {} + (Some(_), None) => { + // no-op + } + (Some(_), Some(_)) => { + return Err("only one of `rpc_secret` and `rpc_secret_file` can be set".into()) + } (None, Some(rpc_secret_file_path_string)) => { let mut rpc_secret_file = std::fs::OpenOptions::new() .read(true) -- cgit v1.2.3 From d6ea0cbefa6dcf89ea30e1cd2d36854d8160d6b1 Mon Sep 17 00:00:00 2001 From: Felix Scheinost Date: Sat, 7 Jan 2023 13:49:15 +0100 Subject: Add tests for `rpc_secret_file` --- src/util/Cargo.toml | 2 + src/util/config.rs | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) (limited to 'src/util') diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 32e9c851..1017b1ce 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -47,6 +47,8 @@ hyper = "0.14" opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] } +[dev-dependencies] +mktemp = "0.4" [features] k2v = [] diff --git a/src/util/config.rs b/src/util/config.rs index 5471fc41..f0a881aa 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -261,3 +261,123 @@ where deserializer.deserialize_any(OptionVisitor) } + +#[cfg(test)] +mod tests { + use crate::error::Error; + use std::fs::File; + use std::io::Write; + + #[test] + fn test_rpc_secret_is_required() -> Result<(), Error> { + let path1 = mktemp::Temp::new_file()?; + let mut file1 = File::create(path1.as_path())?; + writeln!( + file1, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_mode = "3" + rpc_bind_addr = "[::]:3901" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "# + )?; + assert_eq!( + "either `rpc_secret` or `rpc_secret_file` needs to be set", + super::read_config(path1.to_path_buf()) + .unwrap_err() + .to_string() + ); + drop(path1); + drop(file1); + + let path2 = mktemp::Temp::new_file()?; + let mut file2 = File::create(path2.as_path())?; + writeln!( + file2, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_mode = "3" + rpc_bind_addr = "[::]:3901" + rpc_secret = "foo" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "# + )?; + + let config = super::read_config(path2.to_path_buf())?; + assert_eq!("foo", config.rpc_secret.unwrap()); + drop(path2); + drop(file2); + + Ok(()) + } + + #[test] + fn test_rpc_secret_file_works() -> Result<(), Error> { + let path_secret = mktemp::Temp::new_file()?; + let mut file_secret = File::create(path_secret.as_path())?; + writeln!(file_secret, "foo")?; + drop(file_secret); + + let path_config = mktemp::Temp::new_file()?; + let mut file_config = File::create(path_config.as_path())?; + let path_secret_path = path_secret.as_path().display(); + writeln!( + file_config, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_mode = "3" + rpc_bind_addr = "[::]:3901" + rpc_secret_file = "{path_secret_path}" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "# + )?; + let config = super::read_config(path_config.to_path_buf())?; + assert_eq!("foo", config.rpc_secret.unwrap()); + drop(path_config); + drop(path_secret); + drop(file_config); + Ok(()) + } + + #[test] + fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> { + let path_config = mktemp::Temp::new_file()?; + let mut file_config = File::create(path_config.as_path())?; + writeln!( + file_config, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_mode = "3" + rpc_bind_addr = "[::]:3901" + rpc_secret= "dummy" + rpc_secret_file = "dummy" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "# + )?; + assert_eq!( + "only one of `rpc_secret` and `rpc_secret_file` can be set", + super::read_config(path_config.to_path_buf()) + .unwrap_err() + .to_string() + ); + drop(path_config); + drop(file_config); + Ok(()) + } +} -- cgit v1.2.3 From b4a1a6a32f3581bdf1ac673066ef3b4ebd0a2fdc Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 23 Jan 2023 19:53:12 +0000 Subject: util/time.rs: Updated deprecated associated function to timestamp_opt(). --- src/util/time.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/util') diff --git a/src/util/time.rs b/src/util/time.rs index 257b4d2a..42f41a44 100644 --- a/src/util/time.rs +++ b/src/util/time.rs @@ -25,6 +25,6 @@ pub fn increment_logical_clock_2(prev: u64, prev2: u64) -> u64 { pub fn msec_to_rfc3339(msecs: u64) -> String { let secs = msecs as i64 / 1000; let nanos = (msecs as i64 % 1000) as u32 * 1_000_000; - let timestamp = Utc.timestamp(secs, nanos); + let timestamp = Utc.timestamp_opt(secs, nanos).unwrap(); timestamp.to_rfc3339_opts(SecondsFormat::Millis, true) } -- cgit v1.2.3 From d3b2a68988b71b46f1a13a3db202fa03ea63849b Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 23 Jan 2023 22:00:17 +0000 Subject: {garage,util}/Cargo.toml: Updated toml from 0.5 to 0.6. --- src/util/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/util') diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 1017b1ce..33580a43 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -35,7 +35,7 @@ chrono = "0.4" rmp-serde = "0.15" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_json = "1.0" -toml = "0.5" +toml = "0.6" futures = "0.3" tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -- cgit v1.2.3 From fbafa76284e35b11b3862c1b8aa7ba85939799d9 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 23 Jan 2023 22:13:07 +0000 Subject: {db,util}/Cargo.toml: Updated mktemp from 0.4 to 0.5. --- src/util/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/util') diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 33580a43..5a36386d 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -48,7 +48,7 @@ hyper = "0.14" opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] } [dev-dependencies] -mktemp = "0.4" +mktemp = "0.5" [features] k2v = [] -- cgit v1.2.3 From f952e37ba772b9286ba814ddb29f17d0ec7731c6 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 23 Jan 2023 22:19:37 +0000 Subject: {model,util}/Cargo.toml: Updated blake2 from 0.9 to 0.10. --- src/util/Cargo.toml | 2 +- src/util/data.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src/util') diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 5a36386d..cbda43c0 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -18,7 +18,7 @@ garage_db = { version = "0.8.1", path = "../db" } arc-swap = "1.0" async-trait = "0.1" -blake2 = "0.9" +blake2 = "0.10" bytes = "1.0" digest = "0.10" err-derive = "0.3" diff --git a/src/util/data.rs b/src/util/data.rs index 3f61e301..bdd8daee 100644 --- a/src/util/data.rs +++ b/src/util/data.rs @@ -115,9 +115,9 @@ pub fn sha256sum(data: &[u8]) -> Hash { /// Compute the blake2 of a slice pub fn blake2sum(data: &[u8]) -> Hash { - use blake2::{Blake2b, Digest}; + use blake2::{Blake2b512, Digest}; - let mut hasher = Blake2b::new(); + let mut hasher = Blake2b512::new(); hasher.update(data); let mut hash = [0u8; 32]; hash.copy_from_slice(&hasher.finalize()[..32]); -- cgit v1.2.3 From 20c1cdf662a0b5368a8020526f43e08baedfedaa Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 23 Jan 2023 22:27:44 +0000 Subject: Cargo.toml: Loosen tracing dependency to just 0.1. --- src/util/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/util') diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index cbda43c0..abeccbbd 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -27,7 +27,7 @@ hexdump = "0.1" xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] } hex = "0.4" lazy_static = "1.4" -tracing = "0.1.30" +tracing = "0.1" rand = "0.8" sha2 = "0.10" -- cgit v1.2.3 From 656b8d42de2fc945c988094418c90d29d000be32 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 3 Feb 2023 15:27:39 +0100 Subject: secrets can be passed directly in config, as file, or as env --- src/util/config.rs | 116 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 46 deletions(-) (limited to 'src/util') diff --git a/src/util/config.rs b/src/util/config.rs index f0a881aa..c59e9c1d 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -34,9 +34,7 @@ pub struct Config { pub compression_level: Option, /// RPC secret key: 32 bytes hex encoded - /// Note: When using `read_config` this should never be `None` pub rpc_secret: Option, - /// Optional file where RPC secret key is read from pub rpc_secret_file: Option, @@ -122,10 +120,17 @@ pub struct WebConfig { pub struct AdminConfig { /// Address and port to bind for admin API serving pub api_bind_addr: Option, + /// Bearer token to use to scrape metrics pub metrics_token: Option, + /// File to read metrics token from + pub metrics_token_file: Option, + /// Bearer token to use to access Admin API endpoints pub admin_token: Option, + /// File to read admin token from + pub admin_token_file: Option, + /// OTLP server to where to export traces pub trace_sink: Option, } @@ -183,29 +188,55 @@ pub fn read_config(config_file: PathBuf) -> Result { let mut parsed_config: Config = toml::from_str(&config)?; - match (&parsed_config.rpc_secret, &parsed_config.rpc_secret_file) { - (Some(_), None) => { + secret_from_file( + &mut parsed_config.rpc_secret, + &mut parsed_config.rpc_secret_file, + "rpc_secret", + )?; + secret_from_file( + &mut parsed_config.admin.metrics_token, + &mut parsed_config.admin.metrics_token_file, + "admin.metrics_token", + )?; + secret_from_file( + &mut parsed_config.admin.admin_token, + &mut parsed_config.admin.admin_token_file, + "admin.admin_token", + )?; + + Ok(parsed_config) +} + +fn secret_from_file( + secret: &mut Option, + secret_file: &mut Option, + name: &'static str, +) -> Result<(), Error> { + match (&secret, &secret_file) { + (_, None) => { // no-op } (Some(_), Some(_)) => { - return Err("only one of `rpc_secret` and `rpc_secret_file` can be set".into()) + return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); } - (None, Some(rpc_secret_file_path_string)) => { - let mut rpc_secret_file = std::fs::OpenOptions::new() - .read(true) - .open(rpc_secret_file_path_string)?; - let mut rpc_secret_from_file = String::new(); - rpc_secret_file.read_to_string(&mut rpc_secret_from_file)?; + (None, Some(file_path)) => { + #[cfg(unix)] + if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::metadata(&file_path)?; + if metadata.mode() & 0o077 != 0 { + return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into()); + } + } + let mut file = std::fs::OpenOptions::new().read(true).open(file_path)?; + let mut secret_buf = String::new(); + file.read_to_string(&mut secret_buf)?; // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`. // also editors sometimes add a trailing newline - parsed_config.rpc_secret = Some(String::from(rpc_secret_from_file.trim_end())); - } - (None, None) => { - return Err("either `rpc_secret` or `rpc_secret_file` needs to be set".into()) + *secret = Some(String::from(secret_buf.trim_end())); } - }; - - Ok(parsed_config) + } + Ok(()) } fn default_compression() -> Option { @@ -269,31 +300,7 @@ mod tests { use std::io::Write; #[test] - fn test_rpc_secret_is_required() -> Result<(), Error> { - let path1 = mktemp::Temp::new_file()?; - let mut file1 = File::create(path1.as_path())?; - writeln!( - file1, - r#" - metadata_dir = "/tmp/garage/meta" - data_dir = "/tmp/garage/data" - replication_mode = "3" - rpc_bind_addr = "[::]:3901" - - [s3_api] - s3_region = "garage" - api_bind_addr = "[::]:3900" - "# - )?; - assert_eq!( - "either `rpc_secret` or `rpc_secret_file` needs to be set", - super::read_config(path1.to_path_buf()) - .unwrap_err() - .to_string() - ); - drop(path1); - drop(file1); - + fn test_rpc_secret() -> Result<(), Error> { let path2 = mktemp::Temp::new_file()?; let mut file2 = File::create(path2.as_path())?; writeln!( @@ -328,7 +335,7 @@ mod tests { let path_config = mktemp::Temp::new_file()?; let mut file_config = File::create(path_config.as_path())?; - let path_secret_path = path_secret.as_path().display(); + let path_secret_path = path_secret.as_path(); writeln!( file_config, r#" @@ -336,15 +343,32 @@ mod tests { data_dir = "/tmp/garage/data" replication_mode = "3" rpc_bind_addr = "[::]:3901" - rpc_secret_file = "{path_secret_path}" + rpc_secret_file = "{}" [s3_api] s3_region = "garage" api_bind_addr = "[::]:3900" - "# + "#, + path_secret_path.display() )?; let config = super::read_config(path_config.to_path_buf())?; assert_eq!("foo", config.rpc_secret.unwrap()); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let metadata = std::fs::metadata(&path_secret_path)?; + let mut perm = metadata.permissions(); + perm.set_mode(0o660); + std::fs::set_permissions(&path_secret_path, perm)?; + + std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "false"); + assert!(super::read_config(path_config.to_path_buf()).is_err()); + + std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "true"); + assert!(super::read_config(path_config.to_path_buf()).is_ok()); + } + drop(path_config); drop(path_secret); drop(file_config); -- cgit v1.2.3 From 80e232699825c5c512e8714e08b6a80992a06498 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 6 Feb 2023 12:23:55 +0100 Subject: fixes for pr 499 --- src/util/config.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/util') diff --git a/src/util/config.rs b/src/util/config.rs index c59e9c1d..2176353e 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -190,17 +190,17 @@ pub fn read_config(config_file: PathBuf) -> Result { secret_from_file( &mut parsed_config.rpc_secret, - &mut parsed_config.rpc_secret_file, + &parsed_config.rpc_secret_file, "rpc_secret", )?; secret_from_file( &mut parsed_config.admin.metrics_token, - &mut parsed_config.admin.metrics_token_file, + &parsed_config.admin.metrics_token_file, "admin.metrics_token", )?; secret_from_file( &mut parsed_config.admin.admin_token, - &mut parsed_config.admin.admin_token_file, + &parsed_config.admin.admin_token_file, "admin.admin_token", )?; @@ -209,7 +209,7 @@ pub fn read_config(config_file: PathBuf) -> Result { fn secret_from_file( secret: &mut Option, - secret_file: &mut Option, + secret_file: &Option, name: &'static str, ) -> Result<(), Error> { match (&secret, &secret_file) { -- cgit v1.2.3 From ff70e09aa090a140729483783076e7938fad0d39 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 9 Feb 2023 13:48:48 +0000 Subject: util/forwarded_headers.rs: Generalized handle_forwarded_for_headers() here. --- src/util/forwarded_headers.rs | 63 +++++++++++++++++++++++++++++++++++++++++++ src/util/lib.rs | 1 + 2 files changed, 64 insertions(+) create mode 100644 src/util/forwarded_headers.rs (limited to 'src/util') diff --git a/src/util/forwarded_headers.rs b/src/util/forwarded_headers.rs new file mode 100644 index 00000000..6ae275aa --- /dev/null +++ b/src/util/forwarded_headers.rs @@ -0,0 +1,63 @@ +use http::{HeaderMap, HeaderValue}; +use std::net::IpAddr; +use std::str::FromStr; + +use crate::error::{Error, OkOrMessage}; + +pub fn handle_forwarded_for_headers(headers: &HeaderMap) -> Result { + let forwarded_for_header = headers + .get("x-forwarded-for") + .ok_or_message("X-Forwarded-For header not provided")?; + + let forwarded_for_ip_str = forwarded_for_header + .to_str() + .ok_or_message("Error parsing X-Forwarded-For header")?; + + let client_ip = IpAddr::from_str(&forwarded_for_ip_str) + .ok_or_message("Valid IP address not found in X-Forwarded-For header")?; + + Ok(client_ip.to_string()) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_handle_forwarded_for_headers_ipv4_client() { + let mut test_headers = HeaderMap::new(); + test_headers.insert("X-Forwarded-For", "192.0.2.100".parse().unwrap()); + + if let Ok(forwarded_ip) = handle_forwarded_for_headers(&test_headers) { + assert_eq!(forwarded_ip, "192.0.2.100"); + } + } + + #[test] + fn test_handle_forwarded_for_headers_ipv6_client() { + let mut test_headers = HeaderMap::new(); + test_headers.insert("X-Forwarded-For", "2001:db8::f00d:cafe".parse().unwrap()); + + if let Ok(forwarded_ip) = handle_forwarded_for_headers(&test_headers) { + assert_eq!(forwarded_ip, "2001:db8::f00d:cafe"); + } + } + + #[test] + fn test_handle_forwarded_for_headers_invalid_ip() { + let mut test_headers = HeaderMap::new(); + test_headers.insert("X-Forwarded-For", "www.example.com".parse().unwrap()); + + let result = handle_forwarded_for_headers(&test_headers); + assert!(result.is_err()); + } + + #[test] + fn test_handle_forwarded_for_headers_missing() { + let mut test_headers = HeaderMap::new(); + test_headers.insert("Host", "www.deuxfleurs.fr".parse().unwrap()); + + let result = handle_forwarded_for_headers(&test_headers); + assert!(result.is_err()); + } +} diff --git a/src/util/lib.rs b/src/util/lib.rs index d35ca72f..c9110fb2 100644 --- a/src/util/lib.rs +++ b/src/util/lib.rs @@ -11,6 +11,7 @@ pub mod data; pub mod encode; pub mod error; pub mod formater; +pub mod forwarded_headers; pub mod metrics; pub mod migrate; pub mod persister; -- cgit v1.2.3 From 25f2a46fc3e50c882cd518244bb49d44d01ca442 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Fri, 10 Mar 2023 11:40:58 +0000 Subject: rpc/system_metrics.rs: Added rustversion label to garage_build_info metric. --- src/util/Cargo.toml | 3 +++ src/util/build.rs | 8 ++++++++ src/util/version.rs | 4 ++++ 3 files changed, 15 insertions(+) create mode 100644 src/util/build.rs (limited to 'src/util') diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index abeccbbd..9c182fd6 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -47,6 +47,9 @@ hyper = "0.14" opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] } +[build-dependencies] +rustc_version = "0.4.0" + [dev-dependencies] mktemp = "0.5" diff --git a/src/util/build.rs b/src/util/build.rs new file mode 100644 index 00000000..a4e955b8 --- /dev/null +++ b/src/util/build.rs @@ -0,0 +1,8 @@ +use rustc_version::version; + +fn main() { + // Acquire the version of Rust used to compile, this is added as a label to + // the garage_build_info metric. + let v = version().unwrap(); + println!("cargo:rustc-env=RUSTC_VERSION={v}"); +} diff --git a/src/util/version.rs b/src/util/version.rs index b515dccc..2b2ea271 100644 --- a/src/util/version.rs +++ b/src/util/version.rs @@ -26,3 +26,7 @@ pub fn init_version(version: &'static str) { pub fn init_features(features: &'static [&'static str]) { FEATURES.store(Some(Arc::new(features))); } + +pub fn rust_version() -> &'static str { + env!("RUSTC_VERSION") +} -- cgit v1.2.3 From 0a1ddcf6301359cde654003b46636497f6a417a4 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 13 Mar 2023 18:44:09 +0100 Subject: Prepare for v0.8.2 --- src/util/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/util') diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 9c182fd6..2e6231f6 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_util" -version = "0.8.1" +version = "0.8.2" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,7 +14,7 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_db = { version = "0.8.1", path = "../db" } +garage_db = { version = "0.8.2", path = "../db" } arc-swap = "1.0" async-trait = "0.1" -- cgit v1.2.3