diff options
48 files changed, 1526 insertions, 290 deletions
@@ -1198,7 +1198,7 @@ dependencies = [ [[package]] name = "garage" -version = "0.9.0" +version = "0.9.1" dependencies = [ "assert-json-diff", "async-trait", @@ -1227,6 +1227,7 @@ dependencies = [ "hyper", "k2v-client", "kuska-sodiumoxide", + "mktemp", "netapp", "opentelemetry", "opentelemetry-otlp", @@ -1249,7 +1250,7 @@ dependencies = [ [[package]] name = "garage_api" -version = "0.9.0" +version = "0.9.1" dependencies = [ "async-trait", "base64 0.21.3", @@ -1295,7 +1296,7 @@ dependencies = [ [[package]] name = "garage_block" -version = "0.9.0" +version = "0.9.1" dependencies = [ "arc-swap", "async-compression", @@ -1321,7 +1322,7 @@ dependencies = [ [[package]] name = "garage_db" -version = "0.9.0" +version = "0.9.1" dependencies = [ "clap 4.4.0", "err-derive", @@ -1336,7 +1337,7 @@ dependencies = [ [[package]] name = "garage_model" -version = "0.9.0" +version = "0.9.1" dependencies = [ "arc-swap", "async-trait", @@ -1364,7 +1365,7 @@ dependencies = [ [[package]] name = "garage_rpc" -version = "0.9.0" +version = "0.9.1" dependencies = [ "arc-swap", "async-trait", @@ -1399,7 +1400,7 @@ dependencies = [ [[package]] name = "garage_table" -version = "0.9.0" +version = "0.9.1" dependencies = [ "arc-swap", "async-trait", @@ -1421,7 +1422,7 @@ dependencies = [ [[package]] name = "garage_util" -version = "0.9.0" +version = "0.9.1" dependencies = [ "arc-swap", "async-trait", @@ -1455,7 +1456,7 @@ dependencies = [ [[package]] name = "garage_web" -version = "0.9.0" +version = "0.9.1" dependencies = [ "err-derive", "futures", @@ -33,7 +33,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "1a87886681a3ef0b83c95addc26674a538b8a93d35bc80db8998e1fcd0821f6c"; + nixifiedLockHash = "f8da48e74313b18ee372a07d6a7d63a27c7effde24ed8599dfb91ea311df10f8"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -57,16 +57,16 @@ in { cargo2nixVersion = "0.11.0"; workspace = { - garage_db = rustPackages.unknown.garage_db."0.9.0"; - garage_util = rustPackages.unknown.garage_util."0.9.0"; - garage_rpc = rustPackages.unknown.garage_rpc."0.9.0"; + garage_db = rustPackages.unknown.garage_db."0.9.1"; + garage_util = rustPackages.unknown.garage_util."0.9.1"; + garage_rpc = rustPackages.unknown.garage_rpc."0.9.1"; format_table = rustPackages.unknown.format_table."0.1.1"; - garage_table = rustPackages.unknown.garage_table."0.9.0"; - garage_block = rustPackages.unknown.garage_block."0.9.0"; - garage_model = rustPackages.unknown.garage_model."0.9.0"; - garage_api = rustPackages.unknown.garage_api."0.9.0"; - garage_web = rustPackages.unknown.garage_web."0.9.0"; - garage = rustPackages.unknown.garage."0.9.0"; + garage_table = rustPackages.unknown.garage_table."0.9.1"; + garage_block = rustPackages.unknown.garage_block."0.9.1"; + garage_model = rustPackages.unknown.garage_model."0.9.1"; + garage_api = rustPackages.unknown.garage_api."0.9.1"; + garage_web = rustPackages.unknown.garage_web."0.9.1"; + garage = rustPackages.unknown.garage."0.9.1"; k2v-client = rustPackages.unknown.k2v-client."0.0.4"; }; "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.21.0" = overridableMkRustCrate (profileName: rec { @@ -1705,9 +1705,9 @@ in }; }); - "unknown".garage."0.9.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage."0.9.1" = overridableMkRustCrate (profileName: rec { name = "garage"; - version = "0.9.0"; + version = "0.9.1"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/garage"); features = builtins.concatLists [ @@ -1734,14 +1734,14 @@ in format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_api = (rustPackages."unknown".garage_api."0.9.0" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.9.0" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.9.0" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out; - garage_web = (rustPackages."unknown".garage_web."0.9.0" { inherit profileName; }).out; + garage_api = (rustPackages."unknown".garage_api."0.9.1" { inherit profileName; }).out; + garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; + garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; + garage_web = (rustPackages."unknown".garage_web."0.9.1" { inherit profileName; }).out; git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; @@ -1771,15 +1771,16 @@ in http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; k2v_client = (rustPackages."unknown".k2v-client."0.0.4" { inherit profileName; }).out; + mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" { inherit profileName; }).out; serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out; sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out; static_init = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.3" { inherit profileName; }).out; }; }); - "unknown".garage_api."0.9.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_api."0.9.1" = overridableMkRustCrate (profileName: rec { name = "garage_api"; - version = "0.9.0"; + version = "0.9.1"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/api"); features = builtins.concatLists [ @@ -1798,11 +1799,11 @@ in form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.0" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.9.0" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.9.0" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out; + garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out; + garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; @@ -1832,9 +1833,9 @@ in }; }); - "unknown".garage_block."0.9.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_block."0.9.1" = overridableMkRustCrate (profileName: rec { name = "garage_block"; - version = "0.9.0"; + version = "0.9.1"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/block"); features = builtins.concatLists [ @@ -1848,10 +1849,10 @@ in bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; @@ -1864,9 +1865,9 @@ in }; }); - "unknown".garage_db."0.9.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_db."0.9.1" = overridableMkRustCrate (profileName: rec { name = "garage_db"; - version = "0.9.0"; + version = "0.9.1"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/db"); features = builtins.concatLists [ @@ -1896,9 +1897,9 @@ in }; }); - "unknown".garage_model."0.9.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_model."0.9.1" = overridableMkRustCrate (profileName: rec { name = "garage_model"; - version = "0.9.0"; + version = "0.9.1"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/model"); features = builtins.concatLists [ @@ -1917,11 +1918,11 @@ in err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_block = (rustPackages."unknown".garage_block."0.9.0" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out; + garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.10.0" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; @@ -1934,9 +1935,9 @@ in }; }); - "unknown".garage_rpc."0.9.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_rpc."0.9.1" = overridableMkRustCrate (profileName: rec { name = "garage_rpc"; - version = "0.9.0"; + version = "0.9.1"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/rpc"); features = builtins.concatLists [ @@ -1958,8 +1959,8 @@ in format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" { inherit profileName; }).out; @@ -1982,9 +1983,9 @@ in }; }); - "unknown".garage_table."0.9.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_table."0.9.1" = overridableMkRustCrate (profileName: rec { name = "garage_table"; - version = "0.9.0"; + version = "0.9.1"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/table"); dependencies = { @@ -1993,9 +1994,9 @@ in bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out; - garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; + garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; @@ -2007,9 +2008,9 @@ in }; }); - "unknown".garage_util."0.9.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_util."0.9.1" = overridableMkRustCrate (profileName: rec { name = "garage_util"; - version = "0.9.0"; + version = "0.9.1"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/util"); features = builtins.concatLists [ @@ -2025,7 +2026,7 @@ in digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out; + garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; @@ -2051,18 +2052,18 @@ in }; }); - "unknown".garage_web."0.9.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_web."0.9.1" = overridableMkRustCrate (profileName: rec { name = "garage_web"; - version = "0.9.0"; + version = "0.9.1"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/web"); dependencies = { err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; - garage_api = (rustPackages."unknown".garage_api."0.9.0" { inherit profileName; }).out; - garage_model = (rustPackages."unknown".garage_model."0.9.0" { inherit profileName; }).out; - garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out; - garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out; + garage_api = (rustPackages."unknown".garage_api."0.9.1" { inherit profileName; }).out; + garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out; + garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out; + garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out; hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out; hyperlocal = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyperlocal."0.8.0" { inherit profileName; }).out; @@ -18,14 +18,14 @@ default-members = ["src/garage"] [workspace.dependencies] format_table = { version = "0.1.1", path = "src/format-table" } -garage_api = { version = "0.9.0", path = "src/api" } -garage_block = { version = "0.9.0", path = "src/block" } -garage_db = { version = "0.9.0", path = "src/db", default-features = false } -garage_model = { version = "0.9.0", path = "src/model", default-features = false } -garage_rpc = { version = "0.9.0", path = "src/rpc" } -garage_table = { version = "0.9.0", path = "src/table" } -garage_util = { version = "0.9.0", path = "src/util" } -garage_web = { version = "0.9.0", path = "src/web" } +garage_api = { version = "0.9.1", path = "src/api" } +garage_block = { version = "0.9.1", path = "src/block" } +garage_db = { version = "0.9.1", path = "src/db", default-features = false } +garage_model = { version = "0.9.1", path = "src/model", default-features = false } +garage_rpc = { version = "0.9.1", path = "src/rpc" } +garage_table = { version = "0.9.1", path = "src/table" } +garage_util = { version = "0.9.1", path = "src/util" } +garage_web = { version = "0.9.1", path = "src/web" } k2v-client = { version = "0.0.4", path = "src/k2v-client" } [profile.dev] diff --git a/doc/book/connect/apps/index.md b/doc/book/connect/apps/index.md index f67a29c9..c8571fac 100644 --- a/doc/book/connect/apps/index.md +++ b/doc/book/connect/apps/index.md @@ -146,7 +146,7 @@ Keep the Key ID and the Secret key in a pad, they will be needed later. We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist). ```bash -garage bucket create peertube-video +garage bucket create peertube-videos garage bucket create peertube-playlist ``` @@ -216,7 +216,7 @@ object_storage: # Same settings but for webtorrent videos videos: - bucket_name: 'peertube-video' + bucket_name: 'peertube-videos' prefix: '' # You must fill this field to make Peertube use our reverse proxy/website logic base_url: 'http://peertube-videos.web.garage.localhost' diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md index ea4ce1f9..ce0abddd 100644 --- a/doc/book/cookbook/real-world.md +++ b/doc/book/cookbook/real-world.md @@ -85,14 +85,14 @@ to store 2 TB of data in total. ## Get a Docker image Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). -We encourage you to use a fixed tag (eg. `v0.9.0`) and not the `latest` tag. -For this example, we will use the latest published version at the time of the writing which is `v0.9.0` but it's up to you +We encourage you to use a fixed tag (eg. `v0.9.1`) and not the `latest` tag. +For this example, we will use the latest published version at the time of the writing which is `v0.9.1` but it's up to you to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). For example: ``` -sudo docker pull dxflrs/garage:v0.9.0 +sudo docker pull dxflrs/garage:v0.9.1 ``` ## Deploying and configuring Garage @@ -157,7 +157,7 @@ docker run \ -v /etc/garage.toml:/etc/garage.toml \ -v /var/lib/garage/meta:/var/lib/garage/meta \ -v /var/lib/garage/data:/var/lib/garage/data \ - dxflrs/garage:v0.9.0 + dxflrs/garage:v0.9.1 ``` With this command line, Garage should be started automatically at each boot. @@ -171,7 +171,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y version: "3" services: garage: - image: dxflrs/garage:v0.9.0 + image: dxflrs/garage:v0.9.1 network_mode: "host" restart: unless-stopped volumes: diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index 18d160bb..5e12a7da 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -394,7 +394,7 @@ Compression is done synchronously, setting a value too high will add latency to This value can be different between nodes, compression is done by the node which receive the API call. -#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET` (env) {#rpc_secret} +#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET`, `GARAGE_RPC_SECRET_FILE` (env) {#rpc_secret} Garage uses a secret key, called an RPC secret, that is shared between all nodes of the cluster in order to identify these nodes and allow them to @@ -406,6 +406,9 @@ Since Garage `v0.8.2`, the RPC secret can also be stored in a file whose path is given in the configuration variable `rpc_secret_file`, or specified as an environment variable `GARAGE_RPC_SECRET`. +Since Garage `v0.8.5` and `v0.9.1`, you can also specify the path of a file +storing the secret as the `GARAGE_RPC_SECRET_FILE` environment variable. + #### `rpc_bind_addr` {#rpc_bind_addr} The address and port on which to bind for inter-cluster communcations @@ -438,6 +441,17 @@ be obtained by running `garage node id` and then included directly in the key will be returned by `garage node id` and you will have to add the IP yourself. +### `allow_world_readable_secrets` + +Garage checks the permissions of your secret files to make sure they're not +world-readable. In some cases, the check might fail and consider your files as +world-readable even if they're not, for instance when using Posix ACLs. + +Setting `allow_world_readable_secrets` to `true` bypass this +permission verification. + +Alternatively, you can set the `GARAGE_ALLOW_WORLD_READABLE_SECRETS` +environment variable to `true` to bypass the permissions check. ### The `[consul_discovery]` section @@ -583,7 +597,7 @@ See [administration API reference](@/documentation/reference-manual/admin-api.md Alternatively, since `v0.8.5`, a path can be used to create a unix socket. Note that for security reasons, the socket will have 0220 mode. Make sure to set user and group permissions accordingly. -#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN` (env) {#admin_metrics_token} +#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN`, `GARAGE_METRICS_TOKEN_FILE` (env) {#admin_metrics_token} The token for accessing the Metrics endpoint. If this token is not set, the Metrics endpoint can be accessed without access control. @@ -593,8 +607,9 @@ You can use any random string for this value. We recommend generating a random t `metrics_token` was introduced in Garage `v0.7.2`. `metrics_token_file` and the `GARAGE_METRICS_TOKEN` environment variable are supported since Garage `v0.8.2`. +`GARAGE_METRICS_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`. -#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN` (env) {#admin_token} +#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env) {#admin_token} The token for accessing all of the other administration endpoints. If this token is not set, access to these endpoints is disabled entirely. @@ -604,6 +619,7 @@ You can use any random string for this value. We recommend generating a random t `admin_token` was introduced in Garage `v0.7.2`. `admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`. +`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`. #### `trace_sink` {#admin_trace_sink} diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml index 346e68ad..31b75c1f 100644 --- a/script/helm/garage/Chart.yaml +++ b/script/helm/garage/Chart.yaml @@ -21,4 +21,4 @@ version: 0.4.1 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v0.9.0" +appVersion: "v0.9.1" diff --git a/script/jepsen.garage/.envrc b/script/jepsen.garage/.envrc new file mode 100644 index 00000000..1d953f4b --- /dev/null +++ b/script/jepsen.garage/.envrc @@ -0,0 +1 @@ +use nix diff --git a/script/jepsen.garage/.gitignore b/script/jepsen.garage/.gitignore new file mode 100644 index 00000000..31842a96 --- /dev/null +++ b/script/jepsen.garage/.gitignore @@ -0,0 +1,17 @@ +/target +/classes +/checkouts +profiles.clj +pom.xml +pom.xml.asc +*.jar +*.class +/.lein-* +/.nrepl-port +/.prepl-port +.hgignore +.hg/ +.direnv +/store +/store.* +.vagrant diff --git a/script/jepsen.garage/README.md b/script/jepsen.garage/README.md new file mode 100644 index 00000000..50c7eb38 --- /dev/null +++ b/script/jepsen.garage/README.md @@ -0,0 +1,166 @@ +# jepsen.garage + +Jepsen checking of Garage consistency properties. + +## Usage + +Requirements: + +- vagrant +- VirtualBox, configured so that nodes can take an IP in a private network `192.168.56.0/24` (it's the default) +- a user that can create VirtualBox VMs +- leiningen +- gnuplot + +Set up VMs before running tests: + +``` +vagrant up +``` + +Run tests: see commands below. + + +## Results + +### Register linear, without timestamp patch + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 20 --workload reg1 --ops-per-key 100` + +Results without timestamp patch: + +- Fails with a simple clock-scramble nemesis (`--scenario c`). + Explanation: without the timestamp patch, nodes will create objects using their + local clock only as a timestamp, so the ordering will be all over the place if + clocks are scrambled. + +Results with timestamp patch (`--patch tsfix2`): + +- No failure with clock-scramble nemesis + +- Fails with clock-scramble nemesis + partition nemesis (`--scenario cp`). + +**This test is expected to fail.** +Indeed, S3 objects are not meant to behave like linearizable registers. +TODO explain using a counter-example + + +### Read-after-write CRDT register model + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 100 --workload reg2 --ops-per-key 100` + +Results without timestamp patch: + +- Fails with a simple clock-scramble nemesis (`--scenario c`). + Explanation: old values are not overwritten correctly when their timestamps are in the future. + +Results with timestamp patch (`--patch tsfix2`): + +- No failures with clock-scramble nemesis + partition nemesis (`--scenario cp`). + This proves that `tsfix2` (PR#543) does improve consistency. + +- **Fails with layout reconfiguration nemesis** (`--scenario r`). + Example of a failed run: `garage reg2/20231024T120806.899+0200`. + This is the failure mode we are looking for and trying to fix for NLnet task 3. + +Results with NLnet task 3 code (commit 707442f5de, `--patch task3a`): + +- No failures with `--scenario r` (0 of 10 runs), `--scenario pr` (0 of 10 runs), + `--scenario cpr` (0 of 10 runs) and `--scenario dpr` (0 of 10 runs). + +- Same with `--patch task3c` (commit `0041b013`, the final version). + + +### Set, basic test (write some items, then read) + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 200 --concurrency 200 --workload set1 --ops-per-key 100` + +Results without NLnet task3 code (`--patch tsfix2`): + +- For now, no failures with clock-scramble nemesis + partition nemesis -> TODO long test run + +- Does not seem to fail with only the layout reconfiguation nemesis (<10 runs), although theoretically it could + +- **Fails with the partition + layout reconfiguration nemesis** (`--scenario pr`). + Example of a failed run: `garage set1/20231024T172214.488+0200` (1 failure in 4 runs). + This is the failure mode we are looking for and trying to fix for NLnet task 3. + +Results with NLnet task 3 code (commit 707442f5de, `--patch task3a`): + +- The tests are buggy and often result in an "unknown" validity status, which + is caused by some requests not returning results during network partitions or + other nemesis-induced broken cluster states. However, when the tests were + able to finish, there were no failures with scenarios `r`, `pr`, `cpr`, + `dpr`. + + +### Set, continuous test (interspersed reads and writes) + +Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 100 --workload set2 --ops-per-key 100` + +Results without NLnet task3 code (`--patch tsfix2`): + +- No failures with clock-scramble nemesis + db nemesis + partition nemesis (`--scenario cdp`) (0 failures in 10 runs). + +- **Fails with just layout reconfiguration nemesis** (`--scenario r`). + Example of a failed run: `garage set2/20231025T141940.198+0200` (10 failures in 10 runs). + This is the failure mode we are looking for and trying to fix for NLnet task 3. + +Results with NLnet task3 code (commit 707442f5de, `--patch task3a`): + +- No failures with `--scenario r` (0 of 10 runs), `--scenario pr` (0 of 10 runs), + `--scenario cpr` (0 of 10 runs) and `--scenario dpr` (0 of 10 runs). + +- Same with `--patch task3c` (commit `0041b013`, the final version). + + +## NLnet task 3 final results + +- With code from task3 (`--patch task3c`): [reg2 and set2](results/Results-2023-12-13-task3c.png), [set1](results/Results-2023-12-14-task3-set1.png). +- Without (`--patch tsfix2`): [reg2 and set2](results/Results-2023-12-13-tsfix2.png), set1 TBD. + +## Investigating (and fixing) errors + +### Segfaults + +They are due to the download being interrupted in the middle (^C during first launch on clean VMs), the `garage` binary is truncated. +Add `:force?` to the `cached-wget!` call in `daemon.clj` to re-download the binary, +or restar the VMs to clear temporary files. + +### In `jepsen.garage`: prefix wierdness + +In `store/garage set1/20231019T163358.615+0200`: + +``` +INFO [2023-10-19 16:35:20,977] clojure-agent-send-off-pool-207 - jepsen.garage.set list results for prefix set20/ : (set13/0 set13/1 set13/10 set13/11 set13/12 set13/13 set13/14 set13/15 set13/16 set13/17 set13/18 set13/19 set13/2 set13/20 set13/21 set13/22 set13/23 set13/24 set13/25 set13/26 set13/27 set13/28 set13/29 set13/3 set13/30 set13/31 set13/32 set13/33 set13/34 set13/35 set13/36 set13/37 set13/38 set13/39 set13/4 set13/40 set13/41 set13/42 set13/43 set13/44 set13/45 set13/46 set13/47 set13/48 set13/49 set13/5 set13/50 set13/51 set13/52 set13/53 set13/54 set13/55 set13/56 set13/57 set13/58 set13/59 set13/6 set13/60 set13/61 set13/62 set13/63 set13/64 set13/65 set13/66 set13/67 set13/68 set13/69 set13/7 set13/70 set13/71 set13/72 set13/73 set13/74 set13/75 set13/76 set13/77 set13/78 set13/79 set13/8 set13/80 set13/81 set13/82 set13/83 set13/84 set13/85 set13/86 set13/87 set13/88 set13/89 set13/9 set13/90 set13/91 set13/92 set13/93 set13/94 set13/95 set13/96 set13/97 set13/98 set13/99) (node: http://192.168.56.25:3900 ) +``` + +After inspecting, the actual S3 call made was with prefix "set13/", so at least this is not an error in Garage itself but in the jepsen code. + +Finally found out that this was due to closures not correctly capturing their context in the list function in s3api.clj (wtf clojure?) +Not sure exactly where it came from but it seems to have been fixed by making list-inner a separate function and not a sub-function, +and passing all values that were previously in the context (creds and prefix) as additional arguments. + +### `reg2` test inconsistency, even with timestamp fix + +The reg2 test is our custom checker for CRDT read-after-write on individual object keys, acting as registers which can be updated. +The test fails without the timestamp fix, which is expected as the clock scrambler will prevent nodes from having a correct ordering of objects. + +With the timestamp fix (`--patch tsfix1`), the happenned-before relationship should at least be respected, meaning that when a PutObject call starts +after another PutObject call has ended, the second call should overwrite the value of the first call, and that value should not be +readable by future GetObject calls. +However, we observed inconsistencies even with the timestamp fix. + +The inconsistencies seemed to always happenned after writing a nil value, which translates to a DeleteObject call +instead of a PutObject. By removing the possibility of writing nil values, therefore only doing +PutObject calls, the issue disappears. There is therefore an issue to fix in DeleteObject. + +The issue in DeleteObject seems to have been fixed by commit `c82d91c6bccf307186332b6c5c6fc0b128b1b2b1`, which can be used using `--patch tsfix2`. + + +## License + +Copyright © 2023 Alex Auvolat + +This program and the accompanying materials are made available under the +terms of the GNU Affero General Public License v3.0. diff --git a/script/jepsen.garage/Vagrantfile b/script/jepsen.garage/Vagrantfile new file mode 100644 index 00000000..b54c2426 --- /dev/null +++ b/script/jepsen.garage/Vagrantfile @@ -0,0 +1,40 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# + +def vm(config, hostname, ip) + config.vm.hostname = hostname + config.vm.network "private_network", ip: ip +end + +Vagrant.configure("2") do |config| + config.vm.box = "generic/debian10" + + config.vm.provider "virtualbox" do |vb| + vb.gui = false + vb.memory = "512" + vb.customize ["modifyvm", :id, "--vram=12"] + end + + config.vm.provision "shell", inline: <<-SHELL + echo "root:root" | chpasswd + mkdir -p /root/.ssh + echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy" >> /root/.ssh/authorized_keys + SHELL + + config.vm.define "n1" do |config| vm(config, "n1", "192.168.56.21") end + config.vm.define "n2" do |config| vm(config, "n2", "192.168.56.22") end + config.vm.define "n3" do |config| vm(config, "n3", "192.168.56.23") end + config.vm.define "n4" do |config| vm(config, "n4", "192.168.56.24") end + config.vm.define "n5" do |config| vm(config, "n5", "192.168.56.25") end + config.vm.define "n6" do |config| vm(config, "n6", "192.168.56.26") end + config.vm.define "n7" do |config| vm(config, "n7", "192.168.56.27") end + + config.vm.define "n8" do |config| vm(config, "n8", "192.168.56.28") end + config.vm.define "n9" do |config| vm(config, "n9", "192.168.56.29") end + config.vm.define "n10" do |config| vm(config, "n10", "192.168.56.30") end + config.vm.define "n11" do |config| vm(config, "n11", "192.168.56.31") end + config.vm.define "n12" do |config| vm(config, "n12", "192.168.56.32") end + config.vm.define "n13" do |config| vm(config, "n13", "192.168.56.33") end + config.vm.define "n14" do |config| vm(config, "n14", "192.168.56.34") end +end diff --git a/script/jepsen.garage/all_tests_1.sh b/script/jepsen.garage/all_tests_1.sh new file mode 100755 index 00000000..b5397d13 --- /dev/null +++ b/script/jepsen.garage/all_tests_1.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -x + +#for ppatch in task3c task3a tsfix2; do +for ppatch in tsfix2; do + #for psc in c cp cdp r pr cpr dpr; do + for psc in cdp r pr cpr dpr; do + #for ptsk in reg2 set1 set2; do + for ptsk in set1; do + for irun in $(seq 10); do + lein run test --nodes-file nodes.vagrant \ + --time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \ + --workload $ptsk --patch $ppatch --scenario $psc + done + done + done +done diff --git a/script/jepsen.garage/all_tests_2.sh b/script/jepsen.garage/all_tests_2.sh new file mode 100755 index 00000000..641643ed --- /dev/null +++ b/script/jepsen.garage/all_tests_2.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -x + +#for ppatch in task3c tsfix2; do +for ppatch in tsfix2; do + for psc in cdp r pr cpr dpr; do + for ptsk in set1; do + for irun in $(seq 10); do + lein run test --nodes-file nodes2.vagrant \ + --time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \ + --workload $ptsk --patch $ppatch --scenario $psc + done + done + done +done diff --git a/script/jepsen.garage/jaeger.sh b/script/jepsen.garage/jaeger.sh new file mode 100644 index 00000000..7f67b61b --- /dev/null +++ b/script/jepsen.garage/jaeger.sh @@ -0,0 +1,13 @@ +docker stop jaeger +docker rm jaeger + +# UI is on localhost:16686 +# otel-grpc collector is on localhost:4317 +# otel-http collector is on localhost:4318 + +docker run -d --name jaeger \ + -e COLLECTOR_OTLP_ENABLED=true \ + -p 4317:4317 \ + -p 4318:4318 \ + -p 16686:16686 \ + jaegertracing/all-in-one:1.50 diff --git a/script/jepsen.garage/nodes.vagrant b/script/jepsen.garage/nodes.vagrant new file mode 100644 index 00000000..9e5694e6 --- /dev/null +++ b/script/jepsen.garage/nodes.vagrant @@ -0,0 +1,7 @@ +192.168.56.21 +192.168.56.22 +192.168.56.23 +192.168.56.24 +192.168.56.25 +192.168.56.26 +192.168.56.27 diff --git a/script/jepsen.garage/nodes2.vagrant b/script/jepsen.garage/nodes2.vagrant new file mode 100644 index 00000000..842bf276 --- /dev/null +++ b/script/jepsen.garage/nodes2.vagrant @@ -0,0 +1,7 @@ +192.168.56.28 +192.168.56.29 +192.168.56.30 +192.168.56.31 +192.168.56.32 +192.168.56.33 +192.168.56.34 diff --git a/script/jepsen.garage/project.clj b/script/jepsen.garage/project.clj new file mode 100644 index 00000000..59d45484 --- /dev/null +++ b/script/jepsen.garage/project.clj @@ -0,0 +1,10 @@ +(defproject jepsen.garage "0.1.0-SNAPSHOT" + :description "Jepsen testing for Garage" + :url "https://git.deuxfleurs.fr/Deuxfleurs/garage" + :license {:name "AGPLv3" + :url "https://www.gnu.org/licenses/agpl-3.0.en.html"} + :main jepsen.garage + :dependencies [[org.clojure/clojure "1.11.1"] + [jepsen "0.3.3-SNAPSHOT"] + [amazonica "0.3.163"]] + :repl-options {:init-ns jepsen.garage}) diff --git a/script/jepsen.garage/results/Results-2023-11-16.png b/script/jepsen.garage/results/Results-2023-11-16.png Binary files differnew file mode 100644 index 00000000..26dac833 --- /dev/null +++ b/script/jepsen.garage/results/Results-2023-11-16.png diff --git a/script/jepsen.garage/results/Results-2023-12-13-task3c.png b/script/jepsen.garage/results/Results-2023-12-13-task3c.png Binary files differnew file mode 100644 index 00000000..216043c3 --- /dev/null +++ b/script/jepsen.garage/results/Results-2023-12-13-task3c.png diff --git a/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png b/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png Binary files differnew file mode 100644 index 00000000..147d25e9 --- /dev/null +++ b/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png diff --git a/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png b/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png Binary files differnew file mode 100644 index 00000000..dbff3a95 --- /dev/null +++ b/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png diff --git a/script/jepsen.garage/shell.nix b/script/jepsen.garage/shell.nix new file mode 100644 index 00000000..01e4c845 --- /dev/null +++ b/script/jepsen.garage/shell.nix @@ -0,0 +1,18 @@ +{ pkgs ? import <nixpkgs> { + overlays = [ + (self: super: { + jdk = super.jdk11; + jre = super.jre11; + }) + ]; +} }: +pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + leiningen + jdk + jna + vagrant + gnuplot + graphviz + ]; +} diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj new file mode 100644 index 00000000..446b81de --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage.clj @@ -0,0 +1,105 @@ +(ns jepsen.garage + (:require + [clojure.string :as str] + [jepsen + [checker :as checker] + [cli :as cli] + [generator :as gen] + [nemesis :as nemesis] + [tests :as tests]] + [jepsen.os.debian :as debian] + [jepsen.garage + [daemon :as grg] + [nemesis :as grgNemesis] + [reg :as reg] + [set :as set]])) + +(def workloads + "A map of workload names to functions that construct workloads, given opts." + {"reg1" reg/workload1 + "reg2" reg/workload2 + "set1" set/workload1 + "set2" set/workload2}) + +(def scenari + "A map of scenari to the associated nemesis" + {"c" grgNemesis/scenario-c + "cp" grgNemesis/scenario-cp + "r" grgNemesis/scenario-r + "pr" grgNemesis/scenario-pr + "cpr" grgNemesis/scenario-cpr + "cdp" grgNemesis/scenario-cdp + "dpr" grgNemesis/scenario-dpr}) + +(def patches + "A map of patch names to Garage builds" + {"default" "v0.9.0" + "tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09" + "tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1" + "task3a" "707442f5de416fdbed4681a33b739f0a787b7834" + "task3b" "431b28e0cfdc9cac6c649193cf602108a8b02997" + "task3c" "0041b013a473e3ae72f50209d8f79db75a72848b"}) + +(def cli-opts + "Additional command line options." + [["-p" "--patch NAME" "Garage patch to use" + :default "default" + :validate [patches (cli/one-of patches)]] + ["-s" "--scenario NAME" "Nemesis scenario to run" + :default "cp" + :validate [scenari (cli/one-of scenari)]] + ["-r" "--rate HZ" "Approximate number of requests per second, per thread." + :default 10 + :parse-fn read-string + :validate [#(and (number? %) (pos? %)) "Must be a positive number"]] + [nil "--ops-per-key NUM" "Maximum number of operations on any given key." + :default 100 + :parse-fn parse-long + :validate [pos? "Must be a positive integer."]] + ["-w" "--workload NAME" "Workload of test to run" + :default "reg1" + :validate [workloads (cli/one-of workloads)]]]) + +(defn garage-test + "Given an options map from the command line runner (e.g. :nodes, :ssh, + :concurrency, ...), constructs a test map." + [opts] + (let [garage-version (get patches (:patch opts)) + db (grg/db garage-version) + workload ((get workloads (:workload opts)) opts) + scenario ((get scenari (:scenario opts)) (assoc opts :db db))] + (merge tests/noop-test + opts + {:pure-generators true + :name (str "garage-" (name (:patch opts)) " " (name (:workload opts)) " " (name (:scenario opts))) + :os debian/os + :db db + :client (:client workload) + :generator (gen/phases + (->> + (:generator workload) + (gen/stagger (/ (:rate opts))) + (gen/nemesis (:generator scenario)) + (gen/time-limit (:time-limit opts))) + (gen/log "Healing cluster") + (gen/nemesis (:final-generator scenario)) + (gen/log "Waiting for recovery") + (gen/sleep 10) + (gen/log "Running final generator") + (gen/clients (:final-generator workload)) + (gen/log "Generators all done")) + :nemesis (:nemesis scenario) + :checker (checker/compose + {:perf (checker/perf (:perf scenario)) + :workload (:checker workload)}) + }))) + + +(defn -main + "Handles command line arguments. Can either run a test, or a web server for + browsing results." + [& args] + (cli/run! (merge (cli/single-test-cmd {:test-fn garage-test + :opt-spec cli-opts}) + (cli/serve-cmd)) + args)) diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj new file mode 100644 index 00000000..d407dd29 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj @@ -0,0 +1,152 @@ +(ns jepsen.garage.daemon + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c] + [core :as jepsen] + [db :as db]] + [jepsen.control.util :as cu])) + +; CONSTANTS -- HOW GARAGE IS SET UP + +(def base-dir "/opt/garage") +(def data-dir (str base-dir "/data")) +(def meta-dir (str base-dir "/meta")) +(def binary (str base-dir "/garage")) +(def logfile (str base-dir "/garage.log")) +(def pidfile (str base-dir "/garage.pid")) + +(def admin-token "icanhazadmin") +(def access-key-id "GK8bfb6a51286071c6c9cd8bc3") +(def secret-access-key "b0be95f71c1c6f16858a9edf395078b75c12ecb6b1c03385c4ae92076e4994a3") +(def bucket-name "jepsen") + +; THE GARAGE DB + +(defn install! + "Download and install Garage" + [node version] + (c/su + (c/trace + (info node "installing garage" version) + (c/exec :mkdir :-p base-dir) + (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") + cache (cu/cached-wget! url)] + (c/exec :cp cache binary)) + (c/exec :chmod :+x binary)))) + +(defn configure! + "Configure Garage" + [node] + (c/su + (c/trace + (cu/write-file! + (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" + "rpc_bind_addr = \"0.0.0.0:3901\"\n" + "rpc_public_addr = \"" node ":3901\"\n" + "db_engine = \"lmdb\"\n" + "replication_mode = \"2\"\n" + "data_dir = \"" data-dir "\"\n" + "metadata_dir = \"" meta-dir "\"\n" + "[s3_api]\n" + "s3_region = \"us-east-1\"\n" + "api_bind_addr = \"0.0.0.0:3900\"\n" + "[k2v_api]\n" + "api_bind_addr = \"0.0.0.0:3902\"\n" + "[admin]\n" + "api_bind_addr = \"0.0.0.0:3903\"\n" + "admin_token = \"" admin-token "\"\n" + "trace_sink = \"http://192.168.56.1:4317\"\n") + "/etc/garage.toml")))) + +(defn connect-node! + "Connect a Garage node to the rest of the cluster" + [test node] + (c/trace + (let [node-id (c/exec binary :node :id :-q)] + (info node "node id:" node-id) + (c/on-many (:nodes test) + (c/exec binary :node :connect node-id))))) + +(defn configure-node! + "Configure a Garage node to be part of a cluster layout" + [test node] + (c/trace + (let [node-id (c/exec binary :node :id :-q)] + (c/on (jepsen/primary test) + (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))))) + +(defn finalize-config! + "Apply the layout and create a key/bucket pair in the cluster" + [node] + (c/trace + (c/exec binary :layout :apply :--version 1) + (info node "garage status:" (c/exec binary :status)) + (c/exec binary :key :import access-key-id secret-access-key :--yes) + (c/exec binary :bucket :create bucket-name) + (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key-id) + (info node "key info: " (c/exec binary :key :info access-key-id)))) + +(defn db + "Garage DB for a particular version" + [version] + (reify db/DB + (setup! [_ test node] + (install! node version) + (configure! node) + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir base-dir + :env {:RUST_LOG "garage=debug,garage_api=trace"}} + binary + :server) + (c/exec :sleep 3) + + (jepsen/synchronize test) + (connect-node! test node) + + (jepsen/synchronize test) + (configure-node! test node) + + (jepsen/synchronize test) + (when (= node (jepsen/primary test)) + (finalize-config! node))) + + (teardown! [_ test node] + (info node "tearing down garage" version) + (c/su + (cu/stop-daemon! binary pidfile) + (c/exec :rm :-rf logfile) + (c/exec :rm :-rf data-dir) + (c/exec :rm :-rf meta-dir))) + + db/Pause + (pause! [_ test node] + (cu/grepkill! :stop binary)) + (resume! [_ test node] + (cu/grepkill! :cont binary)) + + db/Kill + (kill! [_ test node] + (cu/stop-daemon! binary pidfile)) + (start! [_ test node] + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir base-dir + :env {:RUST_LOG "garage=debug,garage_api=trace"}} + binary + :server)) + + db/LogFiles + (log-files [_ test node] + [logfile]))) + +(defn creds + "Obtain Garage credentials for node" + [node] + {:access-key access-key-id + :secret-key secret-access-key + :endpoint (str "http://" node ":3900") + :bucket bucket-name + :client-config {:path-style-access-enabled true}}) + diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj new file mode 100644 index 00000000..dfce0255 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -0,0 +1,142 @@ +(ns jepsen.garage.nemesis + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c] + [core :as jepsen] + [generator :as gen] + [nemesis :as nemesis]] + [jepsen.nemesis.combined :as combined] + [jepsen.garage.daemon :as grg] + [jepsen.control.util :as cu])) + +; ---- reconfiguration nemesis ---- + +(defn configure-present! + "Configure node to be active in new cluster layout" + [test nodes] + (info "configure-present!" nodes) + (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q)) + node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)] + (c/on + (jepsen/primary test) + (apply c/exec (concat [grg/binary :layout :assign :-c :1G] node-id-strs))))) + +(defn configure-absent! + "Configure nodes to be active in new cluster layout" + [test nodes] + (info "configure-absent!" nodes) + (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q)) + node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)] + (c/on + (jepsen/primary test) + (apply c/exec (concat [grg/binary :layout :assign :-g] node-id-strs))))) + +(defn finalize-config! + "Apply the proposed cluster layout" + [test] + (let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show)) + [_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)] + (if layout-next-version + (do + (info "layout show: " layout-show "; next-version: " layout-next-version) + (c/on (jepsen/primary test) + (c/exec grg/binary :layout :apply :--version layout-next-version))) + (info "no layout changes to apply")))) + +(defn reconfigure-subset + "Reconfigure cluster with only a subset of nodes" + [cnt] + (reify nemesis/Nemesis + (setup! [this test] this) + + (invoke! [this test op] op + (case (:f op) + :start + (let [[keep-nodes remove-nodes] + (->> (:nodes test) + shuffle + (split-at cnt))] + (info "layout split: keep " keep-nodes ", remove " remove-nodes) + (configure-present! test keep-nodes) + (configure-absent! test remove-nodes) + (finalize-config! test) + (assoc op :value keep-nodes)) + :stop + (do + (info "layout un-split: all nodes=" (:nodes test)) + (configure-present! test (:nodes test)) + (finalize-config! test) + (assoc op :value (:nodes test))))) + + (teardown! [this test] this))) + +; ---- nemesis scenari ---- + +(defn nemesis-op + "A generator for a single nemesis operation" + [op] + (fn [_ _] {:type :info, :f op})) + +(defn reconfiguration-package + "Cluster reconfiguration nemesis package" + [opts] + {:generator (->> + (gen/mix [(nemesis-op :reconfigure-start) + (nemesis-op :reconfigure-stop)]) + (gen/stagger (:interval opts 5))) + :final-generator {:type :info, :f :reconfigure-stop} + :nemesis (nemesis/compose + {{:reconfigure-start :start + :reconfigure-stop :stop} (reconfigure-subset 3)}) + :perf #{{:name "reconfigure" + :start #{:reconfigure-start} + :stop #{:reconfigur-stop} + :color "#A197E9"}}}) + +(defn scenario-c + "Clock modifying scenario" + [opts] + (combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})) + +(defn scenario-cp + "Clock modifying + partition scenario" + [opts] + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})])) + +(defn scenario-r + "Cluster reconfiguration scenario" + [opts] + (reconfiguration-package {:interval 1})) + +(defn scenario-pr + "Partition + cluster reconfiguration scenario" + [opts] + (combined/compose-packages + [(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) + +(defn scenario-cpr + "Clock scramble + partition + cluster reconfiguration scenario" + [opts] + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) + +(defn scenario-cdp + "Clock modifying + db + partition scenario" + [opts] + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})])) + +(defn scenario-dpr + "Db + partition + cluster reconfiguration scenario" + [opts] + (combined/compose-packages + [(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) + diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj new file mode 100644 index 00000000..39708c0b --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -0,0 +1,143 @@ +(ns jepsen.garage.reg + (:require [clojure.tools.logging :refer :all] + [clojure.string :as str] + [clojure.set :as set] + [jepsen [checker :as checker] + [cli :as cli] + [client :as client] + [control :as c] + [db :as db] + [generator :as gen] + [independent :as independent] + [nemesis :as nemesis] + [util :as util] + [tests :as tests]] + [jepsen.checker.timeline :as timeline] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [jepsen.garage.daemon :as grg] + [jepsen.garage.s3api :as s3] + [knossos.model :as model] + [slingshot.slingshot :refer [try+]])) + +(defn op-get [_ _] {:type :invoke, :f :read, :value nil}) +(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 99))}) +(defn op-del [_ _] {:type :invoke, :f :write, :value nil}) + +(defrecord RegClient [creds] + client/Client + (open! [this test node] + (assoc this :creds (grg/creds node))) + (setup! [this test]) + (invoke! [this test op] + (try+ + (let [[k v] (:value op)] + (case (:f op) + :read + (util/timeout + 10000 + (assoc op :type :fail, :error ::timeout) + (let [value (s3/get (:creds this) k)] + (assoc op :type :ok, :value (independent/tuple k value)))) + :write + (util/timeout + 10000 + (assoc op :type :info, :error ::timeout) + (do + (s3/put (:creds this) k v) + (assoc op :type :ok))))) + (catch (re-find #"Unavailable" (.getMessage %)) ex + (assoc op :type :info, :error ::unavailable)) + (catch (re-find #"Broken pipe" (.getMessage %)) ex + (assoc op :type :info, :error ::broken-pipe)) + (catch (re-find #"Connection refused" (.getMessage %)) ex + (assoc op :type :info, :error ::connection-refused)))) + (teardown! [this test]) + (close! [this test])) + +(defn reg-read-after-write + "Read-after-Write checker for register operations" + [] + (reify checker/Checker + (check [this test history opts] + (let [init {:put-values {-1 nil} + :put-done #{-1} + :put-in-progress {} + :read-can-contain {} + :bad-reads #{}} + final (reduce + (fn [state op] + (let [current-values (set/union + (set (map (fn [idx] (get (:put-values state) idx)) (:put-done state))) + (set (map (fn [[_ [idx _]]] (get (:put-values state) idx)) (:put-in-progress state)))) + read-can-contain (reduce + (fn [rcc [idx v]] (assoc rcc idx (set/union current-values v))) + {} (:read-can-contain state))] + (info "--------") + (info "state: " state) + (info "current-values: " current-values) + (info "read-can-contain: " read-can-contain) + (info "op: " op) + (case [(:type op) (:f op)] + ([:invoke :write]) + (assoc state + :read-can-contain read-can-contain + :put-values (assoc (:put-values state) (:index op) (:value op)) + :put-in-progress (assoc (:put-in-progress state) (:process op) [(:index op) (:put-done state)])) + ([:ok :write]) + (let [[index overwrites] (get (:put-in-progress state) (:process op))] + (assoc state + :read-can-contain read-can-contain + :put-in-progress (dissoc (:put-in-progress state) (:process op)) + :put-done + (conj + (set/difference (:put-done state) overwrites) + index))) + ([:invoke :read]) + (assoc state + :read-can-contain (assoc read-can-contain (:process op) current-values)) + ([:ok :read]) + (let [this-read-can-contain (get read-can-contain (:process op)) + bad-reads (if (contains? this-read-can-contain (:value op)) + (:bad-reads state) + (conj (:bad-reads state) [(:process op) (:index op) (:value op) this-read-can-contain]))] + (info "this-read-can-contain: " this-read-can-contain) + (assoc state + :read-can-contain (dissoc read-can-contain (:process op)) + :bad-reads bad-reads)) + state))) + init history) + valid? (empty? (:bad-reads final))] + (assoc final :valid? valid?))))) + +(defn workload-common + "Common parts of workload" + [opts] + {:client (RegClient. nil) + :generator (independent/concurrent-generator + 10 + (range) + (fn [k] + (->> + (gen/mix [op-get op-put op-del]) + (gen/limit (:ops-per-key opts)))))}) + +(defn workload1 + "Tests linearizable reads and writes" + [opts] + (assoc (workload-common opts) + :checker (independent/checker + (checker/compose + {:linear (checker/linearizable + {:model (model/register) + :algorithm :linear}) + :timeline (timeline/html)})))) + +(defn workload2 + "Tests CRDT reads and writes" + [opts] + (assoc (workload-common opts) + :checker (independent/checker + (checker/compose + {:reg-read-after-write (reg-read-after-write) + :timeline (timeline/html)})))) diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj new file mode 100644 index 00000000..531e0157 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj @@ -0,0 +1,48 @@ +(ns jepsen.garage.s3api + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c]] + [amazonica.aws.s3 :as s3] + [slingshot.slingshot :refer [try+]])) + +; GARAGE S3 HELPER FUNCTIONS + +(defn get + "Helper for GetObject" + [creds k] + (try+ + (-> (s3/get-object creds (:bucket creds) k) + :input-stream + slurp) + (catch (re-find #"Key not found" (.getMessage %)) ex + nil))) + +(defn put + "Helper for PutObject or DeleteObject (is a delete if value is nil)" + [creds k v] + (if (= v nil) + (s3/delete-object creds + :bucket-name (:bucket creds) + :key k) + (let [some-bytes (.getBytes v "UTF-8") + bytes-stream (java.io.ByteArrayInputStream. some-bytes)] + (s3/put-object creds + :bucket-name (:bucket creds) + :key k + :input-stream bytes-stream + :metadata {:content-length (count some-bytes)})))) + +(defn list-inner [creds prefix ct accum] + (let [list-result (s3/list-objects-v2 creds + {:bucket-name (:bucket creds) + :prefix prefix + :continuation-token ct}) + new-object-summaries (:object-summaries list-result) + new-objects (map (fn [d] (:key d)) new-object-summaries) + objects (concat new-objects accum)] + (if (:truncated? list-result) + (list-inner creds prefix (:next-continuation-token list-result) objects) + objects))) +(defn list + "Helper for ListObjects -- just lists everything in the bucket" + [creds prefix] + (list-inner creds prefix nil [])) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj new file mode 100644 index 00000000..2c7a2ccd --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -0,0 +1,135 @@ +(ns jepsen.garage.set + (:require [clojure.tools.logging :refer :all] + [clojure.string :as str] + [clojure.set :as set] + [jepsen [checker :as checker] + [cli :as cli] + [client :as client] + [control :as c] + [checker :as checker] + [db :as db] + [generator :as gen] + [independent :as independent] + [nemesis :as nemesis] + [util :as util] + [tests :as tests]] + [jepsen.checker.timeline :as timeline] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [jepsen.garage.daemon :as grg] + [jepsen.garage.s3api :as s3] + [knossos.model :as model] + [slingshot.slingshot :refer [try+]])) + +(defn op-add-rand100 [_ _] {:type :invoke, :f :add, :value (rand-int 100)}) +(defn op-read [_ _] {:type :invoke, :f :read, :value nil}) + +(defrecord SetClient [creds] + client/Client + (open! [this test node] + (assoc this :creds (grg/creds node))) + (setup! [this test]) + (invoke! [this test op] + (try+ + (let [[k v] (:value op) + prefix (str "set" k "/")] + (case (:f op) + :add + (util/timeout + 10000 + (assoc op :type :info, :error ::timeout) + (do + (s3/put (:creds this) (str prefix v) "present") + (assoc op :type :ok))) + :read + (util/timeout + 10000 + (assoc op :type :fail, :error ::timeout) + (do + (let [items (s3/list (:creds this) prefix)] + (let [items-stripped (map (fn [o] + (assert (str/starts-with? o prefix)) + (str/replace-first o prefix "")) items) + items-set (set (map parse-long items-stripped))] + (assoc op :type :ok, :value (independent/tuple k items-set)))))))) + (catch (re-find #"Unavailable" (.getMessage %)) ex + (assoc op :type :info, :error ::unavailable)) + (catch (re-find #"Broken pipe" (.getMessage %)) ex + (assoc op :type :info, :error ::broken-pipe)) + (catch (re-find #"Connection refused" (.getMessage %)) ex + (assoc op :type :info, :error ::connection-refused)))) + (teardown! [this test]) + (close! [this test])) + +(defn set-read-after-write + "Read-after-Write checker for set operations" + [] + (reify checker/Checker + (check [this test history opts] + (let [init {:add-started #{} + :add-done #{} + :read-must-contain {} + :missed #{} + :unexpected #{}} + final (reduce + (fn [state op] + (case [(:type op) (:f op)] + ([:invoke :add]) + (assoc state :add-started (conj (:add-started state) (:value op))) + ([:ok :add]) + (assoc state :add-done (conj (:add-done state) (:value op))) + ([:invoke :read]) + (assoc-in state [:read-must-contain (:process op)] (:add-done state)) + ([:ok :read]) + (let [read-must-contain (get (:read-must-contain state) (:process op)) + new-missed (set/difference read-must-contain (:value op)) + new-unexpected (set/difference (:value op) (:add-started state))] + (assoc state + :read-must-contain (dissoc (:read-must-contain state) (:process op)) + :missed (set/union (:missed state) new-missed), + :unexpected (set/union (:unexpected state) new-unexpected))) + state)) + init history) + valid? (and (empty? (:missed final)) (empty? (:unexpected final)))] + (assoc final :valid? valid?))))) + +(defn workload1 + "Tests insertions and deletions" + [opts] + {:client (SetClient. nil) + :checker (independent/checker + (checker/compose + {:set (checker/set) + :timeline (timeline/html)})) + :generator (independent/concurrent-generator + 10 + (range 100) + (fn [k] + (->> (range) + (map (fn [x] {:type :invoke, :f :add, :value x})) + (gen/limit (:ops-per-key opts))))) + :final-generator (independent/concurrent-generator + 10 + (range 100) + (fn [k] + (gen/phases + (gen/once op-read) + (gen/sleep 5))))}) + +(defn workload2 + "Tests insertions and deletions" + [opts] + {:client (SetClient. nil) + :checker (independent/checker + (checker/compose + {:set-read-after-write (set-read-after-write) + ; :set-full (checker/set-full {:linearizable? false}) + :timeline (timeline/html)})) + :generator (independent/concurrent-generator + 10 + (range) + (fn [k] + (->> (gen/mix [op-add-rand100 op-read]) + (gen/limit (:ops-per-key opts)))))}) + + diff --git a/script/jepsen.garage/test/jepsen/garage_test.clj b/script/jepsen.garage/test/jepsen/garage_test.clj new file mode 100644 index 00000000..055392a1 --- /dev/null +++ b/script/jepsen.garage/test/jepsen/garage_test.clj @@ -0,0 +1,7 @@ +(ns jepsen.garage-test + (:require [clojure.test :refer :all] + [jepsen.garage :refer :all])) + +(deftest a-test + (testing "FIXME, I fail." + (is (= 0 1)))) diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index e8cbc1c8..5a667359 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api" -version = "0.9.0" +version = "0.9.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 542b7a81..f9eccb7f 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -15,6 +15,7 @@ use serde::Deserialize; use garage_model::garage::Garage; +use crate::s3::cors::*; use crate::s3::error::*; use crate::s3::put::{get_headers, save_stream}; use crate::s3::xml as s3_xml; @@ -242,7 +243,7 @@ pub async fn handle_post_object( let etag = format!("\"{}\"", md5); - let resp = if let Some(mut target) = params + let mut resp = if let Some(mut target) = params .get("success_action_redirect") .and_then(|h| h.to_str().ok()) .and_then(|u| url::Url::parse(u).ok()) @@ -262,8 +263,7 @@ pub async fn handle_post_object( } else { let path = head .uri - .into_parts() - .path_and_query + .path_and_query() .map(|paq| paq.path().to_string()) .unwrap_or_else(|| "/".to_string()); let authority = head @@ -308,6 +308,13 @@ pub async fn handle_post_object( } }; + let matching_cors_rule = + find_matching_cors_rule(&bucket, &Request::from_parts(head, Body::empty()))?; + if let Some(rule) = matching_cors_rule { + add_cors_headers(&mut resp, rule) + .ok_or_internal_error("Invalid bucket CORS configuration")?; + } + Ok(resp) } diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml index f6aa5f64..d9bd1ac0 100644 --- a/src/block/Cargo.toml +++ b/src/block/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_block" -version = "0.9.0" +version = "0.9.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index 67af4a7c..470135db 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_db" -version = "0.9.0" +version = "0.9.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 7c3a79cb..35edd30c 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.9.0" +version = "0.9.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" @@ -67,6 +67,7 @@ chrono = "0.4" http = "0.2" hmac = "0.12" hyper = { version = "0.14", features = ["client", "http1", "runtime"] } +mktemp = "0.5" sha2 = "0.10" static_init = "1.0" diff --git a/src/garage/cli/init.rs b/src/garage/cli/init.rs index 20813f1c..43ca5c09 100644 --- a/src/garage/cli/init.rs +++ b/src/garage/cli/init.rs @@ -43,7 +43,7 @@ pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> { idstr ); eprintln!( - "where <remote_node> is their own node identifier in the format: <pubkey>@<ip>:<port>" + "where <remote_node> is their own node identifier in the format: <full-node-id>@<ip>:<port>" ); eprintln!(); eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:"); diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index aba57551..be4d5bd6 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -64,7 +64,8 @@ pub enum Command { #[derive(StructOpt, Debug)] pub enum NodeOperation { - /// Print identifier (public key) of this Garage node + /// Print the full node ID (public key) of this Garage node, and its publicly reachable IP + /// address and port if they are specified in config file under `rpc_public_addr` #[structopt(name = "id", version = garage_version())] NodeId(NodeIdOpt), @@ -82,8 +83,9 @@ pub struct NodeIdOpt { #[derive(StructOpt, Debug)] pub struct ConnectNodeOpt { - /// Node public key and address, in the format: - /// `<public key hexadecimal>@<ip or hostname>:<port>` + /// Full node ID (public key) and IP address and port, in the format: + /// `<full node ID>@<ip or hostname>:<port>`. + /// You can retrieve this information on the target node using `garage node id`. pub(crate) node: String, } diff --git a/src/garage/main.rs b/src/garage/main.rs index 66403d05..5c92dae4 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -7,6 +7,7 @@ extern crate tracing; mod admin; mod cli; mod repair; +mod secrets; mod server; #[cfg(feature = "telemetry-otlp")] mod tracing_setup; @@ -28,7 +29,6 @@ use structopt::StructOpt; use netapp::util::parse_and_resolve_peer_addr; use netapp::NetworkKey; -use garage_util::config::Config; use garage_util::error::*; use garage_rpc::system::*; @@ -38,6 +38,7 @@ use garage_model::helper::error::Error as HelperError; use admin::*; use cli::*; +use secrets::Secrets; #[derive(StructOpt, Debug)] #[structopt( @@ -45,8 +46,7 @@ use cli::*; about = "S3-compatible object store for self-hosted geo-distributed deployments" )] struct Opt { - /// Host to connect to for admin operations, in the format: - /// <public-key>@<ip>:<port> + /// Host to connect to for admin operations, in the format: <full-node-id>@<ip>:<port> #[structopt(short = "h", long = "rpc-host", env = "GARAGE_RPC_HOST")] pub rpc_host: Option<String>, @@ -66,24 +66,6 @@ struct Opt { cmd: Command, } -#[derive(StructOpt, Debug)] -pub struct Secrets { - /// RPC secret network key, used to replace rpc_secret in config.toml when running the - /// daemon or doing admin operations - #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")] - pub rpc_secret: Option<String>, - - /// Metrics API authentication token, replaces admin.metrics_token in config.toml when - /// running the Garage daemon - #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")] - pub admin_token: Option<String>, - - /// Metrics API authentication token, replaces admin.metrics_token in config.toml when - /// running the Garage daemon - #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")] - pub metrics_token: Option<String>, -} - #[tokio::main] async fn main() { // Initialize version and features info @@ -218,7 +200,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { // Find and parse the address of the target host let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host { - let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <pubkey>@<IP or hostname>:<port>.", h))?; + let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <full node id>@<IP or hostname>:<port>.", h))?; (id, addrs[0], false) } else { let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir) @@ -248,7 +230,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { addr ); } - Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?; + Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct full-length node ID (public key).")?; } let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into()); @@ -261,16 +243,3 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { Ok(x) => Ok(x), } } - -fn fill_secrets(mut config: Config, secrets: Secrets) -> Config { - if secrets.rpc_secret.is_some() { - config.rpc_secret = secrets.rpc_secret; - } - if secrets.admin_token.is_some() { - config.admin.admin_token = secrets.admin_token; - } - if secrets.metrics_token.is_some() { - config.admin.metrics_token = secrets.metrics_token; - } - config -} diff --git a/src/garage/repair/offline.rs b/src/garage/repair/offline.rs index f4edcf03..45024e71 100644 --- a/src/garage/repair/offline.rs +++ b/src/garage/repair/offline.rs @@ -6,7 +6,7 @@ use garage_util::error::*; use garage_model::garage::Garage; use crate::cli::structs::*; -use crate::{fill_secrets, Secrets}; +use crate::secrets::{fill_secrets, Secrets}; pub async fn offline_repair( config_file: PathBuf, @@ -20,7 +20,7 @@ pub async fn offline_repair( } info!("Loading configuration..."); - let config = fill_secrets(read_config(config_file)?, secrets); + let config = fill_secrets(read_config(config_file)?, secrets)?; info!("Initializing Garage main data store..."); let garage = Garage::new(config)?; diff --git a/src/garage/secrets.rs b/src/garage/secrets.rs new file mode 100644 index 00000000..e96be9e4 --- /dev/null +++ b/src/garage/secrets.rs @@ -0,0 +1,318 @@ +use structopt::StructOpt; + +use garage_util::config::Config; +use garage_util::error::Error; + +/// Structure for secret values or paths that are passed as CLI arguments or environment +/// variables, instead of in the config file. +#[derive(StructOpt, Debug, Default, Clone)] +pub struct Secrets { + /// Skip permission check on files containing secrets + #[cfg(unix)] + #[structopt( + long = "allow-world-readable-secrets", + env = "GARAGE_ALLOW_WORLD_READABLE_SECRETS" + )] + pub allow_world_readable_secrets: Option<bool>, + + /// RPC secret network key, used to replace rpc_secret in config.toml when running the + /// daemon or doing admin operations + #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")] + pub rpc_secret: Option<String>, + + /// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret + /// when running the daemon or doing admin operations + #[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")] + pub rpc_secret_file: Option<String>, + + /// Admin API authentication token, replaces admin.admin_token in config.toml when + /// running the Garage daemon + #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")] + pub admin_token: Option<String>, + + /// Admin API authentication token file path, replaces admin.admin_token in config.toml + /// and admin-token when running the Garage daemon + #[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")] + pub admin_token_file: Option<String>, + + /// Metrics API authentication token, replaces admin.metrics_token in config.toml when + /// running the Garage daemon + #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")] + pub metrics_token: Option<String>, + + /// Metrics API authentication token file path, replaces admin.metrics_token in config.toml + /// and metrics-token when running the Garage daemon + #[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")] + pub metrics_token_file: Option<String>, +} + +/// Single function to fill all secrets in the Config struct from their correct source (value +/// from config or CLI param or env variable or read from a file specified in config or CLI +/// param or env variable) +pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result<Config, Error> { + let allow_world_readable = secrets + .allow_world_readable_secrets + .unwrap_or(config.allow_world_readable_secrets); + + fill_secret( + &mut config.rpc_secret, + &config.rpc_secret_file, + &secrets.rpc_secret, + &secrets.rpc_secret_file, + "rpc_secret", + allow_world_readable, + )?; + + fill_secret( + &mut config.admin.admin_token, + &config.admin.admin_token_file, + &secrets.admin_token, + &secrets.admin_token_file, + "admin.admin_token", + allow_world_readable, + )?; + fill_secret( + &mut config.admin.metrics_token, + &config.admin.metrics_token_file, + &secrets.metrics_token, + &secrets.metrics_token_file, + "admin.metrics_token", + allow_world_readable, + )?; + + Ok(config) +} + +fn fill_secret( + config_secret: &mut Option<String>, + config_secret_file: &Option<String>, + cli_secret: &Option<String>, + cli_secret_file: &Option<String>, + name: &'static str, + allow_world_readable: bool, +) -> Result<(), Error> { + let cli_value = match (&cli_secret, &cli_secret_file) { + (Some(_), Some(_)) => { + return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); + } + (Some(secret), None) => Some(secret.to_string()), + (None, Some(file)) => Some(read_secret_file(file, allow_world_readable)?), + (None, None) => None, + }; + + if let Some(val) = cli_value { + if config_secret.is_some() || config_secret_file.is_some() { + debug!("Overriding secret `{}` using value specified using CLI argument or environnement variable.", name); + } + + *config_secret = Some(val); + } else if let Some(file_path) = &config_secret_file { + if config_secret.is_some() { + return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); + } + + *config_secret = Some(read_secret_file(file_path, allow_world_readable)?); + } + + Ok(()) +} + +fn read_secret_file(file_path: &String, allow_world_readable: bool) -> Result<String, Error> { + if !allow_world_readable { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::metadata(file_path)?; + if metadata.mode() & 0o077 != 0 { + return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into()); + } + } + } + + let secret_buf = std::fs::read_to_string(file_path)?; + + // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`. + // also editors sometimes add a trailing newline + Ok(String::from(secret_buf.trim_end())) +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::io::Write; + + use garage_util::config::read_config; + use garage_util::error::Error; + + use super::*; + + #[test] + fn test_rpc_secret_file_works() -> Result<(), Error> { + let path_secret = mktemp::Temp::new_file()?; + let mut file_secret = File::create(path_secret.as_path())?; + writeln!(file_secret, "foo")?; + drop(file_secret); + + let path_config = mktemp::Temp::new_file()?; + let mut file_config = File::create(path_config.as_path())?; + let path_secret_path = path_secret.as_path(); + writeln!( + file_config, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_mode = "3" + rpc_bind_addr = "[::]:3901" + rpc_secret_file = "{}" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "#, + path_secret_path.display() + )?; + drop(file_config); + + // Second configuration file, same as previous one + // except it allows world-readable secrets. + let path_config_allow_world_readable = mktemp::Temp::new_file()?; + let mut file_config_allow_world_readable = + File::create(path_config_allow_world_readable.as_path())?; + writeln!( + file_config_allow_world_readable, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_mode = "3" + rpc_bind_addr = "[::]:3901" + rpc_secret_file = "{}" + allow_world_readable_secrets = true + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "#, + path_secret_path.display() + )?; + drop(file_config_allow_world_readable); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets(config, Secrets::default())?; + assert_eq!("foo", config.rpc_secret.unwrap()); + + // ---- Check non world-readable secrets config ---- + #[cfg(unix)] + { + let secrets_allow_world_readable = Secrets { + allow_world_readable_secrets: Some(true), + ..Default::default() + }; + let secrets_no_allow_world_readable = Secrets { + allow_world_readable_secrets: Some(false), + ..Default::default() + }; + + use std::os::unix::fs::PermissionsExt; + let metadata = std::fs::metadata(&path_secret_path)?; + let mut perm = metadata.permissions(); + perm.set_mode(0o660); + std::fs::set_permissions(&path_secret_path, perm)?; + + // Config file that just specifies the path + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, Secrets::default()).is_err()); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, secrets_allow_world_readable.clone()).is_ok()); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets(config, secrets_no_allow_world_readable.clone()).is_err()); + + // Config file that also specifies to allow world_readable_secrets + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, Secrets::default()).is_ok()); + + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, secrets_allow_world_readable).is_ok()); + + let config = read_config(path_config_allow_world_readable.to_path_buf())?; + assert!(fill_secrets(config, secrets_no_allow_world_readable).is_err()); + } + + // ---- Check alternative secrets specified on CLI ---- + + let path_secret2 = mktemp::Temp::new_file()?; + let mut file_secret2 = File::create(path_secret2.as_path())?; + writeln!(file_secret2, "bar")?; + drop(file_secret2); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets( + config, + Secrets { + rpc_secret: Some("baz".into()), + ..Default::default() + }, + )?; + assert_eq!(config.rpc_secret.as_deref(), Some("baz")); + + let config = read_config(path_config.to_path_buf())?; + let config = fill_secrets( + config, + Secrets { + rpc_secret_file: Some(path_secret2.display().to_string()), + ..Default::default() + }, + )?; + assert_eq!(config.rpc_secret.as_deref(), Some("bar")); + + let config = read_config(path_config.to_path_buf())?; + assert!(fill_secrets( + config, + Secrets { + rpc_secret: Some("baz".into()), + rpc_secret_file: Some(path_secret2.display().to_string()), + ..Default::default() + } + ) + .is_err()); + + drop(path_secret); + drop(path_secret2); + drop(path_config); + drop(path_config_allow_world_readable); + + Ok(()) + } + + #[test] + fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> { + let path_config = mktemp::Temp::new_file()?; + let mut file_config = File::create(path_config.as_path())?; + writeln!( + file_config, + r#" + metadata_dir = "/tmp/garage/meta" + data_dir = "/tmp/garage/data" + replication_mode = "3" + rpc_bind_addr = "[::]:3901" + rpc_secret= "dummy" + rpc_secret_file = "dummy" + + [s3_api] + s3_region = "garage" + api_bind_addr = "[::]:3900" + "# + )?; + let config = read_config(path_config.to_path_buf())?; + assert_eq!( + "only one of `rpc_secret` and `rpc_secret_file` can be set", + fill_secrets(config, Secrets::default()) + .unwrap_err() + .to_string() + ); + drop(path_config); + drop(file_config); + Ok(()) + } +} diff --git a/src/garage/server.rs b/src/garage/server.rs index 3ad10b72..ac76a44d 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -15,9 +15,9 @@ use garage_web::WebServer; use garage_api::k2v::api_server::K2VApiServer; use crate::admin::*; +use crate::secrets::{fill_secrets, Secrets}; #[cfg(feature = "telemetry-otlp")] use crate::tracing_setup::*; -use crate::{fill_secrets, Secrets}; async fn wait_from(mut chan: watch::Receiver<bool>) { while !*chan.borrow() { @@ -29,12 +29,19 @@ async fn wait_from(mut chan: watch::Receiver<bool>) { pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Error> { info!("Loading configuration..."); - let config = fill_secrets(read_config(config_file)?, secrets); + let config = fill_secrets(read_config(config_file)?, secrets)?; // ---- Initialize Garage internals ---- #[cfg(feature = "metrics")] - let metrics_exporter = opentelemetry_prometheus::exporter().init(); + let metrics_exporter = opentelemetry_prometheus::exporter() + .with_default_summary_quantiles(vec![0.25, 0.5, 0.75, 0.9, 0.95, 0.99]) + .with_default_histogram_boundaries(vec![ + 0.001, 0.0015, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.02, 0.03, 0.05, 0.07, 0.1, + 0.15, 0.2, 0.3, 0.5, 0.7, 1., 1.5, 2., 3., 5., 7., 10., 15., 20., 30., 40., 50., 60., + 70., 100., + ]) + .init(); info!("Initializing Garage main data store..."); let garage = Garage::new(config.clone())?; diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 42b7ffdb..8ecd5d63 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "0.9.0" +version = "0.9.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index f450718f..0ca72016 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_rpc" -version = "0.9.0" +version = "0.9.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index ab8d1112..71fd71b0 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -148,7 +148,7 @@ impl ConsulDiscovery { ret.push((pubkey, SocketAddr::new(ip, ent.service_port))); } else { warn!( - "Could not process node spec from Consul: {:?} (invalid IP or public key)", + "Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)", ent ); } diff --git a/src/rpc/system.rs b/src/rpc/system.rs index 4b40bec4..4cec369b 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -57,7 +57,7 @@ pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc"; pub enum SystemRpc { /// Response to successfull advertisements Ok, - /// Request to connect to a specific node (in <pubkey>@<host>:<port> format) + /// Request to connect to a specific node (in <pubkey>@<host>:<port> format, pubkey = full-length node ID) Connect(String), /// Ask other node its cluster layout. Answered with AdvertiseClusterLayout PullClusterLayout, diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 08ccec72..f40787d1 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_table" -version = "0.9.0" +version = "0.9.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 6554ac13..85317ec1 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_util" -version = "0.9.0" +version = "0.9.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" diff --git a/src/util/config.rs b/src/util/config.rs index ad5c8e1f..65c0b5c0 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -1,6 +1,5 @@ //! Contains type and functions related to Garage configuration file use std::convert::TryFrom; -use std::io::Read; use std::net::SocketAddr; use std::path::PathBuf; @@ -45,11 +44,15 @@ pub struct Config { )] pub compression_level: Option<i32>, + /// Skip the permission check of secret files. Useful when + /// POSIX ACLs (or more complex chmods) are used. + #[serde(default)] + pub allow_world_readable_secrets: bool, + /// RPC secret key: 32 bytes hex encoded pub rpc_secret: Option<String>, /// Optional file where RPC secret key is read from pub rpc_secret_file: Option<String>, - /// Address to bind for RPC pub rpc_bind_addr: SocketAddr, /// Public IP address of this node @@ -221,6 +224,13 @@ pub struct KubernetesDiscoveryConfig { pub skip_crd: bool, } +/// Read and parse configuration +pub fn read_config(config_file: PathBuf) -> Result<Config, Error> { + let config = std::fs::read_to_string(config_file)?; + + Ok(toml::from_str(&config)?) +} + fn default_db_engine() -> String { "lmdb".into() } @@ -235,68 +245,6 @@ fn default_block_size() -> usize { 1048576 } -/// Read and parse configuration -pub fn read_config(config_file: PathBuf) -> Result<Config, Error> { - let mut file = std::fs::OpenOptions::new() - .read(true) - .open(config_file.as_path())?; - - let mut config = String::new(); - file.read_to_string(&mut config)?; - - let mut parsed_config: Config = toml::from_str(&config)?; - - secret_from_file( - &mut parsed_config.rpc_secret, - &parsed_config.rpc_secret_file, - "rpc_secret", - )?; - secret_from_file( - &mut parsed_config.admin.metrics_token, - &parsed_config.admin.metrics_token_file, - "admin.metrics_token", - )?; - secret_from_file( - &mut parsed_config.admin.admin_token, - &parsed_config.admin.admin_token_file, - "admin.admin_token", - )?; - - Ok(parsed_config) -} - -fn secret_from_file( - secret: &mut Option<String>, - secret_file: &Option<String>, - name: &'static str, -) -> Result<(), Error> { - match (&secret, &secret_file) { - (_, None) => { - // no-op - } - (Some(_), Some(_)) => { - return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into()); - } - (None, Some(file_path)) => { - #[cfg(unix)] - if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") { - use std::os::unix::fs::MetadataExt; - let metadata = std::fs::metadata(file_path)?; - if metadata.mode() & 0o077 != 0 { - return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into()); - } - } - let mut file = std::fs::OpenOptions::new().read(true).open(file_path)?; - let mut secret_buf = String::new(); - file.read_to_string(&mut secret_buf)?; - // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`. - // also editors sometimes add a trailing newline - *secret = Some(String::from(secret_buf.trim_end())); - } - } - Ok(()) -} - fn default_compression() -> Option<i32> { Some(1) } @@ -425,83 +373,4 @@ mod tests { Ok(()) } - - #[test] - fn test_rpc_secret_file_works() -> Result<(), Error> { - let path_secret = mktemp::Temp::new_file()?; - let mut file_secret = File::create(path_secret.as_path())?; - writeln!(file_secret, "foo")?; - drop(file_secret); - - let path_config = mktemp::Temp::new_file()?; - let mut file_config = File::create(path_config.as_path())?; - let path_secret_path = path_secret.as_path(); - writeln!( - file_config, - r#" - metadata_dir = "/tmp/garage/meta" - data_dir = "/tmp/garage/data" - replication_mode = "3" - rpc_bind_addr = "[::]:3901" - rpc_secret_file = "{}" - - [s3_api] - s3_region = "garage" - api_bind_addr = "[::]:3900" - "#, - path_secret_path.display() - )?; - let config = super::read_config(path_config.to_path_buf())?; - assert_eq!("foo", config.rpc_secret.unwrap()); - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let metadata = std::fs::metadata(&path_secret_path)?; - let mut perm = metadata.permissions(); - perm.set_mode(0o660); - std::fs::set_permissions(&path_secret_path, perm)?; - - std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "false"); - assert!(super::read_config(path_config.to_path_buf()).is_err()); - - std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "true"); - assert!(super::read_config(path_config.to_path_buf()).is_ok()); - } - - drop(path_config); - drop(path_secret); - drop(file_config); - Ok(()) - } - - #[test] - fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> { - let path_config = mktemp::Temp::new_file()?; - let mut file_config = File::create(path_config.as_path())?; - writeln!( - file_config, - r#" - metadata_dir = "/tmp/garage/meta" - data_dir = "/tmp/garage/data" - replication_mode = "3" - rpc_bind_addr = "[::]:3901" - rpc_secret= "dummy" - rpc_secret_file = "dummy" - - [s3_api] - s3_region = "garage" - api_bind_addr = "[::]:3900" - "# - )?; - assert_eq!( - "only one of `rpc_secret` and `rpc_secret_file` can be set", - super::read_config(path_config.to_path_buf()) - .unwrap_err() - .to_string() - ); - drop(path_config); - drop(file_config); - Ok(()) - } } diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 88cf1486..cb4b7f2b 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_web" -version = "0.9.0" +version = "0.9.1" authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"] edition = "2018" license = "AGPL-3.0" |
