aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Cargo.lock19
-rw-r--r--Cargo.nix121
-rw-r--r--Cargo.toml16
-rw-r--r--doc/book/connect/apps/index.md4
-rw-r--r--doc/book/cookbook/real-world.md10
-rw-r--r--doc/book/reference-manual/configuration.md22
-rw-r--r--script/helm/garage/Chart.yaml2
-rw-r--r--script/jepsen.garage/.envrc1
-rw-r--r--script/jepsen.garage/.gitignore17
-rw-r--r--script/jepsen.garage/README.md166
-rw-r--r--script/jepsen.garage/Vagrantfile40
-rwxr-xr-xscript/jepsen.garage/all_tests_1.sh18
-rwxr-xr-xscript/jepsen.garage/all_tests_2.sh16
-rw-r--r--script/jepsen.garage/jaeger.sh13
-rw-r--r--script/jepsen.garage/nodes.vagrant7
-rw-r--r--script/jepsen.garage/nodes2.vagrant7
-rw-r--r--script/jepsen.garage/project.clj10
-rw-r--r--script/jepsen.garage/results/Results-2023-11-16.pngbin0 -> 1517471 bytes
-rw-r--r--script/jepsen.garage/results/Results-2023-12-13-task3c.pngbin0 -> 1075969 bytes
-rw-r--r--script/jepsen.garage/results/Results-2023-12-13-tsfix2.pngbin0 -> 1136760 bytes
-rw-r--r--script/jepsen.garage/results/Results-2023-12-14-task3-set1.pngbin0 -> 1075655 bytes
-rw-r--r--script/jepsen.garage/shell.nix18
-rw-r--r--script/jepsen.garage/src/jepsen/garage.clj105
-rw-r--r--script/jepsen.garage/src/jepsen/garage/daemon.clj152
-rw-r--r--script/jepsen.garage/src/jepsen/garage/nemesis.clj142
-rw-r--r--script/jepsen.garage/src/jepsen/garage/reg.clj143
-rw-r--r--script/jepsen.garage/src/jepsen/garage/s3api.clj48
-rw-r--r--script/jepsen.garage/src/jepsen/garage/set.clj135
-rw-r--r--script/jepsen.garage/test/jepsen/garage_test.clj7
-rw-r--r--src/api/Cargo.toml2
-rw-r--r--src/api/s3/post_object.rs13
-rw-r--r--src/block/Cargo.toml2
-rw-r--r--src/db/Cargo.toml2
-rw-r--r--src/garage/Cargo.toml3
-rw-r--r--src/garage/cli/init.rs2
-rw-r--r--src/garage/cli/structs.rs8
-rw-r--r--src/garage/main.rs41
-rw-r--r--src/garage/repair/offline.rs4
-rw-r--r--src/garage/secrets.rs318
-rw-r--r--src/garage/server.rs13
-rw-r--r--src/model/Cargo.toml2
-rw-r--r--src/rpc/Cargo.toml2
-rw-r--r--src/rpc/consul.rs2
-rw-r--r--src/rpc/system.rs2
-rw-r--r--src/table/Cargo.toml2
-rw-r--r--src/util/Cargo.toml2
-rw-r--r--src/util/config.rs155
-rw-r--r--src/web/Cargo.toml2
48 files changed, 1526 insertions, 290 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 7135a4de..98433c54 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1198,7 +1198,7 @@ dependencies = [
[[package]]
name = "garage"
-version = "0.9.0"
+version = "0.9.1"
dependencies = [
"assert-json-diff",
"async-trait",
@@ -1227,6 +1227,7 @@ dependencies = [
"hyper",
"k2v-client",
"kuska-sodiumoxide",
+ "mktemp",
"netapp",
"opentelemetry",
"opentelemetry-otlp",
@@ -1249,7 +1250,7 @@ dependencies = [
[[package]]
name = "garage_api"
-version = "0.9.0"
+version = "0.9.1"
dependencies = [
"async-trait",
"base64 0.21.3",
@@ -1295,7 +1296,7 @@ dependencies = [
[[package]]
name = "garage_block"
-version = "0.9.0"
+version = "0.9.1"
dependencies = [
"arc-swap",
"async-compression",
@@ -1321,7 +1322,7 @@ dependencies = [
[[package]]
name = "garage_db"
-version = "0.9.0"
+version = "0.9.1"
dependencies = [
"clap 4.4.0",
"err-derive",
@@ -1336,7 +1337,7 @@ dependencies = [
[[package]]
name = "garage_model"
-version = "0.9.0"
+version = "0.9.1"
dependencies = [
"arc-swap",
"async-trait",
@@ -1364,7 +1365,7 @@ dependencies = [
[[package]]
name = "garage_rpc"
-version = "0.9.0"
+version = "0.9.1"
dependencies = [
"arc-swap",
"async-trait",
@@ -1399,7 +1400,7 @@ dependencies = [
[[package]]
name = "garage_table"
-version = "0.9.0"
+version = "0.9.1"
dependencies = [
"arc-swap",
"async-trait",
@@ -1421,7 +1422,7 @@ dependencies = [
[[package]]
name = "garage_util"
-version = "0.9.0"
+version = "0.9.1"
dependencies = [
"arc-swap",
"async-trait",
@@ -1455,7 +1456,7 @@ dependencies = [
[[package]]
name = "garage_web"
-version = "0.9.0"
+version = "0.9.1"
dependencies = [
"err-derive",
"futures",
diff --git a/Cargo.nix b/Cargo.nix
index cd8270b6..eda6093b 100644
--- a/Cargo.nix
+++ b/Cargo.nix
@@ -33,7 +33,7 @@ args@{
ignoreLockHash,
}:
let
- nixifiedLockHash = "1a87886681a3ef0b83c95addc26674a538b8a93d35bc80db8998e1fcd0821f6c";
+ nixifiedLockHash = "f8da48e74313b18ee372a07d6a7d63a27c7effde24ed8599dfb91ea311df10f8";
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
lockHashIgnored = if ignoreLockHash
@@ -57,16 +57,16 @@ in
{
cargo2nixVersion = "0.11.0";
workspace = {
- garage_db = rustPackages.unknown.garage_db."0.9.0";
- garage_util = rustPackages.unknown.garage_util."0.9.0";
- garage_rpc = rustPackages.unknown.garage_rpc."0.9.0";
+ garage_db = rustPackages.unknown.garage_db."0.9.1";
+ garage_util = rustPackages.unknown.garage_util."0.9.1";
+ garage_rpc = rustPackages.unknown.garage_rpc."0.9.1";
format_table = rustPackages.unknown.format_table."0.1.1";
- garage_table = rustPackages.unknown.garage_table."0.9.0";
- garage_block = rustPackages.unknown.garage_block."0.9.0";
- garage_model = rustPackages.unknown.garage_model."0.9.0";
- garage_api = rustPackages.unknown.garage_api."0.9.0";
- garage_web = rustPackages.unknown.garage_web."0.9.0";
- garage = rustPackages.unknown.garage."0.9.0";
+ garage_table = rustPackages.unknown.garage_table."0.9.1";
+ garage_block = rustPackages.unknown.garage_block."0.9.1";
+ garage_model = rustPackages.unknown.garage_model."0.9.1";
+ garage_api = rustPackages.unknown.garage_api."0.9.1";
+ garage_web = rustPackages.unknown.garage_web."0.9.1";
+ garage = rustPackages.unknown.garage."0.9.1";
k2v-client = rustPackages.unknown.k2v-client."0.0.4";
};
"registry+https://github.com/rust-lang/crates.io-index".addr2line."0.21.0" = overridableMkRustCrate (profileName: rec {
@@ -1705,9 +1705,9 @@ in
};
});
- "unknown".garage."0.9.0" = overridableMkRustCrate (profileName: rec {
+ "unknown".garage."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage";
- version = "0.9.0";
+ version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/garage");
features = builtins.concatLists [
@@ -1734,14 +1734,14 @@ in
format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
- garage_api = (rustPackages."unknown".garage_api."0.9.0" { inherit profileName; }).out;
- garage_block = (rustPackages."unknown".garage_block."0.9.0" { inherit profileName; }).out;
- garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
- garage_model = (rustPackages."unknown".garage_model."0.9.0" { inherit profileName; }).out;
- garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out;
- garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out;
- garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
- garage_web = (rustPackages."unknown".garage_web."0.9.0" { inherit profileName; }).out;
+ garage_api = (rustPackages."unknown".garage_api."0.9.1" { inherit profileName; }).out;
+ garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out;
+ garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
+ garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out;
+ garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
+ garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
+ garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
+ garage_web = (rustPackages."unknown".garage_web."0.9.1" { inherit profileName; }).out;
git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
@@ -1771,15 +1771,16 @@ in
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
k2v_client = (rustPackages."unknown".k2v-client."0.0.4" { inherit profileName; }).out;
+ mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" { inherit profileName; }).out;
serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out;
sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out;
static_init = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.3" { inherit profileName; }).out;
};
});
- "unknown".garage_api."0.9.0" = overridableMkRustCrate (profileName: rec {
+ "unknown".garage_api."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_api";
- version = "0.9.0";
+ version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/api");
features = builtins.concatLists [
@@ -1798,11 +1799,11 @@ in
form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
- garage_block = (rustPackages."unknown".garage_block."0.9.0" { inherit profileName; }).out;
- garage_model = (rustPackages."unknown".garage_model."0.9.0" { inherit profileName; }).out;
- garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out;
- garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out;
- garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
+ garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out;
+ garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out;
+ garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
+ garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
+ garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
@@ -1832,9 +1833,9 @@ in
};
});
- "unknown".garage_block."0.9.0" = overridableMkRustCrate (profileName: rec {
+ "unknown".garage_block."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_block";
- version = "0.9.0";
+ version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/block");
features = builtins.concatLists [
@@ -1848,10 +1849,10 @@ in
bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
- garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
- garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out;
- garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out;
- garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
+ garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
+ garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
+ garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
+ garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
@@ -1864,9 +1865,9 @@ in
};
});
- "unknown".garage_db."0.9.0" = overridableMkRustCrate (profileName: rec {
+ "unknown".garage_db."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_db";
- version = "0.9.0";
+ version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/db");
features = builtins.concatLists [
@@ -1896,9 +1897,9 @@ in
};
});
- "unknown".garage_model."0.9.0" = overridableMkRustCrate (profileName: rec {
+ "unknown".garage_model."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_model";
- version = "0.9.0";
+ version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/model");
features = builtins.concatLists [
@@ -1917,11 +1918,11 @@ in
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
- garage_block = (rustPackages."unknown".garage_block."0.9.0" { inherit profileName; }).out;
- garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
- garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out;
- garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out;
- garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
+ garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out;
+ garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
+ garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
+ garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
+ garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.10.0" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -1934,9 +1935,9 @@ in
};
});
- "unknown".garage_rpc."0.9.0" = overridableMkRustCrate (profileName: rec {
+ "unknown".garage_rpc."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_rpc";
- version = "0.9.0";
+ version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/rpc");
features = builtins.concatLists [
@@ -1958,8 +1959,8 @@ in
format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
- garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
- garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
+ garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
+ garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" { inherit profileName; }).out;
@@ -1982,9 +1983,9 @@ in
};
});
- "unknown".garage_table."0.9.0" = overridableMkRustCrate (profileName: rec {
+ "unknown".garage_table."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_table";
- version = "0.9.0";
+ version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/table");
dependencies = {
@@ -1993,9 +1994,9 @@ in
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
- garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
- garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out;
- garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
+ garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
+ garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
+ garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -2007,9 +2008,9 @@ in
};
});
- "unknown".garage_util."0.9.0" = overridableMkRustCrate (profileName: rec {
+ "unknown".garage_util."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_util";
- version = "0.9.0";
+ version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/util");
features = builtins.concatLists [
@@ -2025,7 +2026,7 @@ in
digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out;
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
- garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
+ garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
@@ -2051,18 +2052,18 @@ in
};
});
- "unknown".garage_web."0.9.0" = overridableMkRustCrate (profileName: rec {
+ "unknown".garage_web."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_web";
- version = "0.9.0";
+ version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/web");
dependencies = {
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
- garage_api = (rustPackages."unknown".garage_api."0.9.0" { inherit profileName; }).out;
- garage_model = (rustPackages."unknown".garage_model."0.9.0" { inherit profileName; }).out;
- garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out;
- garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
+ garage_api = (rustPackages."unknown".garage_api."0.9.1" { inherit profileName; }).out;
+ garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out;
+ garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
+ garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
hyperlocal = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyperlocal."0.8.0" { inherit profileName; }).out;
diff --git a/Cargo.toml b/Cargo.toml
index e3d111c3..be388362 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,14 +18,14 @@ default-members = ["src/garage"]
[workspace.dependencies]
format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api = { version = "0.9.0", path = "src/api" }
-garage_block = { version = "0.9.0", path = "src/block" }
-garage_db = { version = "0.9.0", path = "src/db", default-features = false }
-garage_model = { version = "0.9.0", path = "src/model", default-features = false }
-garage_rpc = { version = "0.9.0", path = "src/rpc" }
-garage_table = { version = "0.9.0", path = "src/table" }
-garage_util = { version = "0.9.0", path = "src/util" }
-garage_web = { version = "0.9.0", path = "src/web" }
+garage_api = { version = "0.9.1", path = "src/api" }
+garage_block = { version = "0.9.1", path = "src/block" }
+garage_db = { version = "0.9.1", path = "src/db", default-features = false }
+garage_model = { version = "0.9.1", path = "src/model", default-features = false }
+garage_rpc = { version = "0.9.1", path = "src/rpc" }
+garage_table = { version = "0.9.1", path = "src/table" }
+garage_util = { version = "0.9.1", path = "src/util" }
+garage_web = { version = "0.9.1", path = "src/web" }
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
[profile.dev]
diff --git a/doc/book/connect/apps/index.md b/doc/book/connect/apps/index.md
index f67a29c9..c8571fac 100644
--- a/doc/book/connect/apps/index.md
+++ b/doc/book/connect/apps/index.md
@@ -146,7 +146,7 @@ Keep the Key ID and the Secret key in a pad, they will be needed later.
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
```bash
-garage bucket create peertube-video
+garage bucket create peertube-videos
garage bucket create peertube-playlist
```
@@ -216,7 +216,7 @@ object_storage:
# Same settings but for webtorrent videos
videos:
- bucket_name: 'peertube-video'
+ bucket_name: 'peertube-videos'
prefix: ''
# You must fill this field to make Peertube use our reverse proxy/website logic
base_url: 'http://peertube-videos.web.garage.localhost'
diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md
index ea4ce1f9..ce0abddd 100644
--- a/doc/book/cookbook/real-world.md
+++ b/doc/book/cookbook/real-world.md
@@ -85,14 +85,14 @@ to store 2 TB of data in total.
## Get a Docker image
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v0.9.0`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v0.9.0` but it's up to you
+We encourage you to use a fixed tag (eg. `v0.9.1`) and not the `latest` tag.
+For this example, we will use the latest published version at the time of the writing which is `v0.9.1` but it's up to you
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
For example:
```
-sudo docker pull dxflrs/garage:v0.9.0
+sudo docker pull dxflrs/garage:v0.9.1
```
## Deploying and configuring Garage
@@ -157,7 +157,7 @@ docker run \
-v /etc/garage.toml:/etc/garage.toml \
-v /var/lib/garage/meta:/var/lib/garage/meta \
-v /var/lib/garage/data:/var/lib/garage/data \
- dxflrs/garage:v0.9.0
+ dxflrs/garage:v0.9.1
```
With this command line, Garage should be started automatically at each boot.
@@ -171,7 +171,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
version: "3"
services:
garage:
- image: dxflrs/garage:v0.9.0
+ image: dxflrs/garage:v0.9.1
network_mode: "host"
restart: unless-stopped
volumes:
diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md
index 18d160bb..5e12a7da 100644
--- a/doc/book/reference-manual/configuration.md
+++ b/doc/book/reference-manual/configuration.md
@@ -394,7 +394,7 @@ Compression is done synchronously, setting a value too high will add latency to
This value can be different between nodes, compression is done by the node which receive the
API call.
-#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET` (env) {#rpc_secret}
+#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET`, `GARAGE_RPC_SECRET_FILE` (env) {#rpc_secret}
Garage uses a secret key, called an RPC secret, that is shared between all
nodes of the cluster in order to identify these nodes and allow them to
@@ -406,6 +406,9 @@ Since Garage `v0.8.2`, the RPC secret can also be stored in a file whose path is
given in the configuration variable `rpc_secret_file`, or specified as an
environment variable `GARAGE_RPC_SECRET`.
+Since Garage `v0.8.5` and `v0.9.1`, you can also specify the path of a file
+storing the secret as the `GARAGE_RPC_SECRET_FILE` environment variable.
+
#### `rpc_bind_addr` {#rpc_bind_addr}
The address and port on which to bind for inter-cluster communcations
@@ -438,6 +441,17 @@ be obtained by running `garage node id` and then included directly in the
key will be returned by `garage node id` and you will have to add the IP
yourself.
+### `allow_world_readable_secrets`
+
+Garage checks the permissions of your secret files to make sure they're not
+world-readable. In some cases, the check might fail and consider your files as
+world-readable even if they're not, for instance when using Posix ACLs.
+
+Setting `allow_world_readable_secrets` to `true` bypass this
+permission verification.
+
+Alternatively, you can set the `GARAGE_ALLOW_WORLD_READABLE_SECRETS`
+environment variable to `true` to bypass the permissions check.
### The `[consul_discovery]` section
@@ -583,7 +597,7 @@ See [administration API reference](@/documentation/reference-manual/admin-api.md
Alternatively, since `v0.8.5`, a path can be used to create a unix socket. Note that for security reasons,
the socket will have 0220 mode. Make sure to set user and group permissions accordingly.
-#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN` (env) {#admin_metrics_token}
+#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN`, `GARAGE_METRICS_TOKEN_FILE` (env) {#admin_metrics_token}
The token for accessing the Metrics endpoint. If this token is not set, the
Metrics endpoint can be accessed without access control.
@@ -593,8 +607,9 @@ You can use any random string for this value. We recommend generating a random t
`metrics_token` was introduced in Garage `v0.7.2`.
`metrics_token_file` and the `GARAGE_METRICS_TOKEN` environment variable are supported since Garage `v0.8.2`.
+`GARAGE_METRICS_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
-#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN` (env) {#admin_token}
+#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env) {#admin_token}
The token for accessing all of the other administration endpoints. If this
token is not set, access to these endpoints is disabled entirely.
@@ -604,6 +619,7 @@ You can use any random string for this value. We recommend generating a random t
`admin_token` was introduced in Garage `v0.7.2`.
`admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`.
+`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
#### `trace_sink` {#admin_trace_sink}
diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml
index 346e68ad..31b75c1f 100644
--- a/script/helm/garage/Chart.yaml
+++ b/script/helm/garage/Chart.yaml
@@ -21,4 +21,4 @@ version: 0.4.1
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
-appVersion: "v0.9.0"
+appVersion: "v0.9.1"
diff --git a/script/jepsen.garage/.envrc b/script/jepsen.garage/.envrc
new file mode 100644
index 00000000..1d953f4b
--- /dev/null
+++ b/script/jepsen.garage/.envrc
@@ -0,0 +1 @@
+use nix
diff --git a/script/jepsen.garage/.gitignore b/script/jepsen.garage/.gitignore
new file mode 100644
index 00000000..31842a96
--- /dev/null
+++ b/script/jepsen.garage/.gitignore
@@ -0,0 +1,17 @@
+/target
+/classes
+/checkouts
+profiles.clj
+pom.xml
+pom.xml.asc
+*.jar
+*.class
+/.lein-*
+/.nrepl-port
+/.prepl-port
+.hgignore
+.hg/
+.direnv
+/store
+/store.*
+.vagrant
diff --git a/script/jepsen.garage/README.md b/script/jepsen.garage/README.md
new file mode 100644
index 00000000..50c7eb38
--- /dev/null
+++ b/script/jepsen.garage/README.md
@@ -0,0 +1,166 @@
+# jepsen.garage
+
+Jepsen checking of Garage consistency properties.
+
+## Usage
+
+Requirements:
+
+- vagrant
+- VirtualBox, configured so that nodes can take an IP in a private network `192.168.56.0/24` (it's the default)
+- a user that can create VirtualBox VMs
+- leiningen
+- gnuplot
+
+Set up VMs before running tests:
+
+```
+vagrant up
+```
+
+Run tests: see commands below.
+
+
+## Results
+
+### Register linear, without timestamp patch
+
+Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 20 --workload reg1 --ops-per-key 100`
+
+Results without timestamp patch:
+
+- Fails with a simple clock-scramble nemesis (`--scenario c`).
+ Explanation: without the timestamp patch, nodes will create objects using their
+ local clock only as a timestamp, so the ordering will be all over the place if
+ clocks are scrambled.
+
+Results with timestamp patch (`--patch tsfix2`):
+
+- No failure with clock-scramble nemesis
+
+- Fails with clock-scramble nemesis + partition nemesis (`--scenario cp`).
+
+**This test is expected to fail.**
+Indeed, S3 objects are not meant to behave like linearizable registers.
+TODO explain using a counter-example
+
+
+### Read-after-write CRDT register model
+
+Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 100 --workload reg2 --ops-per-key 100`
+
+Results without timestamp patch:
+
+- Fails with a simple clock-scramble nemesis (`--scenario c`).
+ Explanation: old values are not overwritten correctly when their timestamps are in the future.
+
+Results with timestamp patch (`--patch tsfix2`):
+
+- No failures with clock-scramble nemesis + partition nemesis (`--scenario cp`).
+ This proves that `tsfix2` (PR#543) does improve consistency.
+
+- **Fails with layout reconfiguration nemesis** (`--scenario r`).
+ Example of a failed run: `garage reg2/20231024T120806.899+0200`.
+ This is the failure mode we are looking for and trying to fix for NLnet task 3.
+
+Results with NLnet task 3 code (commit 707442f5de, `--patch task3a`):
+
+- No failures with `--scenario r` (0 of 10 runs), `--scenario pr` (0 of 10 runs),
+ `--scenario cpr` (0 of 10 runs) and `--scenario dpr` (0 of 10 runs).
+
+- Same with `--patch task3c` (commit `0041b013`, the final version).
+
+
+### Set, basic test (write some items, then read)
+
+Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 200 --concurrency 200 --workload set1 --ops-per-key 100`
+
+Results without NLnet task3 code (`--patch tsfix2`):
+
+- For now, no failures with clock-scramble nemesis + partition nemesis -> TODO long test run
+
+- Does not seem to fail with only the layout reconfiguation nemesis (<10 runs), although theoretically it could
+
+- **Fails with the partition + layout reconfiguration nemesis** (`--scenario pr`).
+ Example of a failed run: `garage set1/20231024T172214.488+0200` (1 failure in 4 runs).
+ This is the failure mode we are looking for and trying to fix for NLnet task 3.
+
+Results with NLnet task 3 code (commit 707442f5de, `--patch task3a`):
+
+- The tests are buggy and often result in an "unknown" validity status, which
+ is caused by some requests not returning results during network partitions or
+ other nemesis-induced broken cluster states. However, when the tests were
+ able to finish, there were no failures with scenarios `r`, `pr`, `cpr`,
+ `dpr`.
+
+
+### Set, continuous test (interspersed reads and writes)
+
+Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 100 --workload set2 --ops-per-key 100`
+
+Results without NLnet task3 code (`--patch tsfix2`):
+
+- No failures with clock-scramble nemesis + db nemesis + partition nemesis (`--scenario cdp`) (0 failures in 10 runs).
+
+- **Fails with just layout reconfiguration nemesis** (`--scenario r`).
+ Example of a failed run: `garage set2/20231025T141940.198+0200` (10 failures in 10 runs).
+ This is the failure mode we are looking for and trying to fix for NLnet task 3.
+
+Results with NLnet task3 code (commit 707442f5de, `--patch task3a`):
+
+- No failures with `--scenario r` (0 of 10 runs), `--scenario pr` (0 of 10 runs),
+ `--scenario cpr` (0 of 10 runs) and `--scenario dpr` (0 of 10 runs).
+
+- Same with `--patch task3c` (commit `0041b013`, the final version).
+
+
+## NLnet task 3 final results
+
+- With code from task3 (`--patch task3c`): [reg2 and set2](results/Results-2023-12-13-task3c.png), [set1](results/Results-2023-12-14-task3-set1.png).
+- Without (`--patch tsfix2`): [reg2 and set2](results/Results-2023-12-13-tsfix2.png), set1 TBD.
+
+## Investigating (and fixing) errors
+
+### Segfaults
+
+They are due to the download being interrupted in the middle (^C during first launch on clean VMs), the `garage` binary is truncated.
+Add `:force?` to the `cached-wget!` call in `daemon.clj` to re-download the binary,
+or restar the VMs to clear temporary files.
+
+### In `jepsen.garage`: prefix wierdness
+
+In `store/garage set1/20231019T163358.615+0200`:
+
+```
+INFO [2023-10-19 16:35:20,977] clojure-agent-send-off-pool-207 - jepsen.garage.set list results for prefix set20/ : (set13/0 set13/1 set13/10 set13/11 set13/12 set13/13 set13/14 set13/15 set13/16 set13/17 set13/18 set13/19 set13/2 set13/20 set13/21 set13/22 set13/23 set13/24 set13/25 set13/26 set13/27 set13/28 set13/29 set13/3 set13/30 set13/31 set13/32 set13/33 set13/34 set13/35 set13/36 set13/37 set13/38 set13/39 set13/4 set13/40 set13/41 set13/42 set13/43 set13/44 set13/45 set13/46 set13/47 set13/48 set13/49 set13/5 set13/50 set13/51 set13/52 set13/53 set13/54 set13/55 set13/56 set13/57 set13/58 set13/59 set13/6 set13/60 set13/61 set13/62 set13/63 set13/64 set13/65 set13/66 set13/67 set13/68 set13/69 set13/7 set13/70 set13/71 set13/72 set13/73 set13/74 set13/75 set13/76 set13/77 set13/78 set13/79 set13/8 set13/80 set13/81 set13/82 set13/83 set13/84 set13/85 set13/86 set13/87 set13/88 set13/89 set13/9 set13/90 set13/91 set13/92 set13/93 set13/94 set13/95 set13/96 set13/97 set13/98 set13/99) (node: http://192.168.56.25:3900 )
+```
+
+After inspecting, the actual S3 call made was with prefix "set13/", so at least this is not an error in Garage itself but in the jepsen code.
+
+Finally found out that this was due to closures not correctly capturing their context in the list function in s3api.clj (wtf clojure?)
+Not sure exactly where it came from but it seems to have been fixed by making list-inner a separate function and not a sub-function,
+and passing all values that were previously in the context (creds and prefix) as additional arguments.
+
+### `reg2` test inconsistency, even with timestamp fix
+
+The reg2 test is our custom checker for CRDT read-after-write on individual object keys, acting as registers which can be updated.
+The test fails without the timestamp fix, which is expected as the clock scrambler will prevent nodes from having a correct ordering of objects.
+
+With the timestamp fix (`--patch tsfix1`), the happenned-before relationship should at least be respected, meaning that when a PutObject call starts
+after another PutObject call has ended, the second call should overwrite the value of the first call, and that value should not be
+readable by future GetObject calls.
+However, we observed inconsistencies even with the timestamp fix.
+
+The inconsistencies seemed to always happenned after writing a nil value, which translates to a DeleteObject call
+instead of a PutObject. By removing the possibility of writing nil values, therefore only doing
+PutObject calls, the issue disappears. There is therefore an issue to fix in DeleteObject.
+
+The issue in DeleteObject seems to have been fixed by commit `c82d91c6bccf307186332b6c5c6fc0b128b1b2b1`, which can be used using `--patch tsfix2`.
+
+
+## License
+
+Copyright © 2023 Alex Auvolat
+
+This program and the accompanying materials are made available under the
+terms of the GNU Affero General Public License v3.0.
diff --git a/script/jepsen.garage/Vagrantfile b/script/jepsen.garage/Vagrantfile
new file mode 100644
index 00000000..b54c2426
--- /dev/null
+++ b/script/jepsen.garage/Vagrantfile
@@ -0,0 +1,40 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+#
+
+def vm(config, hostname, ip)
+ config.vm.hostname = hostname
+ config.vm.network "private_network", ip: ip
+end
+
+Vagrant.configure("2") do |config|
+ config.vm.box = "generic/debian10"
+
+ config.vm.provider "virtualbox" do |vb|
+ vb.gui = false
+ vb.memory = "512"
+ vb.customize ["modifyvm", :id, "--vram=12"]
+ end
+
+ config.vm.provision "shell", inline: <<-SHELL
+ echo "root:root" | chpasswd
+ mkdir -p /root/.ssh
+ echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy" >> /root/.ssh/authorized_keys
+ SHELL
+
+ config.vm.define "n1" do |config| vm(config, "n1", "192.168.56.21") end
+ config.vm.define "n2" do |config| vm(config, "n2", "192.168.56.22") end
+ config.vm.define "n3" do |config| vm(config, "n3", "192.168.56.23") end
+ config.vm.define "n4" do |config| vm(config, "n4", "192.168.56.24") end
+ config.vm.define "n5" do |config| vm(config, "n5", "192.168.56.25") end
+ config.vm.define "n6" do |config| vm(config, "n6", "192.168.56.26") end
+ config.vm.define "n7" do |config| vm(config, "n7", "192.168.56.27") end
+
+ config.vm.define "n8" do |config| vm(config, "n8", "192.168.56.28") end
+ config.vm.define "n9" do |config| vm(config, "n9", "192.168.56.29") end
+ config.vm.define "n10" do |config| vm(config, "n10", "192.168.56.30") end
+ config.vm.define "n11" do |config| vm(config, "n11", "192.168.56.31") end
+ config.vm.define "n12" do |config| vm(config, "n12", "192.168.56.32") end
+ config.vm.define "n13" do |config| vm(config, "n13", "192.168.56.33") end
+ config.vm.define "n14" do |config| vm(config, "n14", "192.168.56.34") end
+end
diff --git a/script/jepsen.garage/all_tests_1.sh b/script/jepsen.garage/all_tests_1.sh
new file mode 100755
index 00000000..b5397d13
--- /dev/null
+++ b/script/jepsen.garage/all_tests_1.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -x
+
+#for ppatch in task3c task3a tsfix2; do
+for ppatch in tsfix2; do
+ #for psc in c cp cdp r pr cpr dpr; do
+ for psc in cdp r pr cpr dpr; do
+ #for ptsk in reg2 set1 set2; do
+ for ptsk in set1; do
+ for irun in $(seq 10); do
+ lein run test --nodes-file nodes.vagrant \
+ --time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \
+ --workload $ptsk --patch $ppatch --scenario $psc
+ done
+ done
+ done
+done
diff --git a/script/jepsen.garage/all_tests_2.sh b/script/jepsen.garage/all_tests_2.sh
new file mode 100755
index 00000000..641643ed
--- /dev/null
+++ b/script/jepsen.garage/all_tests_2.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -x
+
+#for ppatch in task3c tsfix2; do
+for ppatch in tsfix2; do
+ for psc in cdp r pr cpr dpr; do
+ for ptsk in set1; do
+ for irun in $(seq 10); do
+ lein run test --nodes-file nodes2.vagrant \
+ --time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \
+ --workload $ptsk --patch $ppatch --scenario $psc
+ done
+ done
+ done
+done
diff --git a/script/jepsen.garage/jaeger.sh b/script/jepsen.garage/jaeger.sh
new file mode 100644
index 00000000..7f67b61b
--- /dev/null
+++ b/script/jepsen.garage/jaeger.sh
@@ -0,0 +1,13 @@
+docker stop jaeger
+docker rm jaeger
+
+# UI is on localhost:16686
+# otel-grpc collector is on localhost:4317
+# otel-http collector is on localhost:4318
+
+docker run -d --name jaeger \
+ -e COLLECTOR_OTLP_ENABLED=true \
+ -p 4317:4317 \
+ -p 4318:4318 \
+ -p 16686:16686 \
+ jaegertracing/all-in-one:1.50
diff --git a/script/jepsen.garage/nodes.vagrant b/script/jepsen.garage/nodes.vagrant
new file mode 100644
index 00000000..9e5694e6
--- /dev/null
+++ b/script/jepsen.garage/nodes.vagrant
@@ -0,0 +1,7 @@
+192.168.56.21
+192.168.56.22
+192.168.56.23
+192.168.56.24
+192.168.56.25
+192.168.56.26
+192.168.56.27
diff --git a/script/jepsen.garage/nodes2.vagrant b/script/jepsen.garage/nodes2.vagrant
new file mode 100644
index 00000000..842bf276
--- /dev/null
+++ b/script/jepsen.garage/nodes2.vagrant
@@ -0,0 +1,7 @@
+192.168.56.28
+192.168.56.29
+192.168.56.30
+192.168.56.31
+192.168.56.32
+192.168.56.33
+192.168.56.34
diff --git a/script/jepsen.garage/project.clj b/script/jepsen.garage/project.clj
new file mode 100644
index 00000000..59d45484
--- /dev/null
+++ b/script/jepsen.garage/project.clj
@@ -0,0 +1,10 @@
+(defproject jepsen.garage "0.1.0-SNAPSHOT"
+ :description "Jepsen testing for Garage"
+ :url "https://git.deuxfleurs.fr/Deuxfleurs/garage"
+ :license {:name "AGPLv3"
+ :url "https://www.gnu.org/licenses/agpl-3.0.en.html"}
+ :main jepsen.garage
+ :dependencies [[org.clojure/clojure "1.11.1"]
+ [jepsen "0.3.3-SNAPSHOT"]
+ [amazonica "0.3.163"]]
+ :repl-options {:init-ns jepsen.garage})
diff --git a/script/jepsen.garage/results/Results-2023-11-16.png b/script/jepsen.garage/results/Results-2023-11-16.png
new file mode 100644
index 00000000..26dac833
--- /dev/null
+++ b/script/jepsen.garage/results/Results-2023-11-16.png
Binary files differ
diff --git a/script/jepsen.garage/results/Results-2023-12-13-task3c.png b/script/jepsen.garage/results/Results-2023-12-13-task3c.png
new file mode 100644
index 00000000..216043c3
--- /dev/null
+++ b/script/jepsen.garage/results/Results-2023-12-13-task3c.png
Binary files differ
diff --git a/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png b/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png
new file mode 100644
index 00000000..147d25e9
--- /dev/null
+++ b/script/jepsen.garage/results/Results-2023-12-13-tsfix2.png
Binary files differ
diff --git a/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png b/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png
new file mode 100644
index 00000000..dbff3a95
--- /dev/null
+++ b/script/jepsen.garage/results/Results-2023-12-14-task3-set1.png
Binary files differ
diff --git a/script/jepsen.garage/shell.nix b/script/jepsen.garage/shell.nix
new file mode 100644
index 00000000..01e4c845
--- /dev/null
+++ b/script/jepsen.garage/shell.nix
@@ -0,0 +1,18 @@
+{ pkgs ? import <nixpkgs> {
+ overlays = [
+ (self: super: {
+ jdk = super.jdk11;
+ jre = super.jre11;
+ })
+ ];
+} }:
+pkgs.mkShell {
+ nativeBuildInputs = with pkgs; [
+ leiningen
+ jdk
+ jna
+ vagrant
+ gnuplot
+ graphviz
+ ];
+}
diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
new file mode 100644
index 00000000..446b81de
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -0,0 +1,105 @@
+(ns jepsen.garage
+ (:require
+ [clojure.string :as str]
+ [jepsen
+ [checker :as checker]
+ [cli :as cli]
+ [generator :as gen]
+ [nemesis :as nemesis]
+ [tests :as tests]]
+ [jepsen.os.debian :as debian]
+ [jepsen.garage
+ [daemon :as grg]
+ [nemesis :as grgNemesis]
+ [reg :as reg]
+ [set :as set]]))
+
+(def workloads
+ "A map of workload names to functions that construct workloads, given opts."
+ {"reg1" reg/workload1
+ "reg2" reg/workload2
+ "set1" set/workload1
+ "set2" set/workload2})
+
+(def scenari
+ "A map of scenari to the associated nemesis"
+ {"c" grgNemesis/scenario-c
+ "cp" grgNemesis/scenario-cp
+ "r" grgNemesis/scenario-r
+ "pr" grgNemesis/scenario-pr
+ "cpr" grgNemesis/scenario-cpr
+ "cdp" grgNemesis/scenario-cdp
+ "dpr" grgNemesis/scenario-dpr})
+
+(def patches
+ "A map of patch names to Garage builds"
+ {"default" "v0.9.0"
+ "tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"
+ "tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1"
+ "task3a" "707442f5de416fdbed4681a33b739f0a787b7834"
+ "task3b" "431b28e0cfdc9cac6c649193cf602108a8b02997"
+ "task3c" "0041b013a473e3ae72f50209d8f79db75a72848b"})
+
+(def cli-opts
+ "Additional command line options."
+ [["-p" "--patch NAME" "Garage patch to use"
+ :default "default"
+ :validate [patches (cli/one-of patches)]]
+ ["-s" "--scenario NAME" "Nemesis scenario to run"
+ :default "cp"
+ :validate [scenari (cli/one-of scenari)]]
+ ["-r" "--rate HZ" "Approximate number of requests per second, per thread."
+ :default 10
+ :parse-fn read-string
+ :validate [#(and (number? %) (pos? %)) "Must be a positive number"]]
+ [nil "--ops-per-key NUM" "Maximum number of operations on any given key."
+ :default 100
+ :parse-fn parse-long
+ :validate [pos? "Must be a positive integer."]]
+ ["-w" "--workload NAME" "Workload of test to run"
+ :default "reg1"
+ :validate [workloads (cli/one-of workloads)]]])
+
+(defn garage-test
+ "Given an options map from the command line runner (e.g. :nodes, :ssh,
+ :concurrency, ...), constructs a test map."
+ [opts]
+ (let [garage-version (get patches (:patch opts))
+ db (grg/db garage-version)
+ workload ((get workloads (:workload opts)) opts)
+ scenario ((get scenari (:scenario opts)) (assoc opts :db db))]
+ (merge tests/noop-test
+ opts
+ {:pure-generators true
+ :name (str "garage-" (name (:patch opts)) " " (name (:workload opts)) " " (name (:scenario opts)))
+ :os debian/os
+ :db db
+ :client (:client workload)
+ :generator (gen/phases
+ (->>
+ (:generator workload)
+ (gen/stagger (/ (:rate opts)))
+ (gen/nemesis (:generator scenario))
+ (gen/time-limit (:time-limit opts)))
+ (gen/log "Healing cluster")
+ (gen/nemesis (:final-generator scenario))
+ (gen/log "Waiting for recovery")
+ (gen/sleep 10)
+ (gen/log "Running final generator")
+ (gen/clients (:final-generator workload))
+ (gen/log "Generators all done"))
+ :nemesis (:nemesis scenario)
+ :checker (checker/compose
+ {:perf (checker/perf (:perf scenario))
+ :workload (:checker workload)})
+ })))
+
+
+(defn -main
+ "Handles command line arguments. Can either run a test, or a web server for
+ browsing results."
+ [& args]
+ (cli/run! (merge (cli/single-test-cmd {:test-fn garage-test
+ :opt-spec cli-opts})
+ (cli/serve-cmd))
+ args))
diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj
new file mode 100644
index 00000000..d407dd29
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj
@@ -0,0 +1,152 @@
+(ns jepsen.garage.daemon
+ (:require [clojure.tools.logging :refer :all]
+ [jepsen [control :as c]
+ [core :as jepsen]
+ [db :as db]]
+ [jepsen.control.util :as cu]))
+
+; CONSTANTS -- HOW GARAGE IS SET UP
+
+(def base-dir "/opt/garage")
+(def data-dir (str base-dir "/data"))
+(def meta-dir (str base-dir "/meta"))
+(def binary (str base-dir "/garage"))
+(def logfile (str base-dir "/garage.log"))
+(def pidfile (str base-dir "/garage.pid"))
+
+(def admin-token "icanhazadmin")
+(def access-key-id "GK8bfb6a51286071c6c9cd8bc3")
+(def secret-access-key "b0be95f71c1c6f16858a9edf395078b75c12ecb6b1c03385c4ae92076e4994a3")
+(def bucket-name "jepsen")
+
+; THE GARAGE DB
+
+(defn install!
+ "Download and install Garage"
+ [node version]
+ (c/su
+ (c/trace
+ (info node "installing garage" version)
+ (c/exec :mkdir :-p base-dir)
+ (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
+ cache (cu/cached-wget! url)]
+ (c/exec :cp cache binary))
+ (c/exec :chmod :+x binary))))
+
+(defn configure!
+ "Configure Garage"
+ [node]
+ (c/su
+ (c/trace
+ (cu/write-file!
+ (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
+ "rpc_bind_addr = \"0.0.0.0:3901\"\n"
+ "rpc_public_addr = \"" node ":3901\"\n"
+ "db_engine = \"lmdb\"\n"
+ "replication_mode = \"2\"\n"
+ "data_dir = \"" data-dir "\"\n"
+ "metadata_dir = \"" meta-dir "\"\n"
+ "[s3_api]\n"
+ "s3_region = \"us-east-1\"\n"
+ "api_bind_addr = \"0.0.0.0:3900\"\n"
+ "[k2v_api]\n"
+ "api_bind_addr = \"0.0.0.0:3902\"\n"
+ "[admin]\n"
+ "api_bind_addr = \"0.0.0.0:3903\"\n"
+ "admin_token = \"" admin-token "\"\n"
+ "trace_sink = \"http://192.168.56.1:4317\"\n")
+ "/etc/garage.toml"))))
+
+(defn connect-node!
+ "Connect a Garage node to the rest of the cluster"
+ [test node]
+ (c/trace
+ (let [node-id (c/exec binary :node :id :-q)]
+ (info node "node id:" node-id)
+ (c/on-many (:nodes test)
+ (c/exec binary :node :connect node-id)))))
+
+(defn configure-node!
+ "Configure a Garage node to be part of a cluster layout"
+ [test node]
+ (c/trace
+ (let [node-id (c/exec binary :node :id :-q)]
+ (c/on (jepsen/primary test)
+ (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)))))
+
+(defn finalize-config!
+ "Apply the layout and create a key/bucket pair in the cluster"
+ [node]
+ (c/trace
+ (c/exec binary :layout :apply :--version 1)
+ (info node "garage status:" (c/exec binary :status))
+ (c/exec binary :key :import access-key-id secret-access-key :--yes)
+ (c/exec binary :bucket :create bucket-name)
+ (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key-id)
+ (info node "key info: " (c/exec binary :key :info access-key-id))))
+
+(defn db
+ "Garage DB for a particular version"
+ [version]
+ (reify db/DB
+ (setup! [_ test node]
+ (install! node version)
+ (configure! node)
+ (cu/start-daemon!
+ {:logfile logfile
+ :pidfile pidfile
+ :chdir base-dir
+ :env {:RUST_LOG "garage=debug,garage_api=trace"}}
+ binary
+ :server)
+ (c/exec :sleep 3)
+
+ (jepsen/synchronize test)
+ (connect-node! test node)
+
+ (jepsen/synchronize test)
+ (configure-node! test node)
+
+ (jepsen/synchronize test)
+ (when (= node (jepsen/primary test))
+ (finalize-config! node)))
+
+ (teardown! [_ test node]
+ (info node "tearing down garage" version)
+ (c/su
+ (cu/stop-daemon! binary pidfile)
+ (c/exec :rm :-rf logfile)
+ (c/exec :rm :-rf data-dir)
+ (c/exec :rm :-rf meta-dir)))
+
+ db/Pause
+ (pause! [_ test node]
+ (cu/grepkill! :stop binary))
+ (resume! [_ test node]
+ (cu/grepkill! :cont binary))
+
+ db/Kill
+ (kill! [_ test node]
+ (cu/stop-daemon! binary pidfile))
+ (start! [_ test node]
+ (cu/start-daemon!
+ {:logfile logfile
+ :pidfile pidfile
+ :chdir base-dir
+ :env {:RUST_LOG "garage=debug,garage_api=trace"}}
+ binary
+ :server))
+
+ db/LogFiles
+ (log-files [_ test node]
+ [logfile])))
+
+(defn creds
+ "Obtain Garage credentials for node"
+ [node]
+ {:access-key access-key-id
+ :secret-key secret-access-key
+ :endpoint (str "http://" node ":3900")
+ :bucket bucket-name
+ :client-config {:path-style-access-enabled true}})
+
diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
new file mode 100644
index 00000000..dfce0255
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
@@ -0,0 +1,142 @@
+(ns jepsen.garage.nemesis
+ (:require [clojure.tools.logging :refer :all]
+ [jepsen [control :as c]
+ [core :as jepsen]
+ [generator :as gen]
+ [nemesis :as nemesis]]
+ [jepsen.nemesis.combined :as combined]
+ [jepsen.garage.daemon :as grg]
+ [jepsen.control.util :as cu]))
+
+; ---- reconfiguration nemesis ----
+
+(defn configure-present!
+ "Configure node to be active in new cluster layout"
+ [test nodes]
+ (info "configure-present!" nodes)
+ (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q))
+ node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)]
+ (c/on
+ (jepsen/primary test)
+ (apply c/exec (concat [grg/binary :layout :assign :-c :1G] node-id-strs)))))
+
+(defn configure-absent!
+ "Configure nodes to be active in new cluster layout"
+ [test nodes]
+ (info "configure-absent!" nodes)
+ (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q))
+ node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)]
+ (c/on
+ (jepsen/primary test)
+ (apply c/exec (concat [grg/binary :layout :assign :-g] node-id-strs)))))
+
+(defn finalize-config!
+ "Apply the proposed cluster layout"
+ [test]
+ (let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show))
+ [_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)]
+ (if layout-next-version
+ (do
+ (info "layout show: " layout-show "; next-version: " layout-next-version)
+ (c/on (jepsen/primary test)
+ (c/exec grg/binary :layout :apply :--version layout-next-version)))
+ (info "no layout changes to apply"))))
+
+(defn reconfigure-subset
+ "Reconfigure cluster with only a subset of nodes"
+ [cnt]
+ (reify nemesis/Nemesis
+ (setup! [this test] this)
+
+ (invoke! [this test op] op
+ (case (:f op)
+ :start
+ (let [[keep-nodes remove-nodes]
+ (->> (:nodes test)
+ shuffle
+ (split-at cnt))]
+ (info "layout split: keep " keep-nodes ", remove " remove-nodes)
+ (configure-present! test keep-nodes)
+ (configure-absent! test remove-nodes)
+ (finalize-config! test)
+ (assoc op :value keep-nodes))
+ :stop
+ (do
+ (info "layout un-split: all nodes=" (:nodes test))
+ (configure-present! test (:nodes test))
+ (finalize-config! test)
+ (assoc op :value (:nodes test)))))
+
+ (teardown! [this test] this)))
+
+; ---- nemesis scenari ----
+
+(defn nemesis-op
+ "A generator for a single nemesis operation"
+ [op]
+ (fn [_ _] {:type :info, :f op}))
+
+(defn reconfiguration-package
+ "Cluster reconfiguration nemesis package"
+ [opts]
+ {:generator (->>
+ (gen/mix [(nemesis-op :reconfigure-start)
+ (nemesis-op :reconfigure-stop)])
+ (gen/stagger (:interval opts 5)))
+ :final-generator {:type :info, :f :reconfigure-stop}
+ :nemesis (nemesis/compose
+ {{:reconfigure-start :start
+ :reconfigure-stop :stop} (reconfigure-subset 3)})
+ :perf #{{:name "reconfigure"
+ :start #{:reconfigure-start}
+ :stop #{:reconfigur-stop}
+ :color "#A197E9"}}})
+
+(defn scenario-c
+ "Clock modifying scenario"
+ [opts]
+ (combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}))
+
+(defn scenario-cp
+ "Clock modifying + partition scenario"
+ [opts]
+ (combined/compose-packages
+ [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
+ (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})]))
+
+(defn scenario-r
+ "Cluster reconfiguration scenario"
+ [opts]
+ (reconfiguration-package {:interval 1}))
+
+(defn scenario-pr
+ "Partition + cluster reconfiguration scenario"
+ [opts]
+ (combined/compose-packages
+ [(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
+ (reconfiguration-package {:interval 1})]))
+
+(defn scenario-cpr
+ "Clock scramble + partition + cluster reconfiguration scenario"
+ [opts]
+ (combined/compose-packages
+ [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
+ (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
+ (reconfiguration-package {:interval 1})]))
+
+(defn scenario-cdp
+ "Clock modifying + db + partition scenario"
+ [opts]
+ (combined/compose-packages
+ [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
+ (combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}})
+ (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})]))
+
+(defn scenario-dpr
+ "Db + partition + cluster reconfiguration scenario"
+ [opts]
+ (combined/compose-packages
+ [(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}})
+ (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
+ (reconfiguration-package {:interval 1})]))
+
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
new file mode 100644
index 00000000..39708c0b
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -0,0 +1,143 @@
+(ns jepsen.garage.reg
+ (:require [clojure.tools.logging :refer :all]
+ [clojure.string :as str]
+ [clojure.set :as set]
+ [jepsen [checker :as checker]
+ [cli :as cli]
+ [client :as client]
+ [control :as c]
+ [db :as db]
+ [generator :as gen]
+ [independent :as independent]
+ [nemesis :as nemesis]
+ [util :as util]
+ [tests :as tests]]
+ [jepsen.checker.timeline :as timeline]
+ [jepsen.control.util :as cu]
+ [jepsen.os.debian :as debian]
+ [jepsen.garage.daemon :as grg]
+ [jepsen.garage.s3api :as s3]
+ [knossos.model :as model]
+ [slingshot.slingshot :refer [try+]]))
+
+(defn op-get [_ _] {:type :invoke, :f :read, :value nil})
+(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 99))})
+(defn op-del [_ _] {:type :invoke, :f :write, :value nil})
+
+(defrecord RegClient [creds]
+ client/Client
+ (open! [this test node]
+ (assoc this :creds (grg/creds node)))
+ (setup! [this test])
+ (invoke! [this test op]
+ (try+
+ (let [[k v] (:value op)]
+ (case (:f op)
+ :read
+ (util/timeout
+ 10000
+ (assoc op :type :fail, :error ::timeout)
+ (let [value (s3/get (:creds this) k)]
+ (assoc op :type :ok, :value (independent/tuple k value))))
+ :write
+ (util/timeout
+ 10000
+ (assoc op :type :info, :error ::timeout)
+ (do
+ (s3/put (:creds this) k v)
+ (assoc op :type :ok)))))
+ (catch (re-find #"Unavailable" (.getMessage %)) ex
+ (assoc op :type :info, :error ::unavailable))
+ (catch (re-find #"Broken pipe" (.getMessage %)) ex
+ (assoc op :type :info, :error ::broken-pipe))
+ (catch (re-find #"Connection refused" (.getMessage %)) ex
+ (assoc op :type :info, :error ::connection-refused))))
+ (teardown! [this test])
+ (close! [this test]))
+
+(defn reg-read-after-write
+ "Read-after-Write checker for register operations"
+ []
+ (reify checker/Checker
+ (check [this test history opts]
+ (let [init {:put-values {-1 nil}
+ :put-done #{-1}
+ :put-in-progress {}
+ :read-can-contain {}
+ :bad-reads #{}}
+ final (reduce
+ (fn [state op]
+ (let [current-values (set/union
+ (set (map (fn [idx] (get (:put-values state) idx)) (:put-done state)))
+ (set (map (fn [[_ [idx _]]] (get (:put-values state) idx)) (:put-in-progress state))))
+ read-can-contain (reduce
+ (fn [rcc [idx v]] (assoc rcc idx (set/union current-values v)))
+ {} (:read-can-contain state))]
+ (info "--------")
+ (info "state: " state)
+ (info "current-values: " current-values)
+ (info "read-can-contain: " read-can-contain)
+ (info "op: " op)
+ (case [(:type op) (:f op)]
+ ([:invoke :write])
+ (assoc state
+ :read-can-contain read-can-contain
+ :put-values (assoc (:put-values state) (:index op) (:value op))
+ :put-in-progress (assoc (:put-in-progress state) (:process op) [(:index op) (:put-done state)]))
+ ([:ok :write])
+ (let [[index overwrites] (get (:put-in-progress state) (:process op))]
+ (assoc state
+ :read-can-contain read-can-contain
+ :put-in-progress (dissoc (:put-in-progress state) (:process op))
+ :put-done
+ (conj
+ (set/difference (:put-done state) overwrites)
+ index)))
+ ([:invoke :read])
+ (assoc state
+ :read-can-contain (assoc read-can-contain (:process op) current-values))
+ ([:ok :read])
+ (let [this-read-can-contain (get read-can-contain (:process op))
+ bad-reads (if (contains? this-read-can-contain (:value op))
+ (:bad-reads state)
+ (conj (:bad-reads state) [(:process op) (:index op) (:value op) this-read-can-contain]))]
+ (info "this-read-can-contain: " this-read-can-contain)
+ (assoc state
+ :read-can-contain (dissoc read-can-contain (:process op))
+ :bad-reads bad-reads))
+ state)))
+ init history)
+ valid? (empty? (:bad-reads final))]
+ (assoc final :valid? valid?)))))
+
+(defn workload-common
+ "Common parts of workload"
+ [opts]
+ {:client (RegClient. nil)
+ :generator (independent/concurrent-generator
+ 10
+ (range)
+ (fn [k]
+ (->>
+ (gen/mix [op-get op-put op-del])
+ (gen/limit (:ops-per-key opts)))))})
+
+(defn workload1
+ "Tests linearizable reads and writes"
+ [opts]
+ (assoc (workload-common opts)
+ :checker (independent/checker
+ (checker/compose
+ {:linear (checker/linearizable
+ {:model (model/register)
+ :algorithm :linear})
+ :timeline (timeline/html)}))))
+
+(defn workload2
+ "Tests CRDT reads and writes"
+ [opts]
+ (assoc (workload-common opts)
+ :checker (independent/checker
+ (checker/compose
+ {:reg-read-after-write (reg-read-after-write)
+ :timeline (timeline/html)}))))
diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj
new file mode 100644
index 00000000..531e0157
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj
@@ -0,0 +1,48 @@
+(ns jepsen.garage.s3api
+ (:require [clojure.tools.logging :refer :all]
+ [jepsen [control :as c]]
+ [amazonica.aws.s3 :as s3]
+ [slingshot.slingshot :refer [try+]]))
+
+; GARAGE S3 HELPER FUNCTIONS
+
+(defn get
+ "Helper for GetObject"
+ [creds k]
+ (try+
+ (-> (s3/get-object creds (:bucket creds) k)
+ :input-stream
+ slurp)
+ (catch (re-find #"Key not found" (.getMessage %)) ex
+ nil)))
+
+(defn put
+ "Helper for PutObject or DeleteObject (is a delete if value is nil)"
+ [creds k v]
+ (if (= v nil)
+ (s3/delete-object creds
+ :bucket-name (:bucket creds)
+ :key k)
+ (let [some-bytes (.getBytes v "UTF-8")
+ bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
+ (s3/put-object creds
+ :bucket-name (:bucket creds)
+ :key k
+ :input-stream bytes-stream
+ :metadata {:content-length (count some-bytes)}))))
+
+(defn list-inner [creds prefix ct accum]
+ (let [list-result (s3/list-objects-v2 creds
+ {:bucket-name (:bucket creds)
+ :prefix prefix
+ :continuation-token ct})
+ new-object-summaries (:object-summaries list-result)
+ new-objects (map (fn [d] (:key d)) new-object-summaries)
+ objects (concat new-objects accum)]
+ (if (:truncated? list-result)
+ (list-inner creds prefix (:next-continuation-token list-result) objects)
+ objects)))
+(defn list
+ "Helper for ListObjects -- just lists everything in the bucket"
+ [creds prefix]
+ (list-inner creds prefix nil []))
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
new file mode 100644
index 00000000..2c7a2ccd
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -0,0 +1,135 @@
+(ns jepsen.garage.set
+ (:require [clojure.tools.logging :refer :all]
+ [clojure.string :as str]
+ [clojure.set :as set]
+ [jepsen [checker :as checker]
+ [cli :as cli]
+ [client :as client]
+ [control :as c]
+ [checker :as checker]
+ [db :as db]
+ [generator :as gen]
+ [independent :as independent]
+ [nemesis :as nemesis]
+ [util :as util]
+ [tests :as tests]]
+ [jepsen.checker.timeline :as timeline]
+ [jepsen.control.util :as cu]
+ [jepsen.os.debian :as debian]
+ [jepsen.garage.daemon :as grg]
+ [jepsen.garage.s3api :as s3]
+ [knossos.model :as model]
+ [slingshot.slingshot :refer [try+]]))
+
+(defn op-add-rand100 [_ _] {:type :invoke, :f :add, :value (rand-int 100)})
+(defn op-read [_ _] {:type :invoke, :f :read, :value nil})
+
+(defrecord SetClient [creds]
+ client/Client
+ (open! [this test node]
+ (assoc this :creds (grg/creds node)))
+ (setup! [this test])
+ (invoke! [this test op]
+ (try+
+ (let [[k v] (:value op)
+ prefix (str "set" k "/")]
+ (case (:f op)
+ :add
+ (util/timeout
+ 10000
+ (assoc op :type :info, :error ::timeout)
+ (do
+ (s3/put (:creds this) (str prefix v) "present")
+ (assoc op :type :ok)))
+ :read
+ (util/timeout
+ 10000
+ (assoc op :type :fail, :error ::timeout)
+ (do
+ (let [items (s3/list (:creds this) prefix)]
+ (let [items-stripped (map (fn [o]
+ (assert (str/starts-with? o prefix))
+ (str/replace-first o prefix "")) items)
+ items-set (set (map parse-long items-stripped))]
+ (assoc op :type :ok, :value (independent/tuple k items-set))))))))
+ (catch (re-find #"Unavailable" (.getMessage %)) ex
+ (assoc op :type :info, :error ::unavailable))
+ (catch (re-find #"Broken pipe" (.getMessage %)) ex
+ (assoc op :type :info, :error ::broken-pipe))
+ (catch (re-find #"Connection refused" (.getMessage %)) ex
+ (assoc op :type :info, :error ::connection-refused))))
+ (teardown! [this test])
+ (close! [this test]))
+
+(defn set-read-after-write
+ "Read-after-Write checker for set operations"
+ []
+ (reify checker/Checker
+ (check [this test history opts]
+ (let [init {:add-started #{}
+ :add-done #{}
+ :read-must-contain {}
+ :missed #{}
+ :unexpected #{}}
+ final (reduce
+ (fn [state op]
+ (case [(:type op) (:f op)]
+ ([:invoke :add])
+ (assoc state :add-started (conj (:add-started state) (:value op)))
+ ([:ok :add])
+ (assoc state :add-done (conj (:add-done state) (:value op)))
+ ([:invoke :read])
+ (assoc-in state [:read-must-contain (:process op)] (:add-done state))
+ ([:ok :read])
+ (let [read-must-contain (get (:read-must-contain state) (:process op))
+ new-missed (set/difference read-must-contain (:value op))
+ new-unexpected (set/difference (:value op) (:add-started state))]
+ (assoc state
+ :read-must-contain (dissoc (:read-must-contain state) (:process op))
+ :missed (set/union (:missed state) new-missed),
+ :unexpected (set/union (:unexpected state) new-unexpected)))
+ state))
+ init history)
+ valid? (and (empty? (:missed final)) (empty? (:unexpected final)))]
+ (assoc final :valid? valid?)))))
+
+(defn workload1
+ "Tests insertions and deletions"
+ [opts]
+ {:client (SetClient. nil)
+ :checker (independent/checker
+ (checker/compose
+ {:set (checker/set)
+ :timeline (timeline/html)}))
+ :generator (independent/concurrent-generator
+ 10
+ (range 100)
+ (fn [k]
+ (->> (range)
+ (map (fn [x] {:type :invoke, :f :add, :value x}))
+ (gen/limit (:ops-per-key opts)))))
+ :final-generator (independent/concurrent-generator
+ 10
+ (range 100)
+ (fn [k]
+ (gen/phases
+ (gen/once op-read)
+ (gen/sleep 5))))})
+
+(defn workload2
+ "Tests insertions and deletions"
+ [opts]
+ {:client (SetClient. nil)
+ :checker (independent/checker
+ (checker/compose
+ {:set-read-after-write (set-read-after-write)
+ ; :set-full (checker/set-full {:linearizable? false})
+ :timeline (timeline/html)}))
+ :generator (independent/concurrent-generator
+ 10
+ (range)
+ (fn [k]
+ (->> (gen/mix [op-add-rand100 op-read])
+ (gen/limit (:ops-per-key opts)))))})
+
+
diff --git a/script/jepsen.garage/test/jepsen/garage_test.clj b/script/jepsen.garage/test/jepsen/garage_test.clj
new file mode 100644
index 00000000..055392a1
--- /dev/null
+++ b/script/jepsen.garage/test/jepsen/garage_test.clj
@@ -0,0 +1,7 @@
+(ns jepsen.garage-test
+ (:require [clojure.test :refer :all]
+ [jepsen.garage :refer :all]))
+
+(deftest a-test
+ (testing "FIXME, I fail."
+ (is (= 0 1))))
diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml
index e8cbc1c8..5a667359 100644
--- a/src/api/Cargo.toml
+++ b/src/api/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "garage_api"
-version = "0.9.0"
+version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs
index 542b7a81..f9eccb7f 100644
--- a/src/api/s3/post_object.rs
+++ b/src/api/s3/post_object.rs
@@ -15,6 +15,7 @@ use serde::Deserialize;
use garage_model::garage::Garage;
+use crate::s3::cors::*;
use crate::s3::error::*;
use crate::s3::put::{get_headers, save_stream};
use crate::s3::xml as s3_xml;
@@ -242,7 +243,7 @@ pub async fn handle_post_object(
let etag = format!("\"{}\"", md5);
- let resp = if let Some(mut target) = params
+ let mut resp = if let Some(mut target) = params
.get("success_action_redirect")
.and_then(|h| h.to_str().ok())
.and_then(|u| url::Url::parse(u).ok())
@@ -262,8 +263,7 @@ pub async fn handle_post_object(
} else {
let path = head
.uri
- .into_parts()
- .path_and_query
+ .path_and_query()
.map(|paq| paq.path().to_string())
.unwrap_or_else(|| "/".to_string());
let authority = head
@@ -308,6 +308,13 @@ pub async fn handle_post_object(
}
};
+ let matching_cors_rule =
+ find_matching_cors_rule(&bucket, &Request::from_parts(head, Body::empty()))?;
+ if let Some(rule) = matching_cors_rule {
+ add_cors_headers(&mut resp, rule)
+ .ok_or_internal_error("Invalid bucket CORS configuration")?;
+ }
+
Ok(resp)
}
diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml
index f6aa5f64..d9bd1ac0 100644
--- a/src/block/Cargo.toml
+++ b/src/block/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "garage_block"
-version = "0.9.0"
+version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml
index 67af4a7c..470135db 100644
--- a/src/db/Cargo.toml
+++ b/src/db/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "garage_db"
-version = "0.9.0"
+version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml
index 7c3a79cb..35edd30c 100644
--- a/src/garage/Cargo.toml
+++ b/src/garage/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "garage"
-version = "0.9.0"
+version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -67,6 +67,7 @@ chrono = "0.4"
http = "0.2"
hmac = "0.12"
hyper = { version = "0.14", features = ["client", "http1", "runtime"] }
+mktemp = "0.5"
sha2 = "0.10"
static_init = "1.0"
diff --git a/src/garage/cli/init.rs b/src/garage/cli/init.rs
index 20813f1c..43ca5c09 100644
--- a/src/garage/cli/init.rs
+++ b/src/garage/cli/init.rs
@@ -43,7 +43,7 @@ pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
idstr
);
eprintln!(
- "where <remote_node> is their own node identifier in the format: <pubkey>@<ip>:<port>"
+ "where <remote_node> is their own node identifier in the format: <full-node-id>@<ip>:<port>"
);
eprintln!();
eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:");
diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs
index aba57551..be4d5bd6 100644
--- a/src/garage/cli/structs.rs
+++ b/src/garage/cli/structs.rs
@@ -64,7 +64,8 @@ pub enum Command {
#[derive(StructOpt, Debug)]
pub enum NodeOperation {
- /// Print identifier (public key) of this Garage node
+ /// Print the full node ID (public key) of this Garage node, and its publicly reachable IP
+ /// address and port if they are specified in config file under `rpc_public_addr`
#[structopt(name = "id", version = garage_version())]
NodeId(NodeIdOpt),
@@ -82,8 +83,9 @@ pub struct NodeIdOpt {
#[derive(StructOpt, Debug)]
pub struct ConnectNodeOpt {
- /// Node public key and address, in the format:
- /// `<public key hexadecimal>@<ip or hostname>:<port>`
+ /// Full node ID (public key) and IP address and port, in the format:
+ /// `<full node ID>@<ip or hostname>:<port>`.
+ /// You can retrieve this information on the target node using `garage node id`.
pub(crate) node: String,
}
diff --git a/src/garage/main.rs b/src/garage/main.rs
index 66403d05..5c92dae4 100644
--- a/src/garage/main.rs
+++ b/src/garage/main.rs
@@ -7,6 +7,7 @@ extern crate tracing;
mod admin;
mod cli;
mod repair;
+mod secrets;
mod server;
#[cfg(feature = "telemetry-otlp")]
mod tracing_setup;
@@ -28,7 +29,6 @@ use structopt::StructOpt;
use netapp::util::parse_and_resolve_peer_addr;
use netapp::NetworkKey;
-use garage_util::config::Config;
use garage_util::error::*;
use garage_rpc::system::*;
@@ -38,6 +38,7 @@ use garage_model::helper::error::Error as HelperError;
use admin::*;
use cli::*;
+use secrets::Secrets;
#[derive(StructOpt, Debug)]
#[structopt(
@@ -45,8 +46,7 @@ use cli::*;
about = "S3-compatible object store for self-hosted geo-distributed deployments"
)]
struct Opt {
- /// Host to connect to for admin operations, in the format:
- /// <public-key>@<ip>:<port>
+ /// Host to connect to for admin operations, in the format: <full-node-id>@<ip>:<port>
#[structopt(short = "h", long = "rpc-host", env = "GARAGE_RPC_HOST")]
pub rpc_host: Option<String>,
@@ -66,24 +66,6 @@ struct Opt {
cmd: Command,
}
-#[derive(StructOpt, Debug)]
-pub struct Secrets {
- /// RPC secret network key, used to replace rpc_secret in config.toml when running the
- /// daemon or doing admin operations
- #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
- pub rpc_secret: Option<String>,
-
- /// Metrics API authentication token, replaces admin.metrics_token in config.toml when
- /// running the Garage daemon
- #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
- pub admin_token: Option<String>,
-
- /// Metrics API authentication token, replaces admin.metrics_token in config.toml when
- /// running the Garage daemon
- #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
- pub metrics_token: Option<String>,
-}
-
#[tokio::main]
async fn main() {
// Initialize version and features info
@@ -218,7 +200,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
// Find and parse the address of the target host
let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host {
- let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <pubkey>@<IP or hostname>:<port>.", h))?;
+ let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <full node id>@<IP or hostname>:<port>.", h))?;
(id, addrs[0], false)
} else {
let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir)
@@ -248,7 +230,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
addr
);
}
- Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
+ Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct full-length node ID (public key).")?;
}
let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
@@ -261,16 +243,3 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
Ok(x) => Ok(x),
}
}
-
-fn fill_secrets(mut config: Config, secrets: Secrets) -> Config {
- if secrets.rpc_secret.is_some() {
- config.rpc_secret = secrets.rpc_secret;
- }
- if secrets.admin_token.is_some() {
- config.admin.admin_token = secrets.admin_token;
- }
- if secrets.metrics_token.is_some() {
- config.admin.metrics_token = secrets.metrics_token;
- }
- config
-}
diff --git a/src/garage/repair/offline.rs b/src/garage/repair/offline.rs
index f4edcf03..45024e71 100644
--- a/src/garage/repair/offline.rs
+++ b/src/garage/repair/offline.rs
@@ -6,7 +6,7 @@ use garage_util::error::*;
use garage_model::garage::Garage;
use crate::cli::structs::*;
-use crate::{fill_secrets, Secrets};
+use crate::secrets::{fill_secrets, Secrets};
pub async fn offline_repair(
config_file: PathBuf,
@@ -20,7 +20,7 @@ pub async fn offline_repair(
}
info!("Loading configuration...");
- let config = fill_secrets(read_config(config_file)?, secrets);
+ let config = fill_secrets(read_config(config_file)?, secrets)?;
info!("Initializing Garage main data store...");
let garage = Garage::new(config)?;
diff --git a/src/garage/secrets.rs b/src/garage/secrets.rs
new file mode 100644
index 00000000..e96be9e4
--- /dev/null
+++ b/src/garage/secrets.rs
@@ -0,0 +1,318 @@
+use structopt::StructOpt;
+
+use garage_util::config::Config;
+use garage_util::error::Error;
+
+/// Structure for secret values or paths that are passed as CLI arguments or environment
+/// variables, instead of in the config file.
+#[derive(StructOpt, Debug, Default, Clone)]
+pub struct Secrets {
+ /// Skip permission check on files containing secrets
+ #[cfg(unix)]
+ #[structopt(
+ long = "allow-world-readable-secrets",
+ env = "GARAGE_ALLOW_WORLD_READABLE_SECRETS"
+ )]
+ pub allow_world_readable_secrets: Option<bool>,
+
+ /// RPC secret network key, used to replace rpc_secret in config.toml when running the
+ /// daemon or doing admin operations
+ #[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
+ pub rpc_secret: Option<String>,
+
+ /// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret
+ /// when running the daemon or doing admin operations
+ #[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")]
+ pub rpc_secret_file: Option<String>,
+
+ /// Admin API authentication token, replaces admin.admin_token in config.toml when
+ /// running the Garage daemon
+ #[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
+ pub admin_token: Option<String>,
+
+ /// Admin API authentication token file path, replaces admin.admin_token in config.toml
+ /// and admin-token when running the Garage daemon
+ #[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")]
+ pub admin_token_file: Option<String>,
+
+ /// Metrics API authentication token, replaces admin.metrics_token in config.toml when
+ /// running the Garage daemon
+ #[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
+ pub metrics_token: Option<String>,
+
+ /// Metrics API authentication token file path, replaces admin.metrics_token in config.toml
+ /// and metrics-token when running the Garage daemon
+ #[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")]
+ pub metrics_token_file: Option<String>,
+}
+
+/// Single function to fill all secrets in the Config struct from their correct source (value
+/// from config or CLI param or env variable or read from a file specified in config or CLI
+/// param or env variable)
+pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result<Config, Error> {
+ let allow_world_readable = secrets
+ .allow_world_readable_secrets
+ .unwrap_or(config.allow_world_readable_secrets);
+
+ fill_secret(
+ &mut config.rpc_secret,
+ &config.rpc_secret_file,
+ &secrets.rpc_secret,
+ &secrets.rpc_secret_file,
+ "rpc_secret",
+ allow_world_readable,
+ )?;
+
+ fill_secret(
+ &mut config.admin.admin_token,
+ &config.admin.admin_token_file,
+ &secrets.admin_token,
+ &secrets.admin_token_file,
+ "admin.admin_token",
+ allow_world_readable,
+ )?;
+ fill_secret(
+ &mut config.admin.metrics_token,
+ &config.admin.metrics_token_file,
+ &secrets.metrics_token,
+ &secrets.metrics_token_file,
+ "admin.metrics_token",
+ allow_world_readable,
+ )?;
+
+ Ok(config)
+}
+
+fn fill_secret(
+ config_secret: &mut Option<String>,
+ config_secret_file: &Option<String>,
+ cli_secret: &Option<String>,
+ cli_secret_file: &Option<String>,
+ name: &'static str,
+ allow_world_readable: bool,
+) -> Result<(), Error> {
+ let cli_value = match (&cli_secret, &cli_secret_file) {
+ (Some(_), Some(_)) => {
+ return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
+ }
+ (Some(secret), None) => Some(secret.to_string()),
+ (None, Some(file)) => Some(read_secret_file(file, allow_world_readable)?),
+ (None, None) => None,
+ };
+
+ if let Some(val) = cli_value {
+ if config_secret.is_some() || config_secret_file.is_some() {
+ debug!("Overriding secret `{}` using value specified using CLI argument or environnement variable.", name);
+ }
+
+ *config_secret = Some(val);
+ } else if let Some(file_path) = &config_secret_file {
+ if config_secret.is_some() {
+ return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
+ }
+
+ *config_secret = Some(read_secret_file(file_path, allow_world_readable)?);
+ }
+
+ Ok(())
+}
+
+fn read_secret_file(file_path: &String, allow_world_readable: bool) -> Result<String, Error> {
+ if !allow_world_readable {
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::MetadataExt;
+ let metadata = std::fs::metadata(file_path)?;
+ if metadata.mode() & 0o077 != 0 {
+ return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
+ }
+ }
+ }
+
+ let secret_buf = std::fs::read_to_string(file_path)?;
+
+ // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
+ // also editors sometimes add a trailing newline
+ Ok(String::from(secret_buf.trim_end()))
+}
+
+#[cfg(test)]
+mod tests {
+ use std::fs::File;
+ use std::io::Write;
+
+ use garage_util::config::read_config;
+ use garage_util::error::Error;
+
+ use super::*;
+
+ #[test]
+ fn test_rpc_secret_file_works() -> Result<(), Error> {
+ let path_secret = mktemp::Temp::new_file()?;
+ let mut file_secret = File::create(path_secret.as_path())?;
+ writeln!(file_secret, "foo")?;
+ drop(file_secret);
+
+ let path_config = mktemp::Temp::new_file()?;
+ let mut file_config = File::create(path_config.as_path())?;
+ let path_secret_path = path_secret.as_path();
+ writeln!(
+ file_config,
+ r#"
+ metadata_dir = "/tmp/garage/meta"
+ data_dir = "/tmp/garage/data"
+ replication_mode = "3"
+ rpc_bind_addr = "[::]:3901"
+ rpc_secret_file = "{}"
+
+ [s3_api]
+ s3_region = "garage"
+ api_bind_addr = "[::]:3900"
+ "#,
+ path_secret_path.display()
+ )?;
+ drop(file_config);
+
+ // Second configuration file, same as previous one
+ // except it allows world-readable secrets.
+ let path_config_allow_world_readable = mktemp::Temp::new_file()?;
+ let mut file_config_allow_world_readable =
+ File::create(path_config_allow_world_readable.as_path())?;
+ writeln!(
+ file_config_allow_world_readable,
+ r#"
+ metadata_dir = "/tmp/garage/meta"
+ data_dir = "/tmp/garage/data"
+ replication_mode = "3"
+ rpc_bind_addr = "[::]:3901"
+ rpc_secret_file = "{}"
+ allow_world_readable_secrets = true
+
+ [s3_api]
+ s3_region = "garage"
+ api_bind_addr = "[::]:3900"
+ "#,
+ path_secret_path.display()
+ )?;
+ drop(file_config_allow_world_readable);
+
+ let config = read_config(path_config.to_path_buf())?;
+ let config = fill_secrets(config, Secrets::default())?;
+ assert_eq!("foo", config.rpc_secret.unwrap());
+
+ // ---- Check non world-readable secrets config ----
+ #[cfg(unix)]
+ {
+ let secrets_allow_world_readable = Secrets {
+ allow_world_readable_secrets: Some(true),
+ ..Default::default()
+ };
+ let secrets_no_allow_world_readable = Secrets {
+ allow_world_readable_secrets: Some(false),
+ ..Default::default()
+ };
+
+ use std::os::unix::fs::PermissionsExt;
+ let metadata = std::fs::metadata(&path_secret_path)?;
+ let mut perm = metadata.permissions();
+ perm.set_mode(0o660);
+ std::fs::set_permissions(&path_secret_path, perm)?;
+
+ // Config file that just specifies the path
+ let config = read_config(path_config.to_path_buf())?;
+ assert!(fill_secrets(config, Secrets::default()).is_err());
+
+ let config = read_config(path_config.to_path_buf())?;
+ assert!(fill_secrets(config, secrets_allow_world_readable.clone()).is_ok());
+
+ let config = read_config(path_config.to_path_buf())?;
+ assert!(fill_secrets(config, secrets_no_allow_world_readable.clone()).is_err());
+
+ // Config file that also specifies to allow world_readable_secrets
+ let config = read_config(path_config_allow_world_readable.to_path_buf())?;
+ assert!(fill_secrets(config, Secrets::default()).is_ok());
+
+ let config = read_config(path_config_allow_world_readable.to_path_buf())?;
+ assert!(fill_secrets(config, secrets_allow_world_readable).is_ok());
+
+ let config = read_config(path_config_allow_world_readable.to_path_buf())?;
+ assert!(fill_secrets(config, secrets_no_allow_world_readable).is_err());
+ }
+
+ // ---- Check alternative secrets specified on CLI ----
+
+ let path_secret2 = mktemp::Temp::new_file()?;
+ let mut file_secret2 = File::create(path_secret2.as_path())?;
+ writeln!(file_secret2, "bar")?;
+ drop(file_secret2);
+
+ let config = read_config(path_config.to_path_buf())?;
+ let config = fill_secrets(
+ config,
+ Secrets {
+ rpc_secret: Some("baz".into()),
+ ..Default::default()
+ },
+ )?;
+ assert_eq!(config.rpc_secret.as_deref(), Some("baz"));
+
+ let config = read_config(path_config.to_path_buf())?;
+ let config = fill_secrets(
+ config,
+ Secrets {
+ rpc_secret_file: Some(path_secret2.display().to_string()),
+ ..Default::default()
+ },
+ )?;
+ assert_eq!(config.rpc_secret.as_deref(), Some("bar"));
+
+ let config = read_config(path_config.to_path_buf())?;
+ assert!(fill_secrets(
+ config,
+ Secrets {
+ rpc_secret: Some("baz".into()),
+ rpc_secret_file: Some(path_secret2.display().to_string()),
+ ..Default::default()
+ }
+ )
+ .is_err());
+
+ drop(path_secret);
+ drop(path_secret2);
+ drop(path_config);
+ drop(path_config_allow_world_readable);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> {
+ let path_config = mktemp::Temp::new_file()?;
+ let mut file_config = File::create(path_config.as_path())?;
+ writeln!(
+ file_config,
+ r#"
+ metadata_dir = "/tmp/garage/meta"
+ data_dir = "/tmp/garage/data"
+ replication_mode = "3"
+ rpc_bind_addr = "[::]:3901"
+ rpc_secret= "dummy"
+ rpc_secret_file = "dummy"
+
+ [s3_api]
+ s3_region = "garage"
+ api_bind_addr = "[::]:3900"
+ "#
+ )?;
+ let config = read_config(path_config.to_path_buf())?;
+ assert_eq!(
+ "only one of `rpc_secret` and `rpc_secret_file` can be set",
+ fill_secrets(config, Secrets::default())
+ .unwrap_err()
+ .to_string()
+ );
+ drop(path_config);
+ drop(file_config);
+ Ok(())
+ }
+}
diff --git a/src/garage/server.rs b/src/garage/server.rs
index 3ad10b72..ac76a44d 100644
--- a/src/garage/server.rs
+++ b/src/garage/server.rs
@@ -15,9 +15,9 @@ use garage_web::WebServer;
use garage_api::k2v::api_server::K2VApiServer;
use crate::admin::*;
+use crate::secrets::{fill_secrets, Secrets};
#[cfg(feature = "telemetry-otlp")]
use crate::tracing_setup::*;
-use crate::{fill_secrets, Secrets};
async fn wait_from(mut chan: watch::Receiver<bool>) {
while !*chan.borrow() {
@@ -29,12 +29,19 @@ async fn wait_from(mut chan: watch::Receiver<bool>) {
pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Error> {
info!("Loading configuration...");
- let config = fill_secrets(read_config(config_file)?, secrets);
+ let config = fill_secrets(read_config(config_file)?, secrets)?;
// ---- Initialize Garage internals ----
#[cfg(feature = "metrics")]
- let metrics_exporter = opentelemetry_prometheus::exporter().init();
+ let metrics_exporter = opentelemetry_prometheus::exporter()
+ .with_default_summary_quantiles(vec![0.25, 0.5, 0.75, 0.9, 0.95, 0.99])
+ .with_default_histogram_boundaries(vec![
+ 0.001, 0.0015, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.02, 0.03, 0.05, 0.07, 0.1,
+ 0.15, 0.2, 0.3, 0.5, 0.7, 1., 1.5, 2., 3., 5., 7., 10., 15., 20., 30., 40., 50., 60.,
+ 70., 100.,
+ ])
+ .init();
info!("Initializing Garage main data store...");
let garage = Garage::new(config.clone())?;
diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml
index 42b7ffdb..8ecd5d63 100644
--- a/src/model/Cargo.toml
+++ b/src/model/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "garage_model"
-version = "0.9.0"
+version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml
index f450718f..0ca72016 100644
--- a/src/rpc/Cargo.toml
+++ b/src/rpc/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "garage_rpc"
-version = "0.9.0"
+version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs
index ab8d1112..71fd71b0 100644
--- a/src/rpc/consul.rs
+++ b/src/rpc/consul.rs
@@ -148,7 +148,7 @@ impl ConsulDiscovery {
ret.push((pubkey, SocketAddr::new(ip, ent.service_port)));
} else {
warn!(
- "Could not process node spec from Consul: {:?} (invalid IP or public key)",
+ "Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)",
ent
);
}
diff --git a/src/rpc/system.rs b/src/rpc/system.rs
index 4b40bec4..4cec369b 100644
--- a/src/rpc/system.rs
+++ b/src/rpc/system.rs
@@ -57,7 +57,7 @@ pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc";
pub enum SystemRpc {
/// Response to successfull advertisements
Ok,
- /// Request to connect to a specific node (in <pubkey>@<host>:<port> format)
+ /// Request to connect to a specific node (in <pubkey>@<host>:<port> format, pubkey = full-length node ID)
Connect(String),
/// Ask other node its cluster layout. Answered with AdvertiseClusterLayout
PullClusterLayout,
diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml
index 08ccec72..f40787d1 100644
--- a/src/table/Cargo.toml
+++ b/src/table/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "garage_table"
-version = "0.9.0"
+version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml
index 6554ac13..85317ec1 100644
--- a/src/util/Cargo.toml
+++ b/src/util/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "garage_util"
-version = "0.9.0"
+version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
diff --git a/src/util/config.rs b/src/util/config.rs
index ad5c8e1f..65c0b5c0 100644
--- a/src/util/config.rs
+++ b/src/util/config.rs
@@ -1,6 +1,5 @@
//! Contains type and functions related to Garage configuration file
use std::convert::TryFrom;
-use std::io::Read;
use std::net::SocketAddr;
use std::path::PathBuf;
@@ -45,11 +44,15 @@ pub struct Config {
)]
pub compression_level: Option<i32>,
+ /// Skip the permission check of secret files. Useful when
+ /// POSIX ACLs (or more complex chmods) are used.
+ #[serde(default)]
+ pub allow_world_readable_secrets: bool,
+
/// RPC secret key: 32 bytes hex encoded
pub rpc_secret: Option<String>,
/// Optional file where RPC secret key is read from
pub rpc_secret_file: Option<String>,
-
/// Address to bind for RPC
pub rpc_bind_addr: SocketAddr,
/// Public IP address of this node
@@ -221,6 +224,13 @@ pub struct KubernetesDiscoveryConfig {
pub skip_crd: bool,
}
+/// Read and parse configuration
+pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
+ let config = std::fs::read_to_string(config_file)?;
+
+ Ok(toml::from_str(&config)?)
+}
+
fn default_db_engine() -> String {
"lmdb".into()
}
@@ -235,68 +245,6 @@ fn default_block_size() -> usize {
1048576
}
-/// Read and parse configuration
-pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
- let mut file = std::fs::OpenOptions::new()
- .read(true)
- .open(config_file.as_path())?;
-
- let mut config = String::new();
- file.read_to_string(&mut config)?;
-
- let mut parsed_config: Config = toml::from_str(&config)?;
-
- secret_from_file(
- &mut parsed_config.rpc_secret,
- &parsed_config.rpc_secret_file,
- "rpc_secret",
- )?;
- secret_from_file(
- &mut parsed_config.admin.metrics_token,
- &parsed_config.admin.metrics_token_file,
- "admin.metrics_token",
- )?;
- secret_from_file(
- &mut parsed_config.admin.admin_token,
- &parsed_config.admin.admin_token_file,
- "admin.admin_token",
- )?;
-
- Ok(parsed_config)
-}
-
-fn secret_from_file(
- secret: &mut Option<String>,
- secret_file: &Option<String>,
- name: &'static str,
-) -> Result<(), Error> {
- match (&secret, &secret_file) {
- (_, None) => {
- // no-op
- }
- (Some(_), Some(_)) => {
- return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
- }
- (None, Some(file_path)) => {
- #[cfg(unix)]
- if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") {
- use std::os::unix::fs::MetadataExt;
- let metadata = std::fs::metadata(file_path)?;
- if metadata.mode() & 0o077 != 0 {
- return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
- }
- }
- let mut file = std::fs::OpenOptions::new().read(true).open(file_path)?;
- let mut secret_buf = String::new();
- file.read_to_string(&mut secret_buf)?;
- // trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
- // also editors sometimes add a trailing newline
- *secret = Some(String::from(secret_buf.trim_end()));
- }
- }
- Ok(())
-}
-
fn default_compression() -> Option<i32> {
Some(1)
}
@@ -425,83 +373,4 @@ mod tests {
Ok(())
}
-
- #[test]
- fn test_rpc_secret_file_works() -> Result<(), Error> {
- let path_secret = mktemp::Temp::new_file()?;
- let mut file_secret = File::create(path_secret.as_path())?;
- writeln!(file_secret, "foo")?;
- drop(file_secret);
-
- let path_config = mktemp::Temp::new_file()?;
- let mut file_config = File::create(path_config.as_path())?;
- let path_secret_path = path_secret.as_path();
- writeln!(
- file_config,
- r#"
- metadata_dir = "/tmp/garage/meta"
- data_dir = "/tmp/garage/data"
- replication_mode = "3"
- rpc_bind_addr = "[::]:3901"
- rpc_secret_file = "{}"
-
- [s3_api]
- s3_region = "garage"
- api_bind_addr = "[::]:3900"
- "#,
- path_secret_path.display()
- )?;
- let config = super::read_config(path_config.to_path_buf())?;
- assert_eq!("foo", config.rpc_secret.unwrap());
-
- #[cfg(unix)]
- {
- use std::os::unix::fs::PermissionsExt;
- let metadata = std::fs::metadata(&path_secret_path)?;
- let mut perm = metadata.permissions();
- perm.set_mode(0o660);
- std::fs::set_permissions(&path_secret_path, perm)?;
-
- std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "false");
- assert!(super::read_config(path_config.to_path_buf()).is_err());
-
- std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "true");
- assert!(super::read_config(path_config.to_path_buf()).is_ok());
- }
-
- drop(path_config);
- drop(path_secret);
- drop(file_config);
- Ok(())
- }
-
- #[test]
- fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> {
- let path_config = mktemp::Temp::new_file()?;
- let mut file_config = File::create(path_config.as_path())?;
- writeln!(
- file_config,
- r#"
- metadata_dir = "/tmp/garage/meta"
- data_dir = "/tmp/garage/data"
- replication_mode = "3"
- rpc_bind_addr = "[::]:3901"
- rpc_secret= "dummy"
- rpc_secret_file = "dummy"
-
- [s3_api]
- s3_region = "garage"
- api_bind_addr = "[::]:3900"
- "#
- )?;
- assert_eq!(
- "only one of `rpc_secret` and `rpc_secret_file` can be set",
- super::read_config(path_config.to_path_buf())
- .unwrap_err()
- .to_string()
- );
- drop(path_config);
- drop(file_config);
- Ok(())
- }
}
diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml
index 88cf1486..cb4b7f2b 100644
--- a/src/web/Cargo.toml
+++ b/src/web/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "garage_web"
-version = "0.9.0"
+version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018"
license = "AGPL-3.0"