aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlex Auvolat <alex@adnab.me>2023-10-18 15:34:12 +0200
committerAlex Auvolat <alex@adnab.me>2023-10-18 15:34:12 +0200
commit84d43501ce6dfd645d844ada0fcbff932d6fea76 (patch)
tree0e4c1a429ba1bf56c841375320520b1c0fd29beb
parent012ade5d4b81cc623f45042a1d3ca5c3ddafa569 (diff)
downloadgarage-84d43501ce6dfd645d844ada0fcbff932d6fea76.tar.gz
garage-84d43501ce6dfd645d844ada0fcbff932d6fea76.zip
refactor jepsen setup logic
-rw-r--r--script/jepsen.garage/shell.nix12
-rw-r--r--script/jepsen.garage/src/jepsen/garage/grg.clj131
2 files changed, 96 insertions, 47 deletions
diff --git a/script/jepsen.garage/shell.nix b/script/jepsen.garage/shell.nix
index 1f2514c6..01e4c845 100644
--- a/script/jepsen.garage/shell.nix
+++ b/script/jepsen.garage/shell.nix
@@ -1,8 +1,18 @@
-{ pkgs ? import <nixpkgs> { } }:
+{ pkgs ? import <nixpkgs> {
+ overlays = [
+ (self: super: {
+ jdk = super.jdk11;
+ jre = super.jre11;
+ })
+ ];
+} }:
pkgs.mkShell {
nativeBuildInputs = with pkgs; [
leiningen
+ jdk
+ jna
vagrant
gnuplot
+ graphviz
];
}
diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
index 4347b1a7..f6075762 100644
--- a/script/jepsen.garage/src/jepsen/garage/grg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/grg.clj
@@ -1,6 +1,7 @@
(ns jepsen.garage.grg
(:require [clojure.tools.logging :refer :all]
[jepsen [control :as c]
+ [core :as jepsen]
[db :as db]]
[jepsen.control.util :as cu]
[amazonica.aws.s3 :as s3]
@@ -21,63 +22,101 @@
; THE GARAGE DB
+(defn install!
+ "Download and install Garage"
+ [node version]
+ (c/su
+ (c/trace
+ (info node "installing garage" version)
+ (c/exec :mkdir :-p dir)
+ (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
+ cache (cu/cached-wget! url)]
+ (c/exec :cp cache binary))
+ (c/exec :chmod :+x binary))))
+
+(defn configure!
+ "Configure Garage"
+ [node]
+ (c/su
+ (c/trace
+ (cu/write-file!
+ (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
+ "rpc_bind_addr = \"0.0.0.0:3901\"\n"
+ "rpc_public_addr = \"" node ":3901\"\n"
+ "db_engine = \"lmdb\"\n"
+ "replication_mode = \"3\"\n"
+ "data_dir = \"" dir "/data\"\n"
+ "metadata_dir = \"" dir "/meta\"\n"
+ "[s3_api]\n"
+ "s3_region = \"us-east-1\"\n"
+ "api_bind_addr = \"0.0.0.0:3900\"\n"
+ "[k2v_api]\n"
+ "api_bind_addr = \"0.0.0.0:3902\"\n"
+ "[admin]\n"
+ "api_bind_addr = \"0.0.0.0:3903\"\n"
+ "admin_token = \"" grg-admin-token "\"\n")
+ "/etc/garage.toml"))))
+
+(defn connect-node!
+ "Connect a Garage node to the rest of the cluster"
+ [test node]
+ (c/trace
+ (let [node-id (c/exec binary :node :id :-q)]
+ (info node "node id:" node-id)
+ (c/on-many (:nodes test)
+ (c/exec binary :node :connect node-id)))))
+
+(defn configure-node!
+ "Configure a Garage node to be part of a cluster layout"
+ [test node]
+ (c/trace
+ (let [node-id (c/exec binary :node :id :-q)]
+ (c/on (jepsen/primary test)
+ (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)))))
+
+(defn finalize-config!
+ "Apply the layout and create a key/bucket pair in the cluster"
+ [node]
+ (c/trace
+ (c/exec binary :layout :apply :--version 1)
+ (info node "garage status:" (c/exec binary :status))
+ (c/exec binary :key :create grg-key)
+ (c/exec binary :bucket :create grg-bucket)
+ (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
+ (info node "key info: " (c/exec binary :key :info grg-key))))
+
(defn db
"Garage DB for a particular version"
[version]
(reify db/DB
(setup! [_ test node]
- (info node "installing garage" version)
- (c/su
- (c/exec :mkdir :-p dir)
- (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
- cache (cu/wget! url)]
- (c/exec :cp cache binary))
- (c/exec :chmod :+x binary)
- (cu/write-file!
- (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
- "rpc_bind_addr = \"0.0.0.0:3901\"\n"
- "rpc_public_addr = \"" node ":3901\"\n"
- "db_engine = \"lmdb\"\n"
- "replication_mode = \"3\"\n"
- "data_dir = \"" dir "/data\"\n"
- "metadata_dir = \"" dir "/meta\"\n"
- "[s3_api]\n"
- "s3_region = \"us-east-1\"\n"
- "api_bind_addr = \"0.0.0.0:3900\"\n"
- "[k2v_api]\n"
- "api_bind_addr = \"0.0.0.0:3902\"\n"
- "[admin]\n"
- "api_bind_addr = \"0.0.0.0:3903\"\n"
- "admin_token = \"" grg-admin-token "\"\n")
- "/etc/garage.toml")
- (cu/start-daemon!
- {:logfile logfile
- :pidfile pidfile
- :chdir dir}
- binary
- :server)
- (info node "garage daemon started")
- (c/exec :sleep 5)
- (let [node-id (c/exec binary :node :id :-q)]
- (info node "node id:" node-id)
- (c/on-many (:nodes test)
- (c/exec binary :node :connect node-id))
- (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))
- (if (= node (first (:nodes test)))
- (do
- (c/exec :sleep 5)
- (c/exec binary :layout :apply :--version 1)
- (info node "garage status:" (c/exec binary :status))
- (c/exec binary :key :create grg-key)
- (c/exec binary :bucket :create grg-bucket)
- (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
- (info node "key info: " (c/exec binary :key :info grg-key))))))
+ (install! node version)
+ (configure! node)
+ (cu/start-daemon!
+ {:logfile logfile
+ :pidfile pidfile
+ :chdir dir}
+ binary
+ :server)
+ (c/exec :sleep 3)
+
+ (jepsen/synchronize test)
+ (connect-node! test node)
+
+ (jepsen/synchronize test)
+ (configure-node! test node)
+
+ (jepsen/synchronize test)
+ (when (= node (jepsen/primary test))
+ (finalize-config! node)))
+
(teardown! [_ test node]
(info node "tearing down garage" version)
(c/su
(cu/stop-daemon! binary pidfile)
(c/exec :rm :-rf data-dir)
(c/exec :rm :-rf meta-dir)))
+
db/LogFiles
(log-files [_ test node]
[logfile])))