refactor jepsen setup logic

2 files changed, 96 insertions, 47 deletions

diff --git a/script/jepsen.garage/shell.nix b/script/jepsen.garage/shell.nix
index 1f2514c6..01e4c845 100644
--- a/script/jepsen.garage/shell.nix
+++ b/script/jepsen.garage/shell.nix
@@ -1,8 +1,18 @@
-{ pkgs ? import <nixpkgs> { } }:
+{ pkgs ? import <nixpkgs> {
+  overlays = [
+    (self: super: {
+      jdk = super.jdk11;
+      jre = super.jre11;
+    })
+  ];
+} }:
 pkgs.mkShell {
   nativeBuildInputs = with pkgs; [
     leiningen
+    jdk
+    jna
     vagrant
     gnuplot
+    graphviz
   ];
 }
diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
index 4347b1a7..f6075762 100644
--- a/script/jepsen.garage/src/jepsen/garage/grg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/grg.clj
@@ -1,6 +1,7 @@
 (ns jepsen.garage.grg
   (:require [clojure.tools.logging :refer :all]
             [jepsen [control :as c]
+             [core :as jepsen]
              [db :as db]]
             [jepsen.control.util :as cu]
             [amazonica.aws.s3 :as s3]
@@ -21,63 +22,101 @@
 
 ; THE GARAGE DB
 
+(defn install!
+  "Download and install Garage"
+  [node version]
+  (c/su
+    (c/trace
+      (info node "installing garage" version)
+      (c/exec :mkdir :-p dir)
+      (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
+            cache (cu/cached-wget! url)]
+        (c/exec :cp cache binary))
+      (c/exec :chmod :+x binary))))
+
+(defn configure!
+  "Configure Garage"
+  [node]
+  (c/su
+    (c/trace
+      (cu/write-file!
+        (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
+             "rpc_bind_addr = \"0.0.0.0:3901\"\n"
+             "rpc_public_addr = \"" node ":3901\"\n"
+             "db_engine = \"lmdb\"\n"
+             "replication_mode = \"3\"\n"
+             "data_dir = \"" dir "/data\"\n"
+             "metadata_dir = \"" dir "/meta\"\n"
+             "[s3_api]\n"
+             "s3_region = \"us-east-1\"\n"
+             "api_bind_addr = \"0.0.0.0:3900\"\n"
+             "[k2v_api]\n"
+             "api_bind_addr = \"0.0.0.0:3902\"\n"
+             "[admin]\n"
+             "api_bind_addr = \"0.0.0.0:3903\"\n"
+             "admin_token = \"" grg-admin-token "\"\n")
+        "/etc/garage.toml"))))
+
+(defn connect-node!
+  "Connect a Garage node to the rest of the cluster"
+  [test node]
+  (c/trace
+    (let [node-id (c/exec binary :node :id :-q)]
+      (info node "node id:" node-id)
+      (c/on-many (:nodes test)
+                 (c/exec binary :node :connect node-id)))))
+
+(defn configure-node!
+  "Configure a Garage node to be part of a cluster layout"
+  [test node]
+  (c/trace
+    (let [node-id (c/exec binary :node :id :-q)]
+      (c/on (jepsen/primary test)
+          (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)))))
+
+(defn finalize-config!
+  "Apply the layout and create a key/bucket pair in the cluster"
+  [node]
+  (c/trace
+    (c/exec binary :layout :apply :--version 1)
+    (info node "garage status:" (c/exec binary :status))
+    (c/exec binary :key :create grg-key)
+    (c/exec binary :bucket :create grg-bucket)
+    (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
+    (info node "key info: " (c/exec binary :key :info grg-key))))
+
 (defn db
   "Garage DB for a particular version"
   [version]
   (reify db/DB
     (setup! [_ test node]
-      (info node "installing garage" version)
-      (c/su
-        (c/exec :mkdir :-p dir)
-        (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
-              cache (cu/wget! url)]
-          (c/exec :cp cache binary))
-        (c/exec :chmod :+x binary)
-        (cu/write-file!
-          (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
-               "rpc_bind_addr = \"0.0.0.0:3901\"\n"
-               "rpc_public_addr = \"" node ":3901\"\n"
-               "db_engine = \"lmdb\"\n"
-               "replication_mode = \"3\"\n"
-               "data_dir = \"" dir "/data\"\n"
-               "metadata_dir = \"" dir "/meta\"\n"
-               "[s3_api]\n"
-               "s3_region = \"us-east-1\"\n"
-               "api_bind_addr = \"0.0.0.0:3900\"\n"
-               "[k2v_api]\n"
-               "api_bind_addr = \"0.0.0.0:3902\"\n"
-               "[admin]\n"
-               "api_bind_addr = \"0.0.0.0:3903\"\n"
-               "admin_token = \"" grg-admin-token "\"\n")
-          "/etc/garage.toml")
-        (cu/start-daemon!
-          {:logfile logfile
-           :pidfile pidfile
-           :chdir dir}
-          binary
-          :server)
-        (info node "garage daemon started")
-        (c/exec :sleep 5)
-        (let [node-id (c/exec binary :node :id :-q)]
-          (info node "node id:" node-id)
-          (c/on-many (:nodes test)
-                     (c/exec binary :node :connect node-id))
-          (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))
-        (if (= node (first (:nodes test)))
-          (do
-            (c/exec :sleep 5)
-            (c/exec binary :layout :apply :--version 1)
-            (info node "garage status:" (c/exec binary :status))
-            (c/exec binary :key :create grg-key)
-            (c/exec binary :bucket :create grg-bucket)
-            (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
-            (info node "key info: " (c/exec binary :key :info grg-key))))))
+      (install! node version)
+      (configure! node)
+      (cu/start-daemon!
+        {:logfile logfile
+         :pidfile pidfile
+         :chdir dir}
+        binary
+        :server)
+      (c/exec :sleep 3)
+
+      (jepsen/synchronize test)
+      (connect-node! test node)
+
+      (jepsen/synchronize test)
+      (configure-node! test node)
+
+      (jepsen/synchronize test)
+      (when (= node (jepsen/primary test))
+        (finalize-config! node)))
+
     (teardown! [_ test node]
       (info node "tearing down garage" version)
       (c/su
         (cu/stop-daemon! binary pidfile)
         (c/exec :rm :-rf data-dir)
         (c/exec :rm :-rf meta-dir)))
+
     db/LogFiles
     (log-files [_ test node]
       [logfile])))