From 7011b71fbd782e199417ce9afa44a8c220885b4a Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 18 Apr 2023 12:14:13 +0200
Subject: jepsen: wip

---
 script/jepsen.garage/src/jepsen/garage.clj | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 script/jepsen.garage/src/jepsen/garage.clj

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
new file mode 100644
index 00000000..1351c4a7
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -0,0 +1,18 @@
+(ns jepsen.garage
+  (:require [jepsen.cli :as cli]
+            [jepsen.tests :as tests]))
+
+(defn garage-test
+  "Given an options map from the command line runner (e.g. :nodes, :ssh,
+  :concurrency, ...), constructs a test map."
+  [opts]
+  (merge tests/noop-test
+         {:pure-generators true}
+         opts))
+
+(defn -main
+  "Handles command line arguments. Can either run a test, or a web server for
+  browsing results."
+  [& args]
+  (cli/run! (cli/single-test-cmd {:test-fn garage-test})
+            args))
-- 
cgit v1.2.3


From ca4cc7e44f6143ec24be626c683ed5b14ff40295 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 18 Apr 2023 13:59:03 +0200
Subject: jepsen connects to vagrant vms

---
 script/jepsen.garage/src/jepsen/garage.clj | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 1351c4a7..6617064e 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -14,5 +14,6 @@
   "Handles command line arguments. Can either run a test, or a web server for
   browsing results."
   [& args]
-  (cli/run! (cli/single-test-cmd {:test-fn garage-test})
+  (cli/run! (merge (cli/single-test-cmd {:test-fn garage-test})
+                   (cli/serve-cmd))
             args))
-- 
cgit v1.2.3


From bc11701999ececdb4571119082945cbe86f098aa Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 18 Apr 2023 16:10:07 +0200
Subject: jepsen: s3 gets and puts

---
 script/jepsen.garage/src/jepsen/garage.clj | 140 ++++++++++++++++++++++++++++-
 1 file changed, 136 insertions(+), 4 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 6617064e..26c58097 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -1,14 +1,146 @@
 (ns jepsen.garage
-  (:require [jepsen.cli :as cli]
-            [jepsen.tests :as tests]))
+  (:require [clojure.tools.logging :refer :all]
+            [clojure.string :as str]
+            [jepsen [cli :as cli]
+                    [client :as client]
+                    [control :as c]
+                    [db :as db]
+                    [generator :as gen]
+                    [tests :as tests]]
+            [jepsen.control.util :as cu]
+            [jepsen.os.debian :as debian]
+            [amazonica.aws.s3 :as s3]
+            [amazonica.aws.s3transfer :as s3transfer]))
+
+(def dir "/opt/garage")
+(def binary (str dir "/garage"))
+(def logfile (str dir "/garage.log"))
+(def pidfile (str dir "/garage.pid"))
+
+(def grg-admin-token "icanhazadmin")
+(def grg-key "jepsen")
+(def grg-bucket "jepsen")
+(def grg-object "1")
+
+(defn db
+  "Garage DB for a particular version"
+  [version]
+  (reify db/DB
+    (setup! [_ test node]
+      (info node "installing garage" version)
+      (c/su
+        (c/exec :mkdir :-p dir)
+        (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
+              cache (cu/wget! url)]
+          (c/exec :cp cache binary))
+        (c/exec :chmod :+x binary)
+        (cu/write-file!
+          (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
+               "rpc_bind_addr = \"0.0.0.0:3901\"\n"
+               "rpc_public_addr = \"" node ":3901\"\n"
+               "db_engine = \"lmdb\"\n"
+               "replication_mode = \"3\"\n"
+               "data_dir = \"" dir "/data\"\n"
+               "metadata_dir = \"" dir "/meta\"\n"
+               "[s3_api]\n"
+               "s3_region = \"us-east-1\"\n"
+               "api_bind_addr = \"0.0.0.0:3900\"\n"
+               "[k2v_api]\n"
+               "api_bind_addr = \"0.0.0.0:3902\"\n"
+               "[admin]\n"
+               "api_bind_addr = \"0.0.0.0:3903\"\n"
+               "admin_token = \"" grg-admin-token "\"\n")
+          "/etc/garage.toml")
+        (cu/start-daemon!
+          {:logfile logfile
+           :pidfile pidfile
+           :chdir dir}
+          binary
+          :server)
+        (Thread/sleep 100)
+        (let [node-id (c/exec binary :node :id :-q)]
+          (info node "node id:" node-id)
+          (c/on-many (:nodes test)
+                     (c/exec binary :node :connect node-id))
+          (c/exec binary :layout :assign (subs node-id 0 16) :-c 1 :-z :dc1 :-t node))
+        (if (= node (first (:nodes test)))
+          (do
+            (Thread/sleep 2000)
+            (c/exec binary :layout :apply :--version 1)
+            (info node "garage status:" (c/exec binary :status))
+            (c/exec binary :key :new :--name grg-key)
+            (c/exec binary :bucket :create grg-bucket)
+            (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
+            (info node "key info: " (c/exec binary :key :info grg-key))))))
+    (teardown! [_ test node]
+      (info node "tearing down garage" version)
+      (c/su
+        (cu/stop-daemon! binary pidfile)
+        (c/exec :rm :-rf dir)))
+    db/LogFiles
+    (log-files [_ test node]
+      [logfile])))
+
+(defn op-get [_ _] {:type :invoke, :f :get-object, :value nil})
+(defn op-put [_ _] {:type :invoke, :f :put-object, :value (str (rand-int 50))})
+(defn op-del [_ _] {:type :invoke, :f :del-object, :value nil})
+
+(defrecord Client [creds]
+  client/Client
+  (open! [this test node]
+    (let [key-info (c/on node (c/exec binary :key :info grg-key))
+          [_ ak sk] (re-matches
+                      #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
+                      key-info)
+          creds {:access-key ak
+                 :secret-key sk
+                 :endpoint (str "http://" node ":3900")
+                 :client-config {:path-style-access-enabled true}}]
+      (info node "s3 credentials:" creds)
+      (assoc this :creds creds)))
+  (setup! [this test])
+  (invoke! [this test op]
+    (case (:f op)
+      :get-object
+        (let [value-bytes (try
+                      (-> (s3/get-object (:creds this) grg-bucket grg-object)
+                          :input-stream
+                          slurp)
+                      (catch Exception e nil))]
+          (assoc op :type :ok, :value value-bytes))
+      :put-object
+        (let [some-bytes (.getBytes (:value op) "UTF-8")
+              bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
+          (s3/put-object (:creds this)
+                         :bucket-name grg-bucket
+                         :key grg-object
+                         :input-stream bytes-stream
+                         :metadata {:content-length (count some-bytes)})
+          (assoc op :type :ok))
+      :del-object
+      (do
+        (s3/delete-object (:creds this)
+                          :bucket-name grg-bucket
+                          :key grg-object)
+        (assoc op :type :ok, :value nil))))
+  (teardown! [this test])
+  (close! [this test]))
 
 (defn garage-test
   "Given an options map from the command line runner (e.g. :nodes, :ssh,
   :concurrency, ...), constructs a test map."
   [opts]
   (merge tests/noop-test
-         {:pure-generators true}
-         opts))
+         opts
+         {:pure-generators  true
+          :name             "garage"
+          :os               debian/os
+          :db               (db "v0.8.2")
+          :client           (Client. nil)
+          :generator        (->> (gen/mix [op-get op-put])
+                                 (gen/stagger 1)
+                                 (gen/nemesis nil)
+                                 (gen/time-limit 15))}))
 
 (defn -main
   "Handles command line arguments. Can either run a test, or a web server for
-- 
cgit v1.2.3


From 70c1d3db46b3bbdfad55b61c29e346c629380a32 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 18 Apr 2023 16:19:35 +0200
Subject: better match exceptions

---
 script/jepsen.garage/src/jepsen/garage.clj | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 26c58097..9d6f891e 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -9,6 +9,7 @@
                     [tests :as tests]]
             [jepsen.control.util :as cu]
             [jepsen.os.debian :as debian]
+            [slingshot.slingshot :refer [try+]]
             [amazonica.aws.s3 :as s3]
             [amazonica.aws.s3transfer :as s3transfer]))
 
@@ -101,13 +102,14 @@
   (setup! [this test])
   (invoke! [this test op]
     (case (:f op)
-      :get-object
-        (let [value-bytes (try
-                      (-> (s3/get-object (:creds this) grg-bucket grg-object)
-                          :input-stream
-                          slurp)
-                      (catch Exception e nil))]
-          (assoc op :type :ok, :value value-bytes))
+      :get-object (try+
+                    (let [value
+                          (-> (s3/get-object (:creds this) grg-bucket grg-object)
+                              :input-stream
+                              slurp)]
+                      (assoc op :type :ok, :value value))
+                    (catch (re-find #"Key not found" (.getMessage %))  ex
+                      (assoc op :type :ok, :value nil)))
       :put-object
         (let [some-bytes (.getBytes (:value op) "UTF-8")
               bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
@@ -137,10 +139,10 @@
           :os               debian/os
           :db               (db "v0.8.2")
           :client           (Client. nil)
-          :generator        (->> (gen/mix [op-get op-put])
+          :generator        (->> (gen/mix [op-get op-put op-del])
                                  (gen/stagger 1)
                                  (gen/nemesis nil)
-                                 (gen/time-limit 15))}))
+                                 (gen/time-limit 20))}))
 
 (defn -main
   "Handles command line arguments. Can either run a test, or a web server for
-- 
cgit v1.2.3


From dc5245ce65e6acc4c2b1f81dfdf38fc76fe06d3f Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 18 Apr 2023 17:47:53 +0200
Subject: even without nemesis, s3 get/put/delete is not linearizable (is this
 normal?)

---
 script/jepsen.garage/src/jepsen/garage.clj | 60 +++++++++++++++++++-----------
 1 file changed, 38 insertions(+), 22 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 9d6f891e..df3c8f7d 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -1,14 +1,18 @@
 (ns jepsen.garage
   (:require [clojure.tools.logging :refer :all]
             [clojure.string :as str]
-            [jepsen [cli :as cli]
+            [jepsen [checker :as checker]
+                    [cli :as cli]
                     [client :as client]
                     [control :as c]
                     [db :as db]
                     [generator :as gen]
+                    [nemesis :as nemesis]
                     [tests :as tests]]
+            [jepsen.checker.timeline :as timeline]
             [jepsen.control.util :as cu]
             [jepsen.os.debian :as debian]
+            [knossos.model :as model]
             [slingshot.slingshot :refer [try+]]
             [amazonica.aws.s3 :as s3]
             [amazonica.aws.s3transfer :as s3transfer]))
@@ -82,9 +86,9 @@
     (log-files [_ test node]
       [logfile])))
 
-(defn op-get [_ _] {:type :invoke, :f :get-object, :value nil})
-(defn op-put [_ _] {:type :invoke, :f :put-object, :value (str (rand-int 50))})
-(defn op-del [_ _] {:type :invoke, :f :del-object, :value nil})
+(defn op-get [_ _] {:type :invoke, :f :read, :value nil})
+(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 9))})
+(defn op-del [_ _] {:type :invoke, :f :write, :value nil})
 
 (defrecord Client [creds]
   client/Client
@@ -102,7 +106,7 @@
   (setup! [this test])
   (invoke! [this test op]
     (case (:f op)
-      :get-object (try+
+      :read (try+
                     (let [value
                           (-> (s3/get-object (:creds this) grg-bucket grg-object)
                               :input-stream
@@ -110,21 +114,21 @@
                       (assoc op :type :ok, :value value))
                     (catch (re-find #"Key not found" (.getMessage %))  ex
                       (assoc op :type :ok, :value nil)))
-      :put-object
-        (let [some-bytes (.getBytes (:value op) "UTF-8")
-              bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
-          (s3/put-object (:creds this)
-                         :bucket-name grg-bucket
-                         :key grg-object
-                         :input-stream bytes-stream
-                         :metadata {:content-length (count some-bytes)})
-          (assoc op :type :ok))
-      :del-object
-      (do
-        (s3/delete-object (:creds this)
-                          :bucket-name grg-bucket
-                          :key grg-object)
-        (assoc op :type :ok, :value nil))))
+      :write
+        (if (= (:value op) nil)
+          (do
+            (s3/delete-object (:creds this)
+                              :bucket-name grg-bucket
+                              :key grg-object)
+            (assoc op :type :ok, :value nil))
+          (let [some-bytes (.getBytes (:value op) "UTF-8")
+                bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
+            (s3/put-object (:creds this)
+                           :bucket-name grg-bucket
+                           :key grg-object
+                           :input-stream bytes-stream
+                           :metadata {:content-length (count some-bytes)})
+            (assoc op :type :ok)))))
   (teardown! [this test])
   (close! [this test]))
 
@@ -139,10 +143,22 @@
           :os               debian/os
           :db               (db "v0.8.2")
           :client           (Client. nil)
+          :nemesis          (nemesis/partition-random-halves)
+          :checker          (checker/compose
+                              {:perf (checker/perf)
+                               :timeline (timeline/html)
+                               :linear (checker/linearizable
+                                         {:model     (model/register)
+                                          :algorithm :linear})})
           :generator        (->> (gen/mix [op-get op-put op-del])
-                                 (gen/stagger 1)
+                                 (gen/stagger 0.02)
                                  (gen/nemesis nil)
-                                 (gen/time-limit 20))}))
+                                 ; (gen/nemesis
+                                 ;   (cycle [(gen/sleep 5)
+                                 ;           {:type :info, :f :start}
+                                 ;           (gen/sleep 5)
+                                 ;           {:type :info, :f :stop}]))
+                                 (gen/time-limit (+ (:time-limit opts) 5)))}))
 
 (defn -main
   "Handles command line arguments. Can either run a test, or a web server for
-- 
cgit v1.2.3


From 93a7132b4c5fb63966702f15ec995d805a40c8f9 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 18 Apr 2023 18:21:35 +0200
Subject: the fix for increasing timestamps does not make things linearizable

---
 script/jepsen.garage/src/jepsen/garage.clj | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index df3c8f7d..492dad4c 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -27,7 +27,7 @@
 (def grg-bucket "jepsen")
 (def grg-object "1")
 
-(defn db
+(defn garage
   "Garage DB for a particular version"
   [version]
   (reify db/DB
@@ -141,7 +141,8 @@
          {:pure-generators  true
           :name             "garage"
           :os               debian/os
-          :db               (db "v0.8.2")
+          :db               (garage "v0.8.2")
+          ; :db               (garage "d39c5c6984c581e16932aaa07e3687e7b5ce266d")     ; fixed for increasing timestamps
           :client           (Client. nil)
           :nemesis          (nemesis/partition-random-halves)
           :checker          (checker/compose
-- 
cgit v1.2.3


From eb86eaa6d2702ad5171c4fdfcbceb3dfba21833c Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 19 Apr 2023 14:14:22 +0200
Subject: refactor jepsen test

---
 script/jepsen.garage/src/jepsen/garage.clj     | 205 ++++++-------------------
 script/jepsen.garage/src/jepsen/garage/grg.clj | 125 +++++++++++++++
 script/jepsen.garage/src/jepsen/garage/reg.clj |  69 +++++++++
 3 files changed, 244 insertions(+), 155 deletions(-)
 create mode 100644 script/jepsen.garage/src/jepsen/garage/grg.clj
 create mode 100644 script/jepsen.garage/src/jepsen/garage/reg.clj

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 492dad4c..cd30dab8 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -1,170 +1,65 @@
 (ns jepsen.garage
-  (:require [clojure.tools.logging :refer :all]
-            [clojure.string :as str]
-            [jepsen [checker :as checker]
-                    [cli :as cli]
-                    [client :as client]
-                    [control :as c]
-                    [db :as db]
-                    [generator :as gen]
-                    [nemesis :as nemesis]
-                    [tests :as tests]]
-            [jepsen.checker.timeline :as timeline]
-            [jepsen.control.util :as cu]
-            [jepsen.os.debian :as debian]
-            [knossos.model :as model]
-            [slingshot.slingshot :refer [try+]]
-            [amazonica.aws.s3 :as s3]
-            [amazonica.aws.s3transfer :as s3transfer]))
+  (:require
+    [clojure.string :as str]
+    [jepsen
+     [checker :as checker]
+     [cli :as cli]
+     [generator :as gen]
+     [nemesis :as nemesis]
+     [tests :as tests]]
+    [jepsen.os.debian :as debian]
+    [jepsen.garage
+     [grg :as grg]
+     [reg :as reg]]))
 
-(def dir "/opt/garage")
-(def binary (str dir "/garage"))
-(def logfile (str dir "/garage.log"))
-(def pidfile (str dir "/garage.pid"))
+(def workloads
+  "A map of workload names to functions that construct workloads, given opts."
+  {"reg"  reg/workload})
 
-(def grg-admin-token "icanhazadmin")
-(def grg-key "jepsen")
-(def grg-bucket "jepsen")
-(def grg-object "1")
-
-(defn garage
-  "Garage DB for a particular version"
-  [version]
-  (reify db/DB
-    (setup! [_ test node]
-      (info node "installing garage" version)
-      (c/su
-        (c/exec :mkdir :-p dir)
-        (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
-              cache (cu/wget! url)]
-          (c/exec :cp cache binary))
-        (c/exec :chmod :+x binary)
-        (cu/write-file!
-          (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
-               "rpc_bind_addr = \"0.0.0.0:3901\"\n"
-               "rpc_public_addr = \"" node ":3901\"\n"
-               "db_engine = \"lmdb\"\n"
-               "replication_mode = \"3\"\n"
-               "data_dir = \"" dir "/data\"\n"
-               "metadata_dir = \"" dir "/meta\"\n"
-               "[s3_api]\n"
-               "s3_region = \"us-east-1\"\n"
-               "api_bind_addr = \"0.0.0.0:3900\"\n"
-               "[k2v_api]\n"
-               "api_bind_addr = \"0.0.0.0:3902\"\n"
-               "[admin]\n"
-               "api_bind_addr = \"0.0.0.0:3903\"\n"
-               "admin_token = \"" grg-admin-token "\"\n")
-          "/etc/garage.toml")
-        (cu/start-daemon!
-          {:logfile logfile
-           :pidfile pidfile
-           :chdir dir}
-          binary
-          :server)
-        (Thread/sleep 100)
-        (let [node-id (c/exec binary :node :id :-q)]
-          (info node "node id:" node-id)
-          (c/on-many (:nodes test)
-                     (c/exec binary :node :connect node-id))
-          (c/exec binary :layout :assign (subs node-id 0 16) :-c 1 :-z :dc1 :-t node))
-        (if (= node (first (:nodes test)))
-          (do
-            (Thread/sleep 2000)
-            (c/exec binary :layout :apply :--version 1)
-            (info node "garage status:" (c/exec binary :status))
-            (c/exec binary :key :new :--name grg-key)
-            (c/exec binary :bucket :create grg-bucket)
-            (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
-            (info node "key info: " (c/exec binary :key :info grg-key))))))
-    (teardown! [_ test node]
-      (info node "tearing down garage" version)
-      (c/su
-        (cu/stop-daemon! binary pidfile)
-        (c/exec :rm :-rf dir)))
-    db/LogFiles
-    (log-files [_ test node]
-      [logfile])))
-
-(defn op-get [_ _] {:type :invoke, :f :read, :value nil})
-(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 9))})
-(defn op-del [_ _] {:type :invoke, :f :write, :value nil})
-
-(defrecord Client [creds]
-  client/Client
-  (open! [this test node]
-    (let [key-info (c/on node (c/exec binary :key :info grg-key))
-          [_ ak sk] (re-matches
-                      #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
-                      key-info)
-          creds {:access-key ak
-                 :secret-key sk
-                 :endpoint (str "http://" node ":3900")
-                 :client-config {:path-style-access-enabled true}}]
-      (info node "s3 credentials:" creds)
-      (assoc this :creds creds)))
-  (setup! [this test])
-  (invoke! [this test op]
-    (case (:f op)
-      :read (try+
-                    (let [value
-                          (-> (s3/get-object (:creds this) grg-bucket grg-object)
-                              :input-stream
-                              slurp)]
-                      (assoc op :type :ok, :value value))
-                    (catch (re-find #"Key not found" (.getMessage %))  ex
-                      (assoc op :type :ok, :value nil)))
-      :write
-        (if (= (:value op) nil)
-          (do
-            (s3/delete-object (:creds this)
-                              :bucket-name grg-bucket
-                              :key grg-object)
-            (assoc op :type :ok, :value nil))
-          (let [some-bytes (.getBytes (:value op) "UTF-8")
-                bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
-            (s3/put-object (:creds this)
-                           :bucket-name grg-bucket
-                           :key grg-object
-                           :input-stream bytes-stream
-                           :metadata {:content-length (count some-bytes)})
-            (assoc op :type :ok)))))
-  (teardown! [this test])
-  (close! [this test]))
+(def cli-opts
+  "Additional command line options."
+  [["-I" "--increasing-timestamps" "Garage version with increasing timestamps on PutObject"
+    :default false]
+   ["-r" "--rate HZ" "Approximate number of requests per second, per thread."
+    :default  10
+    :parse-fn read-string
+    :validate [#(and (number? %) (pos? %)) "Must be a positive number"]]
+   [nil "--ops-per-key NUM" "Maximum number of operations on any given key."
+    :default  100
+    :parse-fn parse-long
+    :validate [pos? "Must be a positive integer."]]
+   ["-w" "--workload NAME" "Workload of test to run"
+    :default "reg"
+    :validate [workloads (cli/one-of workloads)]]])
 
 (defn garage-test
   "Given an options map from the command line runner (e.g. :nodes, :ssh,
   :concurrency, ...), constructs a test map."
   [opts]
-  (merge tests/noop-test
-         opts
-         {:pure-generators  true
-          :name             "garage"
-          :os               debian/os
-          :db               (garage "v0.8.2")
-          ; :db               (garage "d39c5c6984c581e16932aaa07e3687e7b5ce266d")     ; fixed for increasing timestamps
-          :client           (Client. nil)
-          :nemesis          (nemesis/partition-random-halves)
-          :checker          (checker/compose
-                              {:perf (checker/perf)
-                               :timeline (timeline/html)
-                               :linear (checker/linearizable
-                                         {:model     (model/register)
-                                          :algorithm :linear})})
-          :generator        (->> (gen/mix [op-get op-put op-del])
-                                 (gen/stagger 0.02)
-                                 (gen/nemesis nil)
-                                 ; (gen/nemesis
-                                 ;   (cycle [(gen/sleep 5)
-                                 ;           {:type :info, :f :start}
-                                 ;           (gen/sleep 5)
-                                 ;           {:type :info, :f :stop}]))
-                                 (gen/time-limit (+ (:time-limit opts) 5)))}))
+  (let [workload ((get workloads (:workload opts)) opts)
+        garage-version (if (:increasing-timestamps opts)
+                         "03490d41d58576d7b3bcf977b2726d72a3a66ada"
+                         "v0.8.2")]
+    (merge tests/noop-test
+           opts
+           {:pure-generators  true
+            :name             (str "garage " (name (:workload opts)))
+            :os               debian/os
+            :db               (grg/db garage-version)
+            :client           (:client workload)
+            :generator        (:generator workload)
+            :nemesis          (nemesis/partition-random-halves)
+            :checker          (checker/compose
+                                {:perf (checker/perf)
+                                 :workload (:checker workload)})
+            })))
+
 
 (defn -main
   "Handles command line arguments. Can either run a test, or a web server for
   browsing results."
   [& args]
-  (cli/run! (merge (cli/single-test-cmd {:test-fn garage-test})
+  (cli/run! (merge (cli/single-test-cmd {:test-fn garage-test
+                                         :opt-spec cli-opts})
                    (cli/serve-cmd))
             args))
diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
new file mode 100644
index 00000000..2309d824
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/grg.clj
@@ -0,0 +1,125 @@
+(ns jepsen.garage.grg
+  (:require [clojure.tools.logging :refer :all]
+            [jepsen [control :as c]
+             [db :as db]]
+            [jepsen.control.util :as cu]
+            [amazonica.aws.s3 :as s3]
+            [slingshot.slingshot :refer [try+]]))
+
+; CONSTANTS -- HOW GARAGE IS SET UP
+
+(def dir "/opt/garage")
+(def binary (str dir "/garage"))
+(def logfile (str dir "/garage.log"))
+(def pidfile (str dir "/garage.pid"))
+
+(def grg-admin-token "icanhazadmin")
+(def grg-key "jepsen")
+(def grg-bucket "jepsen")
+
+; THE GARAGE DB
+
+(defn db
+  "Garage DB for a particular version"
+  [version]
+  (reify db/DB
+    (setup! [_ test node]
+      (info node "installing garage" version)
+      (c/su
+        (c/exec :mkdir :-p dir)
+        (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
+              cache (cu/wget! url)]
+          (c/exec :cp cache binary))
+        (c/exec :chmod :+x binary)
+        (cu/write-file!
+          (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
+               "rpc_bind_addr = \"0.0.0.0:3901\"\n"
+               "rpc_public_addr = \"" node ":3901\"\n"
+               "db_engine = \"lmdb\"\n"
+               "replication_mode = \"3\"\n"
+               "data_dir = \"" dir "/data\"\n"
+               "metadata_dir = \"" dir "/meta\"\n"
+               "[s3_api]\n"
+               "s3_region = \"us-east-1\"\n"
+               "api_bind_addr = \"0.0.0.0:3900\"\n"
+               "[k2v_api]\n"
+               "api_bind_addr = \"0.0.0.0:3902\"\n"
+               "[admin]\n"
+               "api_bind_addr = \"0.0.0.0:3903\"\n"
+               "admin_token = \"" grg-admin-token "\"\n")
+          "/etc/garage.toml")
+        (cu/start-daemon!
+          {:logfile logfile
+           :pidfile pidfile
+           :chdir dir}
+          binary
+          :server)
+        (Thread/sleep 100)
+        (let [node-id (c/exec binary :node :id :-q)]
+          (info node "node id:" node-id)
+          (c/on-many (:nodes test)
+                     (c/exec binary :node :connect node-id))
+          (c/exec binary :layout :assign (subs node-id 0 16) :-c 1 :-z :dc1 :-t node))
+        (if (= node (first (:nodes test)))
+          (do
+            (Thread/sleep 2000)
+            (c/exec binary :layout :apply :--version 1)
+            (info node "garage status:" (c/exec binary :status))
+            (c/exec binary :key :new :--name grg-key)
+            (c/exec binary :bucket :create grg-bucket)
+            (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
+            (info node "key info: " (c/exec binary :key :info grg-key))))))
+    (teardown! [_ test node]
+      (info node "tearing down garage" version)
+      (c/su
+        (cu/stop-daemon! binary pidfile)
+        (c/exec :rm :-rf dir)))
+    db/LogFiles
+    (log-files [_ test node]
+      [logfile])))
+
+; GARAGE S3 HELPER FUNCTIONS
+
+(defn s3-creds
+  "Get S3 credentials for node"
+  [node]
+  (let [key-info (c/on node (c/exec binary :key :info grg-key))
+        [_ ak sk] (re-matches
+                    #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
+                    key-info)]
+    {:access-key ak
+     :secret-key sk
+     :endpoint (str "http://" node ":3900")
+     :bucket grg-bucket
+     :client-config {:path-style-access-enabled true}}))
+
+(defn s3-get
+  "Helper for GetObject"
+  [creds k]
+  (try+
+    (-> (s3/get-object creds (:bucket creds) k)
+        :input-stream
+        slurp)
+    (catch (re-find #"Key not found" (.getMessage %)) ex
+      nil)))
+
+(defn s3-put
+  "Helper for PutObject or DeleteObject (is a delete if value is nil)"
+  [creds k v]
+  (if (= v nil)
+    (s3/delete-object creds
+                      :bucket-name (:bucket creds)
+                      :key k)
+    (let [some-bytes (.getBytes v "UTF-8")
+          bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
+      (s3/put-object creds
+                     :bucket-name (:bucket creds)
+                     :key k
+                     :input-stream bytes-stream
+                     :metadata {:content-length (count some-bytes)}))))
+
+(defn s3-list
+  "Helper for ListObjects -- just lists everything in the bucket"
+  [creds]
+  (s3/list-objects-v2 creds
+                      {:bucket-name (:bucket creds)}))
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
new file mode 100644
index 00000000..557b4a99
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -0,0 +1,69 @@
+(ns jepsen.garage.reg
+  (:require [clojure.tools.logging :refer :all]
+            [clojure.string :as str]
+            [jepsen [checker :as checker]
+             [cli :as cli]
+             [client :as client]
+             [control :as c]
+             [db :as db]
+             [generator :as gen]
+             [independent :as independent]
+             [nemesis :as nemesis]
+             [tests :as tests]]
+            [jepsen.checker.timeline :as timeline]
+            [jepsen.control.util :as cu]
+            [jepsen.os.debian :as debian]
+            [jepsen.garage.grg :as grg]
+            [knossos.model :as model]
+            [slingshot.slingshot :refer [try+]]))
+
+(defn op-get [_ _] {:type :invoke, :f :read, :value nil})
+(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 9))})
+(defn op-del [_ _] {:type :invoke, :f :write, :value nil})
+
+(defrecord RegClient [creds]
+  client/Client
+  (open! [this test node]
+    (let [creds (grg/s3-creds node)]
+      (info node "s3 credentials:" creds)
+      (assoc this :creds creds)))
+  (setup! [this test])
+  (invoke! [this test op]
+    (let [[k v] (:value op)]
+      (case (:f op)
+        :read
+          (let [value (grg/s3-get (:creds this) k)]
+            (assoc op :type :ok, :value (independent/tuple k value)))
+        :write
+          (do
+            (grg/s3-put (:creds this) k v)
+            (assoc op :type :ok)))))
+  (teardown! [this test])
+  (close! [this test]))
+
+(defn workload
+  "Tests linearizable reads and writes"
+  [opts]
+  {:client           (RegClient. nil)
+   :checker          (independent/checker
+                       (checker/compose
+                         {:linear (checker/linearizable
+                                    {:model     (model/register)
+                                     :algorithm :linear})
+                          :timeline (timeline/html)}))
+   :generator        (->> (independent/concurrent-generator
+                            10
+                            (range)
+                            (fn [k]
+                              (->>
+                                (gen/mix [op-get op-put op-del])
+                                (gen/stagger (/ (:rate opts)))
+                                (gen/limit (:ops-per-key opts)))))
+                          (gen/nemesis
+                            (cycle [(gen/sleep 5)
+                                    {:type :info, :f :start}
+                                    (gen/sleep 5)
+                                    {:type :info, :f :stop}]))
+                          (gen/time-limit (:time-limit opts)))})
+
+
-- 
cgit v1.2.3


From 6eb26be548c08707b59473e6086f3f5eee89fe47 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 19 Apr 2023 15:27:26 +0200
Subject: Add garage set test (this one works :p)

---
 script/jepsen.garage/src/jepsen/garage.clj     | 22 ++++++++--
 script/jepsen.garage/src/jepsen/garage/grg.clj | 13 +++++-
 script/jepsen.garage/src/jepsen/garage/reg.clj | 21 ++++------
 script/jepsen.garage/src/jepsen/garage/set.clj | 56 ++++++++++++++++++++++++++
 4 files changed, 93 insertions(+), 19 deletions(-)
 create mode 100644 script/jepsen.garage/src/jepsen/garage/set.clj

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index cd30dab8..7f762923 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -10,11 +10,13 @@
     [jepsen.os.debian :as debian]
     [jepsen.garage
      [grg :as grg]
-     [reg :as reg]]))
+     [reg :as reg]
+     [set :as set]]))
 
 (def workloads
   "A map of workload names to functions that construct workloads, given opts."
-  {"reg"  reg/workload})
+  {"reg"  reg/workload
+   "set"  set/workload})
 
 (def cli-opts
   "Additional command line options."
@@ -47,7 +49,21 @@
             :os               debian/os
             :db               (grg/db garage-version)
             :client           (:client workload)
-            :generator        (:generator workload)
+            :generator        (gen/phases
+                                (->>
+                                  (:generator workload)
+                                  (gen/stagger (/ (:rate opts)))
+                                  (gen/nemesis
+                                    (cycle [(gen/sleep 5)
+                                            {:type :info, :f :start}
+                                            (gen/sleep 5)
+                                            {:type :info, :f :stop}]))
+                                  (gen/time-limit (:time-limit opts)))
+                                (gen/log "Healing cluster")
+                                (gen/nemesis (gen/once {:type :info, :f :stop}))
+                                (gen/log "Waiting for recovery")
+                                (gen/sleep 10)
+                                (gen/clients (:final-generator workload)))
             :nemesis          (nemesis/partition-random-halves)
             :checker          (checker/compose
                                 {:perf (checker/perf)
diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
index 2309d824..2e9bcc72 100644
--- a/script/jepsen.garage/src/jepsen/garage/grg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/grg.clj
@@ -121,5 +121,14 @@
 (defn s3-list
   "Helper for ListObjects -- just lists everything in the bucket"
   [creds]
-  (s3/list-objects-v2 creds
-                      {:bucket-name (:bucket creds)}))
+  (defn list-inner [ct accum]
+    (let [list-result (s3/list-objects-v2 creds
+                                          {:bucket-name (:bucket creds)
+                                           :continuation-token ct})
+          new-object-summaries (:object-summaries list-result)
+          new-objects (map (fn [d] (:key d)) new-object-summaries)
+          objects (concat new-objects accum)]
+      (if (:truncated? list-result)
+        (list-inner (:next-continuation-token list-result) objects)
+        objects)))
+  (list-inner nil []))
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index 557b4a99..0b370b36 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -51,19 +51,12 @@
                                     {:model     (model/register)
                                      :algorithm :linear})
                           :timeline (timeline/html)}))
-   :generator        (->> (independent/concurrent-generator
-                            10
-                            (range)
-                            (fn [k]
-                              (->>
-                                (gen/mix [op-get op-put op-del])
-                                (gen/stagger (/ (:rate opts)))
-                                (gen/limit (:ops-per-key opts)))))
-                          (gen/nemesis
-                            (cycle [(gen/sleep 5)
-                                    {:type :info, :f :start}
-                                    (gen/sleep 5)
-                                    {:type :info, :f :stop}]))
-                          (gen/time-limit (:time-limit opts)))})
+   :generator        (independent/concurrent-generator
+                       10
+                       (range)
+                       (fn [k]
+                         (->>
+                           (gen/mix [op-get op-put op-del])
+                           (gen/limit (:ops-per-key opts)))))})
 
 
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
new file mode 100644
index 00000000..ab99e6ef
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -0,0 +1,56 @@
+(ns jepsen.garage.set
+  (:require [clojure.tools.logging :refer :all]
+            [clojure.string :as str]
+            [jepsen [checker :as checker]
+             [cli :as cli]
+             [client :as client]
+             [control :as c]
+             [db :as db]
+             [generator :as gen]
+             [independent :as independent]
+             [nemesis :as nemesis]
+             [tests :as tests]]
+            [jepsen.checker.timeline :as timeline]
+            [jepsen.control.util :as cu]
+            [jepsen.os.debian :as debian]
+            [jepsen.garage.grg :as grg]
+            [knossos.model :as model]
+            [slingshot.slingshot :refer [try+]]))
+
+(defn op-add [_ _] {:type :invoke, :f :add, :value (rand-int 100000)})
+(defn op-read [_ _] {:type :invoke, :f :read, :value nil})
+
+(defrecord SetClient [creds]
+  client/Client
+  (open! [this test node]
+    (let [creds (grg/s3-creds node)]
+      (info node "s3 credentials:" creds)
+      (assoc this :creds creds)))
+  (setup! [this test])
+  (invoke! [this test op]
+    (case (:f op)
+      :add
+        (do
+          (grg/s3-put (:creds this) (str (:value op)) "present")
+          (assoc op :type :ok))
+      :read
+        (let [items (grg/s3-list (:creds this))]
+          (assoc op :type :ok, :value (set (map read-string items))))))
+  (teardown! [this test])
+  (close! [this test]))
+
+(defn workload
+  "Tests insertions and deletions"
+  [opts]
+  {:client            (SetClient. nil)
+   :checker           (checker/compose
+                        {:set (checker/set)
+                         :timeline (timeline/html)})
+   ; :generator         (gen/mix [op-add op-read])
+   ; :generator         (->> (range)
+   ;                         (map (fn [x] {:type :invoke, :f :add, :value x})))
+   :generator         (gen/mix [op-read
+                        (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})))])
+   :final-generator   (gen/once op-read)})
+
+
-- 
cgit v1.2.3


From 0bb1577ae12b2dde313e036399f252266bc04d0a Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 19 Apr 2023 15:59:30 +0200
Subject: two set workloads with different checkers

---
 script/jepsen.garage/src/jepsen/garage.clj     |  3 ++-
 script/jepsen.garage/src/jepsen/garage/grg.clj |  3 ++-
 script/jepsen.garage/src/jepsen/garage/set.clj | 20 ++++++++++++++------
 3 files changed, 18 insertions(+), 8 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 7f762923..dbaf265b 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -16,7 +16,8 @@
 (def workloads
   "A map of workload names to functions that construct workloads, given opts."
   {"reg"  reg/workload
-   "set"  set/workload})
+   "set1" set/workload1
+   "set2" set/workload2})
 
 (def cli-opts
   "Additional command line options."
diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
index 2e9bcc72..4597fb14 100644
--- a/script/jepsen.garage/src/jepsen/garage/grg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/grg.clj
@@ -120,10 +120,11 @@
 
 (defn s3-list
   "Helper for ListObjects -- just lists everything in the bucket"
-  [creds]
+  [creds prefix]
   (defn list-inner [ct accum]
     (let [list-result (s3/list-objects-v2 creds
                                           {:bucket-name (:bucket creds)
+                                           :prefix prefix
                                            :continuation-token ct})
           new-object-summaries (:object-summaries list-result)
           new-objects (map (fn [d] (:key d)) new-object-summaries)
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index ab99e6ef..f8c92580 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -34,12 +34,12 @@
           (grg/s3-put (:creds this) (str (:value op)) "present")
           (assoc op :type :ok))
       :read
-        (let [items (grg/s3-list (:creds this))]
+        (let [items (grg/s3-list (:creds this) "")]
           (assoc op :type :ok, :value (set (map read-string items))))))
   (teardown! [this test])
   (close! [this test]))
 
-(defn workload
+(defn workload1
   "Tests insertions and deletions"
   [opts]
   {:client            (SetClient. nil)
@@ -47,10 +47,18 @@
                         {:set (checker/set)
                          :timeline (timeline/html)})
    ; :generator         (gen/mix [op-add op-read])
-   ; :generator         (->> (range)
-   ;                         (map (fn [x] {:type :invoke, :f :add, :value x})))
-   :generator         (gen/mix [op-read
-                        (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})))])
+   :generator         (->> (range)
+                           (map (fn [x] {:type :invoke, :f :add, :value x})))
    :final-generator   (gen/once op-read)})
 
+(defn workload2
+  "Tests insertions and deletions"
+  [opts]
+  {:client            (SetClient. nil)
+   :checker           (checker/compose
+                        {:set (checker/set-full {:linearizable? false})
+                         :timeline (timeline/html)})
+   :generator         (gen/mix [op-read
+                        (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})))])})
+
 
-- 
cgit v1.2.3


From 55eb4e87c42bf0da88186eb5b2fe1fbbbdf9ed43 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 19 Apr 2023 16:16:34 +0200
Subject: set tests with independant tests together

---
 script/jepsen.garage/src/jepsen/garage/set.clj | 55 ++++++++++++++++----------
 1 file changed, 35 insertions(+), 20 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index f8c92580..156493bc 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -28,14 +28,18 @@
       (assoc this :creds creds)))
   (setup! [this test])
   (invoke! [this test op]
-    (case (:f op)
-      :add
-        (do
-          (grg/s3-put (:creds this) (str (:value op)) "present")
-          (assoc op :type :ok))
-      :read
-        (let [items (grg/s3-list (:creds this) "")]
-          (assoc op :type :ok, :value (set (map read-string items))))))
+    (let [[k v] (:value op)
+          prefix (str "set" k "/")]
+      (case (:f op)
+        :add
+          (do
+            (grg/s3-put (:creds this) (str prefix v) "present")
+            (assoc op :type :ok))
+        :read
+          (let [items (grg/s3-list (:creds this) prefix)
+                items-stripped (map (fn [o] (str/replace-first o prefix "")) items)
+                items-set (set (map read-string items-stripped))]
+            (assoc op :type :ok, :value (independent/tuple k items-set))))))
   (teardown! [this test])
   (close! [this test]))
 
@@ -43,22 +47,33 @@
   "Tests insertions and deletions"
   [opts]
   {:client            (SetClient. nil)
-   :checker           (checker/compose
-                        {:set (checker/set)
-                         :timeline (timeline/html)})
-   ; :generator         (gen/mix [op-add op-read])
-   :generator         (->> (range)
-                           (map (fn [x] {:type :invoke, :f :add, :value x})))
-   :final-generator   (gen/once op-read)})
+   :checker           (independent/checker
+                        (checker/compose
+                          {:set (checker/set)
+                           :timeline (timeline/html)}))
+   :generator         (independent/concurrent-generator
+                        10
+                        (range 100)
+                        (fn [k]
+                          (->>
+                           (gen/mix [op-add])
+                           (gen/limit (:ops-per-key opts)))))
+   :final-generator   (independent/sequential-generator
+                        (range 100)
+                        (fn [k] (gen/once op-read)))})
 
 (defn workload2
   "Tests insertions and deletions"
   [opts]
   {:client            (SetClient. nil)
-   :checker           (checker/compose
-                        {:set (checker/set-full {:linearizable? false})
-                         :timeline (timeline/html)})
-   :generator         (gen/mix [op-read
-                        (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})))])})
+   :checker           (independent/checker
+                        (checker/compose
+                          {:set (checker/set-full {:linearizable? false})
+                           :timeline (timeline/html)}))
+   :generator         (independent/concurrent-generator
+                        10
+                        (range 100)
+                        (fn [k]
+                          (gen/mix [op-add op-read])))})
 
 
-- 
cgit v1.2.3


From 18ee8efb5f7bfcd64bf1a1336ffd6f54574aa79f Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 19 Apr 2023 22:02:03 +0200
Subject: Check read-after-write property for sets

---
 script/jepsen.garage/src/jepsen/garage/set.clj | 48 +++++++++++++++++++++++---
 1 file changed, 43 insertions(+), 5 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index 156493bc..9b21d50e 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -1,10 +1,12 @@
 (ns jepsen.garage.set
   (:require [clojure.tools.logging :refer :all]
             [clojure.string :as str]
+            [clojure.set :as set]
             [jepsen [checker :as checker]
              [cli :as cli]
              [client :as client]
              [control :as c]
+             [checker :as checker]
              [db :as db]
              [generator :as gen]
              [independent :as independent]
@@ -17,7 +19,7 @@
             [knossos.model :as model]
             [slingshot.slingshot :refer [try+]]))
 
-(defn op-add [_ _] {:type :invoke, :f :add, :value (rand-int 100000)})
+(defn op-add-rand100 [_ _] {:type :invoke, :f :add, :value (rand-int 100)})
 (defn op-read [_ _] {:type :invoke, :f :read, :value nil})
 
 (defrecord SetClient [creds]
@@ -43,6 +45,41 @@
   (teardown! [this test])
   (close! [this test]))
 
+(defn set-read-after-write
+  "Read-after-Write checker for set operations"
+  []
+  (reify checker/Checker
+    (check [this test history opts]
+      (let [init {:add-started #{}
+                  :add-done #{}
+                  :read-must-contain {}
+                  :missed #{}
+                  :unexpected #{}}
+            final (reduce
+                    (fn [state op]
+                      (info "state:" state)
+                      (info "operation:" op)
+                      (case [(:type op) (:f op)]
+                        ([:invoke :add])
+                          (assoc state :add-started (conj (:add-started state) (:value op)))
+                        ([:ok :add])
+                          (assoc state :add-done (conj (:add-done state) (:value op)))
+                        ([:invoke :read])
+                          (assoc-in state [:read-must-contain (:process op)] (:add-done state))
+                        ([:ok :read])
+                          (let [read-must-contain (get (:process op) (:read-must-contain state))
+                                new-missed (set/difference read-must-contain (:value op))
+                                new-unexpected (set/difference (:value op) (:add-started state))]
+                            (assoc state
+                                   :read-must-contain (dissoc (:read-must-contain state) (:process op))
+                                   :missed (set/union (:missed state) new-missed),
+                                   :unexpected (set/union (:unexpected state) new-unexpected)))
+                         state))
+                    init history)
+            valid? (and (empty? (:missed final)) (empty? (:unexpected final)))]
+        (info "final state:" final)
+        (assoc final :valid? valid?)))))
+
 (defn workload1
   "Tests insertions and deletions"
   [opts]
@@ -55,8 +92,8 @@
                         10
                         (range 100)
                         (fn [k]
-                          (->>
-                           (gen/mix [op-add])
+                          (->> (range)
+                           (map (fn [x] {:type :invoke, :f :add, :value x}))
                            (gen/limit (:ops-per-key opts)))))
    :final-generator   (independent/sequential-generator
                         (range 100)
@@ -68,12 +105,13 @@
   {:client            (SetClient. nil)
    :checker           (independent/checker
                         (checker/compose
-                          {:set (checker/set-full {:linearizable? false})
+                          {:set-full (checker/set-full {:linearizable? false})
+                           :set-read-after-write (set-read-after-write)
                            :timeline (timeline/html)}))
    :generator         (independent/concurrent-generator
                         10
                         (range 100)
                         (fn [k]
-                          (gen/mix [op-add op-read])))})
+                          (gen/mix [op-add-rand100 op-read])))})
 
 
-- 
cgit v1.2.3


From 9ec4cca3340cfd03deb55cd58d6b354630e7eb5f Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 19 Apr 2023 22:17:20 +0200
Subject: reformatting

---
 script/jepsen.garage/src/jepsen/garage/set.clj | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index 9b21d50e..5f76d1ac 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -57,8 +57,6 @@
                   :unexpected #{}}
             final (reduce
                     (fn [state op]
-                      (info "state:" state)
-                      (info "operation:" op)
                       (case [(:type op) (:f op)]
                         ([:invoke :add])
                           (assoc state :add-started (conj (:add-started state) (:value op)))
@@ -77,7 +75,6 @@
                          state))
                     init history)
             valid? (and (empty? (:missed final)) (empty? (:unexpected final)))]
-        (info "final state:" final)
         (assoc final :valid? valid?)))))
 
 (defn workload1
@@ -110,7 +107,7 @@
                            :timeline (timeline/html)}))
    :generator         (independent/concurrent-generator
                         10
-                        (range 100)
+                        (range)
                         (fn [k]
                           (gen/mix [op-add-rand100 op-read])))})
 
-- 
cgit v1.2.3


From ef5ca86dfc2678e8fc16b99daffeac40093945ae Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 18 Oct 2023 14:01:18 +0200
Subject: jepsen: update to garage 0.9.0

---
 script/jepsen.garage/src/jepsen/garage.clj     |  2 +-
 script/jepsen.garage/src/jepsen/garage/grg.clj | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index dbaf265b..754ddf7d 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -42,7 +42,7 @@
   (let [workload ((get workloads (:workload opts)) opts)
         garage-version (if (:increasing-timestamps opts)
                          "03490d41d58576d7b3bcf977b2726d72a3a66ada"
-                         "v0.8.2")]
+                         "v0.9.0")]
     (merge tests/noop-test
            opts
            {:pure-generators  true
diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
index 4597fb14..31f20852 100644
--- a/script/jepsen.garage/src/jepsen/garage/grg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/grg.clj
@@ -9,6 +9,8 @@
 ; CONSTANTS -- HOW GARAGE IS SET UP
 
 (def dir "/opt/garage")
+(def data-dir (str dir "/data"))
+(def meta-dir (str dir "/meta"))
 (def binary (str dir "/garage"))
 (def logfile (str dir "/garage.log"))
 (def pidfile (str dir "/garage.pid"))
@@ -54,18 +56,19 @@
            :chdir dir}
           binary
           :server)
-        (Thread/sleep 100)
+        (info node "garage daemon started")
+        (c/exec :sleep 5)
         (let [node-id (c/exec binary :node :id :-q)]
           (info node "node id:" node-id)
           (c/on-many (:nodes test)
                      (c/exec binary :node :connect node-id))
-          (c/exec binary :layout :assign (subs node-id 0 16) :-c 1 :-z :dc1 :-t node))
+          (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))
         (if (= node (first (:nodes test)))
           (do
-            (Thread/sleep 2000)
+            (c/exec :sleep 5)
             (c/exec binary :layout :apply :--version 1)
             (info node "garage status:" (c/exec binary :status))
-            (c/exec binary :key :new :--name grg-key)
+            (c/exec binary :key :create grg-key)
             (c/exec binary :bucket :create grg-bucket)
             (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
             (info node "key info: " (c/exec binary :key :info grg-key))))))
@@ -73,7 +76,8 @@
       (info node "tearing down garage" version)
       (c/su
         (cu/stop-daemon! binary pidfile)
-        (c/exec :rm :-rf dir)))
+        (c/exec :rm :-rf data-dir)
+        (c/exec :rm :-rf meta-dir)))
     db/LogFiles
     (log-files [_ test node]
       [logfile])))
-- 
cgit v1.2.3


From 012ade5d4b81cc623f45042a1d3ca5c3ddafa569 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 18 Oct 2023 14:06:32 +0200
Subject: jepsen: update jepsen and fix garage key info

---
 script/jepsen.garage/src/jepsen/garage/grg.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
index 31f20852..4347b1a7 100644
--- a/script/jepsen.garage/src/jepsen/garage/grg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/grg.clj
@@ -87,7 +87,7 @@
 (defn s3-creds
   "Get S3 credentials for node"
   [node]
-  (let [key-info (c/on node (c/exec binary :key :info grg-key))
+  (let [key-info (c/on node (c/exec binary :key :info grg-key :--show-secret))
         [_ ak sk] (re-matches
                     #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
                     key-info)]
-- 
cgit v1.2.3


From 84d43501ce6dfd645d844ada0fcbff932d6fea76 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 18 Oct 2023 15:34:12 +0200
Subject: refactor jepsen setup logic

---
 script/jepsen.garage/src/jepsen/garage/grg.clj | 131 ++++++++++++++++---------
 1 file changed, 85 insertions(+), 46 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
index 4347b1a7..f6075762 100644
--- a/script/jepsen.garage/src/jepsen/garage/grg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/grg.clj
@@ -1,6 +1,7 @@
 (ns jepsen.garage.grg
   (:require [clojure.tools.logging :refer :all]
             [jepsen [control :as c]
+             [core :as jepsen]
              [db :as db]]
             [jepsen.control.util :as cu]
             [amazonica.aws.s3 :as s3]
@@ -21,63 +22,101 @@
 
 ; THE GARAGE DB
 
+(defn install!
+  "Download and install Garage"
+  [node version]
+  (c/su
+    (c/trace
+      (info node "installing garage" version)
+      (c/exec :mkdir :-p dir)
+      (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
+            cache (cu/cached-wget! url)]
+        (c/exec :cp cache binary))
+      (c/exec :chmod :+x binary))))
+
+(defn configure!
+  "Configure Garage"
+  [node]
+  (c/su
+    (c/trace
+      (cu/write-file!
+        (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
+             "rpc_bind_addr = \"0.0.0.0:3901\"\n"
+             "rpc_public_addr = \"" node ":3901\"\n"
+             "db_engine = \"lmdb\"\n"
+             "replication_mode = \"3\"\n"
+             "data_dir = \"" dir "/data\"\n"
+             "metadata_dir = \"" dir "/meta\"\n"
+             "[s3_api]\n"
+             "s3_region = \"us-east-1\"\n"
+             "api_bind_addr = \"0.0.0.0:3900\"\n"
+             "[k2v_api]\n"
+             "api_bind_addr = \"0.0.0.0:3902\"\n"
+             "[admin]\n"
+             "api_bind_addr = \"0.0.0.0:3903\"\n"
+             "admin_token = \"" grg-admin-token "\"\n")
+        "/etc/garage.toml"))))
+
+(defn connect-node!
+  "Connect a Garage node to the rest of the cluster"
+  [test node]
+  (c/trace
+    (let [node-id (c/exec binary :node :id :-q)]
+      (info node "node id:" node-id)
+      (c/on-many (:nodes test)
+                 (c/exec binary :node :connect node-id)))))
+
+(defn configure-node!
+  "Configure a Garage node to be part of a cluster layout"
+  [test node]
+  (c/trace
+    (let [node-id (c/exec binary :node :id :-q)]
+      (c/on (jepsen/primary test)
+          (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)))))
+
+(defn finalize-config!
+  "Apply the layout and create a key/bucket pair in the cluster"
+  [node]
+  (c/trace
+    (c/exec binary :layout :apply :--version 1)
+    (info node "garage status:" (c/exec binary :status))
+    (c/exec binary :key :create grg-key)
+    (c/exec binary :bucket :create grg-bucket)
+    (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
+    (info node "key info: " (c/exec binary :key :info grg-key))))
+
 (defn db
   "Garage DB for a particular version"
   [version]
   (reify db/DB
     (setup! [_ test node]
-      (info node "installing garage" version)
-      (c/su
-        (c/exec :mkdir :-p dir)
-        (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
-              cache (cu/wget! url)]
-          (c/exec :cp cache binary))
-        (c/exec :chmod :+x binary)
-        (cu/write-file!
-          (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
-               "rpc_bind_addr = \"0.0.0.0:3901\"\n"
-               "rpc_public_addr = \"" node ":3901\"\n"
-               "db_engine = \"lmdb\"\n"
-               "replication_mode = \"3\"\n"
-               "data_dir = \"" dir "/data\"\n"
-               "metadata_dir = \"" dir "/meta\"\n"
-               "[s3_api]\n"
-               "s3_region = \"us-east-1\"\n"
-               "api_bind_addr = \"0.0.0.0:3900\"\n"
-               "[k2v_api]\n"
-               "api_bind_addr = \"0.0.0.0:3902\"\n"
-               "[admin]\n"
-               "api_bind_addr = \"0.0.0.0:3903\"\n"
-               "admin_token = \"" grg-admin-token "\"\n")
-          "/etc/garage.toml")
-        (cu/start-daemon!
-          {:logfile logfile
-           :pidfile pidfile
-           :chdir dir}
-          binary
-          :server)
-        (info node "garage daemon started")
-        (c/exec :sleep 5)
-        (let [node-id (c/exec binary :node :id :-q)]
-          (info node "node id:" node-id)
-          (c/on-many (:nodes test)
-                     (c/exec binary :node :connect node-id))
-          (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))
-        (if (= node (first (:nodes test)))
-          (do
-            (c/exec :sleep 5)
-            (c/exec binary :layout :apply :--version 1)
-            (info node "garage status:" (c/exec binary :status))
-            (c/exec binary :key :create grg-key)
-            (c/exec binary :bucket :create grg-bucket)
-            (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
-            (info node "key info: " (c/exec binary :key :info grg-key))))))
+      (install! node version)
+      (configure! node)
+      (cu/start-daemon!
+        {:logfile logfile
+         :pidfile pidfile
+         :chdir dir}
+        binary
+        :server)
+      (c/exec :sleep 3)
+
+      (jepsen/synchronize test)
+      (connect-node! test node)
+
+      (jepsen/synchronize test)
+      (configure-node! test node)
+
+      (jepsen/synchronize test)
+      (when (= node (jepsen/primary test))
+        (finalize-config! node)))
+
     (teardown! [_ test node]
       (info node "tearing down garage" version)
       (c/su
         (cu/stop-daemon! binary pidfile)
         (c/exec :rm :-rf data-dir)
         (c/exec :rm :-rf meta-dir)))
+
     db/LogFiles
     (log-files [_ test node]
       [logfile])))
-- 
cgit v1.2.3


From ddd3de7fcef2a2e05805766085c156d7fde9e12e Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 18 Oct 2023 16:30:45 +0200
Subject: refactor jepsen code

---
 script/jepsen.garage/src/jepsen/garage.clj        |   2 +-
 script/jepsen.garage/src/jepsen/garage/daemon.clj | 134 ++++++++++++++++
 script/jepsen.garage/src/jepsen/garage/grg.clj    | 178 ----------------------
 script/jepsen.garage/src/jepsen/garage/reg.clj    |   9 +-
 script/jepsen.garage/src/jepsen/garage/s3api.clj  |  48 ++++++
 script/jepsen.garage/src/jepsen/garage/set.clj    |   9 +-
 6 files changed, 193 insertions(+), 187 deletions(-)
 create mode 100644 script/jepsen.garage/src/jepsen/garage/daemon.clj
 delete mode 100644 script/jepsen.garage/src/jepsen/garage/grg.clj
 create mode 100644 script/jepsen.garage/src/jepsen/garage/s3api.clj

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 754ddf7d..5816512b 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -9,7 +9,7 @@
      [tests :as tests]]
     [jepsen.os.debian :as debian]
     [jepsen.garage
-     [grg :as grg]
+     [daemon :as grg]
      [reg :as reg]
      [set :as set]]))
 
diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj
new file mode 100644
index 00000000..81163521
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj
@@ -0,0 +1,134 @@
+(ns jepsen.garage.daemon
+  (:require [clojure.tools.logging :refer :all]
+            [jepsen [control :as c]
+             [core :as jepsen]
+             [db :as db]]
+            [jepsen.control.util :as cu]))
+
+; CONSTANTS -- HOW GARAGE IS SET UP
+
+(def base-dir "/opt/garage")
+(def data-dir (str base-dir "/data"))
+(def meta-dir (str base-dir "/meta"))
+(def binary (str base-dir "/garage"))
+(def logfile (str base-dir "/garage.log"))
+(def pidfile (str base-dir "/garage.pid"))
+
+(def admin-token "icanhazadmin")
+(def access-key "jepsen")
+(def bucket-name "jepsen")
+
+; THE GARAGE DB
+
+(defn install!
+  "Download and install Garage"
+  [node version]
+  (c/su
+    (c/trace
+      (info node "installing garage" version)
+      (c/exec :mkdir :-p base-dir)
+      (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
+            cache (cu/cached-wget! url)]
+        (c/exec :cp cache binary))
+      (c/exec :chmod :+x binary))))
+
+(defn configure!
+  "Configure Garage"
+  [node]
+  (c/su
+    (c/trace
+      (cu/write-file!
+        (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
+             "rpc_bind_addr = \"0.0.0.0:3901\"\n"
+             "rpc_public_addr = \"" node ":3901\"\n"
+             "db_engine = \"lmdb\"\n"
+             "replication_mode = \"3\"\n"
+             "data_dir = \"" data-dir "\"\n"
+             "metadata_dir = \"" meta-dir "\"\n"
+             "[s3_api]\n"
+             "s3_region = \"us-east-1\"\n"
+             "api_bind_addr = \"0.0.0.0:3900\"\n"
+             "[k2v_api]\n"
+             "api_bind_addr = \"0.0.0.0:3902\"\n"
+             "[admin]\n"
+             "api_bind_addr = \"0.0.0.0:3903\"\n"
+             "admin_token = \"" admin-token "\"\n")
+        "/etc/garage.toml"))))
+
+(defn connect-node!
+  "Connect a Garage node to the rest of the cluster"
+  [test node]
+  (c/trace
+    (let [node-id (c/exec binary :node :id :-q)]
+      (info node "node id:" node-id)
+      (c/on-many (:nodes test)
+                 (c/exec binary :node :connect node-id)))))
+
+(defn configure-node!
+  "Configure a Garage node to be part of a cluster layout"
+  [test node]
+  (c/trace
+    (let [node-id (c/exec binary :node :id :-q)]
+      (c/on (jepsen/primary test)
+          (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)))))
+
+(defn finalize-config!
+  "Apply the layout and create a key/bucket pair in the cluster"
+  [node]
+  (c/trace
+    (c/exec binary :layout :apply :--version 1)
+    (info node "garage status:" (c/exec binary :status))
+    (c/exec binary :key :create access-key)
+    (c/exec binary :bucket :create bucket-name)
+    (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key)
+    (info node "key info: " (c/exec binary :key :info access-key))))
+
+(defn db
+  "Garage DB for a particular version"
+  [version]
+  (reify db/DB
+    (setup! [_ test node]
+      (install! node version)
+      (configure! node)
+      (cu/start-daemon!
+        {:logfile logfile
+         :pidfile pidfile
+         :chdir base-dir}
+        binary
+        :server)
+      (c/exec :sleep 3)
+
+      (jepsen/synchronize test)
+      (connect-node! test node)
+
+      (jepsen/synchronize test)
+      (configure-node! test node)
+
+      (jepsen/synchronize test)
+      (when (= node (jepsen/primary test))
+        (finalize-config! node)))
+
+    (teardown! [_ test node]
+      (info node "tearing down garage" version)
+      (c/su
+        (cu/stop-daemon! binary pidfile)
+        (c/exec :rm :-rf data-dir)
+        (c/exec :rm :-rf meta-dir)))
+
+    db/LogFiles
+    (log-files [_ test node]
+      [logfile])))
+
+(defn creds
+  "Obtain Garage credentials for node"
+  [node]
+  (let [key-info (c/on node (c/exec binary :key :info access-key :--show-secret))
+        [_ ak sk] (re-matches
+                    #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
+                    key-info)]
+    {:access-key ak
+     :secret-key sk
+     :endpoint (str "http://" node ":3900")
+     :bucket bucket-name
+     :client-config {:path-style-access-enabled true}}))
+
diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
deleted file mode 100644
index f6075762..00000000
--- a/script/jepsen.garage/src/jepsen/garage/grg.clj
+++ /dev/null
@@ -1,178 +0,0 @@
-(ns jepsen.garage.grg
-  (:require [clojure.tools.logging :refer :all]
-            [jepsen [control :as c]
-             [core :as jepsen]
-             [db :as db]]
-            [jepsen.control.util :as cu]
-            [amazonica.aws.s3 :as s3]
-            [slingshot.slingshot :refer [try+]]))
-
-; CONSTANTS -- HOW GARAGE IS SET UP
-
-(def dir "/opt/garage")
-(def data-dir (str dir "/data"))
-(def meta-dir (str dir "/meta"))
-(def binary (str dir "/garage"))
-(def logfile (str dir "/garage.log"))
-(def pidfile (str dir "/garage.pid"))
-
-(def grg-admin-token "icanhazadmin")
-(def grg-key "jepsen")
-(def grg-bucket "jepsen")
-
-; THE GARAGE DB
-
-(defn install!
-  "Download and install Garage"
-  [node version]
-  (c/su
-    (c/trace
-      (info node "installing garage" version)
-      (c/exec :mkdir :-p dir)
-      (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
-            cache (cu/cached-wget! url)]
-        (c/exec :cp cache binary))
-      (c/exec :chmod :+x binary))))
-
-(defn configure!
-  "Configure Garage"
-  [node]
-  (c/su
-    (c/trace
-      (cu/write-file!
-        (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
-             "rpc_bind_addr = \"0.0.0.0:3901\"\n"
-             "rpc_public_addr = \"" node ":3901\"\n"
-             "db_engine = \"lmdb\"\n"
-             "replication_mode = \"3\"\n"
-             "data_dir = \"" dir "/data\"\n"
-             "metadata_dir = \"" dir "/meta\"\n"
-             "[s3_api]\n"
-             "s3_region = \"us-east-1\"\n"
-             "api_bind_addr = \"0.0.0.0:3900\"\n"
-             "[k2v_api]\n"
-             "api_bind_addr = \"0.0.0.0:3902\"\n"
-             "[admin]\n"
-             "api_bind_addr = \"0.0.0.0:3903\"\n"
-             "admin_token = \"" grg-admin-token "\"\n")
-        "/etc/garage.toml"))))
-
-(defn connect-node!
-  "Connect a Garage node to the rest of the cluster"
-  [test node]
-  (c/trace
-    (let [node-id (c/exec binary :node :id :-q)]
-      (info node "node id:" node-id)
-      (c/on-many (:nodes test)
-                 (c/exec binary :node :connect node-id)))))
-
-(defn configure-node!
-  "Configure a Garage node to be part of a cluster layout"
-  [test node]
-  (c/trace
-    (let [node-id (c/exec binary :node :id :-q)]
-      (c/on (jepsen/primary test)
-          (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)))))
-
-(defn finalize-config!
-  "Apply the layout and create a key/bucket pair in the cluster"
-  [node]
-  (c/trace
-    (c/exec binary :layout :apply :--version 1)
-    (info node "garage status:" (c/exec binary :status))
-    (c/exec binary :key :create grg-key)
-    (c/exec binary :bucket :create grg-bucket)
-    (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
-    (info node "key info: " (c/exec binary :key :info grg-key))))
-
-(defn db
-  "Garage DB for a particular version"
-  [version]
-  (reify db/DB
-    (setup! [_ test node]
-      (install! node version)
-      (configure! node)
-      (cu/start-daemon!
-        {:logfile logfile
-         :pidfile pidfile
-         :chdir dir}
-        binary
-        :server)
-      (c/exec :sleep 3)
-
-      (jepsen/synchronize test)
-      (connect-node! test node)
-
-      (jepsen/synchronize test)
-      (configure-node! test node)
-
-      (jepsen/synchronize test)
-      (when (= node (jepsen/primary test))
-        (finalize-config! node)))
-
-    (teardown! [_ test node]
-      (info node "tearing down garage" version)
-      (c/su
-        (cu/stop-daemon! binary pidfile)
-        (c/exec :rm :-rf data-dir)
-        (c/exec :rm :-rf meta-dir)))
-
-    db/LogFiles
-    (log-files [_ test node]
-      [logfile])))
-
-; GARAGE S3 HELPER FUNCTIONS
-
-(defn s3-creds
-  "Get S3 credentials for node"
-  [node]
-  (let [key-info (c/on node (c/exec binary :key :info grg-key :--show-secret))
-        [_ ak sk] (re-matches
-                    #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
-                    key-info)]
-    {:access-key ak
-     :secret-key sk
-     :endpoint (str "http://" node ":3900")
-     :bucket grg-bucket
-     :client-config {:path-style-access-enabled true}}))
-
-(defn s3-get
-  "Helper for GetObject"
-  [creds k]
-  (try+
-    (-> (s3/get-object creds (:bucket creds) k)
-        :input-stream
-        slurp)
-    (catch (re-find #"Key not found" (.getMessage %)) ex
-      nil)))
-
-(defn s3-put
-  "Helper for PutObject or DeleteObject (is a delete if value is nil)"
-  [creds k v]
-  (if (= v nil)
-    (s3/delete-object creds
-                      :bucket-name (:bucket creds)
-                      :key k)
-    (let [some-bytes (.getBytes v "UTF-8")
-          bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
-      (s3/put-object creds
-                     :bucket-name (:bucket creds)
-                     :key k
-                     :input-stream bytes-stream
-                     :metadata {:content-length (count some-bytes)}))))
-
-(defn s3-list
-  "Helper for ListObjects -- just lists everything in the bucket"
-  [creds prefix]
-  (defn list-inner [ct accum]
-    (let [list-result (s3/list-objects-v2 creds
-                                          {:bucket-name (:bucket creds)
-                                           :prefix prefix
-                                           :continuation-token ct})
-          new-object-summaries (:object-summaries list-result)
-          new-objects (map (fn [d] (:key d)) new-object-summaries)
-          objects (concat new-objects accum)]
-      (if (:truncated? list-result)
-        (list-inner (:next-continuation-token list-result) objects)
-        objects)))
-  (list-inner nil []))
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index 0b370b36..3a2c1d55 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -13,7 +13,8 @@
             [jepsen.checker.timeline :as timeline]
             [jepsen.control.util :as cu]
             [jepsen.os.debian :as debian]
-            [jepsen.garage.grg :as grg]
+            [jepsen.garage.daemon :as grg]
+            [jepsen.garage.s3api :as s3]
             [knossos.model :as model]
             [slingshot.slingshot :refer [try+]]))
 
@@ -24,7 +25,7 @@
 (defrecord RegClient [creds]
   client/Client
   (open! [this test node]
-    (let [creds (grg/s3-creds node)]
+    (let [creds (grg/creds node)]
       (info node "s3 credentials:" creds)
       (assoc this :creds creds)))
   (setup! [this test])
@@ -32,11 +33,11 @@
     (let [[k v] (:value op)]
       (case (:f op)
         :read
-          (let [value (grg/s3-get (:creds this) k)]
+          (let [value (s3/get (:creds this) k)]
             (assoc op :type :ok, :value (independent/tuple k value)))
         :write
           (do
-            (grg/s3-put (:creds this) k v)
+            (s3/put (:creds this) k v)
             (assoc op :type :ok)))))
   (teardown! [this test])
   (close! [this test]))
diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj
new file mode 100644
index 00000000..239d5423
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj
@@ -0,0 +1,48 @@
+(ns jepsen.garage.s3api
+  (:require [clojure.tools.logging :refer :all]
+            [jepsen [control :as c]]
+            [amazonica.aws.s3 :as s3]
+            [slingshot.slingshot :refer [try+]]))
+
+; GARAGE S3 HELPER FUNCTIONS
+
+(defn get
+  "Helper for GetObject"
+  [creds k]
+  (try+
+    (-> (s3/get-object creds (:bucket creds) k)
+        :input-stream
+        slurp)
+    (catch (re-find #"Key not found" (.getMessage %)) ex
+      nil)))
+
+(defn put
+  "Helper for PutObject or DeleteObject (is a delete if value is nil)"
+  [creds k v]
+  (if (= v nil)
+    (s3/delete-object creds
+                      :bucket-name (:bucket creds)
+                      :key k)
+    (let [some-bytes (.getBytes v "UTF-8")
+          bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
+      (s3/put-object creds
+                     :bucket-name (:bucket creds)
+                     :key k
+                     :input-stream bytes-stream
+                     :metadata {:content-length (count some-bytes)}))))
+
+(defn list
+  "Helper for ListObjects -- just lists everything in the bucket"
+  [creds prefix]
+  (defn list-inner [ct accum]
+    (let [list-result (s3/list-objects-v2 creds
+                                          {:bucket-name (:bucket creds)
+                                           :prefix prefix
+                                           :continuation-token ct})
+          new-object-summaries (:object-summaries list-result)
+          new-objects (map (fn [d] (:key d)) new-object-summaries)
+          objects (concat new-objects accum)]
+      (if (:truncated? list-result)
+        (list-inner (:next-continuation-token list-result) objects)
+        objects)))
+  (list-inner nil []))
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index 5f76d1ac..6bbc1ee0 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -15,7 +15,8 @@
             [jepsen.checker.timeline :as timeline]
             [jepsen.control.util :as cu]
             [jepsen.os.debian :as debian]
-            [jepsen.garage.grg :as grg]
+            [jepsen.garage.daemon :as grg]
+            [jepsen.garage.s3api :as s3]
             [knossos.model :as model]
             [slingshot.slingshot :refer [try+]]))
 
@@ -25,7 +26,7 @@
 (defrecord SetClient [creds]
   client/Client
   (open! [this test node]
-    (let [creds (grg/s3-creds node)]
+    (let [creds (grg/creds node)]
       (info node "s3 credentials:" creds)
       (assoc this :creds creds)))
   (setup! [this test])
@@ -35,10 +36,10 @@
       (case (:f op)
         :add
           (do
-            (grg/s3-put (:creds this) (str prefix v) "present")
+            (s3/put (:creds this) (str prefix v) "present")
             (assoc op :type :ok))
         :read
-          (let [items (grg/s3-list (:creds this) prefix)
+          (let [items (s3/list (:creds this) prefix)
                 items-stripped (map (fn [o] (str/replace-first o prefix "")) items)
                 items-set (set (map read-string items-stripped))]
             (assoc op :type :ok, :value (independent/tuple k items-set))))))
-- 
cgit v1.2.3


From b3bf16ee27b061a3a091022e718b2994365f945c Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 18 Oct 2023 17:51:34 +0200
Subject: make jepsen test more robust: handle errors and timeouts, fixed
 access key

---
 script/jepsen.garage/src/jepsen/garage.clj        | 17 ++++++++++++-----
 script/jepsen.garage/src/jepsen/garage/daemon.clj | 23 ++++++++++-------------
 script/jepsen.garage/src/jepsen/garage/reg.clj    | 22 ++++++++++++++--------
 3 files changed, 36 insertions(+), 26 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 5816512b..891fdf12 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -41,7 +41,7 @@
   [opts]
   (let [workload ((get workloads (:workload opts)) opts)
         garage-version (if (:increasing-timestamps opts)
-                         "03490d41d58576d7b3bcf977b2726d72a3a66ada"
+                         "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"
                          "v0.9.0")]
     (merge tests/noop-test
            opts
@@ -56,16 +56,23 @@
                                   (gen/stagger (/ (:rate opts)))
                                   (gen/nemesis
                                     (cycle [(gen/sleep 5)
-                                            {:type :info, :f :start}
+                                            ;{:type :info, :f :partition-start}
+                                            ;(gen/sleep 5)
+                                            {:type :info, :f :clock-scramble}
                                             (gen/sleep 5)
-                                            {:type :info, :f :stop}]))
+                                            ;{:type :info, :f :partition-stop}
+                                            ;(gen/sleep 5)
+                                            {:type :info, :f :clock-scramble}]))
                                   (gen/time-limit (:time-limit opts)))
                                 (gen/log "Healing cluster")
-                                (gen/nemesis (gen/once {:type :info, :f :stop}))
+                                (gen/nemesis (gen/once {:type :info, :f :partition-stop}))
                                 (gen/log "Waiting for recovery")
                                 (gen/sleep 10)
                                 (gen/clients (:final-generator workload)))
-            :nemesis          (nemesis/partition-random-halves)
+            :nemesis          (nemesis/compose
+                                {{:partition-start :start
+                                  :partition-stop :stop} (nemesis/partition-random-halves)
+                                 {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})
             :checker          (checker/compose
                                 {:perf (checker/perf)
                                  :workload (:checker workload)})
diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj
index 81163521..2dcfa94f 100644
--- a/script/jepsen.garage/src/jepsen/garage/daemon.clj
+++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj
@@ -15,7 +15,8 @@
 (def pidfile (str base-dir "/garage.pid"))
 
 (def admin-token "icanhazadmin")
-(def access-key "jepsen")
+(def access-key-id "GK8bfb6a51286071c6c9cd8bc3")
+(def secret-access-key "b0be95f71c1c6f16858a9edf395078b75c12ecb6b1c03385c4ae92076e4994a3")
 (def bucket-name "jepsen")
 
 ; THE GARAGE DB
@@ -78,10 +79,10 @@
   (c/trace
     (c/exec binary :layout :apply :--version 1)
     (info node "garage status:" (c/exec binary :status))
-    (c/exec binary :key :create access-key)
+    (c/exec binary :key :import access-key-id secret-access-key :--yes)
     (c/exec binary :bucket :create bucket-name)
-    (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key)
-    (info node "key info: " (c/exec binary :key :info access-key))))
+    (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key-id)
+    (info node "key info: " (c/exec binary :key :info access-key-id))))
 
 (defn db
   "Garage DB for a particular version"
@@ -122,13 +123,9 @@
 (defn creds
   "Obtain Garage credentials for node"
   [node]
-  (let [key-info (c/on node (c/exec binary :key :info access-key :--show-secret))
-        [_ ak sk] (re-matches
-                    #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
-                    key-info)]
-    {:access-key ak
-     :secret-key sk
-     :endpoint (str "http://" node ":3900")
-     :bucket bucket-name
-     :client-config {:path-style-access-enabled true}}))
+  {:access-key access-key-id
+   :secret-key secret-access-key
+   :endpoint (str "http://" node ":3900")
+   :bucket bucket-name
+   :client-config {:path-style-access-enabled true}})
 
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index 3a2c1d55..d5404c6b 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -33,19 +33,25 @@
     (let [[k v] (:value op)]
       (case (:f op)
         :read
-          (let [value (s3/get (:creds this) k)]
-            (assoc op :type :ok, :value (independent/tuple k value)))
+          (try+
+            (let [value (s3/get (:creds this) k)]
+              (assoc op :type :ok, :value (independent/tuple k value)))
+            (catch (re-find #"Unavailable" (.getMessage %)) ex
+              (assoc op :type :fail, :error [:s3-error (.getMessage ex)])))
         :write
-          (do
-            (s3/put (:creds this) k v)
-            (assoc op :type :ok)))))
+          (try+
+            (do
+              (s3/put (:creds this) k v)
+              (assoc op :type :ok))
+            (catch (re-find #"Unavailable" (.getMessage %)) ex
+              (assoc op :type :fail, :error [:s3-error (.getMessage ex)]))))))
   (teardown! [this test])
   (close! [this test]))
 
 (defn workload
   "Tests linearizable reads and writes"
   [opts]
-  {:client           (RegClient. nil)
+  {:client           (client/timeout 10 (RegClient. nil))
    :checker          (independent/checker
                        (checker/compose
                          {:linear (checker/linearizable
@@ -53,8 +59,8 @@
                                      :algorithm :linear})
                           :timeline (timeline/html)}))
    :generator        (independent/concurrent-generator
-                       10
-                       (range)
+                       (/ (:concurrency opts) 10) ; divide threads in 10 groups
+                       (range)                    ; working on 10 keys
                        (fn [k]
                          (->>
                            (gen/mix [op-get op-put op-del])
-- 
cgit v1.2.3


From 74e50eddddf319ce1a32a9b57b3825ea40db3a6c Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Thu, 19 Oct 2023 14:34:19 +0200
Subject: jepsen: refactoring

---
 script/jepsen.garage/src/jepsen/garage.clj        |  8 +++---
 script/jepsen.garage/src/jepsen/garage/daemon.clj |  2 +-
 script/jepsen.garage/src/jepsen/garage/reg.clj    | 27 ++++++++++----------
 script/jepsen.garage/src/jepsen/garage/set.clj    | 30 ++++++++++++++---------
 4 files changed, 37 insertions(+), 30 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 891fdf12..c8865248 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -56,12 +56,12 @@
                                   (gen/stagger (/ (:rate opts)))
                                   (gen/nemesis
                                     (cycle [(gen/sleep 5)
-                                            ;{:type :info, :f :partition-start}
-                                            ;(gen/sleep 5)
+                                            {:type :info, :f :partition-start}
+                                            (gen/sleep 5)
                                             {:type :info, :f :clock-scramble}
                                             (gen/sleep 5)
-                                            ;{:type :info, :f :partition-stop}
-                                            ;(gen/sleep 5)
+                                            {:type :info, :f :partition-stop}
+                                            (gen/sleep 5)
                                             {:type :info, :f :clock-scramble}]))
                                   (gen/time-limit (:time-limit opts)))
                                 (gen/log "Healing cluster")
diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj
index 2dcfa94f..f924dcd6 100644
--- a/script/jepsen.garage/src/jepsen/garage/daemon.clj
+++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj
@@ -43,7 +43,7 @@
              "rpc_bind_addr = \"0.0.0.0:3901\"\n"
              "rpc_public_addr = \"" node ":3901\"\n"
              "db_engine = \"lmdb\"\n"
-             "replication_mode = \"3\"\n"
+             "replication_mode = \"2\"\n"
              "data_dir = \"" data-dir "\"\n"
              "metadata_dir = \"" meta-dir "\"\n"
              "[s3_api]\n"
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index d5404c6b..f3d5cec5 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -9,6 +9,7 @@
              [generator :as gen]
              [independent :as independent]
              [nemesis :as nemesis]
+             [util :as util]
              [tests :as tests]]
             [jepsen.checker.timeline :as timeline]
             [jepsen.control.util :as cu]
@@ -25,33 +26,31 @@
 (defrecord RegClient [creds]
   client/Client
   (open! [this test node]
-    (let [creds (grg/creds node)]
-      (info node "s3 credentials:" creds)
-      (assoc this :creds creds)))
+    (assoc this :creds (grg/creds node)))
   (setup! [this test])
   (invoke! [this test op]
     (let [[k v] (:value op)]
       (case (:f op)
         :read
-          (try+
+          (util/timeout
+            10000
+            (assoc op :type :fail, :error ::timeout)
             (let [value (s3/get (:creds this) k)]
-              (assoc op :type :ok, :value (independent/tuple k value)))
-            (catch (re-find #"Unavailable" (.getMessage %)) ex
-              (assoc op :type :fail, :error [:s3-error (.getMessage ex)])))
+              (assoc op :type :ok, :value (independent/tuple k value))))
         :write
-          (try+
+          (util/timeout
+            10000
+            (assoc op :type :info, :error ::timeout)
             (do
               (s3/put (:creds this) k v)
-              (assoc op :type :ok))
-            (catch (re-find #"Unavailable" (.getMessage %)) ex
-              (assoc op :type :fail, :error [:s3-error (.getMessage ex)]))))))
+              (assoc op :type :ok))))))
   (teardown! [this test])
   (close! [this test]))
 
 (defn workload
   "Tests linearizable reads and writes"
   [opts]
-  {:client           (client/timeout 10 (RegClient. nil))
+  {:client           (RegClient. nil)
    :checker          (independent/checker
                        (checker/compose
                          {:linear (checker/linearizable
@@ -59,8 +58,8 @@
                                      :algorithm :linear})
                           :timeline (timeline/html)}))
    :generator        (independent/concurrent-generator
-                       (/ (:concurrency opts) 10) ; divide threads in 10 groups
-                       (range)                    ; working on 10 keys
+                       10
+                       (range)
                        (fn [k]
                          (->>
                            (gen/mix [op-get op-put op-del])
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index 6bbc1ee0..ff597095 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -11,6 +11,7 @@
              [generator :as gen]
              [independent :as independent]
              [nemesis :as nemesis]
+             [util :as util]
              [tests :as tests]]
             [jepsen.checker.timeline :as timeline]
             [jepsen.control.util :as cu]
@@ -26,23 +27,29 @@
 (defrecord SetClient [creds]
   client/Client
   (open! [this test node]
-    (let [creds (grg/creds node)]
-      (info node "s3 credentials:" creds)
-      (assoc this :creds creds)))
+    (assoc this :creds (grg/creds node)))
   (setup! [this test])
   (invoke! [this test op]
     (let [[k v] (:value op)
           prefix (str "set" k "/")]
       (case (:f op)
         :add
-          (do
-            (s3/put (:creds this) (str prefix v) "present")
-            (assoc op :type :ok))
+          (util/timeout
+            10000
+            (assoc op :type :info, :error ::timeout)
+            (do
+              (s3/put (:creds this) (str prefix v) "present")
+              (assoc op :type :ok)))
         :read
-          (let [items (s3/list (:creds this) prefix)
-                items-stripped (map (fn [o] (str/replace-first o prefix "")) items)
-                items-set (set (map read-string items-stripped))]
-            (assoc op :type :ok, :value (independent/tuple k items-set))))))
+          (util/timeout
+            10000
+            (assoc op :type :fail, :error ::timeout)
+            (let [items (s3/list (:creds this) prefix)
+                  items-stripped (map (fn [o]
+                                        (assert (str/starts-with? o prefix))
+                                        (str/replace-first o prefix "")) items)
+                  items-set (set (map parse-long items-stripped))]
+              (assoc op :type :ok, :value (independent/tuple k items-set)))))))
   (teardown! [this test])
   (close! [this test]))
 
@@ -110,6 +117,7 @@
                         10
                         (range)
                         (fn [k]
-                          (gen/mix [op-add-rand100 op-read])))})
+                          (->> (gen/mix [op-add-rand100 op-read])
+                               (gen/limit (:ops-per-key opts)))))})
 
 
-- 
cgit v1.2.3


From da8b1707489b70c25395ee49383ecbbd8c9f9404 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Thu, 19 Oct 2023 16:45:24 +0200
Subject: jepsen: investigating listobjects error

---
 script/jepsen.garage/src/jepsen/garage/daemon.clj |  7 +++++--
 script/jepsen.garage/src/jepsen/garage/s3api.clj  |  1 +
 script/jepsen.garage/src/jepsen/garage/set.clj    | 17 ++++++++++-------
 3 files changed, 16 insertions(+), 9 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj
index f924dcd6..7c581ba1 100644
--- a/script/jepsen.garage/src/jepsen/garage/daemon.clj
+++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj
@@ -53,7 +53,8 @@
              "api_bind_addr = \"0.0.0.0:3902\"\n"
              "[admin]\n"
              "api_bind_addr = \"0.0.0.0:3903\"\n"
-             "admin_token = \"" admin-token "\"\n")
+             "admin_token = \"" admin-token "\"\n"
+             "trace_sink = \"http://192.168.56.1:4317\"\n")
         "/etc/garage.toml"))))
 
 (defn connect-node!
@@ -94,7 +95,8 @@
       (cu/start-daemon!
         {:logfile logfile
          :pidfile pidfile
-         :chdir base-dir}
+         :chdir base-dir
+         :env {:RUST_LOG "garage=debug,garage_api=trace"}}
         binary
         :server)
       (c/exec :sleep 3)
@@ -113,6 +115,7 @@
       (info node "tearing down garage" version)
       (c/su
         (cu/stop-daemon! binary pidfile)
+        (c/exec :rm :-rf logfile)
         (c/exec :rm :-rf data-dir)
         (c/exec :rm :-rf meta-dir)))
 
diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj
index 239d5423..977de7dc 100644
--- a/script/jepsen.garage/src/jepsen/garage/s3api.clj
+++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj
@@ -42,6 +42,7 @@
           new-object-summaries (:object-summaries list-result)
           new-objects (map (fn [d] (:key d)) new-object-summaries)
           objects (concat new-objects accum)]
+      (info (:endpoint creds) "ListObjectsV2 prefix(" prefix "), ct(" ct "): " new-objects)
       (if (:truncated? list-result)
         (list-inner (:next-continuation-token list-result) objects)
         objects)))
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index ff597095..8a1ab83f 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -44,12 +44,13 @@
           (util/timeout
             10000
             (assoc op :type :fail, :error ::timeout)
-            (let [items (s3/list (:creds this) prefix)
-                  items-stripped (map (fn [o]
+            (let [items (s3/list (:creds this) prefix)]
+              (info "list results for prefix" prefix ":" items " (node:" (:endpoint (:creds this)) ")")
+              (let [items-stripped (map (fn [o]
                                         (assert (str/starts-with? o prefix))
                                         (str/replace-first o prefix "")) items)
-                  items-set (set (map parse-long items-stripped))]
-              (assoc op :type :ok, :value (independent/tuple k items-set)))))))
+                    items-set (set (map parse-long items-stripped))]
+                (assoc op :type :ok, :value (independent/tuple k items-set))))))))
   (teardown! [this test])
   (close! [this test]))
 
@@ -100,9 +101,11 @@
                           (->> (range)
                            (map (fn [x] {:type :invoke, :f :add, :value x}))
                            (gen/limit (:ops-per-key opts)))))
-   :final-generator   (independent/sequential-generator
-                        (range 100)
-                        (fn [k] (gen/once op-read)))})
+   :final-generator   (gen/phases
+                        (independent/sequential-generator
+                          (range 100)
+                          (fn [k] (gen/once op-read)))
+                        (gen/sleep 5))})
 
 (defn workload2
   "Tests insertions and deletions"
-- 
cgit v1.2.3


From ef662822c9e48ff7cfd9300590617e089c0a9498 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Thu, 19 Oct 2023 23:40:55 +0200
Subject: jepsen: fix the list-objects call (?)

---
 script/jepsen.garage/src/jepsen/garage/s3api.clj | 27 ++++++++++++------------
 script/jepsen.garage/src/jepsen/garage/set.clj   | 16 ++++++++------
 2 files changed, 23 insertions(+), 20 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj
index 977de7dc..4f292ac0 100644
--- a/script/jepsen.garage/src/jepsen/garage/s3api.clj
+++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj
@@ -31,19 +31,20 @@
                      :input-stream bytes-stream
                      :metadata {:content-length (count some-bytes)}))))
 
+(defn list-inner [creds prefix ct accum]
+  (let [list-result (s3/list-objects-v2 creds
+                                        {:bucket-name (:bucket creds)
+                                         :prefix prefix
+                                         :continuation-token ct})
+        new-object-summaries (:object-summaries list-result)
+        new-objects (map (fn [d] (:key d)) new-object-summaries)
+        objects (concat new-objects accum)]
+    (info (:endpoint creds) "ListObjectsV2 prefix(" prefix "), ct(" ct "): " new-objects)
+    (if (:truncated? list-result)
+      (list-inner creds prefix (:next-continuation-token list-result) objects)
+      objects)))
 (defn list
   "Helper for ListObjects -- just lists everything in the bucket"
   [creds prefix]
-  (defn list-inner [ct accum]
-    (let [list-result (s3/list-objects-v2 creds
-                                          {:bucket-name (:bucket creds)
-                                           :prefix prefix
-                                           :continuation-token ct})
-          new-object-summaries (:object-summaries list-result)
-          new-objects (map (fn [d] (:key d)) new-object-summaries)
-          objects (concat new-objects accum)]
-      (info (:endpoint creds) "ListObjectsV2 prefix(" prefix "), ct(" ct "): " new-objects)
-      (if (:truncated? list-result)
-        (list-inner (:next-continuation-token list-result) objects)
-        objects)))
-  (list-inner nil []))
+  (info "in s3/list creds:" creds ", prefix:" prefix)
+  (list-inner creds prefix nil []))
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index 8a1ab83f..c5387462 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -44,13 +44,15 @@
           (util/timeout
             10000
             (assoc op :type :fail, :error ::timeout)
-            (let [items (s3/list (:creds this) prefix)]
-              (info "list results for prefix" prefix ":" items " (node:" (:endpoint (:creds this)) ")")
-              (let [items-stripped (map (fn [o]
-                                        (assert (str/starts-with? o prefix))
-                                        (str/replace-first o prefix "")) items)
-                    items-set (set (map parse-long items-stripped))]
-                (assoc op :type :ok, :value (independent/tuple k items-set))))))))
+            (do
+              (info "call s3/list creds: " (:creds this) ", prefix:" prefix)
+              (let [items (s3/list (:creds this) prefix)]
+                (info "list results for prefix" prefix ":" items " (node:" (:endpoint (:creds this)) ")")
+                (let [items-stripped (map (fn [o]
+                                          (assert (str/starts-with? o prefix))
+                                          (str/replace-first o prefix "")) items)
+                      items-set (set (map parse-long items-stripped))]
+                  (assoc op :type :ok, :value (independent/tuple k items-set)))))))))
   (teardown! [this test])
   (close! [this test]))
 
-- 
cgit v1.2.3


From 4ba18ce9cca1b828edcf3f8c8770d49c75ed3083 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Fri, 20 Oct 2023 12:13:11 +0200
Subject: jepsen: wip checker for register-like behavior

---
 script/jepsen.garage/src/jepsen/garage/reg.clj | 66 ++++++++++++++++++++++++--
 1 file changed, 62 insertions(+), 4 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index f3d5cec5..b5bf28ff 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -1,6 +1,7 @@
 (ns jepsen.garage.reg
   (:require [clojure.tools.logging :refer :all]
             [clojure.string :as str]
+            [clojure.set :as set]
             [jepsen [checker :as checker]
              [cli :as cli]
              [client :as client]
@@ -20,7 +21,7 @@
             [slingshot.slingshot :refer [try+]]))
 
 (defn op-get [_ _] {:type :invoke, :f :read, :value nil})
-(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 9))})
+(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 99))})
 (defn op-del [_ _] {:type :invoke, :f :write, :value nil})
 
 (defrecord RegClient [creds]
@@ -47,15 +48,72 @@
   (teardown! [this test])
   (close! [this test]))
 
+(defn reg-read-after-write
+  "Read-after-Write checker for register operations"
+  []
+  (reify checker/Checker
+    (check [this test history opts]
+      (let [init {:put-values {-1 nil}
+                  :put-done #{-1}
+                  :put-in-progress {}
+                  :read-can-contain {}
+                  :bad-reads #{}}
+            final (reduce
+                    (fn [state op]
+                      (let [current-values (set/union
+                                             (set (map (fn [idx] (get (:put-values state) idx)) (:put-done state)))
+                                             (set (map (fn [[_ [idx _]]] (get (:put-values state) idx)) (:put-in-progress state))))
+                            read-can-contain (reduce
+                                               (fn [rcc [idx v]] (assoc rcc idx (set/union current-values v)))
+                                               {} (:read-can-contain state))]
+                      (info "--------")
+                      (info "state: " state)
+                      (info "current-values: " current-values)
+                      (info "read-can-contain: " read-can-contain)
+                      (info "op: " op)
+                      (case [(:type op) (:f op)]
+                        ([:invoke :write])
+                          (assoc state
+                                 :read-can-contain read-can-contain
+                                 :put-values (assoc (:put-values state) (:index op) (:value op))
+                                 :put-in-progress (assoc (:put-in-progress state) (:process op) [(:index op) (:put-done state)]))
+                        ([:ok :write])
+                          (let [[index overwrites] (get (:put-in-progress state) (:process op))]
+                            (assoc state
+                                   :read-can-contain read-can-contain
+                                   :put-in-progress (dissoc (:put-in-progress state) (:process op))
+                                   :put-done
+                                     (conj
+                                       (set/difference (:put-done state) overwrites)
+                                       index)))
+                        ([:invoke :read])
+                         (assoc state
+                                :read-can-contain (assoc read-can-contain (:process op) current-values))
+                        ([:ok :read])
+                         (let [this-read-can-contain (get read-can-contain (:process op))
+                               bad-reads (if (contains? this-read-can-contain (:value op))
+                                           (:bad-reads state)
+                                           (conj (:bad-reads state) [(:process op) (:index op) (:value op) this-read-can-contain]))]
+                           (info "this-read-can-contain: " this-read-can-contain)
+                           (assoc state
+                                  :read-can-contain (dissoc read-can-contain (:process op))
+                                  :bad-reads bad-reads))
+                        state)))
+                    init history)
+            valid? (empty? (:bad-reads final))]
+        (assoc final :valid? valid?)))))
+
 (defn workload
   "Tests linearizable reads and writes"
   [opts]
   {:client           (RegClient. nil)
    :checker          (independent/checker
                        (checker/compose
-                         {:linear (checker/linearizable
-                                    {:model     (model/register)
-                                     :algorithm :linear})
+                         {:reg-read-after-write (reg-read-after-write)
+                          ; linear test is desactivated, indeed Garage is not linear
+                          ;:linear (checker/linearizable
+                          ;          {:model     (model/register)
+                          ;           :algorithm :linear})
                           :timeline (timeline/html)}))
    :generator        (independent/concurrent-generator
                        10
-- 
cgit v1.2.3


From 4b93ce179a3777c8461f3b5843dc3802bddc739c Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Fri, 20 Oct 2023 12:56:45 +0200
Subject: jepsen: errors in reg2 workload under investigation

---
 script/jepsen.garage/src/jepsen/garage.clj     |  3 ++-
 script/jepsen.garage/src/jepsen/garage/reg.clj | 30 +++++++++++++++++---------
 2 files changed, 22 insertions(+), 11 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index c8865248..be192a7f 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -15,7 +15,8 @@
 
 (def workloads
   "A map of workload names to functions that construct workloads, given opts."
-  {"reg"  reg/workload
+  {"reg1" reg/workload1
+   "reg2" reg/workload2
    "set1" set/workload1
    "set2" set/workload2})
 
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index b5bf28ff..6772abfe 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -103,18 +103,10 @@
             valid? (empty? (:bad-reads final))]
         (assoc final :valid? valid?)))))
 
-(defn workload
-  "Tests linearizable reads and writes"
+(defn workload-common
+  "Common parts of workload"
   [opts]
   {:client           (RegClient. nil)
-   :checker          (independent/checker
-                       (checker/compose
-                         {:reg-read-after-write (reg-read-after-write)
-                          ; linear test is desactivated, indeed Garage is not linear
-                          ;:linear (checker/linearizable
-                          ;          {:model     (model/register)
-                          ;           :algorithm :linear})
-                          :timeline (timeline/html)}))
    :generator        (independent/concurrent-generator
                        10
                        (range)
@@ -123,4 +115,22 @@
                            (gen/mix [op-get op-put op-del])
                            (gen/limit (:ops-per-key opts)))))})
 
+(defn workload1
+  "Tests linearizable reads and writes"
+  [opts]
+  (assoc (workload-common opts)
+         :checker (independent/checker
+                    (checker/compose
+                      {:linear (checker/linearizable
+                                 {:model     (model/register)
+                                  :algorithm :linear})
+                       :timeline (timeline/html)}))))
 
+(defn workload2
+  "Tests CRDT reads and writes"
+  [opts]
+  (assoc (workload-common opts)
+         :checker (independent/checker
+                    (checker/compose
+                      {:reg-read-after-write (reg-read-after-write)
+                       :timeline (timeline/html)}))))
-- 
cgit v1.2.3


From d148b83d4f440dc79b2ed08eaa171aca0e2037b0 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Fri, 20 Oct 2023 13:36:48 +0200
Subject: jepsen: reg2 failure seems to happen only with deleteobject

---
 script/jepsen.garage/src/jepsen/garage.clj     | 14 +++++++++-----
 script/jepsen.garage/src/jepsen/garage/reg.clj |  3 ++-
 2 files changed, 11 insertions(+), 6 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index be192a7f..ce02b7f7 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -20,10 +20,16 @@
    "set1" set/workload1
    "set2" set/workload2})
 
+(def patches
+  "A map of patch names to Garage builds"
+  {"default" "v0.9.0"
+   "tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"})
+
 (def cli-opts
   "Additional command line options."
-  [["-I" "--increasing-timestamps" "Garage version with increasing timestamps on PutObject"
-    :default false]
+  [["-p" "--patch NAME" "Garage patch to use"
+    :default "default"
+    :validate [patches (cli/one-of patches)]]
    ["-r" "--rate HZ" "Approximate number of requests per second, per thread."
     :default  10
     :parse-fn read-string
@@ -41,9 +47,7 @@
   :concurrency, ...), constructs a test map."
   [opts]
   (let [workload ((get workloads (:workload opts)) opts)
-        garage-version (if (:increasing-timestamps opts)
-                         "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"
-                         "v0.9.0")]
+        garage-version (get patches (:patch opts))]
     (merge tests/noop-test
            opts
            {:pure-generators  true
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index 6772abfe..ecc96590 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -112,7 +112,8 @@
                        (range)
                        (fn [k]
                          (->>
-                           (gen/mix [op-get op-put op-del])
+                           ; (gen/mix [op-get op-put op-del])
+                           (gen/mix [op-get op-put])
                            (gen/limit (:ops-per-key opts)))))})
 
 (defn workload1
-- 
cgit v1.2.3


From f5b09727815523a1bd4ba5f62d892b2b45b5bed6 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Fri, 20 Oct 2023 15:00:10 +0200
Subject: jepsen: register crdt read-after-write is fixed with deleteobject
 patch

---
 script/jepsen.garage/src/jepsen/garage.clj     | 3 ++-
 script/jepsen.garage/src/jepsen/garage/reg.clj | 3 +--
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index ce02b7f7..a566d9be 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -23,7 +23,8 @@
 (def patches
   "A map of patch names to Garage builds"
   {"default" "v0.9.0"
-   "tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"})
+   "tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"
+   "tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1"})
 
 (def cli-opts
   "Additional command line options."
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index ecc96590..6772abfe 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -112,8 +112,7 @@
                        (range)
                        (fn [k]
                          (->>
-                           ; (gen/mix [op-get op-put op-del])
-                           (gen/mix [op-get op-put])
+                           (gen/mix [op-get op-put op-del])
                            (gen/limit (:ops-per-key opts)))))})
 
 (defn workload1
-- 
cgit v1.2.3


From 654775308ed03abd68941b07b2ad367a4de5c57f Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Fri, 20 Oct 2023 15:48:37 +0200
Subject: jepsen: add cluster reconfiguration nemesis

---
 script/jepsen.garage/src/jepsen/garage.clj         | 27 ++++---
 script/jepsen.garage/src/jepsen/garage/nemesis.clj | 91 ++++++++++++++++++++++
 2 files changed, 104 insertions(+), 14 deletions(-)
 create mode 100644 script/jepsen.garage/src/jepsen/garage/nemesis.clj

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index a566d9be..65a92a76 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -10,6 +10,7 @@
     [jepsen.os.debian :as debian]
     [jepsen.garage
      [daemon :as grg]
+     [nemesis :as grgNemesis]
      [reg :as reg]
      [set :as set]]))
 
@@ -20,6 +21,11 @@
    "set1" set/workload1
    "set2" set/workload2})
 
+(def scenari
+  "A map of scenari to the associated nemesis"
+  {"cp" grgNemesis/scenario-cp
+   "r"  grgNemesis/scenario-r})
+
 (def patches
   "A map of patch names to Garage builds"
   {"default" "v0.9.0"
@@ -31,6 +37,9 @@
   [["-p" "--patch NAME" "Garage patch to use"
     :default "default"
     :validate [patches (cli/one-of patches)]]
+   ["-s" "--scenario NAME" "Nemesis scenario to run"
+    :default "cp"
+    :validate [scenari (cli/one-of scenari)]]
    ["-r" "--rate HZ" "Approximate number of requests per second, per thread."
     :default  10
     :parse-fn read-string
@@ -40,7 +49,7 @@
     :parse-fn parse-long
     :validate [pos? "Must be a positive integer."]]
    ["-w" "--workload NAME" "Workload of test to run"
-    :default "reg"
+    :default "reg1"
     :validate [workloads (cli/one-of workloads)]]])
 
 (defn garage-test
@@ -48,6 +57,7 @@
   :concurrency, ...), constructs a test map."
   [opts]
   (let [workload ((get workloads (:workload opts)) opts)
+        scenario ((get scenari (:scenario opts)) opts)
         garage-version (get patches (:patch opts))]
     (merge tests/noop-test
            opts
@@ -60,25 +70,14 @@
                                 (->>
                                   (:generator workload)
                                   (gen/stagger (/ (:rate opts)))
-                                  (gen/nemesis
-                                    (cycle [(gen/sleep 5)
-                                            {:type :info, :f :partition-start}
-                                            (gen/sleep 5)
-                                            {:type :info, :f :clock-scramble}
-                                            (gen/sleep 5)
-                                            {:type :info, :f :partition-stop}
-                                            (gen/sleep 5)
-                                            {:type :info, :f :clock-scramble}]))
+                                  (gen/nemesis (:generator scenario))
                                   (gen/time-limit (:time-limit opts)))
                                 (gen/log "Healing cluster")
                                 (gen/nemesis (gen/once {:type :info, :f :partition-stop}))
                                 (gen/log "Waiting for recovery")
                                 (gen/sleep 10)
                                 (gen/clients (:final-generator workload)))
-            :nemesis          (nemesis/compose
-                                {{:partition-start :start
-                                  :partition-stop :stop} (nemesis/partition-random-halves)
-                                 {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})
+            :nemesis          (:nemesis scenario)
             :checker          (checker/compose
                                 {:perf (checker/perf)
                                  :workload (:checker workload)})
diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
new file mode 100644
index 00000000..7cd9306e
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
@@ -0,0 +1,91 @@
+(ns jepsen.garage.nemesis
+  (:require [clojure.tools.logging :refer :all]
+            [jepsen [control :as c]
+             [core :as jepsen]
+             [generator :as gen]
+             [nemesis :as nemesis]]
+            [jepsen.garage.daemon :as grg]
+            [jepsen.control.util :as cu]))
+
+(defn configure-present!
+  "Configure node to be active in new cluster layout"
+  [test node]
+  (info "configure-present!" node)
+  (let [node-id (c/on node (c/exec grg/binary :node :id :-q))]
+   (c/on
+     (jepsen/primary test)
+     (c/exec grg/binary :layout :assign (subs node-id 0 16) :-c :1G))))
+
+(defn configure-absent!
+  "Configure node to be active in new cluster layout"
+  [test node]
+  (info "configure-absent!" node)
+  (let [node-id (c/on node (c/exec grg/binary :node :id :-q))]
+   (c/on
+     (jepsen/primary test)
+     (c/exec grg/binary :layout :assign (subs node-id 0 16) :-g))))
+
+(defn finalize-config!
+  "Apply the proposed cluster layout"
+  [test]
+  (let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show))
+        [_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)]
+    (info "layout show: " layout-show "; next-version: " layout-next-version)
+    (c/on (jepsen/primary test)
+          (c/exec grg/binary :layout :apply :--version layout-next-version))))
+
+(defn reconfigure-subset
+  "Reconfigure cluster with only a subset of nodes"
+  [cnt]
+  (reify nemesis/Nemesis
+    (setup! [this test] this)
+
+    (invoke! [this test op] op
+      (case (:f op)
+        :start
+          (let [[keep-nodes remove-nodes]
+                (->> (:nodes test)
+                     shuffle
+                     (split-at cnt))]
+            (info "layout split: keep " keep-nodes ", remove " remove-nodes)
+            (run! #(configure-present! test %) keep-nodes)
+            (run! #(configure-absent! test %) remove-nodes)
+            (finalize-config! test)
+            (assoc op :value keep-nodes))
+        :stop
+          (do
+            (info "layout un-split: all nodes=" (:nodes test))
+            (run! #(configure-present! test %) (:nodes test))
+            (finalize-config! test)
+            (assoc op :value (:nodes test)))))
+
+    (teardown! [this test] this)))
+
+(defn scenario-cp
+  "Clock scramble + parittion scenario"
+  [opts]
+  {:generator      (cycle [(gen/sleep 5)
+                           {:type :info, :f :partition-start}
+                           (gen/sleep 5)
+                           {:type :info, :f :clock-scramble}
+                           (gen/sleep 5)
+                           {:type :info, :f :partition-stop}
+                           (gen/sleep 5)
+                           {:type :info, :f :clock-scramble}])
+   :nemesis        (nemesis/compose
+                     {{:partition-start :start
+                       :partition-stop :stop} (nemesis/partition-random-halves)
+                      {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
+
+(defn scenario-r
+  "Cluster reconfiguration scenario"
+  [opts]
+  {:generator      (cycle [(gen/sleep 5)
+                           {:type :info, :f :reconfigure-start}
+                           (gen/sleep 5)
+                           {:type :info, :f :reconfigure-start}
+                           (gen/sleep 5)
+                           {:type :info, :f :reconfigure-stop}])
+   :nemesis        (nemesis/compose
+                     {{:reconfigure-start :start
+                       :reconfigure-stop :stop} (reconfigure-subset 3)})})
-- 
cgit v1.2.3


From 9030c1eef8f4c7c3435835f15e6b236497f93323 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Fri, 20 Oct 2023 15:53:46 +0200
Subject: jepsen: code path for nemesis final generator

---
 script/jepsen.garage/src/jepsen/garage.clj         |  2 +-
 script/jepsen.garage/src/jepsen/garage/nemesis.clj | 44 +++++++++++-----------
 2 files changed, 24 insertions(+), 22 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 65a92a76..ace3e66a 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -73,7 +73,7 @@
                                   (gen/nemesis (:generator scenario))
                                   (gen/time-limit (:time-limit opts)))
                                 (gen/log "Healing cluster")
-                                (gen/nemesis (gen/once {:type :info, :f :partition-stop}))
+                                (gen/nemesis (:final-generator scenario))
                                 (gen/log "Waiting for recovery")
                                 (gen/sleep 10)
                                 (gen/clients (:final-generator workload)))
diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
index 7cd9306e..9edfe418 100644
--- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj
+++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
@@ -64,28 +64,30 @@
 (defn scenario-cp
   "Clock scramble + parittion scenario"
   [opts]
-  {:generator      (cycle [(gen/sleep 5)
-                           {:type :info, :f :partition-start}
-                           (gen/sleep 5)
-                           {:type :info, :f :clock-scramble}
-                           (gen/sleep 5)
-                           {:type :info, :f :partition-stop}
-                           (gen/sleep 5)
-                           {:type :info, :f :clock-scramble}])
-   :nemesis        (nemesis/compose
-                     {{:partition-start :start
-                       :partition-stop :stop} (nemesis/partition-random-halves)
-                      {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
+  {:generator        (cycle [(gen/sleep 5)
+                             {:type :info, :f :partition-start}
+                             (gen/sleep 5)
+                             {:type :info, :f :clock-scramble}
+                             (gen/sleep 5)
+                             {:type :info, :f :partition-stop}
+                             (gen/sleep 5)
+                             {:type :info, :f :clock-scramble}])
+   :final-generator  (gen/once {:type :info, :f :partition-stop})
+   :nemesis          (nemesis/compose
+                       {{:partition-start :start
+                         :partition-stop :stop} (nemesis/partition-random-halves)
+                        {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
 
 (defn scenario-r
   "Cluster reconfiguration scenario"
   [opts]
-  {:generator      (cycle [(gen/sleep 5)
-                           {:type :info, :f :reconfigure-start}
-                           (gen/sleep 5)
-                           {:type :info, :f :reconfigure-start}
-                           (gen/sleep 5)
-                           {:type :info, :f :reconfigure-stop}])
-   :nemesis        (nemesis/compose
-                     {{:reconfigure-start :start
-                       :reconfigure-stop :stop} (reconfigure-subset 3)})})
+  {:generator        (cycle [(gen/sleep 5)
+                             {:type :info, :f :reconfigure-start}
+                             (gen/sleep 5)
+                             {:type :info, :f :reconfigure-start}
+                             (gen/sleep 5)
+                             {:type :info, :f :reconfigure-stop}])
+   :final-generator  (gen/once {:type :info, :f :reconfigure-stop})
+   :nemesis          (nemesis/compose
+                       {{:reconfigure-start :start
+                         :reconfigure-stop :stop} (reconfigure-subset 3)})})
-- 
cgit v1.2.3


From d2c365767b0a4cb70dcbb1d20b75f41e0f9c20c8 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 24 Oct 2023 11:39:45 +0200
Subject: jepsen: more testing

---
 script/jepsen.garage/src/jepsen/garage.clj         |  6 ++--
 script/jepsen.garage/src/jepsen/garage/nemesis.clj | 34 +++++++++++++++++++++-
 script/jepsen.garage/src/jepsen/garage/s3api.clj   |  2 --
 script/jepsen.garage/src/jepsen/garage/set.clj     |  6 ++--
 4 files changed, 39 insertions(+), 9 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index ace3e66a..6d64a1b8 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -23,8 +23,10 @@
 
 (def scenari
   "A map of scenari to the associated nemesis"
-  {"cp" grgNemesis/scenario-cp
-   "r"  grgNemesis/scenario-r})
+  {"c" grgNemesis/scenario-c
+   "cp" grgNemesis/scenario-cp
+   "r"  grgNemesis/scenario-r
+   "pr"  grgNemesis/scenario-pr})
 
 (def patches
   "A map of patch names to Garage builds"
diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
index 9edfe418..e64bcaf1 100644
--- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj
+++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
@@ -7,6 +7,8 @@
             [jepsen.garage.daemon :as grg]
             [jepsen.control.util :as cu]))
 
+; ---- reconfiguration nemesis ----
+
 (defn configure-present!
   "Configure node to be active in new cluster layout"
   [test node]
@@ -61,8 +63,18 @@
 
     (teardown! [this test] this)))
 
+; ---- nemesis scenari ----
+
+(defn scenario-c
+  "Clock scramble scenario"
+  [opts]
+  {:generator        (cycle [(gen/sleep 5)
+                             {:type :info, :f :clock-scramble}])
+   :nemesis          (nemesis/compose
+                        {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
+
 (defn scenario-cp
-  "Clock scramble + parittion scenario"
+  "Clock scramble + partition scenario"
   [opts]
   {:generator        (cycle [(gen/sleep 5)
                              {:type :info, :f :partition-start}
@@ -91,3 +103,23 @@
    :nemesis          (nemesis/compose
                        {{:reconfigure-start :start
                          :reconfigure-stop :stop} (reconfigure-subset 3)})})
+
+(defn scenario-pr
+  "Partition + cluster reconfiguration scenario"
+  [opts]
+  {:generator        (cycle [(gen/sleep 3)
+                             {:type :info, :f :reconfigure-start}
+                             (gen/sleep 3)
+                             {:type :info, :f :partition-start}
+                             (gen/sleep 3)
+                             {:type :info, :f :reconfigure-start}
+                             (gen/sleep 3)
+                             {:type :info, :f :partition-stop}
+                             (gen/sleep 3)
+                             {:type :info, :f :reconfigure-stop}])
+   :final-generator  (gen/once {:type :info, :f :partition-stop})
+   :nemesis          (nemesis/compose
+                       {{:partition-start :start
+                         :partition-stop :stop} (nemesis/partition-random-halves)
+                        {:reconfigure-start :start
+                         :reconfigure-stop :stop} (reconfigure-subset 3)})})
diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj
index 4f292ac0..531e0157 100644
--- a/script/jepsen.garage/src/jepsen/garage/s3api.clj
+++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj
@@ -39,12 +39,10 @@
         new-object-summaries (:object-summaries list-result)
         new-objects (map (fn [d] (:key d)) new-object-summaries)
         objects (concat new-objects accum)]
-    (info (:endpoint creds) "ListObjectsV2 prefix(" prefix "), ct(" ct "): " new-objects)
     (if (:truncated? list-result)
       (list-inner creds prefix (:next-continuation-token list-result) objects)
       objects)))
 (defn list
   "Helper for ListObjects -- just lists everything in the bucket"
   [creds prefix]
-  (info "in s3/list creds:" creds ", prefix:" prefix)
   (list-inner creds prefix nil []))
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index c5387462..f625e672 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -45,9 +45,7 @@
             10000
             (assoc op :type :fail, :error ::timeout)
             (do
-              (info "call s3/list creds: " (:creds this) ", prefix:" prefix)
               (let [items (s3/list (:creds this) prefix)]
-                (info "list results for prefix" prefix ":" items " (node:" (:endpoint (:creds this)) ")")
                 (let [items-stripped (map (fn [o]
                                           (assert (str/starts-with? o prefix))
                                           (str/replace-first o prefix "")) items)
@@ -115,8 +113,8 @@
   {:client            (SetClient. nil)
    :checker           (independent/checker
                         (checker/compose
-                          {:set-full (checker/set-full {:linearizable? false})
-                           :set-read-after-write (set-read-after-write)
+                          {:set-read-after-write (set-read-after-write)
+                           ; :set-full (checker/set-full {:linearizable? false})
                            :timeline (timeline/html)}))
    :generator         (independent/concurrent-generator
                         10
-- 
cgit v1.2.3


From d13bde5e26098313e789dd3793368a635cf1cc16 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 24 Oct 2023 15:44:05 +0200
Subject: jepsen: set1 and set2 don't fail anymore ??

---
 script/jepsen.garage/src/jepsen/garage.clj         |  3 +-
 script/jepsen.garage/src/jepsen/garage/nemesis.clj | 64 ++++++++++++----------
 2 files changed, 38 insertions(+), 29 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 6d64a1b8..a67399e0 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -26,7 +26,8 @@
   {"c" grgNemesis/scenario-c
    "cp" grgNemesis/scenario-cp
    "r"  grgNemesis/scenario-r
-   "pr"  grgNemesis/scenario-pr})
+   "pr"  grgNemesis/scenario-pr
+   "cpr"  grgNemesis/scenario-cpr})
 
 (def patches
   "A map of patch names to Garage builds"
diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
index e64bcaf1..07083038 100644
--- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj
+++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
@@ -76,30 +76,24 @@
 (defn scenario-cp
   "Clock scramble + partition scenario"
   [opts]
-  {:generator        (cycle [(gen/sleep 5)
-                             {:type :info, :f :partition-start}
-                             (gen/sleep 5)
-                             {:type :info, :f :clock-scramble}
-                             (gen/sleep 5)
-                             {:type :info, :f :partition-stop}
-                             (gen/sleep 5)
-                             {:type :info, :f :clock-scramble}])
+  {:generator        (->>
+                       (gen/mix [{:type :info, :f :clock-scramble}
+                                 {:type :info, :f :partition-stop}
+                                 {:type :info, :f :partition-start}])
+                       (gen/stagger 3))
    :final-generator  (gen/once {:type :info, :f :partition-stop})
    :nemesis          (nemesis/compose
-                       {{:partition-start :start
-                         :partition-stop :stop} (nemesis/partition-random-halves)
-                        {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
+                       {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)
+                        {:partition-start :start
+                         :partition-stop :stop} (nemesis/partition-random-halves)})})
 
 (defn scenario-r
   "Cluster reconfiguration scenario"
   [opts]
-  {:generator        (cycle [(gen/sleep 5)
-                             {:type :info, :f :reconfigure-start}
-                             (gen/sleep 5)
-                             {:type :info, :f :reconfigure-start}
-                             (gen/sleep 5)
-                             {:type :info, :f :reconfigure-stop}])
-   :final-generator  (gen/once {:type :info, :f :reconfigure-stop})
+  {:generator        (->>
+                       (gen/mix [{:type :info, :f :reconfigure-start}
+                                 {:type :info, :f :reconfigure-stop}])
+                       (gen/stagger 3))
    :nemesis          (nemesis/compose
                        {{:reconfigure-start :start
                          :reconfigure-stop :stop} (reconfigure-subset 3)})})
@@ -107,19 +101,33 @@
 (defn scenario-pr
   "Partition + cluster reconfiguration scenario"
   [opts]
-  {:generator        (cycle [(gen/sleep 3)
-                             {:type :info, :f :reconfigure-start}
-                             (gen/sleep 3)
-                             {:type :info, :f :partition-start}
-                             (gen/sleep 3)
-                             {:type :info, :f :reconfigure-start}
-                             (gen/sleep 3)
-                             {:type :info, :f :partition-stop}
-                             (gen/sleep 3)
-                             {:type :info, :f :reconfigure-stop}])
+  {:generator        (->>
+                       (gen/mix [{:type :info, :f :partition-start}
+                                 {:type :info, :f :partition-stop}
+                                 {:type :info, :f :reconfigure-start}
+                                 {:type :info, :f :reconfigure-stop}])
+                       (gen/stagger 3))
    :final-generator  (gen/once {:type :info, :f :partition-stop})
    :nemesis          (nemesis/compose
                        {{:partition-start :start
                          :partition-stop :stop} (nemesis/partition-random-halves)
                         {:reconfigure-start :start
                          :reconfigure-stop :stop} (reconfigure-subset 3)})})
+
+(defn scenario-cpr
+  "Clock scramble + partition + cluster reconfiguration scenario"
+  [opts]
+  {:generator        (->>
+                       (gen/mix [{:type :info, :f :clock-scramble}
+                                 {:type :info, :f :partition-start}
+                                 {:type :info, :f :partition-stop}
+                                 {:type :info, :f :reconfigure-start}
+                                 {:type :info, :f :reconfigure-stop}])
+                       (gen/stagger 3))
+   :final-generator  (gen/once {:type :info, :f :partition-stop})
+   :nemesis          (nemesis/compose
+                       {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)
+                        {:partition-start :start
+                         :partition-stop :stop} (nemesis/partition-random-halves)
+                        {:reconfigure-start :start
+                         :reconfigure-stop :stop} (reconfigure-subset 3)})})
-- 
cgit v1.2.3


From d7ab2c639e330cb3214d83bf98107573ef41f6db Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Tue, 24 Oct 2023 16:39:50 +0200
Subject: jepsen: fix nemesis to actually generate many operations

---
 script/jepsen.garage/src/jepsen/garage/nemesis.clj | 57 +++++++++++++---------
 1 file changed, 33 insertions(+), 24 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
index 07083038..6a2e1935 100644
--- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj
+++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
@@ -32,9 +32,12 @@
   [test]
   (let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show))
         [_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)]
-    (info "layout show: " layout-show "; next-version: " layout-next-version)
-    (c/on (jepsen/primary test)
-          (c/exec grg/binary :layout :apply :--version layout-next-version))))
+    (if layout-next-version
+      (do
+        (info "layout show: " layout-show "; next-version: " layout-next-version)
+        (c/on (jepsen/primary test)
+              (c/exec grg/binary :layout :apply :--version layout-next-version)))
+      (info "no layout changes to apply"))))
 
 (defn reconfigure-subset
   "Reconfigure cluster with only a subset of nodes"
@@ -65,22 +68,28 @@
 
 ; ---- nemesis scenari ----
 
+(defn nemesis-op
+  "A generator for a single nemesis operation"
+  [op]
+  (fn [_ _] {:type :info, :f op}))
+
 (defn scenario-c
   "Clock scramble scenario"
   [opts]
-  {:generator        (cycle [(gen/sleep 5)
-                             {:type :info, :f :clock-scramble}])
+  {:generator        (->>
+                       (nemesis-op :clock-scramble)
+                       (gen/stagger 5))
    :nemesis          (nemesis/compose
-                        {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
+                       {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
 
 (defn scenario-cp
   "Clock scramble + partition scenario"
   [opts]
   {:generator        (->>
-                       (gen/mix [{:type :info, :f :clock-scramble}
-                                 {:type :info, :f :partition-stop}
-                                 {:type :info, :f :partition-start}])
-                       (gen/stagger 3))
+                       (gen/mix [(nemesis-op :clock-scramble)
+                                 (nemesis-op :partition-stop)
+                                 (nemesis-op :partition-start)])
+                       (gen/stagger 5))
    :final-generator  (gen/once {:type :info, :f :partition-stop})
    :nemesis          (nemesis/compose
                        {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)
@@ -91,9 +100,9 @@
   "Cluster reconfiguration scenario"
   [opts]
   {:generator        (->>
-                       (gen/mix [{:type :info, :f :reconfigure-start}
-                                 {:type :info, :f :reconfigure-stop}])
-                       (gen/stagger 3))
+                       (gen/mix [(nemesis-op :reconfigure-start)
+                                 (nemesis-op :reconfigure-stop)])
+                       (gen/stagger 5))
    :nemesis          (nemesis/compose
                        {{:reconfigure-start :start
                          :reconfigure-stop :stop} (reconfigure-subset 3)})})
@@ -102,11 +111,11 @@
   "Partition + cluster reconfiguration scenario"
   [opts]
   {:generator        (->>
-                       (gen/mix [{:type :info, :f :partition-start}
-                                 {:type :info, :f :partition-stop}
-                                 {:type :info, :f :reconfigure-start}
-                                 {:type :info, :f :reconfigure-stop}])
-                       (gen/stagger 3))
+                       (gen/mix [(nemesis-op :partition-start)
+                                 (nemesis-op :partition-stop)
+                                 (nemesis-op :reconfigure-start)
+                                 (nemesis-op :reconfigure-stop)])
+                       (gen/stagger 5))
    :final-generator  (gen/once {:type :info, :f :partition-stop})
    :nemesis          (nemesis/compose
                        {{:partition-start :start
@@ -118,12 +127,12 @@
   "Clock scramble + partition + cluster reconfiguration scenario"
   [opts]
   {:generator        (->>
-                       (gen/mix [{:type :info, :f :clock-scramble}
-                                 {:type :info, :f :partition-start}
-                                 {:type :info, :f :partition-stop}
-                                 {:type :info, :f :reconfigure-start}
-                                 {:type :info, :f :reconfigure-stop}])
-                       (gen/stagger 3))
+                       (gen/mix [(nemesis-op :clock-scramble)
+                                 (nemesis-op :partition-start)
+                                 (nemesis-op :partition-stop)
+                                 (nemesis-op :reconfigure-start)
+                                 (nemesis-op :reconfigure-stop)])
+                       (gen/stagger 5))
    :final-generator  (gen/once {:type :info, :f :partition-stop})
    :nemesis          (nemesis/compose
                        {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)
-- 
cgit v1.2.3


From db921cc05f8bcfccd0d0ba1d90b6dcd77f06dcdd Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 25 Oct 2023 11:41:34 +0200
Subject: jepsen: reconfigure nemesis + add db nemesis

---
 script/jepsen.garage/src/jepsen/garage.clj         |  14 ++-
 script/jepsen.garage/src/jepsen/garage/daemon.clj  |  18 +++
 script/jepsen.garage/src/jepsen/garage/nemesis.clj | 121 ++++++++++-----------
 script/jepsen.garage/src/jepsen/garage/reg.clj     |  37 ++++---
 script/jepsen.garage/src/jepsen/garage/set.clj     |  49 +++++----
 5 files changed, 132 insertions(+), 107 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index a67399e0..3fe527a6 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -27,7 +27,8 @@
    "cp" grgNemesis/scenario-cp
    "r"  grgNemesis/scenario-r
    "pr"  grgNemesis/scenario-pr
-   "cpr"  grgNemesis/scenario-cpr})
+   "cpr"  grgNemesis/scenario-cpr
+   "dpr"  grgNemesis/scenario-dpr})
 
 (def patches
   "A map of patch names to Garage builds"
@@ -59,15 +60,16 @@
   "Given an options map from the command line runner (e.g. :nodes, :ssh,
   :concurrency, ...), constructs a test map."
   [opts]
-  (let [workload ((get workloads (:workload opts)) opts)
-        scenario ((get scenari (:scenario opts)) opts)
-        garage-version (get patches (:patch opts))]
+  (let [garage-version (get patches (:patch opts))
+        db (grg/db garage-version)
+        workload ((get workloads (:workload opts)) opts)
+        scenario ((get scenari (:scenario opts)) (assoc opts :db db))]
     (merge tests/noop-test
            opts
            {:pure-generators  true
             :name             (str "garage " (name (:workload opts)))
             :os               debian/os
-            :db               (grg/db garage-version)
+            :db               db
             :client           (:client workload)
             :generator        (gen/phases
                                 (->>
@@ -82,7 +84,7 @@
                                 (gen/clients (:final-generator workload)))
             :nemesis          (:nemesis scenario)
             :checker          (checker/compose
-                                {:perf (checker/perf)
+                                {:perf (checker/perf (:perf scenario))
                                  :workload (:checker workload)})
             })))
 
diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj
index 7c581ba1..d407dd29 100644
--- a/script/jepsen.garage/src/jepsen/garage/daemon.clj
+++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj
@@ -119,6 +119,24 @@
         (c/exec :rm :-rf data-dir)
         (c/exec :rm :-rf meta-dir)))
 
+    db/Pause
+    (pause! [_ test node]
+      (cu/grepkill! :stop binary))
+    (resume! [_ test node]
+      (cu/grepkill! :cont binary))
+
+    db/Kill
+    (kill! [_ test node]
+      (cu/stop-daemon! binary pidfile))
+    (start! [_ test node]
+      (cu/start-daemon!
+        {:logfile logfile
+         :pidfile pidfile
+         :chdir base-dir
+         :env {:RUST_LOG "garage=debug,garage_api=trace"}}
+        binary
+        :server))
+
     db/LogFiles
     (log-files [_ test node]
       [logfile])))
diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
index 6a2e1935..0222e463 100644
--- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj
+++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
@@ -4,6 +4,7 @@
              [core :as jepsen]
              [generator :as gen]
              [nemesis :as nemesis]]
+            [jepsen.nemesis.combined :as combined]
             [jepsen.garage.daemon :as grg]
             [jepsen.control.util :as cu]))
 
@@ -11,21 +12,23 @@
 
 (defn configure-present!
   "Configure node to be active in new cluster layout"
-  [test node]
-  (info "configure-present!" node)
-  (let [node-id (c/on node (c/exec grg/binary :node :id :-q))]
-   (c/on
-     (jepsen/primary test)
-     (c/exec grg/binary :layout :assign (subs node-id 0 16) :-c :1G))))
+  [test nodes]
+  (info "configure-present!" nodes)
+  (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q))
+        node-id-strs (map (fn [[_ v]] (subs v  0 16)) node-ids)]
+    (c/on
+      (jepsen/primary test)
+      (apply c/exec (concat [grg/binary :layout :assign :-c :1G] node-id-strs)))))
 
 (defn configure-absent!
-  "Configure node to be active in new cluster layout"
-  [test node]
-  (info "configure-absent!" node)
-  (let [node-id (c/on node (c/exec grg/binary :node :id :-q))]
-   (c/on
-     (jepsen/primary test)
-     (c/exec grg/binary :layout :assign (subs node-id 0 16) :-g))))
+  "Configure nodes to be active in new cluster layout"
+  [test nodes]
+  (info "configure-absent!" nodes)
+  (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q))
+        node-id-strs (map (fn [[_ v]] (subs v  0 16)) node-ids)]
+    (c/on
+      (jepsen/primary test)
+      (apply c/exec (concat [grg/binary :layout :assign :-g] node-id-strs)))))
 
 (defn finalize-config!
   "Apply the proposed cluster layout"
@@ -53,14 +56,14 @@
                      shuffle
                      (split-at cnt))]
             (info "layout split: keep " keep-nodes ", remove " remove-nodes)
-            (run! #(configure-present! test %) keep-nodes)
-            (run! #(configure-absent! test %) remove-nodes)
+            (configure-present! test keep-nodes)
+            (configure-absent! test remove-nodes)
             (finalize-config! test)
             (assoc op :value keep-nodes))
         :stop
           (do
             (info "layout un-split: all nodes=" (:nodes test))
-            (run! #(configure-present! test %) (:nodes test))
+            (configure-present! test (:nodes test))
             (finalize-config! test)
             (assoc op :value (:nodes test)))))
 
@@ -73,70 +76,58 @@
   [op]
   (fn [_ _] {:type :info, :f op}))
 
-(defn scenario-c
-  "Clock scramble scenario"
+(defn reconfiguration-package
+  "Cluster reconfiguration nemesis package"
   [opts]
   {:generator        (->>
-                       (nemesis-op :clock-scramble)
-                       (gen/stagger 5))
+                       (gen/mix [(nemesis-op :reconfigure-start)
+                                 (nemesis-op :reconfigure-stop)])
+                       (gen/stagger (:interval opts 5)))
+   :final-generator  {:type :info, :f :reconfigure-stop}
    :nemesis          (nemesis/compose
-                       {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
+                       {{:reconfigure-start :start
+                         :reconfigure-stop :stop} (reconfigure-subset 3)})
+   :perf              #{{:name  "reconfigure"
+                         :start #{:reconfigure-start}
+                         :stop  #{:reconfigur-stop}
+                         :color "#A197E9"}}})
+
+(defn scenario-c
+  "Clock modifying scenario"
+  [opts]
+  (combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}))
 
 (defn scenario-cp
-  "Clock scramble + partition scenario"
+  "Clock modifying + partition scenario"
   [opts]
-  {:generator        (->>
-                       (gen/mix [(nemesis-op :clock-scramble)
-                                 (nemesis-op :partition-stop)
-                                 (nemesis-op :partition-start)])
-                       (gen/stagger 5))
-   :final-generator  (gen/once {:type :info, :f :partition-stop})
-   :nemesis          (nemesis/compose
-                       {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)
-                        {:partition-start :start
-                         :partition-stop :stop} (nemesis/partition-random-halves)})})
+  (combined/compose-packages
+    [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
+     (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})]))
 
 (defn scenario-r
   "Cluster reconfiguration scenario"
   [opts]
-  {:generator        (->>
-                       (gen/mix [(nemesis-op :reconfigure-start)
-                                 (nemesis-op :reconfigure-stop)])
-                       (gen/stagger 5))
-   :nemesis          (nemesis/compose
-                       {{:reconfigure-start :start
-                         :reconfigure-stop :stop} (reconfigure-subset 3)})})
+  (reconfiguration-package {:interval 1}))
 
 (defn scenario-pr
   "Partition + cluster reconfiguration scenario"
   [opts]
-  {:generator        (->>
-                       (gen/mix [(nemesis-op :partition-start)
-                                 (nemesis-op :partition-stop)
-                                 (nemesis-op :reconfigure-start)
-                                 (nemesis-op :reconfigure-stop)])
-                       (gen/stagger 5))
-   :final-generator  (gen/once {:type :info, :f :partition-stop})
-   :nemesis          (nemesis/compose
-                       {{:partition-start :start
-                         :partition-stop :stop} (nemesis/partition-random-halves)
-                        {:reconfigure-start :start
-                         :reconfigure-stop :stop} (reconfigure-subset 3)})})
+  (combined/compose-packages
+    [(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
+     (reconfiguration-package {:interval 1})]))
 
 (defn scenario-cpr
   "Clock scramble + partition + cluster reconfiguration scenario"
   [opts]
-  {:generator        (->>
-                       (gen/mix [(nemesis-op :clock-scramble)
-                                 (nemesis-op :partition-start)
-                                 (nemesis-op :partition-stop)
-                                 (nemesis-op :reconfigure-start)
-                                 (nemesis-op :reconfigure-stop)])
-                       (gen/stagger 5))
-   :final-generator  (gen/once {:type :info, :f :partition-stop})
-   :nemesis          (nemesis/compose
-                       {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)
-                        {:partition-start :start
-                         :partition-stop :stop} (nemesis/partition-random-halves)
-                        {:reconfigure-start :start
-                         :reconfigure-stop :stop} (reconfigure-subset 3)})})
+  (combined/compose-packages
+    [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
+     (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
+     (reconfiguration-package {:interval 1})]))
+
+(defn scenario-dpr
+  "Db + partition + cluster reconfiguration scenario"
+  [opts]
+  (combined/compose-packages
+    [(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}})
+     (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
+     (reconfiguration-package {:interval 1})]))
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index 6772abfe..39708c0b 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -30,21 +30,28 @@
     (assoc this :creds (grg/creds node)))
   (setup! [this test])
   (invoke! [this test op]
-    (let [[k v] (:value op)]
-      (case (:f op)
-        :read
-          (util/timeout
-            10000
-            (assoc op :type :fail, :error ::timeout)
-            (let [value (s3/get (:creds this) k)]
-              (assoc op :type :ok, :value (independent/tuple k value))))
-        :write
-          (util/timeout
-            10000
-            (assoc op :type :info, :error ::timeout)
-            (do
-              (s3/put (:creds this) k v)
-              (assoc op :type :ok))))))
+    (try+
+      (let [[k v] (:value op)]
+        (case (:f op)
+          :read
+            (util/timeout
+              10000
+              (assoc op :type :fail, :error ::timeout)
+              (let [value (s3/get (:creds this) k)]
+                (assoc op :type :ok, :value (independent/tuple k value))))
+          :write
+            (util/timeout
+              10000
+              (assoc op :type :info, :error ::timeout)
+              (do
+                (s3/put (:creds this) k v)
+                (assoc op :type :ok)))))
+      (catch (re-find #"Unavailable" (.getMessage %)) ex
+        (assoc op :type :info, :error ::unavailable))
+      (catch (re-find #"Broken pipe" (.getMessage %)) ex
+        (assoc op :type :info, :error ::broken-pipe))
+      (catch (re-find #"Connection refused" (.getMessage %)) ex
+        (assoc op :type :info, :error ::connection-refused))))
   (teardown! [this test])
   (close! [this test]))
 
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index f625e672..670c73f2 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -30,27 +30,34 @@
     (assoc this :creds (grg/creds node)))
   (setup! [this test])
   (invoke! [this test op]
-    (let [[k v] (:value op)
-          prefix (str "set" k "/")]
-      (case (:f op)
-        :add
-          (util/timeout
-            10000
-            (assoc op :type :info, :error ::timeout)
-            (do
-              (s3/put (:creds this) (str prefix v) "present")
-              (assoc op :type :ok)))
-        :read
-          (util/timeout
-            10000
-            (assoc op :type :fail, :error ::timeout)
-            (do
-              (let [items (s3/list (:creds this) prefix)]
-                (let [items-stripped (map (fn [o]
-                                          (assert (str/starts-with? o prefix))
-                                          (str/replace-first o prefix "")) items)
-                      items-set (set (map parse-long items-stripped))]
-                  (assoc op :type :ok, :value (independent/tuple k items-set)))))))))
+    (try+
+      (let [[k v] (:value op)
+            prefix (str "set" k "/")]
+        (case (:f op)
+          :add
+            (util/timeout
+              10000
+              (assoc op :type :info, :error ::timeout)
+              (do
+                (s3/put (:creds this) (str prefix v) "present")
+                (assoc op :type :ok)))
+          :read
+            (util/timeout
+              10000
+              (assoc op :type :fail, :error ::timeout)
+              (do
+                (let [items (s3/list (:creds this) prefix)]
+                  (let [items-stripped (map (fn [o]
+                                            (assert (str/starts-with? o prefix))
+                                            (str/replace-first o prefix "")) items)
+                        items-set (set (map parse-long items-stripped))]
+                    (assoc op :type :ok, :value (independent/tuple k items-set))))))))
+      (catch (re-find #"Unavailable" (.getMessage %)) ex
+        (assoc op :type :info, :error ::unavailable))
+      (catch (re-find #"Broken pipe" (.getMessage %)) ex
+        (assoc op :type :info, :error ::broken-pipe))
+      (catch (re-find #"Connection refused" (.getMessage %)) ex
+        (assoc op :type :info, :error ::connection-refused))))
   (teardown! [this test])
   (close! [this test]))
 
-- 
cgit v1.2.3


From cfbfa09d24727e83bc042764dad2751e944fc939 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 25 Oct 2023 11:50:16 +0200
Subject: jepsen: fix set2 test omg finally this is so stupid

---
 script/jepsen.garage/src/jepsen/garage/set.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index 670c73f2..a73b8efc 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -81,7 +81,7 @@
                         ([:invoke :read])
                           (assoc-in state [:read-must-contain (:process op)] (:add-done state))
                         ([:ok :read])
-                          (let [read-must-contain (get (:process op) (:read-must-contain state))
+                          (let [read-must-contain (get (:read-must-contain state) (:process op))
                                 new-missed (set/difference read-must-contain (:value op))
                                 new-unexpected (set/difference (:value op) (:add-started state))]
                             (assoc state
-- 
cgit v1.2.3


From 5b1f50be65c251a1dc0a4358c706c409f17a82c0 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Wed, 25 Oct 2023 14:43:24 +0200
Subject: jepsen: testing

---
 script/jepsen.garage/src/jepsen/garage.clj         | 1 +
 script/jepsen.garage/src/jepsen/garage/nemesis.clj | 9 +++++++++
 2 files changed, 10 insertions(+)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 3fe527a6..17363c9d 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -28,6 +28,7 @@
    "r"  grgNemesis/scenario-r
    "pr"  grgNemesis/scenario-pr
    "cpr"  grgNemesis/scenario-cpr
+   "cdp"  grgNemesis/scenario-cdp
    "dpr"  grgNemesis/scenario-dpr})
 
 (def patches
diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
index 0222e463..dfce0255 100644
--- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj
+++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj
@@ -124,6 +124,14 @@
      (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
      (reconfiguration-package {:interval 1})]))
 
+(defn scenario-cdp
+  "Clock modifying + db + partition scenario"
+  [opts]
+  (combined/compose-packages
+    [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
+     (combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}})
+     (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})]))
+
 (defn scenario-dpr
   "Db + partition + cluster reconfiguration scenario"
   [opts]
@@ -131,3 +139,4 @@
     [(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}})
      (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
      (reconfiguration-package {:interval 1})]))
+
-- 
cgit v1.2.3


From 18e58111593d5bf978d8c603e0dc2633ffcd91c4 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Thu, 16 Nov 2023 12:57:21 +0100
Subject: jepsen: add patch and use more complete names

---
 script/jepsen.garage/src/jepsen/garage.clj | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 17363c9d..48659d6c 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -35,7 +35,8 @@
   "A map of patch names to Garage builds"
   {"default" "v0.9.0"
    "tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"
-   "tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1"})
+   "tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1"
+   "task3a" "d4df03424f1c7f3cc1eaba9e16d2e1d049131b97"})
 
 (def cli-opts
   "Additional command line options."
@@ -68,7 +69,7 @@
     (merge tests/noop-test
            opts
            {:pure-generators  true
-            :name             (str "garage " (name (:workload opts)))
+            :name             (str "garage " (name (:workload opts)) " " (name (:scenario opts)) " " (name (:patch opts)))
             :os               debian/os
             :db               db
             :client           (:client workload)
-- 
cgit v1.2.3


From 92dd2bbe15357a24eb68a3d3d6220c4758bb81a7 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Thu, 16 Nov 2023 18:09:13 +0100
Subject: jepsen: nlnet task3a seems to fix things

---
 script/jepsen.garage/src/jepsen/garage.clj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 48659d6c..174e8df0 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -36,7 +36,7 @@
   {"default" "v0.9.0"
    "tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"
    "tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1"
-   "task3a" "d4df03424f1c7f3cc1eaba9e16d2e1d049131b97"})
+   "task3a" "707442f5de416fdbed4681a33b739f0a787b7834"})
 
 (def cli-opts
   "Additional command line options."
-- 
cgit v1.2.3


From fa9247f11b89c960dffe82d6bf990ed4335788e3 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Thu, 14 Dec 2023 16:23:48 +0100
Subject: jepsen: updated results, confirming that task3 works

---
 script/jepsen.garage/src/jepsen/garage.clj     | 10 +++++++---
 script/jepsen.garage/src/jepsen/garage/set.clj | 12 +++++++-----
 2 files changed, 14 insertions(+), 8 deletions(-)

(limited to 'script/jepsen.garage/src/jepsen')

diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 174e8df0..446b81de 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -36,7 +36,9 @@
   {"default" "v0.9.0"
    "tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"
    "tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1"
-   "task3a" "707442f5de416fdbed4681a33b739f0a787b7834"})
+   "task3a" "707442f5de416fdbed4681a33b739f0a787b7834"
+   "task3b" "431b28e0cfdc9cac6c649193cf602108a8b02997"
+   "task3c" "0041b013a473e3ae72f50209d8f79db75a72848b"})
 
 (def cli-opts
   "Additional command line options."
@@ -69,7 +71,7 @@
     (merge tests/noop-test
            opts
            {:pure-generators  true
-            :name             (str "garage " (name (:workload opts)) " " (name (:scenario opts)) " " (name (:patch opts)))
+            :name             (str "garage-" (name (:patch opts)) " " (name (:workload opts)) " " (name (:scenario opts)))
             :os               debian/os
             :db               db
             :client           (:client workload)
@@ -83,7 +85,9 @@
                                 (gen/nemesis (:final-generator scenario))
                                 (gen/log "Waiting for recovery")
                                 (gen/sleep 10)
-                                (gen/clients (:final-generator workload)))
+                                (gen/log "Running final generator")
+                                (gen/clients (:final-generator workload))
+                                (gen/log "Generators all done"))
             :nemesis          (:nemesis scenario)
             :checker          (checker/compose
                                 {:perf (checker/perf (:perf scenario))
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index a73b8efc..2c7a2ccd 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -108,11 +108,13 @@
                           (->> (range)
                            (map (fn [x] {:type :invoke, :f :add, :value x}))
                            (gen/limit (:ops-per-key opts)))))
-   :final-generator   (gen/phases
-                        (independent/sequential-generator
-                          (range 100)
-                          (fn [k] (gen/once op-read)))
-                        (gen/sleep 5))})
+   :final-generator   (independent/concurrent-generator
+                        10
+                        (range 100)
+                        (fn [k]
+                          (gen/phases
+                            (gen/once op-read)
+                            (gen/sleep 5))))})
 
 (defn workload2
   "Tests insertions and deletions"
-- 
cgit v1.2.3