Add garage set test (this one works :p)

5 files changed, 95 insertions, 21 deletions

diff --git a/script/jepsen.garage/README.md b/script/jepsen.garage/README.md
index 460f0b9e..800dde94 100644
--- a/script/jepsen.garage/README.md
+++ b/script/jepsen.garage/README.md
@@ -18,10 +18,10 @@ Set up VMs:
 vagrant up
 ```
 
-Run tests:
+Run tests (this one should fail):
 
 ```
-lein run test --nodes-file nodes.vagrant
+lein run test --nodes-file nodes.vagrant --time-limit 64 --concurrency 50 --rate 50 --workload reg
 ```
 
 ## License
diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index cd30dab8..7f762923 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -10,11 +10,13 @@
     [jepsen.os.debian :as debian]
     [jepsen.garage
      [grg :as grg]
-     [reg :as reg]]))
+     [reg :as reg]
+     [set :as set]]))
 
 (def workloads
   "A map of workload names to functions that construct workloads, given opts."
-  {"reg"  reg/workload})
+  {"reg"  reg/workload
+   "set"  set/workload})
 
 (def cli-opts
   "Additional command line options."
@@ -47,7 +49,21 @@
             :os               debian/os
             :db               (grg/db garage-version)
             :client           (:client workload)
-            :generator        (:generator workload)
+            :generator        (gen/phases
+                                (->>
+                                  (:generator workload)
+                                  (gen/stagger (/ (:rate opts)))
+                                  (gen/nemesis
+                                    (cycle [(gen/sleep 5)
+                                            {:type :info, :f :start}
+                                            (gen/sleep 5)
+                                            {:type :info, :f :stop}]))
+                                  (gen/time-limit (:time-limit opts)))
+                                (gen/log "Healing cluster")
+                                (gen/nemesis (gen/once {:type :info, :f :stop}))
+                                (gen/log "Waiting for recovery")
+                                (gen/sleep 10)
+                                (gen/clients (:final-generator workload)))
             :nemesis          (nemesis/partition-random-halves)
             :checker          (checker/compose
                                 {:perf (checker/perf)
diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
index 2309d824..2e9bcc72 100644
--- a/script/jepsen.garage/src/jepsen/garage/grg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/grg.clj
@@ -121,5 +121,14 @@
 (defn s3-list
   "Helper for ListObjects -- just lists everything in the bucket"
   [creds]
-  (s3/list-objects-v2 creds
-                      {:bucket-name (:bucket creds)}))
+  (defn list-inner [ct accum]
+    (let [list-result (s3/list-objects-v2 creds
+                                          {:bucket-name (:bucket creds)
+                                           :continuation-token ct})
+          new-object-summaries (:object-summaries list-result)
+          new-objects (map (fn [d] (:key d)) new-object-summaries)
+          objects (concat new-objects accum)]
+      (if (:truncated? list-result)
+        (list-inner (:next-continuation-token list-result) objects)
+        objects)))
+  (list-inner nil []))
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index 557b4a99..0b370b36 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -51,19 +51,12 @@
                                     {:model     (model/register)
                                      :algorithm :linear})
                           :timeline (timeline/html)}))
-   :generator        (->> (independent/concurrent-generator
-                            10
-                            (range)
-                            (fn [k]
-                              (->>
-                                (gen/mix [op-get op-put op-del])
-                                (gen/stagger (/ (:rate opts)))
-                                (gen/limit (:ops-per-key opts)))))
-                          (gen/nemesis
-                            (cycle [(gen/sleep 5)
-                                    {:type :info, :f :start}
-                                    (gen/sleep 5)
-                                    {:type :info, :f :stop}]))
-                          (gen/time-limit (:time-limit opts)))})
+   :generator        (independent/concurrent-generator
+                       10
+                       (range)
+                       (fn [k]
+                         (->>
+                           (gen/mix [op-get op-put op-del])
+                           (gen/limit (:ops-per-key opts)))))})
 
 
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
new file mode 100644
index 00000000..ab99e6ef
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -0,0 +1,56 @@
+(ns jepsen.garage.set
+  (:require [clojure.tools.logging :refer :all]
+            [clojure.string :as str]
+            [jepsen [checker :as checker]
+             [cli :as cli]
+             [client :as client]
+             [control :as c]
+             [db :as db]
+             [generator :as gen]
+             [independent :as independent]
+             [nemesis :as nemesis]
+             [tests :as tests]]
+            [jepsen.checker.timeline :as timeline]
+            [jepsen.control.util :as cu]
+            [jepsen.os.debian :as debian]
+            [jepsen.garage.grg :as grg]
+            [knossos.model :as model]
+            [slingshot.slingshot :refer [try+]]))
+
+(defn op-add [_ _] {:type :invoke, :f :add, :value (rand-int 100000)})
+(defn op-read [_ _] {:type :invoke, :f :read, :value nil})
+
+(defrecord SetClient [creds]
+  client/Client
+  (open! [this test node]
+    (let [creds (grg/s3-creds node)]
+      (info node "s3 credentials:" creds)
+      (assoc this :creds creds)))
+  (setup! [this test])
+  (invoke! [this test op]
+    (case (:f op)
+      :add
+        (do
+          (grg/s3-put (:creds this) (str (:value op)) "present")
+          (assoc op :type :ok))
+      :read
+        (let [items (grg/s3-list (:creds this))]
+          (assoc op :type :ok, :value (set (map read-string items))))))
+  (teardown! [this test])
+  (close! [this test]))
+
+(defn workload
+  "Tests insertions and deletions"
+  [opts]
+  {:client            (SetClient. nil)
+   :checker           (checker/compose
+                        {:set (checker/set)
+                         :timeline (timeline/html)})
+   ; :generator         (gen/mix [op-add op-read])
+   ; :generator         (->> (range)
+   ;                         (map (fn [x] {:type :invoke, :f :add, :value x})))
+   :generator         (gen/mix [op-read
+                        (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})))])
+   :final-generator   (gen/once op-read)})
+
+