refactor jepsen test

2 files changed, 194 insertions, 0 deletions

diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj
new file mode 100644
index 00000000..2309d824
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/grg.clj
@@ -0,0 +1,125 @@
+(ns jepsen.garage.grg
+  (:require [clojure.tools.logging :refer :all]
+            [jepsen [control :as c]
+             [db :as db]]
+            [jepsen.control.util :as cu]
+            [amazonica.aws.s3 :as s3]
+            [slingshot.slingshot :refer [try+]]))
+
+; CONSTANTS -- HOW GARAGE IS SET UP
+
+(def dir "/opt/garage")
+(def binary (str dir "/garage"))
+(def logfile (str dir "/garage.log"))
+(def pidfile (str dir "/garage.pid"))
+
+(def grg-admin-token "icanhazadmin")
+(def grg-key "jepsen")
+(def grg-bucket "jepsen")
+
+; THE GARAGE DB
+
+(defn db
+  "Garage DB for a particular version"
+  [version]
+  (reify db/DB
+    (setup! [_ test node]
+      (info node "installing garage" version)
+      (c/su
+        (c/exec :mkdir :-p dir)
+        (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
+              cache (cu/wget! url)]
+          (c/exec :cp cache binary))
+        (c/exec :chmod :+x binary)
+        (cu/write-file!
+          (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
+               "rpc_bind_addr = \"0.0.0.0:3901\"\n"
+               "rpc_public_addr = \"" node ":3901\"\n"
+               "db_engine = \"lmdb\"\n"
+               "replication_mode = \"3\"\n"
+               "data_dir = \"" dir "/data\"\n"
+               "metadata_dir = \"" dir "/meta\"\n"
+               "[s3_api]\n"
+               "s3_region = \"us-east-1\"\n"
+               "api_bind_addr = \"0.0.0.0:3900\"\n"
+               "[k2v_api]\n"
+               "api_bind_addr = \"0.0.0.0:3902\"\n"
+               "[admin]\n"
+               "api_bind_addr = \"0.0.0.0:3903\"\n"
+               "admin_token = \"" grg-admin-token "\"\n")
+          "/etc/garage.toml")
+        (cu/start-daemon!
+          {:logfile logfile
+           :pidfile pidfile
+           :chdir dir}
+          binary
+          :server)
+        (Thread/sleep 100)
+        (let [node-id (c/exec binary :node :id :-q)]
+          (info node "node id:" node-id)
+          (c/on-many (:nodes test)
+                     (c/exec binary :node :connect node-id))
+          (c/exec binary :layout :assign (subs node-id 0 16) :-c 1 :-z :dc1 :-t node))
+        (if (= node (first (:nodes test)))
+          (do
+            (Thread/sleep 2000)
+            (c/exec binary :layout :apply :--version 1)
+            (info node "garage status:" (c/exec binary :status))
+            (c/exec binary :key :new :--name grg-key)
+            (c/exec binary :bucket :create grg-bucket)
+            (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
+            (info node "key info: " (c/exec binary :key :info grg-key))))))
+    (teardown! [_ test node]
+      (info node "tearing down garage" version)
+      (c/su
+        (cu/stop-daemon! binary pidfile)
+        (c/exec :rm :-rf dir)))
+    db/LogFiles
+    (log-files [_ test node]
+      [logfile])))
+
+; GARAGE S3 HELPER FUNCTIONS
+
+(defn s3-creds
+  "Get S3 credentials for node"
+  [node]
+  (let [key-info (c/on node (c/exec binary :key :info grg-key))
+        [_ ak sk] (re-matches
+                    #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
+                    key-info)]
+    {:access-key ak
+     :secret-key sk
+     :endpoint (str "http://" node ":3900")
+     :bucket grg-bucket
+     :client-config {:path-style-access-enabled true}}))
+
+(defn s3-get
+  "Helper for GetObject"
+  [creds k]
+  (try+
+    (-> (s3/get-object creds (:bucket creds) k)
+        :input-stream
+        slurp)
+    (catch (re-find #"Key not found" (.getMessage %)) ex
+      nil)))
+
+(defn s3-put
+  "Helper for PutObject or DeleteObject (is a delete if value is nil)"
+  [creds k v]
+  (if (= v nil)
+    (s3/delete-object creds
+                      :bucket-name (:bucket creds)
+                      :key k)
+    (let [some-bytes (.getBytes v "UTF-8")
+          bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
+      (s3/put-object creds
+                     :bucket-name (:bucket creds)
+                     :key k
+                     :input-stream bytes-stream
+                     :metadata {:content-length (count some-bytes)}))))
+
+(defn s3-list
+  "Helper for ListObjects -- just lists everything in the bucket"
+  [creds]
+  (s3/list-objects-v2 creds
+                      {:bucket-name (:bucket creds)}))
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
new file mode 100644
index 00000000..557b4a99
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -0,0 +1,69 @@
+(ns jepsen.garage.reg
+  (:require [clojure.tools.logging :refer :all]
+            [clojure.string :as str]
+            [jepsen [checker :as checker]
+             [cli :as cli]
+             [client :as client]
+             [control :as c]
+             [db :as db]
+             [generator :as gen]
+             [independent :as independent]
+             [nemesis :as nemesis]
+             [tests :as tests]]
+            [jepsen.checker.timeline :as timeline]
+            [jepsen.control.util :as cu]
+            [jepsen.os.debian :as debian]
+            [jepsen.garage.grg :as grg]
+            [knossos.model :as model]
+            [slingshot.slingshot :refer [try+]]))
+
+(defn op-get [_ _] {:type :invoke, :f :read, :value nil})
+(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 9))})
+(defn op-del [_ _] {:type :invoke, :f :write, :value nil})
+
+(defrecord RegClient [creds]
+  client/Client
+  (open! [this test node]
+    (let [creds (grg/s3-creds node)]
+      (info node "s3 credentials:" creds)
+      (assoc this :creds creds)))
+  (setup! [this test])
+  (invoke! [this test op]
+    (let [[k v] (:value op)]
+      (case (:f op)
+        :read
+          (let [value (grg/s3-get (:creds this) k)]
+            (assoc op :type :ok, :value (independent/tuple k value)))
+        :write
+          (do
+            (grg/s3-put (:creds this) k v)
+            (assoc op :type :ok)))))
+  (teardown! [this test])
+  (close! [this test]))
+
+(defn workload
+  "Tests linearizable reads and writes"
+  [opts]
+  {:client           (RegClient. nil)
+   :checker          (independent/checker
+                       (checker/compose
+                         {:linear (checker/linearizable
+                                    {:model     (model/register)
+                                     :algorithm :linear})
+                          :timeline (timeline/html)}))
+   :generator        (->> (independent/concurrent-generator
+                            10
+                            (range)
+                            (fn [k]
+                              (->>
+                                (gen/mix [op-get op-put op-del])
+                                (gen/stagger (/ (:rate opts)))
+                                (gen/limit (:ops-per-key opts)))))
+                          (gen/nemesis
+                            (cycle [(gen/sleep 5)
+                                    {:type :info, :f :start}
+                                    (gen/sleep 5)
+                                    {:type :info, :f :stop}]))
+                          (gen/time-limit (:time-limit opts)))})
+
+