From eb86eaa6d2702ad5171c4fdfcbceb3dfba21833c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 19 Apr 2023 14:14:22 +0200 Subject: refactor jepsen test --- script/jepsen.garage/src/jepsen/garage/grg.clj | 125 +++++++++++++++++++++++++ script/jepsen.garage/src/jepsen/garage/reg.clj | 69 ++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 script/jepsen.garage/src/jepsen/garage/grg.clj create mode 100644 script/jepsen.garage/src/jepsen/garage/reg.clj (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj new file mode 100644 index 00000000..2309d824 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/grg.clj @@ -0,0 +1,125 @@ +(ns jepsen.garage.grg + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c] + [db :as db]] + [jepsen.control.util :as cu] + [amazonica.aws.s3 :as s3] + [slingshot.slingshot :refer [try+]])) + +; CONSTANTS -- HOW GARAGE IS SET UP + +(def dir "/opt/garage") +(def binary (str dir "/garage")) +(def logfile (str dir "/garage.log")) +(def pidfile (str dir "/garage.pid")) + +(def grg-admin-token "icanhazadmin") +(def grg-key "jepsen") +(def grg-bucket "jepsen") + +; THE GARAGE DB + +(defn db + "Garage DB for a particular version" + [version] + (reify db/DB + (setup! [_ test node] + (info node "installing garage" version) + (c/su + (c/exec :mkdir :-p dir) + (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") + cache (cu/wget! url)] + (c/exec :cp cache binary)) + (c/exec :chmod :+x binary) + (cu/write-file! + (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" + "rpc_bind_addr = \"0.0.0.0:3901\"\n" + "rpc_public_addr = \"" node ":3901\"\n" + "db_engine = \"lmdb\"\n" + "replication_mode = \"3\"\n" + "data_dir = \"" dir "/data\"\n" + "metadata_dir = \"" dir "/meta\"\n" + "[s3_api]\n" + "s3_region = \"us-east-1\"\n" + "api_bind_addr = \"0.0.0.0:3900\"\n" + "[k2v_api]\n" + "api_bind_addr = \"0.0.0.0:3902\"\n" + "[admin]\n" + "api_bind_addr = \"0.0.0.0:3903\"\n" + "admin_token = \"" grg-admin-token "\"\n") + "/etc/garage.toml") + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir dir} + binary + :server) + (Thread/sleep 100) + (let [node-id (c/exec binary :node :id :-q)] + (info node "node id:" node-id) + (c/on-many (:nodes test) + (c/exec binary :node :connect node-id)) + (c/exec binary :layout :assign (subs node-id 0 16) :-c 1 :-z :dc1 :-t node)) + (if (= node (first (:nodes test))) + (do + (Thread/sleep 2000) + (c/exec binary :layout :apply :--version 1) + (info node "garage status:" (c/exec binary :status)) + (c/exec binary :key :new :--name grg-key) + (c/exec binary :bucket :create grg-bucket) + (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key) + (info node "key info: " (c/exec binary :key :info grg-key)))))) + (teardown! [_ test node] + (info node "tearing down garage" version) + (c/su + (cu/stop-daemon! binary pidfile) + (c/exec :rm :-rf dir))) + db/LogFiles + (log-files [_ test node] + [logfile]))) + +; GARAGE S3 HELPER FUNCTIONS + +(defn s3-creds + "Get S3 credentials for node" + [node] + (let [key-info (c/on node (c/exec binary :key :info grg-key)) + [_ ak sk] (re-matches + #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*" + key-info)] + {:access-key ak + :secret-key sk + :endpoint (str "http://" node ":3900") + :bucket grg-bucket + :client-config {:path-style-access-enabled true}})) + +(defn s3-get + "Helper for GetObject" + [creds k] + (try+ + (-> (s3/get-object creds (:bucket creds) k) + :input-stream + slurp) + (catch (re-find #"Key not found" (.getMessage %)) ex + nil))) + +(defn s3-put + "Helper for PutObject or DeleteObject (is a delete if value is nil)" + [creds k v] + (if (= v nil) + (s3/delete-object creds + :bucket-name (:bucket creds) + :key k) + (let [some-bytes (.getBytes v "UTF-8") + bytes-stream (java.io.ByteArrayInputStream. some-bytes)] + (s3/put-object creds + :bucket-name (:bucket creds) + :key k + :input-stream bytes-stream + :metadata {:content-length (count some-bytes)})))) + +(defn s3-list + "Helper for ListObjects -- just lists everything in the bucket" + [creds] + (s3/list-objects-v2 creds + {:bucket-name (:bucket creds)})) diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj new file mode 100644 index 00000000..557b4a99 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -0,0 +1,69 @@ +(ns jepsen.garage.reg + (:require [clojure.tools.logging :refer :all] + [clojure.string :as str] + [jepsen [checker :as checker] + [cli :as cli] + [client :as client] + [control :as c] + [db :as db] + [generator :as gen] + [independent :as independent] + [nemesis :as nemesis] + [tests :as tests]] + [jepsen.checker.timeline :as timeline] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [jepsen.garage.grg :as grg] + [knossos.model :as model] + [slingshot.slingshot :refer [try+]])) + +(defn op-get [_ _] {:type :invoke, :f :read, :value nil}) +(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 9))}) +(defn op-del [_ _] {:type :invoke, :f :write, :value nil}) + +(defrecord RegClient [creds] + client/Client + (open! [this test node] + (let [creds (grg/s3-creds node)] + (info node "s3 credentials:" creds) + (assoc this :creds creds))) + (setup! [this test]) + (invoke! [this test op] + (let [[k v] (:value op)] + (case (:f op) + :read + (let [value (grg/s3-get (:creds this) k)] + (assoc op :type :ok, :value (independent/tuple k value))) + :write + (do + (grg/s3-put (:creds this) k v) + (assoc op :type :ok))))) + (teardown! [this test]) + (close! [this test])) + +(defn workload + "Tests linearizable reads and writes" + [opts] + {:client (RegClient. nil) + :checker (independent/checker + (checker/compose + {:linear (checker/linearizable + {:model (model/register) + :algorithm :linear}) + :timeline (timeline/html)})) + :generator (->> (independent/concurrent-generator + 10 + (range) + (fn [k] + (->> + (gen/mix [op-get op-put op-del]) + (gen/stagger (/ (:rate opts))) + (gen/limit (:ops-per-key opts))))) + (gen/nemesis + (cycle [(gen/sleep 5) + {:type :info, :f :start} + (gen/sleep 5) + {:type :info, :f :stop}])) + (gen/time-limit (:time-limit opts)))}) + + -- cgit v1.2.3 From 6eb26be548c08707b59473e6086f3f5eee89fe47 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 19 Apr 2023 15:27:26 +0200 Subject: Add garage set test (this one works :p) --- script/jepsen.garage/src/jepsen/garage/grg.clj | 13 +++++- script/jepsen.garage/src/jepsen/garage/reg.clj | 21 ++++------ script/jepsen.garage/src/jepsen/garage/set.clj | 56 ++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 16 deletions(-) create mode 100644 script/jepsen.garage/src/jepsen/garage/set.clj (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj index 2309d824..2e9bcc72 100644 --- a/script/jepsen.garage/src/jepsen/garage/grg.clj +++ b/script/jepsen.garage/src/jepsen/garage/grg.clj @@ -121,5 +121,14 @@ (defn s3-list "Helper for ListObjects -- just lists everything in the bucket" [creds] - (s3/list-objects-v2 creds - {:bucket-name (:bucket creds)})) + (defn list-inner [ct accum] + (let [list-result (s3/list-objects-v2 creds + {:bucket-name (:bucket creds) + :continuation-token ct}) + new-object-summaries (:object-summaries list-result) + new-objects (map (fn [d] (:key d)) new-object-summaries) + objects (concat new-objects accum)] + (if (:truncated? list-result) + (list-inner (:next-continuation-token list-result) objects) + objects))) + (list-inner nil [])) diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj index 557b4a99..0b370b36 100644 --- a/script/jepsen.garage/src/jepsen/garage/reg.clj +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -51,19 +51,12 @@ {:model (model/register) :algorithm :linear}) :timeline (timeline/html)})) - :generator (->> (independent/concurrent-generator - 10 - (range) - (fn [k] - (->> - (gen/mix [op-get op-put op-del]) - (gen/stagger (/ (:rate opts))) - (gen/limit (:ops-per-key opts))))) - (gen/nemesis - (cycle [(gen/sleep 5) - {:type :info, :f :start} - (gen/sleep 5) - {:type :info, :f :stop}])) - (gen/time-limit (:time-limit opts)))}) + :generator (independent/concurrent-generator + 10 + (range) + (fn [k] + (->> + (gen/mix [op-get op-put op-del]) + (gen/limit (:ops-per-key opts)))))}) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj new file mode 100644 index 00000000..ab99e6ef --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -0,0 +1,56 @@ +(ns jepsen.garage.set + (:require [clojure.tools.logging :refer :all] + [clojure.string :as str] + [jepsen [checker :as checker] + [cli :as cli] + [client :as client] + [control :as c] + [db :as db] + [generator :as gen] + [independent :as independent] + [nemesis :as nemesis] + [tests :as tests]] + [jepsen.checker.timeline :as timeline] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [jepsen.garage.grg :as grg] + [knossos.model :as model] + [slingshot.slingshot :refer [try+]])) + +(defn op-add [_ _] {:type :invoke, :f :add, :value (rand-int 100000)}) +(defn op-read [_ _] {:type :invoke, :f :read, :value nil}) + +(defrecord SetClient [creds] + client/Client + (open! [this test node] + (let [creds (grg/s3-creds node)] + (info node "s3 credentials:" creds) + (assoc this :creds creds))) + (setup! [this test]) + (invoke! [this test op] + (case (:f op) + :add + (do + (grg/s3-put (:creds this) (str (:value op)) "present") + (assoc op :type :ok)) + :read + (let [items (grg/s3-list (:creds this))] + (assoc op :type :ok, :value (set (map read-string items)))))) + (teardown! [this test]) + (close! [this test])) + +(defn workload + "Tests insertions and deletions" + [opts] + {:client (SetClient. nil) + :checker (checker/compose + {:set (checker/set) + :timeline (timeline/html)}) + ; :generator (gen/mix [op-add op-read]) + ; :generator (->> (range) + ; (map (fn [x] {:type :invoke, :f :add, :value x}))) + :generator (gen/mix [op-read + (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})))]) + :final-generator (gen/once op-read)}) + + -- cgit v1.2.3 From 0bb1577ae12b2dde313e036399f252266bc04d0a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 19 Apr 2023 15:59:30 +0200 Subject: two set workloads with different checkers --- script/jepsen.garage/src/jepsen/garage/grg.clj | 3 ++- script/jepsen.garage/src/jepsen/garage/set.clj | 20 ++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj index 2e9bcc72..4597fb14 100644 --- a/script/jepsen.garage/src/jepsen/garage/grg.clj +++ b/script/jepsen.garage/src/jepsen/garage/grg.clj @@ -120,10 +120,11 @@ (defn s3-list "Helper for ListObjects -- just lists everything in the bucket" - [creds] + [creds prefix] (defn list-inner [ct accum] (let [list-result (s3/list-objects-v2 creds {:bucket-name (:bucket creds) + :prefix prefix :continuation-token ct}) new-object-summaries (:object-summaries list-result) new-objects (map (fn [d] (:key d)) new-object-summaries) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index ab99e6ef..f8c92580 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -34,12 +34,12 @@ (grg/s3-put (:creds this) (str (:value op)) "present") (assoc op :type :ok)) :read - (let [items (grg/s3-list (:creds this))] + (let [items (grg/s3-list (:creds this) "")] (assoc op :type :ok, :value (set (map read-string items)))))) (teardown! [this test]) (close! [this test])) -(defn workload +(defn workload1 "Tests insertions and deletions" [opts] {:client (SetClient. nil) @@ -47,10 +47,18 @@ {:set (checker/set) :timeline (timeline/html)}) ; :generator (gen/mix [op-add op-read]) - ; :generator (->> (range) - ; (map (fn [x] {:type :invoke, :f :add, :value x}))) - :generator (gen/mix [op-read - (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})))]) + :generator (->> (range) + (map (fn [x] {:type :invoke, :f :add, :value x}))) :final-generator (gen/once op-read)}) +(defn workload2 + "Tests insertions and deletions" + [opts] + {:client (SetClient. nil) + :checker (checker/compose + {:set (checker/set-full {:linearizable? false}) + :timeline (timeline/html)}) + :generator (gen/mix [op-read + (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})))])}) + -- cgit v1.2.3 From 55eb4e87c42bf0da88186eb5b2fe1fbbbdf9ed43 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 19 Apr 2023 16:16:34 +0200 Subject: set tests with independant tests together --- script/jepsen.garage/src/jepsen/garage/set.clj | 55 ++++++++++++++++---------- 1 file changed, 35 insertions(+), 20 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index f8c92580..156493bc 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -28,14 +28,18 @@ (assoc this :creds creds))) (setup! [this test]) (invoke! [this test op] - (case (:f op) - :add - (do - (grg/s3-put (:creds this) (str (:value op)) "present") - (assoc op :type :ok)) - :read - (let [items (grg/s3-list (:creds this) "")] - (assoc op :type :ok, :value (set (map read-string items)))))) + (let [[k v] (:value op) + prefix (str "set" k "/")] + (case (:f op) + :add + (do + (grg/s3-put (:creds this) (str prefix v) "present") + (assoc op :type :ok)) + :read + (let [items (grg/s3-list (:creds this) prefix) + items-stripped (map (fn [o] (str/replace-first o prefix "")) items) + items-set (set (map read-string items-stripped))] + (assoc op :type :ok, :value (independent/tuple k items-set)))))) (teardown! [this test]) (close! [this test])) @@ -43,22 +47,33 @@ "Tests insertions and deletions" [opts] {:client (SetClient. nil) - :checker (checker/compose - {:set (checker/set) - :timeline (timeline/html)}) - ; :generator (gen/mix [op-add op-read]) - :generator (->> (range) - (map (fn [x] {:type :invoke, :f :add, :value x}))) - :final-generator (gen/once op-read)}) + :checker (independent/checker + (checker/compose + {:set (checker/set) + :timeline (timeline/html)})) + :generator (independent/concurrent-generator + 10 + (range 100) + (fn [k] + (->> + (gen/mix [op-add]) + (gen/limit (:ops-per-key opts))))) + :final-generator (independent/sequential-generator + (range 100) + (fn [k] (gen/once op-read)))}) (defn workload2 "Tests insertions and deletions" [opts] {:client (SetClient. nil) - :checker (checker/compose - {:set (checker/set-full {:linearizable? false}) - :timeline (timeline/html)}) - :generator (gen/mix [op-read - (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})))])}) + :checker (independent/checker + (checker/compose + {:set (checker/set-full {:linearizable? false}) + :timeline (timeline/html)})) + :generator (independent/concurrent-generator + 10 + (range 100) + (fn [k] + (gen/mix [op-add op-read])))}) -- cgit v1.2.3 From 18ee8efb5f7bfcd64bf1a1336ffd6f54574aa79f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 19 Apr 2023 22:02:03 +0200 Subject: Check read-after-write property for sets --- script/jepsen.garage/src/jepsen/garage/set.clj | 48 +++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 5 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index 156493bc..9b21d50e 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -1,10 +1,12 @@ (ns jepsen.garage.set (:require [clojure.tools.logging :refer :all] [clojure.string :as str] + [clojure.set :as set] [jepsen [checker :as checker] [cli :as cli] [client :as client] [control :as c] + [checker :as checker] [db :as db] [generator :as gen] [independent :as independent] @@ -17,7 +19,7 @@ [knossos.model :as model] [slingshot.slingshot :refer [try+]])) -(defn op-add [_ _] {:type :invoke, :f :add, :value (rand-int 100000)}) +(defn op-add-rand100 [_ _] {:type :invoke, :f :add, :value (rand-int 100)}) (defn op-read [_ _] {:type :invoke, :f :read, :value nil}) (defrecord SetClient [creds] @@ -43,6 +45,41 @@ (teardown! [this test]) (close! [this test])) +(defn set-read-after-write + "Read-after-Write checker for set operations" + [] + (reify checker/Checker + (check [this test history opts] + (let [init {:add-started #{} + :add-done #{} + :read-must-contain {} + :missed #{} + :unexpected #{}} + final (reduce + (fn [state op] + (info "state:" state) + (info "operation:" op) + (case [(:type op) (:f op)] + ([:invoke :add]) + (assoc state :add-started (conj (:add-started state) (:value op))) + ([:ok :add]) + (assoc state :add-done (conj (:add-done state) (:value op))) + ([:invoke :read]) + (assoc-in state [:read-must-contain (:process op)] (:add-done state)) + ([:ok :read]) + (let [read-must-contain (get (:process op) (:read-must-contain state)) + new-missed (set/difference read-must-contain (:value op)) + new-unexpected (set/difference (:value op) (:add-started state))] + (assoc state + :read-must-contain (dissoc (:read-must-contain state) (:process op)) + :missed (set/union (:missed state) new-missed), + :unexpected (set/union (:unexpected state) new-unexpected))) + state)) + init history) + valid? (and (empty? (:missed final)) (empty? (:unexpected final)))] + (info "final state:" final) + (assoc final :valid? valid?))))) + (defn workload1 "Tests insertions and deletions" [opts] @@ -55,8 +92,8 @@ 10 (range 100) (fn [k] - (->> - (gen/mix [op-add]) + (->> (range) + (map (fn [x] {:type :invoke, :f :add, :value x})) (gen/limit (:ops-per-key opts))))) :final-generator (independent/sequential-generator (range 100) @@ -68,12 +105,13 @@ {:client (SetClient. nil) :checker (independent/checker (checker/compose - {:set (checker/set-full {:linearizable? false}) + {:set-full (checker/set-full {:linearizable? false}) + :set-read-after-write (set-read-after-write) :timeline (timeline/html)})) :generator (independent/concurrent-generator 10 (range 100) (fn [k] - (gen/mix [op-add op-read])))}) + (gen/mix [op-add-rand100 op-read])))}) -- cgit v1.2.3 From 9ec4cca3340cfd03deb55cd58d6b354630e7eb5f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 19 Apr 2023 22:17:20 +0200 Subject: reformatting --- script/jepsen.garage/src/jepsen/garage/set.clj | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index 9b21d50e..5f76d1ac 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -57,8 +57,6 @@ :unexpected #{}} final (reduce (fn [state op] - (info "state:" state) - (info "operation:" op) (case [(:type op) (:f op)] ([:invoke :add]) (assoc state :add-started (conj (:add-started state) (:value op))) @@ -77,7 +75,6 @@ state)) init history) valid? (and (empty? (:missed final)) (empty? (:unexpected final)))] - (info "final state:" final) (assoc final :valid? valid?))))) (defn workload1 @@ -110,7 +107,7 @@ :timeline (timeline/html)})) :generator (independent/concurrent-generator 10 - (range 100) + (range) (fn [k] (gen/mix [op-add-rand100 op-read])))}) -- cgit v1.2.3 From ef5ca86dfc2678e8fc16b99daffeac40093945ae Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 18 Oct 2023 14:01:18 +0200 Subject: jepsen: update to garage 0.9.0 --- script/jepsen.garage/src/jepsen/garage/grg.clj | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj index 4597fb14..31f20852 100644 --- a/script/jepsen.garage/src/jepsen/garage/grg.clj +++ b/script/jepsen.garage/src/jepsen/garage/grg.clj @@ -9,6 +9,8 @@ ; CONSTANTS -- HOW GARAGE IS SET UP (def dir "/opt/garage") +(def data-dir (str dir "/data")) +(def meta-dir (str dir "/meta")) (def binary (str dir "/garage")) (def logfile (str dir "/garage.log")) (def pidfile (str dir "/garage.pid")) @@ -54,18 +56,19 @@ :chdir dir} binary :server) - (Thread/sleep 100) + (info node "garage daemon started") + (c/exec :sleep 5) (let [node-id (c/exec binary :node :id :-q)] (info node "node id:" node-id) (c/on-many (:nodes test) (c/exec binary :node :connect node-id)) - (c/exec binary :layout :assign (subs node-id 0 16) :-c 1 :-z :dc1 :-t node)) + (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)) (if (= node (first (:nodes test))) (do - (Thread/sleep 2000) + (c/exec :sleep 5) (c/exec binary :layout :apply :--version 1) (info node "garage status:" (c/exec binary :status)) - (c/exec binary :key :new :--name grg-key) + (c/exec binary :key :create grg-key) (c/exec binary :bucket :create grg-bucket) (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key) (info node "key info: " (c/exec binary :key :info grg-key)))))) @@ -73,7 +76,8 @@ (info node "tearing down garage" version) (c/su (cu/stop-daemon! binary pidfile) - (c/exec :rm :-rf dir))) + (c/exec :rm :-rf data-dir) + (c/exec :rm :-rf meta-dir))) db/LogFiles (log-files [_ test node] [logfile]))) -- cgit v1.2.3 From 012ade5d4b81cc623f45042a1d3ca5c3ddafa569 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 18 Oct 2023 14:06:32 +0200 Subject: jepsen: update jepsen and fix garage key info --- script/jepsen.garage/src/jepsen/garage/grg.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj index 31f20852..4347b1a7 100644 --- a/script/jepsen.garage/src/jepsen/garage/grg.clj +++ b/script/jepsen.garage/src/jepsen/garage/grg.clj @@ -87,7 +87,7 @@ (defn s3-creds "Get S3 credentials for node" [node] - (let [key-info (c/on node (c/exec binary :key :info grg-key)) + (let [key-info (c/on node (c/exec binary :key :info grg-key :--show-secret)) [_ ak sk] (re-matches #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*" key-info)] -- cgit v1.2.3 From 84d43501ce6dfd645d844ada0fcbff932d6fea76 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 18 Oct 2023 15:34:12 +0200 Subject: refactor jepsen setup logic --- script/jepsen.garage/src/jepsen/garage/grg.clj | 131 ++++++++++++++++--------- 1 file changed, 85 insertions(+), 46 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj index 4347b1a7..f6075762 100644 --- a/script/jepsen.garage/src/jepsen/garage/grg.clj +++ b/script/jepsen.garage/src/jepsen/garage/grg.clj @@ -1,6 +1,7 @@ (ns jepsen.garage.grg (:require [clojure.tools.logging :refer :all] [jepsen [control :as c] + [core :as jepsen] [db :as db]] [jepsen.control.util :as cu] [amazonica.aws.s3 :as s3] @@ -21,63 +22,101 @@ ; THE GARAGE DB +(defn install! + "Download and install Garage" + [node version] + (c/su + (c/trace + (info node "installing garage" version) + (c/exec :mkdir :-p dir) + (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") + cache (cu/cached-wget! url)] + (c/exec :cp cache binary)) + (c/exec :chmod :+x binary)))) + +(defn configure! + "Configure Garage" + [node] + (c/su + (c/trace + (cu/write-file! + (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" + "rpc_bind_addr = \"0.0.0.0:3901\"\n" + "rpc_public_addr = \"" node ":3901\"\n" + "db_engine = \"lmdb\"\n" + "replication_mode = \"3\"\n" + "data_dir = \"" dir "/data\"\n" + "metadata_dir = \"" dir "/meta\"\n" + "[s3_api]\n" + "s3_region = \"us-east-1\"\n" + "api_bind_addr = \"0.0.0.0:3900\"\n" + "[k2v_api]\n" + "api_bind_addr = \"0.0.0.0:3902\"\n" + "[admin]\n" + "api_bind_addr = \"0.0.0.0:3903\"\n" + "admin_token = \"" grg-admin-token "\"\n") + "/etc/garage.toml")))) + +(defn connect-node! + "Connect a Garage node to the rest of the cluster" + [test node] + (c/trace + (let [node-id (c/exec binary :node :id :-q)] + (info node "node id:" node-id) + (c/on-many (:nodes test) + (c/exec binary :node :connect node-id))))) + +(defn configure-node! + "Configure a Garage node to be part of a cluster layout" + [test node] + (c/trace + (let [node-id (c/exec binary :node :id :-q)] + (c/on (jepsen/primary test) + (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))))) + +(defn finalize-config! + "Apply the layout and create a key/bucket pair in the cluster" + [node] + (c/trace + (c/exec binary :layout :apply :--version 1) + (info node "garage status:" (c/exec binary :status)) + (c/exec binary :key :create grg-key) + (c/exec binary :bucket :create grg-bucket) + (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key) + (info node "key info: " (c/exec binary :key :info grg-key)))) + (defn db "Garage DB for a particular version" [version] (reify db/DB (setup! [_ test node] - (info node "installing garage" version) - (c/su - (c/exec :mkdir :-p dir) - (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") - cache (cu/wget! url)] - (c/exec :cp cache binary)) - (c/exec :chmod :+x binary) - (cu/write-file! - (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" - "rpc_bind_addr = \"0.0.0.0:3901\"\n" - "rpc_public_addr = \"" node ":3901\"\n" - "db_engine = \"lmdb\"\n" - "replication_mode = \"3\"\n" - "data_dir = \"" dir "/data\"\n" - "metadata_dir = \"" dir "/meta\"\n" - "[s3_api]\n" - "s3_region = \"us-east-1\"\n" - "api_bind_addr = \"0.0.0.0:3900\"\n" - "[k2v_api]\n" - "api_bind_addr = \"0.0.0.0:3902\"\n" - "[admin]\n" - "api_bind_addr = \"0.0.0.0:3903\"\n" - "admin_token = \"" grg-admin-token "\"\n") - "/etc/garage.toml") - (cu/start-daemon! - {:logfile logfile - :pidfile pidfile - :chdir dir} - binary - :server) - (info node "garage daemon started") - (c/exec :sleep 5) - (let [node-id (c/exec binary :node :id :-q)] - (info node "node id:" node-id) - (c/on-many (:nodes test) - (c/exec binary :node :connect node-id)) - (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)) - (if (= node (first (:nodes test))) - (do - (c/exec :sleep 5) - (c/exec binary :layout :apply :--version 1) - (info node "garage status:" (c/exec binary :status)) - (c/exec binary :key :create grg-key) - (c/exec binary :bucket :create grg-bucket) - (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key) - (info node "key info: " (c/exec binary :key :info grg-key)))))) + (install! node version) + (configure! node) + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir dir} + binary + :server) + (c/exec :sleep 3) + + (jepsen/synchronize test) + (connect-node! test node) + + (jepsen/synchronize test) + (configure-node! test node) + + (jepsen/synchronize test) + (when (= node (jepsen/primary test)) + (finalize-config! node))) + (teardown! [_ test node] (info node "tearing down garage" version) (c/su (cu/stop-daemon! binary pidfile) (c/exec :rm :-rf data-dir) (c/exec :rm :-rf meta-dir))) + db/LogFiles (log-files [_ test node] [logfile]))) -- cgit v1.2.3 From ddd3de7fcef2a2e05805766085c156d7fde9e12e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 18 Oct 2023 16:30:45 +0200 Subject: refactor jepsen code --- script/jepsen.garage/src/jepsen/garage/daemon.clj | 134 ++++++++++++++++ script/jepsen.garage/src/jepsen/garage/grg.clj | 178 ---------------------- script/jepsen.garage/src/jepsen/garage/reg.clj | 9 +- script/jepsen.garage/src/jepsen/garage/s3api.clj | 48 ++++++ script/jepsen.garage/src/jepsen/garage/set.clj | 9 +- 5 files changed, 192 insertions(+), 186 deletions(-) create mode 100644 script/jepsen.garage/src/jepsen/garage/daemon.clj delete mode 100644 script/jepsen.garage/src/jepsen/garage/grg.clj create mode 100644 script/jepsen.garage/src/jepsen/garage/s3api.clj (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj new file mode 100644 index 00000000..81163521 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj @@ -0,0 +1,134 @@ +(ns jepsen.garage.daemon + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c] + [core :as jepsen] + [db :as db]] + [jepsen.control.util :as cu])) + +; CONSTANTS -- HOW GARAGE IS SET UP + +(def base-dir "/opt/garage") +(def data-dir (str base-dir "/data")) +(def meta-dir (str base-dir "/meta")) +(def binary (str base-dir "/garage")) +(def logfile (str base-dir "/garage.log")) +(def pidfile (str base-dir "/garage.pid")) + +(def admin-token "icanhazadmin") +(def access-key "jepsen") +(def bucket-name "jepsen") + +; THE GARAGE DB + +(defn install! + "Download and install Garage" + [node version] + (c/su + (c/trace + (info node "installing garage" version) + (c/exec :mkdir :-p base-dir) + (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") + cache (cu/cached-wget! url)] + (c/exec :cp cache binary)) + (c/exec :chmod :+x binary)))) + +(defn configure! + "Configure Garage" + [node] + (c/su + (c/trace + (cu/write-file! + (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" + "rpc_bind_addr = \"0.0.0.0:3901\"\n" + "rpc_public_addr = \"" node ":3901\"\n" + "db_engine = \"lmdb\"\n" + "replication_mode = \"3\"\n" + "data_dir = \"" data-dir "\"\n" + "metadata_dir = \"" meta-dir "\"\n" + "[s3_api]\n" + "s3_region = \"us-east-1\"\n" + "api_bind_addr = \"0.0.0.0:3900\"\n" + "[k2v_api]\n" + "api_bind_addr = \"0.0.0.0:3902\"\n" + "[admin]\n" + "api_bind_addr = \"0.0.0.0:3903\"\n" + "admin_token = \"" admin-token "\"\n") + "/etc/garage.toml")))) + +(defn connect-node! + "Connect a Garage node to the rest of the cluster" + [test node] + (c/trace + (let [node-id (c/exec binary :node :id :-q)] + (info node "node id:" node-id) + (c/on-many (:nodes test) + (c/exec binary :node :connect node-id))))) + +(defn configure-node! + "Configure a Garage node to be part of a cluster layout" + [test node] + (c/trace + (let [node-id (c/exec binary :node :id :-q)] + (c/on (jepsen/primary test) + (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))))) + +(defn finalize-config! + "Apply the layout and create a key/bucket pair in the cluster" + [node] + (c/trace + (c/exec binary :layout :apply :--version 1) + (info node "garage status:" (c/exec binary :status)) + (c/exec binary :key :create access-key) + (c/exec binary :bucket :create bucket-name) + (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key) + (info node "key info: " (c/exec binary :key :info access-key)))) + +(defn db + "Garage DB for a particular version" + [version] + (reify db/DB + (setup! [_ test node] + (install! node version) + (configure! node) + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir base-dir} + binary + :server) + (c/exec :sleep 3) + + (jepsen/synchronize test) + (connect-node! test node) + + (jepsen/synchronize test) + (configure-node! test node) + + (jepsen/synchronize test) + (when (= node (jepsen/primary test)) + (finalize-config! node))) + + (teardown! [_ test node] + (info node "tearing down garage" version) + (c/su + (cu/stop-daemon! binary pidfile) + (c/exec :rm :-rf data-dir) + (c/exec :rm :-rf meta-dir))) + + db/LogFiles + (log-files [_ test node] + [logfile]))) + +(defn creds + "Obtain Garage credentials for node" + [node] + (let [key-info (c/on node (c/exec binary :key :info access-key :--show-secret)) + [_ ak sk] (re-matches + #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*" + key-info)] + {:access-key ak + :secret-key sk + :endpoint (str "http://" node ":3900") + :bucket bucket-name + :client-config {:path-style-access-enabled true}})) + diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/grg.clj deleted file mode 100644 index f6075762..00000000 --- a/script/jepsen.garage/src/jepsen/garage/grg.clj +++ /dev/null @@ -1,178 +0,0 @@ -(ns jepsen.garage.grg - (:require [clojure.tools.logging :refer :all] - [jepsen [control :as c] - [core :as jepsen] - [db :as db]] - [jepsen.control.util :as cu] - [amazonica.aws.s3 :as s3] - [slingshot.slingshot :refer [try+]])) - -; CONSTANTS -- HOW GARAGE IS SET UP - -(def dir "/opt/garage") -(def data-dir (str dir "/data")) -(def meta-dir (str dir "/meta")) -(def binary (str dir "/garage")) -(def logfile (str dir "/garage.log")) -(def pidfile (str dir "/garage.pid")) - -(def grg-admin-token "icanhazadmin") -(def grg-key "jepsen") -(def grg-bucket "jepsen") - -; THE GARAGE DB - -(defn install! - "Download and install Garage" - [node version] - (c/su - (c/trace - (info node "installing garage" version) - (c/exec :mkdir :-p dir) - (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") - cache (cu/cached-wget! url)] - (c/exec :cp cache binary)) - (c/exec :chmod :+x binary)))) - -(defn configure! - "Configure Garage" - [node] - (c/su - (c/trace - (cu/write-file! - (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" - "rpc_bind_addr = \"0.0.0.0:3901\"\n" - "rpc_public_addr = \"" node ":3901\"\n" - "db_engine = \"lmdb\"\n" - "replication_mode = \"3\"\n" - "data_dir = \"" dir "/data\"\n" - "metadata_dir = \"" dir "/meta\"\n" - "[s3_api]\n" - "s3_region = \"us-east-1\"\n" - "api_bind_addr = \"0.0.0.0:3900\"\n" - "[k2v_api]\n" - "api_bind_addr = \"0.0.0.0:3902\"\n" - "[admin]\n" - "api_bind_addr = \"0.0.0.0:3903\"\n" - "admin_token = \"" grg-admin-token "\"\n") - "/etc/garage.toml")))) - -(defn connect-node! - "Connect a Garage node to the rest of the cluster" - [test node] - (c/trace - (let [node-id (c/exec binary :node :id :-q)] - (info node "node id:" node-id) - (c/on-many (:nodes test) - (c/exec binary :node :connect node-id))))) - -(defn configure-node! - "Configure a Garage node to be part of a cluster layout" - [test node] - (c/trace - (let [node-id (c/exec binary :node :id :-q)] - (c/on (jepsen/primary test) - (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))))) - -(defn finalize-config! - "Apply the layout and create a key/bucket pair in the cluster" - [node] - (c/trace - (c/exec binary :layout :apply :--version 1) - (info node "garage status:" (c/exec binary :status)) - (c/exec binary :key :create grg-key) - (c/exec binary :bucket :create grg-bucket) - (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key) - (info node "key info: " (c/exec binary :key :info grg-key)))) - -(defn db - "Garage DB for a particular version" - [version] - (reify db/DB - (setup! [_ test node] - (install! node version) - (configure! node) - (cu/start-daemon! - {:logfile logfile - :pidfile pidfile - :chdir dir} - binary - :server) - (c/exec :sleep 3) - - (jepsen/synchronize test) - (connect-node! test node) - - (jepsen/synchronize test) - (configure-node! test node) - - (jepsen/synchronize test) - (when (= node (jepsen/primary test)) - (finalize-config! node))) - - (teardown! [_ test node] - (info node "tearing down garage" version) - (c/su - (cu/stop-daemon! binary pidfile) - (c/exec :rm :-rf data-dir) - (c/exec :rm :-rf meta-dir))) - - db/LogFiles - (log-files [_ test node] - [logfile]))) - -; GARAGE S3 HELPER FUNCTIONS - -(defn s3-creds - "Get S3 credentials for node" - [node] - (let [key-info (c/on node (c/exec binary :key :info grg-key :--show-secret)) - [_ ak sk] (re-matches - #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*" - key-info)] - {:access-key ak - :secret-key sk - :endpoint (str "http://" node ":3900") - :bucket grg-bucket - :client-config {:path-style-access-enabled true}})) - -(defn s3-get - "Helper for GetObject" - [creds k] - (try+ - (-> (s3/get-object creds (:bucket creds) k) - :input-stream - slurp) - (catch (re-find #"Key not found" (.getMessage %)) ex - nil))) - -(defn s3-put - "Helper for PutObject or DeleteObject (is a delete if value is nil)" - [creds k v] - (if (= v nil) - (s3/delete-object creds - :bucket-name (:bucket creds) - :key k) - (let [some-bytes (.getBytes v "UTF-8") - bytes-stream (java.io.ByteArrayInputStream. some-bytes)] - (s3/put-object creds - :bucket-name (:bucket creds) - :key k - :input-stream bytes-stream - :metadata {:content-length (count some-bytes)})))) - -(defn s3-list - "Helper for ListObjects -- just lists everything in the bucket" - [creds prefix] - (defn list-inner [ct accum] - (let [list-result (s3/list-objects-v2 creds - {:bucket-name (:bucket creds) - :prefix prefix - :continuation-token ct}) - new-object-summaries (:object-summaries list-result) - new-objects (map (fn [d] (:key d)) new-object-summaries) - objects (concat new-objects accum)] - (if (:truncated? list-result) - (list-inner (:next-continuation-token list-result) objects) - objects))) - (list-inner nil [])) diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj index 0b370b36..3a2c1d55 100644 --- a/script/jepsen.garage/src/jepsen/garage/reg.clj +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -13,7 +13,8 @@ [jepsen.checker.timeline :as timeline] [jepsen.control.util :as cu] [jepsen.os.debian :as debian] - [jepsen.garage.grg :as grg] + [jepsen.garage.daemon :as grg] + [jepsen.garage.s3api :as s3] [knossos.model :as model] [slingshot.slingshot :refer [try+]])) @@ -24,7 +25,7 @@ (defrecord RegClient [creds] client/Client (open! [this test node] - (let [creds (grg/s3-creds node)] + (let [creds (grg/creds node)] (info node "s3 credentials:" creds) (assoc this :creds creds))) (setup! [this test]) @@ -32,11 +33,11 @@ (let [[k v] (:value op)] (case (:f op) :read - (let [value (grg/s3-get (:creds this) k)] + (let [value (s3/get (:creds this) k)] (assoc op :type :ok, :value (independent/tuple k value))) :write (do - (grg/s3-put (:creds this) k v) + (s3/put (:creds this) k v) (assoc op :type :ok))))) (teardown! [this test]) (close! [this test])) diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj new file mode 100644 index 00000000..239d5423 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj @@ -0,0 +1,48 @@ +(ns jepsen.garage.s3api + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c]] + [amazonica.aws.s3 :as s3] + [slingshot.slingshot :refer [try+]])) + +; GARAGE S3 HELPER FUNCTIONS + +(defn get + "Helper for GetObject" + [creds k] + (try+ + (-> (s3/get-object creds (:bucket creds) k) + :input-stream + slurp) + (catch (re-find #"Key not found" (.getMessage %)) ex + nil))) + +(defn put + "Helper for PutObject or DeleteObject (is a delete if value is nil)" + [creds k v] + (if (= v nil) + (s3/delete-object creds + :bucket-name (:bucket creds) + :key k) + (let [some-bytes (.getBytes v "UTF-8") + bytes-stream (java.io.ByteArrayInputStream. some-bytes)] + (s3/put-object creds + :bucket-name (:bucket creds) + :key k + :input-stream bytes-stream + :metadata {:content-length (count some-bytes)})))) + +(defn list + "Helper for ListObjects -- just lists everything in the bucket" + [creds prefix] + (defn list-inner [ct accum] + (let [list-result (s3/list-objects-v2 creds + {:bucket-name (:bucket creds) + :prefix prefix + :continuation-token ct}) + new-object-summaries (:object-summaries list-result) + new-objects (map (fn [d] (:key d)) new-object-summaries) + objects (concat new-objects accum)] + (if (:truncated? list-result) + (list-inner (:next-continuation-token list-result) objects) + objects))) + (list-inner nil [])) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index 5f76d1ac..6bbc1ee0 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -15,7 +15,8 @@ [jepsen.checker.timeline :as timeline] [jepsen.control.util :as cu] [jepsen.os.debian :as debian] - [jepsen.garage.grg :as grg] + [jepsen.garage.daemon :as grg] + [jepsen.garage.s3api :as s3] [knossos.model :as model] [slingshot.slingshot :refer [try+]])) @@ -25,7 +26,7 @@ (defrecord SetClient [creds] client/Client (open! [this test node] - (let [creds (grg/s3-creds node)] + (let [creds (grg/creds node)] (info node "s3 credentials:" creds) (assoc this :creds creds))) (setup! [this test]) @@ -35,10 +36,10 @@ (case (:f op) :add (do - (grg/s3-put (:creds this) (str prefix v) "present") + (s3/put (:creds this) (str prefix v) "present") (assoc op :type :ok)) :read - (let [items (grg/s3-list (:creds this) prefix) + (let [items (s3/list (:creds this) prefix) items-stripped (map (fn [o] (str/replace-first o prefix "")) items) items-set (set (map read-string items-stripped))] (assoc op :type :ok, :value (independent/tuple k items-set)))))) -- cgit v1.2.3 From b3bf16ee27b061a3a091022e718b2994365f945c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 18 Oct 2023 17:51:34 +0200 Subject: make jepsen test more robust: handle errors and timeouts, fixed access key --- script/jepsen.garage/src/jepsen/garage/daemon.clj | 23 ++++++++++------------- script/jepsen.garage/src/jepsen/garage/reg.clj | 22 ++++++++++++++-------- 2 files changed, 24 insertions(+), 21 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj index 81163521..2dcfa94f 100644 --- a/script/jepsen.garage/src/jepsen/garage/daemon.clj +++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj @@ -15,7 +15,8 @@ (def pidfile (str base-dir "/garage.pid")) (def admin-token "icanhazadmin") -(def access-key "jepsen") +(def access-key-id "GK8bfb6a51286071c6c9cd8bc3") +(def secret-access-key "b0be95f71c1c6f16858a9edf395078b75c12ecb6b1c03385c4ae92076e4994a3") (def bucket-name "jepsen") ; THE GARAGE DB @@ -78,10 +79,10 @@ (c/trace (c/exec binary :layout :apply :--version 1) (info node "garage status:" (c/exec binary :status)) - (c/exec binary :key :create access-key) + (c/exec binary :key :import access-key-id secret-access-key :--yes) (c/exec binary :bucket :create bucket-name) - (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key) - (info node "key info: " (c/exec binary :key :info access-key)))) + (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key-id) + (info node "key info: " (c/exec binary :key :info access-key-id)))) (defn db "Garage DB for a particular version" @@ -122,13 +123,9 @@ (defn creds "Obtain Garage credentials for node" [node] - (let [key-info (c/on node (c/exec binary :key :info access-key :--show-secret)) - [_ ak sk] (re-matches - #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*" - key-info)] - {:access-key ak - :secret-key sk - :endpoint (str "http://" node ":3900") - :bucket bucket-name - :client-config {:path-style-access-enabled true}})) + {:access-key access-key-id + :secret-key secret-access-key + :endpoint (str "http://" node ":3900") + :bucket bucket-name + :client-config {:path-style-access-enabled true}}) diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj index 3a2c1d55..d5404c6b 100644 --- a/script/jepsen.garage/src/jepsen/garage/reg.clj +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -33,19 +33,25 @@ (let [[k v] (:value op)] (case (:f op) :read - (let [value (s3/get (:creds this) k)] - (assoc op :type :ok, :value (independent/tuple k value))) + (try+ + (let [value (s3/get (:creds this) k)] + (assoc op :type :ok, :value (independent/tuple k value))) + (catch (re-find #"Unavailable" (.getMessage %)) ex + (assoc op :type :fail, :error [:s3-error (.getMessage ex)]))) :write - (do - (s3/put (:creds this) k v) - (assoc op :type :ok))))) + (try+ + (do + (s3/put (:creds this) k v) + (assoc op :type :ok)) + (catch (re-find #"Unavailable" (.getMessage %)) ex + (assoc op :type :fail, :error [:s3-error (.getMessage ex)])))))) (teardown! [this test]) (close! [this test])) (defn workload "Tests linearizable reads and writes" [opts] - {:client (RegClient. nil) + {:client (client/timeout 10 (RegClient. nil)) :checker (independent/checker (checker/compose {:linear (checker/linearizable @@ -53,8 +59,8 @@ :algorithm :linear}) :timeline (timeline/html)})) :generator (independent/concurrent-generator - 10 - (range) + (/ (:concurrency opts) 10) ; divide threads in 10 groups + (range) ; working on 10 keys (fn [k] (->> (gen/mix [op-get op-put op-del]) -- cgit v1.2.3 From 74e50eddddf319ce1a32a9b57b3825ea40db3a6c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 19 Oct 2023 14:34:19 +0200 Subject: jepsen: refactoring --- script/jepsen.garage/src/jepsen/garage/daemon.clj | 2 +- script/jepsen.garage/src/jepsen/garage/reg.clj | 27 ++++++++++---------- script/jepsen.garage/src/jepsen/garage/set.clj | 30 ++++++++++++++--------- 3 files changed, 33 insertions(+), 26 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj index 2dcfa94f..f924dcd6 100644 --- a/script/jepsen.garage/src/jepsen/garage/daemon.clj +++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj @@ -43,7 +43,7 @@ "rpc_bind_addr = \"0.0.0.0:3901\"\n" "rpc_public_addr = \"" node ":3901\"\n" "db_engine = \"lmdb\"\n" - "replication_mode = \"3\"\n" + "replication_mode = \"2\"\n" "data_dir = \"" data-dir "\"\n" "metadata_dir = \"" meta-dir "\"\n" "[s3_api]\n" diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj index d5404c6b..f3d5cec5 100644 --- a/script/jepsen.garage/src/jepsen/garage/reg.clj +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -9,6 +9,7 @@ [generator :as gen] [independent :as independent] [nemesis :as nemesis] + [util :as util] [tests :as tests]] [jepsen.checker.timeline :as timeline] [jepsen.control.util :as cu] @@ -25,33 +26,31 @@ (defrecord RegClient [creds] client/Client (open! [this test node] - (let [creds (grg/creds node)] - (info node "s3 credentials:" creds) - (assoc this :creds creds))) + (assoc this :creds (grg/creds node))) (setup! [this test]) (invoke! [this test op] (let [[k v] (:value op)] (case (:f op) :read - (try+ + (util/timeout + 10000 + (assoc op :type :fail, :error ::timeout) (let [value (s3/get (:creds this) k)] - (assoc op :type :ok, :value (independent/tuple k value))) - (catch (re-find #"Unavailable" (.getMessage %)) ex - (assoc op :type :fail, :error [:s3-error (.getMessage ex)]))) + (assoc op :type :ok, :value (independent/tuple k value)))) :write - (try+ + (util/timeout + 10000 + (assoc op :type :info, :error ::timeout) (do (s3/put (:creds this) k v) - (assoc op :type :ok)) - (catch (re-find #"Unavailable" (.getMessage %)) ex - (assoc op :type :fail, :error [:s3-error (.getMessage ex)])))))) + (assoc op :type :ok)))))) (teardown! [this test]) (close! [this test])) (defn workload "Tests linearizable reads and writes" [opts] - {:client (client/timeout 10 (RegClient. nil)) + {:client (RegClient. nil) :checker (independent/checker (checker/compose {:linear (checker/linearizable @@ -59,8 +58,8 @@ :algorithm :linear}) :timeline (timeline/html)})) :generator (independent/concurrent-generator - (/ (:concurrency opts) 10) ; divide threads in 10 groups - (range) ; working on 10 keys + 10 + (range) (fn [k] (->> (gen/mix [op-get op-put op-del]) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index 6bbc1ee0..ff597095 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -11,6 +11,7 @@ [generator :as gen] [independent :as independent] [nemesis :as nemesis] + [util :as util] [tests :as tests]] [jepsen.checker.timeline :as timeline] [jepsen.control.util :as cu] @@ -26,23 +27,29 @@ (defrecord SetClient [creds] client/Client (open! [this test node] - (let [creds (grg/creds node)] - (info node "s3 credentials:" creds) - (assoc this :creds creds))) + (assoc this :creds (grg/creds node))) (setup! [this test]) (invoke! [this test op] (let [[k v] (:value op) prefix (str "set" k "/")] (case (:f op) :add - (do - (s3/put (:creds this) (str prefix v) "present") - (assoc op :type :ok)) + (util/timeout + 10000 + (assoc op :type :info, :error ::timeout) + (do + (s3/put (:creds this) (str prefix v) "present") + (assoc op :type :ok))) :read - (let [items (s3/list (:creds this) prefix) - items-stripped (map (fn [o] (str/replace-first o prefix "")) items) - items-set (set (map read-string items-stripped))] - (assoc op :type :ok, :value (independent/tuple k items-set)))))) + (util/timeout + 10000 + (assoc op :type :fail, :error ::timeout) + (let [items (s3/list (:creds this) prefix) + items-stripped (map (fn [o] + (assert (str/starts-with? o prefix)) + (str/replace-first o prefix "")) items) + items-set (set (map parse-long items-stripped))] + (assoc op :type :ok, :value (independent/tuple k items-set))))))) (teardown! [this test]) (close! [this test])) @@ -110,6 +117,7 @@ 10 (range) (fn [k] - (gen/mix [op-add-rand100 op-read])))}) + (->> (gen/mix [op-add-rand100 op-read]) + (gen/limit (:ops-per-key opts)))))}) -- cgit v1.2.3 From da8b1707489b70c25395ee49383ecbbd8c9f9404 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 19 Oct 2023 16:45:24 +0200 Subject: jepsen: investigating listobjects error --- script/jepsen.garage/src/jepsen/garage/daemon.clj | 7 +++++-- script/jepsen.garage/src/jepsen/garage/s3api.clj | 1 + script/jepsen.garage/src/jepsen/garage/set.clj | 17 ++++++++++------- 3 files changed, 16 insertions(+), 9 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj index f924dcd6..7c581ba1 100644 --- a/script/jepsen.garage/src/jepsen/garage/daemon.clj +++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj @@ -53,7 +53,8 @@ "api_bind_addr = \"0.0.0.0:3902\"\n" "[admin]\n" "api_bind_addr = \"0.0.0.0:3903\"\n" - "admin_token = \"" admin-token "\"\n") + "admin_token = \"" admin-token "\"\n" + "trace_sink = \"http://192.168.56.1:4317\"\n") "/etc/garage.toml")))) (defn connect-node! @@ -94,7 +95,8 @@ (cu/start-daemon! {:logfile logfile :pidfile pidfile - :chdir base-dir} + :chdir base-dir + :env {:RUST_LOG "garage=debug,garage_api=trace"}} binary :server) (c/exec :sleep 3) @@ -113,6 +115,7 @@ (info node "tearing down garage" version) (c/su (cu/stop-daemon! binary pidfile) + (c/exec :rm :-rf logfile) (c/exec :rm :-rf data-dir) (c/exec :rm :-rf meta-dir))) diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj index 239d5423..977de7dc 100644 --- a/script/jepsen.garage/src/jepsen/garage/s3api.clj +++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj @@ -42,6 +42,7 @@ new-object-summaries (:object-summaries list-result) new-objects (map (fn [d] (:key d)) new-object-summaries) objects (concat new-objects accum)] + (info (:endpoint creds) "ListObjectsV2 prefix(" prefix "), ct(" ct "): " new-objects) (if (:truncated? list-result) (list-inner (:next-continuation-token list-result) objects) objects))) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index ff597095..8a1ab83f 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -44,12 +44,13 @@ (util/timeout 10000 (assoc op :type :fail, :error ::timeout) - (let [items (s3/list (:creds this) prefix) - items-stripped (map (fn [o] + (let [items (s3/list (:creds this) prefix)] + (info "list results for prefix" prefix ":" items " (node:" (:endpoint (:creds this)) ")") + (let [items-stripped (map (fn [o] (assert (str/starts-with? o prefix)) (str/replace-first o prefix "")) items) - items-set (set (map parse-long items-stripped))] - (assoc op :type :ok, :value (independent/tuple k items-set))))))) + items-set (set (map parse-long items-stripped))] + (assoc op :type :ok, :value (independent/tuple k items-set)))))))) (teardown! [this test]) (close! [this test])) @@ -100,9 +101,11 @@ (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})) (gen/limit (:ops-per-key opts))))) - :final-generator (independent/sequential-generator - (range 100) - (fn [k] (gen/once op-read)))}) + :final-generator (gen/phases + (independent/sequential-generator + (range 100) + (fn [k] (gen/once op-read))) + (gen/sleep 5))}) (defn workload2 "Tests insertions and deletions" -- cgit v1.2.3 From ef662822c9e48ff7cfd9300590617e089c0a9498 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 19 Oct 2023 23:40:55 +0200 Subject: jepsen: fix the list-objects call (?) --- script/jepsen.garage/src/jepsen/garage/s3api.clj | 27 ++++++++++++------------ script/jepsen.garage/src/jepsen/garage/set.clj | 16 ++++++++------ 2 files changed, 23 insertions(+), 20 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj index 977de7dc..4f292ac0 100644 --- a/script/jepsen.garage/src/jepsen/garage/s3api.clj +++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj @@ -31,19 +31,20 @@ :input-stream bytes-stream :metadata {:content-length (count some-bytes)})))) +(defn list-inner [creds prefix ct accum] + (let [list-result (s3/list-objects-v2 creds + {:bucket-name (:bucket creds) + :prefix prefix + :continuation-token ct}) + new-object-summaries (:object-summaries list-result) + new-objects (map (fn [d] (:key d)) new-object-summaries) + objects (concat new-objects accum)] + (info (:endpoint creds) "ListObjectsV2 prefix(" prefix "), ct(" ct "): " new-objects) + (if (:truncated? list-result) + (list-inner creds prefix (:next-continuation-token list-result) objects) + objects))) (defn list "Helper for ListObjects -- just lists everything in the bucket" [creds prefix] - (defn list-inner [ct accum] - (let [list-result (s3/list-objects-v2 creds - {:bucket-name (:bucket creds) - :prefix prefix - :continuation-token ct}) - new-object-summaries (:object-summaries list-result) - new-objects (map (fn [d] (:key d)) new-object-summaries) - objects (concat new-objects accum)] - (info (:endpoint creds) "ListObjectsV2 prefix(" prefix "), ct(" ct "): " new-objects) - (if (:truncated? list-result) - (list-inner (:next-continuation-token list-result) objects) - objects))) - (list-inner nil [])) + (info "in s3/list creds:" creds ", prefix:" prefix) + (list-inner creds prefix nil [])) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index 8a1ab83f..c5387462 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -44,13 +44,15 @@ (util/timeout 10000 (assoc op :type :fail, :error ::timeout) - (let [items (s3/list (:creds this) prefix)] - (info "list results for prefix" prefix ":" items " (node:" (:endpoint (:creds this)) ")") - (let [items-stripped (map (fn [o] - (assert (str/starts-with? o prefix)) - (str/replace-first o prefix "")) items) - items-set (set (map parse-long items-stripped))] - (assoc op :type :ok, :value (independent/tuple k items-set)))))))) + (do + (info "call s3/list creds: " (:creds this) ", prefix:" prefix) + (let [items (s3/list (:creds this) prefix)] + (info "list results for prefix" prefix ":" items " (node:" (:endpoint (:creds this)) ")") + (let [items-stripped (map (fn [o] + (assert (str/starts-with? o prefix)) + (str/replace-first o prefix "")) items) + items-set (set (map parse-long items-stripped))] + (assoc op :type :ok, :value (independent/tuple k items-set))))))))) (teardown! [this test]) (close! [this test])) -- cgit v1.2.3 From 4ba18ce9cca1b828edcf3f8c8770d49c75ed3083 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 20 Oct 2023 12:13:11 +0200 Subject: jepsen: wip checker for register-like behavior --- script/jepsen.garage/src/jepsen/garage/reg.clj | 66 ++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj index f3d5cec5..b5bf28ff 100644 --- a/script/jepsen.garage/src/jepsen/garage/reg.clj +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -1,6 +1,7 @@ (ns jepsen.garage.reg (:require [clojure.tools.logging :refer :all] [clojure.string :as str] + [clojure.set :as set] [jepsen [checker :as checker] [cli :as cli] [client :as client] @@ -20,7 +21,7 @@ [slingshot.slingshot :refer [try+]])) (defn op-get [_ _] {:type :invoke, :f :read, :value nil}) -(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 9))}) +(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 99))}) (defn op-del [_ _] {:type :invoke, :f :write, :value nil}) (defrecord RegClient [creds] @@ -47,15 +48,72 @@ (teardown! [this test]) (close! [this test])) +(defn reg-read-after-write + "Read-after-Write checker for register operations" + [] + (reify checker/Checker + (check [this test history opts] + (let [init {:put-values {-1 nil} + :put-done #{-1} + :put-in-progress {} + :read-can-contain {} + :bad-reads #{}} + final (reduce + (fn [state op] + (let [current-values (set/union + (set (map (fn [idx] (get (:put-values state) idx)) (:put-done state))) + (set (map (fn [[_ [idx _]]] (get (:put-values state) idx)) (:put-in-progress state)))) + read-can-contain (reduce + (fn [rcc [idx v]] (assoc rcc idx (set/union current-values v))) + {} (:read-can-contain state))] + (info "--------") + (info "state: " state) + (info "current-values: " current-values) + (info "read-can-contain: " read-can-contain) + (info "op: " op) + (case [(:type op) (:f op)] + ([:invoke :write]) + (assoc state + :read-can-contain read-can-contain + :put-values (assoc (:put-values state) (:index op) (:value op)) + :put-in-progress (assoc (:put-in-progress state) (:process op) [(:index op) (:put-done state)])) + ([:ok :write]) + (let [[index overwrites] (get (:put-in-progress state) (:process op))] + (assoc state + :read-can-contain read-can-contain + :put-in-progress (dissoc (:put-in-progress state) (:process op)) + :put-done + (conj + (set/difference (:put-done state) overwrites) + index))) + ([:invoke :read]) + (assoc state + :read-can-contain (assoc read-can-contain (:process op) current-values)) + ([:ok :read]) + (let [this-read-can-contain (get read-can-contain (:process op)) + bad-reads (if (contains? this-read-can-contain (:value op)) + (:bad-reads state) + (conj (:bad-reads state) [(:process op) (:index op) (:value op) this-read-can-contain]))] + (info "this-read-can-contain: " this-read-can-contain) + (assoc state + :read-can-contain (dissoc read-can-contain (:process op)) + :bad-reads bad-reads)) + state))) + init history) + valid? (empty? (:bad-reads final))] + (assoc final :valid? valid?))))) + (defn workload "Tests linearizable reads and writes" [opts] {:client (RegClient. nil) :checker (independent/checker (checker/compose - {:linear (checker/linearizable - {:model (model/register) - :algorithm :linear}) + {:reg-read-after-write (reg-read-after-write) + ; linear test is desactivated, indeed Garage is not linear + ;:linear (checker/linearizable + ; {:model (model/register) + ; :algorithm :linear}) :timeline (timeline/html)})) :generator (independent/concurrent-generator 10 -- cgit v1.2.3 From 4b93ce179a3777c8461f3b5843dc3802bddc739c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 20 Oct 2023 12:56:45 +0200 Subject: jepsen: errors in reg2 workload under investigation --- script/jepsen.garage/src/jepsen/garage/reg.clj | 30 +++++++++++++++++--------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj index b5bf28ff..6772abfe 100644 --- a/script/jepsen.garage/src/jepsen/garage/reg.clj +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -103,18 +103,10 @@ valid? (empty? (:bad-reads final))] (assoc final :valid? valid?))))) -(defn workload - "Tests linearizable reads and writes" +(defn workload-common + "Common parts of workload" [opts] {:client (RegClient. nil) - :checker (independent/checker - (checker/compose - {:reg-read-after-write (reg-read-after-write) - ; linear test is desactivated, indeed Garage is not linear - ;:linear (checker/linearizable - ; {:model (model/register) - ; :algorithm :linear}) - :timeline (timeline/html)})) :generator (independent/concurrent-generator 10 (range) @@ -123,4 +115,22 @@ (gen/mix [op-get op-put op-del]) (gen/limit (:ops-per-key opts)))))}) +(defn workload1 + "Tests linearizable reads and writes" + [opts] + (assoc (workload-common opts) + :checker (independent/checker + (checker/compose + {:linear (checker/linearizable + {:model (model/register) + :algorithm :linear}) + :timeline (timeline/html)})))) +(defn workload2 + "Tests CRDT reads and writes" + [opts] + (assoc (workload-common opts) + :checker (independent/checker + (checker/compose + {:reg-read-after-write (reg-read-after-write) + :timeline (timeline/html)})))) -- cgit v1.2.3 From d148b83d4f440dc79b2ed08eaa171aca0e2037b0 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 20 Oct 2023 13:36:48 +0200 Subject: jepsen: reg2 failure seems to happen only with deleteobject --- script/jepsen.garage/src/jepsen/garage/reg.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj index 6772abfe..ecc96590 100644 --- a/script/jepsen.garage/src/jepsen/garage/reg.clj +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -112,7 +112,8 @@ (range) (fn [k] (->> - (gen/mix [op-get op-put op-del]) + ; (gen/mix [op-get op-put op-del]) + (gen/mix [op-get op-put]) (gen/limit (:ops-per-key opts)))))}) (defn workload1 -- cgit v1.2.3 From f5b09727815523a1bd4ba5f62d892b2b45b5bed6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 20 Oct 2023 15:00:10 +0200 Subject: jepsen: register crdt read-after-write is fixed with deleteobject patch --- script/jepsen.garage/src/jepsen/garage/reg.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj index ecc96590..6772abfe 100644 --- a/script/jepsen.garage/src/jepsen/garage/reg.clj +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -112,8 +112,7 @@ (range) (fn [k] (->> - ; (gen/mix [op-get op-put op-del]) - (gen/mix [op-get op-put]) + (gen/mix [op-get op-put op-del]) (gen/limit (:ops-per-key opts)))))}) (defn workload1 -- cgit v1.2.3 From 654775308ed03abd68941b07b2ad367a4de5c57f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 20 Oct 2023 15:48:37 +0200 Subject: jepsen: add cluster reconfiguration nemesis --- script/jepsen.garage/src/jepsen/garage/nemesis.clj | 91 ++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 script/jepsen.garage/src/jepsen/garage/nemesis.clj (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj new file mode 100644 index 00000000..7cd9306e --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -0,0 +1,91 @@ +(ns jepsen.garage.nemesis + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c] + [core :as jepsen] + [generator :as gen] + [nemesis :as nemesis]] + [jepsen.garage.daemon :as grg] + [jepsen.control.util :as cu])) + +(defn configure-present! + "Configure node to be active in new cluster layout" + [test node] + (info "configure-present!" node) + (let [node-id (c/on node (c/exec grg/binary :node :id :-q))] + (c/on + (jepsen/primary test) + (c/exec grg/binary :layout :assign (subs node-id 0 16) :-c :1G)))) + +(defn configure-absent! + "Configure node to be active in new cluster layout" + [test node] + (info "configure-absent!" node) + (let [node-id (c/on node (c/exec grg/binary :node :id :-q))] + (c/on + (jepsen/primary test) + (c/exec grg/binary :layout :assign (subs node-id 0 16) :-g)))) + +(defn finalize-config! + "Apply the proposed cluster layout" + [test] + (let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show)) + [_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)] + (info "layout show: " layout-show "; next-version: " layout-next-version) + (c/on (jepsen/primary test) + (c/exec grg/binary :layout :apply :--version layout-next-version)))) + +(defn reconfigure-subset + "Reconfigure cluster with only a subset of nodes" + [cnt] + (reify nemesis/Nemesis + (setup! [this test] this) + + (invoke! [this test op] op + (case (:f op) + :start + (let [[keep-nodes remove-nodes] + (->> (:nodes test) + shuffle + (split-at cnt))] + (info "layout split: keep " keep-nodes ", remove " remove-nodes) + (run! #(configure-present! test %) keep-nodes) + (run! #(configure-absent! test %) remove-nodes) + (finalize-config! test) + (assoc op :value keep-nodes)) + :stop + (do + (info "layout un-split: all nodes=" (:nodes test)) + (run! #(configure-present! test %) (:nodes test)) + (finalize-config! test) + (assoc op :value (:nodes test))))) + + (teardown! [this test] this))) + +(defn scenario-cp + "Clock scramble + parittion scenario" + [opts] + {:generator (cycle [(gen/sleep 5) + {:type :info, :f :partition-start} + (gen/sleep 5) + {:type :info, :f :clock-scramble} + (gen/sleep 5) + {:type :info, :f :partition-stop} + (gen/sleep 5) + {:type :info, :f :clock-scramble}]) + :nemesis (nemesis/compose + {{:partition-start :start + :partition-stop :stop} (nemesis/partition-random-halves) + {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})}) + +(defn scenario-r + "Cluster reconfiguration scenario" + [opts] + {:generator (cycle [(gen/sleep 5) + {:type :info, :f :reconfigure-start} + (gen/sleep 5) + {:type :info, :f :reconfigure-start} + (gen/sleep 5) + {:type :info, :f :reconfigure-stop}]) + :nemesis (nemesis/compose + {{:reconfigure-start :start + :reconfigure-stop :stop} (reconfigure-subset 3)})}) -- cgit v1.2.3 From 9030c1eef8f4c7c3435835f15e6b236497f93323 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 20 Oct 2023 15:53:46 +0200 Subject: jepsen: code path for nemesis final generator --- script/jepsen.garage/src/jepsen/garage/nemesis.clj | 44 +++++++++++----------- 1 file changed, 23 insertions(+), 21 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj index 7cd9306e..9edfe418 100644 --- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -64,28 +64,30 @@ (defn scenario-cp "Clock scramble + parittion scenario" [opts] - {:generator (cycle [(gen/sleep 5) - {:type :info, :f :partition-start} - (gen/sleep 5) - {:type :info, :f :clock-scramble} - (gen/sleep 5) - {:type :info, :f :partition-stop} - (gen/sleep 5) - {:type :info, :f :clock-scramble}]) - :nemesis (nemesis/compose - {{:partition-start :start - :partition-stop :stop} (nemesis/partition-random-halves) - {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})}) + {:generator (cycle [(gen/sleep 5) + {:type :info, :f :partition-start} + (gen/sleep 5) + {:type :info, :f :clock-scramble} + (gen/sleep 5) + {:type :info, :f :partition-stop} + (gen/sleep 5) + {:type :info, :f :clock-scramble}]) + :final-generator (gen/once {:type :info, :f :partition-stop}) + :nemesis (nemesis/compose + {{:partition-start :start + :partition-stop :stop} (nemesis/partition-random-halves) + {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})}) (defn scenario-r "Cluster reconfiguration scenario" [opts] - {:generator (cycle [(gen/sleep 5) - {:type :info, :f :reconfigure-start} - (gen/sleep 5) - {:type :info, :f :reconfigure-start} - (gen/sleep 5) - {:type :info, :f :reconfigure-stop}]) - :nemesis (nemesis/compose - {{:reconfigure-start :start - :reconfigure-stop :stop} (reconfigure-subset 3)})}) + {:generator (cycle [(gen/sleep 5) + {:type :info, :f :reconfigure-start} + (gen/sleep 5) + {:type :info, :f :reconfigure-start} + (gen/sleep 5) + {:type :info, :f :reconfigure-stop}]) + :final-generator (gen/once {:type :info, :f :reconfigure-stop}) + :nemesis (nemesis/compose + {{:reconfigure-start :start + :reconfigure-stop :stop} (reconfigure-subset 3)})}) -- cgit v1.2.3 From d2c365767b0a4cb70dcbb1d20b75f41e0f9c20c8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 24 Oct 2023 11:39:45 +0200 Subject: jepsen: more testing --- script/jepsen.garage/src/jepsen/garage/nemesis.clj | 34 +++++++++++++++++++++- script/jepsen.garage/src/jepsen/garage/s3api.clj | 2 -- script/jepsen.garage/src/jepsen/garage/set.clj | 6 ++-- 3 files changed, 35 insertions(+), 7 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj index 9edfe418..e64bcaf1 100644 --- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -7,6 +7,8 @@ [jepsen.garage.daemon :as grg] [jepsen.control.util :as cu])) +; ---- reconfiguration nemesis ---- + (defn configure-present! "Configure node to be active in new cluster layout" [test node] @@ -61,8 +63,18 @@ (teardown! [this test] this))) +; ---- nemesis scenari ---- + +(defn scenario-c + "Clock scramble scenario" + [opts] + {:generator (cycle [(gen/sleep 5) + {:type :info, :f :clock-scramble}]) + :nemesis (nemesis/compose + {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})}) + (defn scenario-cp - "Clock scramble + parittion scenario" + "Clock scramble + partition scenario" [opts] {:generator (cycle [(gen/sleep 5) {:type :info, :f :partition-start} @@ -91,3 +103,23 @@ :nemesis (nemesis/compose {{:reconfigure-start :start :reconfigure-stop :stop} (reconfigure-subset 3)})}) + +(defn scenario-pr + "Partition + cluster reconfiguration scenario" + [opts] + {:generator (cycle [(gen/sleep 3) + {:type :info, :f :reconfigure-start} + (gen/sleep 3) + {:type :info, :f :partition-start} + (gen/sleep 3) + {:type :info, :f :reconfigure-start} + (gen/sleep 3) + {:type :info, :f :partition-stop} + (gen/sleep 3) + {:type :info, :f :reconfigure-stop}]) + :final-generator (gen/once {:type :info, :f :partition-stop}) + :nemesis (nemesis/compose + {{:partition-start :start + :partition-stop :stop} (nemesis/partition-random-halves) + {:reconfigure-start :start + :reconfigure-stop :stop} (reconfigure-subset 3)})}) diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj index 4f292ac0..531e0157 100644 --- a/script/jepsen.garage/src/jepsen/garage/s3api.clj +++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj @@ -39,12 +39,10 @@ new-object-summaries (:object-summaries list-result) new-objects (map (fn [d] (:key d)) new-object-summaries) objects (concat new-objects accum)] - (info (:endpoint creds) "ListObjectsV2 prefix(" prefix "), ct(" ct "): " new-objects) (if (:truncated? list-result) (list-inner creds prefix (:next-continuation-token list-result) objects) objects))) (defn list "Helper for ListObjects -- just lists everything in the bucket" [creds prefix] - (info "in s3/list creds:" creds ", prefix:" prefix) (list-inner creds prefix nil [])) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index c5387462..f625e672 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -45,9 +45,7 @@ 10000 (assoc op :type :fail, :error ::timeout) (do - (info "call s3/list creds: " (:creds this) ", prefix:" prefix) (let [items (s3/list (:creds this) prefix)] - (info "list results for prefix" prefix ":" items " (node:" (:endpoint (:creds this)) ")") (let [items-stripped (map (fn [o] (assert (str/starts-with? o prefix)) (str/replace-first o prefix "")) items) @@ -115,8 +113,8 @@ {:client (SetClient. nil) :checker (independent/checker (checker/compose - {:set-full (checker/set-full {:linearizable? false}) - :set-read-after-write (set-read-after-write) + {:set-read-after-write (set-read-after-write) + ; :set-full (checker/set-full {:linearizable? false}) :timeline (timeline/html)})) :generator (independent/concurrent-generator 10 -- cgit v1.2.3 From d13bde5e26098313e789dd3793368a635cf1cc16 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 24 Oct 2023 15:44:05 +0200 Subject: jepsen: set1 and set2 don't fail anymore ?? --- script/jepsen.garage/src/jepsen/garage/nemesis.clj | 64 ++++++++++++---------- 1 file changed, 36 insertions(+), 28 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj index e64bcaf1..07083038 100644 --- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -76,30 +76,24 @@ (defn scenario-cp "Clock scramble + partition scenario" [opts] - {:generator (cycle [(gen/sleep 5) - {:type :info, :f :partition-start} - (gen/sleep 5) - {:type :info, :f :clock-scramble} - (gen/sleep 5) - {:type :info, :f :partition-stop} - (gen/sleep 5) - {:type :info, :f :clock-scramble}]) + {:generator (->> + (gen/mix [{:type :info, :f :clock-scramble} + {:type :info, :f :partition-stop} + {:type :info, :f :partition-start}]) + (gen/stagger 3)) :final-generator (gen/once {:type :info, :f :partition-stop}) :nemesis (nemesis/compose - {{:partition-start :start - :partition-stop :stop} (nemesis/partition-random-halves) - {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})}) + {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0) + {:partition-start :start + :partition-stop :stop} (nemesis/partition-random-halves)})}) (defn scenario-r "Cluster reconfiguration scenario" [opts] - {:generator (cycle [(gen/sleep 5) - {:type :info, :f :reconfigure-start} - (gen/sleep 5) - {:type :info, :f :reconfigure-start} - (gen/sleep 5) - {:type :info, :f :reconfigure-stop}]) - :final-generator (gen/once {:type :info, :f :reconfigure-stop}) + {:generator (->> + (gen/mix [{:type :info, :f :reconfigure-start} + {:type :info, :f :reconfigure-stop}]) + (gen/stagger 3)) :nemesis (nemesis/compose {{:reconfigure-start :start :reconfigure-stop :stop} (reconfigure-subset 3)})}) @@ -107,19 +101,33 @@ (defn scenario-pr "Partition + cluster reconfiguration scenario" [opts] - {:generator (cycle [(gen/sleep 3) - {:type :info, :f :reconfigure-start} - (gen/sleep 3) - {:type :info, :f :partition-start} - (gen/sleep 3) - {:type :info, :f :reconfigure-start} - (gen/sleep 3) - {:type :info, :f :partition-stop} - (gen/sleep 3) - {:type :info, :f :reconfigure-stop}]) + {:generator (->> + (gen/mix [{:type :info, :f :partition-start} + {:type :info, :f :partition-stop} + {:type :info, :f :reconfigure-start} + {:type :info, :f :reconfigure-stop}]) + (gen/stagger 3)) :final-generator (gen/once {:type :info, :f :partition-stop}) :nemesis (nemesis/compose {{:partition-start :start :partition-stop :stop} (nemesis/partition-random-halves) {:reconfigure-start :start :reconfigure-stop :stop} (reconfigure-subset 3)})}) + +(defn scenario-cpr + "Clock scramble + partition + cluster reconfiguration scenario" + [opts] + {:generator (->> + (gen/mix [{:type :info, :f :clock-scramble} + {:type :info, :f :partition-start} + {:type :info, :f :partition-stop} + {:type :info, :f :reconfigure-start} + {:type :info, :f :reconfigure-stop}]) + (gen/stagger 3)) + :final-generator (gen/once {:type :info, :f :partition-stop}) + :nemesis (nemesis/compose + {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0) + {:partition-start :start + :partition-stop :stop} (nemesis/partition-random-halves) + {:reconfigure-start :start + :reconfigure-stop :stop} (reconfigure-subset 3)})}) -- cgit v1.2.3 From d7ab2c639e330cb3214d83bf98107573ef41f6db Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 24 Oct 2023 16:39:50 +0200 Subject: jepsen: fix nemesis to actually generate many operations --- script/jepsen.garage/src/jepsen/garage/nemesis.clj | 57 +++++++++++++--------- 1 file changed, 33 insertions(+), 24 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj index 07083038..6a2e1935 100644 --- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -32,9 +32,12 @@ [test] (let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show)) [_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)] - (info "layout show: " layout-show "; next-version: " layout-next-version) - (c/on (jepsen/primary test) - (c/exec grg/binary :layout :apply :--version layout-next-version)))) + (if layout-next-version + (do + (info "layout show: " layout-show "; next-version: " layout-next-version) + (c/on (jepsen/primary test) + (c/exec grg/binary :layout :apply :--version layout-next-version))) + (info "no layout changes to apply")))) (defn reconfigure-subset "Reconfigure cluster with only a subset of nodes" @@ -65,22 +68,28 @@ ; ---- nemesis scenari ---- +(defn nemesis-op + "A generator for a single nemesis operation" + [op] + (fn [_ _] {:type :info, :f op})) + (defn scenario-c "Clock scramble scenario" [opts] - {:generator (cycle [(gen/sleep 5) - {:type :info, :f :clock-scramble}]) + {:generator (->> + (nemesis-op :clock-scramble) + (gen/stagger 5)) :nemesis (nemesis/compose - {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})}) + {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})}) (defn scenario-cp "Clock scramble + partition scenario" [opts] {:generator (->> - (gen/mix [{:type :info, :f :clock-scramble} - {:type :info, :f :partition-stop} - {:type :info, :f :partition-start}]) - (gen/stagger 3)) + (gen/mix [(nemesis-op :clock-scramble) + (nemesis-op :partition-stop) + (nemesis-op :partition-start)]) + (gen/stagger 5)) :final-generator (gen/once {:type :info, :f :partition-stop}) :nemesis (nemesis/compose {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0) @@ -91,9 +100,9 @@ "Cluster reconfiguration scenario" [opts] {:generator (->> - (gen/mix [{:type :info, :f :reconfigure-start} - {:type :info, :f :reconfigure-stop}]) - (gen/stagger 3)) + (gen/mix [(nemesis-op :reconfigure-start) + (nemesis-op :reconfigure-stop)]) + (gen/stagger 5)) :nemesis (nemesis/compose {{:reconfigure-start :start :reconfigure-stop :stop} (reconfigure-subset 3)})}) @@ -102,11 +111,11 @@ "Partition + cluster reconfiguration scenario" [opts] {:generator (->> - (gen/mix [{:type :info, :f :partition-start} - {:type :info, :f :partition-stop} - {:type :info, :f :reconfigure-start} - {:type :info, :f :reconfigure-stop}]) - (gen/stagger 3)) + (gen/mix [(nemesis-op :partition-start) + (nemesis-op :partition-stop) + (nemesis-op :reconfigure-start) + (nemesis-op :reconfigure-stop)]) + (gen/stagger 5)) :final-generator (gen/once {:type :info, :f :partition-stop}) :nemesis (nemesis/compose {{:partition-start :start @@ -118,12 +127,12 @@ "Clock scramble + partition + cluster reconfiguration scenario" [opts] {:generator (->> - (gen/mix [{:type :info, :f :clock-scramble} - {:type :info, :f :partition-start} - {:type :info, :f :partition-stop} - {:type :info, :f :reconfigure-start} - {:type :info, :f :reconfigure-stop}]) - (gen/stagger 3)) + (gen/mix [(nemesis-op :clock-scramble) + (nemesis-op :partition-start) + (nemesis-op :partition-stop) + (nemesis-op :reconfigure-start) + (nemesis-op :reconfigure-stop)]) + (gen/stagger 5)) :final-generator (gen/once {:type :info, :f :partition-stop}) :nemesis (nemesis/compose {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0) -- cgit v1.2.3 From db921cc05f8bcfccd0d0ba1d90b6dcd77f06dcdd Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 25 Oct 2023 11:41:34 +0200 Subject: jepsen: reconfigure nemesis + add db nemesis --- script/jepsen.garage/src/jepsen/garage/daemon.clj | 18 +++ script/jepsen.garage/src/jepsen/garage/nemesis.clj | 121 ++++++++++----------- script/jepsen.garage/src/jepsen/garage/reg.clj | 37 ++++--- script/jepsen.garage/src/jepsen/garage/set.clj | 49 +++++---- 4 files changed, 124 insertions(+), 101 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/daemon.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj index 7c581ba1..d407dd29 100644 --- a/script/jepsen.garage/src/jepsen/garage/daemon.clj +++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj @@ -119,6 +119,24 @@ (c/exec :rm :-rf data-dir) (c/exec :rm :-rf meta-dir))) + db/Pause + (pause! [_ test node] + (cu/grepkill! :stop binary)) + (resume! [_ test node] + (cu/grepkill! :cont binary)) + + db/Kill + (kill! [_ test node] + (cu/stop-daemon! binary pidfile)) + (start! [_ test node] + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir base-dir + :env {:RUST_LOG "garage=debug,garage_api=trace"}} + binary + :server)) + db/LogFiles (log-files [_ test node] [logfile]))) diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj index 6a2e1935..0222e463 100644 --- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -4,6 +4,7 @@ [core :as jepsen] [generator :as gen] [nemesis :as nemesis]] + [jepsen.nemesis.combined :as combined] [jepsen.garage.daemon :as grg] [jepsen.control.util :as cu])) @@ -11,21 +12,23 @@ (defn configure-present! "Configure node to be active in new cluster layout" - [test node] - (info "configure-present!" node) - (let [node-id (c/on node (c/exec grg/binary :node :id :-q))] - (c/on - (jepsen/primary test) - (c/exec grg/binary :layout :assign (subs node-id 0 16) :-c :1G)))) + [test nodes] + (info "configure-present!" nodes) + (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q)) + node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)] + (c/on + (jepsen/primary test) + (apply c/exec (concat [grg/binary :layout :assign :-c :1G] node-id-strs))))) (defn configure-absent! - "Configure node to be active in new cluster layout" - [test node] - (info "configure-absent!" node) - (let [node-id (c/on node (c/exec grg/binary :node :id :-q))] - (c/on - (jepsen/primary test) - (c/exec grg/binary :layout :assign (subs node-id 0 16) :-g)))) + "Configure nodes to be active in new cluster layout" + [test nodes] + (info "configure-absent!" nodes) + (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q)) + node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)] + (c/on + (jepsen/primary test) + (apply c/exec (concat [grg/binary :layout :assign :-g] node-id-strs))))) (defn finalize-config! "Apply the proposed cluster layout" @@ -53,14 +56,14 @@ shuffle (split-at cnt))] (info "layout split: keep " keep-nodes ", remove " remove-nodes) - (run! #(configure-present! test %) keep-nodes) - (run! #(configure-absent! test %) remove-nodes) + (configure-present! test keep-nodes) + (configure-absent! test remove-nodes) (finalize-config! test) (assoc op :value keep-nodes)) :stop (do (info "layout un-split: all nodes=" (:nodes test)) - (run! #(configure-present! test %) (:nodes test)) + (configure-present! test (:nodes test)) (finalize-config! test) (assoc op :value (:nodes test))))) @@ -73,70 +76,58 @@ [op] (fn [_ _] {:type :info, :f op})) -(defn scenario-c - "Clock scramble scenario" +(defn reconfiguration-package + "Cluster reconfiguration nemesis package" [opts] {:generator (->> - (nemesis-op :clock-scramble) - (gen/stagger 5)) + (gen/mix [(nemesis-op :reconfigure-start) + (nemesis-op :reconfigure-stop)]) + (gen/stagger (:interval opts 5))) + :final-generator {:type :info, :f :reconfigure-stop} :nemesis (nemesis/compose - {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})}) + {{:reconfigure-start :start + :reconfigure-stop :stop} (reconfigure-subset 3)}) + :perf #{{:name "reconfigure" + :start #{:reconfigure-start} + :stop #{:reconfigur-stop} + :color "#A197E9"}}}) + +(defn scenario-c + "Clock modifying scenario" + [opts] + (combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})) (defn scenario-cp - "Clock scramble + partition scenario" + "Clock modifying + partition scenario" [opts] - {:generator (->> - (gen/mix [(nemesis-op :clock-scramble) - (nemesis-op :partition-stop) - (nemesis-op :partition-start)]) - (gen/stagger 5)) - :final-generator (gen/once {:type :info, :f :partition-stop}) - :nemesis (nemesis/compose - {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0) - {:partition-start :start - :partition-stop :stop} (nemesis/partition-random-halves)})}) + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})])) (defn scenario-r "Cluster reconfiguration scenario" [opts] - {:generator (->> - (gen/mix [(nemesis-op :reconfigure-start) - (nemesis-op :reconfigure-stop)]) - (gen/stagger 5)) - :nemesis (nemesis/compose - {{:reconfigure-start :start - :reconfigure-stop :stop} (reconfigure-subset 3)})}) + (reconfiguration-package {:interval 1})) (defn scenario-pr "Partition + cluster reconfiguration scenario" [opts] - {:generator (->> - (gen/mix [(nemesis-op :partition-start) - (nemesis-op :partition-stop) - (nemesis-op :reconfigure-start) - (nemesis-op :reconfigure-stop)]) - (gen/stagger 5)) - :final-generator (gen/once {:type :info, :f :partition-stop}) - :nemesis (nemesis/compose - {{:partition-start :start - :partition-stop :stop} (nemesis/partition-random-halves) - {:reconfigure-start :start - :reconfigure-stop :stop} (reconfigure-subset 3)})}) + (combined/compose-packages + [(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) (defn scenario-cpr "Clock scramble + partition + cluster reconfiguration scenario" [opts] - {:generator (->> - (gen/mix [(nemesis-op :clock-scramble) - (nemesis-op :partition-start) - (nemesis-op :partition-stop) - (nemesis-op :reconfigure-start) - (nemesis-op :reconfigure-stop)]) - (gen/stagger 5)) - :final-generator (gen/once {:type :info, :f :partition-stop}) - :nemesis (nemesis/compose - {{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0) - {:partition-start :start - :partition-stop :stop} (nemesis/partition-random-halves) - {:reconfigure-start :start - :reconfigure-stop :stop} (reconfigure-subset 3)})}) + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) + +(defn scenario-dpr + "Db + partition + cluster reconfiguration scenario" + [opts] + (combined/compose-packages + [(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) + (reconfiguration-package {:interval 1})])) diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj index 6772abfe..39708c0b 100644 --- a/script/jepsen.garage/src/jepsen/garage/reg.clj +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -30,21 +30,28 @@ (assoc this :creds (grg/creds node))) (setup! [this test]) (invoke! [this test op] - (let [[k v] (:value op)] - (case (:f op) - :read - (util/timeout - 10000 - (assoc op :type :fail, :error ::timeout) - (let [value (s3/get (:creds this) k)] - (assoc op :type :ok, :value (independent/tuple k value)))) - :write - (util/timeout - 10000 - (assoc op :type :info, :error ::timeout) - (do - (s3/put (:creds this) k v) - (assoc op :type :ok)))))) + (try+ + (let [[k v] (:value op)] + (case (:f op) + :read + (util/timeout + 10000 + (assoc op :type :fail, :error ::timeout) + (let [value (s3/get (:creds this) k)] + (assoc op :type :ok, :value (independent/tuple k value)))) + :write + (util/timeout + 10000 + (assoc op :type :info, :error ::timeout) + (do + (s3/put (:creds this) k v) + (assoc op :type :ok))))) + (catch (re-find #"Unavailable" (.getMessage %)) ex + (assoc op :type :info, :error ::unavailable)) + (catch (re-find #"Broken pipe" (.getMessage %)) ex + (assoc op :type :info, :error ::broken-pipe)) + (catch (re-find #"Connection refused" (.getMessage %)) ex + (assoc op :type :info, :error ::connection-refused)))) (teardown! [this test]) (close! [this test])) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index f625e672..670c73f2 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -30,27 +30,34 @@ (assoc this :creds (grg/creds node))) (setup! [this test]) (invoke! [this test op] - (let [[k v] (:value op) - prefix (str "set" k "/")] - (case (:f op) - :add - (util/timeout - 10000 - (assoc op :type :info, :error ::timeout) - (do - (s3/put (:creds this) (str prefix v) "present") - (assoc op :type :ok))) - :read - (util/timeout - 10000 - (assoc op :type :fail, :error ::timeout) - (do - (let [items (s3/list (:creds this) prefix)] - (let [items-stripped (map (fn [o] - (assert (str/starts-with? o prefix)) - (str/replace-first o prefix "")) items) - items-set (set (map parse-long items-stripped))] - (assoc op :type :ok, :value (independent/tuple k items-set))))))))) + (try+ + (let [[k v] (:value op) + prefix (str "set" k "/")] + (case (:f op) + :add + (util/timeout + 10000 + (assoc op :type :info, :error ::timeout) + (do + (s3/put (:creds this) (str prefix v) "present") + (assoc op :type :ok))) + :read + (util/timeout + 10000 + (assoc op :type :fail, :error ::timeout) + (do + (let [items (s3/list (:creds this) prefix)] + (let [items-stripped (map (fn [o] + (assert (str/starts-with? o prefix)) + (str/replace-first o prefix "")) items) + items-set (set (map parse-long items-stripped))] + (assoc op :type :ok, :value (independent/tuple k items-set)))))))) + (catch (re-find #"Unavailable" (.getMessage %)) ex + (assoc op :type :info, :error ::unavailable)) + (catch (re-find #"Broken pipe" (.getMessage %)) ex + (assoc op :type :info, :error ::broken-pipe)) + (catch (re-find #"Connection refused" (.getMessage %)) ex + (assoc op :type :info, :error ::connection-refused)))) (teardown! [this test]) (close! [this test])) -- cgit v1.2.3 From cfbfa09d24727e83bc042764dad2751e944fc939 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 25 Oct 2023 11:50:16 +0200 Subject: jepsen: fix set2 test omg finally this is so stupid --- script/jepsen.garage/src/jepsen/garage/set.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index 670c73f2..a73b8efc 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -81,7 +81,7 @@ ([:invoke :read]) (assoc-in state [:read-must-contain (:process op)] (:add-done state)) ([:ok :read]) - (let [read-must-contain (get (:process op) (:read-must-contain state)) + (let [read-must-contain (get (:read-must-contain state) (:process op)) new-missed (set/difference read-must-contain (:value op)) new-unexpected (set/difference (:value op) (:add-started state))] (assoc state -- cgit v1.2.3 From 5b1f50be65c251a1dc0a4358c706c409f17a82c0 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 25 Oct 2023 14:43:24 +0200 Subject: jepsen: testing --- script/jepsen.garage/src/jepsen/garage/nemesis.clj | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj index 0222e463..dfce0255 100644 --- a/script/jepsen.garage/src/jepsen/garage/nemesis.clj +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -124,6 +124,14 @@ (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) (reconfiguration-package {:interval 1})])) +(defn scenario-cdp + "Clock modifying + db + partition scenario" + [opts] + (combined/compose-packages + [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) + (combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) + (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})])) + (defn scenario-dpr "Db + partition + cluster reconfiguration scenario" [opts] @@ -131,3 +139,4 @@ [(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) (reconfiguration-package {:interval 1})])) + -- cgit v1.2.3 From fa9247f11b89c960dffe82d6bf990ed4335788e3 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 14 Dec 2023 16:23:48 +0100 Subject: jepsen: updated results, confirming that task3 works --- script/jepsen.garage/src/jepsen/garage/set.clj | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'script/jepsen.garage/src/jepsen/garage') diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index a73b8efc..2c7a2ccd 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -108,11 +108,13 @@ (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})) (gen/limit (:ops-per-key opts))))) - :final-generator (gen/phases - (independent/sequential-generator - (range 100) - (fn [k] (gen/once op-read))) - (gen/sleep 5))}) + :final-generator (independent/concurrent-generator + 10 + (range 100) + (fn [k] + (gen/phases + (gen/once op-read) + (gen/sleep 5))))}) (defn workload2 "Tests insertions and deletions" -- cgit v1.2.3