aboutsummaryrefslogtreecommitdiff
path: root/script/jepsen.garage/src/jepsen
diff options
context:
space:
mode:
Diffstat (limited to 'script/jepsen.garage/src/jepsen')
-rw-r--r--script/jepsen.garage/src/jepsen/garage.clj2
-rw-r--r--script/jepsen.garage/src/jepsen/garage/daemon.clj (renamed from script/jepsen.garage/src/jepsen/garage/grg.clj)92
-rw-r--r--script/jepsen.garage/src/jepsen/garage/reg.clj9
-rw-r--r--script/jepsen.garage/src/jepsen/garage/s3api.clj48
-rw-r--r--script/jepsen.garage/src/jepsen/garage/set.clj9
5 files changed, 83 insertions, 77 deletions
diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj
index 754ddf7d..5816512b 100644
--- a/script/jepsen.garage/src/jepsen/garage.clj
+++ b/script/jepsen.garage/src/jepsen/garage.clj
@@ -9,7 +9,7 @@
[tests :as tests]]
[jepsen.os.debian :as debian]
[jepsen.garage
- [grg :as grg]
+ [daemon :as grg]
[reg :as reg]
[set :as set]]))
diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj
index f6075762..81163521 100644
--- a/script/jepsen.garage/src/jepsen/garage/grg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj
@@ -1,24 +1,22 @@
-(ns jepsen.garage.grg
+(ns jepsen.garage.daemon
(:require [clojure.tools.logging :refer :all]
[jepsen [control :as c]
[core :as jepsen]
[db :as db]]
- [jepsen.control.util :as cu]
- [amazonica.aws.s3 :as s3]
- [slingshot.slingshot :refer [try+]]))
+ [jepsen.control.util :as cu]))
; CONSTANTS -- HOW GARAGE IS SET UP
-(def dir "/opt/garage")
-(def data-dir (str dir "/data"))
-(def meta-dir (str dir "/meta"))
-(def binary (str dir "/garage"))
-(def logfile (str dir "/garage.log"))
-(def pidfile (str dir "/garage.pid"))
+(def base-dir "/opt/garage")
+(def data-dir (str base-dir "/data"))
+(def meta-dir (str base-dir "/meta"))
+(def binary (str base-dir "/garage"))
+(def logfile (str base-dir "/garage.log"))
+(def pidfile (str base-dir "/garage.pid"))
-(def grg-admin-token "icanhazadmin")
-(def grg-key "jepsen")
-(def grg-bucket "jepsen")
+(def admin-token "icanhazadmin")
+(def access-key "jepsen")
+(def bucket-name "jepsen")
; THE GARAGE DB
@@ -28,7 +26,7 @@
(c/su
(c/trace
(info node "installing garage" version)
- (c/exec :mkdir :-p dir)
+ (c/exec :mkdir :-p base-dir)
(let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
cache (cu/cached-wget! url)]
(c/exec :cp cache binary))
@@ -45,8 +43,8 @@
"rpc_public_addr = \"" node ":3901\"\n"
"db_engine = \"lmdb\"\n"
"replication_mode = \"3\"\n"
- "data_dir = \"" dir "/data\"\n"
- "metadata_dir = \"" dir "/meta\"\n"
+ "data_dir = \"" data-dir "\"\n"
+ "metadata_dir = \"" meta-dir "\"\n"
"[s3_api]\n"
"s3_region = \"us-east-1\"\n"
"api_bind_addr = \"0.0.0.0:3900\"\n"
@@ -54,7 +52,7 @@
"api_bind_addr = \"0.0.0.0:3902\"\n"
"[admin]\n"
"api_bind_addr = \"0.0.0.0:3903\"\n"
- "admin_token = \"" grg-admin-token "\"\n")
+ "admin_token = \"" admin-token "\"\n")
"/etc/garage.toml"))))
(defn connect-node!
@@ -80,10 +78,10 @@
(c/trace
(c/exec binary :layout :apply :--version 1)
(info node "garage status:" (c/exec binary :status))
- (c/exec binary :key :create grg-key)
- (c/exec binary :bucket :create grg-bucket)
- (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
- (info node "key info: " (c/exec binary :key :info grg-key))))
+ (c/exec binary :key :create access-key)
+ (c/exec binary :bucket :create bucket-name)
+ (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key)
+ (info node "key info: " (c/exec binary :key :info access-key))))
(defn db
"Garage DB for a particular version"
@@ -95,7 +93,7 @@
(cu/start-daemon!
{:logfile logfile
:pidfile pidfile
- :chdir dir}
+ :chdir base-dir}
binary
:server)
(c/exec :sleep 3)
@@ -121,58 +119,16 @@
(log-files [_ test node]
[logfile])))
-; GARAGE S3 HELPER FUNCTIONS
-
-(defn s3-creds
- "Get S3 credentials for node"
+(defn creds
+ "Obtain Garage credentials for node"
[node]
- (let [key-info (c/on node (c/exec binary :key :info grg-key :--show-secret))
+ (let [key-info (c/on node (c/exec binary :key :info access-key :--show-secret))
[_ ak sk] (re-matches
#"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
key-info)]
{:access-key ak
:secret-key sk
:endpoint (str "http://" node ":3900")
- :bucket grg-bucket
+ :bucket bucket-name
:client-config {:path-style-access-enabled true}}))
-(defn s3-get
- "Helper for GetObject"
- [creds k]
- (try+
- (-> (s3/get-object creds (:bucket creds) k)
- :input-stream
- slurp)
- (catch (re-find #"Key not found" (.getMessage %)) ex
- nil)))
-
-(defn s3-put
- "Helper for PutObject or DeleteObject (is a delete if value is nil)"
- [creds k v]
- (if (= v nil)
- (s3/delete-object creds
- :bucket-name (:bucket creds)
- :key k)
- (let [some-bytes (.getBytes v "UTF-8")
- bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
- (s3/put-object creds
- :bucket-name (:bucket creds)
- :key k
- :input-stream bytes-stream
- :metadata {:content-length (count some-bytes)}))))
-
-(defn s3-list
- "Helper for ListObjects -- just lists everything in the bucket"
- [creds prefix]
- (defn list-inner [ct accum]
- (let [list-result (s3/list-objects-v2 creds
- {:bucket-name (:bucket creds)
- :prefix prefix
- :continuation-token ct})
- new-object-summaries (:object-summaries list-result)
- new-objects (map (fn [d] (:key d)) new-object-summaries)
- objects (concat new-objects accum)]
- (if (:truncated? list-result)
- (list-inner (:next-continuation-token list-result) objects)
- objects)))
- (list-inner nil []))
diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj
index 0b370b36..3a2c1d55 100644
--- a/script/jepsen.garage/src/jepsen/garage/reg.clj
+++ b/script/jepsen.garage/src/jepsen/garage/reg.clj
@@ -13,7 +13,8 @@
[jepsen.checker.timeline :as timeline]
[jepsen.control.util :as cu]
[jepsen.os.debian :as debian]
- [jepsen.garage.grg :as grg]
+ [jepsen.garage.daemon :as grg]
+ [jepsen.garage.s3api :as s3]
[knossos.model :as model]
[slingshot.slingshot :refer [try+]]))
@@ -24,7 +25,7 @@
(defrecord RegClient [creds]
client/Client
(open! [this test node]
- (let [creds (grg/s3-creds node)]
+ (let [creds (grg/creds node)]
(info node "s3 credentials:" creds)
(assoc this :creds creds)))
(setup! [this test])
@@ -32,11 +33,11 @@
(let [[k v] (:value op)]
(case (:f op)
:read
- (let [value (grg/s3-get (:creds this) k)]
+ (let [value (s3/get (:creds this) k)]
(assoc op :type :ok, :value (independent/tuple k value)))
:write
(do
- (grg/s3-put (:creds this) k v)
+ (s3/put (:creds this) k v)
(assoc op :type :ok)))))
(teardown! [this test])
(close! [this test]))
diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj
new file mode 100644
index 00000000..239d5423
--- /dev/null
+++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj
@@ -0,0 +1,48 @@
+(ns jepsen.garage.s3api
+ (:require [clojure.tools.logging :refer :all]
+ [jepsen [control :as c]]
+ [amazonica.aws.s3 :as s3]
+ [slingshot.slingshot :refer [try+]]))
+
+; GARAGE S3 HELPER FUNCTIONS
+
+(defn get
+ "Helper for GetObject"
+ [creds k]
+ (try+
+ (-> (s3/get-object creds (:bucket creds) k)
+ :input-stream
+ slurp)
+ (catch (re-find #"Key not found" (.getMessage %)) ex
+ nil)))
+
+(defn put
+ "Helper for PutObject or DeleteObject (is a delete if value is nil)"
+ [creds k v]
+ (if (= v nil)
+ (s3/delete-object creds
+ :bucket-name (:bucket creds)
+ :key k)
+ (let [some-bytes (.getBytes v "UTF-8")
+ bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
+ (s3/put-object creds
+ :bucket-name (:bucket creds)
+ :key k
+ :input-stream bytes-stream
+ :metadata {:content-length (count some-bytes)}))))
+
+(defn list
+ "Helper for ListObjects -- just lists everything in the bucket"
+ [creds prefix]
+ (defn list-inner [ct accum]
+ (let [list-result (s3/list-objects-v2 creds
+ {:bucket-name (:bucket creds)
+ :prefix prefix
+ :continuation-token ct})
+ new-object-summaries (:object-summaries list-result)
+ new-objects (map (fn [d] (:key d)) new-object-summaries)
+ objects (concat new-objects accum)]
+ (if (:truncated? list-result)
+ (list-inner (:next-continuation-token list-result) objects)
+ objects)))
+ (list-inner nil []))
diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj
index 5f76d1ac..6bbc1ee0 100644
--- a/script/jepsen.garage/src/jepsen/garage/set.clj
+++ b/script/jepsen.garage/src/jepsen/garage/set.clj
@@ -15,7 +15,8 @@
[jepsen.checker.timeline :as timeline]
[jepsen.control.util :as cu]
[jepsen.os.debian :as debian]
- [jepsen.garage.grg :as grg]
+ [jepsen.garage.daemon :as grg]
+ [jepsen.garage.s3api :as s3]
[knossos.model :as model]
[slingshot.slingshot :refer [try+]]))
@@ -25,7 +26,7 @@
(defrecord SetClient [creds]
client/Client
(open! [this test node]
- (let [creds (grg/s3-creds node)]
+ (let [creds (grg/creds node)]
(info node "s3 credentials:" creds)
(assoc this :creds creds)))
(setup! [this test])
@@ -35,10 +36,10 @@
(case (:f op)
:add
(do
- (grg/s3-put (:creds this) (str prefix v) "present")
+ (s3/put (:creds this) (str prefix v) "present")
(assoc op :type :ok))
:read
- (let [items (grg/s3-list (:creds this) prefix)
+ (let [items (s3/list (:creds this) prefix)
items-stripped (map (fn [o] (str/replace-first o prefix "")) items)
items-set (set (map read-string items-stripped))]
(assoc op :type :ok, :value (independent/tuple k items-set))))))
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-02 17:21:59 +0000