(ns jepsen.garage.daemon (:require [clojure.tools.logging :refer :all] [jepsen [control :as c] [core :as jepsen] [db :as db]] [jepsen.control.util :as cu])) ; CONSTANTS -- HOW GARAGE IS SET UP (def base-dir "/opt/garage") (def data-dir (str base-dir "/data")) (def meta-dir (str base-dir "/meta")) (def binary (str base-dir "/garage")) (def logfile (str base-dir "/garage.log")) (def pidfile (str base-dir "/garage.pid")) (def admin-token "icanhazadmin") (def access-key-id "GK8bfb6a51286071c6c9cd8bc3") (def secret-access-key "b0be95f71c1c6f16858a9edf395078b75c12ecb6b1c03385c4ae92076e4994a3") (def bucket-name "jepsen") ; THE GARAGE DB (defn install! "Download and install Garage" [node version] (c/su (c/trace (info node "installing garage" version) (c/exec :mkdir :-p base-dir) (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") cache (cu/cached-wget! url)] (c/exec :cp cache binary)) (c/exec :chmod :+x binary)))) (defn configure! "Configure Garage" [node] (c/su (c/trace (cu/write-file! (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" "rpc_bind_addr = \"0.0.0.0:3901\"\n" "rpc_public_addr = \"" node ":3901\"\n" "db_engine = \"lmdb\"\n" "replication_mode = \"3\"\n" "data_dir = \"" data-dir "\"\n" "metadata_dir = \"" meta-dir "\"\n" "[s3_api]\n" "s3_region = \"us-east-1\"\n" "api_bind_addr = \"0.0.0.0:3900\"\n" "[k2v_api]\n" "api_bind_addr = \"0.0.0.0:3902\"\n" "[admin]\n" "api_bind_addr = \"0.0.0.0:3903\"\n" "admin_token = \"" admin-token "\"\n" "trace_sink = \"http://192.168.56.1:4317\"\n") "/etc/garage.toml")))) (defn connect-node! "Connect a Garage node to the rest of the cluster" [test node] (c/trace (let [node-id (c/exec binary :node :id :-q)] (info node "node id:" node-id) (c/on-many (:nodes test) (c/exec binary :node :connect node-id))))) (defn configure-node! "Configure a Garage node to be part of a cluster layout" [test node] (c/trace (let [node-id (c/exec binary :node :id :-q)] (c/on (jepsen/primary test) (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))))) (defn finalize-config! "Apply the layout and create a key/bucket pair in the cluster" [node] (c/trace (c/exec binary :layout :apply :--version 1) (info node "garage status:" (c/exec binary :status)) (c/exec binary :key :import access-key-id secret-access-key :--yes) (c/exec binary :bucket :create bucket-name) (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key-id) (info node "key info: " (c/exec binary :key :info access-key-id)))) (defn db "Garage DB for a particular version" [version] (reify db/DB (setup! [_ test node] (install! node version) (configure! node) (cu/start-daemon! {:logfile logfile :pidfile pidfile :chdir base-dir :env {:RUST_LOG "garage=debug,garage_api=trace"}} binary :server) (c/exec :sleep 3) (jepsen/synchronize test) (connect-node! test node) (jepsen/synchronize test) (configure-node! test node) (jepsen/synchronize test) (when (= node (jepsen/primary test)) (finalize-config! node))) (teardown! [_ test node] (info node "tearing down garage" version) (c/su (cu/stop-daemon! binary pidfile) (c/exec :rm :-rf logfile) (c/exec :rm :-rf data-dir) (c/exec :rm :-rf meta-dir))) db/Pause (pause! [_ test node] (cu/grepkill! :stop binary)) (resume! [_ test node] (cu/grepkill! :cont binary)) db/Kill (kill! [_ test node] (cu/stop-daemon! binary pidfile)) (start! [_ test node] (cu/start-daemon! {:logfile logfile :pidfile pidfile :chdir base-dir :env {:RUST_LOG "garage=debug,garage_api=trace"}} binary :server)) db/LogFiles (log-files [_ test node] [logfile]))) (defn creds "Obtain Garage credentials for node" [node] {:access-key access-key-id :secret-key secret-access-key :endpoint (str "http://" node ":3900") :bucket bucket-name :client-config {:path-style-access-enabled true}})