diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj index 754ddf7d..5816512b 100644 --- a/script/jepsen.garage/src/jepsen/garage.clj +++ b/script/jepsen.garage/src/jepsen/garage.clj @@ -9,7 +9,7 @@ [tests :as tests]] [jepsen.os.debian :as debian] [jepsen.garage - [grg :as grg] + [daemon :as grg] [reg :as reg] [set :as set]])) diff --git a/script/jepsen.garage/src/jepsen/garage/grg.clj b/script/jepsen.garage/src/jepsen/garage/daemon.clj similarity index 55% rename from script/jepsen.garage/src/jepsen/garage/grg.clj rename to script/jepsen.garage/src/jepsen/garage/daemon.clj index f6075762..81163521 100644 --- a/script/jepsen.garage/src/jepsen/garage/grg.clj +++ b/script/jepsen.garage/src/jepsen/garage/daemon.clj @@ -1,24 +1,22 @@ -(ns jepsen.garage.grg +(ns jepsen.garage.daemon (:require [clojure.tools.logging :refer :all] [jepsen [control :as c] [core :as jepsen] [db :as db]] - [jepsen.control.util :as cu] - [amazonica.aws.s3 :as s3] - [slingshot.slingshot :refer [try+]])) + [jepsen.control.util :as cu])) ; CONSTANTS -- HOW GARAGE IS SET UP -(def dir "/opt/garage") -(def data-dir (str dir "/data")) -(def meta-dir (str dir "/meta")) -(def binary (str dir "/garage")) -(def logfile (str dir "/garage.log")) -(def pidfile (str dir "/garage.pid")) +(def base-dir "/opt/garage") +(def data-dir (str base-dir "/data")) +(def meta-dir (str base-dir "/meta")) +(def binary (str base-dir "/garage")) +(def logfile (str base-dir "/garage.log")) +(def pidfile (str base-dir "/garage.pid")) -(def grg-admin-token "icanhazadmin") -(def grg-key "jepsen") -(def grg-bucket "jepsen") +(def admin-token "icanhazadmin") +(def access-key "jepsen") +(def bucket-name "jepsen") ; THE GARAGE DB @@ -28,7 +26,7 @@ (c/su (c/trace (info node "installing garage" version) - (c/exec :mkdir :-p dir) + (c/exec :mkdir :-p base-dir) (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") cache (cu/cached-wget! url)] (c/exec :cp cache binary)) @@ -45,8 +43,8 @@ "rpc_public_addr = \"" node ":3901\"\n" "db_engine = \"lmdb\"\n" "replication_mode = \"3\"\n" - "data_dir = \"" dir "/data\"\n" - "metadata_dir = \"" dir "/meta\"\n" + "data_dir = \"" data-dir "\"\n" + "metadata_dir = \"" meta-dir "\"\n" "[s3_api]\n" "s3_region = \"us-east-1\"\n" "api_bind_addr = \"0.0.0.0:3900\"\n" @@ -54,7 +52,7 @@ "api_bind_addr = \"0.0.0.0:3902\"\n" "[admin]\n" "api_bind_addr = \"0.0.0.0:3903\"\n" - "admin_token = \"" grg-admin-token "\"\n") + "admin_token = \"" admin-token "\"\n") "/etc/garage.toml")))) (defn connect-node! @@ -80,10 +78,10 @@ (c/trace (c/exec binary :layout :apply :--version 1) (info node "garage status:" (c/exec binary :status)) - (c/exec binary :key :create grg-key) - (c/exec binary :bucket :create grg-bucket) - (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key) - (info node "key info: " (c/exec binary :key :info grg-key)))) + (c/exec binary :key :create access-key) + (c/exec binary :bucket :create bucket-name) + (c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key) + (info node "key info: " (c/exec binary :key :info access-key)))) (defn db "Garage DB for a particular version" @@ -95,7 +93,7 @@ (cu/start-daemon! {:logfile logfile :pidfile pidfile - :chdir dir} + :chdir base-dir} binary :server) (c/exec :sleep 3) @@ -121,58 +119,16 @@ (log-files [_ test node] [logfile]))) -; GARAGE S3 HELPER FUNCTIONS - -(defn s3-creds - "Get S3 credentials for node" +(defn creds + "Obtain Garage credentials for node" [node] - (let [key-info (c/on node (c/exec binary :key :info grg-key :--show-secret)) + (let [key-info (c/on node (c/exec binary :key :info access-key :--show-secret)) [_ ak sk] (re-matches #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*" key-info)] {:access-key ak :secret-key sk :endpoint (str "http://" node ":3900") - :bucket grg-bucket + :bucket bucket-name :client-config {:path-style-access-enabled true}})) -(defn s3-get - "Helper for GetObject" - [creds k] - (try+ - (-> (s3/get-object creds (:bucket creds) k) - :input-stream - slurp) - (catch (re-find #"Key not found" (.getMessage %)) ex - nil))) - -(defn s3-put - "Helper for PutObject or DeleteObject (is a delete if value is nil)" - [creds k v] - (if (= v nil) - (s3/delete-object creds - :bucket-name (:bucket creds) - :key k) - (let [some-bytes (.getBytes v "UTF-8") - bytes-stream (java.io.ByteArrayInputStream. some-bytes)] - (s3/put-object creds - :bucket-name (:bucket creds) - :key k - :input-stream bytes-stream - :metadata {:content-length (count some-bytes)})))) - -(defn s3-list - "Helper for ListObjects -- just lists everything in the bucket" - [creds prefix] - (defn list-inner [ct accum] - (let [list-result (s3/list-objects-v2 creds - {:bucket-name (:bucket creds) - :prefix prefix - :continuation-token ct}) - new-object-summaries (:object-summaries list-result) - new-objects (map (fn [d] (:key d)) new-object-summaries) - objects (concat new-objects accum)] - (if (:truncated? list-result) - (list-inner (:next-continuation-token list-result) objects) - objects))) - (list-inner nil [])) diff --git a/script/jepsen.garage/src/jepsen/garage/reg.clj b/script/jepsen.garage/src/jepsen/garage/reg.clj index 0b370b36..3a2c1d55 100644 --- a/script/jepsen.garage/src/jepsen/garage/reg.clj +++ b/script/jepsen.garage/src/jepsen/garage/reg.clj @@ -13,7 +13,8 @@ [jepsen.checker.timeline :as timeline] [jepsen.control.util :as cu] [jepsen.os.debian :as debian] - [jepsen.garage.grg :as grg] + [jepsen.garage.daemon :as grg] + [jepsen.garage.s3api :as s3] [knossos.model :as model] [slingshot.slingshot :refer [try+]])) @@ -24,7 +25,7 @@ (defrecord RegClient [creds] client/Client (open! [this test node] - (let [creds (grg/s3-creds node)] + (let [creds (grg/creds node)] (info node "s3 credentials:" creds) (assoc this :creds creds))) (setup! [this test]) @@ -32,11 +33,11 @@ (let [[k v] (:value op)] (case (:f op) :read - (let [value (grg/s3-get (:creds this) k)] + (let [value (s3/get (:creds this) k)] (assoc op :type :ok, :value (independent/tuple k value))) :write (do - (grg/s3-put (:creds this) k v) + (s3/put (:creds this) k v) (assoc op :type :ok))))) (teardown! [this test]) (close! [this test])) diff --git a/script/jepsen.garage/src/jepsen/garage/s3api.clj b/script/jepsen.garage/src/jepsen/garage/s3api.clj new file mode 100644 index 00000000..239d5423 --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/s3api.clj @@ -0,0 +1,48 @@ +(ns jepsen.garage.s3api + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c]] + [amazonica.aws.s3 :as s3] + [slingshot.slingshot :refer [try+]])) + +; GARAGE S3 HELPER FUNCTIONS + +(defn get + "Helper for GetObject" + [creds k] + (try+ + (-> (s3/get-object creds (:bucket creds) k) + :input-stream + slurp) + (catch (re-find #"Key not found" (.getMessage %)) ex + nil))) + +(defn put + "Helper for PutObject or DeleteObject (is a delete if value is nil)" + [creds k v] + (if (= v nil) + (s3/delete-object creds + :bucket-name (:bucket creds) + :key k) + (let [some-bytes (.getBytes v "UTF-8") + bytes-stream (java.io.ByteArrayInputStream. some-bytes)] + (s3/put-object creds + :bucket-name (:bucket creds) + :key k + :input-stream bytes-stream + :metadata {:content-length (count some-bytes)})))) + +(defn list + "Helper for ListObjects -- just lists everything in the bucket" + [creds prefix] + (defn list-inner [ct accum] + (let [list-result (s3/list-objects-v2 creds + {:bucket-name (:bucket creds) + :prefix prefix + :continuation-token ct}) + new-object-summaries (:object-summaries list-result) + new-objects (map (fn [d] (:key d)) new-object-summaries) + objects (concat new-objects accum)] + (if (:truncated? list-result) + (list-inner (:next-continuation-token list-result) objects) + objects))) + (list-inner nil [])) diff --git a/script/jepsen.garage/src/jepsen/garage/set.clj b/script/jepsen.garage/src/jepsen/garage/set.clj index 5f76d1ac..6bbc1ee0 100644 --- a/script/jepsen.garage/src/jepsen/garage/set.clj +++ b/script/jepsen.garage/src/jepsen/garage/set.clj @@ -15,7 +15,8 @@ [jepsen.checker.timeline :as timeline] [jepsen.control.util :as cu] [jepsen.os.debian :as debian] - [jepsen.garage.grg :as grg] + [jepsen.garage.daemon :as grg] + [jepsen.garage.s3api :as s3] [knossos.model :as model] [slingshot.slingshot :refer [try+]])) @@ -25,7 +26,7 @@ (defrecord SetClient [creds] client/Client (open! [this test node] - (let [creds (grg/s3-creds node)] + (let [creds (grg/creds node)] (info node "s3 credentials:" creds) (assoc this :creds creds))) (setup! [this test]) @@ -35,10 +36,10 @@ (case (:f op) :add (do - (grg/s3-put (:creds this) (str prefix v) "present") + (s3/put (:creds this) (str prefix v) "present") (assoc op :type :ok)) :read - (let [items (grg/s3-list (:creds this) prefix) + (let [items (s3/list (:creds this) prefix) items-stripped (map (fn [o] (str/replace-first o prefix "")) items) items-set (set (map read-string items-stripped))] (assoc op :type :ok, :value (independent/tuple k items-set))))))