(ns jepsen.garage (:require [clojure.tools.logging :refer :all] [clojure.string :as str] [jepsen [checker :as checker] [cli :as cli] [client :as client] [control :as c] [db :as db] [generator :as gen] [nemesis :as nemesis] [tests :as tests]] [jepsen.checker.timeline :as timeline] [jepsen.control.util :as cu] [jepsen.os.debian :as debian] [knossos.model :as model] [slingshot.slingshot :refer [try+]] [amazonica.aws.s3 :as s3] [amazonica.aws.s3transfer :as s3transfer])) (def dir "/opt/garage") (def binary (str dir "/garage")) (def logfile (str dir "/garage.log")) (def pidfile (str dir "/garage.pid")) (def grg-admin-token "icanhazadmin") (def grg-key "jepsen") (def grg-bucket "jepsen") (def grg-object "1") (defn garage "Garage DB for a particular version" [version] (reify db/DB (setup! [_ test node] (info node "installing garage" version) (c/su (c/exec :mkdir :-p dir) (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") cache (cu/wget! url)] (c/exec :cp cache binary)) (c/exec :chmod :+x binary) (cu/write-file! (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" "rpc_bind_addr = \"0.0.0.0:3901\"\n" "rpc_public_addr = \"" node ":3901\"\n" "db_engine = \"lmdb\"\n" "replication_mode = \"3\"\n" "data_dir = \"" dir "/data\"\n" "metadata_dir = \"" dir "/meta\"\n" "[s3_api]\n" "s3_region = \"us-east-1\"\n" "api_bind_addr = \"0.0.0.0:3900\"\n" "[k2v_api]\n" "api_bind_addr = \"0.0.0.0:3902\"\n" "[admin]\n" "api_bind_addr = \"0.0.0.0:3903\"\n" "admin_token = \"" grg-admin-token "\"\n") "/etc/garage.toml") (cu/start-daemon! {:logfile logfile :pidfile pidfile :chdir dir} binary :server) (Thread/sleep 100) (let [node-id (c/exec binary :node :id :-q)] (info node "node id:" node-id) (c/on-many (:nodes test) (c/exec binary :node :connect node-id)) (c/exec binary :layout :assign (subs node-id 0 16) :-c 1 :-z :dc1 :-t node)) (if (= node (first (:nodes test))) (do (Thread/sleep 2000) (c/exec binary :layout :apply :--version 1) (info node "garage status:" (c/exec binary :status)) (c/exec binary :key :new :--name grg-key) (c/exec binary :bucket :create grg-bucket) (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key) (info node "key info: " (c/exec binary :key :info grg-key)))))) (teardown! [_ test node] (info node "tearing down garage" version) (c/su (cu/stop-daemon! binary pidfile) (c/exec :rm :-rf dir))) db/LogFiles (log-files [_ test node] [logfile]))) (defn op-get [_ _] {:type :invoke, :f :read, :value nil}) (defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 9))}) (defn op-del [_ _] {:type :invoke, :f :write, :value nil}) (defrecord Client [creds] client/Client (open! [this test node] (let [key-info (c/on node (c/exec binary :key :info grg-key)) [_ ak sk] (re-matches #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*" key-info) creds {:access-key ak :secret-key sk :endpoint (str "http://" node ":3900") :client-config {:path-style-access-enabled true}}] (info node "s3 credentials:" creds) (assoc this :creds creds))) (setup! [this test]) (invoke! [this test op] (case (:f op) :read (try+ (let [value (-> (s3/get-object (:creds this) grg-bucket grg-object) :input-stream slurp)] (assoc op :type :ok, :value value)) (catch (re-find #"Key not found" (.getMessage %)) ex (assoc op :type :ok, :value nil))) :write (if (= (:value op) nil) (do (s3/delete-object (:creds this) :bucket-name grg-bucket :key grg-object) (assoc op :type :ok, :value nil)) (let [some-bytes (.getBytes (:value op) "UTF-8") bytes-stream (java.io.ByteArrayInputStream. some-bytes)] (s3/put-object (:creds this) :bucket-name grg-bucket :key grg-object :input-stream bytes-stream :metadata {:content-length (count some-bytes)}) (assoc op :type :ok))))) (teardown! [this test]) (close! [this test])) (defn garage-test "Given an options map from the command line runner (e.g. :nodes, :ssh, :concurrency, ...), constructs a test map." [opts] (merge tests/noop-test opts {:pure-generators true :name "garage" :os debian/os :db (garage "v0.8.2") ; :db (garage "d39c5c6984c581e16932aaa07e3687e7b5ce266d") ; fixed for increasing timestamps :client (Client. nil) :nemesis (nemesis/partition-random-halves) :checker (checker/compose {:perf (checker/perf) :timeline (timeline/html) :linear (checker/linearizable {:model (model/register) :algorithm :linear})}) :generator (->> (gen/mix [op-get op-put op-del]) (gen/stagger 0.02) (gen/nemesis nil) ; (gen/nemesis ; (cycle [(gen/sleep 5) ; {:type :info, :f :start} ; (gen/sleep 5) ; {:type :info, :f :stop}])) (gen/time-limit (+ (:time-limit opts) 5)))})) (defn -main "Handles command line arguments. Can either run a test, or a web server for browsing results." [& args] (cli/run! (merge (cli/single-test-cmd {:test-fn garage-test}) (cli/serve-cmd)) args))