refactor jepsen setup logic

script/jepsen.garage/shell.nix

@@ -1,8 +1,18 @@
-{ pkgs ? import <nixpkgs> { } }:
+{ pkgs ? import <nixpkgs> {
+  overlays = [
+    (self: super: {
+      jdk = super.jdk11;
+      jre = super.jre11;
+    })
+  ];
+} }:
 pkgs.mkShell {
   nativeBuildInputs = with pkgs; [
     leiningen
+    jdk
+    jna
     vagrant
     gnuplot
+    graphviz
   ];
 }

script/jepsen.garage/src/jepsen/garage/grg.clj

@@ -1,6 +1,7 @@
 (ns jepsen.garage.grg
   (:require [clojure.tools.logging :refer :all]
             [jepsen [control :as c]
+             [core :as jepsen]
              [db :as db]]
             [jepsen.control.util :as cu]
             [amazonica.aws.s3 :as s3]
@@ -21,18 +22,23 @@
 
 ; THE GARAGE DB
 
-(defn db
+(defn install!
-  "Garage DB for a particular version"
+  "Download and install Garage"
-  [version]
+  [node version]
-  (reify db/DB
-    (setup! [_ test node]
-      (info node "installing garage" version)
   (c/su
+    (c/trace
+      (info node "installing garage" version)
       (c/exec :mkdir :-p dir)
       (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
-              cache (cu/wget! url)]
+            cache (cu/cached-wget! url)]
         (c/exec :cp cache binary))
-        (c/exec :chmod :+x binary)
+      (c/exec :chmod :+x binary))))
+
+(defn configure!
+  "Configure Garage"
+  [node]
+  (c/su
+    (c/trace
       (cu/write-file!
         (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
              "rpc_bind_addr = \"0.0.0.0:3901\"\n"
@@ -49,35 +55,68 @@
              "[admin]\n"
              "api_bind_addr = \"0.0.0.0:3903\"\n"
              "admin_token = \"" grg-admin-token "\"\n")
-          "/etc/garage.toml")
+        "/etc/garage.toml"))))
+
+(defn connect-node!
+  "Connect a Garage node to the rest of the cluster"
+  [test node]
+  (c/trace
+    (let [node-id (c/exec binary :node :id :-q)]
+      (info node "node id:" node-id)
+      (c/on-many (:nodes test)
+                 (c/exec binary :node :connect node-id)))))
+
+(defn configure-node!
+  "Configure a Garage node to be part of a cluster layout"
+  [test node]
+  (c/trace
+    (let [node-id (c/exec binary :node :id :-q)]
+      (c/on (jepsen/primary test)
+          (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)))))
+
+(defn finalize-config!
+  "Apply the layout and create a key/bucket pair in the cluster"
+  [node]
+  (c/trace
+    (c/exec binary :layout :apply :--version 1)
+    (info node "garage status:" (c/exec binary :status))
+    (c/exec binary :key :create grg-key)
+    (c/exec binary :bucket :create grg-bucket)
+    (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
+    (info node "key info: " (c/exec binary :key :info grg-key))))
+
+(defn db
+  "Garage DB for a particular version"
+  [version]
+  (reify db/DB
+    (setup! [_ test node]
+      (install! node version)
+      (configure! node)
       (cu/start-daemon!
         {:logfile logfile
          :pidfile pidfile
          :chdir dir}
         binary
         :server)
-        (info node "garage daemon started")
+      (c/exec :sleep 3)
-        (c/exec :sleep 5)
+
-        (let [node-id (c/exec binary :node :id :-q)]
+      (jepsen/synchronize test)
-          (info node "node id:" node-id)
+      (connect-node! test node)
-          (c/on-many (:nodes test)
+
-                     (c/exec binary :node :connect node-id))
+      (jepsen/synchronize test)
-          (c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))
+      (configure-node! test node)
-        (if (= node (first (:nodes test)))
+
-          (do
+      (jepsen/synchronize test)
-            (c/exec :sleep 5)
+      (when (= node (jepsen/primary test))
-            (c/exec binary :layout :apply :--version 1)
+        (finalize-config! node)))
-            (info node "garage status:" (c/exec binary :status))
+
-            (c/exec binary :key :create grg-key)
-            (c/exec binary :bucket :create grg-bucket)
-            (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
-            (info node "key info: " (c/exec binary :key :info grg-key))))))
     (teardown! [_ test node]
       (info node "tearing down garage" version)
       (c/su
         (cu/stop-daemon! binary pidfile)
         (c/exec :rm :-rf data-dir)
         (c/exec :rm :-rf meta-dir)))
+
     db/LogFiles
     (log-files [_ test node]
       [logfile])))