refactor jepsen setup logic

This commit is contained in:
Alex 2023-10-18 15:34:12 +02:00
parent 012ade5d4b
commit 84d43501ce
2 changed files with 96 additions and 47 deletions

View file

@ -1,8 +1,18 @@
{ pkgs ? import <nixpkgs> { } }: { pkgs ? import <nixpkgs> {
overlays = [
(self: super: {
jdk = super.jdk11;
jre = super.jre11;
})
];
} }:
pkgs.mkShell { pkgs.mkShell {
nativeBuildInputs = with pkgs; [ nativeBuildInputs = with pkgs; [
leiningen leiningen
jdk
jna
vagrant vagrant
gnuplot gnuplot
graphviz
]; ];
} }

View file

@ -1,6 +1,7 @@
(ns jepsen.garage.grg (ns jepsen.garage.grg
(:require [clojure.tools.logging :refer :all] (:require [clojure.tools.logging :refer :all]
[jepsen [control :as c] [jepsen [control :as c]
[core :as jepsen]
[db :as db]] [db :as db]]
[jepsen.control.util :as cu] [jepsen.control.util :as cu]
[amazonica.aws.s3 :as s3] [amazonica.aws.s3 :as s3]
@ -21,63 +22,101 @@
; THE GARAGE DB ; THE GARAGE DB
(defn install!
"Download and install Garage"
[node version]
(c/su
(c/trace
(info node "installing garage" version)
(c/exec :mkdir :-p dir)
(let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
cache (cu/cached-wget! url)]
(c/exec :cp cache binary))
(c/exec :chmod :+x binary))))
(defn configure!
"Configure Garage"
[node]
(c/su
(c/trace
(cu/write-file!
(str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
"rpc_bind_addr = \"0.0.0.0:3901\"\n"
"rpc_public_addr = \"" node ":3901\"\n"
"db_engine = \"lmdb\"\n"
"replication_mode = \"3\"\n"
"data_dir = \"" dir "/data\"\n"
"metadata_dir = \"" dir "/meta\"\n"
"[s3_api]\n"
"s3_region = \"us-east-1\"\n"
"api_bind_addr = \"0.0.0.0:3900\"\n"
"[k2v_api]\n"
"api_bind_addr = \"0.0.0.0:3902\"\n"
"[admin]\n"
"api_bind_addr = \"0.0.0.0:3903\"\n"
"admin_token = \"" grg-admin-token "\"\n")
"/etc/garage.toml"))))
(defn connect-node!
"Connect a Garage node to the rest of the cluster"
[test node]
(c/trace
(let [node-id (c/exec binary :node :id :-q)]
(info node "node id:" node-id)
(c/on-many (:nodes test)
(c/exec binary :node :connect node-id)))))
(defn configure-node!
"Configure a Garage node to be part of a cluster layout"
[test node]
(c/trace
(let [node-id (c/exec binary :node :id :-q)]
(c/on (jepsen/primary test)
(c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)))))
(defn finalize-config!
"Apply the layout and create a key/bucket pair in the cluster"
[node]
(c/trace
(c/exec binary :layout :apply :--version 1)
(info node "garage status:" (c/exec binary :status))
(c/exec binary :key :create grg-key)
(c/exec binary :bucket :create grg-bucket)
(c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
(info node "key info: " (c/exec binary :key :info grg-key))))
(defn db (defn db
"Garage DB for a particular version" "Garage DB for a particular version"
[version] [version]
(reify db/DB (reify db/DB
(setup! [_ test node] (setup! [_ test node]
(info node "installing garage" version) (install! node version)
(c/su (configure! node)
(c/exec :mkdir :-p dir) (cu/start-daemon!
(let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") {:logfile logfile
cache (cu/wget! url)] :pidfile pidfile
(c/exec :cp cache binary)) :chdir dir}
(c/exec :chmod :+x binary) binary
(cu/write-file! :server)
(str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" (c/exec :sleep 3)
"rpc_bind_addr = \"0.0.0.0:3901\"\n"
"rpc_public_addr = \"" node ":3901\"\n" (jepsen/synchronize test)
"db_engine = \"lmdb\"\n" (connect-node! test node)
"replication_mode = \"3\"\n"
"data_dir = \"" dir "/data\"\n" (jepsen/synchronize test)
"metadata_dir = \"" dir "/meta\"\n" (configure-node! test node)
"[s3_api]\n"
"s3_region = \"us-east-1\"\n" (jepsen/synchronize test)
"api_bind_addr = \"0.0.0.0:3900\"\n" (when (= node (jepsen/primary test))
"[k2v_api]\n" (finalize-config! node)))
"api_bind_addr = \"0.0.0.0:3902\"\n"
"[admin]\n"
"api_bind_addr = \"0.0.0.0:3903\"\n"
"admin_token = \"" grg-admin-token "\"\n")
"/etc/garage.toml")
(cu/start-daemon!
{:logfile logfile
:pidfile pidfile
:chdir dir}
binary
:server)
(info node "garage daemon started")
(c/exec :sleep 5)
(let [node-id (c/exec binary :node :id :-q)]
(info node "node id:" node-id)
(c/on-many (:nodes test)
(c/exec binary :node :connect node-id))
(c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node))
(if (= node (first (:nodes test)))
(do
(c/exec :sleep 5)
(c/exec binary :layout :apply :--version 1)
(info node "garage status:" (c/exec binary :status))
(c/exec binary :key :create grg-key)
(c/exec binary :bucket :create grg-bucket)
(c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key)
(info node "key info: " (c/exec binary :key :info grg-key))))))
(teardown! [_ test node] (teardown! [_ test node]
(info node "tearing down garage" version) (info node "tearing down garage" version)
(c/su (c/su
(cu/stop-daemon! binary pidfile) (cu/stop-daemon! binary pidfile)
(c/exec :rm :-rf data-dir) (c/exec :rm :-rf data-dir)
(c/exec :rm :-rf meta-dir))) (c/exec :rm :-rf meta-dir)))
db/LogFiles db/LogFiles
(log-files [_ test node] (log-files [_ test node]
[logfile]))) [logfile])))