Deuxfleurs/garage
44
56
Fork 73
Code Issues 100 Pull requests 14 Releases 30 Wiki Activity

Jepsen testing (NLnet task 3 subtask 1) #544

Merged
lx merged 41 commits from jepsen into main 2024-01-11 10:52:13 +00:00
Conversation 0 Commits 41 Files changed 22 +138 -111
6 changed files with 138 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit db921cc05f - Show all commits

2
script/jepsen.garage/README.md
View file

@ -97,6 +97,8 @@ Results:
- For now, no failures with clock-scramble nemesis + partition nemesis -> TODO long test run - For now, no failures with clock-scramble nemesis + partition nemesis -> TODO long test run
- Does not seem to fail with partition + layout reconfiguration nemesis (>100 runs)
- Does not seem to fail with the clock scrambler + partition + layout reconfiguation nemesis (>10 runs), although theoretically it could - Does not seem to fail with the clock scrambler + partition + layout reconfiguation nemesis (>10 runs), although theoretically it could
TODO: make it fail!!! TODO: make it fail!!!

14
script/jepsen.garage/src/jepsen/garage.clj
View file

@ -27,7 +27,8 @@
"cp" grgNemesis/scenario-cp "cp" grgNemesis/scenario-cp
"r" grgNemesis/scenario-r "r" grgNemesis/scenario-r
"pr" grgNemesis/scenario-pr "pr" grgNemesis/scenario-pr
"cpr" grgNemesis/scenario-cpr}) "cpr" grgNemesis/scenario-cpr
"dpr" grgNemesis/scenario-dpr})
(def patches (def patches
"A map of patch names to Garage builds" "A map of patch names to Garage builds"
@ -59,15 +60,16 @@
"Given an options map from the command line runner (e.g. :nodes, :ssh, "Given an options map from the command line runner (e.g. :nodes, :ssh,
:concurrency, ...), constructs a test map." :concurrency, ...), constructs a test map."
[opts] [opts]
(let [workload ((get workloads (:workload opts)) opts) (let [garage-version (get patches (:patch opts))
scenario ((get scenari (:scenario opts)) opts) db (grg/db garage-version)
garage-version (get patches (:patch opts))] workload ((get workloads (:workload opts)) opts)
scenario ((get scenari (:scenario opts)) (assoc opts :db db))]
(merge tests/noop-test (merge tests/noop-test
opts opts
{:pure-generators true {:pure-generators true
:name (str "garage " (name (:workload opts))) :name (str "garage " (name (:workload opts)))
:os debian/os :os debian/os
:db (grg/db garage-version) :db db
:client (:client workload) :client (:client workload)
:generator (gen/phases :generator (gen/phases
(->> (->>
@ -82,7 +84,7 @@
(gen/clients (:final-generator workload))) (gen/clients (:final-generator workload)))
:nemesis (:nemesis scenario) :nemesis (:nemesis scenario)
:checker (checker/compose :checker (checker/compose
{:perf (checker/perf) {:perf (checker/perf (:perf scenario))
:workload (:checker workload)}) :workload (:checker workload)})
}))) })))

18
script/jepsen.garage/src/jepsen/garage/daemon.clj
View file

@ -119,6 +119,24 @@
(c/exec :rm :-rf data-dir) (c/exec :rm :-rf data-dir)
(c/exec :rm :-rf meta-dir))) (c/exec :rm :-rf meta-dir)))
db/Pause
(pause! [_ test node]
(cu/grepkill! :stop binary))
(resume! [_ test node]
(cu/grepkill! :cont binary))
db/Kill
(kill! [_ test node]
(cu/stop-daemon! binary pidfile))
(start! [_ test node]
(cu/start-daemon!
{:logfile logfile
:pidfile pidfile
:chdir base-dir
:env {:RUST_LOG "garage=debug,garage_api=trace"}}
binary
:server))
db/LogFiles db/LogFiles
(log-files [_ test node] (log-files [_ test node]
[logfile]))) [logfile])))

129
script/jepsen.garage/src/jepsen/garage/nemesis.clj
View file

@ -4,6 +4,7 @@
[core :as jepsen] [core :as jepsen]
[generator :as gen] [generator :as gen]
[nemesis :as nemesis]] [nemesis :as nemesis]]
[jepsen.nemesis.combined :as combined]
[jepsen.garage.daemon :as grg] [jepsen.garage.daemon :as grg]
[jepsen.control.util :as cu])) [jepsen.control.util :as cu]))
@ -11,21 +12,23 @@
(defn configure-present! (defn configure-present!
"Configure node to be active in new cluster layout" "Configure node to be active in new cluster layout"
[test node] [test nodes]
(info "configure-present!" node) (info "configure-present!" nodes)
(let [node-id (c/on node (c/exec grg/binary :node :id :-q))] (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q))
(c/on node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)]
(jepsen/primary test) (c/on
(c/exec grg/binary :layout :assign (subs node-id 0 16) :-c :1G)))) (jepsen/primary test)
(apply c/exec (concat [grg/binary :layout :assign :-c :1G] node-id-strs)))))
(defn configure-absent! (defn configure-absent!
"Configure node to be active in new cluster layout" "Configure nodes to be active in new cluster layout"
[test node] [test nodes]
(info "configure-absent!" node) (info "configure-absent!" nodes)
(let [node-id (c/on node (c/exec grg/binary :node :id :-q))] (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q))
(c/on node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)]
(jepsen/primary test) (c/on
(c/exec grg/binary :layout :assign (subs node-id 0 16) :-g)))) (jepsen/primary test)
(apply c/exec (concat [grg/binary :layout :assign :-g] node-id-strs)))))
(defn finalize-config! (defn finalize-config!
"Apply the proposed cluster layout" "Apply the proposed cluster layout"
@ -53,14 +56,14 @@
shuffle shuffle
(split-at cnt))] (split-at cnt))]
(info "layout split: keep " keep-nodes ", remove " remove-nodes) (info "layout split: keep " keep-nodes ", remove " remove-nodes)
(run! #(configure-present! test %) keep-nodes) (configure-present! test keep-nodes)
(run! #(configure-absent! test %) remove-nodes) (configure-absent! test remove-nodes)
(finalize-config! test) (finalize-config! test)
(assoc op :value keep-nodes)) (assoc op :value keep-nodes))
:stop :stop
(do (do
(info "layout un-split: all nodes=" (:nodes test)) (info "layout un-split: all nodes=" (:nodes test))
(run! #(configure-present! test %) (:nodes test)) (configure-present! test (:nodes test))
(finalize-config! test) (finalize-config! test)
(assoc op :value (:nodes test))))) (assoc op :value (:nodes test)))))
@ -73,70 +76,58 @@
[op] [op]
(fn [_ _] {:type :info, :f op})) (fn [_ _] {:type :info, :f op}))
(defn scenario-c (defn reconfiguration-package
"Clock scramble scenario" "Cluster reconfiguration nemesis package"
[opts]
{:generator (->>
(nemesis-op :clock-scramble)
(gen/stagger 5))
:nemesis (nemesis/compose
{{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})})
(defn scenario-cp
"Clock scramble + partition scenario"
[opts]
{:generator (->>
(gen/mix [(nemesis-op :clock-scramble)
(nemesis-op :partition-stop)
(nemesis-op :partition-start)])
(gen/stagger 5))
:final-generator (gen/once {:type :info, :f :partition-stop})
:nemesis (nemesis/compose
{{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)
{:partition-start :start
:partition-stop :stop} (nemesis/partition-random-halves)})})
(defn scenario-r
"Cluster reconfiguration scenario"
[opts] [opts]
{:generator (->> {:generator (->>
(gen/mix [(nemesis-op :reconfigure-start) (gen/mix [(nemesis-op :reconfigure-start)
(nemesis-op :reconfigure-stop)]) (nemesis-op :reconfigure-stop)])
(gen/stagger 5)) (gen/stagger (:interval opts 5)))
:final-generator {:type :info, :f :reconfigure-stop}
:nemesis (nemesis/compose :nemesis (nemesis/compose
{{:reconfigure-start :start {{:reconfigure-start :start
:reconfigure-stop :stop} (reconfigure-subset 3)})}) :reconfigure-stop :stop} (reconfigure-subset 3)})
:perf #{{:name "reconfigure"
:start #{:reconfigure-start}
:stop #{:reconfigur-stop}
:color "#A197E9"}}})
(defn scenario-c
"Clock modifying scenario"
[opts]
(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}))
(defn scenario-cp
"Clock modifying + partition scenario"
[opts]
(combined/compose-packages
[(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})]))
(defn scenario-r
"Cluster reconfiguration scenario"
[opts]
(reconfiguration-package {:interval 1}))
(defn scenario-pr (defn scenario-pr
"Partition + cluster reconfiguration scenario" "Partition + cluster reconfiguration scenario"
[opts] [opts]
{:generator (->> (combined/compose-packages
(gen/mix [(nemesis-op :partition-start) [(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
(nemesis-op :partition-stop) (reconfiguration-package {:interval 1})]))
(nemesis-op :reconfigure-start)
(nemesis-op :reconfigure-stop)])
(gen/stagger 5))
:final-generator (gen/once {:type :info, :f :partition-stop})
:nemesis (nemesis/compose
{{:partition-start :start
:partition-stop :stop} (nemesis/partition-random-halves)
{:reconfigure-start :start
:reconfigure-stop :stop} (reconfigure-subset 3)})})
(defn scenario-cpr (defn scenario-cpr
"Clock scramble + partition + cluster reconfiguration scenario" "Clock scramble + partition + cluster reconfiguration scenario"
[opts] [opts]
{:generator (->> (combined/compose-packages
(gen/mix [(nemesis-op :clock-scramble) [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
(nemesis-op :partition-start) (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
(nemesis-op :partition-stop) (reconfiguration-package {:interval 1})]))
(nemesis-op :reconfigure-start)
(nemesis-op :reconfigure-stop)]) (defn scenario-dpr
(gen/stagger 5)) "Db + partition + cluster reconfiguration scenario"
:final-generator (gen/once {:type :info, :f :partition-stop}) [opts]
:nemesis (nemesis/compose (combined/compose-packages
{{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0) [(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}})
{:partition-start :start (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
:partition-stop :stop} (nemesis/partition-random-halves) (reconfiguration-package {:interval 1})]))
{:reconfigure-start :start
:reconfigure-stop :stop} (reconfigure-subset 3)})})

37
script/jepsen.garage/src/jepsen/garage/reg.clj
View file

@ -30,21 +30,28 @@
(assoc this :creds (grg/creds node))) (assoc this :creds (grg/creds node)))
(setup! [this test]) (setup! [this test])
(invoke! [this test op] (invoke! [this test op]
(let [[k v] (:value op)] (try+
(case (:f op) (let [[k v] (:value op)]
:read (case (:f op)
(util/timeout :read
10000 (util/timeout
(assoc op :type :fail, :error ::timeout) 10000
(let [value (s3/get (:creds this) k)] (assoc op :type :fail, :error ::timeout)
(assoc op :type :ok, :value (independent/tuple k value)))) (let [value (s3/get (:creds this) k)]
:write (assoc op :type :ok, :value (independent/tuple k value))))
(util/timeout :write
10000 (util/timeout
(assoc op :type :info, :error ::timeout) 10000
(do (assoc op :type :info, :error ::timeout)
(s3/put (:creds this) k v) (do
(assoc op :type :ok)))))) (s3/put (:creds this) k v)
(assoc op :type :ok)))))
(catch (re-find #"Unavailable" (.getMessage %)) ex
(assoc op :type :info, :error ::unavailable))
(catch (re-find #"Broken pipe" (.getMessage %)) ex
(assoc op :type :info, :error ::broken-pipe))
(catch (re-find #"Connection refused" (.getMessage %)) ex
(assoc op :type :info, :error ::connection-refused))))
(teardown! [this test]) (teardown! [this test])
(close! [this test])) (close! [this test]))

49
script/jepsen.garage/src/jepsen/garage/set.clj
View file

@ -30,27 +30,34 @@
(assoc this :creds (grg/creds node))) (assoc this :creds (grg/creds node)))
(setup! [this test]) (setup! [this test])
(invoke! [this test op] (invoke! [this test op]
(let [[k v] (:value op) (try+
prefix (str "set" k "/")] (let [[k v] (:value op)
(case (:f op) prefix (str "set" k "/")]
:add (case (:f op)
(util/timeout :add
10000 (util/timeout
(assoc op :type :info, :error ::timeout) 10000
(do (assoc op :type :info, :error ::timeout)
(s3/put (:creds this) (str prefix v) "present") (do
(assoc op :type :ok))) (s3/put (:creds this) (str prefix v) "present")
:read (assoc op :type :ok)))
(util/timeout :read
10000 (util/timeout
(assoc op :type :fail, :error ::timeout) 10000
(do (assoc op :type :fail, :error ::timeout)
(let [items (s3/list (:creds this) prefix)] (do
(let [items-stripped (map (fn [o] (let [items (s3/list (:creds this) prefix)]
(assert (str/starts-with? o prefix)) (let [items-stripped (map (fn [o]
(str/replace-first o prefix "")) items) (assert (str/starts-with? o prefix))
items-set (set (map parse-long items-stripped))] (str/replace-first o prefix "")) items)
(assoc op :type :ok, :value (independent/tuple k items-set))))))))) items-set (set (map parse-long items-stripped))]
(assoc op :type :ok, :value (independent/tuple k items-set))))))))
(catch (re-find #"Unavailable" (.getMessage %)) ex
(assoc op :type :info, :error ::unavailable))
(catch (re-find #"Broken pipe" (.getMessage %)) ex
(assoc op :type :info, :error ::broken-pipe))
(catch (re-find #"Connection refused" (.getMessage %)) ex
(assoc op :type :info, :error ::connection-refused))))
(teardown! [this test]) (teardown! [this test])
(close! [this test])) (close! [this test]))