diff --git a/script/jepsen.garage/Vagrantfile b/script/jepsen.garage/Vagrantfile index d0e545d3..1125bccf 100644 --- a/script/jepsen.garage/Vagrantfile +++ b/script/jepsen.garage/Vagrantfile @@ -27,4 +27,5 @@ Vagrant.configure("2") do |config| config.vm.define "n3" do |config| vm(config, "n3", "192.168.56.23") end config.vm.define "n4" do |config| vm(config, "n4", "192.168.56.24") end config.vm.define "n5" do |config| vm(config, "n5", "192.168.56.25") end + config.vm.define "n6" do |config| vm(config, "n6", "192.168.56.26") end end diff --git a/script/jepsen.garage/nodes.vagrant b/script/jepsen.garage/nodes.vagrant index df7c4622..3f7e2b42 100644 --- a/script/jepsen.garage/nodes.vagrant +++ b/script/jepsen.garage/nodes.vagrant @@ -3,3 +3,4 @@ 192.168.56.23 192.168.56.24 192.168.56.25 +192.168.56.26 diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj index a566d9be..65a92a76 100644 --- a/script/jepsen.garage/src/jepsen/garage.clj +++ b/script/jepsen.garage/src/jepsen/garage.clj @@ -10,6 +10,7 @@ [jepsen.os.debian :as debian] [jepsen.garage [daemon :as grg] + [nemesis :as grgNemesis] [reg :as reg] [set :as set]])) @@ -20,6 +21,11 @@ "set1" set/workload1 "set2" set/workload2}) +(def scenari + "A map of scenari to the associated nemesis" + {"cp" grgNemesis/scenario-cp + "r" grgNemesis/scenario-r}) + (def patches "A map of patch names to Garage builds" {"default" "v0.9.0" @@ -31,6 +37,9 @@ [["-p" "--patch NAME" "Garage patch to use" :default "default" :validate [patches (cli/one-of patches)]] + ["-s" "--scenario NAME" "Nemesis scenario to run" + :default "cp" + :validate [scenari (cli/one-of scenari)]] ["-r" "--rate HZ" "Approximate number of requests per second, per thread." :default 10 :parse-fn read-string @@ -40,7 +49,7 @@ :parse-fn parse-long :validate [pos? "Must be a positive integer."]] ["-w" "--workload NAME" "Workload of test to run" - :default "reg" + :default "reg1" :validate [workloads (cli/one-of workloads)]]]) (defn garage-test @@ -48,6 +57,7 @@ :concurrency, ...), constructs a test map." [opts] (let [workload ((get workloads (:workload opts)) opts) + scenario ((get scenari (:scenario opts)) opts) garage-version (get patches (:patch opts))] (merge tests/noop-test opts @@ -60,25 +70,14 @@ (->> (:generator workload) (gen/stagger (/ (:rate opts))) - (gen/nemesis - (cycle [(gen/sleep 5) - {:type :info, :f :partition-start} - (gen/sleep 5) - {:type :info, :f :clock-scramble} - (gen/sleep 5) - {:type :info, :f :partition-stop} - (gen/sleep 5) - {:type :info, :f :clock-scramble}])) + (gen/nemesis (:generator scenario)) (gen/time-limit (:time-limit opts))) (gen/log "Healing cluster") (gen/nemesis (gen/once {:type :info, :f :partition-stop})) (gen/log "Waiting for recovery") (gen/sleep 10) (gen/clients (:final-generator workload))) - :nemesis (nemesis/compose - {{:partition-start :start - :partition-stop :stop} (nemesis/partition-random-halves) - {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)}) + :nemesis (:nemesis scenario) :checker (checker/compose {:perf (checker/perf) :workload (:checker workload)}) diff --git a/script/jepsen.garage/src/jepsen/garage/nemesis.clj b/script/jepsen.garage/src/jepsen/garage/nemesis.clj new file mode 100644 index 00000000..7cd9306e --- /dev/null +++ b/script/jepsen.garage/src/jepsen/garage/nemesis.clj @@ -0,0 +1,91 @@ +(ns jepsen.garage.nemesis + (:require [clojure.tools.logging :refer :all] + [jepsen [control :as c] + [core :as jepsen] + [generator :as gen] + [nemesis :as nemesis]] + [jepsen.garage.daemon :as grg] + [jepsen.control.util :as cu])) + +(defn configure-present! + "Configure node to be active in new cluster layout" + [test node] + (info "configure-present!" node) + (let [node-id (c/on node (c/exec grg/binary :node :id :-q))] + (c/on + (jepsen/primary test) + (c/exec grg/binary :layout :assign (subs node-id 0 16) :-c :1G)))) + +(defn configure-absent! + "Configure node to be active in new cluster layout" + [test node] + (info "configure-absent!" node) + (let [node-id (c/on node (c/exec grg/binary :node :id :-q))] + (c/on + (jepsen/primary test) + (c/exec grg/binary :layout :assign (subs node-id 0 16) :-g)))) + +(defn finalize-config! + "Apply the proposed cluster layout" + [test] + (let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show)) + [_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)] + (info "layout show: " layout-show "; next-version: " layout-next-version) + (c/on (jepsen/primary test) + (c/exec grg/binary :layout :apply :--version layout-next-version)))) + +(defn reconfigure-subset + "Reconfigure cluster with only a subset of nodes" + [cnt] + (reify nemesis/Nemesis + (setup! [this test] this) + + (invoke! [this test op] op + (case (:f op) + :start + (let [[keep-nodes remove-nodes] + (->> (:nodes test) + shuffle + (split-at cnt))] + (info "layout split: keep " keep-nodes ", remove " remove-nodes) + (run! #(configure-present! test %) keep-nodes) + (run! #(configure-absent! test %) remove-nodes) + (finalize-config! test) + (assoc op :value keep-nodes)) + :stop + (do + (info "layout un-split: all nodes=" (:nodes test)) + (run! #(configure-present! test %) (:nodes test)) + (finalize-config! test) + (assoc op :value (:nodes test))))) + + (teardown! [this test] this))) + +(defn scenario-cp + "Clock scramble + parittion scenario" + [opts] + {:generator (cycle [(gen/sleep 5) + {:type :info, :f :partition-start} + (gen/sleep 5) + {:type :info, :f :clock-scramble} + (gen/sleep 5) + {:type :info, :f :partition-stop} + (gen/sleep 5) + {:type :info, :f :clock-scramble}]) + :nemesis (nemesis/compose + {{:partition-start :start + :partition-stop :stop} (nemesis/partition-random-halves) + {:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})}) + +(defn scenario-r + "Cluster reconfiguration scenario" + [opts] + {:generator (cycle [(gen/sleep 5) + {:type :info, :f :reconfigure-start} + (gen/sleep 5) + {:type :info, :f :reconfigure-start} + (gen/sleep 5) + {:type :info, :f :reconfigure-stop}]) + :nemesis (nemesis/compose + {{:reconfigure-start :start + :reconfigure-stop :stop} (reconfigure-subset 3)})})