(ns jepsen.garage.nemesis (:require [clojure.tools.logging :refer :all] [jepsen [control :as c] [core :as jepsen] [generator :as gen] [nemesis :as nemesis]] [jepsen.nemesis.combined :as combined] [jepsen.garage.daemon :as grg] [jepsen.control.util :as cu])) ; ---- reconfiguration nemesis ---- (defn configure-present! "Configure node to be active in new cluster layout" [test nodes] (info "configure-present!" nodes) (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q)) node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)] (c/on (jepsen/primary test) (apply c/exec (concat [grg/binary :layout :assign :-c :1G] node-id-strs))))) (defn configure-absent! "Configure nodes to be active in new cluster layout" [test nodes] (info "configure-absent!" nodes) (let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q)) node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)] (c/on (jepsen/primary test) (apply c/exec (concat [grg/binary :layout :assign :-g] node-id-strs))))) (defn finalize-config! "Apply the proposed cluster layout" [test] (let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show)) [_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)] (if layout-next-version (do (info "layout show: " layout-show "; next-version: " layout-next-version) (c/on (jepsen/primary test) (c/exec grg/binary :layout :apply :--version layout-next-version))) (info "no layout changes to apply")))) (defn reconfigure-subset "Reconfigure cluster with only a subset of nodes" [cnt] (reify nemesis/Nemesis (setup! [this test] this) (invoke! [this test op] op (case (:f op) :start (let [[keep-nodes remove-nodes] (->> (:nodes test) shuffle (split-at cnt))] (info "layout split: keep " keep-nodes ", remove " remove-nodes) (configure-present! test keep-nodes) (configure-absent! test remove-nodes) (finalize-config! test) (assoc op :value keep-nodes)) :stop (do (info "layout un-split: all nodes=" (:nodes test)) (configure-present! test (:nodes test)) (finalize-config! test) (assoc op :value (:nodes test))))) (teardown! [this test] this))) ; ---- nemesis scenari ---- (defn nemesis-op "A generator for a single nemesis operation" [op] (fn [_ _] {:type :info, :f op})) (defn reconfiguration-package "Cluster reconfiguration nemesis package" [opts] {:generator (->> (gen/mix [(nemesis-op :reconfigure-start) (nemesis-op :reconfigure-stop)]) (gen/stagger (:interval opts 5))) :final-generator {:type :info, :f :reconfigure-stop} :nemesis (nemesis/compose {{:reconfigure-start :start :reconfigure-stop :stop} (reconfigure-subset 3)}) :perf #{{:name "reconfigure" :start #{:reconfigure-start} :stop #{:reconfigur-stop} :color "#A197E9"}}}) (defn scenario-c "Clock modifying scenario" [opts] (combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})) (defn scenario-cp "Clock modifying + partition scenario" [opts] (combined/compose-packages [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})])) (defn scenario-r "Cluster reconfiguration scenario" [opts] (reconfiguration-package {:interval 1})) (defn scenario-pr "Partition + cluster reconfiguration scenario" [opts] (combined/compose-packages [(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) (reconfiguration-package {:interval 1})])) (defn scenario-cpr "Clock scramble + partition + cluster reconfiguration scenario" [opts] (combined/compose-packages [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) (reconfiguration-package {:interval 1})])) (defn scenario-cdp "Clock modifying + db + partition scenario" [opts] (combined/compose-packages [(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}) (combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})])) (defn scenario-dpr "Db + partition + cluster reconfiguration scenario" [opts] (combined/compose-packages [(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}}) (combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}}) (reconfiguration-package {:interval 1})]))