forked from Deuxfleurs/garage
make jepsen test more robust: handle errors and timeouts, fixed access key
This commit is contained in:
parent
ddd3de7fce
commit
b3bf16ee27
3 changed files with 36 additions and 26 deletions
|
@ -41,7 +41,7 @@
|
||||||
[opts]
|
[opts]
|
||||||
(let [workload ((get workloads (:workload opts)) opts)
|
(let [workload ((get workloads (:workload opts)) opts)
|
||||||
garage-version (if (:increasing-timestamps opts)
|
garage-version (if (:increasing-timestamps opts)
|
||||||
"03490d41d58576d7b3bcf977b2726d72a3a66ada"
|
"d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"
|
||||||
"v0.9.0")]
|
"v0.9.0")]
|
||||||
(merge tests/noop-test
|
(merge tests/noop-test
|
||||||
opts
|
opts
|
||||||
|
@ -56,16 +56,23 @@
|
||||||
(gen/stagger (/ (:rate opts)))
|
(gen/stagger (/ (:rate opts)))
|
||||||
(gen/nemesis
|
(gen/nemesis
|
||||||
(cycle [(gen/sleep 5)
|
(cycle [(gen/sleep 5)
|
||||||
{:type :info, :f :start}
|
;{:type :info, :f :partition-start}
|
||||||
|
;(gen/sleep 5)
|
||||||
|
{:type :info, :f :clock-scramble}
|
||||||
(gen/sleep 5)
|
(gen/sleep 5)
|
||||||
{:type :info, :f :stop}]))
|
;{:type :info, :f :partition-stop}
|
||||||
|
;(gen/sleep 5)
|
||||||
|
{:type :info, :f :clock-scramble}]))
|
||||||
(gen/time-limit (:time-limit opts)))
|
(gen/time-limit (:time-limit opts)))
|
||||||
(gen/log "Healing cluster")
|
(gen/log "Healing cluster")
|
||||||
(gen/nemesis (gen/once {:type :info, :f :stop}))
|
(gen/nemesis (gen/once {:type :info, :f :partition-stop}))
|
||||||
(gen/log "Waiting for recovery")
|
(gen/log "Waiting for recovery")
|
||||||
(gen/sleep 10)
|
(gen/sleep 10)
|
||||||
(gen/clients (:final-generator workload)))
|
(gen/clients (:final-generator workload)))
|
||||||
:nemesis (nemesis/partition-random-halves)
|
:nemesis (nemesis/compose
|
||||||
|
{{:partition-start :start
|
||||||
|
:partition-stop :stop} (nemesis/partition-random-halves)
|
||||||
|
{:clock-scramble :scramble} (nemesis/clock-scrambler 20.0)})
|
||||||
:checker (checker/compose
|
:checker (checker/compose
|
||||||
{:perf (checker/perf)
|
{:perf (checker/perf)
|
||||||
:workload (:checker workload)})
|
:workload (:checker workload)})
|
||||||
|
|
|
@ -15,7 +15,8 @@
|
||||||
(def pidfile (str base-dir "/garage.pid"))
|
(def pidfile (str base-dir "/garage.pid"))
|
||||||
|
|
||||||
(def admin-token "icanhazadmin")
|
(def admin-token "icanhazadmin")
|
||||||
(def access-key "jepsen")
|
(def access-key-id "GK8bfb6a51286071c6c9cd8bc3")
|
||||||
|
(def secret-access-key "b0be95f71c1c6f16858a9edf395078b75c12ecb6b1c03385c4ae92076e4994a3")
|
||||||
(def bucket-name "jepsen")
|
(def bucket-name "jepsen")
|
||||||
|
|
||||||
; THE GARAGE DB
|
; THE GARAGE DB
|
||||||
|
@ -78,10 +79,10 @@
|
||||||
(c/trace
|
(c/trace
|
||||||
(c/exec binary :layout :apply :--version 1)
|
(c/exec binary :layout :apply :--version 1)
|
||||||
(info node "garage status:" (c/exec binary :status))
|
(info node "garage status:" (c/exec binary :status))
|
||||||
(c/exec binary :key :create access-key)
|
(c/exec binary :key :import access-key-id secret-access-key :--yes)
|
||||||
(c/exec binary :bucket :create bucket-name)
|
(c/exec binary :bucket :create bucket-name)
|
||||||
(c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key)
|
(c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key-id)
|
||||||
(info node "key info: " (c/exec binary :key :info access-key))))
|
(info node "key info: " (c/exec binary :key :info access-key-id))))
|
||||||
|
|
||||||
(defn db
|
(defn db
|
||||||
"Garage DB for a particular version"
|
"Garage DB for a particular version"
|
||||||
|
@ -122,13 +123,9 @@
|
||||||
(defn creds
|
(defn creds
|
||||||
"Obtain Garage credentials for node"
|
"Obtain Garage credentials for node"
|
||||||
[node]
|
[node]
|
||||||
(let [key-info (c/on node (c/exec binary :key :info access-key :--show-secret))
|
{:access-key access-key-id
|
||||||
[_ ak sk] (re-matches
|
:secret-key secret-access-key
|
||||||
#"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*"
|
:endpoint (str "http://" node ":3900")
|
||||||
key-info)]
|
:bucket bucket-name
|
||||||
{:access-key ak
|
:client-config {:path-style-access-enabled true}})
|
||||||
:secret-key sk
|
|
||||||
:endpoint (str "http://" node ":3900")
|
|
||||||
:bucket bucket-name
|
|
||||||
:client-config {:path-style-access-enabled true}}))
|
|
||||||
|
|
||||||
|
|
|
@ -33,19 +33,25 @@
|
||||||
(let [[k v] (:value op)]
|
(let [[k v] (:value op)]
|
||||||
(case (:f op)
|
(case (:f op)
|
||||||
:read
|
:read
|
||||||
(let [value (s3/get (:creds this) k)]
|
(try+
|
||||||
(assoc op :type :ok, :value (independent/tuple k value)))
|
(let [value (s3/get (:creds this) k)]
|
||||||
|
(assoc op :type :ok, :value (independent/tuple k value)))
|
||||||
|
(catch (re-find #"Unavailable" (.getMessage %)) ex
|
||||||
|
(assoc op :type :fail, :error [:s3-error (.getMessage ex)])))
|
||||||
:write
|
:write
|
||||||
(do
|
(try+
|
||||||
(s3/put (:creds this) k v)
|
(do
|
||||||
(assoc op :type :ok)))))
|
(s3/put (:creds this) k v)
|
||||||
|
(assoc op :type :ok))
|
||||||
|
(catch (re-find #"Unavailable" (.getMessage %)) ex
|
||||||
|
(assoc op :type :fail, :error [:s3-error (.getMessage ex)]))))))
|
||||||
(teardown! [this test])
|
(teardown! [this test])
|
||||||
(close! [this test]))
|
(close! [this test]))
|
||||||
|
|
||||||
(defn workload
|
(defn workload
|
||||||
"Tests linearizable reads and writes"
|
"Tests linearizable reads and writes"
|
||||||
[opts]
|
[opts]
|
||||||
{:client (RegClient. nil)
|
{:client (client/timeout 10 (RegClient. nil))
|
||||||
:checker (independent/checker
|
:checker (independent/checker
|
||||||
(checker/compose
|
(checker/compose
|
||||||
{:linear (checker/linearizable
|
{:linear (checker/linearizable
|
||||||
|
@ -53,8 +59,8 @@
|
||||||
:algorithm :linear})
|
:algorithm :linear})
|
||||||
:timeline (timeline/html)}))
|
:timeline (timeline/html)}))
|
||||||
:generator (independent/concurrent-generator
|
:generator (independent/concurrent-generator
|
||||||
10
|
(/ (:concurrency opts) 10) ; divide threads in 10 groups
|
||||||
(range)
|
(range) ; working on 10 keys
|
||||||
(fn [k]
|
(fn [k]
|
||||||
(->>
|
(->>
|
||||||
(gen/mix [op-get op-put op-del])
|
(gen/mix [op-get op-put op-del])
|
||||||
|
|
Loading…
Reference in a new issue