Deuxfleurs/garage
41
56
Fork 73
Code Issues 98 Pull requests 14 Releases 30 Wiki Activity

Add garage set test (this one works :p)

Browse source
This commit is contained in:
Alex 2023-04-19 15:27:26 +02:00
parent eb86eaa6d2
commit 6eb26be548
5 changed files with 95 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
script/jepsen.garage/README.md
View file

@ -18,10 +18,10 @@ Set up VMs:
vagrant up vagrant up
``` ```
Run tests: Run tests (this one should fail):
``` ```
lein run test --nodes-file nodes.vagrant lein run test --nodes-file nodes.vagrant --time-limit 64 --concurrency 50 --rate 50 --workload reg
``` ```
## License ## License

22
script/jepsen.garage/src/jepsen/garage.clj
View file

@ -10,11 +10,13 @@
[jepsen.os.debian :as debian] [jepsen.os.debian :as debian]
[jepsen.garage [jepsen.garage
[grg :as grg] [grg :as grg]
[reg :as reg]])) [reg :as reg]
[set :as set]]))
(def workloads (def workloads
"A map of workload names to functions that construct workloads, given opts." "A map of workload names to functions that construct workloads, given opts."
{"reg" reg/workload}) {"reg" reg/workload
"set" set/workload})
(def cli-opts (def cli-opts
"Additional command line options." "Additional command line options."
@ -47,7 +49,21 @@
:os debian/os :os debian/os
:db (grg/db garage-version) :db (grg/db garage-version)
:client (:client workload) :client (:client workload)
:generator (:generator workload) :generator (gen/phases
(->>
(:generator workload)
(gen/stagger (/ (:rate opts)))
(gen/nemesis
(cycle [(gen/sleep 5)
{:type :info, :f :start}
(gen/sleep 5)
{:type :info, :f :stop}]))
(gen/time-limit (:time-limit opts)))
(gen/log "Healing cluster")
(gen/nemesis (gen/once {:type :info, :f :stop}))
(gen/log "Waiting for recovery")
(gen/sleep 10)
(gen/clients (:final-generator workload)))
:nemesis (nemesis/partition-random-halves) :nemesis (nemesis/partition-random-halves)
:checker (checker/compose :checker (checker/compose
{:perf (checker/perf) {:perf (checker/perf)

13
script/jepsen.garage/src/jepsen/garage/grg.clj
View file

@ -121,5 +121,14 @@
(defn s3-list (defn s3-list
"Helper for ListObjects -- just lists everything in the bucket" "Helper for ListObjects -- just lists everything in the bucket"
[creds] [creds]
(s3/list-objects-v2 creds (defn list-inner [ct accum]
{:bucket-name (:bucket creds)})) (let [list-result (s3/list-objects-v2 creds
{:bucket-name (:bucket creds)
:continuation-token ct})
new-object-summaries (:object-summaries list-result)
new-objects (map (fn [d] (:key d)) new-object-summaries)
objects (concat new-objects accum)]
(if (:truncated? list-result)
(list-inner (:next-continuation-token list-result) objects)
objects)))
(list-inner nil []))

21
script/jepsen.garage/src/jepsen/garage/reg.clj
View file

@ -51,19 +51,12 @@
{:model (model/register) {:model (model/register)
:algorithm :linear}) :algorithm :linear})
:timeline (timeline/html)})) :timeline (timeline/html)}))
:generator (->> (independent/concurrent-generator :generator (independent/concurrent-generator
10 10
(range) (range)
(fn [k] (fn [k]
(->> (->>
(gen/mix [op-get op-put op-del]) (gen/mix [op-get op-put op-del])
(gen/stagger (/ (:rate opts))) (gen/limit (:ops-per-key opts)))))})
(gen/limit (:ops-per-key opts)))))
(gen/nemesis
(cycle [(gen/sleep 5)
{:type :info, :f :start}
(gen/sleep 5)
{:type :info, :f :stop}]))
(gen/time-limit (:time-limit opts)))})

56
script/jepsen.garage/src/jepsen/garage/set.clj Normal file
View file

@ -0,0 +1,56 @@
(ns jepsen.garage.set
(:require [clojure.tools.logging :refer :all]
[clojure.string :as str]
[jepsen [checker :as checker]
[cli :as cli]
[client :as client]
[control :as c]
[db :as db]
[generator :as gen]
[independent :as independent]
[nemesis :as nemesis]
[tests :as tests]]
[jepsen.checker.timeline :as timeline]
[jepsen.control.util :as cu]
[jepsen.os.debian :as debian]
[jepsen.garage.grg :as grg]
[knossos.model :as model]
[slingshot.slingshot :refer [try+]]))
(defn op-add [_ _] {:type :invoke, :f :add, :value (rand-int 100000)})
(defn op-read [_ _] {:type :invoke, :f :read, :value nil})
(defrecord SetClient [creds]
client/Client
(open! [this test node]
(let [creds (grg/s3-creds node)]
(info node "s3 credentials:" creds)
(assoc this :creds creds)))
(setup! [this test])
(invoke! [this test op]
(case (:f op)
:add
(do
(grg/s3-put (:creds this) (str (:value op)) "present")
(assoc op :type :ok))
:read
(let [items (grg/s3-list (:creds this))]
(assoc op :type :ok, :value (set (map read-string items))))))
(teardown! [this test])
(close! [this test]))
(defn workload
"Tests insertions and deletions"
[opts]
{:client (SetClient. nil)
:checker (checker/compose
{:set (checker/set)
:timeline (timeline/html)})
; :generator (gen/mix [op-add op-read])
; :generator (->> (range)
; (map (fn [x] {:type :invoke, :f :add, :value x})))
:generator (gen/mix [op-read
(->> (range) (map (fn [x] {:type :invoke, :f :add, :value x})))])
:final-generator (gen/once op-read)})