From bc11701999ececdb4571119082945cbe86f098aa Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 18 Apr 2023 16:10:07 +0200 Subject: [PATCH] jepsen: s3 gets and puts --- script/jepsen.garage/Vagrantfile | 6 +- script/jepsen.garage/destroy-tap.sh | 9 -- script/jepsen.garage/garage-cluster.nix | 70 ----------- script/jepsen.garage/nodes.containers | 5 - script/jepsen.garage/project.clj | 3 +- script/jepsen.garage/setup-tap.sh | 10 -- script/jepsen.garage/src/jepsen/garage.clj | 140 ++++++++++++++++++++- 7 files changed, 141 insertions(+), 102 deletions(-) delete mode 100755 script/jepsen.garage/destroy-tap.sh delete mode 100644 script/jepsen.garage/garage-cluster.nix delete mode 100644 script/jepsen.garage/nodes.containers delete mode 100755 script/jepsen.garage/setup-tap.sh diff --git a/script/jepsen.garage/Vagrantfile b/script/jepsen.garage/Vagrantfile index da6a277f..c40c600d 100644 --- a/script/jepsen.garage/Vagrantfile +++ b/script/jepsen.garage/Vagrantfile @@ -8,7 +8,7 @@ def vm(config, hostname, ip) end Vagrant.configure("2") do |config| - config.vm.box = "generic/alpine38" + config.vm.box = "generic/debian10" config.vm.provider "virtualbox" do |vb| vb.gui = false @@ -17,8 +17,8 @@ Vagrant.configure("2") do |config| config.vm.provision "shell", inline: <<-SHELL echo "root:root" | chpasswd - apk update - apk add wget + mkdir -p /root/.ssh + echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy" >> /root/.ssh/authorized_keys SHELL config.vm.define "n1" do |config| vm(config, "n1", "192.168.56.21") end diff --git a/script/jepsen.garage/destroy-tap.sh b/script/jepsen.garage/destroy-tap.sh deleted file mode 100755 index 544b3053..00000000 --- a/script/jepsen.garage/destroy-tap.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -USER=$(whoami) - -for NODE in 1 2 3 4 5; do - sudo ip link delete microvm-n$NODE -done - - diff --git a/script/jepsen.garage/garage-cluster.nix b/script/jepsen.garage/garage-cluster.nix deleted file mode 100644 index ebc73aaf..00000000 --- a/script/jepsen.garage/garage-cluster.nix +++ /dev/null @@ -1,70 +0,0 @@ -{ config, lib, pkgs, ... }: -let - unstable = import ./unstable.nix; - addressMap = - { - "n1" = { localAddress = "10.233.0.101"; hostAddress = "10.233.1.101"; }; - "n2" = { localAddress = "10.233.0.102"; hostAddress = "10.233.1.102"; }; - "n3" = { localAddress = "10.233.0.103"; hostAddress = "10.233.1.103"; }; - "n4" = { localAddress = "10.233.0.104"; hostAddress = "10.233.1.104"; }; - "n5" = { localAddress = "10.233.0.105"; hostAddress = "10.233.1.105"; }; - }; - toHostsEntry = name: { localAddress, ... }: "${localAddress} ${name}"; - extraHosts = - builtins.concatStringsSep "\n" - (lib.attrsets.mapAttrsToList toHostsEntry addressMap); - nodeConfig = hostName: { localAddress, hostAddress }: { - inherit localAddress hostAddress; - - ephemeral = true; - autoStart = true; - privateNetwork = true; - - config = { config, pkgs, ... }: - { - networking = { - inherit hostName extraHosts; - }; - - services.openssh = { - enable = true; - permitRootLogin = "yes"; - kexAlgorithms = [ "curve25519-sha256@libssh.org" "ecdh-sha2-nistp256" "ecdh-sha2-nistp384" "ecdh-sha2-nistp521" "diffie-hellman-group-exchange-sha256" "diffie-hellman-group14-sha1" "diffie-hellman-group-exchange-sha1" "diffie-hellman-group1-sha1" ]; - }; - users.users.root.initialPassword = "root"; - users.users.root.openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9" - ]; - - system.stateVersion = "22.11"; - - services.garage = { - enable = true; - logLevel = "debug"; - settings = { - replication_mode = "3"; - db_engine = "lmdb"; - rpc_secret = "b597bb28ebdc90cdc4f15712733ca678cfb9a7e0311e0b9e93db9610fc3685e6"; - rpc_bind_addr = "0.0.0.0:3901"; - s3_api = { - region = "garage"; - api_bind_addr = "0.0.0.0:3900"; - }; - k2v_api.api_bind_addr = "0.0.0.0:3902"; - admin = { - api_bind_addr = "0.0.0.0:3903"; - admin_token = "icanhazadmin"; - }; - }; - }; - - networking.firewall.allowedTCPPorts = [ 3901 3900 3902 3903 ]; - }; - }; -in -{ - containers = lib.attrsets.mapAttrs nodeConfig addressMap; - networking = { - inherit extraHosts; - }; -} diff --git a/script/jepsen.garage/nodes.containers b/script/jepsen.garage/nodes.containers deleted file mode 100644 index b8f3eeb1..00000000 --- a/script/jepsen.garage/nodes.containers +++ /dev/null @@ -1,5 +0,0 @@ -n1.containers -n2.containers -n3.containers -n4.containers -n5.containers diff --git a/script/jepsen.garage/project.clj b/script/jepsen.garage/project.clj index 2ab03e49..04ae4cc3 100644 --- a/script/jepsen.garage/project.clj +++ b/script/jepsen.garage/project.clj @@ -5,5 +5,6 @@ :url "https://www.gnu.org/licenses/gpl-3.0.en.html"} :main jepsen.garage :dependencies [[org.clojure/clojure "1.11.1"] - [jepsen "0.3.2-SNAPSHOT"]] + [jepsen "0.3.2-SNAPSHOT"] + [amazonica "0.3.163"]] :repl-options {:init-ns jepsen.garage}) diff --git a/script/jepsen.garage/setup-tap.sh b/script/jepsen.garage/setup-tap.sh deleted file mode 100755 index 7e09abc3..00000000 --- a/script/jepsen.garage/setup-tap.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -USER=$(whoami) - -for NODE in 1 2 3 4 5; do - sudo ip tuntap add microvm-n$NODE mode tap user $USER - sudo ip addr add dev microvm-n$NODE 10.$NODE.0.1 -done - - diff --git a/script/jepsen.garage/src/jepsen/garage.clj b/script/jepsen.garage/src/jepsen/garage.clj index 6617064e..26c58097 100644 --- a/script/jepsen.garage/src/jepsen/garage.clj +++ b/script/jepsen.garage/src/jepsen/garage.clj @@ -1,14 +1,146 @@ (ns jepsen.garage - (:require [jepsen.cli :as cli] - [jepsen.tests :as tests])) + (:require [clojure.tools.logging :refer :all] + [clojure.string :as str] + [jepsen [cli :as cli] + [client :as client] + [control :as c] + [db :as db] + [generator :as gen] + [tests :as tests]] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian] + [amazonica.aws.s3 :as s3] + [amazonica.aws.s3transfer :as s3transfer])) + +(def dir "/opt/garage") +(def binary (str dir "/garage")) +(def logfile (str dir "/garage.log")) +(def pidfile (str dir "/garage.pid")) + +(def grg-admin-token "icanhazadmin") +(def grg-key "jepsen") +(def grg-bucket "jepsen") +(def grg-object "1") + +(defn db + "Garage DB for a particular version" + [version] + (reify db/DB + (setup! [_ test node] + (info node "installing garage" version) + (c/su + (c/exec :mkdir :-p dir) + (let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage") + cache (cu/wget! url)] + (c/exec :cp cache binary)) + (c/exec :chmod :+x binary) + (cu/write-file! + (str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n" + "rpc_bind_addr = \"0.0.0.0:3901\"\n" + "rpc_public_addr = \"" node ":3901\"\n" + "db_engine = \"lmdb\"\n" + "replication_mode = \"3\"\n" + "data_dir = \"" dir "/data\"\n" + "metadata_dir = \"" dir "/meta\"\n" + "[s3_api]\n" + "s3_region = \"us-east-1\"\n" + "api_bind_addr = \"0.0.0.0:3900\"\n" + "[k2v_api]\n" + "api_bind_addr = \"0.0.0.0:3902\"\n" + "[admin]\n" + "api_bind_addr = \"0.0.0.0:3903\"\n" + "admin_token = \"" grg-admin-token "\"\n") + "/etc/garage.toml") + (cu/start-daemon! + {:logfile logfile + :pidfile pidfile + :chdir dir} + binary + :server) + (Thread/sleep 100) + (let [node-id (c/exec binary :node :id :-q)] + (info node "node id:" node-id) + (c/on-many (:nodes test) + (c/exec binary :node :connect node-id)) + (c/exec binary :layout :assign (subs node-id 0 16) :-c 1 :-z :dc1 :-t node)) + (if (= node (first (:nodes test))) + (do + (Thread/sleep 2000) + (c/exec binary :layout :apply :--version 1) + (info node "garage status:" (c/exec binary :status)) + (c/exec binary :key :new :--name grg-key) + (c/exec binary :bucket :create grg-bucket) + (c/exec binary :bucket :allow :--read :--write grg-bucket :--key grg-key) + (info node "key info: " (c/exec binary :key :info grg-key)))))) + (teardown! [_ test node] + (info node "tearing down garage" version) + (c/su + (cu/stop-daemon! binary pidfile) + (c/exec :rm :-rf dir))) + db/LogFiles + (log-files [_ test node] + [logfile]))) + +(defn op-get [_ _] {:type :invoke, :f :get-object, :value nil}) +(defn op-put [_ _] {:type :invoke, :f :put-object, :value (str (rand-int 50))}) +(defn op-del [_ _] {:type :invoke, :f :del-object, :value nil}) + +(defrecord Client [creds] + client/Client + (open! [this test node] + (let [key-info (c/on node (c/exec binary :key :info grg-key)) + [_ ak sk] (re-matches + #"(?s).*Key ID: (.*)\nSecret key: (.*)\nCan create.*" + key-info) + creds {:access-key ak + :secret-key sk + :endpoint (str "http://" node ":3900") + :client-config {:path-style-access-enabled true}}] + (info node "s3 credentials:" creds) + (assoc this :creds creds))) + (setup! [this test]) + (invoke! [this test op] + (case (:f op) + :get-object + (let [value-bytes (try + (-> (s3/get-object (:creds this) grg-bucket grg-object) + :input-stream + slurp) + (catch Exception e nil))] + (assoc op :type :ok, :value value-bytes)) + :put-object + (let [some-bytes (.getBytes (:value op) "UTF-8") + bytes-stream (java.io.ByteArrayInputStream. some-bytes)] + (s3/put-object (:creds this) + :bucket-name grg-bucket + :key grg-object + :input-stream bytes-stream + :metadata {:content-length (count some-bytes)}) + (assoc op :type :ok)) + :del-object + (do + (s3/delete-object (:creds this) + :bucket-name grg-bucket + :key grg-object) + (assoc op :type :ok, :value nil)))) + (teardown! [this test]) + (close! [this test])) (defn garage-test "Given an options map from the command line runner (e.g. :nodes, :ssh, :concurrency, ...), constructs a test map." [opts] (merge tests/noop-test - {:pure-generators true} - opts)) + opts + {:pure-generators true + :name "garage" + :os debian/os + :db (db "v0.8.2") + :client (Client. nil) + :generator (->> (gen/mix [op-get op-put]) + (gen/stagger 1) + (gen/nemesis nil) + (gen/time-limit 15))})) (defn -main "Handles command line arguments. Can either run a test, or a web server for