WIP reproduce stats

garage-latency-simple.R

@@ -0,0 +1,67 @@
+#!/usr/bin/env Rscript
+.libPaths(c("~/R", .libPaths()))
+repo <- "https://cran.r-project.org"
+
+if (!require(tidyverse)) install.packages("tidyverse", repos=repo); library(tidyverse)
+if (!require(svglite)) install.packages("svglite", repos=repo); library(svglite)
+
+system("mkdir -p $HOME/.local/bin")
+Sys.getenv("HOME") -> home
+Sys.setenv(PATH = paste(paste(home, "/go/bin", sep=""), paste(home, "/.local/bin", sep=""), Sys.getenv("PATH"), sep=":"))
+system("echo $PATH")
+
+if (!file.exists("~/.local/bin/garage")) {
+  system("wget https://garagehq.deuxfleurs.fr/_releases/v0.5.0/x86_64-unknown-linux-musl/garage -O $HOME/.local/bin/garage")
+  system("chmod +x $HOME/.local/bin/garage")
+}
+
+if (!file.exists("~/.local/bin/minio")) {
+  system("wget https://dl.min.io/server/minio/release/linux-amd64/minio.RELEASE.2021-11-24T23-19-33Z -O $HOME/.local/bin/minio")
+  system("chmod +x $HOME/.local/bin/minio")
+}
+
+if (!file.exists("~/.local/bin/mknet")) {
+  system("pip3 install --user git+https://git.deuxfleurs.fr/quentin/mknet")
+}
+
+if (!file.exists("~/go/bin/s3lat")) {
+  system("go install git.deuxfleurs.fr/quentin/s3lat@latest")
+}
+
+if (!file.exists("50ms.garage.csv")) {
+  system("mknet create ./mknet/single-dc.yml")
+  system("mknet run-all ./mknet/deploy_garage.sh")
+  system("mknet run dc1:dc1s1 ./mknet/bench_garage.sh")
+  system("mknet destroy")
+}
+
+if (!file.exists("50ms.minio.csv")) {
+  system("mknet create ./mknet/single-dc.yaml")
+  system("mknet run-all ./mknet/deploy_minio.py")
+  system("mknet run dc1:dc1s1 ./mknet/bench_minio.sh")
+  system("mknet destroy")
+}
+
+read_csv("50ms.garage.csv") %>% add_column(daemon="garage 0.5.0") -> a
+read_csv("50ms.minio.csv") %>% add_column(daemon="minio RELEASE.2021-11-24T23-19-33Z") -> b
+bind_rows(a,b) %>% group_by(daemon,op) %>% summarise(
+  time_mean = mean(time),
+  time_max = max(time),
+  time_min = min(time)
+) -> c
+
+ggplot(c, aes(x=op,y=time_mean,fill=daemon,ymin=time_min,ymax=time_max)) +
+      geom_bar(stat="identity", position=position_dodge(),color="black") +
+      geom_errorbar(position=position_dodge(.9),width=.2) + 
+      scale_y_continuous(expand=c(0,0))+
+      coord_flip() +
+      labs(
+        x="S3 Endpoint", 
+        y="Latency (ms)",
+        fill="Daemon",
+        caption="Get the code to reproduce this graph at https://git.deuxfleurs.fr/quentin/benchmarks",
+        title="S3 endpoint latency in a simulated geo-distributed cluster",
+        subtitle="100 measurements, 5 nodes, 100ms RTT + 20ms jitter between nodes\nno contention: latency is due to intra-cluster communications\ncolored bar = mean latency, error bar = min and max") + 
+      theme_classic() +
+      ggsave("endpoint-latency.png", width=200, height=110, units="mm")
+       

mknet/bench_garage.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+socat "openssl-listen:4443,\
+reuseaddr,\
+fork,\
+verify=0,\
+cert=/tmp/garage.pem" tcp4-connect:localhost:3900 &
+
+sleep 1
+
+export SSL=1
+export SSL_INSECURE=1
+export REGION=garage
+export ENDPOINT=localhost:4443
+export AWS_ACCESS_KEY_ID=GK4ea45d2f25091883071e0b73
+export AWS_SECRET_ACCESS_KEY=334f7b0ef233571c67b302c5197e6b8c7150da1907e4e530caa82e041d725ecf
+
+/home/quentin/go/bin/bench-garage | tee 50ms.garage.csv

mknet/bench_minio.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+export ENDPOINT=[fc00:9a7a:9e::1]:9000
+export AWS_ACCESS_KEY_ID=minioadmin
+export AWS_SECRET_ACCESS_KEY=minioadmin
+
+s3lat | tee 50ms.minio.csv

mknet/deploy_garage.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -euo pipefail
+IFS=$'\n\t'
+
+GARAGE_PATH=garage
+STORAGE_PATH=/tmp/garage-testnet
+export RPC_SECRET=3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba
+
+if [ -z "$ZONE" ]; then
+	NODE_STORAGE_PATH=${STORAGE_PATH}/${HOST}
+else
+	NODE_STORAGE_PATH=${STORAGE_PATH}/${ZONE}/${HOST}
+fi
+BOOTSTRAP_FILE=${STORAGE_PATH}/bootstrap_peer
+export GARAGE_CONFIG_FILE=${NODE_STORAGE_PATH}/garage.toml 
+
+
+mkdir -p ${NODE_STORAGE_PATH}
+cd ${NODE_STORAGE_PATH}
+rm ${BOOTSTRAP_FILE} 2>/dev/null || true
+
+cat > ${GARAGE_CONFIG_FILE} << EOF
+metadata_dir = "${NODE_STORAGE_PATH}/meta"
+data_dir = "${NODE_STORAGE_PATH}/data"
+
+replication_mode = "3"
+
+rpc_bind_addr = "[::]:3901"
+rpc_public_addr = "[${IP}]:3901"
+rpc_secret = "${RPC_SECRET}"
+
+bootstrap_peers=[]
+
+[s3_api]
+s3_region = "garage"
+api_bind_addr = "[::]:3900"
+root_domain = ".s3.garage"
+
+[s3_web]
+bind_addr = "[::]:3902"
+root_domain = ".web.garage"
+index = "index.html"
+EOF
+
+RUST_LOG=garage=debug ${GARAGE_PATH} server 2>> ${NODE_STORAGE_PATH}/logs & disown
+sleep 2
+
+CONFIG_NODE_FPATH=$(find /tmp/garage-testnet/ -maxdepth 3 -name garage.toml|head -n 1)
+
+SELF_ID=$(${GARAGE_PATH} node id 2>/dev/null)
+SHORT_ID=$(echo ${SELF_ID} | cut -c-64)
+
+${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} node connect ${SELF_ID}
+${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} layout assign ${SHORT_ID} -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST}
+
+if [ ${CONFIG_NODE_FPATH} == ${GARAGE_CONFIG_FILE} ]; then
+	sleep 2
+	${GARAGE_PATH} layout show
+	${GARAGE_PATH} layout apply --version 1
+fi

mknet/deploy_minio.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+import json, os, sys, time, pathlib, socket, shutil
+
+STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet')
+HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt')
+UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock')
+DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid))
+
+def main():
+    if int(os.environ['ID']) == 1: leader()
+    else: follower()
+
+def leader():
+    shutil.rmtree(STORAGE_PATH, ignore_errors=True)
+    os.makedirs(STORAGE_PATH)
+    print(STORAGE_PATH)
+
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.bind(UNIX_SOCK)
+    sock.listen()
+
+    n_serv = os.environ['SIZE']
+    fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]]
+
+    identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ]
+    print(f"ident: {identities}")
+    msg = f"{json.dumps(identities)}\n".encode()
+    [ co.send(msg) for co in fl ]
+
+    run_minio(identities)
+
+def follower():
+    co = None
+    while True:
+        time.sleep(1)
+        try:
+            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            sock.connect(UNIX_SOCK)
+            co = sock.makefile()
+            break
+        except Exception as err:
+            print('conn failed, wait,', err)
+    my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() })
+    sock.send(f"{my_identity}\n".encode())
+    identities = json.loads(co.readline())
+
+    run_minio(identities)
+
+def make_data():
+    data_path = DATA_PATH(os.environ['ID'])
+    os.makedirs(data_path)
+    return data_path
+
+def run_minio(identities):
+    cmd = f"minio server --console-address ':9001' --address [{os.environ['IP']}]:9000"
+    for ident in identities:
+        cmd += f" http://[{ident['ip']}]:9000{ident['path']}"
+    cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1"
+    print("launch: ", cmd)
+    os.system(cmd)
+
+__name__ == '__main__' and main()

mknet/single-dc.yml

@@ -0,0 +1,34 @@
+links:
+  - &fiber
+    bandwidth: 100M
+    latency: 50ms
+    jitter: 10ms
+
+zones:
+  - &dc1
+    name: dc1
+    internal: *fiber
+    external: *fiber
+
+servers:
+  - name: dc1s1
+    zone: *dc1
+  - name: dc1s2
+    zone: *dc1
+  - name: dc1s3
+    zone: *dc1
+  - name: dc1s4
+    zone: *dc1
+  - name: dc1s5
+    zone: *dc1
+
+global:
+  subnet:
+    base: 'fc00:9a7a:9e::'
+    local: 64
+    zone: 16
+  latency-offset: 3ms
+  upstream:
+    ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff
+    conn: *fiber
+