WIP reproduce stats

This commit is contained in:
Quentin 2021-12-08 17:09:17 +01:00
commit df4116b174
6 changed files with 249 additions and 0 deletions

67
garage-latency-simple.R Executable file
View file

@ -0,0 +1,67 @@
#!/usr/bin/env Rscript
.libPaths(c("~/R", .libPaths()))
repo <- "https://cran.r-project.org"
if (!require(tidyverse)) install.packages("tidyverse", repos=repo); library(tidyverse)
if (!require(svglite)) install.packages("svglite", repos=repo); library(svglite)
system("mkdir -p $HOME/.local/bin")
Sys.getenv("HOME") -> home
Sys.setenv(PATH = paste(paste(home, "/go/bin", sep=""), paste(home, "/.local/bin", sep=""), Sys.getenv("PATH"), sep=":"))
system("echo $PATH")
if (!file.exists("~/.local/bin/garage")) {
system("wget https://garagehq.deuxfleurs.fr/_releases/v0.5.0/x86_64-unknown-linux-musl/garage -O $HOME/.local/bin/garage")
system("chmod +x $HOME/.local/bin/garage")
}
if (!file.exists("~/.local/bin/minio")) {
system("wget https://dl.min.io/server/minio/release/linux-amd64/minio.RELEASE.2021-11-24T23-19-33Z -O $HOME/.local/bin/minio")
system("chmod +x $HOME/.local/bin/minio")
}
if (!file.exists("~/.local/bin/mknet")) {
system("pip3 install --user git+https://git.deuxfleurs.fr/quentin/mknet")
}
if (!file.exists("~/go/bin/s3lat")) {
system("go install git.deuxfleurs.fr/quentin/s3lat@latest")
}
if (!file.exists("50ms.garage.csv")) {
system("mknet create ./mknet/single-dc.yml")
system("mknet run-all ./mknet/deploy_garage.sh")
system("mknet run dc1:dc1s1 ./mknet/bench_garage.sh")
system("mknet destroy")
}
if (!file.exists("50ms.minio.csv")) {
system("mknet create ./mknet/single-dc.yaml")
system("mknet run-all ./mknet/deploy_minio.py")
system("mknet run dc1:dc1s1 ./mknet/bench_minio.sh")
system("mknet destroy")
}
read_csv("50ms.garage.csv") %>% add_column(daemon="garage 0.5.0") -> a
read_csv("50ms.minio.csv") %>% add_column(daemon="minio RELEASE.2021-11-24T23-19-33Z") -> b
bind_rows(a,b) %>% group_by(daemon,op) %>% summarise(
time_mean = mean(time),
time_max = max(time),
time_min = min(time)
) -> c
ggplot(c, aes(x=op,y=time_mean,fill=daemon,ymin=time_min,ymax=time_max)) +
geom_bar(stat="identity", position=position_dodge(),color="black") +
geom_errorbar(position=position_dodge(.9),width=.2) +
scale_y_continuous(expand=c(0,0))+
coord_flip() +
labs(
x="S3 Endpoint",
y="Latency (ms)",
fill="Daemon",
caption="Get the code to reproduce this graph at https://git.deuxfleurs.fr/quentin/benchmarks",
title="S3 endpoint latency in a simulated geo-distributed cluster",
subtitle="100 measurements, 5 nodes, 100ms RTT + 20ms jitter between nodes\nno contention: latency is due to intra-cluster communications\ncolored bar = mean latency, error bar = min and max") +
theme_classic() +
ggsave("endpoint-latency.png", width=200, height=110, units="mm")

18
mknet/bench_garage.sh Executable file
View file

@ -0,0 +1,18 @@
#!/bin/bash
socat "openssl-listen:4443,\
reuseaddr,\
fork,\
verify=0,\
cert=/tmp/garage.pem" tcp4-connect:localhost:3900 &
sleep 1
export SSL=1
export SSL_INSECURE=1
export REGION=garage
export ENDPOINT=localhost:4443
export AWS_ACCESS_KEY_ID=GK4ea45d2f25091883071e0b73
export AWS_SECRET_ACCESS_KEY=334f7b0ef233571c67b302c5197e6b8c7150da1907e4e530caa82e041d725ecf
/home/quentin/go/bin/bench-garage | tee 50ms.garage.csv

7
mknet/bench_minio.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
export ENDPOINT=[fc00:9a7a:9e::1]:9000
export AWS_ACCESS_KEY_ID=minioadmin
export AWS_SECRET_ACCESS_KEY=minioadmin
s3lat | tee 50ms.minio.csv

61
mknet/deploy_garage.sh Executable file
View file

@ -0,0 +1,61 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'
GARAGE_PATH=garage
STORAGE_PATH=/tmp/garage-testnet
export RPC_SECRET=3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba
if [ -z "$ZONE" ]; then
NODE_STORAGE_PATH=${STORAGE_PATH}/${HOST}
else
NODE_STORAGE_PATH=${STORAGE_PATH}/${ZONE}/${HOST}
fi
BOOTSTRAP_FILE=${STORAGE_PATH}/bootstrap_peer
export GARAGE_CONFIG_FILE=${NODE_STORAGE_PATH}/garage.toml
mkdir -p ${NODE_STORAGE_PATH}
cd ${NODE_STORAGE_PATH}
rm ${BOOTSTRAP_FILE} 2>/dev/null || true
cat > ${GARAGE_CONFIG_FILE} << EOF
metadata_dir = "${NODE_STORAGE_PATH}/meta"
data_dir = "${NODE_STORAGE_PATH}/data"
replication_mode = "3"
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "[${IP}]:3901"
rpc_secret = "${RPC_SECRET}"
bootstrap_peers=[]
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage"
[s3_web]
bind_addr = "[::]:3902"
root_domain = ".web.garage"
index = "index.html"
EOF
RUST_LOG=garage=debug ${GARAGE_PATH} server 2>> ${NODE_STORAGE_PATH}/logs & disown
sleep 2
CONFIG_NODE_FPATH=$(find /tmp/garage-testnet/ -maxdepth 3 -name garage.toml|head -n 1)
SELF_ID=$(${GARAGE_PATH} node id 2>/dev/null)
SHORT_ID=$(echo ${SELF_ID} | cut -c-64)
${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} node connect ${SELF_ID}
${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} layout assign ${SHORT_ID} -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST}
if [ ${CONFIG_NODE_FPATH} == ${GARAGE_CONFIG_FILE} ]; then
sleep 2
${GARAGE_PATH} layout show
${GARAGE_PATH} layout apply --version 1
fi

62
mknet/deploy_minio.py Executable file
View file

@ -0,0 +1,62 @@
#!/usr/bin/env python3
import json, os, sys, time, pathlib, socket, shutil
STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet')
HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt')
UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock')
DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid))
def main():
if int(os.environ['ID']) == 1: leader()
else: follower()
def leader():
shutil.rmtree(STORAGE_PATH, ignore_errors=True)
os.makedirs(STORAGE_PATH)
print(STORAGE_PATH)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(UNIX_SOCK)
sock.listen()
n_serv = os.environ['SIZE']
fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]]
identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ]
print(f"ident: {identities}")
msg = f"{json.dumps(identities)}\n".encode()
[ co.send(msg) for co in fl ]
run_minio(identities)
def follower():
co = None
while True:
time.sleep(1)
try:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(UNIX_SOCK)
co = sock.makefile()
break
except Exception as err:
print('conn failed, wait,', err)
my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() })
sock.send(f"{my_identity}\n".encode())
identities = json.loads(co.readline())
run_minio(identities)
def make_data():
data_path = DATA_PATH(os.environ['ID'])
os.makedirs(data_path)
return data_path
def run_minio(identities):
cmd = f"minio server --console-address ':9001' --address [{os.environ['IP']}]:9000"
for ident in identities:
cmd += f" http://[{ident['ip']}]:9000{ident['path']}"
cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1"
print("launch: ", cmd)
os.system(cmd)
__name__ == '__main__' and main()

34
mknet/single-dc.yml Normal file
View file

@ -0,0 +1,34 @@
links:
- &fiber
bandwidth: 100M
latency: 50ms
jitter: 10ms
zones:
- &dc1
name: dc1
internal: *fiber
external: *fiber
servers:
- name: dc1s1
zone: *dc1
- name: dc1s2
zone: *dc1
- name: dc1s3
zone: *dc1
- name: dc1s4
zone: *dc1
- name: dc1s5
zone: *dc1
global:
subnet:
base: 'fc00:9a7a:9e::'
local: 64
zone: 16
latency-offset: 3ms
upstream:
ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff
conn: *fiber