WIP reproduce stats
This commit is contained in:
commit
df4116b174
|
|
@ -0,0 +1,67 @@
|
|||
#!/usr/bin/env Rscript
|
||||
.libPaths(c("~/R", .libPaths()))
|
||||
repo <- "https://cran.r-project.org"
|
||||
|
||||
if (!require(tidyverse)) install.packages("tidyverse", repos=repo); library(tidyverse)
|
||||
if (!require(svglite)) install.packages("svglite", repos=repo); library(svglite)
|
||||
|
||||
system("mkdir -p $HOME/.local/bin")
|
||||
Sys.getenv("HOME") -> home
|
||||
Sys.setenv(PATH = paste(paste(home, "/go/bin", sep=""), paste(home, "/.local/bin", sep=""), Sys.getenv("PATH"), sep=":"))
|
||||
system("echo $PATH")
|
||||
|
||||
if (!file.exists("~/.local/bin/garage")) {
|
||||
system("wget https://garagehq.deuxfleurs.fr/_releases/v0.5.0/x86_64-unknown-linux-musl/garage -O $HOME/.local/bin/garage")
|
||||
system("chmod +x $HOME/.local/bin/garage")
|
||||
}
|
||||
|
||||
if (!file.exists("~/.local/bin/minio")) {
|
||||
system("wget https://dl.min.io/server/minio/release/linux-amd64/minio.RELEASE.2021-11-24T23-19-33Z -O $HOME/.local/bin/minio")
|
||||
system("chmod +x $HOME/.local/bin/minio")
|
||||
}
|
||||
|
||||
if (!file.exists("~/.local/bin/mknet")) {
|
||||
system("pip3 install --user git+https://git.deuxfleurs.fr/quentin/mknet")
|
||||
}
|
||||
|
||||
if (!file.exists("~/go/bin/s3lat")) {
|
||||
system("go install git.deuxfleurs.fr/quentin/s3lat@latest")
|
||||
}
|
||||
|
||||
if (!file.exists("50ms.garage.csv")) {
|
||||
system("mknet create ./mknet/single-dc.yml")
|
||||
system("mknet run-all ./mknet/deploy_garage.sh")
|
||||
system("mknet run dc1:dc1s1 ./mknet/bench_garage.sh")
|
||||
system("mknet destroy")
|
||||
}
|
||||
|
||||
if (!file.exists("50ms.minio.csv")) {
|
||||
system("mknet create ./mknet/single-dc.yaml")
|
||||
system("mknet run-all ./mknet/deploy_minio.py")
|
||||
system("mknet run dc1:dc1s1 ./mknet/bench_minio.sh")
|
||||
system("mknet destroy")
|
||||
}
|
||||
|
||||
read_csv("50ms.garage.csv") %>% add_column(daemon="garage 0.5.0") -> a
|
||||
read_csv("50ms.minio.csv") %>% add_column(daemon="minio RELEASE.2021-11-24T23-19-33Z") -> b
|
||||
bind_rows(a,b) %>% group_by(daemon,op) %>% summarise(
|
||||
time_mean = mean(time),
|
||||
time_max = max(time),
|
||||
time_min = min(time)
|
||||
) -> c
|
||||
|
||||
ggplot(c, aes(x=op,y=time_mean,fill=daemon,ymin=time_min,ymax=time_max)) +
|
||||
geom_bar(stat="identity", position=position_dodge(),color="black") +
|
||||
geom_errorbar(position=position_dodge(.9),width=.2) +
|
||||
scale_y_continuous(expand=c(0,0))+
|
||||
coord_flip() +
|
||||
labs(
|
||||
x="S3 Endpoint",
|
||||
y="Latency (ms)",
|
||||
fill="Daemon",
|
||||
caption="Get the code to reproduce this graph at https://git.deuxfleurs.fr/quentin/benchmarks",
|
||||
title="S3 endpoint latency in a simulated geo-distributed cluster",
|
||||
subtitle="100 measurements, 5 nodes, 100ms RTT + 20ms jitter between nodes\nno contention: latency is due to intra-cluster communications\ncolored bar = mean latency, error bar = min and max") +
|
||||
theme_classic() +
|
||||
ggsave("endpoint-latency.png", width=200, height=110, units="mm")
|
||||
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
#!/bin/bash
|
||||
|
||||
socat "openssl-listen:4443,\
|
||||
reuseaddr,\
|
||||
fork,\
|
||||
verify=0,\
|
||||
cert=/tmp/garage.pem" tcp4-connect:localhost:3900 &
|
||||
|
||||
sleep 1
|
||||
|
||||
export SSL=1
|
||||
export SSL_INSECURE=1
|
||||
export REGION=garage
|
||||
export ENDPOINT=localhost:4443
|
||||
export AWS_ACCESS_KEY_ID=GK4ea45d2f25091883071e0b73
|
||||
export AWS_SECRET_ACCESS_KEY=334f7b0ef233571c67b302c5197e6b8c7150da1907e4e530caa82e041d725ecf
|
||||
|
||||
/home/quentin/go/bin/bench-garage | tee 50ms.garage.csv
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
export ENDPOINT=[fc00:9a7a:9e::1]:9000
|
||||
export AWS_ACCESS_KEY_ID=minioadmin
|
||||
export AWS_SECRET_ACCESS_KEY=minioadmin
|
||||
|
||||
s3lat | tee 50ms.minio.csv
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
IFS=$'\n\t'
|
||||
|
||||
GARAGE_PATH=garage
|
||||
STORAGE_PATH=/tmp/garage-testnet
|
||||
export RPC_SECRET=3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba
|
||||
|
||||
if [ -z "$ZONE" ]; then
|
||||
NODE_STORAGE_PATH=${STORAGE_PATH}/${HOST}
|
||||
else
|
||||
NODE_STORAGE_PATH=${STORAGE_PATH}/${ZONE}/${HOST}
|
||||
fi
|
||||
BOOTSTRAP_FILE=${STORAGE_PATH}/bootstrap_peer
|
||||
export GARAGE_CONFIG_FILE=${NODE_STORAGE_PATH}/garage.toml
|
||||
|
||||
|
||||
mkdir -p ${NODE_STORAGE_PATH}
|
||||
cd ${NODE_STORAGE_PATH}
|
||||
rm ${BOOTSTRAP_FILE} 2>/dev/null || true
|
||||
|
||||
cat > ${GARAGE_CONFIG_FILE} << EOF
|
||||
metadata_dir = "${NODE_STORAGE_PATH}/meta"
|
||||
data_dir = "${NODE_STORAGE_PATH}/data"
|
||||
|
||||
replication_mode = "3"
|
||||
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_public_addr = "[${IP}]:3901"
|
||||
rpc_secret = "${RPC_SECRET}"
|
||||
|
||||
bootstrap_peers=[]
|
||||
|
||||
[s3_api]
|
||||
s3_region = "garage"
|
||||
api_bind_addr = "[::]:3900"
|
||||
root_domain = ".s3.garage"
|
||||
|
||||
[s3_web]
|
||||
bind_addr = "[::]:3902"
|
||||
root_domain = ".web.garage"
|
||||
index = "index.html"
|
||||
EOF
|
||||
|
||||
RUST_LOG=garage=debug ${GARAGE_PATH} server 2>> ${NODE_STORAGE_PATH}/logs & disown
|
||||
sleep 2
|
||||
|
||||
CONFIG_NODE_FPATH=$(find /tmp/garage-testnet/ -maxdepth 3 -name garage.toml|head -n 1)
|
||||
|
||||
SELF_ID=$(${GARAGE_PATH} node id 2>/dev/null)
|
||||
SHORT_ID=$(echo ${SELF_ID} | cut -c-64)
|
||||
|
||||
${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} node connect ${SELF_ID}
|
||||
${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} layout assign ${SHORT_ID} -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST}
|
||||
|
||||
if [ ${CONFIG_NODE_FPATH} == ${GARAGE_CONFIG_FILE} ]; then
|
||||
sleep 2
|
||||
${GARAGE_PATH} layout show
|
||||
${GARAGE_PATH} layout apply --version 1
|
||||
fi
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
#!/usr/bin/env python3
|
||||
import json, os, sys, time, pathlib, socket, shutil
|
||||
|
||||
STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet')
|
||||
HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt')
|
||||
UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock')
|
||||
DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid))
|
||||
|
||||
def main():
|
||||
if int(os.environ['ID']) == 1: leader()
|
||||
else: follower()
|
||||
|
||||
def leader():
|
||||
shutil.rmtree(STORAGE_PATH, ignore_errors=True)
|
||||
os.makedirs(STORAGE_PATH)
|
||||
print(STORAGE_PATH)
|
||||
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
sock.bind(UNIX_SOCK)
|
||||
sock.listen()
|
||||
|
||||
n_serv = os.environ['SIZE']
|
||||
fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]]
|
||||
|
||||
identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ]
|
||||
print(f"ident: {identities}")
|
||||
msg = f"{json.dumps(identities)}\n".encode()
|
||||
[ co.send(msg) for co in fl ]
|
||||
|
||||
run_minio(identities)
|
||||
|
||||
def follower():
|
||||
co = None
|
||||
while True:
|
||||
time.sleep(1)
|
||||
try:
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
sock.connect(UNIX_SOCK)
|
||||
co = sock.makefile()
|
||||
break
|
||||
except Exception as err:
|
||||
print('conn failed, wait,', err)
|
||||
my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() })
|
||||
sock.send(f"{my_identity}\n".encode())
|
||||
identities = json.loads(co.readline())
|
||||
|
||||
run_minio(identities)
|
||||
|
||||
def make_data():
|
||||
data_path = DATA_PATH(os.environ['ID'])
|
||||
os.makedirs(data_path)
|
||||
return data_path
|
||||
|
||||
def run_minio(identities):
|
||||
cmd = f"minio server --console-address ':9001' --address [{os.environ['IP']}]:9000"
|
||||
for ident in identities:
|
||||
cmd += f" http://[{ident['ip']}]:9000{ident['path']}"
|
||||
cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1"
|
||||
print("launch: ", cmd)
|
||||
os.system(cmd)
|
||||
|
||||
__name__ == '__main__' and main()
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
links:
|
||||
- &fiber
|
||||
bandwidth: 100M
|
||||
latency: 50ms
|
||||
jitter: 10ms
|
||||
|
||||
zones:
|
||||
- &dc1
|
||||
name: dc1
|
||||
internal: *fiber
|
||||
external: *fiber
|
||||
|
||||
servers:
|
||||
- name: dc1s1
|
||||
zone: *dc1
|
||||
- name: dc1s2
|
||||
zone: *dc1
|
||||
- name: dc1s3
|
||||
zone: *dc1
|
||||
- name: dc1s4
|
||||
zone: *dc1
|
||||
- name: dc1s5
|
||||
zone: *dc1
|
||||
|
||||
global:
|
||||
subnet:
|
||||
base: 'fc00:9a7a:9e::'
|
||||
local: 64
|
||||
zone: 16
|
||||
latency-offset: 3ms
|
||||
upstream:
|
||||
ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff
|
||||
conn: *fiber
|
||||
|
||||
Loading…
Reference in New Issue