From df4116b1742d19b80fe157950d209ac1b4ae5834 Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 8 Dec 2021 17:09:17 +0100 Subject: [PATCH] WIP reproduce stats --- garage-latency-simple.R | 67 +++++++++++++++++++++++++++++++++++++++++ mknet/bench_garage.sh | 18 +++++++++++ mknet/bench_minio.sh | 7 +++++ mknet/deploy_garage.sh | 61 +++++++++++++++++++++++++++++++++++++ mknet/deploy_minio.py | 62 ++++++++++++++++++++++++++++++++++++++ mknet/single-dc.yml | 34 +++++++++++++++++++++ 6 files changed, 249 insertions(+) create mode 100755 garage-latency-simple.R create mode 100755 mknet/bench_garage.sh create mode 100755 mknet/bench_minio.sh create mode 100755 mknet/deploy_garage.sh create mode 100755 mknet/deploy_minio.py create mode 100644 mknet/single-dc.yml diff --git a/garage-latency-simple.R b/garage-latency-simple.R new file mode 100755 index 0000000..d24b15a --- /dev/null +++ b/garage-latency-simple.R @@ -0,0 +1,67 @@ +#!/usr/bin/env Rscript +.libPaths(c("~/R", .libPaths())) +repo <- "https://cran.r-project.org" + +if (!require(tidyverse)) install.packages("tidyverse", repos=repo); library(tidyverse) +if (!require(svglite)) install.packages("svglite", repos=repo); library(svglite) + +system("mkdir -p $HOME/.local/bin") +Sys.getenv("HOME") -> home +Sys.setenv(PATH = paste(paste(home, "/go/bin", sep=""), paste(home, "/.local/bin", sep=""), Sys.getenv("PATH"), sep=":")) +system("echo $PATH") + +if (!file.exists("~/.local/bin/garage")) { + system("wget https://garagehq.deuxfleurs.fr/_releases/v0.5.0/x86_64-unknown-linux-musl/garage -O $HOME/.local/bin/garage") + system("chmod +x $HOME/.local/bin/garage") +} + +if (!file.exists("~/.local/bin/minio")) { + system("wget https://dl.min.io/server/minio/release/linux-amd64/minio.RELEASE.2021-11-24T23-19-33Z -O $HOME/.local/bin/minio") + system("chmod +x $HOME/.local/bin/minio") +} + +if (!file.exists("~/.local/bin/mknet")) { + system("pip3 install --user git+https://git.deuxfleurs.fr/quentin/mknet") +} + +if (!file.exists("~/go/bin/s3lat")) { + system("go install git.deuxfleurs.fr/quentin/s3lat@latest") +} + +if (!file.exists("50ms.garage.csv")) { + system("mknet create ./mknet/single-dc.yml") + system("mknet run-all ./mknet/deploy_garage.sh") + system("mknet run dc1:dc1s1 ./mknet/bench_garage.sh") + system("mknet destroy") +} + +if (!file.exists("50ms.minio.csv")) { + system("mknet create ./mknet/single-dc.yaml") + system("mknet run-all ./mknet/deploy_minio.py") + system("mknet run dc1:dc1s1 ./mknet/bench_minio.sh") + system("mknet destroy") +} + +read_csv("50ms.garage.csv") %>% add_column(daemon="garage 0.5.0") -> a +read_csv("50ms.minio.csv") %>% add_column(daemon="minio RELEASE.2021-11-24T23-19-33Z") -> b +bind_rows(a,b) %>% group_by(daemon,op) %>% summarise( + time_mean = mean(time), + time_max = max(time), + time_min = min(time) +) -> c + +ggplot(c, aes(x=op,y=time_mean,fill=daemon,ymin=time_min,ymax=time_max)) + + geom_bar(stat="identity", position=position_dodge(),color="black") + + geom_errorbar(position=position_dodge(.9),width=.2) + + scale_y_continuous(expand=c(0,0))+ + coord_flip() + + labs( + x="S3 Endpoint", + y="Latency (ms)", + fill="Daemon", + caption="Get the code to reproduce this graph at https://git.deuxfleurs.fr/quentin/benchmarks", + title="S3 endpoint latency in a simulated geo-distributed cluster", + subtitle="100 measurements, 5 nodes, 100ms RTT + 20ms jitter between nodes\nno contention: latency is due to intra-cluster communications\ncolored bar = mean latency, error bar = min and max") + + theme_classic() + + ggsave("endpoint-latency.png", width=200, height=110, units="mm") + diff --git a/mknet/bench_garage.sh b/mknet/bench_garage.sh new file mode 100755 index 0000000..3f6e5c0 --- /dev/null +++ b/mknet/bench_garage.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +socat "openssl-listen:4443,\ +reuseaddr,\ +fork,\ +verify=0,\ +cert=/tmp/garage.pem" tcp4-connect:localhost:3900 & + +sleep 1 + +export SSL=1 +export SSL_INSECURE=1 +export REGION=garage +export ENDPOINT=localhost:4443 +export AWS_ACCESS_KEY_ID=GK4ea45d2f25091883071e0b73 +export AWS_SECRET_ACCESS_KEY=334f7b0ef233571c67b302c5197e6b8c7150da1907e4e530caa82e041d725ecf + +/home/quentin/go/bin/bench-garage | tee 50ms.garage.csv diff --git a/mknet/bench_minio.sh b/mknet/bench_minio.sh new file mode 100755 index 0000000..eb00758 --- /dev/null +++ b/mknet/bench_minio.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +export ENDPOINT=[fc00:9a7a:9e::1]:9000 +export AWS_ACCESS_KEY_ID=minioadmin +export AWS_SECRET_ACCESS_KEY=minioadmin + +s3lat | tee 50ms.minio.csv diff --git a/mknet/deploy_garage.sh b/mknet/deploy_garage.sh new file mode 100755 index 0000000..0dc9b38 --- /dev/null +++ b/mknet/deploy_garage.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -euo pipefail +IFS=$'\n\t' + +GARAGE_PATH=garage +STORAGE_PATH=/tmp/garage-testnet +export RPC_SECRET=3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba + +if [ -z "$ZONE" ]; then + NODE_STORAGE_PATH=${STORAGE_PATH}/${HOST} +else + NODE_STORAGE_PATH=${STORAGE_PATH}/${ZONE}/${HOST} +fi +BOOTSTRAP_FILE=${STORAGE_PATH}/bootstrap_peer +export GARAGE_CONFIG_FILE=${NODE_STORAGE_PATH}/garage.toml + + +mkdir -p ${NODE_STORAGE_PATH} +cd ${NODE_STORAGE_PATH} +rm ${BOOTSTRAP_FILE} 2>/dev/null || true + +cat > ${GARAGE_CONFIG_FILE} << EOF +metadata_dir = "${NODE_STORAGE_PATH}/meta" +data_dir = "${NODE_STORAGE_PATH}/data" + +replication_mode = "3" + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "[${IP}]:3901" +rpc_secret = "${RPC_SECRET}" + +bootstrap_peers=[] + +[s3_api] +s3_region = "garage" +api_bind_addr = "[::]:3900" +root_domain = ".s3.garage" + +[s3_web] +bind_addr = "[::]:3902" +root_domain = ".web.garage" +index = "index.html" +EOF + +RUST_LOG=garage=debug ${GARAGE_PATH} server 2>> ${NODE_STORAGE_PATH}/logs & disown +sleep 2 + +CONFIG_NODE_FPATH=$(find /tmp/garage-testnet/ -maxdepth 3 -name garage.toml|head -n 1) + +SELF_ID=$(${GARAGE_PATH} node id 2>/dev/null) +SHORT_ID=$(echo ${SELF_ID} | cut -c-64) + +${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} node connect ${SELF_ID} +${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} layout assign ${SHORT_ID} -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST} + +if [ ${CONFIG_NODE_FPATH} == ${GARAGE_CONFIG_FILE} ]; then + sleep 2 + ${GARAGE_PATH} layout show + ${GARAGE_PATH} layout apply --version 1 +fi diff --git a/mknet/deploy_minio.py b/mknet/deploy_minio.py new file mode 100755 index 0000000..4488974 --- /dev/null +++ b/mknet/deploy_minio.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import json, os, sys, time, pathlib, socket, shutil + +STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet') +HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt') +UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock') +DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid)) + +def main(): + if int(os.environ['ID']) == 1: leader() + else: follower() + +def leader(): + shutil.rmtree(STORAGE_PATH, ignore_errors=True) + os.makedirs(STORAGE_PATH) + print(STORAGE_PATH) + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(UNIX_SOCK) + sock.listen() + + n_serv = os.environ['SIZE'] + fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]] + + identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ] + print(f"ident: {identities}") + msg = f"{json.dumps(identities)}\n".encode() + [ co.send(msg) for co in fl ] + + run_minio(identities) + +def follower(): + co = None + while True: + time.sleep(1) + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(UNIX_SOCK) + co = sock.makefile() + break + except Exception as err: + print('conn failed, wait,', err) + my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() }) + sock.send(f"{my_identity}\n".encode()) + identities = json.loads(co.readline()) + + run_minio(identities) + +def make_data(): + data_path = DATA_PATH(os.environ['ID']) + os.makedirs(data_path) + return data_path + +def run_minio(identities): + cmd = f"minio server --console-address ':9001' --address [{os.environ['IP']}]:9000" + for ident in identities: + cmd += f" http://[{ident['ip']}]:9000{ident['path']}" + cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1" + print("launch: ", cmd) + os.system(cmd) + +__name__ == '__main__' and main() diff --git a/mknet/single-dc.yml b/mknet/single-dc.yml new file mode 100644 index 0000000..39427aa --- /dev/null +++ b/mknet/single-dc.yml @@ -0,0 +1,34 @@ +links: + - &fiber + bandwidth: 100M + latency: 50ms + jitter: 10ms + +zones: + - &dc1 + name: dc1 + internal: *fiber + external: *fiber + +servers: + - name: dc1s1 + zone: *dc1 + - name: dc1s2 + zone: *dc1 + - name: dc1s3 + zone: *dc1 + - name: dc1s4 + zone: *dc1 + - name: dc1s5 + zone: *dc1 + +global: + subnet: + base: 'fc00:9a7a:9e::' + local: 64 + zone: 16 + latency-offset: 3ms + upstream: + ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff + conn: *fiber +