From cc2b090149c47cbac47bbfe872ee5a04c49a00e3 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Mon, 29 Nov 2021 23:51:09 +0100 Subject: [PATCH] WIP --- 50ms.garage.csv | 501 ++++++++++++++++++++++++++++++++++ README.md | 7 + analyse.R | 6 + instrumented/deploy_garage.sh | 3 - instrumented/deploy_minio.py | 57 ++++ instrumented/runner.sh | 15 + main.py | 2 + single-dc.yml | 33 +++ 8 files changed, 621 insertions(+), 3 deletions(-) create mode 100644 50ms.garage.csv create mode 100644 analyse.R create mode 100755 instrumented/deploy_minio.py create mode 100755 instrumented/runner.sh create mode 100644 single-dc.yml diff --git a/50ms.garage.csv b/50ms.garage.csv new file mode 100644 index 0000000..fdf03a6 --- /dev/null +++ b/50ms.garage.csv @@ -0,0 +1,501 @@ +op,time +listbuckets,202.111138 +listbuckets,43.940584 +listbuckets,181.316207 +listbuckets,19.614464 +listbuckets,202.478714 +listbuckets,136.910843 +listbuckets,163.83813 +listbuckets,6.058236 +listbuckets,85.429565 +listbuckets,114.172791 +listbuckets,74.642451 +listbuckets,185.718226 +listbuckets,167.826023 +listbuckets,149.187267 +listbuckets,133.087361 +listbuckets,84.117499 +listbuckets,144.38981 +listbuckets,23.85664 +listbuckets,87.296531 +listbuckets,67.410471 +listbuckets,84.965746 +listbuckets,49.504764 +listbuckets,137.740524 +listbuckets,170.798293 +listbuckets,95.475103 +listbuckets,137.597302 +listbuckets,63.782028 +listbuckets,29.721971 +listbuckets,28.081703 +listbuckets,69.563921 +listbuckets,5.206663 +listbuckets,18.225493 +listbuckets,45.392384 +listbuckets,162.435983 +listbuckets,50.777425 +listbuckets,37.839457 +listbuckets,79.086669 +listbuckets,87.751924 +listbuckets,134.904993 +listbuckets,42.232689 +listbuckets,192.142975 +listbuckets,140.725668 +listbuckets,95.926909 +listbuckets,51.227748 +listbuckets,177.170003 +listbuckets,144.06313 +listbuckets,102.662758 +listbuckets,32.749906 +listbuckets,122.617287 +listbuckets,169.247085 +listbuckets,12.547939 +listbuckets,140.197528 +listbuckets,7.936283 +listbuckets,46.104653 +listbuckets,17.26822 +listbuckets,103.748224 +listbuckets,178.618347 +listbuckets,21.52861 +listbuckets,45.549531 +listbuckets,183.945418 +listbuckets,115.780897 +listbuckets,174.858715 +listbuckets,86.042806 +listbuckets,31.345205 +listbuckets,12.83907 +listbuckets,198.613903 +listbuckets,188.966298 +listbuckets,45.879156 +listbuckets,145.656468 +listbuckets,178.692096 +listbuckets,139.07375 +listbuckets,201.000343 +listbuckets,52.520817 +listbuckets,31.036891 +listbuckets,39.697006 +listbuckets,141.790216 +listbuckets,165.41757 +listbuckets,202.853394 +listbuckets,194.009791 +listbuckets,51.369246 +listbuckets,173.986272 +listbuckets,11.476469 +listbuckets,17.626068 +listbuckets,47.418342 +listbuckets,157.026306 +listbuckets,50.140819 +listbuckets,104.12086 +listbuckets,92.402613 +listbuckets,75.890084 +listbuckets,132.398325 +listbuckets,44.180258 +listbuckets,93.321994 +listbuckets,71.967464 +listbuckets,64.616629 +listbuckets,84.258466 +listbuckets,195.606576 +listbuckets,118.320338 +listbuckets,104.692974 +listbuckets,74.44628 +listbuckets,28.309184 +putobject,165.46463 +putobject,222.517763 +putobject,111.937486 +putobject,184.887542 +putobject,126.217546 +putobject,237.106337 +putobject,116.128475 +putobject,189.31095 +putobject,113.380511 +putobject,103.315545 +putobject,247.080631 +putobject,186.276523 +putobject,99.660792 +putobject,117.760306 +putobject,109.472237 +putobject,264.241006 +putobject,121.447772 +putobject,117.987948 +putobject,111.855782 +putobject,115.530352 +putobject,326.730637 +putobject,120.42312 +putobject,221.54514 +putobject,110.206819 +putobject,123.908553 +putobject,117.173665 +putobject,115.430233 +putobject,110.628507 +putobject,237.560348 +putobject,115.31804 +putobject,308.395616 +putobject,107.843702 +putobject,111.375833 +putobject,96.310716 +putobject,112.016026 +putobject,239.727462 +putobject,112.919547 +putobject,111.346417 +putobject,329.578191 +putobject,106.91322 +putobject,95.694077 +putobject,278.426107 +putobject,105.462491 +putobject,181.24935 +putobject,107.048446 +putobject,218.381427 +putobject,170.658909 +putobject,222.801469 +putobject,120.682382 +putobject,165.640233 +putobject,189.454552 +putobject,152.842501 +putobject,106.93962 +putobject,255.688236 +putobject,111.958125 +putobject,130.607382 +putobject,202.151204 +putobject,114.005915 +putobject,215.592064 +putobject,339.66065 +putobject,116.144346 +putobject,211.735699 +putobject,127.648517 +putobject,105.163645 +putobject,205.664038 +putobject,117.612316 +putobject,95.377638 +putobject,103.966738 +putobject,117.708519 +putobject,198.141979 +putobject,114.142965 +putobject,236.392525 +putobject,179.734151 +putobject,116.104981 +putobject,176.715002 +putobject,116.993234 +putobject,194.191116 +putobject,103.240473 +putobject,149.059204 +putobject,111.387445 +putobject,206.070688 +putobject,116.469762 +putobject,348.528368 +putobject,112.547413 +putobject,217.546766 +putobject,112.896413 +putobject,215.344494 +putobject,115.486139 +putobject,118.703142 +putobject,213.602786 +putobject,109.749772 +putobject,112.005876 +putobject,112.576999 +putobject,115.241835 +putobject,216.114253 +putobject,122.455961 +putobject,107.242653 +putobject,152.564715 +putobject,122.086712 +putobject,199.452321 +listobjects,300.581365 +listobjects,277.790833 +listobjects,300.438323 +listobjects,306.401268 +listobjects,247.070642 +listobjects,265.460266 +listobjects,307.833962 +listobjects,278.348249 +listobjects,289.753623 +listobjects,428.597858 +listobjects,291.670485 +listobjects,291.750707 +listobjects,337.027161 +listobjects,246.554724 +listobjects,257.611418 +listobjects,320.019725 +listobjects,257.59075 +listobjects,295.037894 +listobjects,283.894836 +listobjects,247.228281 +listobjects,253.136202 +listobjects,337.787051 +listobjects,258.82629 +listobjects,268.927815 +listobjects,321.913973 +listobjects,281.985389 +listobjects,260.410992 +listobjects,287.980444 +listobjects,275.53983 +listobjects,269.558019 +listobjects,287.118302 +listobjects,290.645774 +listobjects,285.738097 +listobjects,392.130411 +listobjects,289.995953 +listobjects,262.023637 +listobjects,325.831103 +listobjects,278.981469 +listobjects,284.476107 +listobjects,314.008409 +listobjects,247.46578 +listobjects,241.607423 +listobjects,280.720744 +listobjects,251.493039 +listobjects,251.274956 +listobjects,308.848053 +listobjects,247.605826 +listobjects,254.399835 +listobjects,272.523456 +listobjects,295.263441 +listobjects,287.200498 +listobjects,321.094731 +listobjects,298.581687 +listobjects,267.287678 +listobjects,334.118082 +listobjects,294.63522 +listobjects,396.182216 +listobjects,324.845486 +listobjects,277.918475 +listobjects,276.752988 +listobjects,286.609247 +listobjects,237.788439 +listobjects,291.572429 +listobjects,289.931931 +listobjects,292.7288 +listobjects,278.329784 +listobjects,312.4204 +listobjects,250.387574 +listobjects,266.618319 +listobjects,294.083115 +listobjects,282.236044 +listobjects,246.433896 +listobjects,259.166735 +listobjects,283.552347 +listobjects,262.037944 +listobjects,438.74009 +listobjects,261.966819 +listobjects,272.358954 +listobjects,328.634834 +listobjects,249.669345 +listobjects,271.167948 +listobjects,314.412415 +listobjects,247.691167 +listobjects,283.680419 +listobjects,343.269156 +listobjects,251.896493 +listobjects,252.494486 +listobjects,422.943899 +listobjects,249.298261 +listobjects,261.622207 +listobjects,297.580581 +listobjects,283.851304 +listobjects,271.55979 +listobjects,285.568215 +listobjects,230.497899 +listobjects,263.590013 +listobjects,328.976692 +listobjects,263.056663 +listobjects,290.204427 +listobjects,313.846942 +getobject,182.785981 +getobject,109.499628 +getobject,106.650121 +getobject,121.295002 +getobject,117.047546 +getobject,121.821129 +getobject,99.500457 +getobject,110.204614 +getobject,90.665673 +getobject,103.448016 +getobject,109.513215 +getobject,109.836517 +getobject,108.865037 +getobject,109.142191 +getobject,115.979895 +getobject,114.514819 +getobject,261.303813 +getobject,97.933911 +getobject,105.707616 +getobject,95.21515 +getobject,114.97932 +getobject,97.167349 +getobject,109.995088 +getobject,116.094451 +getobject,92.307803 +getobject,98.966066 +getobject,92.048943 +getobject,108.609352 +getobject,108.804713 +getobject,98.891615 +getobject,115.83029 +getobject,243.530767 +getobject,210.53295 +getobject,112.15031 +getobject,111.292435 +getobject,114.00824 +getobject,104.009179 +getobject,95.101244 +getobject,107.050751 +getobject,120.16432 +getobject,194.587857 +getobject,241.974629 +getobject,106.301621 +getobject,108.892679 +getobject,108.393894 +getobject,246.919505 +getobject,107.829215 +getobject,102.11495 +getobject,103.701736 +getobject,101.141817 +getobject,95.347592 +getobject,103.559998 +getobject,88.331191 +getobject,98.132176 +getobject,108.62405 +getobject,105.357643 +getobject,113.917848 +getobject,97.43226 +getobject,93.802174 +getobject,98.555798 +getobject,255.646637 +getobject,106.634181 +getobject,106.455823 +getobject,111.535305 +getobject,96.665838 +getobject,109.615498 +getobject,105.223109 +getobject,109.366968 +getobject,103.517893 +getobject,102.918258 +getobject,100.815486 +getobject,108.270276 +getobject,96.698476 +getobject,109.806247 +getobject,115.544536 +getobject,112.49002 +getobject,259.839035 +getobject,111.031397 +getobject,106.06039 +getobject,112.722872 +getobject,113.953012 +getobject,103.298047 +getobject,99.28177 +getobject,112.432912 +getobject,96.173452 +getobject,99.882016 +getobject,111.264639 +getobject,108.417085 +getobject,117.353102 +getobject,95.225466 +getobject,111.075511 +getobject,264.20257 +getobject,117.333244 +getobject,120.125003 +getobject,108.663392 +getobject,119.961413 +getobject,95.005941 +getobject,95.4336 +getobject,115.280757 +getobject,97.201088 +removeobject,222.573682 +removeobject,231.761917 +removeobject,442.585842 +removeobject,234.382532 +removeobject,387.512105 +removeobject,305.628182 +removeobject,289.502486 +removeobject,231.169395 +removeobject,316.098005 +removeobject,209.524123 +removeobject,424.370175 +removeobject,216.714103 +removeobject,375.594149 +removeobject,223.780047 +removeobject,350.67269 +removeobject,336.975443 +removeobject,323.65556 +removeobject,292.523519 +removeobject,231.739565 +removeobject,472.605248 +removeobject,205.028165 +removeobject,297.558856 +removeobject,354.25723 +removeobject,358.617621 +removeobject,231.66819 +removeobject,218.014656 +removeobject,293.081194 +removeobject,220.378333 +removeobject,461.275468 +removeobject,294.197941 +removeobject,199.614791 +removeobject,294.474434 +removeobject,418.385018 +removeobject,230.544922 +removeobject,327.569235 +removeobject,218.21218 +removeobject,307.861353 +removeobject,328.311953 +removeobject,216.075602 +removeobject,210.143125 +removeobject,208.936679 +removeobject,190.425894 +removeobject,224.586404 +removeobject,337.963656 +removeobject,444.272797 +removeobject,301.920388 +removeobject,340.421272 +removeobject,338.346501 +removeobject,296.788527 +removeobject,207.559962 +removeobject,457.649247 +removeobject,300.037582 +removeobject,321.593867 +removeobject,292.55592 +removeobject,309.367726 +removeobject,255.574575 +removeobject,426.947087 +removeobject,265.768115 +removeobject,329.29239 +removeobject,308.343145 +removeobject,359.263264 +removeobject,203.957307 +removeobject,365.564661 +removeobject,237.358118 +removeobject,317.935617 +removeobject,313.741912 +removeobject,312.306754 +removeobject,323.298865 +removeobject,309.122932 +removeobject,202.639531 +removeobject,195.346255 +removeobject,218.604323 +removeobject,240.143875 +removeobject,341.057908 +removeobject,307.301071 +removeobject,326.504098 +removeobject,309.864819 +removeobject,300.616459 +removeobject,232.943336 +removeobject,220.08092 +removeobject,237.389609 +removeobject,334.716365 +removeobject,291.190192 +removeobject,206.634058 +removeobject,291.836125 +removeobject,307.390831 +removeobject,197.95132 +removeobject,379.646215 +removeobject,211.924509 +removeobject,244.664849 +removeobject,348.966198 +removeobject,316.319435 +removeobject,291.416401 +removeobject,215.855116 +removeobject,329.053698 +removeobject,322.553745 +removeobject,316.627999 +removeobject,311.277814 +removeobject,330.250545 +removeobject,221.223845 diff --git a/README.md b/README.md index 014d2c4..0438d14 100644 --- a/README.md +++ b/README.md @@ -33,3 +33,10 @@ sudo chmod +x /usr/local/bin/garage garage help ``` + +### Minio + +``` +sudo wget https://dl.min.io/server/minio/release/linux-amd64/minio -O /usr/local/bin/minio +sudo chmod +x /usr/local/bin/minio +``` diff --git a/analyse.R b/analyse.R new file mode 100644 index 0000000..6ddf845 --- /dev/null +++ b/analyse.R @@ -0,0 +1,6 @@ +library(tidyverse) +read_csv("50ms.garage.csv") -> a +ggplot(a, aes(x=op,y=time)) + + geom_violin(adjust = .5,draw_quantiles = c(0.25, 0.5, 0.75)) + + labs(title="Garage v0.5.0 Endpoint Latency\n(5 nodes, 100ms RTT between nodes, 20ms jitter, bars = Q1,median,Q3)", x="S3 Endpoint Name", y="Latency (ms)") + + theme_classic() diff --git a/instrumented/deploy_garage.sh b/instrumented/deploy_garage.sh index 14f36a6..10cd50b 100755 --- a/instrumented/deploy_garage.sh +++ b/instrumented/deploy_garage.sh @@ -51,9 +51,6 @@ CONFIG_NODE_FPATH=($(find /tmp/garage-testnet/ -maxdepth 3 -name garage.toml)) SELF_ID=$(${GARAGE_PATH} node id 2>/dev/null) SHORT_ID=$(echo ${SELF_ID} | cut -c-64) -echo ===DEBUG=== -echo $CONFIG_NODE_FPATH -echo ${CONFIG_NODE_FPATH[0]} ${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} node connect ${SELF_ID} ${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} layout assign ${SHORT_ID} -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST} diff --git a/instrumented/deploy_minio.py b/instrumented/deploy_minio.py new file mode 100755 index 0000000..cbacb1f --- /dev/null +++ b/instrumented/deploy_minio.py @@ -0,0 +1,57 @@ +#!/bin/python3 +import yaml, os, sys, time, pathlib, unshare + +HOST_MARKER = "__AUTO_MKNET_MINIO_DEPLOY_DYYycw9Z5c5PvylU" +storage_path = "/tmp/minio-testnet" + +# Remove any entry we put in /etc/hosts +def cleanup(): + with open('/etc/hosts', 'r') as fin: + with open('/etc/hosts', 'w') as fout: + fout.writelines(filter(lambda line: not HOST_MARKER in line, fin)) + +def wait(p): + print("wait", p) + d = os.path.join(storage_path, p) + while not os.path.exists(d): + time.sleep(1) + #os.unlink(d) +def post(p): + print("post", p) + pathlib.Path(os.path.join(storage_path, p)).touch() + +## Create a working directory +me = int(os.environ['ID']) +node_storage_path = os.path.join(storage_path, str(me)) +os.makedirs(node_storage_path, exist_ok=True) + +## Writing down some info for others +for v in [ 'ZONE', 'HOST', 'IP' ]: + with open(os.path.join(node_storage_path, v+'.txt'), 'w') as f: + f.write(os.environ[v]) +netsize = int(os.environ['SIZE']) +post("init"+str(me)) + +## Delete old hosts +if me == 1: + cleanup() + with open('/etc/hosts', 'a') as f: + for i in range(1,netsize+1): + wait("init"+str(i)) + ip = pathlib.Path(os.path.join(storage_path, str(i), 'IP.txt')).read_text() + host = pathlib.Path(os.path.join(storage_path, str(i), 'HOST.txt')).read_text() + f.write(f"{ip} {host} minio{i} # {HOST_MARKER}\n") + post("hosts") +wait("hosts") + +# Adding entries to the host file + +# Unsharing the filesystem +data=os.path.join(storage_path, "data") +os.makedirs(data, exist_ok=True) +unshare.unshare(unshare.CLONE_NEWNS) +os.system("mount --make-rprivate /") # see https://stackoverflow.com/a/41580252 +os.system("mount -t tmpfs tmpfs "+data) + +last = str(netsize) +os.system("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+last+"}"+data+" > "+node_storage_path + "/minio.log 2>&1") diff --git a/instrumented/runner.sh b/instrumented/runner.sh new file mode 100755 index 0000000..5a83284 --- /dev/null +++ b/instrumented/runner.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +socat "openssl-listen:4443,\ +reuseaddr,\ +fork,\ +verify=0,\ +cert=/tmp/garage.pem" tcp4-connect:localhost:3900 & + +sleep 1 + +export ENDPOINT=localhost:4443 +export AWS_ACCESS_KEY_ID=GK4ea45d2f25091883071e0b73 +export AWS_SECRET_ACCESS_KEY=334f7b0ef233571c67b302c5197e6b8c7150da1907e4e530caa82e041d725ecf + +/home/quentin/go/bin/bench-garage | tee 50ms.garage.csv diff --git a/main.py b/main.py index c243a2f..39dea7e 100755 --- a/main.py +++ b/main.py @@ -321,6 +321,7 @@ def runall(cmd): env["HOST"] = server.name env["IP"] = str(server.ip) env["ID"] = str(number) + env["SIZE"] = str(len(config['servers'])) name = f'testnet-{zone.name}-{server.name}' net.ns.run(name, cmd, env) number +=1 @@ -330,6 +331,7 @@ def runall(cmd): env["HOST"] = zone.name env["IP"] = str(zone.ip) env["ID"] = str(number) + env["SIZE"] = str(len(config['servers'])) name = f'testnet-{zone.name}-{zone.name}' net.ns.run(name, cmd, env) first = False diff --git a/single-dc.yml b/single-dc.yml new file mode 100644 index 0000000..2f55956 --- /dev/null +++ b/single-dc.yml @@ -0,0 +1,33 @@ +links: + - &fiber + bandwidth: 100M + latency: 50ms + jitter: 10ms + +zones: + - &dc1 + name: dc1 + internal: *fiber + external: *fiber + +servers: + - name: dc1s1 + zone: *dc1 + - name: dc1s2 + zone: *dc1 + - name: dc1s3 + zone: *dc1 + - name: dc1s4 + zone: *dc1 + - name: dc1s5 + zone: *dc1 + +global: + subnet: + base: 'fc00:9a7a:9e::' + local: 64 + zone: 16 + latency-offset: 3ms + upstream: + ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff + conn: *fiber