WIP

50ms.garage.csv

@@ -0,0 +1,501 @@
+op,time
+listbuckets,202.111138
+listbuckets,43.940584
+listbuckets,181.316207
+listbuckets,19.614464
+listbuckets,202.478714
+listbuckets,136.910843
+listbuckets,163.83813
+listbuckets,6.058236
+listbuckets,85.429565
+listbuckets,114.172791
+listbuckets,74.642451
+listbuckets,185.718226
+listbuckets,167.826023
+listbuckets,149.187267
+listbuckets,133.087361
+listbuckets,84.117499
+listbuckets,144.38981
+listbuckets,23.85664
+listbuckets,87.296531
+listbuckets,67.410471
+listbuckets,84.965746
+listbuckets,49.504764
+listbuckets,137.740524
+listbuckets,170.798293
+listbuckets,95.475103
+listbuckets,137.597302
+listbuckets,63.782028
+listbuckets,29.721971
+listbuckets,28.081703
+listbuckets,69.563921
+listbuckets,5.206663
+listbuckets,18.225493
+listbuckets,45.392384
+listbuckets,162.435983
+listbuckets,50.777425
+listbuckets,37.839457
+listbuckets,79.086669
+listbuckets,87.751924
+listbuckets,134.904993
+listbuckets,42.232689
+listbuckets,192.142975
+listbuckets,140.725668
+listbuckets,95.926909
+listbuckets,51.227748
+listbuckets,177.170003
+listbuckets,144.06313
+listbuckets,102.662758
+listbuckets,32.749906
+listbuckets,122.617287
+listbuckets,169.247085
+listbuckets,12.547939
+listbuckets,140.197528
+listbuckets,7.936283
+listbuckets,46.104653
+listbuckets,17.26822
+listbuckets,103.748224
+listbuckets,178.618347
+listbuckets,21.52861
+listbuckets,45.549531
+listbuckets,183.945418
+listbuckets,115.780897
+listbuckets,174.858715
+listbuckets,86.042806
+listbuckets,31.345205
+listbuckets,12.83907
+listbuckets,198.613903
+listbuckets,188.966298
+listbuckets,45.879156
+listbuckets,145.656468
+listbuckets,178.692096
+listbuckets,139.07375
+listbuckets,201.000343
+listbuckets,52.520817
+listbuckets,31.036891
+listbuckets,39.697006
+listbuckets,141.790216
+listbuckets,165.41757
+listbuckets,202.853394
+listbuckets,194.009791
+listbuckets,51.369246
+listbuckets,173.986272
+listbuckets,11.476469
+listbuckets,17.626068
+listbuckets,47.418342
+listbuckets,157.026306
+listbuckets,50.140819
+listbuckets,104.12086
+listbuckets,92.402613
+listbuckets,75.890084
+listbuckets,132.398325
+listbuckets,44.180258
+listbuckets,93.321994
+listbuckets,71.967464
+listbuckets,64.616629
+listbuckets,84.258466
+listbuckets,195.606576
+listbuckets,118.320338
+listbuckets,104.692974
+listbuckets,74.44628
+listbuckets,28.309184
+putobject,165.46463
+putobject,222.517763
+putobject,111.937486
+putobject,184.887542
+putobject,126.217546
+putobject,237.106337
+putobject,116.128475
+putobject,189.31095
+putobject,113.380511
+putobject,103.315545
+putobject,247.080631
+putobject,186.276523
+putobject,99.660792
+putobject,117.760306
+putobject,109.472237
+putobject,264.241006
+putobject,121.447772
+putobject,117.987948
+putobject,111.855782
+putobject,115.530352
+putobject,326.730637
+putobject,120.42312
+putobject,221.54514
+putobject,110.206819
+putobject,123.908553
+putobject,117.173665
+putobject,115.430233
+putobject,110.628507
+putobject,237.560348
+putobject,115.31804
+putobject,308.395616
+putobject,107.843702
+putobject,111.375833
+putobject,96.310716
+putobject,112.016026
+putobject,239.727462
+putobject,112.919547
+putobject,111.346417
+putobject,329.578191
+putobject,106.91322
+putobject,95.694077
+putobject,278.426107
+putobject,105.462491
+putobject,181.24935
+putobject,107.048446
+putobject,218.381427
+putobject,170.658909
+putobject,222.801469
+putobject,120.682382
+putobject,165.640233
+putobject,189.454552
+putobject,152.842501
+putobject,106.93962
+putobject,255.688236
+putobject,111.958125
+putobject,130.607382
+putobject,202.151204
+putobject,114.005915
+putobject,215.592064
+putobject,339.66065
+putobject,116.144346
+putobject,211.735699
+putobject,127.648517
+putobject,105.163645
+putobject,205.664038
+putobject,117.612316
+putobject,95.377638
+putobject,103.966738
+putobject,117.708519
+putobject,198.141979
+putobject,114.142965
+putobject,236.392525
+putobject,179.734151
+putobject,116.104981
+putobject,176.715002
+putobject,116.993234
+putobject,194.191116
+putobject,103.240473
+putobject,149.059204
+putobject,111.387445
+putobject,206.070688
+putobject,116.469762
+putobject,348.528368
+putobject,112.547413
+putobject,217.546766
+putobject,112.896413
+putobject,215.344494
+putobject,115.486139
+putobject,118.703142
+putobject,213.602786
+putobject,109.749772
+putobject,112.005876
+putobject,112.576999
+putobject,115.241835
+putobject,216.114253
+putobject,122.455961
+putobject,107.242653
+putobject,152.564715
+putobject,122.086712
+putobject,199.452321
+listobjects,300.581365
+listobjects,277.790833
+listobjects,300.438323
+listobjects,306.401268
+listobjects,247.070642
+listobjects,265.460266
+listobjects,307.833962
+listobjects,278.348249
+listobjects,289.753623
+listobjects,428.597858
+listobjects,291.670485
+listobjects,291.750707
+listobjects,337.027161
+listobjects,246.554724
+listobjects,257.611418
+listobjects,320.019725
+listobjects,257.59075
+listobjects,295.037894
+listobjects,283.894836
+listobjects,247.228281
+listobjects,253.136202
+listobjects,337.787051
+listobjects,258.82629
+listobjects,268.927815
+listobjects,321.913973
+listobjects,281.985389
+listobjects,260.410992
+listobjects,287.980444
+listobjects,275.53983
+listobjects,269.558019
+listobjects,287.118302
+listobjects,290.645774
+listobjects,285.738097
+listobjects,392.130411
+listobjects,289.995953
+listobjects,262.023637
+listobjects,325.831103
+listobjects,278.981469
+listobjects,284.476107
+listobjects,314.008409
+listobjects,247.46578
+listobjects,241.607423
+listobjects,280.720744
+listobjects,251.493039
+listobjects,251.274956
+listobjects,308.848053
+listobjects,247.605826
+listobjects,254.399835
+listobjects,272.523456
+listobjects,295.263441
+listobjects,287.200498
+listobjects,321.094731
+listobjects,298.581687
+listobjects,267.287678
+listobjects,334.118082
+listobjects,294.63522
+listobjects,396.182216
+listobjects,324.845486
+listobjects,277.918475
+listobjects,276.752988
+listobjects,286.609247
+listobjects,237.788439
+listobjects,291.572429
+listobjects,289.931931
+listobjects,292.7288
+listobjects,278.329784
+listobjects,312.4204
+listobjects,250.387574
+listobjects,266.618319
+listobjects,294.083115
+listobjects,282.236044
+listobjects,246.433896
+listobjects,259.166735
+listobjects,283.552347
+listobjects,262.037944
+listobjects,438.74009
+listobjects,261.966819
+listobjects,272.358954
+listobjects,328.634834
+listobjects,249.669345
+listobjects,271.167948
+listobjects,314.412415
+listobjects,247.691167
+listobjects,283.680419
+listobjects,343.269156
+listobjects,251.896493
+listobjects,252.494486
+listobjects,422.943899
+listobjects,249.298261
+listobjects,261.622207
+listobjects,297.580581
+listobjects,283.851304
+listobjects,271.55979
+listobjects,285.568215
+listobjects,230.497899
+listobjects,263.590013
+listobjects,328.976692
+listobjects,263.056663
+listobjects,290.204427
+listobjects,313.846942
+getobject,182.785981
+getobject,109.499628
+getobject,106.650121
+getobject,121.295002
+getobject,117.047546
+getobject,121.821129
+getobject,99.500457
+getobject,110.204614
+getobject,90.665673
+getobject,103.448016
+getobject,109.513215
+getobject,109.836517
+getobject,108.865037
+getobject,109.142191
+getobject,115.979895
+getobject,114.514819
+getobject,261.303813
+getobject,97.933911
+getobject,105.707616
+getobject,95.21515
+getobject,114.97932
+getobject,97.167349
+getobject,109.995088
+getobject,116.094451
+getobject,92.307803
+getobject,98.966066
+getobject,92.048943
+getobject,108.609352
+getobject,108.804713
+getobject,98.891615
+getobject,115.83029
+getobject,243.530767
+getobject,210.53295
+getobject,112.15031
+getobject,111.292435
+getobject,114.00824
+getobject,104.009179
+getobject,95.101244
+getobject,107.050751
+getobject,120.16432
+getobject,194.587857
+getobject,241.974629
+getobject,106.301621
+getobject,108.892679
+getobject,108.393894
+getobject,246.919505
+getobject,107.829215
+getobject,102.11495
+getobject,103.701736
+getobject,101.141817
+getobject,95.347592
+getobject,103.559998
+getobject,88.331191
+getobject,98.132176
+getobject,108.62405
+getobject,105.357643
+getobject,113.917848
+getobject,97.43226
+getobject,93.802174
+getobject,98.555798
+getobject,255.646637
+getobject,106.634181
+getobject,106.455823
+getobject,111.535305
+getobject,96.665838
+getobject,109.615498
+getobject,105.223109
+getobject,109.366968
+getobject,103.517893
+getobject,102.918258
+getobject,100.815486
+getobject,108.270276
+getobject,96.698476
+getobject,109.806247
+getobject,115.544536
+getobject,112.49002
+getobject,259.839035
+getobject,111.031397
+getobject,106.06039
+getobject,112.722872
+getobject,113.953012
+getobject,103.298047
+getobject,99.28177
+getobject,112.432912
+getobject,96.173452
+getobject,99.882016
+getobject,111.264639
+getobject,108.417085
+getobject,117.353102
+getobject,95.225466
+getobject,111.075511
+getobject,264.20257
+getobject,117.333244
+getobject,120.125003
+getobject,108.663392
+getobject,119.961413
+getobject,95.005941
+getobject,95.4336
+getobject,115.280757
+getobject,97.201088
+removeobject,222.573682
+removeobject,231.761917
+removeobject,442.585842
+removeobject,234.382532
+removeobject,387.512105
+removeobject,305.628182
+removeobject,289.502486
+removeobject,231.169395
+removeobject,316.098005
+removeobject,209.524123
+removeobject,424.370175
+removeobject,216.714103
+removeobject,375.594149
+removeobject,223.780047
+removeobject,350.67269
+removeobject,336.975443
+removeobject,323.65556
+removeobject,292.523519
+removeobject,231.739565
+removeobject,472.605248
+removeobject,205.028165
+removeobject,297.558856
+removeobject,354.25723
+removeobject,358.617621
+removeobject,231.66819
+removeobject,218.014656
+removeobject,293.081194
+removeobject,220.378333
+removeobject,461.275468
+removeobject,294.197941
+removeobject,199.614791
+removeobject,294.474434
+removeobject,418.385018
+removeobject,230.544922
+removeobject,327.569235
+removeobject,218.21218
+removeobject,307.861353
+removeobject,328.311953
+removeobject,216.075602
+removeobject,210.143125
+removeobject,208.936679
+removeobject,190.425894
+removeobject,224.586404
+removeobject,337.963656
+removeobject,444.272797
+removeobject,301.920388
+removeobject,340.421272
+removeobject,338.346501
+removeobject,296.788527
+removeobject,207.559962
+removeobject,457.649247
+removeobject,300.037582
+removeobject,321.593867
+removeobject,292.55592
+removeobject,309.367726
+removeobject,255.574575
+removeobject,426.947087
+removeobject,265.768115
+removeobject,329.29239
+removeobject,308.343145
+removeobject,359.263264
+removeobject,203.957307
+removeobject,365.564661
+removeobject,237.358118
+removeobject,317.935617
+removeobject,313.741912
+removeobject,312.306754
+removeobject,323.298865
+removeobject,309.122932
+removeobject,202.639531
+removeobject,195.346255
+removeobject,218.604323
+removeobject,240.143875
+removeobject,341.057908
+removeobject,307.301071
+removeobject,326.504098
+removeobject,309.864819
+removeobject,300.616459
+removeobject,232.943336
+removeobject,220.08092
+removeobject,237.389609
+removeobject,334.716365
+removeobject,291.190192
+removeobject,206.634058
+removeobject,291.836125
+removeobject,307.390831
+removeobject,197.95132
+removeobject,379.646215
+removeobject,211.924509
+removeobject,244.664849
+removeobject,348.966198
+removeobject,316.319435
+removeobject,291.416401
+removeobject,215.855116
+removeobject,329.053698
+removeobject,322.553745
+removeobject,316.627999
+removeobject,311.277814
+removeobject,330.250545
+removeobject,221.223845

README.md

@@ -33,3 +33,10 @@ sudo chmod +x /usr/local/bin/garage
 
 garage help
 ```
+
+### Minio
+
+```
+sudo wget https://dl.min.io/server/minio/release/linux-amd64/minio -O /usr/local/bin/minio
+sudo chmod +x /usr/local/bin/minio
+```

analyse.R

@@ -0,0 +1,6 @@
+library(tidyverse)
+read_csv("50ms.garage.csv") -> a
+ggplot(a, aes(x=op,y=time)) + 
+  geom_violin(adjust = .5,draw_quantiles = c(0.25, 0.5, 0.75)) +
+  labs(title="Garage v0.5.0 Endpoint Latency\n(5 nodes, 100ms RTT between nodes, 20ms jitter, bars = Q1,median,Q3)", x="S3 Endpoint Name", y="Latency (ms)") +
+  theme_classic()

instrumented/deploy_garage.sh

@@ -51,9 +51,6 @@ CONFIG_NODE_FPATH=($(find /tmp/garage-testnet/ -maxdepth 3 -name garage.toml))
 SELF_ID=$(${GARAGE_PATH} node id 2>/dev/null)
 SHORT_ID=$(echo ${SELF_ID} | cut -c-64)
 
-echo ===DEBUG===
-echo $CONFIG_NODE_FPATH
-echo ${CONFIG_NODE_FPATH[0]}
 ${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} node connect ${SELF_ID}
 ${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} layout assign ${SHORT_ID} -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST}
 

instrumented/deploy_minio.py

@@ -0,0 +1,57 @@
+#!/bin/python3
+import yaml, os, sys, time, pathlib, unshare
+
+HOST_MARKER = "__AUTO_MKNET_MINIO_DEPLOY_DYYycw9Z5c5PvylU"
+storage_path = "/tmp/minio-testnet"
+
+# Remove any entry we put in /etc/hosts
+def cleanup():
+    with open('/etc/hosts', 'r') as fin:
+        with open('/etc/hosts', 'w') as fout:
+            fout.writelines(filter(lambda line: not HOST_MARKER in line, fin))
+
+def wait(p):
+    print("wait", p)
+    d = os.path.join(storage_path, p)
+    while not os.path.exists(d):
+        time.sleep(1)
+    #os.unlink(d)
+def post(p): 
+    print("post", p)
+    pathlib.Path(os.path.join(storage_path, p)).touch()
+
+## Create a working directory
+me = int(os.environ['ID'])
+node_storage_path = os.path.join(storage_path, str(me))
+os.makedirs(node_storage_path, exist_ok=True)
+
+## Writing down some info for others
+for v in [ 'ZONE', 'HOST', 'IP' ]:
+     with open(os.path.join(node_storage_path, v+'.txt'), 'w') as f:
+        f.write(os.environ[v])
+netsize = int(os.environ['SIZE'])
+post("init"+str(me))
+
+## Delete old hosts
+if me == 1:
+    cleanup()
+    with open('/etc/hosts', 'a') as f:
+        for i in range(1,netsize+1):
+            wait("init"+str(i))
+            ip = pathlib.Path(os.path.join(storage_path, str(i), 'IP.txt')).read_text()
+            host = pathlib.Path(os.path.join(storage_path, str(i), 'HOST.txt')).read_text()
+            f.write(f"{ip} {host} minio{i} # {HOST_MARKER}\n")
+    post("hosts")
+wait("hosts")
+
+# Adding entries to the host file
+
+# Unsharing the filesystem
+data=os.path.join(storage_path, "data")
+os.makedirs(data, exist_ok=True)
+unshare.unshare(unshare.CLONE_NEWNS)
+os.system("mount --make-rprivate /") # see https://stackoverflow.com/a/41580252
+os.system("mount -t tmpfs tmpfs "+data)
+
+last = str(netsize)
+os.system("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+last+"}"+data+" > "+node_storage_path + "/minio.log 2>&1")

instrumented/runner.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+socat "openssl-listen:4443,\
+reuseaddr,\
+fork,\
+verify=0,\
+cert=/tmp/garage.pem" tcp4-connect:localhost:3900 &
+
+sleep 1
+
+export ENDPOINT=localhost:4443
+export AWS_ACCESS_KEY_ID=GK4ea45d2f25091883071e0b73
+export AWS_SECRET_ACCESS_KEY=334f7b0ef233571c67b302c5197e6b8c7150da1907e4e530caa82e041d725ecf
+
+/home/quentin/go/bin/bench-garage | tee 50ms.garage.csv

main.py

@@ -321,6 +321,7 @@ def runall(cmd):
                 env["HOST"] = server.name
                 env["IP"] = str(server.ip)
                 env["ID"] = str(number)
+                env["SIZE"] = str(len(config['servers']))
                 name = f'testnet-{zone.name}-{server.name}'
                 net.ns.run(name, cmd, env)
                 number +=1
@@ -330,6 +331,7 @@ def runall(cmd):
             env["HOST"] = zone.name
             env["IP"] = str(zone.ip)
             env["ID"] = str(number)
+            env["SIZE"] = str(len(config['servers']))
             name = f'testnet-{zone.name}-{zone.name}'
             net.ns.run(name, cmd, env)
             first = False

single-dc.yml

@@ -0,0 +1,33 @@
+links:
+  - &fiber
+    bandwidth: 100M
+    latency: 50ms
+    jitter: 10ms
+
+zones:
+  - &dc1
+    name: dc1
+    internal: *fiber
+    external: *fiber
+
+servers:
+  - name: dc1s1
+    zone: *dc1
+  - name: dc1s2
+    zone: *dc1
+  - name: dc1s3
+    zone: *dc1
+  - name: dc1s4
+    zone: *dc1
+  - name: dc1s5
+    zone: *dc1
+
+global:
+  subnet:
+    base: 'fc00:9a7a:9e::'
+    local: 64
+    zone: 16
+  latency-offset: 3ms
+  upstream:
+    ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff
+    conn: *fiber