Update R code

50ms.garage.csv

@@ -0,0 +1,501 @@
+op,time
+listbuckets,202.111138
+listbuckets,43.940584
+listbuckets,181.316207
+listbuckets,19.614464
+listbuckets,202.478714
+listbuckets,136.910843
+listbuckets,163.83813
+listbuckets,6.058236
+listbuckets,85.429565
+listbuckets,114.172791
+listbuckets,74.642451
+listbuckets,185.718226
+listbuckets,167.826023
+listbuckets,149.187267
+listbuckets,133.087361
+listbuckets,84.117499
+listbuckets,144.38981
+listbuckets,23.85664
+listbuckets,87.296531
+listbuckets,67.410471
+listbuckets,84.965746
+listbuckets,49.504764
+listbuckets,137.740524
+listbuckets,170.798293
+listbuckets,95.475103
+listbuckets,137.597302
+listbuckets,63.782028
+listbuckets,29.721971
+listbuckets,28.081703
+listbuckets,69.563921
+listbuckets,5.206663
+listbuckets,18.225493
+listbuckets,45.392384
+listbuckets,162.435983
+listbuckets,50.777425
+listbuckets,37.839457
+listbuckets,79.086669
+listbuckets,87.751924
+listbuckets,134.904993
+listbuckets,42.232689
+listbuckets,192.142975
+listbuckets,140.725668
+listbuckets,95.926909
+listbuckets,51.227748
+listbuckets,177.170003
+listbuckets,144.06313
+listbuckets,102.662758
+listbuckets,32.749906
+listbuckets,122.617287
+listbuckets,169.247085
+listbuckets,12.547939
+listbuckets,140.197528
+listbuckets,7.936283
+listbuckets,46.104653
+listbuckets,17.26822
+listbuckets,103.748224
+listbuckets,178.618347
+listbuckets,21.52861
+listbuckets,45.549531
+listbuckets,183.945418
+listbuckets,115.780897
+listbuckets,174.858715
+listbuckets,86.042806
+listbuckets,31.345205
+listbuckets,12.83907
+listbuckets,198.613903
+listbuckets,188.966298
+listbuckets,45.879156
+listbuckets,145.656468
+listbuckets,178.692096
+listbuckets,139.07375
+listbuckets,201.000343
+listbuckets,52.520817
+listbuckets,31.036891
+listbuckets,39.697006
+listbuckets,141.790216
+listbuckets,165.41757
+listbuckets,202.853394
+listbuckets,194.009791
+listbuckets,51.369246
+listbuckets,173.986272
+listbuckets,11.476469
+listbuckets,17.626068
+listbuckets,47.418342
+listbuckets,157.026306
+listbuckets,50.140819
+listbuckets,104.12086
+listbuckets,92.402613
+listbuckets,75.890084
+listbuckets,132.398325
+listbuckets,44.180258
+listbuckets,93.321994
+listbuckets,71.967464
+listbuckets,64.616629
+listbuckets,84.258466
+listbuckets,195.606576
+listbuckets,118.320338
+listbuckets,104.692974
+listbuckets,74.44628
+listbuckets,28.309184
+putobject,165.46463
+putobject,222.517763
+putobject,111.937486
+putobject,184.887542
+putobject,126.217546
+putobject,237.106337
+putobject,116.128475
+putobject,189.31095
+putobject,113.380511
+putobject,103.315545
+putobject,247.080631
+putobject,186.276523
+putobject,99.660792
+putobject,117.760306
+putobject,109.472237
+putobject,264.241006
+putobject,121.447772
+putobject,117.987948
+putobject,111.855782
+putobject,115.530352
+putobject,326.730637
+putobject,120.42312
+putobject,221.54514
+putobject,110.206819
+putobject,123.908553
+putobject,117.173665
+putobject,115.430233
+putobject,110.628507
+putobject,237.560348
+putobject,115.31804
+putobject,308.395616
+putobject,107.843702
+putobject,111.375833
+putobject,96.310716
+putobject,112.016026
+putobject,239.727462
+putobject,112.919547
+putobject,111.346417
+putobject,329.578191
+putobject,106.91322
+putobject,95.694077
+putobject,278.426107
+putobject,105.462491
+putobject,181.24935
+putobject,107.048446
+putobject,218.381427
+putobject,170.658909
+putobject,222.801469
+putobject,120.682382
+putobject,165.640233
+putobject,189.454552
+putobject,152.842501
+putobject,106.93962
+putobject,255.688236
+putobject,111.958125
+putobject,130.607382
+putobject,202.151204
+putobject,114.005915
+putobject,215.592064
+putobject,339.66065
+putobject,116.144346
+putobject,211.735699
+putobject,127.648517
+putobject,105.163645
+putobject,205.664038
+putobject,117.612316
+putobject,95.377638
+putobject,103.966738
+putobject,117.708519
+putobject,198.141979
+putobject,114.142965
+putobject,236.392525
+putobject,179.734151
+putobject,116.104981
+putobject,176.715002
+putobject,116.993234
+putobject,194.191116
+putobject,103.240473
+putobject,149.059204
+putobject,111.387445
+putobject,206.070688
+putobject,116.469762
+putobject,348.528368
+putobject,112.547413
+putobject,217.546766
+putobject,112.896413
+putobject,215.344494
+putobject,115.486139
+putobject,118.703142
+putobject,213.602786
+putobject,109.749772
+putobject,112.005876
+putobject,112.576999
+putobject,115.241835
+putobject,216.114253
+putobject,122.455961
+putobject,107.242653
+putobject,152.564715
+putobject,122.086712
+putobject,199.452321
+listobjects,300.581365
+listobjects,277.790833
+listobjects,300.438323
+listobjects,306.401268
+listobjects,247.070642
+listobjects,265.460266
+listobjects,307.833962
+listobjects,278.348249
+listobjects,289.753623
+listobjects,428.597858
+listobjects,291.670485
+listobjects,291.750707
+listobjects,337.027161
+listobjects,246.554724
+listobjects,257.611418
+listobjects,320.019725
+listobjects,257.59075
+listobjects,295.037894
+listobjects,283.894836
+listobjects,247.228281
+listobjects,253.136202
+listobjects,337.787051
+listobjects,258.82629
+listobjects,268.927815
+listobjects,321.913973
+listobjects,281.985389
+listobjects,260.410992
+listobjects,287.980444
+listobjects,275.53983
+listobjects,269.558019
+listobjects,287.118302
+listobjects,290.645774
+listobjects,285.738097
+listobjects,392.130411
+listobjects,289.995953
+listobjects,262.023637
+listobjects,325.831103
+listobjects,278.981469
+listobjects,284.476107
+listobjects,314.008409
+listobjects,247.46578
+listobjects,241.607423
+listobjects,280.720744
+listobjects,251.493039
+listobjects,251.274956
+listobjects,308.848053
+listobjects,247.605826
+listobjects,254.399835
+listobjects,272.523456
+listobjects,295.263441
+listobjects,287.200498
+listobjects,321.094731
+listobjects,298.581687
+listobjects,267.287678
+listobjects,334.118082
+listobjects,294.63522
+listobjects,396.182216
+listobjects,324.845486
+listobjects,277.918475
+listobjects,276.752988
+listobjects,286.609247
+listobjects,237.788439
+listobjects,291.572429
+listobjects,289.931931
+listobjects,292.7288
+listobjects,278.329784
+listobjects,312.4204
+listobjects,250.387574
+listobjects,266.618319
+listobjects,294.083115
+listobjects,282.236044
+listobjects,246.433896
+listobjects,259.166735
+listobjects,283.552347
+listobjects,262.037944
+listobjects,438.74009
+listobjects,261.966819
+listobjects,272.358954
+listobjects,328.634834
+listobjects,249.669345
+listobjects,271.167948
+listobjects,314.412415
+listobjects,247.691167
+listobjects,283.680419
+listobjects,343.269156
+listobjects,251.896493
+listobjects,252.494486
+listobjects,422.943899
+listobjects,249.298261
+listobjects,261.622207
+listobjects,297.580581
+listobjects,283.851304
+listobjects,271.55979
+listobjects,285.568215
+listobjects,230.497899
+listobjects,263.590013
+listobjects,328.976692
+listobjects,263.056663
+listobjects,290.204427
+listobjects,313.846942
+getobject,182.785981
+getobject,109.499628
+getobject,106.650121
+getobject,121.295002
+getobject,117.047546
+getobject,121.821129
+getobject,99.500457
+getobject,110.204614
+getobject,90.665673
+getobject,103.448016
+getobject,109.513215
+getobject,109.836517
+getobject,108.865037
+getobject,109.142191
+getobject,115.979895
+getobject,114.514819
+getobject,261.303813
+getobject,97.933911
+getobject,105.707616
+getobject,95.21515
+getobject,114.97932
+getobject,97.167349
+getobject,109.995088
+getobject,116.094451
+getobject,92.307803
+getobject,98.966066
+getobject,92.048943
+getobject,108.609352
+getobject,108.804713
+getobject,98.891615
+getobject,115.83029
+getobject,243.530767
+getobject,210.53295
+getobject,112.15031
+getobject,111.292435
+getobject,114.00824
+getobject,104.009179
+getobject,95.101244
+getobject,107.050751
+getobject,120.16432
+getobject,194.587857
+getobject,241.974629
+getobject,106.301621
+getobject,108.892679
+getobject,108.393894
+getobject,246.919505
+getobject,107.829215
+getobject,102.11495
+getobject,103.701736
+getobject,101.141817
+getobject,95.347592
+getobject,103.559998
+getobject,88.331191
+getobject,98.132176
+getobject,108.62405
+getobject,105.357643
+getobject,113.917848
+getobject,97.43226
+getobject,93.802174
+getobject,98.555798
+getobject,255.646637
+getobject,106.634181
+getobject,106.455823
+getobject,111.535305
+getobject,96.665838
+getobject,109.615498
+getobject,105.223109
+getobject,109.366968
+getobject,103.517893
+getobject,102.918258
+getobject,100.815486
+getobject,108.270276
+getobject,96.698476
+getobject,109.806247
+getobject,115.544536
+getobject,112.49002
+getobject,259.839035
+getobject,111.031397
+getobject,106.06039
+getobject,112.722872
+getobject,113.953012
+getobject,103.298047
+getobject,99.28177
+getobject,112.432912
+getobject,96.173452
+getobject,99.882016
+getobject,111.264639
+getobject,108.417085
+getobject,117.353102
+getobject,95.225466
+getobject,111.075511
+getobject,264.20257
+getobject,117.333244
+getobject,120.125003
+getobject,108.663392
+getobject,119.961413
+getobject,95.005941
+getobject,95.4336
+getobject,115.280757
+getobject,97.201088
+removeobject,222.573682
+removeobject,231.761917
+removeobject,442.585842
+removeobject,234.382532
+removeobject,387.512105
+removeobject,305.628182
+removeobject,289.502486
+removeobject,231.169395
+removeobject,316.098005
+removeobject,209.524123
+removeobject,424.370175
+removeobject,216.714103
+removeobject,375.594149
+removeobject,223.780047
+removeobject,350.67269
+removeobject,336.975443
+removeobject,323.65556
+removeobject,292.523519
+removeobject,231.739565
+removeobject,472.605248
+removeobject,205.028165
+removeobject,297.558856
+removeobject,354.25723
+removeobject,358.617621
+removeobject,231.66819
+removeobject,218.014656
+removeobject,293.081194
+removeobject,220.378333
+removeobject,461.275468
+removeobject,294.197941
+removeobject,199.614791
+removeobject,294.474434
+removeobject,418.385018
+removeobject,230.544922
+removeobject,327.569235
+removeobject,218.21218
+removeobject,307.861353
+removeobject,328.311953
+removeobject,216.075602
+removeobject,210.143125
+removeobject,208.936679
+removeobject,190.425894
+removeobject,224.586404
+removeobject,337.963656
+removeobject,444.272797
+removeobject,301.920388
+removeobject,340.421272
+removeobject,338.346501
+removeobject,296.788527
+removeobject,207.559962
+removeobject,457.649247
+removeobject,300.037582
+removeobject,321.593867
+removeobject,292.55592
+removeobject,309.367726
+removeobject,255.574575
+removeobject,426.947087
+removeobject,265.768115
+removeobject,329.29239
+removeobject,308.343145
+removeobject,359.263264
+removeobject,203.957307
+removeobject,365.564661
+removeobject,237.358118
+removeobject,317.935617
+removeobject,313.741912
+removeobject,312.306754
+removeobject,323.298865
+removeobject,309.122932
+removeobject,202.639531
+removeobject,195.346255
+removeobject,218.604323
+removeobject,240.143875
+removeobject,341.057908
+removeobject,307.301071
+removeobject,326.504098
+removeobject,309.864819
+removeobject,300.616459
+removeobject,232.943336
+removeobject,220.08092
+removeobject,237.389609
+removeobject,334.716365
+removeobject,291.190192
+removeobject,206.634058
+removeobject,291.836125
+removeobject,307.390831
+removeobject,197.95132
+removeobject,379.646215
+removeobject,211.924509
+removeobject,244.664849
+removeobject,348.966198
+removeobject,316.319435
+removeobject,291.416401
+removeobject,215.855116
+removeobject,329.053698
+removeobject,322.553745
+removeobject,316.627999
+removeobject,311.277814
+removeobject,330.250545
+removeobject,221.223845

README.md

@@ -0,0 +1,42 @@
+# mk-testnet
+
+Show usage:
+
+```help
+python3 main.py
+```
+
+## Example
+
+```bash
+sudo rm -rf /tmp/garage-testnet/ # always start by deleting previous run
+sudo python3 main.py create ./config.yml
+sudo python3 main.py run-all example/deploy_garage.sh
+sudo python3 main.py run dc1:dc1s1 garage -c /tmp/garage-testnet/dc1/dc1s1/garage.toml status
+sudo python3 main.py destroy
+```
+
+## Instrumented daemons
+
+If you want to use the scripts provided in the `instrumented` folder,
+you must add to your path some tools
+
+### Garage (`deploy_garage.sh`)
+
+```bash
+# see versions on https://garagehq.deuxfleurs.fr/_releases.html
+export GRG_ARCH=x86_64-unknown-linux-musl
+export GRG_VERSION=v0.5.0
+
+sudo wget https://garagehq.deuxfleurs.fr/_releases/${GRG_VERSION}/${GRG_ARCH}/garage -O /usr/local/bin/garage
+sudo chmod +x /usr/local/bin/garage
+
+garage help
+```
+
+### Minio
+
+```
+sudo wget https://dl.min.io/server/minio/release/linux-amd64/minio -O /usr/local/bin/minio
+sudo chmod +x /usr/local/bin/minio
+```

analyse.R

@@ -0,0 +1,28 @@
+install.packages("tidyverse")
+install.packages("svglite")
+library(tidyverse)
+library(svglite)
+read_csv("50ms.garage.csv") %>% add_column(daemon="garage 0.5.0") -> a
+read_csv("50ms.minio.csv") %>% add_column(daemon="minio RELEASE.2021-11-24T23-19-33Z") -> b
+bind_rows(a,b) %>% group_by(daemon,op) %>% summarise(
+  time_mean = mean(time),
+  time_max = max(time),
+  time_min = min(time)
+) -> c
+
+
+ggplot(c, aes(x=op,y=time_mean,fill=daemon,ymin=time_min,ymax=time_max)) +
+      geom_bar(stat="identity", position=position_dodge(),color="black") +
+      geom_errorbar(position=position_dodge(.9),width=.2) + 
+      scale_y_continuous(expand=c(0,0))+
+      coord_flip() +
+      labs(
+        x="S3 Endpoint", 
+        y="Latency (ms)",
+        fill="Daemon",
+        caption="Get the code to reproduce this graph at https://git.deuxfleurs.fr/quentin/benchmarks",
+        title="S3 endpoint latency in a simulated geo-distributed cluster",
+        subtitle="100 measurements, 5 nodes, 100ms RTT + 20ms jitter between nodes\nno contention: latency is due to intra-cluster communications\ncolored bar = mean latency, error bar = min and max") + 
+      theme_classic() +
+      ggsave("endpoint-latency.png", width=200, height=110, units="mm")
+       

instrumented/deploy_garage.sh

@@ -46,7 +46,7 @@ EOF
 RUST_LOG=garage=debug ${GARAGE_PATH} server 2>> ${NODE_STORAGE_PATH}/logs & disown
 sleep 2
 
-CONFIG_NODE_FPATH=(${STORAGE_PATH}/*{,/*}/garage.toml)
+CONFIG_NODE_FPATH=($(find /tmp/garage-testnet/ -maxdepth 3 -name garage.toml))
 
 SELF_ID=$(${GARAGE_PATH} node id 2>/dev/null)
 SHORT_ID=$(echo ${SELF_ID} | cut -c-64)

instrumented/deploy_minio.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+import json, yaml, os, sys, time, pathlib, socket, shutil
+
+STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet')
+#STORAGE_PATH = os.path.join('/mnt/windows', '.minio-testnet')
+MINIO_PATH = '/srv'
+HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt')
+UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock')
+DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid))
+
+def main():
+    if int(os.environ['ID']) == 1: leader()
+    else: follower()
+
+def leader():
+    shutil.rmtree(STORAGE_PATH, ignore_errors=True)
+    os.makedirs(STORAGE_PATH)
+    os.makedirs(MINIO_PATH, exist_ok=True)
+    print(STORAGE_PATH)
+
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.bind(UNIX_SOCK)
+    sock.listen()
+
+    n_serv = n_servers()
+    fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]]
+
+    identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ]
+    print(f"ident: {identities}")
+    msg = f"{json.dumps(identities)}\n".encode()
+    [ co.send(msg) for co in fl ]
+
+    run_minio(identities)
+
+def follower():
+    co = None
+    while True:
+        time.sleep(1)
+        try:
+            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            sock.connect(UNIX_SOCK)
+            co = sock.makefile()
+            break
+        except Exception as err:
+            print('conn failed, wait,', err)
+    my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() })
+    sock.send(f"{my_identity}\n".encode())
+    identities = json.loads(co.readline())
+
+    run_minio(identities)
+
+def make_data():
+    data_path = DATA_PATH(os.environ['ID'])
+    os.makedirs(data_path)
+    return data_path
+
+def run_minio(identities):
+    cmd = f"minio server --console-address ':9001' --address [{os.environ['IP']}]:9000"
+    for ident in identities:
+        cmd += f" http://[{ident['ip']}]:9000{ident['path']}"
+    cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1"
+    print("launch: ", cmd)
+    os.system(cmd)
+
+def n_servers():
+    with open('.current_state.yml', 'r') as f:
+        netw = yaml.safe_load(f)
+        n_servers = len(netw['servers'])
+        return n_servers
+
+__name__ == '__main__' and main()

instrumented/minio-runner.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+export ENDPOINT=[fc00:9a7a:9e::1]:9000
+export AWS_ACCESS_KEY_ID=minioadmin
+export AWS_SECRET_ACCESS_KEY=minioadmin
+
+s3lat | tee 50ms.minio.csv

instrumented/runner.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+socat "openssl-listen:4443,\
+reuseaddr,\
+fork,\
+verify=0,\
+cert=/tmp/garage.pem" tcp4-connect:localhost:3900 &
+
+sleep 1
+
+export SSL=1
+export SSL_INSECURE=1
+export REGION=garage
+export ENDPOINT=localhost:4443
+export AWS_ACCESS_KEY_ID=GK4ea45d2f25091883071e0b73
+export AWS_SECRET_ACCESS_KEY=334f7b0ef233571c67b302c5197e6b8c7150da1907e4e530caa82e041d725ecf
+
+/home/quentin/go/bin/bench-garage | tee 50ms.garage.csv

main.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/usr/bin/env python
 import ipaddress
 import os
 import shutil
@@ -305,7 +305,7 @@ def run(netns, cmd):
 
     if len(cmd) == 0:
         cmd = [os.getenv("SHELL") or "/bin/sh"]
-    os.execve("/bin/env", ["/bin/env", "ip", "netns" , "exec", name ] + cmd, env)
+    os.execve("/usr/bin/env", ["/usr/bin/env", "ip", "netns" , "exec", name ] + cmd, env)
 
 def runall(cmd):
     with open(".current_state.yml", "r") as file:
@@ -321,6 +321,7 @@ def runall(cmd):
                 env["HOST"] = server.name
                 env["IP"] = str(server.ip)
                 env["ID"] = str(number)
+                env["SIZE"] = str(len(config['servers']))
                 name = f'testnet-{zone.name}-{server.name}'
                 net.ns.run(name, cmd, env)
                 number +=1
@@ -330,6 +331,7 @@ def runall(cmd):
             env["HOST"] = zone.name
             env["IP"] = str(zone.ip)
             env["ID"] = str(number)
+            env["SIZE"] = str(len(config['servers']))
             name = f'testnet-{zone.name}-{zone.name}'
             net.ns.run(name, cmd, env)
             first = False

single-dc.yml

@@ -0,0 +1,33 @@
+links:
+  - &fiber
+    bandwidth: 100M
+    latency: 50ms
+    jitter: 10ms
+
+zones:
+  - &dc1
+    name: dc1
+    internal: *fiber
+    external: *fiber
+
+servers:
+  - name: dc1s1
+    zone: *dc1
+  - name: dc1s2
+    zone: *dc1
+  - name: dc1s3
+    zone: *dc1
+  - name: dc1s4
+    zone: *dc1
+  - name: dc1s5
+    zone: *dc1
+
+global:
+  subnet:
+    base: 'fc00:9a7a:9e::'
+    local: 64
+    zone: 16
+  latency-offset: 3ms
+  upstream:
+    ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff
+    conn: *fiber