forked from Deuxfleurs/mknet
Add S3TTFB
This commit is contained in:
parent
700025e479
commit
a40c123542
GPG key ID:
E9602264D639FF68
16 changed files with 656 additions and 10 deletions
61
artifacts/2022-09-23-s3ttfb/garage-v0.7.csv
Normal file
61
artifacts/2022-09-23-s3ttfb/garage-v0.7.csv
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
nanoseconds
|
||||
2043724295
|
||||
1819672397
|
||||
1856153241
|
||||
1814937942
|
||||
1818943177
|
||||
1814245475
|
||||
1828595614
|
||||
1817263829
|
||||
1857481054
|
||||
1816538404
|
||||
1816874169
|
||||
1856305403
|
||||
1857317361
|
||||
1814517488
|
||||
1817156893
|
||||
1815832311
|
||||
1857020274
|
||||
1822556545
|
||||
1816402439
|
||||
1815664395
|
||||
1818129711
|
||||
1855229810
|
||||
1817427508
|
||||
1817253968
|
||||
1855458740
|
||||
1818412087
|
||||
1816422102
|
||||
1814017661
|
||||
1821773596
|
||||
1814924392
|
||||
1819138108
|
||||
1814140079
|
||||
1814253338
|
||||
1814717178
|
||||
1823390172
|
||||
1856664915
|
||||
1817886566
|
||||
1815441938
|
||||
1815381667
|
||||
1864383216
|
||||
1815026715
|
||||
1816346397
|
||||
1814835227
|
||||
1855973303
|
||||
1814632968
|
||||
1823675273
|
||||
1817900846
|
||||
1816282804
|
||||
1817289580
|
||||
1817288129
|
||||
1860637032
|
||||
1855146585
|
||||
1816649086
|
||||
1822524427
|
||||
1814277922
|
||||
1901037764
|
||||
1818903030
|
||||
1815404774
|
||||
1815446844
|
||||
1815223628
|
||||
|
61
artifacts/2022-09-23-s3ttfb/garage-v0.8.csv
Normal file
61
artifacts/2022-09-23-s3ttfb/garage-v0.8.csv
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
nanoseconds
|
||||
376764319
|
||||
268350943
|
||||
310303481
|
||||
266580460
|
||||
267172958
|
||||
46771899
|
||||
271623330
|
||||
270463024
|
||||
47401936
|
||||
45384570
|
||||
129505789
|
||||
46244307
|
||||
270594700
|
||||
267908944
|
||||
310163759
|
||||
271096844
|
||||
268930686
|
||||
309894956
|
||||
273312715
|
||||
269766963
|
||||
267947351
|
||||
270443270
|
||||
266292594
|
||||
215426567
|
||||
383550730
|
||||
84593093
|
||||
380970185
|
||||
419898767
|
||||
381574216
|
||||
422289246
|
||||
380535266
|
||||
268204508
|
||||
46449693
|
||||
270408730
|
||||
268572254
|
||||
308981862
|
||||
266303861
|
||||
46092770
|
||||
308073958
|
||||
383696451
|
||||
381725197
|
||||
379008890
|
||||
379902614
|
||||
382568588
|
||||
45370580
|
||||
83526768
|
||||
381230253
|
||||
267980416
|
||||
269187238
|
||||
84449786
|
||||
267524276
|
||||
46169934
|
||||
43509278
|
||||
308776528
|
||||
270626080
|
||||
271203190
|
||||
216167767
|
||||
239794865
|
||||
46230173
|
||||
267821837
|
||||
|
61
artifacts/2022-09-23-s3ttfb/minio-20220917.csv
Normal file
61
artifacts/2022-09-23-s3ttfb/minio-20220917.csv
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
nanoseconds
|
||||
941737601
|
||||
970059899
|
||||
956848578
|
||||
981034132
|
||||
968456906
|
||||
986209313
|
||||
947371347
|
||||
961890275
|
||||
951011001
|
||||
947766957
|
||||
946381130
|
||||
1172968280
|
||||
944672409
|
||||
946216350
|
||||
960072933
|
||||
947966145
|
||||
947998544
|
||||
957730811
|
||||
960898554
|
||||
950611253
|
||||
953195848
|
||||
974654976
|
||||
982578891
|
||||
957419979
|
||||
947448876
|
||||
945953214
|
||||
936687543
|
||||
950116533
|
||||
979971727
|
||||
941427574
|
||||
931898843
|
||||
928494073
|
||||
927924296
|
||||
940995728
|
||||
927965457
|
||||
928098595
|
||||
933759012
|
||||
929902652
|
||||
953753466
|
||||
931448442
|
||||
934503899
|
||||
933297864
|
||||
934775226
|
||||
936615695
|
||||
950945991
|
||||
935054323
|
||||
1162136321
|
||||
940502403
|
||||
1158490674
|
||||
945337099
|
||||
948016443
|
||||
939863143
|
||||
938112935
|
||||
933271354
|
||||
946325434
|
||||
954355353
|
||||
948997521
|
||||
1124820589
|
||||
989829976
|
||||
938639125
|
||||
|
25
artifacts/2022-09-23-s3ttfb/plot.R
Normal file
25
artifacts/2022-09-23-s3ttfb/plot.R
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
library(tidyverse)
|
||||
|
||||
|
||||
read_csv("garage-v0.7.csv") %>% add_column(daemon="garage 0.7.3") -> a
|
||||
read_csv("garage-v0.8.csv") %>% add_column(daemon="garage 0.8.0 beta") -> b
|
||||
read_csv("minio-20220917.csv") %>% add_column(daemon="minio RELEASE.2022-09-17") -> c
|
||||
bind_rows(a,b,c) %>% mutate(ms = nanoseconds / 1000 / 1000) -> d
|
||||
|
||||
ggplot(d, aes(y = ms, group = daemon, fill=daemon)) +
|
||||
#geom_violin(trim = TRUE, width=2) +
|
||||
geom_histogram(binwidth=50) +
|
||||
scale_y_continuous(expand=c(0,0))+
|
||||
scale_x_continuous(expand=c(0,0))+
|
||||
coord_flip() +
|
||||
labs(
|
||||
y="TTFB (ms)",
|
||||
x="Count",
|
||||
fill="Daemon",
|
||||
caption="Get the code to reproduce this graph at https://git.deuxfleurs.fr/Deuxfleurs/mknet",
|
||||
title="TTFB (Time To First Byte) on GetObject over a slow network (5 Mbps)",
|
||||
subtitle="A 1MB file is uploaded and then fetched 60 times.\nExcept for Minio, the queried node does not store any data (gateway) to force net. communications.") +
|
||||
theme_classic() +
|
||||
theme(legend.position = c(.15, .8))
|
||||
|
||||
ggsave("./plot.png", width=200, height=110, units="mm")
|
||||
BIN
artifacts/2022-09-23-s3ttfb/plot.png
Normal file
BIN
artifacts/2022-09-23-s3ttfb/plot.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 125 KiB |
1
benchmarks/s3ttfb/.gitignore
vendored
Normal file
1
benchmarks/s3ttfb/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
s3ttfb
|
||||
12
benchmarks/s3ttfb/go.mod
Normal file
12
benchmarks/s3ttfb/go.mod
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
module git.deuxfleurs.fr/Deuxfleurs/mknet/benchmarks/s3ttfb
|
||||
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.16
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.7
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11
|
||||
github.com/google/uuid v1.1.1
|
||||
github.com/minio/minio-go/v7 v7.0.16
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
|
||||
)
|
||||
123
benchmarks/s3ttfb/go.sum
Normal file
123
benchmarks/s3ttfb/go.sum
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 h1:tcFliCWne+zOuUfKNRn8JdFBuWPDuISDH08wD2ULkhk=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot91steJeti4ryyu/tLd4Sk84O5W22L7O2EQU=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.7 h1:odVM52tFHhpqZBKNjVW5h+Zt1tKHbhdTQRb+0WHrNtw=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.7/go.mod h1:dN2gja/QXxFF15hQreyrqYhLBaQo1d9ZKe/v/uplQoI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.20 h1:9+ZhlDY7N9dPnUmf7CDfW9In4sW5Ff3bh7oy4DzS1IE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.20/go.mod h1:UKY5HyIux08bbNA7Blv4PcXQ8cTkGh7ghHMFklaviR4=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 h1:r08j4sbZu/RVi+BNxkBJwPMUYY3P8mgSDuKkZ/ZN1lE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 h1:wj5Rwc05hvUSvKuOF29IYb9QrCLjU+rHAy/x/o0DK2c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14 h1:ZSIPAkAsCCjYrhqfw2+lNzWDzxzHXEckFkTePL5RSWQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14/go.mod h1:AyGgqiKv9ECM6IZeNQtdT8NnMvUb3/2wokeq2Fgryto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18 h1:BBYoNQt2kUZUUK4bIPsKrCcjVPUMNsgQpNAwhznK/zo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18/go.mod h1:NS55eQ4YixUJPTC+INxi2/jCqe1y2Uw3rnh9wEOVJxY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 h1:Jrd/oMh0PKQc6+BowB+pLEwLIgaQF29eYbe7E1Av9Ug=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 h1:HfVVR1vItaG6le+Bpw6P4midjBDMKnjMyZnw9MXYUcE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17/go.mod h1:YqMdV+gEKCQ59NrB7rzrJdALeBIsYiVi8Inj3+KcqHI=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 h1:3/gm/JTX9bX8CpzTgIlrtYpB3EVBDxyg/GY/QdcIEZw=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11/go.mod h1:fmgDANqTUCxciViKl9hb/zD5LFbvPINFRgWhDbR+vZo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 h1:pwvCchFUEnlceKIgPUouBJwK81aCkQ8UDMORfeFtW10=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 h1:GUnZ62TevLqIoDyHeiWj2P7EqaosgakBKVvWriIdLQY=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5/go.mod h1:csZuQY65DAdFBt1oIjO5hhBR49kQqop4+lcuCjf2arA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 h1:9pPi0PsFNAGILFfPCk8Y0iyEBGc6lu6OQ97U7hmdesg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM=
|
||||
github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA=
|
||||
github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
|
||||
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
|
||||
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
|
||||
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
|
||||
github.com/minio/minio-go/v7 v7.0.16 h1:GspaSBS8lOuEUCAqMe0W3UxSoyOA4b4F8PTspRVI+k4=
|
||||
github.com/minio/minio-go/v7 v7.0.16/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
|
||||
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
|
||||
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
|
||||
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU=
|
||||
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
|
||||
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
214
benchmarks/s3ttfb/main.go
Normal file
214
benchmarks/s3ttfb/main.go
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/http/httptrace"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
type PRNG struct {
|
||||
rem int64
|
||||
}
|
||||
|
||||
func (r *PRNG) Read(p []byte) (n int, err error) {
|
||||
//log.Printf("rem=%d, buf=%d\n", r.rem, len(p))
|
||||
if int64(len(p)) > r.rem {
|
||||
p = p[:r.rem]
|
||||
}
|
||||
|
||||
if int64(len(p)) > r.rem {
|
||||
log.Fatal("LOGIC ERROR")
|
||||
}
|
||||
|
||||
n, err = rand.Read(p)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
r.rem -= int64(n)
|
||||
if r.rem <= 0 {
|
||||
err = io.EOF
|
||||
//log.Printf("PRNG file has been fully read. rem=%d,n=%d,err=%s\n", r.rem, n, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func main() {
|
||||
var err error
|
||||
fmt.Printf("nanoseconds\n")
|
||||
|
||||
// Initial setup
|
||||
_, isSSL := os.LookupEnv("SSL")
|
||||
opts := minio.Options{
|
||||
Creds: credentials.NewStaticV4(os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"), ""),
|
||||
Secure: isSSL,
|
||||
}
|
||||
|
||||
if region, ok := os.LookupEnv("AWS_REGION"); ok {
|
||||
opts.Region = region
|
||||
}
|
||||
|
||||
if _, ok := os.LookupEnv("SSL_INSECURE"); ok {
|
||||
opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||
}
|
||||
|
||||
size := 1 * 1024 * 1024 // 1 MB
|
||||
if env_size, ok := os.LookupEnv("SIZE"); ok {
|
||||
size, err = strconv.Atoi(env_size)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
repeat := 60
|
||||
if env_repeat, ok := os.LookupEnv("REPEAT"); ok {
|
||||
repeat, err = strconv.Atoi(env_repeat)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
mc, err := minio.New(os.Getenv("ENDPOINT"), &opts)
|
||||
if err != nil {
|
||||
log.Fatal("failed connect", err)
|
||||
return
|
||||
}
|
||||
|
||||
// AWS SDK stuff
|
||||
customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
PartitionID: "aws",
|
||||
URL: "http://"+os.Getenv("ENDPOINT"),
|
||||
SigningRegion: region,
|
||||
}, nil
|
||||
})
|
||||
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithEndpointResolverWithOptions(customResolver))
|
||||
if err != nil {
|
||||
log.Fatalf("failed to load configuration, %v", err)
|
||||
}
|
||||
|
||||
// Create an Amazon S3 service client
|
||||
client := s3.NewFromConfig(cfg, func(opts *s3.Options) {
|
||||
opts.UsePathStyle = true
|
||||
})
|
||||
|
||||
|
||||
// Create Bucket
|
||||
buck := uuid.New().String()
|
||||
err = mc.MakeBucket(context.Background(), buck, minio.MakeBucketOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return
|
||||
}
|
||||
|
||||
// Send an object
|
||||
prng := new(PRNG)
|
||||
prng.rem = int64(size)
|
||||
|
||||
|
||||
key := uuid.New().String()
|
||||
_, err = mc.PutObject(
|
||||
context.Background(),
|
||||
buck,
|
||||
key,
|
||||
prng,
|
||||
int64(size),
|
||||
minio.PutObjectOptions{ContentType: "application/octet-stream"},
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatal("unable to upload our test object")
|
||||
return
|
||||
}
|
||||
|
||||
for i := 0; i < repeat; i++ {
|
||||
// Now, try to fetch it *repeat* times (assume no caching)
|
||||
var elapsed time.Duration
|
||||
|
||||
var start time.Time
|
||||
trace := &httptrace.ClientTrace{
|
||||
GotFirstResponseByte: func() { elapsed = time.Since(start) },
|
||||
}
|
||||
|
||||
start = time.Now()
|
||||
/*object, err := mc.GetObject(httptrace.WithClientTrace(context.Background(), trace), buck, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return
|
||||
}*/
|
||||
|
||||
out, err := client.GetObject(httptrace.WithClientTrace(context.Background(), trace), &s3.GetObjectInput {
|
||||
Bucket: aws.String(buck),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return
|
||||
}
|
||||
object := out.Body
|
||||
|
||||
|
||||
if _, err = io.Copy(ioutil.Discard, object) ; err != nil {
|
||||
log.Fatal("failed getobject: ", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("%v\n", elapsed.Nanoseconds())
|
||||
//time.Sleep(10 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
func timeGet(url string) {
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
|
||||
var start, connect, dns, tlsHandshake time.Time
|
||||
|
||||
trace := &httptrace.ClientTrace{
|
||||
DNSStart: func(dsi httptrace.DNSStartInfo) { dns = time.Now() },
|
||||
DNSDone: func(ddi httptrace.DNSDoneInfo) {
|
||||
fmt.Printf("DNS Done: %v\n", time.Since(dns))
|
||||
},
|
||||
|
||||
TLSHandshakeStart: func() { tlsHandshake = time.Now() },
|
||||
TLSHandshakeDone: func(cs tls.ConnectionState, err error) {
|
||||
fmt.Printf("TLS Handshake: %v\n", time.Since(tlsHandshake))
|
||||
},
|
||||
|
||||
ConnectStart: func(network, addr string) { connect = time.Now() },
|
||||
ConnectDone: func(network, addr string, err error) {
|
||||
fmt.Printf("Connect time: %v\n", time.Since(connect))
|
||||
},
|
||||
|
||||
GotFirstResponseByte: func() {
|
||||
fmt.Printf("Time from start to first byte: %v\n", time.Since(start))
|
||||
},
|
||||
}
|
||||
|
||||
req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
|
||||
start = time.Now()
|
||||
if _, err := http.DefaultTransport.RoundTrip(req); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Printf("Total time: %v\n", time.Since(start))
|
||||
}
|
||||
*/
|
||||
|
|
@ -14,6 +14,8 @@ os.chdir(me / "benchmarks" / "s3concurrent")
|
|||
shared.exec("go build")
|
||||
os.chdir(me / "benchmarks" / "s3lat")
|
||||
shared.exec("go build")
|
||||
os.chdir(me / "benchmarks" / "s3ttfb")
|
||||
shared.exec("go build")
|
||||
os.chdir(me)
|
||||
|
||||
print("--- install python dependencies ---")
|
||||
|
|
@ -29,8 +31,9 @@ print("--- download artifacts ---")
|
|||
from scenarios.fragments import flavor
|
||||
flavor.download()
|
||||
|
||||
print("""done! now, run:
|
||||
print(f"""done! now, run:
|
||||
> sudo -i
|
||||
> cd {os.getcwd()}
|
||||
> source ./.venv/bin/activate
|
||||
> ./mknet scenario ./topo/with-vdsl.yml ./scenarios/garage-s3lat garage-v0.8
|
||||
""")
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ if 'HOST' in env:
|
|||
config = storage_path / "garage.toml"
|
||||
env['GARAGE_CONFIG_FILE'] = str(config)
|
||||
|
||||
def deploy_coord():
|
||||
def deploy_coord(gw=None):
|
||||
destroy()
|
||||
shared.log("start daemon")
|
||||
shared.exec(f"{version['path']} --version")
|
||||
|
|
@ -46,7 +46,7 @@ def deploy_coord():
|
|||
shared.log("discover nodes")
|
||||
connect()
|
||||
shared.log("build layout")
|
||||
create_layout()
|
||||
create_layout(gw=gw)
|
||||
shared.log("create key")
|
||||
create_key()
|
||||
shared.log("ready")
|
||||
|
|
@ -136,26 +136,45 @@ def cluster_info():
|
|||
def connect():
|
||||
cinf = cluster_info()
|
||||
shared.log("start connections...")
|
||||
ret = nodes.add_node([n['node_addr'] for n in cinf])
|
||||
for st in ret:
|
||||
if not st.success:
|
||||
raise Exception("Node connect failed", ret)
|
||||
while True:
|
||||
try:
|
||||
ret = nodes.add_node([n['node_addr'] for n in cinf], _request_timeout=3)
|
||||
except:
|
||||
shared.log("not ready, retry in 1sec")
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
for st in ret:
|
||||
if not st.success:
|
||||
continue
|
||||
#raise Exception("Node connect failed", ret)
|
||||
break
|
||||
|
||||
shared.log("all nodes connected")
|
||||
|
||||
def create_layout():
|
||||
def create_layout(gw=None):
|
||||
if gw is None:
|
||||
gw = []
|
||||
|
||||
v = layout.get_layout().version
|
||||
|
||||
cinf = cluster_info()
|
||||
nlay = dict()
|
||||
for n in cinf:
|
||||
capa = 1
|
||||
if n['host'] in gw:
|
||||
capa = None
|
||||
|
||||
nlay[n['node_id']] = NodeClusterInfo(
|
||||
zone = n['zone'],
|
||||
capacity = 1,
|
||||
capacity = capa,
|
||||
tags = [ n['host'] ],
|
||||
)
|
||||
layout.add_layout(nlay)
|
||||
layout.apply_layout(LayoutVersion(version=v+1))
|
||||
|
||||
shared.log(layout.get_layout())
|
||||
|
||||
def create_key():
|
||||
global key
|
||||
kinfo = shared.fn_retry(lambda: keys.add_key(AddKeyRequest(name="mknet")))
|
||||
|
|
|
|||
26
scenarios/fragments/s3ttfb.py
Normal file
26
scenarios/fragments/s3ttfb.py
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import os
|
||||
from os.path import exists
|
||||
from pathlib import Path
|
||||
from fragments import shared, minio, garage
|
||||
|
||||
s3bin = Path(os.path.dirname(__file__)) / "../../benchmarks/s3ttfb/s3ttfb"
|
||||
|
||||
def common():
|
||||
out = Path(shared.storage_path) / "s3ttfb.csv"
|
||||
shared.log(f"launching s3ttfb ({s3bin})")
|
||||
shared.exec(f"{s3bin} > {out}")
|
||||
shared.log(f"execution done, output written to {out}")
|
||||
|
||||
def on_garage():
|
||||
os.environ['AWS_ACCESS_KEY_ID'] = garage.key.access_key_id
|
||||
os.environ['AWS_SECRET_ACCESS_KEY'] = garage.key.secret_access_key
|
||||
os.environ['ENDPOINT'] = "localhost:3900"
|
||||
os.environ['AWS_REGION'] = "garage"
|
||||
common()
|
||||
|
||||
def on_minio():
|
||||
os.environ['AWS_ACCESS_KEY_ID'] = minio.access_key
|
||||
os.environ['AWS_SECRET_ACCESS_KEY'] = minio.secret_key
|
||||
os.environ['ENDPOINT'] = "localhost:9000"
|
||||
os.environ['AWS_REGION'] = "us-east-1"
|
||||
common()
|
||||
18
scenarios/garage-s3ttfb
Executable file
18
scenarios/garage-s3ttfb
Executable file
|
|
@ -0,0 +1,18 @@
|
|||
#!/usr/bin/env python3
|
||||
from fragments import garage, s3ttfb, shared, flavor
|
||||
import sys
|
||||
|
||||
for fl in sys.argv[1:]:
|
||||
if fl in flavor.garage:
|
||||
garage.version = flavor.garage[fl]
|
||||
|
||||
if shared.id() == 1:
|
||||
garage.deploy_coord(gw = [ 'node1' ])
|
||||
s3ttfb.on_garage()
|
||||
garage.delete_key()
|
||||
garage.destroy()
|
||||
else:
|
||||
garage.deploy_follow()
|
||||
garage.sync_on_key_down()
|
||||
garage.destroy()
|
||||
shared.log("bye")
|
||||
19
scenarios/minio-s3ttfb
Executable file
19
scenarios/minio-s3ttfb
Executable file
|
|
@ -0,0 +1,19 @@
|
|||
#!/usr/bin/env python3
|
||||
from fragments import minio, s3ttfb, shared, flavor
|
||||
import sys
|
||||
|
||||
|
||||
for fl in sys.argv[1:]:
|
||||
if fl in flavor.minio:
|
||||
minio.version = flavor.minio[fl]
|
||||
|
||||
if shared.id() == 1:
|
||||
minio.deploy_coord()
|
||||
s3ttfb.on_minio()
|
||||
minio.delete_sync_bucket()
|
||||
minio.destroy()
|
||||
else:
|
||||
minio.deploy_follow()
|
||||
minio.sync_on_bucket_down()
|
||||
minio.destroy()
|
||||
shared.log("bye")
|
||||
|
|
@ -1,2 +1,2 @@
|
|||
git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@7b1c1faf7a#egg=garage-admin-sdk&subdirectory=python
|
||||
git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@2b685b90be9aa848a9450dd582d1d322b25b5568#egg=garage-admin-sdk&subdirectory=python
|
||||
minio
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ links:
|
|||
- &slow
|
||||
bandwidth: 5M
|
||||
latency: 500us
|
||||
limit: 5
|
||||
- &1000
|
||||
bandwidth: 1000M
|
||||
latency: 100us
|
||||
|
|
@ -13,6 +14,8 @@ servers:
|
|||
<<: *slow
|
||||
- name: node3
|
||||
<<: *slow
|
||||
- name: node4
|
||||
<<: *slow
|
||||
|
||||
global:
|
||||
subnet:
|
||||
|
|
|
|||
Loading…
Reference in a new issue