From a744f72f15f45818e39848bfa4c03b5af6c2cf9a Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Sat, 24 Sep 2022 15:13:40 +0200 Subject: [PATCH] Bench PutObject with many parallel small obj --- artifacts/2022-09-23-warp/plot.R | 1 - benchmarks/s3billion/.gitignore | 1 + benchmarks/s3billion/README.md | 22 ++++ benchmarks/s3billion/go.mod | 8 ++ benchmarks/s3billion/go.sum | 78 ++++++++++++++ benchmarks/s3billion/main.go | 176 +++++++++++++++++++++++++++++++ prepare.py | 2 + scenarios/fragments/s3billion.py | 24 +++++ scenarios/garage-s3billion | 21 ++++ 9 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 benchmarks/s3billion/.gitignore create mode 100644 benchmarks/s3billion/README.md create mode 100644 benchmarks/s3billion/go.mod create mode 100644 benchmarks/s3billion/go.sum create mode 100644 benchmarks/s3billion/main.go create mode 100644 scenarios/fragments/s3billion.py create mode 100755 scenarios/garage-s3billion diff --git a/artifacts/2022-09-23-warp/plot.R b/artifacts/2022-09-23-warp/plot.R index f4fec0a..c5a6bb3 100644 --- a/artifacts/2022-09-23-warp/plot.R +++ b/artifacts/2022-09-23-warp/plot.R @@ -19,4 +19,3 @@ ggplot(s, aes(x=value,y=config)) + theme_classic() ggsave("./plot.png", width=200, height=120, units="mm") -1 \ No newline at end of file diff --git a/benchmarks/s3billion/.gitignore b/benchmarks/s3billion/.gitignore new file mode 100644 index 0000000..86a93cc --- /dev/null +++ b/benchmarks/s3billion/.gitignore @@ -0,0 +1 @@ +s3billion diff --git a/benchmarks/s3billion/README.md b/benchmarks/s3billion/README.md new file mode 100644 index 0000000..95936bc --- /dev/null +++ b/benchmarks/s3billion/README.md @@ -0,0 +1,22 @@ +# s3lat + +## installation + +```bash +go get git.deuxfleurs.fr/quentin/s3lat@latest +``` + +## usage + +```bash +export ENDPOINT=[fc00:9a7a:9e::1]:9000 +export AWS_ACCESS_KEY_ID=minioadmin +export AWS_SECRET_ACCESS_KEY=minioadmin + +s3lat +``` + +## see also + + - https://git.deuxfleurs.fr/quentin/benchmarks + - https://git.deuxfleurs.fr/trinity-1686a/mknet diff --git a/benchmarks/s3billion/go.mod b/benchmarks/s3billion/go.mod new file mode 100644 index 0000000..7c49b15 --- /dev/null +++ b/benchmarks/s3billion/go.mod @@ -0,0 +1,8 @@ +module git.deuxfleurs.fr/Deuxfleurs/mknet/benchmarks/s3billion + +go 1.16 + +require ( + github.com/google/uuid v1.1.1 + github.com/minio/minio-go/v7 v7.0.16 +) diff --git a/benchmarks/s3billion/go.sum b/benchmarks/s3billion/go.sum new file mode 100644 index 0000000..207d6f9 --- /dev/null +++ b/benchmarks/s3billion/go.sum @@ -0,0 +1,78 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= +github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.16 h1:GspaSBS8lOuEUCAqMe0W3UxSoyOA4b4F8PTspRVI+k4= +github.com/minio/minio-go/v7 v7.0.16/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/benchmarks/s3billion/main.go b/benchmarks/s3billion/main.go new file mode 100644 index 0000000..7a2cc6c --- /dev/null +++ b/benchmarks/s3billion/main.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "log" + "math/rand" + "net/http" + "os" + "strconv" + "time" + + "github.com/google/uuid" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type PRNG struct { + rem int64 +} + +func (r *PRNG) Read(p []byte) (n int, err error) { + //log.Printf("rem=%d, buf=%d\n", r.rem, len(p)) + if int64(len(p)) > r.rem { + p = p[:r.rem] + } + + if int64(len(p)) > r.rem { + log.Fatal("LOGIC ERROR") + } + + n, err = rand.Read(p) + if err != nil { + return + } + r.rem -= int64(n) + if r.rem <= 0 { + err = io.EOF + //log.Printf("PRNG file has been fully read. rem=%d,n=%d,err=%s\n", r.rem, n, err) + } + return +} + +func putObj(mc *minio.Client, buck string, size int64) error { + prng := new(PRNG) + prng.rem = size + + key := uuid.New().String() + + _, err := mc.PutObject( + context.Background(), + buck, + key, + prng, + size, + minio.PutObjectOptions{ContentType: "application/octet-stream"}, + ) + + return err +} + +func main() { + fmt.Printf("total_objects,batch_dur_nanoseconds\n") + + minio.MaxRetry = 1 + + _, isSSL := os.LookupEnv("SSL") + opts := minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"), ""), + Secure: isSSL, + } + + if region, ok := os.LookupEnv("AWS_REGION"); ok { + opts.Region = region + } + + if _, ok := os.LookupEnv("SSL_INSECURE"); ok { + opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + } + + mc, err := minio.New(os.Getenv("ENDPOINT"), &opts) + if err != nil { + log.Fatal("failed connect", err) + return + } + + thread := 32 + if env_thread, ok := os.LookupEnv("THREAD"); ok { + tmp, err := strconv.Atoi(env_thread) + if err != nil { + log.Fatalf("invalid value for THREAD: %v\n", env_thread) + } + thread = tmp + } + + batch_size := 256 + if env_batch_size, ok := os.LookupEnv("BATCH_SIZE"); ok { + tmp, err := strconv.Atoi(env_batch_size) + if err != nil { + log.Fatalf("invalid value for BATCH_SIZE: %v\n", env_batch_size) + } + batch_size = tmp + } + + batch_count := 128 + if env_batch_count, ok := os.LookupEnv("BATCH_COUNT"); ok { + tmp, err := strconv.Atoi(env_batch_count) + if err != nil { + log.Fatalf("invalid value for BATCH_COUNT: %v\n", env_batch_count) + } + batch_count = tmp + } + + obj_size := 16 + if env_obj_size, ok := os.LookupEnv("OBJ_SIZE"); ok { + tmp, err := strconv.Atoi(env_obj_size) + if err != nil { + log.Fatalf("invalid value for OBJ_SIZE: %v\n", env_obj_size) + } + obj_size = tmp + } + + total_obj := thread * batch_size * batch_count + total_size := total_obj * obj_size + log.Printf("if bench succeed, %v objects (%v bytes) will be created\n", total_obj, total_size) + + // Create Bucket + buck := uuid.New().String() + err = mc.MakeBucket(context.Background(), buck, minio.MakeBucketOptions{}) + if err != nil { + log.Fatal(err) + return + } + log.Printf("created bucket %s\n", buck) + + // Start sending... + for bc := 0; bc < batch_count; bc++ { + log.Printf("batch %d/%d - start\n", bc+1, batch_count) + + start := time.Now() + syn := make(chan error) + + for tc := 0; tc < thread; tc++ { + go func() { + for bs := 0; bs < batch_size; bs++ { + err := putObj(mc, buck, int64(obj_size)) + if err != nil { + syn <- err + return + } + } + syn <- nil + + }() + } + log.Printf("batch %d/%d - all threads started\n", bc+1, batch_count) + + errCount := 0 + for tc := 0; tc < thread; tc++ { + cerr := <-syn + if cerr != nil { + errCount += 1 + log.Printf("thread %d/%d failed with %s\n", tc, thread, cerr) + } + } + if errCount > 0 { + log.Fatal("Too many errors, exiting...") + return + } + elapsed := time.Since(start) + fmt.Printf("%d,%v\n", bc * thread * batch_size, elapsed.Nanoseconds()) + log.Printf("batch %d/%d - all threads returned\n", bc+1, batch_count) + } +} diff --git a/prepare.py b/prepare.py index d6e2ab9..e7021ea 100755 --- a/prepare.py +++ b/prepare.py @@ -16,6 +16,8 @@ os.chdir(me / "benchmarks" / "s3lat") shared.exec("go build") os.chdir(me / "benchmarks" / "s3ttfb") shared.exec("go build") +os.chdir(me / "benchmarks" / "s3billion") +shared.exec("go build") os.chdir(me) print("--- install python dependencies ---") diff --git a/scenarios/fragments/s3billion.py b/scenarios/fragments/s3billion.py new file mode 100644 index 0000000..7fdb74c --- /dev/null +++ b/scenarios/fragments/s3billion.py @@ -0,0 +1,24 @@ +import os +from os.path import exists +from pathlib import Path +from fragments import shared, minio, garage + +s3bin = Path(os.path.dirname(__file__)) / "../../benchmarks/s3billion/s3billion" + +def common(): + out = Path(shared.storage_path) / "s3billion.csv" + shared.log(f"launching s3billion ({s3bin})") + shared.exec(f"{s3bin} > {out}") + shared.log(f"execution done, output written to {out}") + +def on_garage(): + os.environ['AWS_ACCESS_KEY_ID'] = garage.key.access_key_id + os.environ['AWS_SECRET_ACCESS_KEY'] = garage.key.secret_access_key + os.environ['ENDPOINT'] = f"[{os.environ['IP']}]:3900" + common() + +def on_minio(): + os.environ['AWS_ACCESS_KEY_ID'] = minio.access_key + os.environ['AWS_SECRET_ACCESS_KEY'] = minio.secret_key + os.environ['ENDPOINT'] = f"[{os.environ['IP']}]:9000" + common() diff --git a/scenarios/garage-s3billion b/scenarios/garage-s3billion new file mode 100755 index 0000000..55653f6 --- /dev/null +++ b/scenarios/garage-s3billion @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +from fragments import garage, s3billion, shared, flavor +import sys, os + +for fl in sys.argv[1:]: + if fl in flavor.garage: + garage.version = flavor.garage[fl] + +#os.environ['BATCH_COUNT'] = "2" +shared.exec("ulimit -n 65535") + +if shared.id() == 1: + garage.deploy_coord() + s3billion.on_garage() + garage.delete_key() + garage.destroy() +else: + garage.deploy_follow() + garage.sync_on_key_down() + garage.destroy() +shared.log("bye")