Bench PutObject with many parallel small obj
This commit is contained in:
parent
ac4f100da1
commit
a744f72f15
GPG key ID:
E9602264D639FF68
9 changed files with 332 additions and 1 deletions
|
|
@ -19,4 +19,3 @@ ggplot(s, aes(x=value,y=config)) +
|
||||||
theme_classic()
|
theme_classic()
|
||||||
|
|
||||||
ggsave("./plot.png", width=200, height=120, units="mm")
|
ggsave("./plot.png", width=200, height=120, units="mm")
|
||||||
1
|
|
||||||
1
benchmarks/s3billion/.gitignore
vendored
Normal file
1
benchmarks/s3billion/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
s3billion
|
||||||
22
benchmarks/s3billion/README.md
Normal file
22
benchmarks/s3billion/README.md
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
# s3lat
|
||||||
|
|
||||||
|
## installation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go get git.deuxfleurs.fr/quentin/s3lat@latest
|
||||||
|
```
|
||||||
|
|
||||||
|
## usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export ENDPOINT=[fc00:9a7a:9e::1]:9000
|
||||||
|
export AWS_ACCESS_KEY_ID=minioadmin
|
||||||
|
export AWS_SECRET_ACCESS_KEY=minioadmin
|
||||||
|
|
||||||
|
s3lat
|
||||||
|
```
|
||||||
|
|
||||||
|
## see also
|
||||||
|
|
||||||
|
- https://git.deuxfleurs.fr/quentin/benchmarks
|
||||||
|
- https://git.deuxfleurs.fr/trinity-1686a/mknet
|
||||||
8
benchmarks/s3billion/go.mod
Normal file
8
benchmarks/s3billion/go.mod
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
module git.deuxfleurs.fr/Deuxfleurs/mknet/benchmarks/s3billion
|
||||||
|
|
||||||
|
go 1.16
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/google/uuid v1.1.1
|
||||||
|
github.com/minio/minio-go/v7 v7.0.16
|
||||||
|
)
|
||||||
78
benchmarks/s3billion/go.sum
Normal file
78
benchmarks/s3billion/go.sum
Normal file
|
|
@ -0,0 +1,78 @@
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
|
||||||
|
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||||
|
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||||
|
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||||
|
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||||
|
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||||
|
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||||
|
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||||
|
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||||
|
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||||
|
github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
|
||||||
|
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||||
|
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||||
|
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
|
||||||
|
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
|
||||||
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||||
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||||
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
|
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
|
||||||
|
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
|
||||||
|
github.com/minio/minio-go/v7 v7.0.16 h1:GspaSBS8lOuEUCAqMe0W3UxSoyOA4b4F8PTspRVI+k4=
|
||||||
|
github.com/minio/minio-go/v7 v7.0.16/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
|
||||||
|
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
|
||||||
|
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||||
|
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||||
|
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
|
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||||
|
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
|
||||||
|
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
|
||||||
|
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||||
|
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
|
||||||
|
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||||
|
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||||
|
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||||
|
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||||
|
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
|
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||||
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
|
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU=
|
||||||
|
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||||
|
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
|
||||||
|
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
|
||||||
|
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||||
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
|
||||||
|
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||||
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||||
|
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
176
benchmarks/s3billion/main.go
Normal file
176
benchmarks/s3billion/main.go
Normal file
|
|
@ -0,0 +1,176 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"math/rand"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/minio/minio-go/v7"
|
||||||
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
|
)
|
||||||
|
|
||||||
|
type PRNG struct {
|
||||||
|
rem int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *PRNG) Read(p []byte) (n int, err error) {
|
||||||
|
//log.Printf("rem=%d, buf=%d\n", r.rem, len(p))
|
||||||
|
if int64(len(p)) > r.rem {
|
||||||
|
p = p[:r.rem]
|
||||||
|
}
|
||||||
|
|
||||||
|
if int64(len(p)) > r.rem {
|
||||||
|
log.Fatal("LOGIC ERROR")
|
||||||
|
}
|
||||||
|
|
||||||
|
n, err = rand.Read(p)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.rem -= int64(n)
|
||||||
|
if r.rem <= 0 {
|
||||||
|
err = io.EOF
|
||||||
|
//log.Printf("PRNG file has been fully read. rem=%d,n=%d,err=%s\n", r.rem, n, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func putObj(mc *minio.Client, buck string, size int64) error {
|
||||||
|
prng := new(PRNG)
|
||||||
|
prng.rem = size
|
||||||
|
|
||||||
|
key := uuid.New().String()
|
||||||
|
|
||||||
|
_, err := mc.PutObject(
|
||||||
|
context.Background(),
|
||||||
|
buck,
|
||||||
|
key,
|
||||||
|
prng,
|
||||||
|
size,
|
||||||
|
minio.PutObjectOptions{ContentType: "application/octet-stream"},
|
||||||
|
)
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
fmt.Printf("total_objects,batch_dur_nanoseconds\n")
|
||||||
|
|
||||||
|
minio.MaxRetry = 1
|
||||||
|
|
||||||
|
_, isSSL := os.LookupEnv("SSL")
|
||||||
|
opts := minio.Options{
|
||||||
|
Creds: credentials.NewStaticV4(os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY"), ""),
|
||||||
|
Secure: isSSL,
|
||||||
|
}
|
||||||
|
|
||||||
|
if region, ok := os.LookupEnv("AWS_REGION"); ok {
|
||||||
|
opts.Region = region
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := os.LookupEnv("SSL_INSECURE"); ok {
|
||||||
|
opts.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
|
||||||
|
}
|
||||||
|
|
||||||
|
mc, err := minio.New(os.Getenv("ENDPOINT"), &opts)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("failed connect", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
thread := 32
|
||||||
|
if env_thread, ok := os.LookupEnv("THREAD"); ok {
|
||||||
|
tmp, err := strconv.Atoi(env_thread)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("invalid value for THREAD: %v\n", env_thread)
|
||||||
|
}
|
||||||
|
thread = tmp
|
||||||
|
}
|
||||||
|
|
||||||
|
batch_size := 256
|
||||||
|
if env_batch_size, ok := os.LookupEnv("BATCH_SIZE"); ok {
|
||||||
|
tmp, err := strconv.Atoi(env_batch_size)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("invalid value for BATCH_SIZE: %v\n", env_batch_size)
|
||||||
|
}
|
||||||
|
batch_size = tmp
|
||||||
|
}
|
||||||
|
|
||||||
|
batch_count := 128
|
||||||
|
if env_batch_count, ok := os.LookupEnv("BATCH_COUNT"); ok {
|
||||||
|
tmp, err := strconv.Atoi(env_batch_count)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("invalid value for BATCH_COUNT: %v\n", env_batch_count)
|
||||||
|
}
|
||||||
|
batch_count = tmp
|
||||||
|
}
|
||||||
|
|
||||||
|
obj_size := 16
|
||||||
|
if env_obj_size, ok := os.LookupEnv("OBJ_SIZE"); ok {
|
||||||
|
tmp, err := strconv.Atoi(env_obj_size)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("invalid value for OBJ_SIZE: %v\n", env_obj_size)
|
||||||
|
}
|
||||||
|
obj_size = tmp
|
||||||
|
}
|
||||||
|
|
||||||
|
total_obj := thread * batch_size * batch_count
|
||||||
|
total_size := total_obj * obj_size
|
||||||
|
log.Printf("if bench succeed, %v objects (%v bytes) will be created\n", total_obj, total_size)
|
||||||
|
|
||||||
|
// Create Bucket
|
||||||
|
buck := uuid.New().String()
|
||||||
|
err = mc.MakeBucket(context.Background(), buck, minio.MakeBucketOptions{})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
log.Printf("created bucket %s\n", buck)
|
||||||
|
|
||||||
|
// Start sending...
|
||||||
|
for bc := 0; bc < batch_count; bc++ {
|
||||||
|
log.Printf("batch %d/%d - start\n", bc+1, batch_count)
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
syn := make(chan error)
|
||||||
|
|
||||||
|
for tc := 0; tc < thread; tc++ {
|
||||||
|
go func() {
|
||||||
|
for bs := 0; bs < batch_size; bs++ {
|
||||||
|
err := putObj(mc, buck, int64(obj_size))
|
||||||
|
if err != nil {
|
||||||
|
syn <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
syn <- nil
|
||||||
|
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
log.Printf("batch %d/%d - all threads started\n", bc+1, batch_count)
|
||||||
|
|
||||||
|
errCount := 0
|
||||||
|
for tc := 0; tc < thread; tc++ {
|
||||||
|
cerr := <-syn
|
||||||
|
if cerr != nil {
|
||||||
|
errCount += 1
|
||||||
|
log.Printf("thread %d/%d failed with %s\n", tc, thread, cerr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if errCount > 0 {
|
||||||
|
log.Fatal("Too many errors, exiting...")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
fmt.Printf("%d,%v\n", bc * thread * batch_size, elapsed.Nanoseconds())
|
||||||
|
log.Printf("batch %d/%d - all threads returned\n", bc+1, batch_count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -16,6 +16,8 @@ os.chdir(me / "benchmarks" / "s3lat")
|
||||||
shared.exec("go build")
|
shared.exec("go build")
|
||||||
os.chdir(me / "benchmarks" / "s3ttfb")
|
os.chdir(me / "benchmarks" / "s3ttfb")
|
||||||
shared.exec("go build")
|
shared.exec("go build")
|
||||||
|
os.chdir(me / "benchmarks" / "s3billion")
|
||||||
|
shared.exec("go build")
|
||||||
os.chdir(me)
|
os.chdir(me)
|
||||||
|
|
||||||
print("--- install python dependencies ---")
|
print("--- install python dependencies ---")
|
||||||
|
|
|
||||||
24
scenarios/fragments/s3billion.py
Normal file
24
scenarios/fragments/s3billion.py
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
import os
|
||||||
|
from os.path import exists
|
||||||
|
from pathlib import Path
|
||||||
|
from fragments import shared, minio, garage
|
||||||
|
|
||||||
|
s3bin = Path(os.path.dirname(__file__)) / "../../benchmarks/s3billion/s3billion"
|
||||||
|
|
||||||
|
def common():
|
||||||
|
out = Path(shared.storage_path) / "s3billion.csv"
|
||||||
|
shared.log(f"launching s3billion ({s3bin})")
|
||||||
|
shared.exec(f"{s3bin} > {out}")
|
||||||
|
shared.log(f"execution done, output written to {out}")
|
||||||
|
|
||||||
|
def on_garage():
|
||||||
|
os.environ['AWS_ACCESS_KEY_ID'] = garage.key.access_key_id
|
||||||
|
os.environ['AWS_SECRET_ACCESS_KEY'] = garage.key.secret_access_key
|
||||||
|
os.environ['ENDPOINT'] = f"[{os.environ['IP']}]:3900"
|
||||||
|
common()
|
||||||
|
|
||||||
|
def on_minio():
|
||||||
|
os.environ['AWS_ACCESS_KEY_ID'] = minio.access_key
|
||||||
|
os.environ['AWS_SECRET_ACCESS_KEY'] = minio.secret_key
|
||||||
|
os.environ['ENDPOINT'] = f"[{os.environ['IP']}]:9000"
|
||||||
|
common()
|
||||||
21
scenarios/garage-s3billion
Executable file
21
scenarios/garage-s3billion
Executable file
|
|
@ -0,0 +1,21 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from fragments import garage, s3billion, shared, flavor
|
||||||
|
import sys, os
|
||||||
|
|
||||||
|
for fl in sys.argv[1:]:
|
||||||
|
if fl in flavor.garage:
|
||||||
|
garage.version = flavor.garage[fl]
|
||||||
|
|
||||||
|
#os.environ['BATCH_COUNT'] = "2"
|
||||||
|
shared.exec("ulimit -n 65535")
|
||||||
|
|
||||||
|
if shared.id() == 1:
|
||||||
|
garage.deploy_coord()
|
||||||
|
s3billion.on_garage()
|
||||||
|
garage.delete_key()
|
||||||
|
garage.destroy()
|
||||||
|
else:
|
||||||
|
garage.deploy_follow()
|
||||||
|
garage.sync_on_key_down()
|
||||||
|
garage.destroy()
|
||||||
|
shared.log("bye")
|
||||||
Loading…
Reference in a new issue