Scenario based benchmarks will be ready soon

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "benchmarks/warp"]
 	path = benchmarks/warp
 	url = https://github.com/minio/warp
+[submodule "analysis/quentin"]
+	path = analysis/quentin
+	url = https://git.deuxfleurs.fr/quentin/benchmarks

README.md

@@ -1,43 +1,49 @@
 # mknet
 
-## Installation
+mknet is a tool to simulate various network topologies
+locally thanks to network namespaces and traffic control (tc).
+
+## Prepare your environment
+
+Get the repository and its submodules:
 
 ```bash
-sudo pip3 install git+https://git.deuxfleurs.fr/trinity-1686a/mknet
+git clone https://git.deuxfleurs.fr/Deuxfleurs/mknet.git
+cd mknet
+git submodule update --init
 ```
 
-## Usage
+Compile benchmark tools:
 
 ```bash
-sudo rm -rf /tmp/garage-testnet/ # always start by deleting previous run
+( cd benchmarks/warp && go build )
-sudo mknet create ./config.yml
+( cd benchmarks/s3concurrent && go build )
-sudo mknet run-all example/deploy_garage.sh
+( cd benchmarks/s3lat && go build )
-sudo mknet run dc1:dc1s1 garage -c /tmp/garage-testnet/dc1/dc1s1/garage.toml status
-sudo mknet destroy
 ```
 
-## Instrumented daemons
+Switch to root as it is required by our tool,
-
+and setup your python environment.
-If you want to use the scripts provided in the `example` folder,
-you must add to your path some tools.
-
-### Garage (`deploy_garage.sh`)
 
 ```bash
-# see versions on https://garagehq.deuxfleurs.fr/_releases.html
+sudo -i
-export GRG_ARCH=x86_64-unknown-linux-musl
+pip3 install --user .
-export GRG_VERSION=v0.5.0
+( cd scenarios && pip3 install -r requirements.txt )
-
-sudo wget https://garagehq.deuxfleurs.fr/_releases/${GRG_VERSION}/${GRG_ARCH}/garage -O /usr/local/bin/garage
-sudo chmod +x /usr/local/bin/garage
-
-garage help
 ```
 
-### Minio (`deploy_minio.py`)
+## Run a test
 
-```
+(Not yet implemented)
-sudo wget https://dl.min.io/server/minio/release/linux-amd64/minio -O /usr/local/bin/minio
+
-sudo chmod +x /usr/local/bin/minio
+```bash
+./mknet scenario ./benchmarks/garage-s3lat ./topo/single-dc.yml
 ```
 
+## Manual usage
+
+```bash
+./mknet create ./topo/with-vdsl.yml
+./mknet run-all ./scenarios/garage-manual
+./mknet run dc1:dc1s1 /tmp/mknet-bin/garage* -c /tmp/mknet-store/garage/dc1/dc1s1/garage.toml status
+./mknet run-all ./scenarios/clean
+./mknet destroy
+

analysis/quentin

@@ -0,0 +1 @@
+Subproject commit 902555b9d5db594fdad927f51c2d8984d6e9f6a5

benchmarks/s3concurrent/.gitignore

@@ -0,0 +1 @@
+s3concurrent

benchmarks/s3concurrent/main.go

@@ -1,17 +1,17 @@
 package main
 
 import (
-    "context"
+	"context"
-    "crypto/tls"
+	"crypto/tls"
-    "io"
+	"io"
-    "log"
+	"log"
-    "math/rand"
+	"math/rand"
-    "net/http"
+	"net/http"
-    "os"
+	"os"
 
+	"github.com/google/uuid"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
-    "github.com/google/uuid"
 )
 
 func buildMc() (*minio.Client, error) {
@@ -30,92 +30,93 @@ func buildMc() (*minio.Client, error) {
 	}
 
 	mc, err := minio.New(os.Getenv("ENDPOINT"), &opts)
-    return mc, err
+	return mc, err
 }
 
-type PRNG struct { 
+type PRNG struct {
-    rem int64
+	rem int64
 }
-func (r *PRNG)  Read(p []byte) (n int, err error) {
-    //log.Printf("rem=%d, buf=%d\n", r.rem, len(p))
-    if int64(len(p)) > r.rem {
-        p = p[:r.rem]
-    }
 
-    if int64(len(p)) > r.rem {
+func (r *PRNG) Read(p []byte) (n int, err error) {
-        log.Fatal("LOGIC ERROR")
+	//log.Printf("rem=%d, buf=%d\n", r.rem, len(p))
-    }
+	if int64(len(p)) > r.rem {
+		p = p[:r.rem]
+	}
 
-    n, err = rand.Read(p)
+	if int64(len(p)) > r.rem {
-    if err != nil {
+		log.Fatal("LOGIC ERROR")
-        return
+	}
-    }
+
-    r.rem -= int64(n)
+	n, err = rand.Read(p)
-    if r.rem <= 0 {
+	if err != nil {
-        err = io.EOF
+		return
-        //log.Printf("PRNG file has been fully read. rem=%d,n=%d,err=%s\n", r.rem, n, err)
+	}
-    }
+	r.rem -= int64(n)
-    return
+	if r.rem <= 0 {
+		err = io.EOF
+		//log.Printf("PRNG file has been fully read. rem=%d,n=%d,err=%s\n", r.rem, n, err)
+	}
+	return
 }
 
 func putObj(buck string, size int64) error {
-    mc, err := buildMc()
+	mc, err := buildMc()
-    if err != nil {
+	if err != nil {
-        return err
+		return err
-    }
+	}
 
-    prng := new(PRNG)
+	prng := new(PRNG)
-    prng.rem = size
+	prng.rem = size
 
-    key := uuid.New().String()
+	key := uuid.New().String()
 
-    _, err = mc.PutObject(
+	_, err = mc.PutObject(
-        context.Background(), 
+		context.Background(),
-        buck, 
+		buck,
-        key, 
+		key,
-        prng, 
+		prng,
-        size, 
+		size,
-        minio.PutObjectOptions{ContentType:"application/octet-stream"},
+		minio.PutObjectOptions{ContentType: "application/octet-stream"},
-    )
+	)
 
-    return err
+	return err
 }
 
 func main() {
-    minio.MaxRetry = 1
+	minio.MaxRetry = 1
-    mc, err := buildMc()
+	mc, err := buildMc()
 	if err != nil {
 		log.Fatal("failed connect", err)
 		return
 	}
 
-    // Create Bucket
+	// Create Bucket
-    buck := uuid.New().String()
+	buck := uuid.New().String()
-    err = mc.MakeBucket(context.Background(), buck, minio.MakeBucketOptions{ })
+	err = mc.MakeBucket(context.Background(), buck, minio.MakeBucketOptions{})
-    if err != nil {
+	if err != nil {
-        log.Fatal(err)
+		log.Fatal(err)
-        return
+		return
-    }
+	}
-    log.Printf("created bucket %s\n", buck)
+	log.Printf("created bucket %s\n", buck)
 
-    // Send to bucket
+	// Send to bucket
-    for i := 1; i <= 16; i++ {
+	for i := 1; i <= 16; i++ {
-        log.Printf("start concurrent loop with %d coroutines\n", i)
+		log.Printf("start concurrent loop with %d coroutines\n", i)
-        syn := make(chan error)
+		syn := make(chan error)
-        for j := 1; j <= i; j++ {
+		for j := 1; j <= i; j++ {
-            go func() {
+			go func() {
-                syn <- putObj(buck, 1024 * 1024)
+				syn <- putObj(buck, 1024*1024)
-            }()
+			}()
-        }
+		}
 
-        for j := 1; j <= i; j++ {
+		for j := 1; j <= i; j++ {
-            cerr := <-syn
+			cerr := <-syn
-            if cerr != nil {
+			if cerr != nil {
-                log.Printf("%d/%d failed with %s\n", j, i, cerr)
+				log.Printf("%d/%d failed with %s\n", j, i, cerr)
-            }
+			}
-        }
+		}
-        log.Printf("done, %d coroutines returned\n", i)
+		log.Printf("done, %d coroutines returned\n", i)
-    }
+	}
 
-    log.Println("done")
+	log.Println("done")
 }

benchmarks/s3lat/.gitignore

@@ -0,0 +1 @@
+s3lat

scenarios/fragments/garage.py

@@ -30,7 +30,7 @@ keys = key_api.KeyApi(api)
 
 # Setup, launch on import
 storage_path = Path(shared.storage_path) / "garage" / env['HOST']
-if 'ZONE' in env:
+if 'ZONE' in env and env['ZONE'] != "":
     storage_path = Path(shared.storage_path) / "garage" / env['ZONE'] / env['HOST']
 config = storage_path / "garage.toml"
 env['GARAGE_CONFIG_FILE'] = str(config)
@@ -117,14 +117,17 @@ admin_token = "{admin}"
         f.write(json.dumps({
             "node_addr": f"{node_id}@{env['IP']}:3901",
             "node_id": node_id,
-            "zone": env['ZONE'],
+            "zone": env['ZONE'] if 'ZONE' in env and env['ZONE'] != "" else env['HOST'],
             "host": env['HOST'],
         }))
 
 def destroy():
     dpid = Path(storage_path) / "daemon.pid"
     if exists(dpid):
-        shared.exec(f"kill -9 $(cat {dpid})")
+        try:
+            shared.exec(f"kill -9 $(cat {dpid})")
+        except:
+            pass
         shared.exec(f"rm -f {dpid}")
     if len(str(storage_path)) < 8: # arbitrary, stupid safe guard
         print(storage_path)

scenarios/garage-manual

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+from fragments import garage, s3lat, shared
+
+if shared.id() == 1:
+    garage.deploy_coord()
+else:
+    garage.deploy_follow()