Create a prepare script

.gitignore

@@ -3,3 +3,4 @@ __pycache__
 *.swp
 build
 mknet.egg-info
+.venv

README.md

@@ -5,42 +5,27 @@ locally thanks to network namespaces and traffic control (tc).
 
 ## Prepare your environment
 
-Get the repository and its submodules:
+Get the repository:
 
 ```bash
 git clone https://git.deuxfleurs.fr/Deuxfleurs/mknet.git
 cd mknet
-git submodule update --init
 ```
 
-Compile benchmark tools:
+Run our configuration script:
 
 ```bash
-( cd benchmarks/warp && go build )
-( cd benchmarks/s3concurrent && go build )
-( cd benchmarks/s3lat && go build )
+./prepare.py
 ```
 
-Switch to root as it is required by our tool,
-and setup your python environment.
+Now, you are ready to launch an experiment:
 
 ```bash
 sudo -i
-pip3 install --user .
-( cd scenarios && pip3 install --user -r requirements.txt )
+source .venv/bin/activate
+./mknet scenario ./topo/50ms.yml ./scenarios/garage-s3lat garage-v0.8
 ```
 
-Sometimes, it is required to add manually your local python lib folder to the environment:
-
-```bash
-export PYTHONPATH=$PYTHONPATH:$HOME/.local/lib/python3.10/site-packages/
-```
-
-You can check that everything has been installed by running in a python REPL:
-
-```
-import garage_admin_sdk
-```
 
 ## Topologies
 

prepare.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+from pathlib import Path
+from scenarios.fragments import shared
+import os, venv
+
+me = Path(os.path.dirname(__file__))
+print("--- git submodule ---")
+shared.exec("git submodule update --init")
+
+print("--- compile go benchmarks ---")
+os.chdir(me / "benchmarks" / "warp")
+shared.exec("go build")
+os.chdir(me / "benchmarks" / "s3concurrent")
+shared.exec("go build")
+os.chdir(me / "benchmarks" / "s3lat")
+shared.exec("go build")
+os.chdir(me)
+
+print("--- install python dependencies ---")
+venv.create(".venv", with_pip=True)
+shared.exec("""
+source .venv/bin/activate
+which python3
+python3 -m pip install .
+python3 -m pip install -r scenarios/requirements.txt
+""")
+
+print("--- download garage artifacts ---")
+from scenarios.fragments import garage
+garage.download()
+
+print("""done! now, run:
+> sudo -i
+> source ./.venv/bin/activate
+> ./mknet scenario ./topo/with-vdsl.yml ./scenarios/garage-s3lat garage-v0.8
+""")

scenarios/fragments/garage.py

@@ -1,22 +1,32 @@
-import glob, json, requests, time, garage_admin_sdk
+import glob, json, time
 from os.path import exists
 from os import environ as env
 from pathlib import Path
-from fragments import shared
-from garage_admin_sdk.api import nodes_api, layout_api, key_api
-from garage_admin_sdk.model.node_cluster_info import NodeClusterInfo
-from garage_admin_sdk.model.layout_version import LayoutVersion
-from garage_admin_sdk.model.add_key_request import AddKeyRequest
-from garage_admin_sdk.model.update_key_request import UpdateKeyRequest
-from garage_admin_sdk.model.update_key_request_allow import UpdateKeyRequestAllow
+from . import shared
+
+try:
+    import garage_admin_sdk
+    from garage_admin_sdk.api import nodes_api, layout_api, key_api
+    from garage_admin_sdk.model.node_cluster_info import NodeClusterInfo
+    from garage_admin_sdk.model.layout_version import LayoutVersion
+    from garage_admin_sdk.model.add_key_request import AddKeyRequest
+    from garage_admin_sdk.model.update_key_request import UpdateKeyRequest
+    from garage_admin_sdk.model.update_key_request_allow import UpdateKeyRequestAllow
+except:
+    pass
 
 storage_path = "./i/am/not/defined"
 rpc_secret = "3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba"
 metrics = "cacce0b2de4bc2d9f5b5fdff551e01ac1496055aed248202d415398987e35f81"
 admin = "ae8cb40ea7368bbdbb6430af11cca7da833d3458a5f52086f4e805a570fb5c2a"
-path = None
 key = None
 
+version_flavor = {
+    "garage-local": { "path": "./garage/target/release/garage" },
+    "garage-v0.7": { "version": "v0.7.3", "target": "x86_64-unknown-linux-musl" },
+    "garage-v0.8": { "version": "89b8087ba81c508ba382aa6c9cb6bb3afa6a43c8", "target": "x86_64-unknown-linux-musl" },
+}
+version = version_flavor["garage-v0.7"]
 
 configuration = garage_admin_sdk.Configuration(
     host = "http://localhost:3903/v0",
@@ -29,15 +39,15 @@ keys = key_api.KeyApi(api)
 
 
 # Setup, launch on import
-storage_path = Path(shared.storage_path) / "garage" / env['HOST']
-if 'ZONE' in env and env['ZONE'] != "":
+if 'HOST' in env:
+    storage_path = Path(shared.storage_path) / "garage" / env['HOST']
+    if 'ZONE' in env and env['ZONE'] != "":
         storage_path = Path(shared.storage_path) / "garage" / env['ZONE'] / env['HOST']
-config = storage_path / "garage.toml"
-env['GARAGE_CONFIG_FILE'] = str(config)
+    config = storage_path / "garage.toml"
+    env['GARAGE_CONFIG_FILE'] = str(config)
 
 def deploy_coord(version=None, target=None):
     destroy()
-    from_ci(version, target)
     shared.log("start daemon")
     daemon()
     shared.log("discover nodes")
@@ -50,32 +60,32 @@ def deploy_coord(version=None, target=None):
 
 def deploy_follow(version=None, target=None):
     destroy()
-    from_ci(version, target)
     shared.log("start daemon")
     daemon()
     shared.log("wait for coord")
     sync_on_key_up()
     shared.log("ready")
 
-def from_local(p):
-    global path
-    path = p
-    shared.exec(f"{p} --version")
+def path(vers=None):
+    if vers is None:
+        vers = version
 
-def from_ci(version=None, target=None):
-    global path
-    version = version or "v0.7.3"
-    target = target or "x86_64-unknown-linux-musl"
+    if "path" in vers: return vers["path"]
+    else: 
+      binary = f"garage-{vers['target']}-{vers['version']}"
+      return Path(shared.binary_path) / binary
 
-    binary = f"garage-{target}-{version}"
-    path = Path(shared.binary_path) / binary
-    if shared.id() != 1: return 
+def download():
+    for flav, version in version_flavor.items():
+        if "path" in version: continue
+
+        p = path(vers=version) 
+        if exists(p): continue
         
-    if not exists(path):
         shared.exec(f"mkdir -p {shared.binary_path}")
-        shared.exec(f"wget https://garagehq.deuxfleurs.fr/_releases/{version}/{target}/garage -O {path}")
-    shared.exec(f"chmod +x {path}")
-    shared.exec(f"{path} --version")
+        shared.exec(f"wget https://garagehq.deuxfleurs.fr/_releases/{version['version']}/{version['target']}/garage -O {p}")
+        shared.exec(f"chmod +x {p}")
+        shared.exec(f"{p} --version")
 
 def daemon():
     shared.exec(f"mkdir -p {storage_path}")
@@ -108,7 +118,7 @@ metrics_token = "{metrics}"
 admin_token = "{admin}"
         """)
 
-    shared.exec(f"{path} server 2>> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid")
+    shared.exec(f"{path()} server 2>> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid")
     time.sleep(1)
 
     node_info = storage_path / "node_info"

scenarios/fragments/warp.py

@@ -5,6 +5,12 @@ from fragments import shared, garage
 
 warp_bin = Path(os.path.dirname(__file__)) / "../../benchmarks/warp/warp"
 
+bench_flavor = {
+    "warp-fast": "mixed --obj.size 5M --objects 200 --duration=1m",
+    "warp-default": "mixed"
+}
+bench = warp_bench_flavor["warp-fast"]
+
 def on_garage(params="mixed"):
     shared.log(f"launching warp {warp_bin}")
     shared.exec(f"{warp_bin} {params} --host={os.environ['IP']}:3900  --access-key={garage.key.access_key_id} --secret-key={garage.key.secret_access_key}")

scenarios/garage-s3concurrent

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+from fragments import garage, s3lat, shared
+import sys
+
+for flavor in sys.argv[1:]:
+    if flavor in garage.version_flavor:
+        garage.version = garage.version_flavor[flavor]
+
+if shared.id() == 1:
+    garage.deploy_coord(version=garage.version)
+    s3concurrent.on_garage()
+    garage.delete_key()
+    garage.destroy()
+else:
+    garage.deploy_follow(version=garage.version)
+    garage.sync_on_key_down()
+    garage.destroy()
+shared.log("bye")

scenarios/garage-s3lat

@@ -2,22 +2,17 @@
 from fragments import garage, s3lat, shared
 import sys
 
-garage_version_flavor = {
-    "garage-v0.7": "v0.7.3",
-    "garage-v0.8": "89b8087ba81c508ba382aa6c9cb6bb3afa6a43c8"
-}
-garage_version = garage_version_flavor["garage-v0.7"]
 for flavor in sys.argv[1:]:
-    if flavor in garage_version_flavor:
-        garage_version = garage_version_flavor[flavor]
+    if flavor in garage.version_flavor:
+        garage.version = garage.version_flavor[flavor]
 
 if shared.id() == 1:
-    garage.deploy_coord(version=garage_version)
+    garage.deploy_coord(version=garage.version)
     s3lat.on_garage()
     garage.delete_key()
     garage.destroy()
 else:
-    garage.deploy_follow(version=garage_version)
+    garage.deploy_follow(version=garage.version)
     garage.sync_on_key_down()
     garage.destroy()
 shared.log("bye")

scenarios/garage-warp

@@ -2,32 +2,19 @@
 from fragments import garage, warp, shared
 import sys
 
-garage_version_flavor = {
-    "garage-v0.7": "v0.7.3",
-    "garage-v0.8": "89b8087ba81c508ba382aa6c9cb6bb3afa6a43c8"
-}
-garage_version = garage_version_flavor["garage-v0.7"]
-
-warp_bench_flavor = {
-    "warp-fast": "mixed --obj.size 5M --objects 200 --duration=1m",
-    "warp-default": "mixed"
-}
-warp_bench = warp_bench_flavor["warp-fast"]
-
-
 for flavor in sys.argv[1:]:
-    if flavor in garage_version_flavor:
-        garage_version = garage_version_flavor[flavor]
-    if flavor in warp_bench_flavor:
-        warp_bench = warp_bench_flavor[flavor]
+    if flavor in garage.version_flavor:
+        garage.version = garage.version_flavor[flavor]
+    if flavor in warp.bench_flavor:
+        warp.bench = warp.bench_flavor[flavor]
 
 if shared.id() == 1:
-    garage.deploy_coord(version=garage_version)
+    garage.deploy_coord(version=garage.version)
     warp.on_garage(params=warp_bench)
     garage.delete_key()
     garage.destroy()
 else:
-    garage.deploy_follow(version=garage_version)
+    garage.deploy_follow(version=garage.version)
     garage.sync_on_key_down()
     garage.destroy()
 shared.log("bye")