From e3409ce6b751fa92a98000ca4a2225b2ed10fdee Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Wed, 14 Sep 2022 17:25:16 +0200 Subject: [PATCH] New way to deploy Garage --- benchmarks/clean | 8 + benchmarks/fragments/.shared.py.swo | Bin 0 -> 12288 bytes benchmarks/fragments/garage.py | 202 ++++++++++++++++++ .../fragments/minio.py | 0 benchmarks/fragments/s3lat.py | 18 ++ benchmarks/fragments/shared.py | 28 +++ benchmarks/garage-s3lat | 12 ++ benchmarks/requirements.txt | 1 + example/deploy_garage.sh | 61 ------ setup.py | 4 + shell.nix | 12 ++ topo/multi-dc.yml | 46 ++++ topo/single-dc.yml | 34 +++ topo/slow-net.yml | 25 +++ config.yml => topo/with-vdsl.yml | 0 15 files changed, 390 insertions(+), 61 deletions(-) create mode 100755 benchmarks/clean create mode 100644 benchmarks/fragments/.shared.py.swo create mode 100644 benchmarks/fragments/garage.py rename example/deploy_minio.py => benchmarks/fragments/minio.py (100%) mode change 100755 => 100644 create mode 100644 benchmarks/fragments/s3lat.py create mode 100644 benchmarks/fragments/shared.py create mode 100755 benchmarks/garage-s3lat create mode 100644 benchmarks/requirements.txt delete mode 100755 example/deploy_garage.sh create mode 100644 shell.nix create mode 100644 topo/multi-dc.yml create mode 100644 topo/single-dc.yml create mode 100644 topo/slow-net.yml rename config.yml => topo/with-vdsl.yml (100%) diff --git a/benchmarks/clean b/benchmarks/clean new file mode 100755 index 0000000..325edec --- /dev/null +++ b/benchmarks/clean @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +import os +from fragments import garage, shared + +garage.destroy() +shared.log("clean done") + diff --git a/benchmarks/fragments/.shared.py.swo b/benchmarks/fragments/.shared.py.swo new file mode 100644 index 0000000000000000000000000000000000000000..02b9acbabe2aa9b07d0b9dbcd999207841ec6491 GIT binary patch literal 12288 zcmeI&&1w@-6bJBA_wf@KqKnfMYyxf450DZF2u&ARsLI-r|=!ronXNi@Z9NiVGZfpaxVF2$h~Lo$8T048LaH}AK^x84Y*wg_a72Zrr0^q}S@)`%u-!9pk@4KFozM}d7^jCkg==RAntXde zIv!<_J#g*RM#IxeH;Q`2GLv`~S%`fM1-ioX?z3oDZD0oIYorbDvY=e7p+qj`NUnhx4PF z?+ahwbB;Mf&Ms$%BTWQEKmYKm;y{0G7>0$`NjjW6HfUq3iD+-Kh7c z_x3O=oS!*1AWSYSpj2r}>P>)zQjATZeWTeyOTMrg6@N0WMfyltqn|DAY;EstZSQYB z*?s(U@p-enM0<1L11 zLQ^U7rrgAN${MbT-7c=2Z5hf6ehW9X+#ylkP)%GwKT;0)MdO^}u$VVk4QV+n1c zwbDZ_BzCCwW>_LAVk&#J;ipwDz&D^`9e!MTNR3$gW~Us;Z6I@}GK^>^4Y!0m-d%lA zm4prc^OP^bs7`> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid") + time.sleep(1) + + node_info = storage_path / "node_info" + node_id = nodes.get_nodes().node + with open(node_info, 'w+') as f: + f.write(json.dumps({ + "node_addr": f"{node_id}@{env['IP']}:3901", + "node_id": node_id, + "zone": env['ZONE'], + "host": env['HOST'], + })) + +def destroy(): + dpid = Path(storage_path) / "daemon.pid" + if exists(dpid): + shared.exec(f"kill -9 $(cat {dpid})") + shared.exec(f"rm -f {dpid}") + if len(str(storage_path)) < 8: # arbitrary, stupid safe guard + print(storage_path) + raise Exception("You tried to clean a storage path that might be the root of your FS, panicking...") + shared.exec(f"rm -fr {storage_path}") + +# this function is ugly, sorry :s +_cluster_info = None +def cluster_info(): + global _cluster_info + if _cluster_info is not None: return _cluster_info + + while True: + time.sleep(1) + node_files = glob.glob(f"{shared.storage_path}/**/node_info", recursive=True) + if len(node_files) == shared.count(): break + + _cluster_info = [ json.loads(Path(f).read_text()) for f in node_files ] + return _cluster_info + + +def connect(): + cinf = cluster_info() + ret = nodes.add_node([n['node_addr'] for n in cinf]) + for st in ret: + if not st.success: + raise Exception("Node connect failed", ret) + shared.log("all nodes connected") + +def create_layout(): + v = layout.get_layout().version + + cinf = cluster_info() + nlay = dict() + for n in cinf: + nlay[n['node_id']] = NodeClusterInfo( + zone = n['zone'], + capacity = 1, + tags = [ n['host'] ], + ) + layout.add_layout(nlay) + layout.apply_layout(LayoutVersion(version=v+1)) + +def create_key(): + global key + kinfo = shared.fn_retry(lambda: keys.add_key(AddKeyRequest(name="mknet"))) + allow_create = UpdateKeyRequestAllow(create_bucket=True) + keys.update_key(kinfo.access_key_id, UpdateKeyRequest(allow=allow_create)) + key = kinfo + + +def delete_key(): + global key + delete_key(key.access_key_id) + key = None + +def sync_on_key_up(): + global key + while True: + try: + key = keys.search_key("mknet") + return key + except: + pass + time.sleep(1) + +def sync_on_key_down(): + while True: + try: + keys.search_key("mknet") + except: + return + time.sleep(1) + diff --git a/example/deploy_minio.py b/benchmarks/fragments/minio.py old mode 100755 new mode 100644 similarity index 100% rename from example/deploy_minio.py rename to benchmarks/fragments/minio.py diff --git a/benchmarks/fragments/s3lat.py b/benchmarks/fragments/s3lat.py new file mode 100644 index 0000000..c25594e --- /dev/null +++ b/benchmarks/fragments/s3lat.py @@ -0,0 +1,18 @@ +a = """ +echo "sleep 3 min to wait for minio bootstrap" +sleep 180 + +export ENDPOINT=localhost:9000 +export AWS_ACCESS_KEY_ID=minioadmin +export AWS_SECRET_ACCESS_KEY=minioadmin + +mc alias set minio-bench http://$ENDPOINT $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY +for i in $(seq 1 10); do + mc mb minio-bench/bench$i +done + +s3lat | tee 50ms.minio.csv +""" + +def on_garage(): + raise Exception("Not yet implemented") diff --git a/benchmarks/fragments/shared.py b/benchmarks/fragments/shared.py new file mode 100644 index 0000000..e0cd449 --- /dev/null +++ b/benchmarks/fragments/shared.py @@ -0,0 +1,28 @@ +import os, time + +binary_path = "/tmp/mknet-bin" +storage_path = "/tmp/mknet-store" + +def exec(s): + if os.system(s) != 0: + raise Exception("Command terminated with an error") +def exec_retry(s, cnt=16): + print(s) + for i in range(cnt): + time.sleep(i) # this is expected to sleep before running the command to reduce the noise + if os.system(s) == 0: return + raise Exception("Command terminated with an error too many times") +def fn_retry(f, cnt=5): + for i in range(cnt): + try: + r = f() + return r + except Exception as e: + if i+1 == cnt: raise e + log(f"failed call, retry in {i} sec") + time.sleep(i) + +def id(): return int(os.environ['ID']) +def count(): return int(os.environ['SERVER_COUNT']) +def log(*args): print(f"[{id()}/{count()} - {os.environ['HOST']}]", *args) + diff --git a/benchmarks/garage-s3lat b/benchmarks/garage-s3lat new file mode 100755 index 0000000..361ed26 --- /dev/null +++ b/benchmarks/garage-s3lat @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +from fragments import garage, s3lat, shared + +if shared.id() == 1: + garage.deploy_coord() + s3lat.on_garage() + garage.delete_key() + garage.destroy() +else: + garage.deploy_follow() + garage.sync_on_key_down() + garage.destroy() diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt new file mode 100644 index 0000000..41a5912 --- /dev/null +++ b/benchmarks/requirements.txt @@ -0,0 +1 @@ +git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@7b1c1faf7a#egg=garage-admin-sdk&subdirectory=python diff --git a/example/deploy_garage.sh b/example/deploy_garage.sh deleted file mode 100755 index 0dc9b38..0000000 --- a/example/deploy_garage.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -set -euo pipefail -IFS=$'\n\t' - -GARAGE_PATH=garage -STORAGE_PATH=/tmp/garage-testnet -export RPC_SECRET=3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba - -if [ -z "$ZONE" ]; then - NODE_STORAGE_PATH=${STORAGE_PATH}/${HOST} -else - NODE_STORAGE_PATH=${STORAGE_PATH}/${ZONE}/${HOST} -fi -BOOTSTRAP_FILE=${STORAGE_PATH}/bootstrap_peer -export GARAGE_CONFIG_FILE=${NODE_STORAGE_PATH}/garage.toml - - -mkdir -p ${NODE_STORAGE_PATH} -cd ${NODE_STORAGE_PATH} -rm ${BOOTSTRAP_FILE} 2>/dev/null || true - -cat > ${GARAGE_CONFIG_FILE} << EOF -metadata_dir = "${NODE_STORAGE_PATH}/meta" -data_dir = "${NODE_STORAGE_PATH}/data" - -replication_mode = "3" - -rpc_bind_addr = "[::]:3901" -rpc_public_addr = "[${IP}]:3901" -rpc_secret = "${RPC_SECRET}" - -bootstrap_peers=[] - -[s3_api] -s3_region = "garage" -api_bind_addr = "[::]:3900" -root_domain = ".s3.garage" - -[s3_web] -bind_addr = "[::]:3902" -root_domain = ".web.garage" -index = "index.html" -EOF - -RUST_LOG=garage=debug ${GARAGE_PATH} server 2>> ${NODE_STORAGE_PATH}/logs & disown -sleep 2 - -CONFIG_NODE_FPATH=$(find /tmp/garage-testnet/ -maxdepth 3 -name garage.toml|head -n 1) - -SELF_ID=$(${GARAGE_PATH} node id 2>/dev/null) -SHORT_ID=$(echo ${SELF_ID} | cut -c-64) - -${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} node connect ${SELF_ID} -${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} layout assign ${SHORT_ID} -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST} - -if [ ${CONFIG_NODE_FPATH} == ${GARAGE_CONFIG_FILE} ]; then - sleep 2 - ${GARAGE_PATH} layout show - ${GARAGE_PATH} layout apply --version 1 -fi diff --git a/setup.py b/setup.py index fda29c5..6c58b07 100644 --- a/setup.py +++ b/setup.py @@ -3,4 +3,8 @@ setup(name='mknet', version='1.0', scripts=['mknet'], py_modules=['net'], + install_requires=[ + 'PyYAML', + 'requests' + ], ) diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..a253897 --- /dev/null +++ b/shell.nix @@ -0,0 +1,12 @@ +{ pkgs ? import {} }: + pkgs.mkShell { + nativeBuildInputs = [ + pkgs.python310 + pkgs.python310Packages.pyyaml + pkgs.python310Packages.requests + + # to test the pip setup + pkgs.python310Packages.pip + pkgs.python310Packages.setuptools + ]; +} diff --git a/topo/multi-dc.yml b/topo/multi-dc.yml new file mode 100644 index 0000000..77e695b --- /dev/null +++ b/topo/multi-dc.yml @@ -0,0 +1,46 @@ +links: + - &100 + bandwidth: 100M + latency: 500us + - &wan + bandwidth: 100M + latency: 50ms + jitter: 10ms + +zones: + - &dc1 + name: dc1 + internal: *100 + external: *wan + - &dc2 + name: dc2 + internal: *100 + external: *wan + - &dc3 + name: dc3 + internal: *100 + external: *wan + +servers: + - name: dc1s1 + zone: *dc1 + - name: dc1s2 + zone: *dc1 + - name: dc2s1 + zone: *dc2 + - name: dc2s2 + zone: *dc2 + - name: dc3s1 + zone: *dc3 + - name: dc3s2 + zone: *dc3 + +global: + subnet: + base: 'fc00:9a7a:9e::' + local: 64 + zone: 16 + latency-offset: 3ms + upstream: + ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff + conn: *wan diff --git a/topo/single-dc.yml b/topo/single-dc.yml new file mode 100644 index 0000000..39427aa --- /dev/null +++ b/topo/single-dc.yml @@ -0,0 +1,34 @@ +links: + - &fiber + bandwidth: 100M + latency: 50ms + jitter: 10ms + +zones: + - &dc1 + name: dc1 + internal: *fiber + external: *fiber + +servers: + - name: dc1s1 + zone: *dc1 + - name: dc1s2 + zone: *dc1 + - name: dc1s3 + zone: *dc1 + - name: dc1s4 + zone: *dc1 + - name: dc1s5 + zone: *dc1 + +global: + subnet: + base: 'fc00:9a7a:9e::' + local: 64 + zone: 16 + latency-offset: 3ms + upstream: + ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff + conn: *fiber + diff --git a/topo/slow-net.yml b/topo/slow-net.yml new file mode 100644 index 0000000..75fdfc2 --- /dev/null +++ b/topo/slow-net.yml @@ -0,0 +1,25 @@ +links: + - &slow + bandwidth: 1M + latency: 500us + - &1000 + bandwidth: 1000M + latency: 100us + +servers: + - name: node1 + <<: *slow + - name: node2 + <<: *slow + - name: node3 + <<: *slow + +global: + subnet: + base: 'fc00:9a7a:9e::' + local: 64 + zone: 16 + latency-offset: 3ms + upstream: + ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff + conn: *1000 diff --git a/config.yml b/topo/with-vdsl.yml similarity index 100% rename from config.yml rename to topo/with-vdsl.yml