add support for upstream port and add garage example script

This commit is contained in:
Trinity Pointard 2021-11-08 09:25:14 +01:00
parent 610e554903
commit 179d18fcec
4 changed files with 89 additions and 10 deletions

View file

@ -40,7 +40,10 @@ servers:
global: global:
subnet: subnet:
base: 172.24.0.0 base: fc00:9a7a:9e::
local: 8 local: 64
zone: 8 zone: 16
latency-offset: 3ms latency-offset: 3ms
upstream:
ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff
conn: *fiber

51
example/deploy_garage.sh Executable file
View file

@ -0,0 +1,51 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'
GARAGE_PATH=/home/trinity/tmp/garage/target/release/garage
STORAGE_PATH=/tmp/garage-testnet
export RPC_SECRET=3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba
NODE_STORAGE_PATH=${STORAGE_PATH}/${ZONE}/${HOST}
BOOTSTRAP_FILE=${STORAGE_PATH}/bootstrap_peer
export GARAGE_CONFIG_FILE=${NODE_STORAGE_PATH}/garage.toml
mkdir -p ${NODE_STORAGE_PATH}
cd ${NODE_STORAGE_PATH}
rm ${BOOTSTRAP_FILE} 2>/dev/null || true
cat > ${GARAGE_CONFIG_FILE} << EOF
metadata_dir = "${NODE_STORAGE_PATH}/meta"
data_dir = "${NODE_STORAGE_PATH}/data"
replication_mode = "3"
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "[${IP}]:3901"
rpc_secret = "${RPC_SECRET}"
bootstrap_peers=[]
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
[s3_web]
bind_addr = "[::]:3902"
root_domain = ".web.garage"
index = "index.html"
EOF
RUST_LOG=garage=debug ${GARAGE_PATH} server 2>> ${NODE_STORAGE_PATH}/logs & disown
sleep 2
CONFIG_NODE_FPATH=(${STORAGE_PATH}/*{,/*}/garage.toml)
SELF_ID=$(${GARAGE_PATH} node-id 2>/dev/null)
SHORT_ID=$(echo ${SELF_ID} | cut -c-64)
${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} node connect ${SELF_ID}
sleep ${ID}
${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} node configure -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST} ${SHORT_ID}

25
main.py
View file

@ -149,6 +149,8 @@ class Network:
self.zones = {} self.zones = {}
self.subnet_manager = None self.subnet_manager = None
self.latency_off = Latency(0) self.latency_off = Latency(0)
self.host_ip = None
self.host_link = None
def set_subnet_manager(self, subnet): def set_subnet_manager(self, subnet):
self.subnet_manager = SubnetManager(subnet) self.subnet_manager = SubnetManager(subnet)
@ -184,14 +186,17 @@ class Network:
else: else:
zone.ip = self.subnet_manager.next_local() zone.ip = self.subnet_manager.next_local()
self.subnet_manager.next_zone() self.subnet_manager.next_zone()
if not self.host_ip:
self.host_ip = self.subnet_manager.next_local()
def __str__(self): def __str__(self):
return f'Network{{subnet_manager: {self.subnet_manager}, zones: {list(self.zones.values())}, latency_offset: {self.latency_off}}}' return f'Network{{subnet_manager: {self.subnet_manager}, zones: {list(self.zones.values())}, latency_offset: {self.latency_off}}}'
class NamespaceManager: class NamespaceManager:
def __init__(self): def __init__(self):
self.namespaces = set() self.namespaces = set(["unconfined"])
self.prefixlen = 0 self.prefixlen = 0
net.ns.name_unconfined()
def make_namespace(self, name): def make_namespace(self, name):
if not name in self.namespaces: if not name in self.namespaces:
@ -202,7 +207,7 @@ class NamespaceManager:
self.make_namespace(namespace) self.make_namespace(namespace)
net.create_bridge(name, namespace, ports) net.create_bridge(name, namespace, ports)
def make_veth(self, name1, name2, space1, space2, ip, link): def make_veth(self, name1, name2, space1, space2, ip, link=None):
self.make_namespace(space1) self.make_namespace(space1)
self.make_namespace(space2) self.make_namespace(space2)
net.create_veth(name1, space1, name2, space2, ip, self.prefixlen, link) net.create_veth(name1, space1, name2, space2, ip, self.prefixlen, link)
@ -210,7 +215,8 @@ class NamespaceManager:
def build_network(self, network): def build_network(self, network):
self.prefixlen = network.subnet_manager.prefix self.prefixlen = network.subnet_manager.prefix
netns = "testnet-core" netns = "testnet-core"
ports = [] self.make_veth("veth-testnet", "unconfined", "unconfined", netns, network.host_ip, network.host_link)
ports = ["unconfined"]
for zone in network.zones.values(): for zone in network.zones.values():
if zone.is_zone(): if zone.is_zone():
self.build_zone(zone) self.build_zone(zone)
@ -244,6 +250,10 @@ def parse(yaml):
latency_offset = global_conf.get("latency-offset", 0) latency_offset = global_conf.get("latency-offset", 0)
network = Network() network = Network()
if upstream := global_conf.get("upstream"):
network.host_ip = upstream.get("ip")
if host_link:= upstream.get("conn"):
network.host_link = LinkInfo(latency_offset=latency_offset, **host_link)
network.set_subnet_manager(subnet) network.set_subnet_manager(subnet)
network.set_latency_offset(latency_offset) network.set_latency_offset(latency_offset)
for server in server_list: for server in server_list:
@ -302,6 +312,7 @@ def runall(cmd):
config = yaml.safe_load(file) config = yaml.safe_load(file)
zones = parse(config).zones zones = parse(config).zones
number = 1
for zone in zones.values(): for zone in zones.values():
if zone.is_zone(): if zone.is_zone():
for server in zone.servers.values(): for server in zone.servers.values():
@ -309,19 +320,25 @@ def runall(cmd):
env["ZONE"] = zone.name env["ZONE"] = zone.name
env["HOST"] = server.name env["HOST"] = server.name
env["IP"] = str(server.ip) env["IP"] = str(server.ip)
env["ID"] = str(number)
name = f'testnet-{zone.name}-{server.name}' name = f'testnet-{zone.name}-{server.name}'
net.ns.run(name, cmd, env) net.ns.run(name, cmd, env)
number +=1
else: else:
env = os.environ.copy() env = os.environ.copy()
env["ZONE"] = ""
env["HOST"] = zone.name env["HOST"] = zone.name
env["IP"] = str(zone.ip) env["IP"] = str(zone.ip)
env["ID"] = str(number)
name = f'testnet-{zone.name}-{zone.name}' name = f'testnet-{zone.name}-{zone.name}'
net.ns.run(name, cmd, env) net.ns.run(name, cmd, env)
first = False
number +=1
def destroy(): def destroy():
for ns in net.ns.list(): for ns in net.ns.list():
net.ns.kill(ns) net.ns.kill(ns)
net.ns.forget("unconfined")
os.remove(".current_state.yml") os.remove(".current_state.yml")
if __name__ == "__main__": if __name__ == "__main__":

14
net.py
View file

@ -1,4 +1,5 @@
import subprocess import subprocess
import os
_netns = ["ip", "netns"] _netns = ["ip", "netns"]
def run_netns(*cmd): def run_netns(*cmd):
@ -8,9 +9,16 @@ def run_netns(*cmd):
return process return process
class ns: class ns:
def list(): def name_unconfined():
process = run_netns("list") if "unconfined" not in ns.list(True):
return [ns for ns in process.stdout.split("\n") if ns.startswith("testnet-")] run_netns("attach", "unconfined", str(os.getpid()))
def list(include_unconfined = False):
try:
nss = os.listdir("/var/run/netns")
return [ns for ns in nss if ns.startswith("testnet-") or include_unconfined and ns == "unconfined"]
except FileNotFoundError:
return []
def forget(name): def forget(name):
run_netns("del", name) run_netns("del", name)