mknet/scenarios/fragments/garage.py

209 lines
5.7 KiB
Python
Raw Normal View History

2022-09-16 13:43:58 +00:00
import glob, json, time
2022-09-14 15:25:16 +00:00
from os.path import exists
from os import environ as env
from pathlib import Path
2022-09-16 14:19:09 +00:00
from . import shared, flavor
import garage_admin_sdk
from garage_admin_sdk.api import nodes_api, layout_api, key_api
from garage_admin_sdk.model.node_cluster_info import NodeClusterInfo
from garage_admin_sdk.model.layout_version import LayoutVersion
from garage_admin_sdk.model.add_key_request import AddKeyRequest
from garage_admin_sdk.model.update_key_request import UpdateKeyRequest
from garage_admin_sdk.model.update_key_request_allow import UpdateKeyRequestAllow
2022-09-14 15:25:16 +00:00
storage_path = "./i/am/not/defined"
rpc_secret = "3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba"
metrics = "cacce0b2de4bc2d9f5b5fdff551e01ac1496055aed248202d415398987e35f81"
admin = "ae8cb40ea7368bbdbb6430af11cca7da833d3458a5f52086f4e805a570fb5c2a"
2022-09-14 16:01:44 +00:00
key = None
2022-09-14 15:25:16 +00:00
2022-09-16 14:19:09 +00:00
version = flavor.garage["garage-v0.7"]
2022-09-14 15:25:16 +00:00
configuration = garage_admin_sdk.Configuration(
host = "http://localhost:3903/v0",
access_token = admin
)
api = garage_admin_sdk.ApiClient(configuration)
nodes = nodes_api.NodesApi(api)
layout = layout_api.LayoutApi(api)
keys = key_api.KeyApi(api)
# Setup, launch on import
2022-09-16 13:43:58 +00:00
if 'HOST' in env:
storage_path = Path(shared.storage_path) / "garage" / env['HOST']
if 'ZONE' in env and env['ZONE'] != "":
storage_path = Path(shared.storage_path) / "garage" / env['ZONE'] / env['HOST']
config = storage_path / "garage.toml"
env['GARAGE_CONFIG_FILE'] = str(config)
2022-09-14 15:25:16 +00:00
2022-09-23 20:50:50 +00:00
def deploy_coord(gw=None):
2022-09-14 15:25:16 +00:00
destroy()
shared.log("start daemon")
shared.exec(f"{version['path']} --version")
2022-09-14 15:25:16 +00:00
daemon()
shared.log("discover nodes")
connect()
shared.log("build layout")
2022-09-23 20:50:50 +00:00
create_layout(gw=gw)
2022-09-14 15:25:16 +00:00
shared.log("create key")
create_key()
shared.log("ready")
def deploy_follow():
2022-09-14 15:25:16 +00:00
destroy()
shared.log("start daemon")
daemon()
shared.log("wait for coord")
sync_on_key_up()
shared.log("ready")
def daemon():
shared.exec(f"mkdir -p {storage_path}")
with open(config, 'w+') as f:
f.write(f"""
metadata_dir = "{storage_path}/meta"
data_dir = "{storage_path}/data"
replication_mode = "3"
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "[{env['IP']}]:3901"
rpc_secret = "{rpc_secret}"
bootstrap_peers=[]
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.garage"
[s3_web]
bind_addr = "[::]:3902"
root_domain = ".web.garage"
index = "index.html"
[admin]
api_bind_addr = "0.0.0.0:3903"
metrics_token = "{metrics}"
admin_token = "{admin}"
""")
2022-09-16 14:19:09 +00:00
shared.exec(f"{version['path']} server 2>> {storage_path}/logs.stderr 1>> {storage_path}/logs.stdout & echo $! > {storage_path}/daemon.pid")
2022-09-14 15:25:16 +00:00
time.sleep(1)
node_info = storage_path / "node_info"
2022-09-14 16:01:44 +00:00
node_id = shared.fn_retry(lambda: nodes.get_nodes().node)
2022-09-14 15:25:16 +00:00
with open(node_info, 'w+') as f:
f.write(json.dumps({
"node_addr": f"{node_id}@{env['IP']}:3901",
"node_id": node_id,
"zone": env['ZONE'] if 'ZONE' in env and env['ZONE'] != "" else env['HOST'],
2022-09-14 15:25:16 +00:00
"host": env['HOST'],
}))
def destroy():
dpid = Path(storage_path) / "daemon.pid"
if exists(dpid):
try:
shared.exec(f"kill -9 $(cat {dpid})")
except:
pass
2022-09-14 15:25:16 +00:00
shared.exec(f"rm -f {dpid}")
if len(str(storage_path)) < 8: # arbitrary, stupid safe guard
print(storage_path)
raise Exception("You tried to clean a storage path that might be the root of your FS, panicking...")
shared.exec(f"rm -fr {storage_path}")
# this function is ugly, sorry :s
_cluster_info = None
def cluster_info():
global _cluster_info
if _cluster_info is not None: return _cluster_info
2022-09-19 12:16:08 +00:00
shared.log("fetch cluster info")
2022-09-14 15:25:16 +00:00
while True:
node_files = glob.glob(f"{shared.storage_path}/**/node_info", recursive=True)
2022-09-19 12:16:08 +00:00
if len(node_files) >= shared.count(): break
shared.log(f"found {len(node_files)} over {shared.count()}, wait 1 sec.")
time.sleep(1)
2022-09-14 15:25:16 +00:00
_cluster_info = [ json.loads(Path(f).read_text()) for f in node_files ]
return _cluster_info
def connect():
cinf = cluster_info()
2022-09-19 12:16:08 +00:00
shared.log("start connections...")
2022-09-23 20:50:50 +00:00
while True:
try:
ret = nodes.add_node([n['node_addr'] for n in cinf], _request_timeout=3)
except:
shared.log("not ready, retry in 1sec")
time.sleep(1)
continue
for st in ret:
if not st.success:
continue
#raise Exception("Node connect failed", ret)
break
2022-09-14 15:25:16 +00:00
shared.log("all nodes connected")
2022-09-23 20:50:50 +00:00
def create_layout(gw=None):
if gw is None:
gw = []
2022-09-14 15:25:16 +00:00
v = layout.get_layout().version
cinf = cluster_info()
nlay = dict()
for n in cinf:
2022-09-23 20:50:50 +00:00
capa = 1
if n['host'] in gw:
capa = None
2022-09-14 15:25:16 +00:00
nlay[n['node_id']] = NodeClusterInfo(
zone = n['zone'],
2022-09-23 20:50:50 +00:00
capacity = capa,
2022-09-14 15:25:16 +00:00
tags = [ n['host'] ],
)
layout.add_layout(nlay)
layout.apply_layout(LayoutVersion(version=v+1))
2022-09-23 20:50:50 +00:00
shared.log(layout.get_layout())
2022-09-14 15:25:16 +00:00
def create_key():
global key
kinfo = shared.fn_retry(lambda: keys.add_key(AddKeyRequest(name="mknet")))
allow_create = UpdateKeyRequestAllow(create_bucket=True)
keys.update_key(kinfo.access_key_id, UpdateKeyRequest(allow=allow_create))
key = kinfo
def delete_key():
global key
2022-09-14 16:01:44 +00:00
keys.delete_key(key.access_key_id)
2022-09-14 15:25:16 +00:00
key = None
def sync_on_key_up():
global key
while True:
try:
key = keys.search_key("mknet")
return key
except:
pass
time.sleep(1)
def sync_on_key_down():
while True:
try:
keys.search_key("mknet")
except:
return
time.sleep(1)