Add some support for minio

This commit is contained in:
Quentin 2022-09-23 17:55:45 +02:00
parent 0664442648
commit c9cbe5fc52
Signed by untrusted user: quentin
GPG key ID: E9602264D639FF68
5 changed files with 144 additions and 34 deletions

View file

@ -2,14 +2,20 @@ from pathlib import Path
from . import shared from . import shared
from os.path import exists from os.path import exists
def add_path(d): def grg_path(d):
for flav, desc in d.items(): for flav, desc in d.items():
if "path" in desc: continue if "path" in desc: continue
binary = f"garage-{desc['target']}-{desc['version']}" binary = f"garage-{desc['target']}-{desc['version']}"
desc['path'] = Path(shared.binary_path) / binary desc['path'] = Path(shared.binary_path) / binary
return d return d
garage = add_path({ def minio_path(d):
for flav, desc in d.items():
if "path" in desc: continue
desc['path'] = Path(shared.binary_path) / flav
return d
garage = grg_path({
"garage-local": { "path": "./garage/target/release/garage" }, "garage-local": { "path": "./garage/target/release/garage" },
"garage-v0.7": { "garage-v0.7": {
"version": "v0.7.3", "version": "v0.7.3",
@ -23,6 +29,12 @@ warp = {
"warp-default": "mixed" "warp-default": "mixed"
} }
minio = minio_path({
"minio-20220917": {
"version": "2022-09-17T00-09-45Z"
}
})
def download(): def download():
for flav, desc in garage.items(): for flav, desc in garage.items():
@ -33,3 +45,13 @@ def download():
shared.exec(f"wget https://garagehq.deuxfleurs.fr/_releases/{desc['version']}/{desc['target']}/garage -O {desc['path']}") shared.exec(f"wget https://garagehq.deuxfleurs.fr/_releases/{desc['version']}/{desc['target']}/garage -O {desc['path']}")
shared.exec(f"chmod +x {desc['path']}") shared.exec(f"chmod +x {desc['path']}")
shared.exec(f"{desc['path']} --version") shared.exec(f"{desc['path']} --version")
for flav, desc in minio.items():
if "version" not in desc: continue
if exists(desc['path']): continue
shared.exec(f"mkdir -p {shared.binary_path}")
shared.exec(f"wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.{desc['version']} -O {desc['path']}")
shared.exec(f"chmod +x {desc['path']}")
shared.exec(f"{desc['path']} --version")

View file

@ -1,62 +1,136 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import json, os, sys, time, pathlib, socket, shutil import json, os, sys, time, pathlib, socket, shutil, urllib3
import minio
from os import environ as env
from pathlib import Path
from . import shared, flavor
STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet') storage_path = "./i/am/not/defined"
HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt') version = flavor.minio["minio-20220917"]
UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock') unix_sock = str(Path(shared.storage_path) / "minio.sock")
DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid)) access_key = "minioadmin"
secret_key = "minioadmin"
client = minio.Minio(
f"[{env['IP']}]:9000",
access_key="minioadmin",
secret_key="minioadmin",
secure=False,
http_client=urllib3.PoolManager(
timeout=5,
retries=1,
)
)
def main(): if 'HOST' in env:
if int(os.environ['ID']) == 1: leader() storage_path = str(Path(shared.storage_path) / "minio" / env['HOST'])
else: follower() if 'ZONE' in env and env['ZONE'] != "":
storage_path = str(Path(shared.storage_path) / "minio" / env['ZONE'] / env['HOST'])
def leader(): stdout = Path(storage_path) / "minio.stdout"
shutil.rmtree(STORAGE_PATH, ignore_errors=True) stderr = Path(storage_path) / "minio.stderr"
os.makedirs(STORAGE_PATH) pid = Path(storage_path) / "daemon.pid"
print(STORAGE_PATH)
def destroy():
if os.path.exists(pid):
try:
shared.exec(f"kill -9 `cat {pid}`")
except:
pass
if len(str(storage_path)) > 8:
shutil.rmtree(storage_path, ignore_errors=True)
def deploy_coord():
destroy()
if os.path.exists(unix_sock):
os.unlink(unix_sock)
os.makedirs(storage_path)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(UNIX_SOCK) sock.bind(unix_sock)
sock.listen() sock.listen()
n_serv = int(os.environ['SERVER_COUNT']) # Create sockets
fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]] fl = [ co for co, addr in [ sock.accept() for i in range(1, shared.count()) ]]
identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ] # Receive configurations, centralize them
print(f"ident: {identities}") me = [ { "ip": os.environ['IP'], "path": storage_path } ]
others = [ json.loads(co.makefile().readline()) for co in fl ]
identities = others + me
shared.log(f"ident: {identities}")
# Dispatch them
msg = f"{json.dumps(identities)}\n".encode() msg = f"{json.dumps(identities)}\n".encode()
[ co.send(msg) for co in fl ] [ co.send(msg) for co in fl ]
run_minio(identities) run_minio(identities)
def follower(): while True:
try:
if client.bucket_exists("sync"): break
client.make_bucket("sync")
break
except Exception as e:
shared.log("waiting for bootstrap...", e)
time.sleep(1)
shared.log("ready")
def deploy_follow():
destroy()
os.makedirs(storage_path)
co = None co = None
while True: while True:
time.sleep(1)
try: try:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(UNIX_SOCK) sock.connect(unix_sock)
co = sock.makefile() co = sock.makefile()
break break
except Exception as err: except Exception as err:
print('conn failed, wait,', err) shared.log('conn failed, wait 1 sec, err is', err)
my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() }) time.sleep(1)
# send my identity
my_identity = json.dumps({ "ip": os.environ['IP'], "path": storage_path })
sock.send(f"{my_identity}\n".encode()) sock.send(f"{my_identity}\n".encode())
# get all
identities = json.loads(co.readline()) identities = json.loads(co.readline())
run_minio(identities) run_minio(identities)
sync_on_bucket_up()
shared.log("ready")
def make_data(): def sync_on_bucket_up():
data_path = DATA_PATH(os.environ['ID']) while True:
os.makedirs(data_path) try:
return data_path if client.bucket_exists("sync"): break
except:
pass
shared.log("waiting for bucket 'sync'...")
time.sleep(1)
def sync_on_bucket_down():
while True:
if not client.bucket_exists("sync"): break
time.sleep(1)
def delete_sync_bucket():
client.remove_bucket("sync")
def run_minio(identities): def run_minio(identities):
cmd = f"minio server --console-address ':9001' --address ':9000'"
# Required to prevent Minio error: "/tmp/mknet-store/minio/node1` is part of root drive, will not be used"
# https://github.com/minio/minio/issues/15720
env['CI'] = "true"
env['MINIO_CI_CD'] = "true"
cmd = f"{version['path']} server --console-address ':9001' --address ':9000'"
for ident in identities: for ident in identities:
cmd += f" http://[{ident['ip']}]:9000{ident['path']}" cmd += f" http://[{ident['ip']}]:9000{ident['path']}"
cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1" cmd += f" > {stdout} 2> {stderr}"
print("launch: ", cmd) cmd += f" & echo $! > {pid}"
os.system(cmd)
__name__ == '__main__' and main() shared.log("launch: ", cmd)
os.system(cmd)

12
scenarios/minio-manual Executable file
View file

@ -0,0 +1,12 @@
#!/usr/bin/env python3
from fragments import shared, flavor, minio
import sys
for fl in sys.argv[1:]:
if fl in flavor.minio:
minio.version = flavor.minio[fl]
if shared.id() == 1:
minio.deploy_coord()
else:
minio.deploy_follow()

View file

@ -1 +1,2 @@
git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@7b1c1faf7a#egg=garage-admin-sdk&subdirectory=python git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@7b1c1faf7a#egg=garage-admin-sdk&subdirectory=python
minio

View file

@ -2,7 +2,8 @@ links:
- &slow - &slow
bandwidth: 1M bandwidth: 1M
latency: 500us latency: 500us
txqueuelen: 10 #txqueuelen: 10
limit: 10
- &1000 - &1000
bandwidth: 1000M bandwidth: 1000M
latency: 100us latency: 100us