Add some support for minio
This commit is contained in:
parent
0664442648
commit
c9cbe5fc52
5 changed files with 144 additions and 34 deletions
|
@ -2,14 +2,20 @@ from pathlib import Path
|
|||
from . import shared
|
||||
from os.path import exists
|
||||
|
||||
def add_path(d):
|
||||
def grg_path(d):
|
||||
for flav, desc in d.items():
|
||||
if "path" in desc: continue
|
||||
binary = f"garage-{desc['target']}-{desc['version']}"
|
||||
desc['path'] = Path(shared.binary_path) / binary
|
||||
return d
|
||||
|
||||
garage = add_path({
|
||||
def minio_path(d):
|
||||
for flav, desc in d.items():
|
||||
if "path" in desc: continue
|
||||
desc['path'] = Path(shared.binary_path) / flav
|
||||
return d
|
||||
|
||||
garage = grg_path({
|
||||
"garage-local": { "path": "./garage/target/release/garage" },
|
||||
"garage-v0.7": {
|
||||
"version": "v0.7.3",
|
||||
|
@ -23,6 +29,12 @@ warp = {
|
|||
"warp-default": "mixed"
|
||||
}
|
||||
|
||||
minio = minio_path({
|
||||
"minio-20220917": {
|
||||
"version": "2022-09-17T00-09-45Z"
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
def download():
|
||||
for flav, desc in garage.items():
|
||||
|
@ -33,3 +45,13 @@ def download():
|
|||
shared.exec(f"wget https://garagehq.deuxfleurs.fr/_releases/{desc['version']}/{desc['target']}/garage -O {desc['path']}")
|
||||
shared.exec(f"chmod +x {desc['path']}")
|
||||
shared.exec(f"{desc['path']} --version")
|
||||
|
||||
for flav, desc in minio.items():
|
||||
if "version" not in desc: continue
|
||||
if exists(desc['path']): continue
|
||||
|
||||
shared.exec(f"mkdir -p {shared.binary_path}")
|
||||
shared.exec(f"wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.{desc['version']} -O {desc['path']}")
|
||||
shared.exec(f"chmod +x {desc['path']}")
|
||||
shared.exec(f"{desc['path']} --version")
|
||||
|
||||
|
|
|
@ -1,62 +1,136 @@
|
|||
#!/usr/bin/env python3
|
||||
import json, os, sys, time, pathlib, socket, shutil
|
||||
import json, os, sys, time, pathlib, socket, shutil, urllib3
|
||||
import minio
|
||||
from os import environ as env
|
||||
from pathlib import Path
|
||||
from . import shared, flavor
|
||||
|
||||
STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet')
|
||||
HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt')
|
||||
UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock')
|
||||
DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid))
|
||||
storage_path = "./i/am/not/defined"
|
||||
version = flavor.minio["minio-20220917"]
|
||||
unix_sock = str(Path(shared.storage_path) / "minio.sock")
|
||||
access_key = "minioadmin"
|
||||
secret_key = "minioadmin"
|
||||
client = minio.Minio(
|
||||
f"[{env['IP']}]:9000",
|
||||
access_key="minioadmin",
|
||||
secret_key="minioadmin",
|
||||
secure=False,
|
||||
http_client=urllib3.PoolManager(
|
||||
timeout=5,
|
||||
retries=1,
|
||||
)
|
||||
)
|
||||
|
||||
def main():
|
||||
if int(os.environ['ID']) == 1: leader()
|
||||
else: follower()
|
||||
if 'HOST' in env:
|
||||
storage_path = str(Path(shared.storage_path) / "minio" / env['HOST'])
|
||||
if 'ZONE' in env and env['ZONE'] != "":
|
||||
storage_path = str(Path(shared.storage_path) / "minio" / env['ZONE'] / env['HOST'])
|
||||
|
||||
def leader():
|
||||
shutil.rmtree(STORAGE_PATH, ignore_errors=True)
|
||||
os.makedirs(STORAGE_PATH)
|
||||
print(STORAGE_PATH)
|
||||
stdout = Path(storage_path) / "minio.stdout"
|
||||
stderr = Path(storage_path) / "minio.stderr"
|
||||
pid = Path(storage_path) / "daemon.pid"
|
||||
|
||||
def destroy():
|
||||
if os.path.exists(pid):
|
||||
try:
|
||||
shared.exec(f"kill -9 `cat {pid}`")
|
||||
except:
|
||||
pass
|
||||
|
||||
if len(str(storage_path)) > 8:
|
||||
shutil.rmtree(storage_path, ignore_errors=True)
|
||||
|
||||
def deploy_coord():
|
||||
destroy()
|
||||
if os.path.exists(unix_sock):
|
||||
os.unlink(unix_sock)
|
||||
|
||||
os.makedirs(storage_path)
|
||||
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
sock.bind(UNIX_SOCK)
|
||||
sock.bind(unix_sock)
|
||||
sock.listen()
|
||||
|
||||
n_serv = int(os.environ['SERVER_COUNT'])
|
||||
fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]]
|
||||
# Create sockets
|
||||
fl = [ co for co, addr in [ sock.accept() for i in range(1, shared.count()) ]]
|
||||
|
||||
identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ]
|
||||
print(f"ident: {identities}")
|
||||
# Receive configurations, centralize them
|
||||
me = [ { "ip": os.environ['IP'], "path": storage_path } ]
|
||||
others = [ json.loads(co.makefile().readline()) for co in fl ]
|
||||
identities = others + me
|
||||
shared.log(f"ident: {identities}")
|
||||
|
||||
# Dispatch them
|
||||
msg = f"{json.dumps(identities)}\n".encode()
|
||||
[ co.send(msg) for co in fl ]
|
||||
|
||||
run_minio(identities)
|
||||
|
||||
def follower():
|
||||
while True:
|
||||
try:
|
||||
if client.bucket_exists("sync"): break
|
||||
client.make_bucket("sync")
|
||||
break
|
||||
except Exception as e:
|
||||
shared.log("waiting for bootstrap...", e)
|
||||
time.sleep(1)
|
||||
shared.log("ready")
|
||||
|
||||
def deploy_follow():
|
||||
destroy()
|
||||
os.makedirs(storage_path)
|
||||
|
||||
co = None
|
||||
while True:
|
||||
time.sleep(1)
|
||||
try:
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
sock.connect(UNIX_SOCK)
|
||||
sock.connect(unix_sock)
|
||||
co = sock.makefile()
|
||||
break
|
||||
except Exception as err:
|
||||
print('conn failed, wait,', err)
|
||||
my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() })
|
||||
shared.log('conn failed, wait 1 sec, err is', err)
|
||||
time.sleep(1)
|
||||
|
||||
# send my identity
|
||||
my_identity = json.dumps({ "ip": os.environ['IP'], "path": storage_path })
|
||||
sock.send(f"{my_identity}\n".encode())
|
||||
|
||||
# get all
|
||||
identities = json.loads(co.readline())
|
||||
|
||||
run_minio(identities)
|
||||
sync_on_bucket_up()
|
||||
shared.log("ready")
|
||||
|
||||
def make_data():
|
||||
data_path = DATA_PATH(os.environ['ID'])
|
||||
os.makedirs(data_path)
|
||||
return data_path
|
||||
def sync_on_bucket_up():
|
||||
while True:
|
||||
try:
|
||||
if client.bucket_exists("sync"): break
|
||||
except:
|
||||
pass
|
||||
shared.log("waiting for bucket 'sync'...")
|
||||
time.sleep(1)
|
||||
|
||||
def sync_on_bucket_down():
|
||||
while True:
|
||||
if not client.bucket_exists("sync"): break
|
||||
time.sleep(1)
|
||||
|
||||
def delete_sync_bucket():
|
||||
client.remove_bucket("sync")
|
||||
|
||||
def run_minio(identities):
|
||||
cmd = f"minio server --console-address ':9001' --address ':9000'"
|
||||
|
||||
# Required to prevent Minio error: "/tmp/mknet-store/minio/node1` is part of root drive, will not be used"
|
||||
# https://github.com/minio/minio/issues/15720
|
||||
env['CI'] = "true"
|
||||
env['MINIO_CI_CD'] = "true"
|
||||
|
||||
cmd = f"{version['path']} server --console-address ':9001' --address ':9000'"
|
||||
for ident in identities:
|
||||
cmd += f" http://[{ident['ip']}]:9000{ident['path']}"
|
||||
cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1"
|
||||
print("launch: ", cmd)
|
||||
os.system(cmd)
|
||||
cmd += f" > {stdout} 2> {stderr}"
|
||||
cmd += f" & echo $! > {pid}"
|
||||
|
||||
__name__ == '__main__' and main()
|
||||
shared.log("launch: ", cmd)
|
||||
os.system(cmd)
|
||||
|
|
12
scenarios/minio-manual
Executable file
12
scenarios/minio-manual
Executable file
|
@ -0,0 +1,12 @@
|
|||
#!/usr/bin/env python3
|
||||
from fragments import shared, flavor, minio
|
||||
import sys
|
||||
|
||||
for fl in sys.argv[1:]:
|
||||
if fl in flavor.minio:
|
||||
minio.version = flavor.minio[fl]
|
||||
|
||||
if shared.id() == 1:
|
||||
minio.deploy_coord()
|
||||
else:
|
||||
minio.deploy_follow()
|
|
@ -1 +1,2 @@
|
|||
git+https://git.deuxfleurs.fr/quentin/garage-admin-sdk@7b1c1faf7a#egg=garage-admin-sdk&subdirectory=python
|
||||
minio
|
||||
|
|
|
@ -2,7 +2,8 @@ links:
|
|||
- &slow
|
||||
bandwidth: 1M
|
||||
latency: 500us
|
||||
txqueuelen: 10
|
||||
#txqueuelen: 10
|
||||
limit: 10
|
||||
- &1000
|
||||
bandwidth: 1000M
|
||||
latency: 100us
|
||||
|
|
Loading…
Reference in a new issue