diff --git a/instrumented/deploy_minio.py b/instrumented/deploy_minio.py index cbacb1f..8bd5e66 100755 --- a/instrumented/deploy_minio.py +++ b/instrumented/deploy_minio.py @@ -1,57 +1,78 @@ #!/bin/python3 -import yaml, os, sys, time, pathlib, unshare +import yaml, os, sys, time, pathlib, unshare, socket, shutil -HOST_MARKER = "__AUTO_MKNET_MINIO_DEPLOY_DYYycw9Z5c5PvylU" -storage_path = "/tmp/minio-testnet" +STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet') +MINIO_PATH = '/srv' +HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt') +UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock') +DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid)) -# Remove any entry we put in /etc/hosts -def cleanup(): - with open('/etc/hosts', 'r') as fin: - with open('/etc/hosts', 'w') as fout: - fout.writelines(filter(lambda line: not HOST_MARKER in line, fin)) +def main(): + unshare.unshare(unshare.CLONE_NEWNS) + os.system("mount --make-rprivate /") # see https://stackoverflow.com/a/41580252 + if int(os.environ['ID']) == 1: leader() + else: follower() -def wait(p): - print("wait", p) - d = os.path.join(storage_path, p) - while not os.path.exists(d): - time.sleep(1) - #os.unlink(d) -def post(p): - print("post", p) - pathlib.Path(os.path.join(storage_path, p)).touch() +def leader(): + shutil.rmtree(STORAGE_PATH, ignore_errors=True) + os.makedirs(STORAGE_PATH) + os.makedirs(MINIO_PATH, exist_ok=True) + print(STORAGE_PATH) -## Create a working directory -me = int(os.environ['ID']) -node_storage_path = os.path.join(storage_path, str(me)) -os.makedirs(node_storage_path, exist_ok=True) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(UNIX_SOCK) + sock.listen() -## Writing down some info for others -for v in [ 'ZONE', 'HOST', 'IP' ]: - with open(os.path.join(node_storage_path, v+'.txt'), 'w') as f: - f.write(os.environ[v]) -netsize = int(os.environ['SIZE']) -post("init"+str(me)) + n_serv = n_servers() + fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]] -## Delete old hosts -if me == 1: - cleanup() - with open('/etc/hosts', 'a') as f: - for i in range(1,netsize+1): - wait("init"+str(i)) - ip = pathlib.Path(os.path.join(storage_path, str(i), 'IP.txt')).read_text() - host = pathlib.Path(os.path.join(storage_path, str(i), 'HOST.txt')).read_text() - f.write(f"{ip} {host} minio{i} # {HOST_MARKER}\n") - post("hosts") -wait("hosts") + ips = [ co.makefile().readline().strip() for co in fl ] + [ os.environ['IP'] ] + print(f"ips: {ips}") + gen_hosts(ips) + [ co.send(f"{n_serv}\n".encode()) for co in fl ] -# Adding entries to the host file + os.system(f"mount --bind {HOSTS_PATH} /etc/hosts") -# Unsharing the filesystem -data=os.path.join(storage_path, "data") -os.makedirs(data, exist_ok=True) -unshare.unshare(unshare.CLONE_NEWNS) -os.system("mount --make-rprivate /") # see https://stackoverflow.com/a/41580252 -os.system("mount -t tmpfs tmpfs "+data) + data_path = DATA_PATH(os.environ['ID']) + os.makedirs(data_path) + os.system(f"mount --bind {data_path} {MINIO_PATH}") -last = str(netsize) -os.system("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+last+"}"+data+" > "+node_storage_path + "/minio.log 2>&1") + run_minio(n_serv) + +def follower(): + co = None + while True: + time.sleep(3) + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(UNIX_SOCK) + co = sock.makefile() + break + except Exception as err: + print('conn failed, wait,', err) + sock.send(f"{os.environ['IP']}\n".encode()) + n_serv = int(co.readline()) + + os.system(f"mount --bind {HOSTS_PATH} /etc/hosts") + + data_path = DATA_PATH(os.environ['ID']) + os.makedirs(data_path) + + run_minio(n_serv) + +def run_minio(count): + #print("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+str(count)+"}"+MINIO_PATH+" > "+ STORAGE_PATH + "/minio."+ os.environ['ID'] +".log 2>&1") + os.system("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+str(count)+"}"+MINIO_PATH+" > "+ STORAGE_PATH + "/minio."+ os.environ['ID'] +".log 2>&1") + +def gen_hosts(ips): + with open(HOSTS_PATH, 'w') as f: + for idx,ip in enumerate(ips): + f.write(f"{ip} minio{idx+1}\n") + +def n_servers(): + with open('.current_state.yml', 'r') as f: + netw = yaml.safe_load(f) + n_servers = len(netw['servers']) + return n_servers + +__name__ == '__main__' and main()