Reworked minio deployment script

instrumented/deploy_minio.py

@@ -1,57 +1,78 @@
 #!/bin/python3
-import yaml, os, sys, time, pathlib, unshare
+import yaml, os, sys, time, pathlib, unshare, socket, shutil
 
-HOST_MARKER = "__AUTO_MKNET_MINIO_DEPLOY_DYYycw9Z5c5PvylU"
-storage_path = "/tmp/minio-testnet"
+STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet')
+MINIO_PATH = '/srv'
+HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt')
+UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock')
+DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid))
 
-# Remove any entry we put in /etc/hosts
-def cleanup():
-    with open('/etc/hosts', 'r') as fin:
-        with open('/etc/hosts', 'w') as fout:
-            fout.writelines(filter(lambda line: not HOST_MARKER in line, fin))
+def main():
+    unshare.unshare(unshare.CLONE_NEWNS)
+    os.system("mount --make-rprivate /") # see https://stackoverflow.com/a/41580252
+    if int(os.environ['ID']) == 1: leader()
+    else: follower()
 
-def wait(p):
-    print("wait", p)
-    d = os.path.join(storage_path, p)
-    while not os.path.exists(d):
-        time.sleep(1)
-    #os.unlink(d)
-def post(p): 
-    print("post", p)
-    pathlib.Path(os.path.join(storage_path, p)).touch()
+def leader():
+    shutil.rmtree(STORAGE_PATH, ignore_errors=True)
+    os.makedirs(STORAGE_PATH)
+    os.makedirs(MINIO_PATH, exist_ok=True)
+    print(STORAGE_PATH)
 
-## Create a working directory
-me = int(os.environ['ID'])
-node_storage_path = os.path.join(storage_path, str(me))
-os.makedirs(node_storage_path, exist_ok=True)
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.bind(UNIX_SOCK)
+    sock.listen()
 
-## Writing down some info for others
-for v in [ 'ZONE', 'HOST', 'IP' ]:
-     with open(os.path.join(node_storage_path, v+'.txt'), 'w') as f:
-        f.write(os.environ[v])
-netsize = int(os.environ['SIZE'])
-post("init"+str(me))
+    n_serv = n_servers()
+    fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]]
 
-## Delete old hosts
-if me == 1:
-    cleanup()
-    with open('/etc/hosts', 'a') as f:
-        for i in range(1,netsize+1):
-            wait("init"+str(i))
-            ip = pathlib.Path(os.path.join(storage_path, str(i), 'IP.txt')).read_text()
-            host = pathlib.Path(os.path.join(storage_path, str(i), 'HOST.txt')).read_text()
-            f.write(f"{ip} {host} minio{i} # {HOST_MARKER}\n")
-    post("hosts")
-wait("hosts")
+    ips = [ co.makefile().readline().strip() for co in fl ] + [ os.environ['IP'] ]
+    print(f"ips: {ips}")
+    gen_hosts(ips)
+    [ co.send(f"{n_serv}\n".encode()) for co in fl ]
 
-# Adding entries to the host file
+    os.system(f"mount --bind {HOSTS_PATH} /etc/hosts")
 
-# Unsharing the filesystem
-data=os.path.join(storage_path, "data")
-os.makedirs(data, exist_ok=True)
-unshare.unshare(unshare.CLONE_NEWNS)
-os.system("mount --make-rprivate /") # see https://stackoverflow.com/a/41580252
-os.system("mount -t tmpfs tmpfs "+data)
+    data_path = DATA_PATH(os.environ['ID'])
+    os.makedirs(data_path)
+    os.system(f"mount --bind {data_path} {MINIO_PATH}")
 
-last = str(netsize)
-os.system("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+last+"}"+data+" > "+node_storage_path + "/minio.log 2>&1")
+    run_minio(n_serv)
+
+def follower():
+    co = None
+    while True:
+        time.sleep(3)
+        try:
+            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            sock.connect(UNIX_SOCK)
+            co = sock.makefile()
+            break
+        except Exception as err:
+            print('conn failed, wait,', err)
+    sock.send(f"{os.environ['IP']}\n".encode())
+    n_serv = int(co.readline())
+
+    os.system(f"mount --bind {HOSTS_PATH} /etc/hosts")
+
+    data_path = DATA_PATH(os.environ['ID'])
+    os.makedirs(data_path)
+
+    run_minio(n_serv)
+
+def run_minio(count):
+    #print("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+str(count)+"}"+MINIO_PATH+" > "+ STORAGE_PATH + "/minio."+ os.environ['ID'] +".log 2>&1")
+    os.system("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+str(count)+"}"+MINIO_PATH+" > "+ STORAGE_PATH + "/minio."+ os.environ['ID'] +".log 2>&1")
+
+def gen_hosts(ips):
+    with open(HOSTS_PATH, 'w') as f:
+        for idx,ip in enumerate(ips):
+            f.write(f"{ip} minio{idx+1}\n")
+
+def n_servers():
+    with open('.current_state.yml', 'r') as f:
+        netw = yaml.safe_load(f)
+        n_servers = len(netw['servers'])
+        return n_servers
+
+__name__ == '__main__' and main()