Reworked minio deployment script

This commit is contained in:
Quentin 2021-12-06 11:53:22 +01:00
parent cc2b090149
commit eacb73340c
Signed by: quentin
GPG key ID: A98E9B769E4FF428

View file

@ -1,57 +1,78 @@
#!/bin/python3
import yaml, os, sys, time, pathlib, unshare
import yaml, os, sys, time, pathlib, unshare, socket, shutil
HOST_MARKER = "__AUTO_MKNET_MINIO_DEPLOY_DYYycw9Z5c5PvylU"
storage_path = "/tmp/minio-testnet"
STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet')
MINIO_PATH = '/srv'
HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt')
UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock')
DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid))
# Remove any entry we put in /etc/hosts
def cleanup():
with open('/etc/hosts', 'r') as fin:
with open('/etc/hosts', 'w') as fout:
fout.writelines(filter(lambda line: not HOST_MARKER in line, fin))
def main():
unshare.unshare(unshare.CLONE_NEWNS)
os.system("mount --make-rprivate /") # see https://stackoverflow.com/a/41580252
if int(os.environ['ID']) == 1: leader()
else: follower()
def wait(p):
print("wait", p)
d = os.path.join(storage_path, p)
while not os.path.exists(d):
time.sleep(1)
#os.unlink(d)
def post(p):
print("post", p)
pathlib.Path(os.path.join(storage_path, p)).touch()
def leader():
shutil.rmtree(STORAGE_PATH, ignore_errors=True)
os.makedirs(STORAGE_PATH)
os.makedirs(MINIO_PATH, exist_ok=True)
print(STORAGE_PATH)
## Create a working directory
me = int(os.environ['ID'])
node_storage_path = os.path.join(storage_path, str(me))
os.makedirs(node_storage_path, exist_ok=True)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(UNIX_SOCK)
sock.listen()
## Writing down some info for others
for v in [ 'ZONE', 'HOST', 'IP' ]:
with open(os.path.join(node_storage_path, v+'.txt'), 'w') as f:
f.write(os.environ[v])
netsize = int(os.environ['SIZE'])
post("init"+str(me))
n_serv = n_servers()
fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]]
## Delete old hosts
if me == 1:
cleanup()
with open('/etc/hosts', 'a') as f:
for i in range(1,netsize+1):
wait("init"+str(i))
ip = pathlib.Path(os.path.join(storage_path, str(i), 'IP.txt')).read_text()
host = pathlib.Path(os.path.join(storage_path, str(i), 'HOST.txt')).read_text()
f.write(f"{ip} {host} minio{i} # {HOST_MARKER}\n")
post("hosts")
wait("hosts")
ips = [ co.makefile().readline().strip() for co in fl ] + [ os.environ['IP'] ]
print(f"ips: {ips}")
gen_hosts(ips)
[ co.send(f"{n_serv}\n".encode()) for co in fl ]
# Adding entries to the host file
os.system(f"mount --bind {HOSTS_PATH} /etc/hosts")
# Unsharing the filesystem
data=os.path.join(storage_path, "data")
os.makedirs(data, exist_ok=True)
unshare.unshare(unshare.CLONE_NEWNS)
os.system("mount --make-rprivate /") # see https://stackoverflow.com/a/41580252
os.system("mount -t tmpfs tmpfs "+data)
data_path = DATA_PATH(os.environ['ID'])
os.makedirs(data_path)
os.system(f"mount --bind {data_path} {MINIO_PATH}")
last = str(netsize)
os.system("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+last+"}"+data+" > "+node_storage_path + "/minio.log 2>&1")
run_minio(n_serv)
def follower():
co = None
while True:
time.sleep(3)
try:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(UNIX_SOCK)
co = sock.makefile()
break
except Exception as err:
print('conn failed, wait,', err)
sock.send(f"{os.environ['IP']}\n".encode())
n_serv = int(co.readline())
os.system(f"mount --bind {HOSTS_PATH} /etc/hosts")
data_path = DATA_PATH(os.environ['ID'])
os.makedirs(data_path)
run_minio(n_serv)
def run_minio(count):
#print("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+str(count)+"}"+MINIO_PATH+" > "+ STORAGE_PATH + "/minio."+ os.environ['ID'] +".log 2>&1")
os.system("minio server --console-address ':9001' --address [" + os.environ['IP'] + "]:9000 http://minio{1.."+str(count)+"}"+MINIO_PATH+" > "+ STORAGE_PATH + "/minio."+ os.environ['ID'] +".log 2>&1")
def gen_hosts(ips):
with open(HOSTS_PATH, 'w') as f:
for idx,ip in enumerate(ips):
f.write(f"{ip} minio{idx+1}\n")
def n_servers():
with open('.current_state.yml', 'r') as f:
netw = yaml.safe_load(f)
n_servers = len(netw['servers'])
return n_servers
__name__ == '__main__' and main()