New secretmgr

cluster/staging/app/convertsecrets

@@ -0,0 +1 @@
+../../../secretmgr/convertsecrets

cluster/staging/app/core/secrets.toml

@@ -0,0 +1,4 @@
+[secrets."d53/gandi_api_key"]
+type = 'user'
+description = 'Gandi API key'
+

cluster/staging/app/core/secrets/d53/gandi_api_key

@@ -1 +0,0 @@
-USER Gandi API key

cluster/staging/app/directory/secrets.toml

@@ -0,0 +1,48 @@
+[secrets."directory/ldap_base_dn"]
+type = 'user'
+description = 'LDAP base DN for everything (e.g. dc=example,dc=com)'
+
+[secrets."directory/guichet/smtp_user"]
+type = 'user'
+description = 'SMTP username'
+
+[secrets."directory/guichet/s3_access_key"]
+type = 'user'
+description = 'Garage access key for Guichet profile pictures'
+
+[secrets."directory/guichet/s3_endpoint"]
+type = 'user'
+description = 'S3 endpoint URL'
+
+[secrets."directory/guichet/s3_region"]
+type = 'user'
+description = 'S3 region'
+
+[secrets."directory/guichet/smtp_pass"]
+type = 'user'
+description = 'SMTP password'
+
+[secrets."directory/guichet/web_hostname"]
+type = 'user'
+description = 'Public hostname from which Guichet is accessible via HTTP (e.g. guichet.example.com)'
+
+[secrets."directory/guichet/s3_bucket"]
+type = 'user'
+description = 'S3 bucket in which to store data files (such as profile pictures)'
+
+[secrets."directory/guichet/smtp_server"]
+type = 'user'
+description = 'SMTP server address (hostname:port)'
+
+[secrets."directory/guichet/s3_secret_key"]
+type = 'user'
+description = 'Garage secret key for Guichet profile pictures'
+
+[secrets."directory/guichet/mail_from"]
+type = 'user'
+description = 'E-mail address from which to send welcome emails to new users'
+
+[secrets."directory/guichet/mail_domain"]
+type = 'user'
+description = 'E-mail domain for new users (e.g. example.com)'
+

cluster/staging/app/directory/secrets/directory/guichet/mail_domain

@@ -1 +0,0 @@
-USER E-mail domain for new users (e.g. example.com)

cluster/staging/app/directory/secrets/directory/guichet/mail_from

@@ -1 +0,0 @@
-USER E-mail address from which to send welcome emails to new users

cluster/staging/app/directory/secrets/directory/guichet/s3_access_key

@@ -1 +0,0 @@
-USER Garage access key for Guichet profile pictures

cluster/staging/app/directory/secrets/directory/guichet/s3_bucket

@@ -1 +0,0 @@
-USER S3 bucket in which to store data files (such as profile pictures)

cluster/staging/app/directory/secrets/directory/guichet/s3_endpoint

@@ -1 +0,0 @@
-USER S3 endpoint URL

cluster/staging/app/directory/secrets/directory/guichet/s3_region

@@ -1 +0,0 @@
-USER S3 region

cluster/staging/app/directory/secrets/directory/guichet/s3_secret_key

@@ -1 +0,0 @@
-USER Garage secret key for Guichet profile pictures

cluster/staging/app/directory/secrets/directory/guichet/smtp_pass

@@ -1 +0,0 @@
-USER SMTP password

cluster/staging/app/directory/secrets/directory/guichet/smtp_server

@@ -1 +0,0 @@
-USER SMTP server address (hostname:port)

cluster/staging/app/directory/secrets/directory/guichet/smtp_user

@@ -1 +0,0 @@
-USER SMTP username

cluster/staging/app/directory/secrets/directory/guichet/web_hostname

@@ -1 +0,0 @@
-USER Public hostname from which Guichet is accessible via HTTP (e.g. guichet.example.com)

cluster/staging/app/directory/secrets/directory/ldap_base_dn

@@ -1 +0,0 @@
-USER LDAP base DN for everything (e.g. dc=example,dc=com)

cluster/staging/app/dummy/secrets.toml

@@ -0,0 +1,36 @@
+[service_users."dummy"]
+description = 'Service user for dummy database access'
+username_secret = "dummy/db_username"
+password_secret = "dummy/db_password"
+rotate_password = true
+
+[secrets."dummy/s3_access_key"]
+type = 'user'
+description = 'S3 access key ID for database storage'
+
+[secrets."dummy/s3_secret_key"]
+description = 'S3 secret key for database storage'
+type = 'user'
+
+[secrets."dummy/public_domain"]
+description = 'Publicly accessible domain for dummy resource'
+type = 'user'
+
+[secrets."dummy/form_secret"]
+description = 'Form secret for dummy web pages'
+type = 'command'
+command = 'openssl rand -base64 42'
+rotate = true
+
+[secrets."dummy/signing_key"]
+description = 'Key to use to sign dummy service data'
+type = 'command'
+command = 'openssl rand -base64 42'
+
+[secrets."dummy/rpc_secret"]
+description = 'RPC secret for communication between dummy nodes'
+type = 'command'
+command = 'openssl rand -base64 42'
+rotate = true
+
+

cluster/staging/app/garage/secrets.toml

@@ -0,0 +1,15 @@
+[secrets."garage-staging/admin_token"]
+type = 'command'
+rotate = true
+command = 'openssl rand -hex 32'
+
+[secrets."garage-staging/rpc_secret"]
+type = 'command'
+rotate = true
+command = 'openssl rand -hex 32'
+
+[secrets."garage-staging/metrics_token"]
+type = 'command'
+rotate = true
+command = 'openssl rand -hex 32'
+

cluster/staging/app/garage/secrets/garage-staging/admin_token

@@ -1 +0,0 @@
-CMD openssl rand -hex 32

cluster/staging/app/garage/secrets/garage-staging/metrics_token

@@ -1 +0,0 @@
-CMD openssl rand -hex 32

cluster/staging/app/garage/secrets/garage-staging/rpc_secret

@@ -1 +0,0 @@
-CMD openssl rand -hex 32

cluster/staging/app/im/secrets.toml

@@ -0,0 +1,27 @@
+[secrets."synapse/s3_access_key"]
+type = 'user'
+description = 'S3 access key ID for database storage'
+
+[secrets."synapse/form_secret"]
+type = 'command'
+rotate = true
+command = 'openssl rand -base64 42'
+
+[secrets."synapse/signing_key"]
+type = 'user'
+description = 'Signing key for messages'
+
+[secrets."synapse/macaroon_secret_key"]
+type = 'command'
+rotate = true
+command = 'openssl rand -base64 42'
+
+[secrets."synapse/registration_shared_secret"]
+type = 'command'
+rotate = true
+command = 'openssl rand -base64 42'
+
+[secrets."synapse/s3_secret_key"]
+type = 'user'
+description = 'S3 secret key for database storage'
+

cluster/staging/app/im/secrets/synapse/form_secret

@@ -1 +0,0 @@
-CMD openssl rand -base64 42

cluster/staging/app/im/secrets/synapse/macaroon_secret_key

@@ -1 +0,0 @@
-CMD openssl rand -base64 42

cluster/staging/app/im/secrets/synapse/registration_shared_secret

@@ -1 +0,0 @@
-CMD openssl rand -base64 42

cluster/staging/app/im/secrets/synapse/s3_access_key

@@ -1 +0,0 @@
-USER S3 access key ID for database storage

cluster/staging/app/im/secrets/synapse/s3_secret_key

@@ -1 +0,0 @@
-USER S3 secret key for database storage

cluster/staging/app/im/secrets/synapse/signing_key

@@ -1 +0,0 @@
-USER Signing key for messages

cluster/staging/app/telemetry/secrets.toml

@@ -0,0 +1,13 @@
+[secrets."telemetry/grafana/s3_access_key"]
+type = 'user'
+description = 'S3 access key for grafana db'
+
+[secrets."telemetry/grafana/admin_password"]
+type = 'command'
+rotate = true
+command = 'openssl rand -base64 12'
+
+[secrets."telemetry/grafana/s3_secret_key"]
+type = 'user'
+description = 'S3 secret key for grafana db'
+

cluster/staging/app/telemetry/secrets/telemetry/grafana/admin_password

@@ -1 +0,0 @@
-CMD openssl rand -base64 12

cluster/staging/app/telemetry/secrets/telemetry/grafana/s3_access_key

@@ -1 +0,0 @@
-USER S3 access key for grafana db

cluster/staging/app/telemetry/secrets/telemetry/grafana/s3_secret_key

@@ -1 +0,0 @@
-USER S3 secret key for grafana db

cluster/staging/secretmgr.toml

@@ -0,0 +1,9 @@
+[ldap]
+server = "ldap://localhost:1389"
+service_dn_suffix = "ou=services,ou=users,dc=staging,dc=deuxfleurs,dc=org"
+admin_dn = "cn=admin,dc=staging,dc=deuxfleurs,dc=org"
+admin_password_secret = "directory/admin_password"
+
+[constants]
+"dummy/public_domain" = "dummy.staging.deuxfleurs.org"
+"dummy/test_constant" = "test value"

convertsecrets

@@ -0,0 +1,215 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.pip ps.consul ps.ldap ps.passlib ps.requests ps.six ])"
+
+# DEPENDENCY: python-consul
+import consul
+
+# DEPENDENCY: python-ldap
+import ldap
+
+# DEPENDENCY: passlib
+from passlib.hash import ldap_salted_sha1
+
+import os
+import sys
+import glob
+import subprocess
+import getpass
+import base64
+from secrets import token_bytes
+
+
+"""
+TODO: this will be a utility to handle secrets in the Consul database
+for the various components of the Deuxfleurs infrastructure
+
+Functionnalities:
+- check that secrets are correctly configured
+- help user fill in secrets
+- create LDAP service users and fill in corresponding secrets
+- maybe one day: manage SSL certificates and keys
+
+It uses files placed in <module_name>/secrets/* to know what secrets
+it should handle. These secret files contain directives for what to do
+about these secrets.
+
+Example directives:
+
+USER <description>
+(a secret that must be filled in by the user)
+
+USER_LONG <description>
+(the same, indicates that the secret fits on several lines)
+
+CMD <command>
+(a secret that is generated by running this command)
+
+CMD_ONCE <command>
+(same, but value is not changed when doing a regen)
+
+CONST <constant value>
+(the secret has a constant value set here)
+
+CONST_LONG
+<constant value, several lines>
+(same)
+
+SERVICE_DN <service name> <service description>
+(the LDAP DN of a service user)
+
+SERVICE_PASSWORD <service name>
+(the LDAP password for the corresponding service user)
+
+SSL_CERT <cert name> <list of domains>
+(a SSL domain for the given domains)
+
+SSL_KEY <cert name>
+(the SSL key going with corresponding certificate)
+
+RSA_PUBLIC_KEY <key name> <key description>
+(a public RSA key)
+
+RSA_PRIVATE_KEY <key name>
+(the corresponding private RSA key)
+"""
+
+
+# Parameters
+LDAP_URL = "ldap://localhost:1389"
+SERVICE_DN_SUFFIX = "ou=services,ou=users,dc=deuxfleurs,dc=fr"
+consul_server = consul.Consul()
+
+
+# ----
+
+USER             = "USER"
+USER_LONG        = "USER_LONG"
+CMD              = "CMD"
+CMD_ONCE         = "CMD_ONCE"
+CONST            = "CONST"
+CONST_LONG       = "CONST_LONG"
+SERVICE_DN       = "SERVICE_DN"
+SERVICE_PASSWORD = "SERVICE_PASSWORD"
+SSL_CERT         = "SSL_CERT"
+SSL_KEY          = "SSL_KEY"
+RSA_PUBLIC_KEY   = "RSA_PUBLIC_KEY"
+RSA_PRIVATE_KEY  = "RSA_PRIVATE_KEY"
+
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKCYAN = '\033[96m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+def read_secret(key, file_path):
+    lines = [l.strip() for l in open(file_path, "r")]
+    if len(lines) == 0:
+        print(bcolors.FAIL, "ERROR:", bcolors.ENDC, "Empty file in", file_path)
+        sys.exit(-1)
+    l0 = lines[0].split(" ")
+    stype = l0[0]
+    secret = {"type": stype, "key": key}
+    if stype in [USER, USER_LONG]:
+        secret["desc"] = " ".join(l0[1:])
+    elif stype in [CMD, CMD_ONCE]:
+        secret["cmd"] = " ".join(l0[1:])
+    elif stype == CONST:
+        secret["value"] = " ".join(l0[1:])
+    elif stype == CONST_LONG:
+        secret["value"] = "\n".join(lines[1:])
+    elif stype in [SERVICE_DN, SERVICE_PASSWORD]:
+        secret["service"] = l0[1]
+        if stype == SERVICE_DN:
+            secret["service_desc"] = " ".join(l0[2:])
+    elif stype in [SSL_CERT, SSL_KEY]:
+        secret["cert_name"] = l0[1]
+        if stype == SSL_CERT:
+            secret["cert_domains"] = l0[2:]
+    elif stype in [RSA_PUBLIC_KEY, RSA_PRIVATE_KEY]:
+        secret["key_name"] = l0[1]
+        if stype == RSA_PUBLIC_KEY:
+            secret["key_desc"] = " ".join(l0[2:])
+    else:
+        print(bcolors.FAIL, "ERROR:", bcolors.ENDC, "Invalid secret type", stype, "in", file_path)
+        sys.exit(-1)
+
+    return secret
+
+def read_secrets(module_list):
+    secrets = {}
+    for mod in module_list:
+        for file_path in glob.glob(mod.strip('/') + "/secrets/**", recursive=True):
+            if os.path.isfile(file_path):
+                key = '/'.join(file_path.split("/")[1:])
+                secrets[key] = read_secret(key, file_path)
+    return secrets
+
+def convert_secrets(module_list):
+    for mod in module_list:
+        print("converting module: ", mod)
+        secrets = read_secrets([mod])
+        with open(os.path.join(mod.strip('/'), "secrets.toml"), "w") as file:
+            for (secret_key, secret) in secrets.items():
+                file.write("[secrets.\"{}\"]\n".format("/".join(secret_key.split("/")[1:])))
+                ty = secret["type"]
+                if ty in [CMD, CMD_ONCE]:
+                    newsecret = {
+                            "type": "command",
+                            "rotate": ty != "CMD_ONCE",
+                            "command": secret["cmd"],
+                        }
+                elif ty in [USER, USER_LONG]:
+                    newsecret = {
+                            "type": "user",
+                            "multiline": ty == "USER_LONG",
+                            "description": secret["desc"],
+                        }
+                elif ty in [CONST, CONST_LONG]:
+                    newsecret = {
+                            "type": "constant",
+                            "value": secret["value"],
+                        }
+                else:
+                    newsecret = {
+                            "type": secret["type"],
+                    }
+                    if "service_desc" in secret:
+                        newsecret["description"] = secret["service_desc"]
+                    if "key_desc" in secret:
+                        newsecret["description"] = secret["key_desc"]
+                    if "cert_name" in secret:
+                        newsecret["name"] = secret["cert_name"]
+                    if "key_name" in secret:
+                        newsecret["name"] = secret["key_name"]
+                    for k in ["service", "cert_domains"]:
+                        if k in secret:
+                            newsecret[k] = secret[k]
+                for (k, v) in newsecret.items():
+                    if type(v) == bool:
+                        if v:
+                            file.write("{} = true\n".format(k))
+                    elif type(v) == str:
+                        file.write("{} = {}\n".format(k, repr(v)))
+                    else:
+                        print("invalid value: ", v)
+                file.write("\n")
+
+
+# ---- MAIN ----
+
+if __name__ == "__main__":
+    for i, val in enumerate(sys.argv):
+        if val == "convert":
+            convert_secrets(sys.argv[i+1:])
+            break
+        else:
+            print("Usage:")
+            print("    convertsecrets convert <module name>...")
+
+
+#  vim: set sts=4 ts=4 sw=4 tw=0 ft=python et :

dev_secretmgr

@@ -1,455 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.pip ps.consul ps.ldap ps.passlib ps.requests ps.six ps.toml ])"
-
-# DEPENDENCY: python-consul
-import consul
-
-# DEPENDENCY: python-ldap
-import ldap
-
-# DEPENDENCY: passlib
-from passlib.hash import ldap_salted_sha1
-
-# DEPENDENCY: toml
-import toml
-
-import os
-import sys
-import glob
-import subprocess
-import getpass
-import base64
-from secrets import token_bytes
-
-
-"""
-TODO: this will be a utility to handle secrets in the Consul database
-for the various components of the Deuxfleurs infrastructure
-
-Functionnalities:
-- check that secrets are correctly configured
-- help user fill in secrets
-- create LDAP service users and fill in corresponding secrets
-- maybe one day: manage SSL certificates and keys
-
-It uses files placed in <module_name>/secrets/* to know what secrets
-it should handle. These secret files contain directives for what to do
-about these secrets.
-
-Example directives:
-
-USER <description>
-(a secret that must be filled in by the user)
-
-USER_LONG <description>
-(the same, indicates that the secret fits on several lines)
-
-CMD <command>
-(a secret that is generated by running this command)
-
-CMD_ONCE <command>
-(same, but value is not changed when doing a regen)
-
-CONST <constant value>
-(the secret has a constant value set here)
-
-CONST_LONG
-<constant value, several lines>
-(same)
-
-SERVICE_DN <service name> <service description>
-(the LDAP DN of a service user)
-
-SERVICE_PASSWORD <service name>
-(the LDAP password for the corresponding service user)
-
-SSL_CERT <cert name> <list of domains>
-(a SSL domain for the given domains)
-
-SSL_KEY <cert name>
-(the SSL key going with corresponding certificate)
-
-RSA_PUBLIC_KEY <key name> <key description>
-(a public RSA key)
-
-RSA_PRIVATE_KEY <key name>
-(the corresponding private RSA key)
-"""
-
-
-# Parameters
-LDAP_URL = "ldap://localhost:1389"
-SERVICE_DN_SUFFIX = "ou=services,ou=users,dc=deuxfleurs,dc=fr"
-consul_server = consul.Consul()
-
-
-# ---- UTIL ----
-
-class bcolors:
-    HEADER = '\033[95m'
-    OKBLUE = '\033[94m'
-    OKCYAN = '\033[96m'
-    OKGREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-    BOLD = '\033[1m'
-    UNDERLINE = '\033[4m'
-
-ldap_admin_conn = None
-def get_ldap_admin_conn():
-    global ldap_admin_conn
-    if ldap_admin_conn is None:
-        ldap_admin_conn = ldap.initialize(LDAP_URL)
-        ldap_user = input("LDAP admin user (full DN, please!): ")
-        ldap_pass = getpass.getpass("LDAP admin password: ")
-        ldap_admin_conn.simple_bind_s(ldap_user, ldap_pass)
-    return ldap_admin_conn
-
-# ---- SECRETS ----
-
-class Secret:
-    def __init__(self, key, description=None):
-        self.consul_key = "secrets/" + key
-        if description != None:
-            self.description = description
-
-    def check(self, value):
-        return True
-
-    def generate(self, old_value):
-        pass
-
-class UserSecret(Secret):
-    def __init__(self, example=None, multiline=False, **kwargs):
-        Secret.__init__(self, **kwargs)
-        self.example = example
-        self.multiline = multiline
-
-    def generate(self, old_value):
-            print("Description:", self.description)
-            print("Enter value for secret, or ^C to skip:")
-            if self.multiline:
-                print("THIS IS A LONG VALUE, ENTER SEVERAL LINES AND FINISH WITH A LINE CONTAINING A SINGLE .")
-                try:
-                    lines = []
-                    while True:
-                        line = input().strip()
-                        if line == ".":
-                            break
-                        lines.append(line)
-                    return "\n".join(lines)
-                except KeyboardInterrupt:
-                    return None
-            else:
-                try:
-                    while True:
-                        line = input().strip()
-                        if line != "":
-                            return line
-                        else:
-                            print("Please enter a non-empty value, or ^C to skip:")
-                except KeyboardInterrupt:
-                    return None
-
-class CommandSecret(Secret):
-    def __init__(self, command, rotate=False, **kwargs):
-        Secret.__init__(self, **kwargs)
-        self.command = command
-        self.rotate = rotate
-
-    def generate(self, old_value):
-        if not self.rotate and old_value is not None:
-            return None
-
-        print("Executing command:", self.command)
-        return subprocess.check_output(["sh", "-c", self.command])
-
-class ConstantSecret(Secret):
-    def __init__(self, value, **kwargs):
-        Secret.__init__(self, **kwargs)
-        self.value = value
-
-    def check(self, value):
-        return value == self.value
-
-    def generate(self, old_value):
-        if old_value != self.value:
-            return self.value
-        else:
-            return None
-
-# ---- SERVICE USERS ----
-
-class ServiceUserPasswordSecret:
-    def __init__(self, service_user, **kwargs):
-        Secret.__init__(self, **kwargs)
-        self.service_user = service_user
-
-    def check(self, value):
-        l = ldap.initialize(LDAP_URL)
-        try:
-            l.simple_bind_s(self.service_user.dn, value)
-            return True
-        except Exception as e:
-            return False
-
-    def generate(self, old_value):
-        if old_value != self.service_user.password:
-            return self.service_user.password
-        else:
-            return None
-
-class ServiceUser:
-    def __init__(self, username, password_secret, user_dn_secret=None, user_name_secret=None, rotate_password=False):
-        self.username = username
-        self.password = None
-        self.dn = "cn={},{}".format(self.username, SERVICE_DN_SUFFIX)
-        self.rotate_password = rotate_password
-
-        self.username_secret = username_secret
-        self.password_secret = password_secret
-        self.dn_secret = dn_secret
-
-    def secrets(self):
-        secrets = {}
-        secrets[self.password_secret] = ServiceUserPasswordSecret(
-                service_user=self,
-                key=self.password_secret,
-                description="LDAP password for service user {}".format(self.username),
-                )
-        if self.user_dn_secret != None:
-            secrets[self.dn_secret] = ConstantSecret(
-                    key=self.dn_secret,
-                    value=self.dn,
-                    description="LDAP DN for service user {}".format(self.username),
-                )
-        if self.username_secret != None:
-            secrets[self.username_secret] = ConstantSecret(
-                    key=self.username_secret,
-                    value=self.username,
-                    description="LDAP username for service user {}".format(self.username),
-                    )
-        return secrets
-
-# ---- READING secrets.toml FILES ----
-
-class Config:
-    def __init__(self, cluster_name):
-        self.cluster_name = cluster_name
-        self.app_path = os.path.join(".", "cluster", cluster_name)
-        self.service_users = {}
-        self.secrets = {}
-        self.modules = []
-
-    def load_module(self, module_name)
-        secrets_toml_path = os.path.join(self.app_path, module_name, "secrets.toml")
-
-        with open(secrets_toml_path) as f:
-            secrets_toml = toml.load(f)
-
-        self.modules.append(module_name)
-
-        # Service users, and their associated secrets
-        if "service_users" in secrets_toml:
-            for (uname, uargs) in secrets_toml["service_users"].items():
-                service_user = ServiceUser(uname, **uargs)
-                for (skey, secret) in service_user.secrets():
-                    if skey in self.secrets:
-                        raise Exception("Duplicate secret: {}".format(skey))
-                    self.secrets[skey] = secret
-                self.service_users[uname] = service_user
-
-        # Other secrets
-        if "secrets" in secrets_toml:
-            for (skey, sargs) in secrets_toml["secrets"].items():
-                ty = sargs["type"]
-                del sargs["type"]
-                if ty == "user":
-                    secret = UserSecret(key=skey, **sargs)
-                elif ty == "command":
-                    secret = CommandSecret(key=skey, **sargs)
-                elif ty == "constant":
-                    secret = Constantsecret(key=skey, **sargs)
-                else:
-                    raise Exception("Invalid secret type: {}".format(ty))
-                if skey in self.secrets:
-                    raise Exception("Duplicate secret: {}".format(skey))
-                self.secrets[skey] = secret
-
-    def check_secrets(self):
-        print(":: Checking secrets...")
-        for (_, secret) in self.secrets.items():
-            _, data = consul_server.kv.get(secret.consul_key)
-            if data is None:
-                print(secret.consul_key, bcolors.FAIL, "x", bcolors.ENDC, " missing")
-            elif not secret.check(data["Value"].decode('ascii').strip()): 
-                print(secret.consul_key, bcolors.FAIL, "x", bcolors.ENDC, " wrong value")
-            else:
-                print(secret.consul_key, bcolors.OKGREEN, "✓", bcolors.ENDC)
-        print()
-
-    def gen_secrets(self):
-        print(":: Generating missing secrets...")
-        for (_, secret) in self.secrets.items():
-            _, data = consul_server.kv.get(secret.consul_key)
-            if data is None:
-                value = secret.generate(None)
-                if value != None:
-                    print("PUT {} = {}".format(secret.consul_key, value))
-                    print(bcolors.OKCYAN, "Value set.", bcolors.ENDC)
-                else:
-                    print(bcolors.WARNING, "Skipped.", bcolors.ENDC)
-        print()
-
-    def rotate_secrets(self):
-        print(":: Rotating secrets...")
-        for (_, secret) in self.secrets.items():
-            print(": {} ({})".format(secret.consul_key, secret.__class__.__name__))
-            _, data = consul_server.kv.get(secret.consul_key)
-            if data is None:
-                value = secret.generate(None)
-            else:
-                value = secret.generate(data["Value"].decode('ascii').strip())
-            if value != None:
-                print("PUT {} = {}".format(secret.consul_key, value))
-                print(bcolors.OKCYAN, "Value set.", bcolors.ENDC)
-            else:
-                print(bcolors.WARNING, "Skipped.", bcolors.ENDC)
-            print()
-
-def check_secrets(modules):
-    for mod in modules:
-        print("==== MODULE `{}` ====".format(mod))
-        modcfg = Config(mod)
-        modcfg.check_secrets()
-        print("")
-
-def gen_secrets(modules):
-    for mod in modules:
-        print("==== MODULE `{}` ====".format(mod))
-        modcfg = Config(mod)
-        modcfg.gen_secrets()
-        modcfg.check_secrets()
-        print("")
-
-def rotate_secrets(modules):
-    for mod in modules:
-        print("==== MODULE `{}` ====".format(mod))
-        modcfg = Config(mod)
-        modcfg.rotate_secrets()
-        modcfg.check_secrets()
-        print("")
-
-# ----- old ----
-
-
-# ---- CHECK COMMAND ----
-
-
-def check_secrets_services(secrets):
-    print("Checking secrets for LDAP service users...")
-    services = get_secrets_services(secrets)
-
-    for svc_name, svc in services.items():
-        for dn_key in svc["dn_at"]:
-            _, data = consul_server.kv.get(dn_key)
-            if data is not None:
-                got_val = data["Value"].decode('ascii').strip()
-                if got_val != svc["dn"]:
-                    print(svc_name, "wrong DN at", dn_key, bcolors.FAIL, "x", bcolors.ENDC)
-                    print("got:", got_val, "instead of:", svc["dn"])
-
-        if svc["pass"] is None:
-            print(svc_name, bcolors.FAIL, "no password stored", bcolors.ENDC)
-        else:
-            for pass_key in svc["pass_at"]:
-                _, data = consul_server.kv.get(pass_key)
-                if data is not None:
-                    got_val = data["Value"].decode('ascii').strip()
-                    if got_val != svc["pass"]:
-                        print(svc_name, "wrong pass at", dn_key, bcolors.FAIL, "x", bcolors.ENDC)
-
-            l = ldap.initialize(LDAP_URL)
-            try:
-                l.simple_bind_s(svc["dn"], svc["pass"])
-                print(svc_name, bcolors.OKGREEN, "✓", bcolors.ENDC)
-            except Exception as e:
-                print(svc_name, bcolors.FAIL, e, bcolors.ENDC)
-    print()
-
-
-# ---- GEN COMMAND ----
-
-def gen_secrets_services(secrets, regen):
-    print("Generating LDAP service accounts...")
-    services = get_secrets_services(secrets)
-
-    for svc_name, svc in services.items():
-        print("----")
-        print("Service:", svc_name)
-        print("Description:", svc["desc"])
-
-        for dn_key in svc["dn_at"]:
-            _, data = consul_server.kv.get(dn_key)
-            if data is None or data["Value"].decode('ascii').strip() != svc["dn"]:
-                print(bcolors.OKCYAN, "Setting DN", bcolors.ENDC, "at", dn_key)
-                consul_server.kv.put(dn_key, svc["dn"])
-
-        if svc["pass"] is None or regen:
-            print(bcolors.OKCYAN, "Generating new password", bcolors.ENDC)
-            svc["pass"] = base64.urlsafe_b64encode(token_bytes(12)).decode('ascii')
-
-        l = ldap.initialize(LDAP_URL)
-        try:
-            l.simple_bind_s(svc["dn"], svc["pass"])
-        except:
-            fix_service_user(svc)
-
-        for pass_key in svc["pass_at"]:
-            _, data = consul_server.kv.get(pass_key)
-            if data is None or data["Value"].decode('ascii').strip() != svc["pass"]:
-                print(bcolors.OKCYAN, "Setting password", bcolors.ENDC, "at", pass_key)
-                consul_server.kv.put(pass_key, svc["pass"])
-
-    print()
-
-def fix_service_user(svc):
-    print("Fixing service user", svc["dn"], "...")
-    l = get_ldap_admin_conn()
-    res = l.search_s(svc["dn"], ldap.SCOPE_BASE, "objectclass=*")
-    pass_crypt = ldap_salted_sha1.hash(svc["pass"])
-    if res is None or len(res) == 0:
-        print(bcolors.OKCYAN, "Creating entity...", bcolors.ENDC)
-        l.add_s(svc["dn"],
-                [
-                    ("objectclass",     [b"person", b"top"]),
-                    ("displayname",     [svc["desc"].encode('ascii')]),
-                    ("userpassword",     [pass_crypt.encode('ascii')]),
-                ])
-    else:
-        print(bcolors.OKCYAN, "Resetting entity password", bcolors.ENDC)
-        l.modify_s(svc["dn"],
-                [
-                    (ldap.MOD_REPLACE, "userpassword", [pass_crypt.encode('ascii')])
-                ])
-
-# ---- MAIN ----
-
-if __name__ == "__main__":
-    for i, val in enumerate(sys.argv):
-        if val == "check":
-            check_secrets(sys.argv[i+1:])
-            break
-        elif val == "gen":
-            gen_secrets(sys.argv[i+1:])
-            break
-        elif val == "rotate":
-            rotate_secrets(sys.argv[i+1:])
-            break
-        else:
-            print("Usage:")
-            print("    secretmgr.py [check|gen|rotate] <module name>...")
-
-
-#  vim: set sts=4 ts=4 sw=4 tw=0 ft=python et :

secretmgr

@@ -0,0 +1,444 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.pip ps.consul ps.ldap ps.passlib ps.requests ps.six ps.toml ])"
+
+# DEPENDENCY: python-consul
+import consul
+
+# DEPENDENCY: python-ldap
+import ldap
+
+# DEPENDENCY: passlib
+from passlib.hash import ldap_salted_sha1
+
+# DEPENDENCY: toml
+import toml
+
+import os
+import sys
+import glob
+import subprocess
+import getpass
+import base64
+from secrets import token_bytes
+
+"""
+This is a utility to handle secrets in the Consul database
+for the various components of the Deuxfleurs infrastructure
+
+Functionnalities:
+- check that secrets are correctly configured
+- help user fill in secrets
+- create LDAP service users and fill in corresponding secrets
+- TODO: manage Garage buckets and access keys
+- maybe one day: manage SSL certificates and keys
+"""
+
+# ---- UTIL ----
+
+consul_server = consul.Consul()
+
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKCYAN = '\033[96m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+
+# ---- SECRETS ----
+
+class Secret:
+    def __init__(self, key, config, description=None):
+        self.config = config
+        self.key = key
+        self.consul_key = "secrets/" + key
+        if description != None:
+            self.description = description
+        else:
+            self.description = None
+
+    def check(self, value):
+        return True
+
+    def generate(self):
+        pass
+
+    def rotate(self):
+        return None
+
+    def print_info(self):
+        print("Secret:         {}".format(self.consul_key))
+        print("Type:           {}".format(self.__class__.__name__))
+        if self.description != None:
+            print("Description:    {}".format(self.description))
+
+class UserSecret(Secret):
+    def __init__(self, example=None, multiline=False, **kwargs):
+        Secret.__init__(self, **kwargs)
+        self.example = example
+        self.multiline = multiline
+
+    def generate(self):
+        if self.key in self.config.constants:
+            print("Using constant value from cluster's secretmgr.toml")
+            return self.config.constants[self.key]
+
+        print("Enter value for secret, or ^C to skip:")
+        if self.multiline:
+            print("THIS IS A LONG VALUE, ENTER SEVERAL LINES AND FINISH WITH A LINE CONTAINING A SINGLE .")
+            try:
+                lines = []
+                while True:
+                    line = input().strip()
+                    if line == ".":
+                        break
+                    lines.append(line)
+                return "\n".join(lines)
+            except KeyboardInterrupt:
+                return None
+        else:
+            try:
+                while True:
+                    line = input().strip()
+                    if line != "":
+                        return line
+                    else:
+                        print("Please enter a non-empty value, or ^C to skip:")
+            except KeyboardInterrupt:
+                return None
+
+class CommandSecret(Secret):
+    def __init__(self, command, rotate=False, **kwargs):
+        Secret.__init__(self, **kwargs)
+        self.command = command
+        self.rotate_value = rotate
+
+    def generate(self):
+        print("Executing command:", self.command)
+        return subprocess.check_output(["sh", "-c", self.command])
+
+    def rotate(self):
+        if self.rotate_value:
+            return self.generate()
+        else:
+            return None
+
+class ConstantSecret(Secret):
+    def __init__(self, value, **kwargs):
+        Secret.__init__(self, **kwargs)
+        self.value = value
+
+    def check(self, value):
+        return value == self.value
+
+    def generate(self):
+        return self.value
+
+
+# ---- SERVICE USERS ----
+
+class ServiceUserPasswordSecret(Secret):
+    def __init__(self, service_user, **kwargs):
+        Secret.__init__(self, **kwargs)
+        self.service_user = service_user
+
+    def check(self, value):
+        l = ldap.initialize(self.config.ldap_server)
+        try:
+            l.simple_bind_s(self.service_user.dn, value)
+            return True
+        except Exception as e:
+            return False
+
+    def generate(self):
+        return self.service_user.password
+
+    def rotate(self):
+        return self.service_user.password
+
+class ServiceUser:
+    def __init__(self, username, password_secret, config, description=None, dn_secret=None, username_secret=None, rotate_password=False):
+        self.config = config
+        self.username = username
+        self.description = description
+        self.password = None
+        self.dn = "cn={},{}".format(self.username, self.config.ldap_service_dn_suffix)
+        self.rotate_password = rotate_password
+
+        self.password_secret = ServiceUserPasswordSecret(
+                config=config,
+                service_user=self,
+                key=password_secret,
+                description="LDAP password for service user {}".format(username),
+                )
+
+        self.username_secret = None
+        if username_secret != None:
+            self.username_secret = ConstantSecret(
+                    config=config,
+                    key=username_secret,
+                    value=username,
+                    description="LDAP username for service user {}".format(username),
+                    )
+
+        self.dn_secret = None
+        if dn_secret != None:
+            self.dn_secret = ConstantSecret(
+                    config=config,
+                    key=dn_secret,
+                    value=self.dn,
+                    description="LDAP DN for service user {}".format(username),
+                )
+
+    def secrets(self):
+        secrets = {}
+        secrets[self.password_secret.key] = self.password_secret
+        if self.dn_secret != None:
+            secrets[self.dn_secret.key] = self.dn_secret
+        if self.username_secret != None:
+            secrets[self.username_secret.key] = self.username_secret
+        return secrets
+
+    def configure(self, rotate):
+        _, data = consul_server.kv.get(self.password_secret.consul_key)
+        if data is None:
+            good = False
+        else:
+            l = ldap.initialize(self.config.ldap_server)
+            try:
+                self.password = data["Value"].decode('ascii')
+                l.simple_bind_s(self.dn, self.password)
+                good = True
+            except:
+                good = False
+
+        if not good or (rotate and self.rotate_password):
+            # Reset passsword
+            self.password = base64.urlsafe_b64encode(token_bytes(12)).decode('ascii')
+            pass_crypt = ldap_salted_sha1.hash(self.password).encode('ascii')
+
+            l = self.config.get_ldap_admin_conn()
+            res = l.search_s(self.dn, ldap.SCOPE_BASE, "objectclass=*")
+            if res is None or len(res) == 0:
+                print(bcolors.OKCYAN, "Creating entity", self.dn, bcolors.ENDC)
+                l.add_s(self.dn,
+                        [
+                            ("objectclass",     [b"person", b"top"]),
+                            ("displayname",     [self.description.encode('ascii')]),
+                            ("userpassword",     [pass_crypt]),
+                        ])
+            else:
+                print(bcolors.OKCYAN, "Resetting password for entity", self.dn, bcolors.ENDC)
+                l.modify_s(self.dn,
+                        [
+                            (ldap.MOD_REPLACE, "userpassword", [pass_crypt])
+                        ])
+        else:
+            print(bcolors.OKGREEN, "Entity is good: ", self.dn, bcolors.ENDC)
+
+
+# ---- MAIN CONFIG CLASS ----
+
+class Config:
+    def __init__(self, cluster_name):
+        self.cluster_name = cluster_name
+        self.app_path = os.path.join(".", "cluster", cluster_name, "app")
+
+        self.service_users = {}
+        self.secrets = {}
+        self.modules = []
+
+        # Load config from secretmgr.toml in cluster directory
+        secretmgr_toml_path = os.path.join(".", "cluster", cluster_name, "secretmgr.toml")
+        if os.path.exists(secretmgr_toml_path):
+            with open(secretmgr_toml_path) as f:
+                secretmgr_toml = toml.load(f)
+        else:
+            secretmgr_toml = {}
+
+        if "constants" in secretmgr_toml:
+            self.constants = secretmgr_toml["constants"]
+        else:
+            self.constants = {}
+
+        self.ldap_server = None
+        self.ldap_service_dn_suffix = None
+        self.ldap_admin_dn = None
+        self.ldap_admin_password_secret = None
+        self.ldap_admin_conn = None
+        if "ldap" in secretmgr_toml:
+            if "server" in secretmgr_toml["ldap"]:
+                self.ldap_server = secretmgr_toml["ldap"]["server"]
+            if "service_dn_suffix" in secretmgr_toml["ldap"]:
+                self.ldap_service_dn_suffix = secretmgr_toml["ldap"]["service_dn_suffix"]
+            if "admin_dn" in secretmgr_toml["ldap"]:
+                self.ldap_admin_dn = secretmgr_toml["ldap"]["admin_dn"]
+            if "admin_password_secret" in secretmgr_toml["ldap"]:
+                self.ldap_admin_password_secret = secretmgr_toml["ldap"]["admin_password_secret"]
+
+    def load_module(self, module_name):
+        secrets_toml_path = os.path.join(self.app_path, module_name, "secrets.toml")
+
+        with open(secrets_toml_path) as f:
+            secrets_toml = toml.load(f)
+
+        self.modules.append(module_name)
+
+        # Service users, and their associated secrets
+        if "service_users" in secrets_toml:
+            for (uname, uargs) in secrets_toml["service_users"].items():
+                service_user = ServiceUser(uname, config=self, **uargs)
+                for (skey, secret) in service_user.secrets().items():
+                    if skey in self.secrets:
+                        raise Exception("Duplicate secret: {}".format(skey))
+                    self.secrets[skey] = secret
+                self.service_users[uname] = service_user
+
+        # Other secrets
+        if "secrets" in secrets_toml:
+            for (skey, sargs) in secrets_toml["secrets"].items():
+                ty = sargs["type"]
+                del sargs["type"]
+                if ty == "user":
+                    secret = UserSecret(config=self, key=skey, **sargs)
+                elif ty == "command":
+                    secret = CommandSecret(config=self, key=skey, **sargs)
+                elif ty == "constant":
+                    secret = ConstantSecret(config=self, key=skey, **sargs)
+                else:
+                    raise Exception("Invalid secret type: {}".format(ty))
+                if skey in self.secrets:
+                    raise Exception("Duplicate secret: {}".format(skey))
+                self.secrets[skey] = secret
+
+    def add_constant_secrets(self):
+        for (skey, value) in self.constants.items():
+            self.secrets[skey] = ConstantSecret(
+                config=self,
+                key=skey,
+                value=value,
+                description="Constant value for secret {}".format(skey))
+
+    def get_ldap_admin_conn(self):
+        if self.ldap_admin_conn is None:
+            if self.ldap_admin_password_secret != None:
+                pass_key = "secrets/" + self.ldap_admin_password_secret
+                _, data = consul_server.kv.get(pass_key)
+                if data is None:
+                    raise Exception("LDAP admin password could not be read at: {}".format(pass_key))
+                ldap_pass = data["Value"].decode('ascii').strip()
+            else:
+                ldap_pass = getpass.getpass("LDAP admin password: ")
+
+            self.ldap_admin_conn = ldap.initialize(self.ldap_server)
+            self.ldap_admin_conn.simple_bind_s(self.ldap_admin_dn, ldap_pass)
+        return self.ldap_admin_conn
+
+    def check_secrets(self):
+        print(":: Checking secrets...")
+        must_gen = False
+        for (_, secret) in self.secrets.items():
+            _, data = consul_server.kv.get(secret.consul_key)
+            if data is None:
+                print(secret.consul_key, bcolors.FAIL, "x  missing", bcolors.ENDC)
+                must_gen = True
+            elif not secret.check(data["Value"].decode('ascii').strip()): 
+                print(secret.consul_key, bcolors.WARNING, "x  bad value", bcolors.ENDC)
+                must_gen = True
+            else:
+                print(secret.consul_key, bcolors.OKGREEN, "✓", bcolors.ENDC)
+        print()
+        if must_gen:
+            print("To fix missing or invalid secrets, use `secretmgr gen <cluster_name> <app>...`")
+            print()
+
+    def gen_secrets(self):
+        if len(self.service_users) > 0:
+            print(":: Configuring service users...")
+            for (_, su) in self.service_users.items():
+                su.configure(False)
+            print()
+
+        print(":: Generating missing/invalid secrets...")
+        for (_, secret) in self.secrets.items():
+            _, data = consul_server.kv.get(secret.consul_key)
+            if data is None or not secret.check(data["Value"].decode('ascii').strip()):
+                print()
+                secret.print_info()
+                value = secret.generate()
+                if value != None:
+                    consul_server.kv.put(secret.consul_key, value)
+                    print(bcolors.OKCYAN, "Value set.", bcolors.ENDC)
+                else:
+                    print(bcolors.WARNING, "Skipped.", bcolors.ENDC)
+
+        print()
+        self.check_secrets()
+
+    def rotate_secrets(self):
+        if len(self.service_users) > 0:
+            print(":: Regenerating service user passwords...")
+            for (_, su) in self.service_users.items():
+                su.configure(True)
+            print()
+
+        print(":: Rotating secrets...")
+        for (_, secret) in self.secrets.items():
+            print()
+            secret.print_info()
+
+            _, data = consul_server.kv.get(secret.consul_key)
+            if data is None:
+                old_value = None
+            else:
+                old_value = data["Value"].decode('ascii').strip()
+            new_value = secret.rotate()
+
+            if new_value != None and new_value != old_value:
+                consul_server.kv.put(secret.consul_key, new_value)
+                print(bcolors.OKCYAN, "Value set.", bcolors.ENDC)
+            else:
+                print(bcolors.OKGREEN, "Nothing to do.", bcolors.ENDC)
+
+        print()
+        self.check_secrets()
+
+
+# ---- MAIN ----
+
+def load_config(cluster_name, modules):
+    cfg = Config(cluster_name)
+    if len(modules) > 0:
+        for mod in modules:
+            cfg.load_module(mod)
+    else:
+        cfg.add_constant_secrets()
+    return cfg
+
+if __name__ == "__main__":
+    verb = None
+
+    for i, val in enumerate(sys.argv):
+        if val == "check":
+            verb = lambda cfg: cfg.check_secrets()
+            break
+        elif val == "gen":
+            verb = lambda cfg: cfg.gen_secrets()
+            break
+        elif val == "rotate":
+            verb = lambda cfg: cfg.rotate_secrets()
+            break
+
+    if verb is None:
+        print("Usage:")
+        print("    secretmgr.py [check|gen|rotate] <cluster name> <module name>...")
+    else:
+        cfg = load_config(sys.argv[i+1], sys.argv[i+2:])
+        verb(cfg)
+
+
+#  vim: set sts=4 ts=4 sw=4 tw=0 ft=python et :