Improve secretmgr more, update secrets for staging

cluster/staging/app/directory/secrets.toml

@@ -1,6 +1,7 @@
 [secrets."directory/ldap_base_dn"]
 type = 'user'
-description = 'LDAP base DN for everything (e.g. dc=example,dc=com)'
+description = 'LDAP base DN for everything'
+example = 'dc=example,dc=com'
 
 [secrets."directory/guichet/smtp_user"]
 type = 'user'
@@ -24,7 +25,8 @@ description = 'SMTP password'
 
 [secrets."directory/guichet/web_hostname"]
 type = 'user'
-description = 'Public hostname from which Guichet is accessible via HTTP (e.g. guichet.example.com)'
+description = 'Public hostname from which Guichet is accessible via HTTP'
+example = 'guichet.example.com'
 
 [secrets."directory/guichet/s3_bucket"]
 type = 'user'
@@ -44,5 +46,6 @@ description = 'E-mail address from which to send welcome emails to new users'
 
 [secrets."directory/guichet/mail_domain"]
 type = 'user'
-description = 'E-mail domain for new users (e.g. example.com)'
+description = 'E-mail domain for new users'
+example = 'example.com'
 

cluster/staging/secretmgr.toml

@@ -4,6 +4,16 @@ service_dn_suffix = "ou=services,ou=users,dc=staging,dc=deuxfleurs,dc=org"
 admin_dn = "cn=admin,dc=staging,dc=deuxfleurs,dc=org"
 admin_password_secret = "directory/admin_password"
 
-[constants]
+[user_values]
+"directory/ldap_base_dn" = "dc=staging,dc=deuxfleurs,dc=org"
+"directory/guichet/mail_domain" = "staging.deuxfleurs.org"
+"directory/guichet/mail_from" = "contact@deuxfleurs.org"
+"directory/guichet/s3_bucket" = "bottin-pictures"
+"directory/guichet/s3_endpoint" = "garage.staging.deuxfleurs.org"
+"directory/guichet/s3_region" = "garage-staging"
+"directory/guichet/smtp_server" = "mail.gandi.net:25"
+"directory/guichet/smtp_user" = "contact@deuxfleurs.org"
+"directory/guichet/web_hostname" = "guichet.staging.deuxfleurs.org"
+
 "dummy/public_domain" = "dummy.staging.deuxfleurs.org"
 "dummy/test_constant" = "test value"

convertsecrets

@@ -196,7 +196,9 @@ def convert_secrets(module_list):
                     elif type(v) == str:
                         file.write("{} = {}\n".format(k, repr(v)))
                     else:
-                        print("invalid value: ", v)
+                        print("warning: invalid value: ", v)
+                        print("secret to fix: ", k)
+                        file.write("{} = {}\n".format(k, repr(repr(v))))
                 file.write("\n")
 
 

secretmgr

@@ -55,7 +55,6 @@ class Secret:
     def __init__(self, key, config, description=None):
         self.config = config
         self.key = key
-        self.consul_key = "secrets/" + key
         if description != None:
             self.description = description
         else:
@@ -71,21 +70,30 @@ class Secret:
         return None
 
     def print_info(self):
-        print("Secret:         {}".format(self.consul_key))
+        print("Secret:         {}".format(self.key))
-        print("Type:           {}".format(self.__class__.__name__))
+        print("Type:           {}".format(self.__class__.TYPE))
         if self.description != None:
             print("Description:    {}".format(self.description))
 
 class UserSecret(Secret):
+    TYPE = "user-entered secret"
+
     def __init__(self, example=None, multiline=False, **kwargs):
         Secret.__init__(self, **kwargs)
         self.example = example
         self.multiline = multiline
 
+    def print_info(self):
+        Secret.print_info(self)
+        if self.key in self.config.user_values:
+            print("Cluster value:  {}".format(self.config.user_values[self.key]))
+        elif self.example != None:
+            print("Example:        {}".format(self.example))
+
     def generate(self):
-        if self.key in self.config.constants:
+        if self.key in self.config.user_values:
             print("Using constant value from cluster's secretmgr.toml")
-            return self.config.constants[self.key]
+            return self.config.user_values[self.key]
 
         print("Enter value for secret, or ^C to skip:")
         if self.multiline:
@@ -112,11 +120,19 @@ class UserSecret(Secret):
                 return None
 
 class CommandSecret(Secret):
+    TYPE = "command"
+
     def __init__(self, command, rotate=False, **kwargs):
         Secret.__init__(self, **kwargs)
         self.command = command
         self.rotate_value = rotate
 
+    def print_info(self):
+        Secret.print_info(self)
+        print("Command:        {}".format(self.command))
+        if self.rotate_value:
+            print("Rotate:         True")
+
     def generate(self):
         print("Executing command:", self.command)
         return subprocess.check_output(["sh", "-c", self.command])
@@ -128,10 +144,16 @@ class CommandSecret(Secret):
             return None
 
 class ConstantSecret(Secret):
+    TYPE = "constant value"
+
     def __init__(self, value, **kwargs):
         Secret.__init__(self, **kwargs)
         self.value = value
 
+    def print_info(self):
+        Secret.print_info(self)
+        print("Value:          {}".format(self.value))
+
     def check(self, value):
         return value == self.value
 
@@ -142,10 +164,16 @@ class ConstantSecret(Secret):
 # ---- SERVICE USERS ----
 
 class ServiceUserPasswordSecret(Secret):
+    TYPE = "service user's password"
+
     def __init__(self, service_user, **kwargs):
         Secret.__init__(self, **kwargs)
         self.service_user = service_user
 
+    def print_info(self):
+        Secret.print_info(self)
+        print("Service user:   {}".format(self.service_user.username))
+
     def check(self, value):
         l = ldap.initialize(self.config.ldap_server)
         try:
@@ -160,6 +188,41 @@ class ServiceUserPasswordSecret(Secret):
     def rotate(self):
         return self.service_user.password
 
+class ServiceUserNameSecret(Secret):
+    TYPE = "service user's username (constant value)"
+
+    def __init__(self, service_user, **kwargs):
+        Secret.__init__(self, **kwargs)
+        self.service_user = service_user
+
+    def print_info(self):
+        Secret.print_info(self)
+        print("Value:          {}".format(self.service_user.username))
+
+    def check(self, value):
+        return value == self.service_user.username
+
+    def generate(self):
+        return self.service_user.username
+
+class ServiceUserDNSecret(Secret):
+    TYPE = "service user's DN (constant value)"
+
+    def __init__(self, service_user, **kwargs):
+        Secret.__init__(self, **kwargs)
+        self.service_user = service_user
+
+    def print_info(self):
+        Secret.print_info(self)
+        print("Service user:   {}".format(self.service_user.username))
+        print("Value:          {}".format(self.service_user.dn))
+
+    def check(self, value):
+        return value == self.service_user.dn
+
+    def generate(self):
+        return self.service_user.dn
+
 class ServiceUser:
     def __init__(self, username, password_secret, config, description=None, dn_secret=None, username_secret=None, rotate_password=False):
         self.config = config
@@ -169,30 +232,15 @@ class ServiceUser:
         self.dn = "cn={},{}".format(self.username, self.config.ldap_service_dn_suffix)
         self.rotate_password = rotate_password
 
-        self.password_secret = ServiceUserPasswordSecret(
+        self.password_secret = ServiceUserPasswordSecret(config=config, service_user=self, key=password_secret)
-                config=config,
-                service_user=self,
-                key=password_secret,
-                description="LDAP password for service user {}".format(username),
-                )
 
         self.username_secret = None
         if username_secret != None:
-            self.username_secret = ConstantSecret(
+            self.username_secret = ServiceUserNameSecret(config=config, service_user=self, key=username_secret)
-                    config=config,
-                    key=username_secret,
-                    value=username,
-                    description="LDAP username for service user {}".format(username),
-                    )
 
         self.dn_secret = None
         if dn_secret != None:
-            self.dn_secret = ConstantSecret(
+            self.dn_secret = ServiceUserDNSecret(config=config, service_user=self, key=dn_secret)
-                    config=config,
-                    key=dn_secret,
-                    value=self.dn,
-                    description="LDAP DN for service user {}".format(username),
-                )
 
     def secrets(self):
         secrets = {}
@@ -204,13 +252,12 @@ class ServiceUser:
         return secrets
 
     def configure(self, rotate):
-        _, data = consul_server.kv.get(self.password_secret.consul_key)
+        self.password = self.config.get_secret(self.password_secret.key)
-        if data is None:
+        if self.password is None:
             good = False
         else:
             l = ldap.initialize(self.config.ldap_server)
             try:
-                self.password = data["Value"].decode('ascii')
                 l.simple_bind_s(self.dn, self.password)
                 good = True
             except:
@@ -225,6 +272,9 @@ class ServiceUser:
             res = l.search_s(self.dn, ldap.SCOPE_BASE, "objectclass=*")
             if res is None or len(res) == 0:
                 print(bcolors.OKCYAN, "Creating entity", self.dn, bcolors.ENDC)
+                if self.config.dry_run:
+                    print(bcolors.OKBLUE, "Dry run, skipping. Add --do to actually do something.", bcolors.ENDC)
+                    return
                 l.add_s(self.dn,
                         [
                             ("objectclass",     [b"person", b"top"]),
@@ -233,6 +283,9 @@ class ServiceUser:
                         ])
             else:
                 print(bcolors.OKCYAN, "Resetting password for entity", self.dn, bcolors.ENDC)
+                if self.config.dry_run:
+                    print(bcolors.OKBLUE, "Dry run, skipping. Add --do to actually do something.", bcolors.ENDC)
+                    return
                 l.modify_s(self.dn,
                         [
                             (ldap.MOD_REPLACE, "userpassword", [pass_crypt])
@@ -244,13 +297,14 @@ class ServiceUser:
 # ---- MAIN CONFIG CLASS ----
 
 class Config:
-    def __init__(self, cluster_name):
+    def __init__(self, cluster_name, dry_run):
         self.cluster_name = cluster_name
         self.app_path = os.path.join(".", "cluster", cluster_name, "app")
 
         self.service_users = {}
         self.secrets = {}
         self.modules = []
+        self.dry_run = dry_run
 
         # Load config from secretmgr.toml in cluster directory
         secretmgr_toml_path = os.path.join(".", "cluster", cluster_name, "secretmgr.toml")
@@ -260,10 +314,10 @@ class Config:
         else:
             secretmgr_toml = {}
 
-        if "constants" in secretmgr_toml:
+        if "user_values" in secretmgr_toml:
-            self.constants = secretmgr_toml["constants"]
+            self.user_values = secretmgr_toml["user_values"]
         else:
-            self.constants = {}
+            self.user_values = {}
 
         self.ldap_server = None
         self.ldap_service_dn_suffix = None
@@ -309,28 +363,66 @@ class Config:
                     secret = CommandSecret(config=self, key=skey, **sargs)
                 elif ty == "constant":
                     secret = ConstantSecret(config=self, key=skey, **sargs)
+                elif ty == "service_password":
+                    service = sargs["service"]
+                    del sargs["service"]
+                    secret = ServiceUserPasswordSecret(
+                        config=self,
+                        key=skey,
+                        service_user=self.service_users[service],
+                        **sargs)
+                elif ty == "service_username":
+                    service = sargs["service"]
+                    del sargs["service"]
+                    secret = ServiceUserNameSecret(
+                        config=self,
+                        key=skey,
+                        service_user=self.service_users[service],
+                        **sargs)
+                elif ty == "service_dn":
+                    service = sargs["service"]
+                    del sargs["service"]
+                    secret = ServiceUserDNSecret(
+                        config=self,
+                        key=skey,
+                        service_user=self.service_users[service],
+                        **sargs)
                 else:
-                    raise Exception("Invalid secret type: {}".format(ty))
+                    description = "{}, {}".format(ty,
+                                ", ".join([k + ": " + v for k, v in sargs.items()]))
+                    secret = UserSecret(
+                        config=self,
+                        key=skey,
+                        multiline=True,
+                        description=description)
                 if skey in self.secrets:
                     raise Exception("Duplicate secret: {}".format(skey))
                 self.secrets[skey] = secret
 
-    def add_constant_secrets(self):
+    def add_user_values_secrets(self):
-        for (skey, value) in self.constants.items():
+        for (skey, value) in self.user_values.items():
             self.secrets[skey] = ConstantSecret(
                 config=self,
                 key=skey,
                 value=value,
-                description="Constant value for secret {}".format(skey))
+                description="Cluster-defined user value")
+
+    # -- consul and ldap helpers --
+
+    def check_consul_cluster(self):
+        # Check cluster name we are connected to
+        consul_node = consul_server.agent.self()
+        if consul_node["Config"]["Datacenter"] != self.cluster_name:
+            print("You are not connected to the correct Consul cluster.")
+            print("You are connected to cluster '{}' instead of '{}'.".format(consul_node["Config"]["Datacenter"], self.cluster_name))
+            sys.exit(1)
 
     def get_ldap_admin_conn(self):
         if self.ldap_admin_conn is None:
             if self.ldap_admin_password_secret != None:
-                pass_key = "secrets/" + self.ldap_admin_password_secret
+                ldap_pass = self.get_secret(self.ldap_admin_password_secret)
-                _, data = consul_server.kv.get(pass_key)
+                if ldap_pass is None:
-                if data is None:
                     raise Exception("LDAP admin password could not be read at: {}".format(pass_key))
-                ldap_pass = data["Value"].decode('ascii').strip()
             else:
                 ldap_pass = getpass.getpass("LDAP admin password: ")
 
@@ -338,25 +430,57 @@ class Config:
             self.ldap_admin_conn.simple_bind_s(self.ldap_admin_dn, ldap_pass)
         return self.ldap_admin_conn
 
+    def get_secret(self, key):
+        _, data = consul_server.kv.get("secrets/" + key)
+        if data is None:
+            return None
+        else:
+            return data["Value"].decode('ascii').strip()
+
+    def put_secret(self, key, value):
+        if self.dry_run:
+            print(bcolors.OKBLUE, "Dry run, not updating secrets/{}. Add --do to actually do something.".format(key), bcolors.ENDC)
+            return
+        consul_server.kv.put("secrets/" + key, value)
+
+    # -- user actions --
+
+    def print_info(self):
+        print("== LIST OF SERVICE USERS ==")
+        print()
+        for (_, su) in self.service_users.items():
+            print("Username:      {}".format(su.username))
+            print("DN:            {}".format(su.dn))
+            print("Pass. secret:  {}".format(su.password_secret.key))
+            print()
+
+        print("== LIST OF SECRETS ==")
+        print()
+        for (_, secret) in self.secrets.items():
+            secret.print_info()
+            print()
+
     def check_secrets(self):
+        self.check_consul_cluster()
         print(":: Checking secrets...")
         must_gen = False
         for (_, secret) in self.secrets.items():
-            _, data = consul_server.kv.get(secret.consul_key)
+            value = self.get_secret(secret.key)
-            if data is None:
+            if value is None:
-                print(secret.consul_key, bcolors.FAIL, "x  missing", bcolors.ENDC)
+                print(secret.key, bcolors.FAIL, "x  missing", bcolors.ENDC)
                 must_gen = True
-            elif not secret.check(data["Value"].decode('ascii').strip()): 
+            elif not secret.check(value): 
-                print(secret.consul_key, bcolors.WARNING, "x  bad value", bcolors.ENDC)
+                print(secret.key, bcolors.WARNING, "x  bad value", bcolors.ENDC)
                 must_gen = True
             else:
-                print(secret.consul_key, bcolors.OKGREEN, "✓", bcolors.ENDC)
+                print(secret.key, bcolors.OKGREEN, "✓", bcolors.ENDC)
         print()
         if must_gen:
             print("To fix missing or invalid secrets, use `secretmgr gen <cluster_name> <app>...`")
             print()
 
     def gen_secrets(self):
+        self.check_consul_cluster()
         if len(self.service_users) > 0:
             print(":: Configuring service users...")
             for (_, su) in self.service_users.items():
@@ -365,13 +489,13 @@ class Config:
 
         print(":: Generating missing/invalid secrets...")
         for (_, secret) in self.secrets.items():
-            _, data = consul_server.kv.get(secret.consul_key)
+            old_value = self.get_secret(secret.key)
-            if data is None or not secret.check(data["Value"].decode('ascii').strip()):
+            if old_value is None or not secret.check(old_value):
                 print()
                 secret.print_info()
                 value = secret.generate()
                 if value != None:
-                    consul_server.kv.put(secret.consul_key, value)
+                    self.put_secret(secret.key, value)
                     print(bcolors.OKCYAN, "Value set.", bcolors.ENDC)
                 else:
                     print(bcolors.WARNING, "Skipped.", bcolors.ENDC)
@@ -380,6 +504,7 @@ class Config:
         self.check_secrets()
 
     def rotate_secrets(self):
+        self.check_consul_cluster()
         if len(self.service_users) > 0:
             print(":: Regenerating service user passwords...")
             for (_, su) in self.service_users.items():
@@ -391,15 +516,11 @@ class Config:
             print()
             secret.print_info()
 
-            _, data = consul_server.kv.get(secret.consul_key)
+            old_value = self.get_secret(secret.key)
-            if data is None:
-                old_value = None
-            else:
-                old_value = data["Value"].decode('ascii').strip()
             new_value = secret.rotate()
 
             if new_value != None and new_value != old_value:
-                consul_server.kv.put(secret.consul_key, new_value)
+                self.put_secret(secret.key, new_value)
                 print(bcolors.OKCYAN, "Value set.", bcolors.ENDC)
             else:
                 print(bcolors.OKGREEN, "Nothing to do.", bcolors.ENDC)
@@ -410,20 +531,28 @@ class Config:
 
 # ---- MAIN ----
 
-def load_config(cluster_name, modules):
+def load_config(cluster_name, modules, **kwargs):
-    cfg = Config(cluster_name)
+    # Load config
+    cfg = Config(cluster_name, **kwargs)
     if len(modules) > 0:
         for mod in modules:
             cfg.load_module(mod)
     else:
-        cfg.add_constant_secrets()
+        cfg.add_user_values_secrets()
+
     return cfg
 
 if __name__ == "__main__":
     verb = None
+    dry_run = True
 
     for i, val in enumerate(sys.argv):
-        if val == "check":
+        if val == "--do":
+            dry_run = False
+        elif val == "info":
+            verb = lambda cfg: cfg.print_info()
+            break
+        elif val == "check":
             verb = lambda cfg: cfg.check_secrets()
             break
         elif val == "gen":
@@ -435,9 +564,12 @@ if __name__ == "__main__":
 
     if verb is None:
         print("Usage:")
-        print("    secretmgr.py [check|gen|rotate] <cluster name> <module name>...")
+        print("    secretmgr [--do] info|check|gen|rotate <cluster name> [<module name>...]")
     else:
-        cfg = load_config(sys.argv[i+1], sys.argv[i+2:])
+        cfg = load_config(
+                cluster_name=sys.argv[i+1],
+                modules=sys.argv[i+2:],
+                dry_run=dry_run)
         verb(cfg)