New secretmgr

This commit is contained in:
Alex 2022-12-25 21:03:16 +01:00
parent 7fd81f3470
commit 8d0a7a806d
No known key found for this signature in database
GPG Key ID: 09EC5284AA804D3C
36 changed files with 812 additions and 480 deletions

View File

@ -0,0 +1 @@
../../../secretmgr/convertsecrets

View File

@ -0,0 +1,4 @@
[secrets."d53/gandi_api_key"]
type = 'user'
description = 'Gandi API key'

View File

@ -1 +0,0 @@
USER Gandi API key

View File

@ -0,0 +1,48 @@
[secrets."directory/ldap_base_dn"]
type = 'user'
description = 'LDAP base DN for everything (e.g. dc=example,dc=com)'
[secrets."directory/guichet/smtp_user"]
type = 'user'
description = 'SMTP username'
[secrets."directory/guichet/s3_access_key"]
type = 'user'
description = 'Garage access key for Guichet profile pictures'
[secrets."directory/guichet/s3_endpoint"]
type = 'user'
description = 'S3 endpoint URL'
[secrets."directory/guichet/s3_region"]
type = 'user'
description = 'S3 region'
[secrets."directory/guichet/smtp_pass"]
type = 'user'
description = 'SMTP password'
[secrets."directory/guichet/web_hostname"]
type = 'user'
description = 'Public hostname from which Guichet is accessible via HTTP (e.g. guichet.example.com)'
[secrets."directory/guichet/s3_bucket"]
type = 'user'
description = 'S3 bucket in which to store data files (such as profile pictures)'
[secrets."directory/guichet/smtp_server"]
type = 'user'
description = 'SMTP server address (hostname:port)'
[secrets."directory/guichet/s3_secret_key"]
type = 'user'
description = 'Garage secret key for Guichet profile pictures'
[secrets."directory/guichet/mail_from"]
type = 'user'
description = 'E-mail address from which to send welcome emails to new users'
[secrets."directory/guichet/mail_domain"]
type = 'user'
description = 'E-mail domain for new users (e.g. example.com)'

View File

@ -1 +0,0 @@
USER E-mail domain for new users (e.g. example.com)

View File

@ -1 +0,0 @@
USER E-mail address from which to send welcome emails to new users

View File

@ -1 +0,0 @@
USER Garage access key for Guichet profile pictures

View File

@ -1 +0,0 @@
USER S3 bucket in which to store data files (such as profile pictures)

View File

@ -1 +0,0 @@
USER S3 endpoint URL

View File

@ -1 +0,0 @@
USER Garage secret key for Guichet profile pictures

View File

@ -1 +0,0 @@
USER SMTP password

View File

@ -1 +0,0 @@
USER SMTP server address (hostname:port)

View File

@ -1 +0,0 @@
USER SMTP username

View File

@ -1 +0,0 @@
USER Public hostname from which Guichet is accessible via HTTP (e.g. guichet.example.com)

View File

@ -1 +0,0 @@
USER LDAP base DN for everything (e.g. dc=example,dc=com)

View File

@ -0,0 +1,36 @@
[service_users."dummy"]
description = 'Service user for dummy database access'
username_secret = "dummy/db_username"
password_secret = "dummy/db_password"
rotate_password = true
[secrets."dummy/s3_access_key"]
type = 'user'
description = 'S3 access key ID for database storage'
[secrets."dummy/s3_secret_key"]
description = 'S3 secret key for database storage'
type = 'user'
[secrets."dummy/public_domain"]
description = 'Publicly accessible domain for dummy resource'
type = 'user'
[secrets."dummy/form_secret"]
description = 'Form secret for dummy web pages'
type = 'command'
command = 'openssl rand -base64 42'
rotate = true
[secrets."dummy/signing_key"]
description = 'Key to use to sign dummy service data'
type = 'command'
command = 'openssl rand -base64 42'
[secrets."dummy/rpc_secret"]
description = 'RPC secret for communication between dummy nodes'
type = 'command'
command = 'openssl rand -base64 42'
rotate = true

View File

@ -0,0 +1,15 @@
[secrets."garage-staging/admin_token"]
type = 'command'
rotate = true
command = 'openssl rand -hex 32'
[secrets."garage-staging/rpc_secret"]
type = 'command'
rotate = true
command = 'openssl rand -hex 32'
[secrets."garage-staging/metrics_token"]
type = 'command'
rotate = true
command = 'openssl rand -hex 32'

View File

@ -1 +0,0 @@
CMD openssl rand -hex 32

View File

@ -1 +0,0 @@
CMD openssl rand -hex 32

View File

@ -1 +0,0 @@
CMD openssl rand -hex 32

View File

@ -0,0 +1,27 @@
[secrets."synapse/s3_access_key"]
type = 'user'
description = 'S3 access key ID for database storage'
[secrets."synapse/form_secret"]
type = 'command'
rotate = true
command = 'openssl rand -base64 42'
[secrets."synapse/signing_key"]
type = 'user'
description = 'Signing key for messages'
[secrets."synapse/macaroon_secret_key"]
type = 'command'
rotate = true
command = 'openssl rand -base64 42'
[secrets."synapse/registration_shared_secret"]
type = 'command'
rotate = true
command = 'openssl rand -base64 42'
[secrets."synapse/s3_secret_key"]
type = 'user'
description = 'S3 secret key for database storage'

View File

@ -1 +0,0 @@
CMD openssl rand -base64 42

View File

@ -1 +0,0 @@
CMD openssl rand -base64 42

View File

@ -1 +0,0 @@
CMD openssl rand -base64 42

View File

@ -1 +0,0 @@
USER S3 access key ID for database storage

View File

@ -1 +0,0 @@
USER S3 secret key for database storage

View File

@ -1 +0,0 @@
USER Signing key for messages

View File

@ -0,0 +1,13 @@
[secrets."telemetry/grafana/s3_access_key"]
type = 'user'
description = 'S3 access key for grafana db'
[secrets."telemetry/grafana/admin_password"]
type = 'command'
rotate = true
command = 'openssl rand -base64 12'
[secrets."telemetry/grafana/s3_secret_key"]
type = 'user'
description = 'S3 secret key for grafana db'

View File

@ -1 +0,0 @@
CMD openssl rand -base64 12

View File

@ -1 +0,0 @@
USER S3 access key for grafana db

View File

@ -1 +0,0 @@
USER S3 secret key for grafana db

View File

@ -0,0 +1,9 @@
[ldap]
server = "ldap://localhost:1389"
service_dn_suffix = "ou=services,ou=users,dc=staging,dc=deuxfleurs,dc=org"
admin_dn = "cn=admin,dc=staging,dc=deuxfleurs,dc=org"
admin_password_secret = "directory/admin_password"
[constants]
"dummy/public_domain" = "dummy.staging.deuxfleurs.org"
"dummy/test_constant" = "test value"

215
convertsecrets Executable file
View File

@ -0,0 +1,215 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.pip ps.consul ps.ldap ps.passlib ps.requests ps.six ])"
# DEPENDENCY: python-consul
import consul
# DEPENDENCY: python-ldap
import ldap
# DEPENDENCY: passlib
from passlib.hash import ldap_salted_sha1
import os
import sys
import glob
import subprocess
import getpass
import base64
from secrets import token_bytes
"""
TODO: this will be a utility to handle secrets in the Consul database
for the various components of the Deuxfleurs infrastructure
Functionnalities:
- check that secrets are correctly configured
- help user fill in secrets
- create LDAP service users and fill in corresponding secrets
- maybe one day: manage SSL certificates and keys
It uses files placed in <module_name>/secrets/* to know what secrets
it should handle. These secret files contain directives for what to do
about these secrets.
Example directives:
USER <description>
(a secret that must be filled in by the user)
USER_LONG <description>
(the same, indicates that the secret fits on several lines)
CMD <command>
(a secret that is generated by running this command)
CMD_ONCE <command>
(same, but value is not changed when doing a regen)
CONST <constant value>
(the secret has a constant value set here)
CONST_LONG
<constant value, several lines>
(same)
SERVICE_DN <service name> <service description>
(the LDAP DN of a service user)
SERVICE_PASSWORD <service name>
(the LDAP password for the corresponding service user)
SSL_CERT <cert name> <list of domains>
(a SSL domain for the given domains)
SSL_KEY <cert name>
(the SSL key going with corresponding certificate)
RSA_PUBLIC_KEY <key name> <key description>
(a public RSA key)
RSA_PRIVATE_KEY <key name>
(the corresponding private RSA key)
"""
# Parameters
LDAP_URL = "ldap://localhost:1389"
SERVICE_DN_SUFFIX = "ou=services,ou=users,dc=deuxfleurs,dc=fr"
consul_server = consul.Consul()
# ----
USER = "USER"
USER_LONG = "USER_LONG"
CMD = "CMD"
CMD_ONCE = "CMD_ONCE"
CONST = "CONST"
CONST_LONG = "CONST_LONG"
SERVICE_DN = "SERVICE_DN"
SERVICE_PASSWORD = "SERVICE_PASSWORD"
SSL_CERT = "SSL_CERT"
SSL_KEY = "SSL_KEY"
RSA_PUBLIC_KEY = "RSA_PUBLIC_KEY"
RSA_PRIVATE_KEY = "RSA_PRIVATE_KEY"
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKCYAN = '\033[96m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
def read_secret(key, file_path):
lines = [l.strip() for l in open(file_path, "r")]
if len(lines) == 0:
print(bcolors.FAIL, "ERROR:", bcolors.ENDC, "Empty file in", file_path)
sys.exit(-1)
l0 = lines[0].split(" ")
stype = l0[0]
secret = {"type": stype, "key": key}
if stype in [USER, USER_LONG]:
secret["desc"] = " ".join(l0[1:])
elif stype in [CMD, CMD_ONCE]:
secret["cmd"] = " ".join(l0[1:])
elif stype == CONST:
secret["value"] = " ".join(l0[1:])
elif stype == CONST_LONG:
secret["value"] = "\n".join(lines[1:])
elif stype in [SERVICE_DN, SERVICE_PASSWORD]:
secret["service"] = l0[1]
if stype == SERVICE_DN:
secret["service_desc"] = " ".join(l0[2:])
elif stype in [SSL_CERT, SSL_KEY]:
secret["cert_name"] = l0[1]
if stype == SSL_CERT:
secret["cert_domains"] = l0[2:]
elif stype in [RSA_PUBLIC_KEY, RSA_PRIVATE_KEY]:
secret["key_name"] = l0[1]
if stype == RSA_PUBLIC_KEY:
secret["key_desc"] = " ".join(l0[2:])
else:
print(bcolors.FAIL, "ERROR:", bcolors.ENDC, "Invalid secret type", stype, "in", file_path)
sys.exit(-1)
return secret
def read_secrets(module_list):
secrets = {}
for mod in module_list:
for file_path in glob.glob(mod.strip('/') + "/secrets/**", recursive=True):
if os.path.isfile(file_path):
key = '/'.join(file_path.split("/")[1:])
secrets[key] = read_secret(key, file_path)
return secrets
def convert_secrets(module_list):
for mod in module_list:
print("converting module: ", mod)
secrets = read_secrets([mod])
with open(os.path.join(mod.strip('/'), "secrets.toml"), "w") as file:
for (secret_key, secret) in secrets.items():
file.write("[secrets.\"{}\"]\n".format("/".join(secret_key.split("/")[1:])))
ty = secret["type"]
if ty in [CMD, CMD_ONCE]:
newsecret = {
"type": "command",
"rotate": ty != "CMD_ONCE",
"command": secret["cmd"],
}
elif ty in [USER, USER_LONG]:
newsecret = {
"type": "user",
"multiline": ty == "USER_LONG",
"description": secret["desc"],
}
elif ty in [CONST, CONST_LONG]:
newsecret = {
"type": "constant",
"value": secret["value"],
}
else:
newsecret = {
"type": secret["type"],
}
if "service_desc" in secret:
newsecret["description"] = secret["service_desc"]
if "key_desc" in secret:
newsecret["description"] = secret["key_desc"]
if "cert_name" in secret:
newsecret["name"] = secret["cert_name"]
if "key_name" in secret:
newsecret["name"] = secret["key_name"]
for k in ["service", "cert_domains"]:
if k in secret:
newsecret[k] = secret[k]
for (k, v) in newsecret.items():
if type(v) == bool:
if v:
file.write("{} = true\n".format(k))
elif type(v) == str:
file.write("{} = {}\n".format(k, repr(v)))
else:
print("invalid value: ", v)
file.write("\n")
# ---- MAIN ----
if __name__ == "__main__":
for i, val in enumerate(sys.argv):
if val == "convert":
convert_secrets(sys.argv[i+1:])
break
else:
print("Usage:")
print(" convertsecrets convert <module name>...")
# vim: set sts=4 ts=4 sw=4 tw=0 ft=python et :

View File

@ -1,455 +0,0 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.pip ps.consul ps.ldap ps.passlib ps.requests ps.six ps.toml ])"
# DEPENDENCY: python-consul
import consul
# DEPENDENCY: python-ldap
import ldap
# DEPENDENCY: passlib
from passlib.hash import ldap_salted_sha1
# DEPENDENCY: toml
import toml
import os
import sys
import glob
import subprocess
import getpass
import base64
from secrets import token_bytes
"""
TODO: this will be a utility to handle secrets in the Consul database
for the various components of the Deuxfleurs infrastructure
Functionnalities:
- check that secrets are correctly configured
- help user fill in secrets
- create LDAP service users and fill in corresponding secrets
- maybe one day: manage SSL certificates and keys
It uses files placed in <module_name>/secrets/* to know what secrets
it should handle. These secret files contain directives for what to do
about these secrets.
Example directives:
USER <description>
(a secret that must be filled in by the user)
USER_LONG <description>
(the same, indicates that the secret fits on several lines)
CMD <command>
(a secret that is generated by running this command)
CMD_ONCE <command>
(same, but value is not changed when doing a regen)
CONST <constant value>
(the secret has a constant value set here)
CONST_LONG
<constant value, several lines>
(same)
SERVICE_DN <service name> <service description>
(the LDAP DN of a service user)
SERVICE_PASSWORD <service name>
(the LDAP password for the corresponding service user)
SSL_CERT <cert name> <list of domains>
(a SSL domain for the given domains)
SSL_KEY <cert name>
(the SSL key going with corresponding certificate)
RSA_PUBLIC_KEY <key name> <key description>
(a public RSA key)
RSA_PRIVATE_KEY <key name>
(the corresponding private RSA key)
"""
# Parameters
LDAP_URL = "ldap://localhost:1389"
SERVICE_DN_SUFFIX = "ou=services,ou=users,dc=deuxfleurs,dc=fr"
consul_server = consul.Consul()
# ---- UTIL ----
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKCYAN = '\033[96m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
ldap_admin_conn = None
def get_ldap_admin_conn():
global ldap_admin_conn
if ldap_admin_conn is None:
ldap_admin_conn = ldap.initialize(LDAP_URL)
ldap_user = input("LDAP admin user (full DN, please!): ")
ldap_pass = getpass.getpass("LDAP admin password: ")
ldap_admin_conn.simple_bind_s(ldap_user, ldap_pass)
return ldap_admin_conn
# ---- SECRETS ----
class Secret:
def __init__(self, key, description=None):
self.consul_key = "secrets/" + key
if description != None:
self.description = description
def check(self, value):
return True
def generate(self, old_value):
pass
class UserSecret(Secret):
def __init__(self, example=None, multiline=False, **kwargs):
Secret.__init__(self, **kwargs)
self.example = example
self.multiline = multiline
def generate(self, old_value):
print("Description:", self.description)
print("Enter value for secret, or ^C to skip:")
if self.multiline:
print("THIS IS A LONG VALUE, ENTER SEVERAL LINES AND FINISH WITH A LINE CONTAINING A SINGLE .")
try:
lines = []
while True:
line = input().strip()
if line == ".":
break
lines.append(line)
return "\n".join(lines)
except KeyboardInterrupt:
return None
else:
try:
while True:
line = input().strip()
if line != "":
return line
else:
print("Please enter a non-empty value, or ^C to skip:")
except KeyboardInterrupt:
return None
class CommandSecret(Secret):
def __init__(self, command, rotate=False, **kwargs):
Secret.__init__(self, **kwargs)
self.command = command
self.rotate = rotate
def generate(self, old_value):
if not self.rotate and old_value is not None:
return None
print("Executing command:", self.command)
return subprocess.check_output(["sh", "-c", self.command])
class ConstantSecret(Secret):
def __init__(self, value, **kwargs):
Secret.__init__(self, **kwargs)
self.value = value
def check(self, value):
return value == self.value
def generate(self, old_value):
if old_value != self.value:
return self.value
else:
return None
# ---- SERVICE USERS ----
class ServiceUserPasswordSecret:
def __init__(self, service_user, **kwargs):
Secret.__init__(self, **kwargs)
self.service_user = service_user
def check(self, value):
l = ldap.initialize(LDAP_URL)
try:
l.simple_bind_s(self.service_user.dn, value)
return True
except Exception as e:
return False
def generate(self, old_value):
if old_value != self.service_user.password:
return self.service_user.password
else:
return None
class ServiceUser:
def __init__(self, username, password_secret, user_dn_secret=None, user_name_secret=None, rotate_password=False):
self.username = username
self.password = None
self.dn = "cn={},{}".format(self.username, SERVICE_DN_SUFFIX)
self.rotate_password = rotate_password
self.username_secret = username_secret
self.password_secret = password_secret
self.dn_secret = dn_secret
def secrets(self):
secrets = {}
secrets[self.password_secret] = ServiceUserPasswordSecret(
service_user=self,
key=self.password_secret,
description="LDAP password for service user {}".format(self.username),
)
if self.user_dn_secret != None:
secrets[self.dn_secret] = ConstantSecret(
key=self.dn_secret,
value=self.dn,
description="LDAP DN for service user {}".format(self.username),
)
if self.username_secret != None:
secrets[self.username_secret] = ConstantSecret(
key=self.username_secret,
value=self.username,
description="LDAP username for service user {}".format(self.username),
)
return secrets
# ---- READING secrets.toml FILES ----
class Config:
def __init__(self, cluster_name):
self.cluster_name = cluster_name
self.app_path = os.path.join(".", "cluster", cluster_name)
self.service_users = {}
self.secrets = {}
self.modules = []
def load_module(self, module_name)
secrets_toml_path = os.path.join(self.app_path, module_name, "secrets.toml")
with open(secrets_toml_path) as f:
secrets_toml = toml.load(f)
self.modules.append(module_name)
# Service users, and their associated secrets
if "service_users" in secrets_toml:
for (uname, uargs) in secrets_toml["service_users"].items():
service_user = ServiceUser(uname, **uargs)
for (skey, secret) in service_user.secrets():
if skey in self.secrets:
raise Exception("Duplicate secret: {}".format(skey))
self.secrets[skey] = secret
self.service_users[uname] = service_user
# Other secrets
if "secrets" in secrets_toml:
for (skey, sargs) in secrets_toml["secrets"].items():
ty = sargs["type"]
del sargs["type"]
if ty == "user":
secret = UserSecret(key=skey, **sargs)
elif ty == "command":
secret = CommandSecret(key=skey, **sargs)
elif ty == "constant":
secret = Constantsecret(key=skey, **sargs)
else:
raise Exception("Invalid secret type: {}".format(ty))
if skey in self.secrets:
raise Exception("Duplicate secret: {}".format(skey))
self.secrets[skey] = secret
def check_secrets(self):
print(":: Checking secrets...")
for (_, secret) in self.secrets.items():
_, data = consul_server.kv.get(secret.consul_key)
if data is None:
print(secret.consul_key, bcolors.FAIL, "x", bcolors.ENDC, " missing")
elif not secret.check(data["Value"].decode('ascii').strip()):
print(secret.consul_key, bcolors.FAIL, "x", bcolors.ENDC, " wrong value")
else:
print(secret.consul_key, bcolors.OKGREEN, "✓", bcolors.ENDC)
print()
def gen_secrets(self):
print(":: Generating missing secrets...")
for (_, secret) in self.secrets.items():
_, data = consul_server.kv.get(secret.consul_key)
if data is None:
value = secret.generate(None)
if value != None:
print("PUT {} = {}".format(secret.consul_key, value))
print(bcolors.OKCYAN, "Value set.", bcolors.ENDC)
else:
print(bcolors.WARNING, "Skipped.", bcolors.ENDC)
print()
def rotate_secrets(self):
print(":: Rotating secrets...")
for (_, secret) in self.secrets.items():
print(": {} ({})".format(secret.consul_key, secret.__class__.__name__))
_, data = consul_server.kv.get(secret.consul_key)
if data is None:
value = secret.generate(None)
else:
value = secret.generate(data["Value"].decode('ascii').strip())
if value != None:
print("PUT {} = {}".format(secret.consul_key, value))
print(bcolors.OKCYAN, "Value set.", bcolors.ENDC)
else:
print(bcolors.WARNING, "Skipped.", bcolors.ENDC)
print()
def check_secrets(modules):
for mod in modules:
print("==== MODULE `{}` ====".format(mod))
modcfg = Config(mod)
modcfg.check_secrets()
print("")
def gen_secrets(modules):
for mod in modules:
print("==== MODULE `{}` ====".format(mod))
modcfg = Config(mod)
modcfg.gen_secrets()
modcfg.check_secrets()
print("")
def rotate_secrets(modules):
for mod in modules:
print("==== MODULE `{}` ====".format(mod))
modcfg = Config(mod)
modcfg.rotate_secrets()
modcfg.check_secrets()
print("")
# ----- old ----
# ---- CHECK COMMAND ----
def check_secrets_services(secrets):
print("Checking secrets for LDAP service users...")
services = get_secrets_services(secrets)
for svc_name, svc in services.items():
for dn_key in svc["dn_at"]:
_, data = consul_server.kv.get(dn_key)
if data is not None:
got_val = data["Value"].decode('ascii').strip()
if got_val != svc["dn"]:
print(svc_name, "wrong DN at", dn_key, bcolors.FAIL, "x", bcolors.ENDC)
print("got:", got_val, "instead of:", svc["dn"])
if svc["pass"] is None:
print(svc_name, bcolors.FAIL, "no password stored", bcolors.ENDC)
else:
for pass_key in svc["pass_at"]:
_, data = consul_server.kv.get(pass_key)
if data is not None:
got_val = data["Value"].decode('ascii').strip()
if got_val != svc["pass"]:
print(svc_name, "wrong pass at", dn_key, bcolors.FAIL, "x", bcolors.ENDC)
l = ldap.initialize(LDAP_URL)
try:
l.simple_bind_s(svc["dn"], svc["pass"])
print(svc_name, bcolors.OKGREEN, "✓", bcolors.ENDC)
except Exception as e:
print(svc_name, bcolors.FAIL, e, bcolors.ENDC)
print()
# ---- GEN COMMAND ----
def gen_secrets_services(secrets, regen):
print("Generating LDAP service accounts...")
services = get_secrets_services(secrets)
for svc_name, svc in services.items():
print("----")
print("Service:", svc_name)
print("Description:", svc["desc"])
for dn_key in svc["dn_at"]:
_, data = consul_server.kv.get(dn_key)
if data is None or data["Value"].decode('ascii').strip() != svc["dn"]:
print(bcolors.OKCYAN, "Setting DN", bcolors.ENDC, "at", dn_key)
consul_server.kv.put(dn_key, svc["dn"])
if svc["pass"] is None or regen:
print(bcolors.OKCYAN, "Generating new password", bcolors.ENDC)
svc["pass"] = base64.urlsafe_b64encode(token_bytes(12)).decode('ascii')
l = ldap.initialize(LDAP_URL)
try:
l.simple_bind_s(svc["dn"], svc["pass"])
except:
fix_service_user(svc)
for pass_key in svc["pass_at"]:
_, data = consul_server.kv.get(pass_key)
if data is None or data["Value"].decode('ascii').strip() != svc["pass"]:
print(bcolors.OKCYAN, "Setting password", bcolors.ENDC, "at", pass_key)
consul_server.kv.put(pass_key, svc["pass"])
print()
def fix_service_user(svc):
print("Fixing service user", svc["dn"], "...")
l = get_ldap_admin_conn()
res = l.search_s(svc["dn"], ldap.SCOPE_BASE, "objectclass=*")
pass_crypt = ldap_salted_sha1.hash(svc["pass"])
if res is None or len(res) == 0:
print(bcolors.OKCYAN, "Creating entity...", bcolors.ENDC)
l.add_s(svc["dn"],
[
("objectclass", [b"person", b"top"]),
("displayname", [svc["desc"].encode('ascii')]),
("userpassword", [pass_crypt.encode('ascii')]),
])
else:
print(bcolors.OKCYAN, "Resetting entity password", bcolors.ENDC)
l.modify_s(svc["dn"],
[
(ldap.MOD_REPLACE, "userpassword", [pass_crypt.encode('ascii')])
])
# ---- MAIN ----
if __name__ == "__main__":
for i, val in enumerate(sys.argv):
if val == "check":
check_secrets(sys.argv[i+1:])
break
elif val == "gen":
gen_secrets(sys.argv[i+1:])
break
elif val == "rotate":
rotate_secrets(sys.argv[i+1:])
break
else:
print("Usage:")
print(" secretmgr.py [check|gen|rotate] <module name>...")
# vim: set sts=4 ts=4 sw=4 tw=0 ft=python et :

444
secretmgr Executable file
View File

@ -0,0 +1,444 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.pip ps.consul ps.ldap ps.passlib ps.requests ps.six ps.toml ])"
# DEPENDENCY: python-consul
import consul
# DEPENDENCY: python-ldap
import ldap
# DEPENDENCY: passlib
from passlib.hash import ldap_salted_sha1
# DEPENDENCY: toml
import toml
import os
import sys
import glob
import subprocess
import getpass
import base64
from secrets import token_bytes
"""
This is a utility to handle secrets in the Consul database
for the various components of the Deuxfleurs infrastructure
Functionnalities:
- check that secrets are correctly configured
- help user fill in secrets
- create LDAP service users and fill in corresponding secrets
- TODO: manage Garage buckets and access keys
- maybe one day: manage SSL certificates and keys
"""
# ---- UTIL ----
consul_server = consul.Consul()
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKCYAN = '\033[96m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
# ---- SECRETS ----
class Secret:
def __init__(self, key, config, description=None):
self.config = config
self.key = key
self.consul_key = "secrets/" + key
if description != None:
self.description = description
else:
self.description = None
def check(self, value):
return True
def generate(self):
pass
def rotate(self):
return None
def print_info(self):
print("Secret: {}".format(self.consul_key))
print("Type: {}".format(self.__class__.__name__))
if self.description != None:
print("Description: {}".format(self.description))
class UserSecret(Secret):
def __init__(self, example=None, multiline=False, **kwargs):
Secret.__init__(self, **kwargs)
self.example = example
self.multiline = multiline
def generate(self):
if self.key in self.config.constants:
print("Using constant value from cluster's secretmgr.toml")
return self.config.constants[self.key]
print("Enter value for secret, or ^C to skip:")
if self.multiline:
print("THIS IS A LONG VALUE, ENTER SEVERAL LINES AND FINISH WITH A LINE CONTAINING A SINGLE .")
try:
lines = []
while True:
line = input().strip()
if line == ".":
break
lines.append(line)
return "\n".join(lines)
except KeyboardInterrupt:
return None
else:
try:
while True:
line = input().strip()
if line != "":
return line
else:
print("Please enter a non-empty value, or ^C to skip:")
except KeyboardInterrupt:
return None
class CommandSecret(Secret):
def __init__(self, command, rotate=False, **kwargs):
Secret.__init__(self, **kwargs)
self.command = command
self.rotate_value = rotate
def generate(self):
print("Executing command:", self.command)
return subprocess.check_output(["sh", "-c", self.command])
def rotate(self):
if self.rotate_value:
return self.generate()
else:
return None
class ConstantSecret(Secret):
def __init__(self, value, **kwargs):
Secret.__init__(self, **kwargs)
self.value = value
def check(self, value):
return value == self.value
def generate(self):
return self.value
# ---- SERVICE USERS ----
class ServiceUserPasswordSecret(Secret):
def __init__(self, service_user, **kwargs):
Secret.__init__(self, **kwargs)
self.service_user = service_user
def check(self, value):
l = ldap.initialize(self.config.ldap_server)
try:
l.simple_bind_s(self.service_user.dn, value)
return True
except Exception as e:
return False
def generate(self):
return self.service_user.password
def rotate(self):
return self.service_user.password
class ServiceUser:
def __init__(self, username, password_secret, config, description=None, dn_secret=None, username_secret=None, rotate_password=False):
self.config = config
self.username = username
self.description = description
self.password = None
self.dn = "cn={},{}".format(self.username, self.config.ldap_service_dn_suffix)
self.rotate_password = rotate_password
self.password_secret = ServiceUserPasswordSecret(
config=config,
service_user=self,
key=password_secret,
description="LDAP password for service user {}".format(username),
)
self.username_secret = None
if username_secret != None:
self.username_secret = ConstantSecret(
config=config,
key=username_secret,
value=username,
description="LDAP username for service user {}".format(username),
)
self.dn_secret = None
if dn_secret != None:
self.dn_secret = ConstantSecret(
config=config,
key=dn_secret,
value=self.dn,
description="LDAP DN for service user {}".format(username),
)
def secrets(self):
secrets = {}
secrets[self.password_secret.key] = self.password_secret
if self.dn_secret != None:
secrets[self.dn_secret.key] = self.dn_secret
if self.username_secret != None:
secrets[self.username_secret.key] = self.username_secret
return secrets
def configure(self, rotate):
_, data = consul_server.kv.get(self.password_secret.consul_key)
if data is None:
good = False
else:
l = ldap.initialize(self.config.ldap_server)
try:
self.password = data["Value"].decode('ascii')
l.simple_bind_s(self.dn, self.password)
good = True
except:
good = False
if not good or (rotate and self.rotate_password):
# Reset passsword
self.password = base64.urlsafe_b64encode(token_bytes(12)).decode('ascii')
pass_crypt = ldap_salted_sha1.hash(self.password).encode('ascii')
l = self.config.get_ldap_admin_conn()
res = l.search_s(self.dn, ldap.SCOPE_BASE, "objectclass=*")
if res is None or len(res) == 0:
print(bcolors.OKCYAN, "Creating entity", self.dn, bcolors.ENDC)
l.add_s(self.dn,
[
("objectclass", [b"person", b"top"]),
("displayname", [self.description.encode('ascii')]),
("userpassword", [pass_crypt]),
])
else:
print(bcolors.OKCYAN, "Resetting password for entity", self.dn, bcolors.ENDC)
l.modify_s(self.dn,
[
(ldap.MOD_REPLACE, "userpassword", [pass_crypt])
])
else:
print(bcolors.OKGREEN, "Entity is good: ", self.dn, bcolors.ENDC)
# ---- MAIN CONFIG CLASS ----
class Config:
def __init__(self, cluster_name):
self.cluster_name = cluster_name
self.app_path = os.path.join(".", "cluster", cluster_name, "app")
self.service_users = {}
self.secrets = {}
self.modules = []
# Load config from secretmgr.toml in cluster directory
secretmgr_toml_path = os.path.join(".", "cluster", cluster_name, "secretmgr.toml")
if os.path.exists(secretmgr_toml_path):
with open(secretmgr_toml_path) as f:
secretmgr_toml = toml.load(f)
else:
secretmgr_toml = {}
if "constants" in secretmgr_toml:
self.constants = secretmgr_toml["constants"]
else:
self.constants = {}
self.ldap_server = None
self.ldap_service_dn_suffix = None
self.ldap_admin_dn = None
self.ldap_admin_password_secret = None
self.ldap_admin_conn = None
if "ldap" in secretmgr_toml:
if "server" in secretmgr_toml["ldap"]:
self.ldap_server = secretmgr_toml["ldap"]["server"]
if "service_dn_suffix" in secretmgr_toml["ldap"]:
self.ldap_service_dn_suffix = secretmgr_toml["ldap"]["service_dn_suffix"]
if "admin_dn" in secretmgr_toml["ldap"]:
self.ldap_admin_dn = secretmgr_toml["ldap"]["admin_dn"]
if "admin_password_secret" in secretmgr_toml["ldap"]:
self.ldap_admin_password_secret = secretmgr_toml["ldap"]["admin_password_secret"]
def load_module(self, module_name):
secrets_toml_path = os.path.join(self.app_path, module_name, "secrets.toml")
with open(secrets_toml_path) as f:
secrets_toml = toml.load(f)
self.modules.append(module_name)
# Service users, and their associated secrets
if "service_users" in secrets_toml:
for (uname, uargs) in secrets_toml["service_users"].items():
service_user = ServiceUser(uname, config=self, **uargs)
for (skey, secret) in service_user.secrets().items():
if skey in self.secrets:
raise Exception("Duplicate secret: {}".format(skey))
self.secrets[skey] = secret
self.service_users[uname] = service_user
# Other secrets
if "secrets" in secrets_toml:
for (skey, sargs) in secrets_toml["secrets"].items():
ty = sargs["type"]
del sargs["type"]
if ty == "user":
secret = UserSecret(config=self, key=skey, **sargs)
elif ty == "command":
secret = CommandSecret(config=self, key=skey, **sargs)
elif ty == "constant":
secret = ConstantSecret(config=self, key=skey, **sargs)
else:
raise Exception("Invalid secret type: {}".format(ty))
if skey in self.secrets:
raise Exception("Duplicate secret: {}".format(skey))
self.secrets[skey] = secret
def add_constant_secrets(self):
for (skey, value) in self.constants.items():
self.secrets[skey] = ConstantSecret(
config=self,
key=skey,
value=value,
description="Constant value for secret {}".format(skey))
def get_ldap_admin_conn(self):
if self.ldap_admin_conn is None:
if self.ldap_admin_password_secret != None:
pass_key = "secrets/" + self.ldap_admin_password_secret
_, data = consul_server.kv.get(pass_key)
if data is None:
raise Exception("LDAP admin password could not be read at: {}".format(pass_key))
ldap_pass = data["Value"].decode('ascii').strip()
else:
ldap_pass = getpass.getpass("LDAP admin password: ")
self.ldap_admin_conn = ldap.initialize(self.ldap_server)
self.ldap_admin_conn.simple_bind_s(self.ldap_admin_dn, ldap_pass)
return self.ldap_admin_conn
def check_secrets(self):
print(":: Checking secrets...")
must_gen = False
for (_, secret) in self.secrets.items():
_, data = consul_server.kv.get(secret.consul_key)
if data is None:
print(secret.consul_key, bcolors.FAIL, "x missing", bcolors.ENDC)
must_gen = True
elif not secret.check(data["Value"].decode('ascii').strip()):
print(secret.consul_key, bcolors.WARNING, "x bad value", bcolors.ENDC)
must_gen = True
else:
print(secret.consul_key, bcolors.OKGREEN, "✓", bcolors.ENDC)
print()
if must_gen:
print("To fix missing or invalid secrets, use `secretmgr gen <cluster_name> <app>...`")
print()
def gen_secrets(self):
if len(self.service_users) > 0:
print(":: Configuring service users...")
for (_, su) in self.service_users.items():
su.configure(False)
print()
print(":: Generating missing/invalid secrets...")
for (_, secret) in self.secrets.items():
_, data = consul_server.kv.get(secret.consul_key)
if data is None or not secret.check(data["Value"].decode('ascii').strip()):
print()
secret.print_info()
value = secret.generate()
if value != None:
consul_server.kv.put(secret.consul_key, value)
print(bcolors.OKCYAN, "Value set.", bcolors.ENDC)
else:
print(bcolors.WARNING, "Skipped.", bcolors.ENDC)
print()
self.check_secrets()
def rotate_secrets(self):
if len(self.service_users) > 0:
print(":: Regenerating service user passwords...")
for (_, su) in self.service_users.items():
su.configure(True)
print()
print(":: Rotating secrets...")
for (_, secret) in self.secrets.items():
print()
secret.print_info()
_, data = consul_server.kv.get(secret.consul_key)
if data is None:
old_value = None
else:
old_value = data["Value"].decode('ascii').strip()
new_value = secret.rotate()
if new_value != None and new_value != old_value:
consul_server.kv.put(secret.consul_key, new_value)
print(bcolors.OKCYAN, "Value set.", bcolors.ENDC)
else:
print(bcolors.OKGREEN, "Nothing to do.", bcolors.ENDC)
print()
self.check_secrets()
# ---- MAIN ----
def load_config(cluster_name, modules):
cfg = Config(cluster_name)
if len(modules) > 0:
for mod in modules:
cfg.load_module(mod)
else:
cfg.add_constant_secrets()
return cfg
if __name__ == "__main__":
verb = None
for i, val in enumerate(sys.argv):
if val == "check":
verb = lambda cfg: cfg.check_secrets()
break
elif val == "gen":
verb = lambda cfg: cfg.gen_secrets()
break
elif val == "rotate":
verb = lambda cfg: cfg.rotate_secrets()
break
if verb is None:
print("Usage:")
print(" secretmgr.py [check|gen|rotate] <cluster name> <module name>...")
else:
cfg = load_config(sys.argv[i+1], sys.argv[i+2:])
verb(cfg)
# vim: set sts=4 ts=4 sw=4 tw=0 ft=python et :