From a5a56b6f70bd5b445c3d10d0c5a99011a8bcdd1d Mon Sep 17 00:00:00 2001 From: LUXEY Adrien Date: Mon, 28 Dec 2020 12:04:08 +0100 Subject: [PATCH 1/8] wrote a redirection to deuxfleurs.fr in Treafik config's comments --- app/deployment/web_static.hcl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/deployment/web_static.hcl b/app/deployment/web_static.hcl index a02d48b..01de23e 100644 --- a/app/deployment/web_static.hcl +++ b/app/deployment/web_static.hcl @@ -38,8 +38,13 @@ EOH "traefik.enable=true", "traefik.frontend.entryPoints=https,http", "traefik.frontend.rule=Host:deuxfleurs.fr,www.deuxfleurs.fr,deuxfleurs.org,www.deuxfleurs.org;PathPrefix:/", - # ideally we would have a rewrite regex: ^https?://(www\.deuxfleurs\.fr|deuxfleurs\.org|www\.deuxfleurs\.fr)(.*)$ "traefik.frontend.priority=10" + # Ideally we would have a rewrite regex like this https://regex101.com/r/WHrABU/1 + # See how it does not handle "http://deuxfleurs.fr/"? Not a big deal since HTTPS is redirected somewhere else + #"traefik.frontend.redirect.regex=^https?://(www\.deuxfleurs\.fr|deuxfleurs\.org|www\.deuxfleurs\.org)(.*)$", + #"traefik.frontend.redirect.replacement=https://deuxfleurs.fr/$${2}", + # Only set permanent redirect when it is guaranteed to work + #"traefik.frontend.redirect.permanent=true", ] port = "deuxfleurs_port" address_mode = "host" From 2a346f54301503077def38edc1039230a699958e Mon Sep 17 00:00:00 2001 From: LUXEY Adrien Date: Tue, 19 Jan 2021 14:40:14 +0100 Subject: [PATCH 2/8] coquille --- app/im/secrets/chat/coturn/static-auth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/im/secrets/chat/coturn/static-auth b/app/im/secrets/chat/coturn/static-auth index d23be29..43628ef 100644 --- a/app/im/secrets/chat/coturn/static-auth +++ b/app/im/secrets/chat/coturn/static-auth @@ -1 +1 @@ -USER cotorn static-auth (what is this?) +USER coturn static-auth (what is this?) From 98280c8628bc1df6197a1a830808bcd8631101bf Mon Sep 17 00:00:00 2001 From: LUXEY Adrien Date: Tue, 19 Jan 2021 15:21:23 +0100 Subject: [PATCH 3/8] updated READMEs --- README.md | 33 +++++++++++++++++++++++++++------ app/.gitignore | 1 + app/README.md | 38 ++++++++++++++++++++++++++------------ 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5bf9f58..83aad91 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ To ease the development, we make the choice of a fully integrated environment ### Deploying/Updating new services is done from your machine -*The following instructions are provided for ops that already have access to the servers.* +*The following instructions are provided for ops that already have access to the servers (meaning: their SSH public key is known by the cluster).* Deploy Nomad on your machine: @@ -74,16 +74,37 @@ Create an alias (and put it in your `.bashrc`) to bind APIs on your machine: alias bind_df="ssh \ -p110 \ -N \ - -L 4646:127.0.0.1:4646 \ - -L 8500:127.0.0.1:8500 \ - -L 8082:traefik-admin.service.2.cluster.deuxfleurs.fr:8082 \ - -L 5432:psql-proxy.service.2.cluster.deuxfleurs.fr:5432 \ -L 1389:bottin2.service.2.cluster.deuxfleurs.fr:389 \ + -L 4646:127.0.0.1:4646 \ + -L 5432:psql-proxy.service.2.cluster.deuxfleurs.fr:5432 \ + -L 8082:traefik-admin.service.2.cluster.deuxfleurs.fr:8082 \ + -L 8500:127.0.0.1:8500 \ " ``` and run: + bind_df + +Adrien uses `.ssh/config` configuration instead. I works basically the same. Here it goes: + ``` -bind_df +# in ~/.ssh/config + +Host deuxfleurs + User adrien + Hostname deuxfleurs.fr + # If you don't use the default ~/.ssh/id_rsa to connect to Deuxfleurs + IdentityFile + PubKeyAuthentication yes + ForwardAgent No + LocalForward 1389 bottin2.service.2.cluster.deuxfleurs.fr:389 + LocalForward 4646 127.0.0.1:4646 + LocalForward 5432 psql-proxy.service.2.cluster.deuxfleurs.fr:5432 + LocalForward 8082 traefik-admin.service.2.cluster.deuxfleurs.fr:8082 + LocalForward 8500 127.0.0.1:8500 ``` + +Now, to connect, do the following: + + ssh deuxfleurs -N diff --git a/app/.gitignore b/app/.gitignore index bee8a64..1da68d7 100644 --- a/app/.gitignore +++ b/app/.gitignore @@ -1 +1,2 @@ +env/ __pycache__ diff --git a/app/README.md b/app/README.md index 3049cac..a0dcf43 100644 --- a/app/README.md +++ b/app/README.md @@ -1,6 +1,4 @@ -## Understand this folder hierarchy - -This folder contains the following hierarchy: +# Folder hierarchy - `/build//`: folders with dockerfiles and other necessary resources for building container images - `/config/`: folder containing configuration files, referenced by deployment file @@ -8,18 +6,34 @@ This folder contains the following hierarchy: - `/deploy/`: folder containing the HCL file(s) necessary for deploying the module - `/integration/`: folder containing files for integration testing using docker-compose +# Secret Manager `secretmgr.py` + +The Secret Manager ensures that all secrets are present where they should in the cluster. + +**You need access to the cluster** (SSH port forwarding) for it to find any secret on the cluster. Refer to the previous directory's [README](../README.md), at the bottom of the file. + ## How to install `secretmgr.py` dependencies -How to install its dependencies: - ```bash -# on fedora: -dnf install -y openldap-devel -# on ubuntu: -apt-get install -y libldap2-dev +### Install system dependencies first: +## On fedora -# for eveyrone: -pip3 install --user --requirement requirements.txt +dnf install -y openldap-devel cyrus-sasl-devel +## On ubuntu +apt-get install -y libldap2-dev libsasl2-dev + +### Now install the Python dependencies from requirements.txt: + +## Either using a virtual environment +# (requires virtualenv python module) +python3 -m virtualenv env +# Must be done everytime you create a new terminal window in this folder: +. env/bin/activate +# Install the deps +pip install -r requirements.txt + +## Either by installing the dependencies for your system user: +pip3 install --user -r requirements.txt ``` ## How to use `secretmgr.py` @@ -42,7 +56,7 @@ Rotate secrets for app `dummy`, overwriting existing ones (be careful, this is d ./secretmgr.py regen dummy ``` -## How to upgrade our packaged apps to a new version? +# Upgrading one of our packaged apps to a new version 1. Edit `docker-compose.yml` 2. Change the `VERSION` variable to the desired version From 6e1940061ab6da8021cd9f38818657baa4be2d17 Mon Sep 17 00:00:00 2001 From: LUXEY Adrien Date: Tue, 19 Jan 2021 17:16:58 +0100 Subject: [PATCH 4/8] =?UTF-8?q?coturn=20retourne=20une=20erreur=20bien=20f?= =?UTF-8?q?ormat=C3=A9e=20face=20=C3=A0=20un=20fichier=20vide=20(il=20pour?= =?UTF-8?q?rait=20renvoyer=20autre=20chose),=20plus=20bug=20nom=20de=20var?= =?UTF-8?q?iable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/secretmgr.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/secretmgr.py b/app/secretmgr.py index 6af6d13..950e5c0 100755 --- a/app/secretmgr.py +++ b/app/secretmgr.py @@ -103,6 +103,9 @@ class bcolors: def read_secret(key, file_path): lines = [l.strip() for l in open(file_path, "r")] + if len(lines) == 0: + print(bcolors.FAIL, "ERROR:", bcolors.ENDC, "Empty file in", file_path) + sys.exit(-1) l0 = lines[0].split(" ") stype = l0[0] secret = {"type": stype, "key": key} @@ -275,7 +278,7 @@ def gen_secrets_base(secrets, regen): line = input().strip() if line == ".": break - vals.append(line) + lines.append(line) val = "\n".join(lines) consul_server.kv.put(key, val) print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) From a847a9683f6e8be44f40ee92a44416ab6dce14a1 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 19 Jan 2021 17:27:32 +0100 Subject: [PATCH 5/8] Cleanup op_guide folder --- op_guide/plume/README.md | 2 ++ op_guide/{update_matrix.md => update_matrix/README.md} | 0 2 files changed, 2 insertions(+) rename op_guide/{update_matrix.md => update_matrix/README.md} (100%) diff --git a/op_guide/plume/README.md b/op_guide/plume/README.md index fa6084d..4a8bbac 100644 --- a/op_guide/plume/README.md +++ b/op_guide/plume/README.md @@ -1,3 +1,5 @@ +## Creating a new Plume user + 1. Bind nomad on your machine with SSH (check the README file at the root of this repo) 2. Go to http://127.0.0.1:4646 3. Select `plume` -> click `exec` button (top right) diff --git a/op_guide/update_matrix.md b/op_guide/update_matrix/README.md similarity index 100% rename from op_guide/update_matrix.md rename to op_guide/update_matrix/README.md From 9560f8085292e990949c53b4ba964936a3d3b6e5 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 19 Jan 2021 17:29:37 +0100 Subject: [PATCH 6/8] mention secretmgr.py in create_database --- op_guide/create_database/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/op_guide/create_database/README.md b/op_guide/create_database/README.md index 7d49c97..fb3bdd9 100644 --- a/op_guide/create_database/README.md +++ b/op_guide/create_database/README.md @@ -8,6 +8,8 @@ Go to guichet.deuxfleurs.fr 4. Hash it with `slappasswd` 5. Add a `userpassword` entry with the hash +This step can also be done using the automated tool `secretmgr.py` in the app folder. + ## 2. Connect to postgres with the admin users ```bash From 1c814f002af3aafa76aced040845d6fdeee7953e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 19 Jan 2021 17:53:53 +0100 Subject: [PATCH 7/8] Add CMD_ONCE secret type and fill in/change secret definitions --- app/im/secrets/chat/easybridge/as_token | 1 + app/im/secrets/chat/easybridge/db_pass | 1 + app/im/secrets/chat/easybridge/db_user | 1 + app/im/secrets/chat/easybridge/hs_token | 1 + app/im/secrets/chat/easybridge/web_session_key | 2 ++ app/im/secrets/chat/fb2mx/as_token | 2 +- app/im/secrets/chat/fb2mx/hs_token | 2 +- app/im/secrets/chat/synapse/homeserver.signing.key | 1 + app/im/secrets/chat/synapse/registration_shared_secret | 2 +- app/plume/secrets/plume/pgsql_pw | 2 +- app/secretmgr.py | 9 +++++++-- 11 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 app/im/secrets/chat/easybridge/as_token create mode 100644 app/im/secrets/chat/easybridge/db_pass create mode 100644 app/im/secrets/chat/easybridge/db_user create mode 100644 app/im/secrets/chat/easybridge/hs_token create mode 100644 app/im/secrets/chat/easybridge/web_session_key create mode 100644 app/im/secrets/chat/synapse/homeserver.signing.key diff --git a/app/im/secrets/chat/easybridge/as_token b/app/im/secrets/chat/easybridge/as_token new file mode 100644 index 0000000..5fa4e3c --- /dev/null +++ b/app/im/secrets/chat/easybridge/as_token @@ -0,0 +1 @@ +CMD openssl rand -hex 32 diff --git a/app/im/secrets/chat/easybridge/db_pass b/app/im/secrets/chat/easybridge/db_pass new file mode 100644 index 0000000..7e1f94b --- /dev/null +++ b/app/im/secrets/chat/easybridge/db_pass @@ -0,0 +1 @@ +SERVICE_PASSWORD easybridge diff --git a/app/im/secrets/chat/easybridge/db_user b/app/im/secrets/chat/easybridge/db_user new file mode 100644 index 0000000..436267c --- /dev/null +++ b/app/im/secrets/chat/easybridge/db_user @@ -0,0 +1 @@ +CONST easybridge diff --git a/app/im/secrets/chat/easybridge/hs_token b/app/im/secrets/chat/easybridge/hs_token new file mode 100644 index 0000000..5fa4e3c --- /dev/null +++ b/app/im/secrets/chat/easybridge/hs_token @@ -0,0 +1 @@ +CMD openssl rand -hex 32 diff --git a/app/im/secrets/chat/easybridge/web_session_key b/app/im/secrets/chat/easybridge/web_session_key new file mode 100644 index 0000000..614bed7 --- /dev/null +++ b/app/im/secrets/chat/easybridge/web_session_key @@ -0,0 +1,2 @@ +CMD openssl rand -hex 32 + diff --git a/app/im/secrets/chat/fb2mx/as_token b/app/im/secrets/chat/fb2mx/as_token index 20b76d4..5fa4e3c 100644 --- a/app/im/secrets/chat/fb2mx/as_token +++ b/app/im/secrets/chat/fb2mx/as_token @@ -1 +1 @@ -USER fb2mx API server token +CMD openssl rand -hex 32 diff --git a/app/im/secrets/chat/fb2mx/hs_token b/app/im/secrets/chat/fb2mx/hs_token index 8808f8f..5fa4e3c 100644 --- a/app/im/secrets/chat/fb2mx/hs_token +++ b/app/im/secrets/chat/fb2mx/hs_token @@ -1 +1 @@ -USER fb2mx homeserver token +CMD openssl rand -hex 32 diff --git a/app/im/secrets/chat/synapse/homeserver.signing.key b/app/im/secrets/chat/synapse/homeserver.signing.key new file mode 100644 index 0000000..099bd18 --- /dev/null +++ b/app/im/secrets/chat/synapse/homeserver.signing.key @@ -0,0 +1 @@ +USER Synapse homeserver ed25519 signing key diff --git a/app/im/secrets/chat/synapse/registration_shared_secret b/app/im/secrets/chat/synapse/registration_shared_secret index 395cccc..b82f191 100644 --- a/app/im/secrets/chat/synapse/registration_shared_secret +++ b/app/im/secrets/chat/synapse/registration_shared_secret @@ -1 +1 @@ -USER Shared secret for homeserver registrations (?) +CMD head -c 32 /dev/urandom | base64 diff --git a/app/plume/secrets/plume/pgsql_pw b/app/plume/secrets/plume/pgsql_pw index 978be54..0f831bb 100644 --- a/app/plume/secrets/plume/pgsql_pw +++ b/app/plume/secrets/plume/pgsql_pw @@ -1 +1 @@ -CMD openssl rand -base64 32 +SERVICE_PASSWORD plume diff --git a/app/secretmgr.py b/app/secretmgr.py index 6af6d13..5cf55dc 100755 --- a/app/secretmgr.py +++ b/app/secretmgr.py @@ -43,6 +43,9 @@ USER_LONG CMD (a secret that is generated by running this command) +CMD_ONCE +(same, but value is not changed when doing a regen) + CONST (the secret has a constant value set here) @@ -81,6 +84,7 @@ consul_server = consul.Consul() USER = "USER" USER_LONG = "USER_LONG" CMD = "CMD" +CMD_ONCE = "CMD_ONCE" CONST = "CONST" CONST_LONG = "CONST_LONG" SERVICE_DN = "SERVICE_DN" @@ -108,7 +112,7 @@ def read_secret(key, file_path): secret = {"type": stype, "key": key} if stype in [USER, USER_LONG]: secret["desc"] = " ".join(l0[1:]) - elif stype == CMD: + elif stype in [CMD, CMD_ONCE]: secret["cmd"] = " ".join(l0[1:]) elif stype == CONST: secret["value"] = " ".join(l0[1:]) @@ -151,6 +155,7 @@ def get_secrets_services(secrets): if svc not in services: services[svc] = { "dn": "cn=%s,%s"%(svc, SERVICE_DN_SUFFIX), + "desc": "(not provided)", "pass": None, "dn_at": [], "pass_at": [], @@ -289,7 +294,7 @@ def gen_secrets_base(secrets, regen): consul_server.kv.put(key, secret["value"]) print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) - if secret["type"] == CMD: + if secret["type"] == CMD or (secret["type"] == CMD_ONCE and data is None): print("----") print(key) print("Executing command:", secret["cmd"]) From a2eec38de4acb0a58f22dddc161facf44910085a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 19 Jan 2021 18:02:00 +0100 Subject: [PATCH 8/8] Add a few missing secrets --- app/backup/secrets/backup/id_ed25519 | 1 + app/backup/secrets/backup/id_ed25519.pub | 1 + app/backup/secrets/backup/target_ssh_dir | 1 + app/backup/secrets/backup/target_ssh_fingerprint | 1 + app/backup/secrets/backup/target_ssh_host | 1 + app/backup/secrets/backup/target_ssh_port | 1 + app/backup/secrets/backup/target_ssh_user | 1 + app/garage/secrets/garage/garage-ca.crt | 1 + app/garage/secrets/garage/garage-ca.key | 1 + app/garage/secrets/garage/garage.crt | 1 + app/garage/secrets/garage/garage.key | 1 + 11 files changed, 11 insertions(+) create mode 100644 app/backup/secrets/backup/id_ed25519 create mode 100644 app/backup/secrets/backup/id_ed25519.pub create mode 100644 app/backup/secrets/backup/target_ssh_dir create mode 100644 app/backup/secrets/backup/target_ssh_fingerprint create mode 100644 app/backup/secrets/backup/target_ssh_host create mode 100644 app/backup/secrets/backup/target_ssh_port create mode 100644 app/backup/secrets/backup/target_ssh_user create mode 100644 app/garage/secrets/garage/garage-ca.crt create mode 100644 app/garage/secrets/garage/garage-ca.key create mode 100644 app/garage/secrets/garage/garage.crt create mode 100644 app/garage/secrets/garage/garage.key diff --git a/app/backup/secrets/backup/id_ed25519 b/app/backup/secrets/backup/id_ed25519 new file mode 100644 index 0000000..9d7fd46 --- /dev/null +++ b/app/backup/secrets/backup/id_ed25519 @@ -0,0 +1 @@ +USER_LONG Private ed25519 key of the container doing the backup diff --git a/app/backup/secrets/backup/id_ed25519.pub b/app/backup/secrets/backup/id_ed25519.pub new file mode 100644 index 0000000..0a2ab35 --- /dev/null +++ b/app/backup/secrets/backup/id_ed25519.pub @@ -0,0 +1 @@ +USER Public ed25519 key of the container doing the backup (this key must be in authorized_keys on the backup target host) diff --git a/app/backup/secrets/backup/target_ssh_dir b/app/backup/secrets/backup/target_ssh_dir new file mode 100644 index 0000000..3b2a4da --- /dev/null +++ b/app/backup/secrets/backup/target_ssh_dir @@ -0,0 +1 @@ +USER Directory where to store backups on target host diff --git a/app/backup/secrets/backup/target_ssh_fingerprint b/app/backup/secrets/backup/target_ssh_fingerprint new file mode 100644 index 0000000..608f3ec --- /dev/null +++ b/app/backup/secrets/backup/target_ssh_fingerprint @@ -0,0 +1 @@ +USER SSH fingerprint of the target machine (format: copy here the corresponding line from your known_hosts file) diff --git a/app/backup/secrets/backup/target_ssh_host b/app/backup/secrets/backup/target_ssh_host new file mode 100644 index 0000000..6268f87 --- /dev/null +++ b/app/backup/secrets/backup/target_ssh_host @@ -0,0 +1 @@ +USER Hostname of the backup target host diff --git a/app/backup/secrets/backup/target_ssh_port b/app/backup/secrets/backup/target_ssh_port new file mode 100644 index 0000000..309dd38 --- /dev/null +++ b/app/backup/secrets/backup/target_ssh_port @@ -0,0 +1 @@ +USER SSH port number to connect to the target host diff --git a/app/backup/secrets/backup/target_ssh_user b/app/backup/secrets/backup/target_ssh_user new file mode 100644 index 0000000..98b3046 --- /dev/null +++ b/app/backup/secrets/backup/target_ssh_user @@ -0,0 +1 @@ +USER SSH username to log in as on the target host diff --git a/app/garage/secrets/garage/garage-ca.crt b/app/garage/secrets/garage/garage-ca.crt new file mode 100644 index 0000000..8488ab6 --- /dev/null +++ b/app/garage/secrets/garage/garage-ca.crt @@ -0,0 +1 @@ +USER_LONG garage-ca.crt (generated with Garage's genkeys.sh script) diff --git a/app/garage/secrets/garage/garage-ca.key b/app/garage/secrets/garage/garage-ca.key new file mode 100644 index 0000000..ca3e90c --- /dev/null +++ b/app/garage/secrets/garage/garage-ca.key @@ -0,0 +1 @@ +USER_LONG garage-ca.key (generated with Garage's genkeys.sh script) diff --git a/app/garage/secrets/garage/garage.crt b/app/garage/secrets/garage/garage.crt new file mode 100644 index 0000000..6044ab8 --- /dev/null +++ b/app/garage/secrets/garage/garage.crt @@ -0,0 +1 @@ +USER_LONG garage.crt (generated with Garage's genkeys.sh script) diff --git a/app/garage/secrets/garage/garage.key b/app/garage/secrets/garage/garage.key new file mode 100644 index 0000000..db3cb0e --- /dev/null +++ b/app/garage/secrets/garage/garage.key @@ -0,0 +1 @@ +USER_LONG garage.key (generated with Garage's genkeys.sh script)