diff --git a/README.md b/README.md index 26a7856..5bf9f58 100644 --- a/README.md +++ b/README.md @@ -5,21 +5,25 @@ deuxfleurs.fr ## Our abstraction stack -We try to build a generic abstraction stack between our different resources (CPU, RAM, disk, etc.) and our services (Chat, Storage, etc.): +We try to build a generic abstraction stack between our different resources (CPU, RAM, disk, etc.) and our services (Chat, Storage, etc.), we develop our own tools when needed: - * ansible (physical node conf) - * nomad (schedule containers) - * consul (distributed key value store / lock / service discovery) - * garage/glusterfs (file storage) - * stolon + postgresql (distributed relational database) - * docker (container tool) - * bottin (LDAP server, auth) + * **[garage](https://git.deuxfleurs.fr/Deuxfleurs/garage/):** S3-compatible lightweight object store for self-hosted geo-distributed deployments (we also have a legacy glusterfs cluster) + * **[diplonat](https://git.deuxfleurs.fr/Deuxfleurs/diplonat):** network automation (firewalling, upnp igd) + * **[bottin](https://git.deuxfleurs.fr/Deuxfleurs/bottin):** authentication and authorization (LDAP protocol, consul backend) + * **[guichet](https://git.deuxfleurs.fr/Deuxfleurs/guichet):** a dashboard for our users and administrators + * **ansible:** physical node configuration + * **nomad:** schedule containers and handle their lifecycle + * **consul:** distributed key value store + lock + service discovery + * **stolon + postgresql:** distributed relational database + * **docker:** package, distribute and isolate applications Some services we provide: - * Chat (Matrix/Riot) - * Email (Postfix/Dovecot/Sogo) - * Storage (Seafile) + * **Websites:** garage (static) + fediverse blog (plume) + * **Chat:** Synapse + Element Web (Matrix protocol) + * **Email:** Postfix SMTP + Dovecot IMAP + opendkim DKIM + Sogo webmail (legacy) | Alps webmail (experimental) + * **Storage:** Seafile (legacy) | Nextcloud (experimental) + * **Visio:** Jitsi As a generic abstraction is provided, deploying new services should be easy. @@ -40,14 +44,6 @@ To ease the development, we make the choice of a fully integrated environment ## Start hacking -### Clone the repository - -``` -git clone https://gitlab.com/superboum/deuxfleurs.fr.git -git submodule init -git submodule update -``` - ### Deploying/Updating new services is done from your machine *The following instructions are provided for ops that already have access to the servers.* @@ -82,6 +78,7 @@ alias bind_df="ssh \ -L 8500:127.0.0.1:8500 \ -L 8082:traefik-admin.service.2.cluster.deuxfleurs.fr:8082 \ -L 5432:psql-proxy.service.2.cluster.deuxfleurs.fr:5432 \ + -L 1389:bottin2.service.2.cluster.deuxfleurs.fr:389 \ " ``` diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/app/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/app/README.md b/app/README.md new file mode 100644 index 0000000..3049cac --- /dev/null +++ b/app/README.md @@ -0,0 +1,52 @@ +## Understand this folder hierarchy + +This folder contains the following hierarchy: + +- `/build//`: folders with dockerfiles and other necessary resources for building container images +- `/config/`: folder containing configuration files, referenced by deployment file +- `/secrets/`: folder containing secrets, which can be synchronized with Consul using `secretmgr.py` +- `/deploy/`: folder containing the HCL file(s) necessary for deploying the module +- `/integration/`: folder containing files for integration testing using docker-compose + +## How to install `secretmgr.py` dependencies + +How to install its dependencies: + +```bash +# on fedora: +dnf install -y openldap-devel +# on ubuntu: +apt-get install -y libldap2-dev + +# for eveyrone: +pip3 install --user --requirement requirements.txt +``` + +## How to use `secretmgr.py` + +Check that all secrets are correctly deployed for app `dummy`: + +```bash +./secretmgr.py check dummy +``` + +Generate secrets for app `dummy` if they don't already exist: + +```bash +./secretmgr.py gen dummy +``` + +Rotate secrets for app `dummy`, overwriting existing ones (be careful, this is dangerous!): + +```bash +./secretmgr.py regen dummy +``` + +## How to upgrade our packaged apps to a new version? + + 1. Edit `docker-compose.yml` + 2. Change the `VERSION` variable to the desired version + 3. Increment the docker image tag by 1 (eg: superboum/riot:v13 -> superboum/riot:v14) + 4. Run `docker-compose build` + 5. Run `docker-compose push` + 6. Done diff --git a/app/build/backup-consul/Dockerfile b/app/backup/build/backup-consul/Dockerfile similarity index 100% rename from app/build/backup-consul/Dockerfile rename to app/backup/build/backup-consul/Dockerfile diff --git a/app/build/backup-consul/do_backup.sh b/app/backup/build/backup-consul/do_backup.sh similarity index 100% rename from app/build/backup-consul/do_backup.sh rename to app/backup/build/backup-consul/do_backup.sh diff --git a/app/deployment/backup.hcl b/app/backup/deploy/backup.hcl similarity index 100% rename from app/deployment/backup.hcl rename to app/backup/deploy/backup.hcl diff --git a/app/build/README.md b/app/build/README.md deleted file mode 100644 index a877cfa..0000000 --- a/app/build/README.md +++ /dev/null @@ -1,8 +0,0 @@ -## How to upgrade our packaged apps to a new version? - - 1. Edit `docker-compose.yml` - 2. Change the `VERSION` variable to the desired version - 3. Increment the docker image tag by 1 (eg: superboum/riot:v13 -> superboum/riot:v14) - 4. Run `docker-compose build` - 5. Run `docker-compose push` - 6. Done diff --git a/app/build/blog-quentin/.dockerenv b/app/build/blog-quentin/.dockerenv deleted file mode 100755 index e69de29..0000000 diff --git a/app/build/blog-quentin/Dockerfile b/app/build/blog-quentin/Dockerfile deleted file mode 100644 index 61f5c40..0000000 --- a/app/build/blog-quentin/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM amd64/debian:stretch as builder - -COPY ./quentin.dufour.io/Gemfile /root/quentin.dufour.io/Gemfile - -WORKDIR /root/quentin.dufour.io - -RUN apt-get update && \ - apt-get install -y ruby-dev gem build-essential bundler zlib1g-dev libxml2-dev && \ - bundle install - -COPY ./quentin.dufour.io/ /root/quentin.dufour.io/ -RUN bundle exec jekyll build - -FROM superboum/amd64_webserver:v2 -COPY --from=builder /root/quentin.dufour.io/_site /srv/http - diff --git a/app/build/blog-quentin/README.md b/app/build/blog-quentin/README.md deleted file mode 100644 index 25ac463..0000000 --- a/app/build/blog-quentin/README.md +++ /dev/null @@ -1 +0,0 @@ -sudo docker build -t superboum/amd64_blog:v19 . diff --git a/app/build/coturn/Dockerfile b/app/build/coturn/Dockerfile deleted file mode 100644 index 0d23161..0000000 --- a/app/build/coturn/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM amd64/debian:buster - -RUN apt-get update && \ - apt-get dist-upgrade -y && \ - apt-get install -y \ - coturn - -CMD ["/usr/bin/turnserver"] diff --git a/app/build/coturn/README.md b/app/build/coturn/README.md deleted file mode 100644 index e882146..0000000 --- a/app/build/coturn/README.md +++ /dev/null @@ -1,17 +0,0 @@ - -## Génère l'image -``` -sudo docker build -t registry.gitlab.com/superboum/ankh-morpork/amd64_coturn:v1 . -``` - -## Run bash dans le container -``` -sudo docker run --rm -t -i registry.gitlab.com/superboum/ankh-morpork/amd64_coturn:v1 bash -sudo docker run --rm -t -i -p 3478:3478/udp -p 3479:3479/udp -p 3478:3478/tcp -p 3479:3479/tcp registry.gitlab.com/superboum/ankh-morpork/amd64_coturn:v1 -``` - -## Used ports -- udp/tcp 3478 3479 - -## Publish -sudo docker push registry.gitlab.com/superboum/ankh-morpork/amd64_coturn:v1 diff --git a/app/build/landing/README.md b/app/build/landing/README.md deleted file mode 100644 index 5d2cb2b..0000000 --- a/app/build/landing/README.md +++ /dev/null @@ -1,3 +0,0 @@ -``` -docker build -t superboum/amd64_landing:v8 . -``` diff --git a/app/build/static/Dockerfile b/app/build/static/Dockerfile deleted file mode 100644 index cdba59a..0000000 --- a/app/build/static/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM golang:1.11.1-stretch as builder - -COPY ./goStatic /goStatic -WORKDIR /goStatic -RUN CGO_ENABLED=0 go build -a -o web-server . - -FROM scratch -COPY --from=builder /goStatic/web-server / -ENTRYPOINT ["/web-server"] diff --git a/app/build/static/README.md b/app/build/static/README.md deleted file mode 100644 index d50390c..0000000 --- a/app/build/static/README.md +++ /dev/null @@ -1,5 +0,0 @@ - -``` -sudo docker build -t superboum/amd64_webserver:v3 . -sudo docker push superboum/amd64_webserver:v3 -``` diff --git a/app/build/static/goStatic b/app/build/static/goStatic deleted file mode 160000 index 3f97f57..0000000 --- a/app/build/static/goStatic +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3f97f57aaee09a142afe3ca0f1a5d51acd856436 diff --git a/app/build/webpull/.gitignore b/app/build/webpull/.gitignore deleted file mode 100644 index ba2906d..0000000 --- a/app/build/webpull/.gitignore +++ /dev/null @@ -1 +0,0 @@ -main diff --git a/app/build/webpull/Dockerfile.nodejs b/app/build/webpull/Dockerfile.nodejs deleted file mode 100644 index acc7e74..0000000 --- a/app/build/webpull/Dockerfile.nodejs +++ /dev/null @@ -1,9 +0,0 @@ -FROM node:13.8-buster - -RUN apt-get update && \ - apt-get install -y git - -COPY ./main /srv/httpd -WORKDIR /srv -CMD ["/srv/httpd"] - diff --git a/app/build/webpull/Dockerfile.ruby b/app/build/webpull/Dockerfile.ruby deleted file mode 100644 index 7578cca..0000000 --- a/app/build/webpull/Dockerfile.ruby +++ /dev/null @@ -1,12 +0,0 @@ -FROM fedora:32 - -ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 -ENV LANGUAGE=en_US.UTF-8 -ENV RUBYOPT --disable-did_you_mean - -RUN dnf install -y git ruby ruby-devel rubygems rubygem-bundler @development-tools redhat-rpm-config gcc-c++ zlib-devel - -COPY ./main /srv/httpd -WORKDIR /srv -CMD ["/srv/httpd"] diff --git a/app/build/webpull/README.md b/app/build/webpull/README.md deleted file mode 100644 index 5d17d17..0000000 --- a/app/build/webpull/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# webpull - -Webpull allows you to update your live website without deploying a new docker container but by simply calling an URL - -You need to specify a secret token at boot: - -``` -WEBPULL_TOKEN=s3cr3et ./webpull -``` - -## Node.js version - -``` -go build ./main.go -sudo docker build -f ./Dockerfile.nodejs -t superboum/amd64_webpull_pug:v1 . -``` - -## Ruby version - -``` -go build ./main.go -sudo docker build -f ./Dockerfile.ruby -t superboum/amd64_webpull_ruby:v1 . -``` diff --git a/app/build/webpull/main.go b/app/build/webpull/main.go deleted file mode 100644 index 46c90b9..0000000 --- a/app/build/webpull/main.go +++ /dev/null @@ -1,100 +0,0 @@ -package main - -import ( - "fmt" - "errors" - "io" - "os/exec" - "os" - "log" - "net/http" - "strings" -) - -func myexec(w io.Writer, main string, params ...string) error { - cmd := exec.Command(main, params...) - cmd.Stdout = w - cmd.Stderr = w - err := cmd.Run() - if err != nil { - fmt.Fprintf(w, "Failed to run: %s %s\n", main, strings.Join(params, " ")) - } - return err -} - -func update(w io.Writer) error { - fmt.Fprintf(w, "Start update...\n") - _, err := os.Stat("./.git") - if err != nil { - fmt.Fprintf(w, ".git folder does not exist, creating it...\n") - err := myexec(w, "git", "init") - if err != nil { - return err - } - } - - err = myexec(w, "git", "remote", "get-url", "origin") - if err != nil { - repo, exists := os.LookupEnv("WEBPULL_REPO") - if !exists { - fmt.Fprintf(w, "You must define WEBPULL_REPO env variable...\n") - return errors.New("Missing environment variable WEBPULL_REPO") - } - fmt.Fprintf(w, "git remote is not yet set...\n") - err := myexec(w, "git", "remote", "add", "origin", repo) - if err != nil { - return err - } - } - - err = myexec(w, "git", "pull", "origin", "master") - if err != nil { - fmt.Fprintf(w, "Failed to pull...\n") - return err - } - - _, err = os.Stat("./.webpull") - if err != nil { - fmt.Fprintf(w, "You must create an executable file named '.webpull' at the root of your repository.\nIf you have nothing to run, just create an empty bash script...\n") - return err - } - - err = myexec(w, "./.webpull") - if err != nil { - fmt.Fprintf(w, "An error occured during script execution\n") - return err - } - - fmt.Fprintf(w, "Success.\n") - return nil -} - -func main() { - token, exists := os.LookupEnv("WEBPULL_TOKEN") - if !exists { - log.Fatal("Environment variable 'WEBPULL_TOKEN' must be defined") - } - - if update(os.Stdout) != nil { - log.Fatal("Initial 'update' failed") - } - - fs := http.FileServer(http.Dir("./static")) - http.HandleFunc("/update", func(w http.ResponseWriter, r *http.Request) { - keys, ok := r.URL.Query()["token"] - if !ok || len(keys[0]) < 1 { - http.Error(w, "Missing 'token' query parameter", 401) - return - } - - if keys[0] != token { - http.Error(w, "Wrong token", 401) - return - } - - update(w) - }) - http.Handle("/", fs) - - log.Fatal(http.ListenAndServe(":8080", nil)) -} diff --git a/app/config/configuration/email/dkim/smtp.private.sample b/app/config/configuration/email/dkim/smtp.private.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/configuration/email/dkim/smtp.txt.sample b/app/config/configuration/email/dkim/smtp.txt.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/configuration/seafile/ccnet/mykey.peer.sample b/app/config/configuration/seafile/ccnet/mykey.peer.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/configuration/seafile/conf/mykey.peer.sample b/app/config/configuration/seafile/conf/mykey.peer.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/.gitignore b/app/config/secrets/.gitignore deleted file mode 100644 index 2ff3cd5..0000000 --- a/app/config/secrets/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Blacklist everything cleverly -* -!*/ - -# Whitelist some patterns -!*.sample -!*.gen -!*.sh -!.gitignore - -# Whitelist specific files diff --git a/app/config/secrets/chat/coturn/static-auth.sample b/app/config/secrets/chat/coturn/static-auth.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/fb2mx/as_token.sample b/app/config/secrets/chat/fb2mx/as_token.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/fb2mx/db_url.sample b/app/config/secrets/chat/fb2mx/db_url.sample deleted file mode 100644 index aff4635..0000000 --- a/app/config/secrets/chat/fb2mx/db_url.sample +++ /dev/null @@ -1 +0,0 @@ -postgres://username:password@hostname/dbname diff --git a/app/config/secrets/chat/fb2mx/hs_token.sample b/app/config/secrets/chat/fb2mx/hs_token.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/synapse/homeserver.tls.crt.sample b/app/config/secrets/chat/synapse/homeserver.tls.crt.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/synapse/homeserver.tls.dh.sample b/app/config/secrets/chat/synapse/homeserver.tls.dh.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/synapse/homeserver.tls.key.sample b/app/config/secrets/chat/synapse/homeserver.tls.key.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/synapse/ldap_binddn.sample b/app/config/secrets/chat/synapse/ldap_binddn.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/synapse/ldap_bindpw.sample b/app/config/secrets/chat/synapse/ldap_bindpw.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/synapse/postgres_db.sample b/app/config/secrets/chat/synapse/postgres_db.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/synapse/postgres_pwd.sample b/app/config/secrets/chat/synapse/postgres_pwd.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/synapse/postgres_user.sample b/app/config/secrets/chat/synapse/postgres_user.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/chat/synapse/registration_shared_secret.sample b/app/config/secrets/chat/synapse/registration_shared_secret.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/email/dkim/smtp.private.sample b/app/config/secrets/email/dkim/smtp.private.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/email/dovecot/dovecot.crt.sample b/app/config/secrets/email/dovecot/dovecot.crt.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/email/dovecot/dovecot.key.sample b/app/config/secrets/email/dovecot/dovecot.key.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/email/dovecot/ldap_binddn.sample b/app/config/secrets/email/dovecot/ldap_binddn.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/email/dovecot/ldap_bindpwd.sample b/app/config/secrets/email/dovecot/ldap_bindpwd.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/email/postfix/postfix.crt.sample b/app/config/secrets/email/postfix/postfix.crt.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/email/postfix/postfix.key.sample b/app/config/secrets/email/postfix/postfix.key.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/email/sogo/ldap_binddn.sample b/app/config/secrets/email/sogo/ldap_binddn.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/email/sogo/ldap_bindpw.sample b/app/config/secrets/email/sogo/ldap_bindpw.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/email/sogo/postgre_auth.sample b/app/config/secrets/email/sogo/postgre_auth.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/jitsi/auth.jitsi.deuxfleurs.fr.crt.sample b/app/config/secrets/jitsi/auth.jitsi.deuxfleurs.fr.crt.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/jitsi/auth.jitsi.deuxfleurs.fr.key.sample b/app/config/secrets/jitsi/auth.jitsi.deuxfleurs.fr.key.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/jitsi/jitsi.deuxfleurs.fr.crt.sample b/app/config/secrets/jitsi/jitsi.deuxfleurs.fr.crt.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/jitsi/jitsi.deuxfleurs.fr.key.sample b/app/config/secrets/jitsi/jitsi.deuxfleurs.fr.key.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/mariadb/main/ldap_binddn.sample b/app/config/secrets/mariadb/main/ldap_binddn.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/mariadb/main/ldap_bindpwd.sample b/app/config/secrets/mariadb/main/ldap_bindpwd.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/mariadb/main/mysql_pwd.sample b/app/config/secrets/mariadb/main/mysql_pwd.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/platoo/bddpw.sample b/app/config/secrets/platoo/bddpw.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/plume/pgsql_pw.sh b/app/config/secrets/plume/pgsql_pw.sh deleted file mode 100755 index 519a30a..0000000 --- a/app/config/secrets/plume/pgsql_pw.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -openssl rand -base64 32 > pgsql_pw diff --git a/app/config/secrets/plume/secret_key.sh b/app/config/secrets/plume/secret_key.sh deleted file mode 100755 index f4bbee5..0000000 --- a/app/config/secrets/plume/secret_key.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -openssl rand -base64 32 > secret_key diff --git a/app/config/secrets/postgres/keeper/pg_repl_pwd.sample b/app/config/secrets/postgres/keeper/pg_repl_pwd.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/postgres/keeper/pg_repl_username.sample b/app/config/secrets/postgres/keeper/pg_repl_username.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/postgres/keeper/pg_su_pwd.sample b/app/config/secrets/postgres/keeper/pg_su_pwd.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/seafile/conf/mykey.peer.sample b/app/config/secrets/seafile/conf/mykey.peer.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/web/home_token.sample b/app/config/secrets/web/home_token.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/config/secrets/web/quentin.dufour.io_token.sample b/app/config/secrets/web/quentin.dufour.io_token.sample deleted file mode 100644 index e69de29..0000000 diff --git a/app/deployment/core.hcl b/app/core/deploy/core.hcl similarity index 100% rename from app/deployment/core.hcl rename to app/core/deploy/core.hcl diff --git a/app/deployment/web_static.hcl b/app/deployment/web_static.hcl deleted file mode 100644 index 01de23e..0000000 --- a/app/deployment/web_static.hcl +++ /dev/null @@ -1,117 +0,0 @@ -job "web_static" { - datacenters = ["dc1"] - type = "service" - - constraint { - attribute = "${attr.cpu.arch}" - value = "amd64" - } - - group "landing" { - network { - port "deuxfleurs_port" { to = 8080 } - } - - task "server" { - driver = "docker" - config { - image = "superboum/amd64_webpull_pug:v4" - ports = [ "deuxfleurs_port" ] - } - - template { - data = <) (TODO: replace this with two separate files and change template) diff --git a/app/config/configuration/garage/garage.toml b/app/garage/config/garage.toml similarity index 62% rename from app/config/configuration/garage/garage.toml rename to app/garage/config/garage.toml index 51ae81f..4d08cf2 100644 --- a/app/config/configuration/garage/garage.toml +++ b/app/garage/config/garage.toml @@ -8,7 +8,12 @@ rpc_bind_addr = "[::]:3901" consul_host = "consul.service.2.cluster.deuxfleurs.fr:8500" consul_service_name = "garage-rpc" -bootstrap_peers = [ ] +bootstrap_peers = [] + +max_concurrent_rpc_requests = 12 +data_replication_factor = 3 +meta_replication_factor = 3 +meta_epidemic_fanout = 3 [rpc_tls] ca_cert = "/garage/garage-ca.crt" @@ -19,3 +24,7 @@ node_key = "/garage/garage.key" s3_region = "garage" api_bind_addr = "[::]:3900" +[s3_web] +bind_addr = "[::]:3902" +root_domain = ".web.deuxfleurs.fr" +index = "index.html" diff --git a/app/deployment/garage.hcl b/app/garage/deploy/garage.hcl similarity index 94% rename from app/deployment/garage.hcl rename to app/garage/deploy/garage.hcl index d4c7c9e..1be68aa 100644 --- a/app/deployment/garage.hcl +++ b/app/garage/deploy/garage.hcl @@ -12,13 +12,14 @@ job "garage" { network { port "s3" { static = 3900 } port "rpc" { static = 3901 } + port "web" { static = 3902 } } task "server" { driver = "docker" config { advertise_ipv6_address = true - image = "lxpz/garage_amd64:4" + image = "lxpz/garage_amd64:v0.1.1b" network_mode = "host" volumes = [ "/mnt/storage/garage/data:/garage/data", @@ -31,7 +32,7 @@ job "garage" { } template { - data = file("../config/configuration/garage/garage.toml") + data = file("../config/garage.toml") destination = "secrets/garage.toml" } diff --git a/app/build/matrix-synapse/Dockerfile b/app/im/build/matrix-synapse/Dockerfile similarity index 100% rename from app/build/matrix-synapse/Dockerfile rename to app/im/build/matrix-synapse/Dockerfile diff --git a/app/build/matrix-synapse/entrypoint.sh b/app/im/build/matrix-synapse/entrypoint.sh similarity index 100% rename from app/build/matrix-synapse/entrypoint.sh rename to app/im/build/matrix-synapse/entrypoint.sh diff --git a/app/build/riotweb/Dockerfile b/app/im/build/riotweb/Dockerfile similarity index 51% rename from app/build/riotweb/Dockerfile rename to app/im/build/riotweb/Dockerfile index 862e2e5..c768e87 100644 --- a/app/build/riotweb/Dockerfile +++ b/app/im/build/riotweb/Dockerfile @@ -5,9 +5,9 @@ WORKDIR /root RUN apt-get update && \ apt-get install -y wget && \ - wget https://github.com/vector-im/riot-web/releases/download/v${VERSION}/riot-v${VERSION}.tar.gz && \ - tar xf riot-v${VERSION}.tar.gz && \ - mv riot-v${VERSION}/ riot/ + wget https://github.com/vector-im/element-web/releases/download/v${VERSION}/element-v${VERSION}.tar.gz && \ + tar xf element-v${VERSION}.tar.gz && \ + mv element-v${VERSION}/ riot/ FROM superboum/amd64_webserver:v3 COPY --from=builder /root/riot /srv/http diff --git a/app/config/configuration/chat/coturn/turnserver.conf.tpl b/app/im/config/coturn/turnserver.conf.tpl similarity index 100% rename from app/config/configuration/chat/coturn/turnserver.conf.tpl rename to app/im/config/coturn/turnserver.conf.tpl diff --git a/app/config/configuration/chat/easybridge/config.json.tpl b/app/im/config/easybridge/config.json.tpl similarity index 100% rename from app/config/configuration/chat/easybridge/config.json.tpl rename to app/im/config/easybridge/config.json.tpl diff --git a/app/config/configuration/chat/easybridge/registration.yaml.tpl b/app/im/config/easybridge/registration.yaml.tpl similarity index 100% rename from app/config/configuration/chat/easybridge/registration.yaml.tpl rename to app/im/config/easybridge/registration.yaml.tpl diff --git a/app/config/configuration/chat/fb2mx/config.yaml b/app/im/config/fb2mx/config.yaml similarity index 100% rename from app/config/configuration/chat/fb2mx/config.yaml rename to app/im/config/fb2mx/config.yaml diff --git a/app/config/configuration/chat/fb2mx/registration.yaml b/app/im/config/fb2mx/registration.yaml similarity index 100% rename from app/config/configuration/chat/fb2mx/registration.yaml rename to app/im/config/fb2mx/registration.yaml diff --git a/app/config/configuration/chat/riot_web/config.json b/app/im/config/riot_web/config.json similarity index 100% rename from app/config/configuration/chat/riot_web/config.json rename to app/im/config/riot_web/config.json diff --git a/app/config/configuration/chat/synapse/conf.d/report_stats.yaml b/app/im/config/synapse/conf.d/report_stats.yaml similarity index 100% rename from app/config/configuration/chat/synapse/conf.d/report_stats.yaml rename to app/im/config/synapse/conf.d/report_stats.yaml diff --git a/app/config/configuration/chat/synapse/conf.d/server_name.yaml b/app/im/config/synapse/conf.d/server_name.yaml similarity index 100% rename from app/config/configuration/chat/synapse/conf.d/server_name.yaml rename to app/im/config/synapse/conf.d/server_name.yaml diff --git a/app/config/configuration/chat/synapse/homeserver.yaml b/app/im/config/synapse/homeserver.yaml similarity index 100% rename from app/config/configuration/chat/synapse/homeserver.yaml rename to app/im/config/synapse/homeserver.yaml diff --git a/app/config/configuration/chat/synapse/log.yaml b/app/im/config/synapse/log.yaml similarity index 100% rename from app/config/configuration/chat/synapse/log.yaml rename to app/im/config/synapse/log.yaml diff --git a/app/deployment/im.hcl b/app/im/deploy/im.hcl similarity index 88% rename from app/deployment/im.hcl rename to app/im/deploy/im.hcl index cb14144..c9591e6 100644 --- a/app/deployment/im.hcl +++ b/app/im/deploy/im.hcl @@ -15,7 +15,7 @@ job "im" { driver = "docker" config { - image = "particallydone/amd64_synapse:v39" + image = "superboum/amd64_synapse:v40" network_mode = "host" readonly_rootfs = true ports = [ "client_port", "federation_port" ] @@ -35,27 +35,27 @@ job "im" { } template { - data = file("../config/configuration/chat/synapse/homeserver.yaml") + data = file("../config/synapse/homeserver.yaml") destination = "secrets/conf/homeserver.yaml" } template { - data = file("../config/configuration/chat/easybridge/registration.yaml.tpl") + data = file("../config/easybridge/registration.yaml.tpl") destination = "secrets/conf/easybridge_registration.yaml" } template { - data = file("../config/configuration/chat/synapse/log.yaml") + data = file("../config/synapse/log.yaml") destination = "secrets/conf/log.yaml" } template { - data = file("../config/configuration/chat/synapse/conf.d/server_name.yaml") + data = file("../config/synapse/conf.d/server_name.yaml") destination = "secrets/conf/server_name.yaml" } template { - data = file("../config/configuration/chat/synapse/conf.d/report_stats.yaml") + data = file("../config/synapse/conf.d/report_stats.yaml") destination = "secrets/conf/report_stats.yaml" } @@ -152,12 +152,12 @@ job "im" { } template { - data = file("../config/configuration/chat/easybridge/registration.yaml.tpl") + data = file("../config/easybridge/registration.yaml.tpl") destination = "secrets/conf/registration.yaml" } template { - data = file("../config/configuration/chat/easybridge/config.json.tpl") + data = file("../config/easybridge/config.json.tpl") destination = "secrets/conf/config.json" } @@ -220,7 +220,7 @@ job "im" { task "server" { driver = "docker" config { - image = "particallydone/amd64_riotweb:v18" + image = "superboum/amd64_riotweb:v19" ports = [ "web_port" ] volumes = [ "secrets/config.json:/srv/http/config.json" @@ -228,7 +228,7 @@ job "im" { } template { - data = file("../config/configuration/chat/riot_web/config.json") + data = file("../config/riot_web/config.json") destination = "secrets/config.json" } diff --git a/app/im/secrets/chat/coturn/static-auth b/app/im/secrets/chat/coturn/static-auth new file mode 100644 index 0000000..d23be29 --- /dev/null +++ b/app/im/secrets/chat/coturn/static-auth @@ -0,0 +1 @@ +USER cotorn static-auth (what is this?) diff --git a/app/im/secrets/chat/fb2mx/as_token b/app/im/secrets/chat/fb2mx/as_token new file mode 100644 index 0000000..20b76d4 --- /dev/null +++ b/app/im/secrets/chat/fb2mx/as_token @@ -0,0 +1 @@ +USER fb2mx API server token diff --git a/app/im/secrets/chat/fb2mx/db_url b/app/im/secrets/chat/fb2mx/db_url new file mode 100644 index 0000000..f06e265 --- /dev/null +++ b/app/im/secrets/chat/fb2mx/db_url @@ -0,0 +1 @@ +USER fb2mx database URL, format: postgres://username:password@hostname/dbname diff --git a/app/im/secrets/chat/fb2mx/hs_token b/app/im/secrets/chat/fb2mx/hs_token new file mode 100644 index 0000000..8808f8f --- /dev/null +++ b/app/im/secrets/chat/fb2mx/hs_token @@ -0,0 +1 @@ +USER fb2mx homeserver token diff --git a/app/im/secrets/chat/synapse/homeserver.tls.crt b/app/im/secrets/chat/synapse/homeserver.tls.crt new file mode 100644 index 0000000..b696093 --- /dev/null +++ b/app/im/secrets/chat/synapse/homeserver.tls.crt @@ -0,0 +1 @@ +SSL_CERT synapse im.deuxfleurs.fr diff --git a/app/im/secrets/chat/synapse/homeserver.tls.dh b/app/im/secrets/chat/synapse/homeserver.tls.dh new file mode 100644 index 0000000..0231fed --- /dev/null +++ b/app/im/secrets/chat/synapse/homeserver.tls.dh @@ -0,0 +1 @@ +USER_LONG DH parameters for matrix ssl key? how does this work? diff --git a/app/im/secrets/chat/synapse/homeserver.tls.key b/app/im/secrets/chat/synapse/homeserver.tls.key new file mode 100644 index 0000000..feee544 --- /dev/null +++ b/app/im/secrets/chat/synapse/homeserver.tls.key @@ -0,0 +1 @@ +SSL_KEY synapse im.deuxfleurs.fr diff --git a/app/im/secrets/chat/synapse/ldap_binddn b/app/im/secrets/chat/synapse/ldap_binddn new file mode 100644 index 0000000..2631bef --- /dev/null +++ b/app/im/secrets/chat/synapse/ldap_binddn @@ -0,0 +1 @@ +SERVICE_DN matrix Matrix chat server diff --git a/app/im/secrets/chat/synapse/ldap_bindpw b/app/im/secrets/chat/synapse/ldap_bindpw new file mode 100644 index 0000000..ba07446 --- /dev/null +++ b/app/im/secrets/chat/synapse/ldap_bindpw @@ -0,0 +1 @@ +SERVICE_PASSWORD matrix diff --git a/app/im/secrets/chat/synapse/postgres_db b/app/im/secrets/chat/synapse/postgres_db new file mode 100644 index 0000000..74eefa7 --- /dev/null +++ b/app/im/secrets/chat/synapse/postgres_db @@ -0,0 +1 @@ +CONST synapse diff --git a/app/im/secrets/chat/synapse/postgres_pwd b/app/im/secrets/chat/synapse/postgres_pwd new file mode 100644 index 0000000..ba07446 --- /dev/null +++ b/app/im/secrets/chat/synapse/postgres_pwd @@ -0,0 +1 @@ +SERVICE_PASSWORD matrix diff --git a/app/im/secrets/chat/synapse/postgres_user b/app/im/secrets/chat/synapse/postgres_user new file mode 100644 index 0000000..b08e86a --- /dev/null +++ b/app/im/secrets/chat/synapse/postgres_user @@ -0,0 +1 @@ +CONST matrix diff --git a/app/im/secrets/chat/synapse/registration_shared_secret b/app/im/secrets/chat/synapse/registration_shared_secret new file mode 100644 index 0000000..395cccc --- /dev/null +++ b/app/im/secrets/chat/synapse/registration_shared_secret @@ -0,0 +1 @@ +USER Shared secret for homeserver registrations (?) diff --git a/app/build/jitsi-conference-focus/Dockerfile b/app/jitsi/build/jitsi-conference-focus/Dockerfile similarity index 100% rename from app/build/jitsi-conference-focus/Dockerfile rename to app/jitsi/build/jitsi-conference-focus/Dockerfile diff --git a/app/build/jitsi-conference-focus/jicofo b/app/jitsi/build/jitsi-conference-focus/jicofo similarity index 100% rename from app/build/jitsi-conference-focus/jicofo rename to app/jitsi/build/jitsi-conference-focus/jicofo diff --git a/app/build/jitsi-conference-focus/sip-communicator.properties b/app/jitsi/build/jitsi-conference-focus/sip-communicator.properties similarity index 100% rename from app/build/jitsi-conference-focus/sip-communicator.properties rename to app/jitsi/build/jitsi-conference-focus/sip-communicator.properties diff --git a/app/build/jitsi-meet/Dockerfile b/app/jitsi/build/jitsi-meet/Dockerfile similarity index 100% rename from app/build/jitsi-meet/Dockerfile rename to app/jitsi/build/jitsi-meet/Dockerfile diff --git a/app/build/jitsi-meet/config.js b/app/jitsi/build/jitsi-meet/config.js similarity index 100% rename from app/build/jitsi-meet/config.js rename to app/jitsi/build/jitsi-meet/config.js diff --git a/app/build/jitsi-meet/entrypoint.sh b/app/jitsi/build/jitsi-meet/entrypoint.sh similarity index 100% rename from app/build/jitsi-meet/entrypoint.sh rename to app/jitsi/build/jitsi-meet/entrypoint.sh diff --git a/app/build/jitsi-videobridge/Dockerfile b/app/jitsi/build/jitsi-videobridge/Dockerfile similarity index 100% rename from app/build/jitsi-videobridge/Dockerfile rename to app/jitsi/build/jitsi-videobridge/Dockerfile diff --git a/app/build/jitsi-videobridge/jvb_run b/app/jitsi/build/jitsi-videobridge/jvb_run similarity index 100% rename from app/build/jitsi-videobridge/jvb_run rename to app/jitsi/build/jitsi-videobridge/jvb_run diff --git a/app/build/jitsi-xmpp/Dockerfile b/app/jitsi/build/jitsi-xmpp/Dockerfile similarity index 100% rename from app/build/jitsi-xmpp/Dockerfile rename to app/jitsi/build/jitsi-xmpp/Dockerfile diff --git a/app/build/jitsi-xmpp/external_components.cfg.lua b/app/jitsi/build/jitsi-xmpp/external_components.cfg.lua similarity index 100% rename from app/build/jitsi-xmpp/external_components.cfg.lua rename to app/jitsi/build/jitsi-xmpp/external_components.cfg.lua diff --git a/app/build/jitsi-xmpp/xmpp_conf b/app/jitsi/build/jitsi-xmpp/xmpp_conf similarity index 100% rename from app/build/jitsi-xmpp/xmpp_conf rename to app/jitsi/build/jitsi-xmpp/xmpp_conf diff --git a/app/build/jitsi-xmpp/xmpp_gen b/app/jitsi/build/jitsi-xmpp/xmpp_gen similarity index 100% rename from app/build/jitsi-xmpp/xmpp_gen rename to app/jitsi/build/jitsi-xmpp/xmpp_gen diff --git a/app/build/jitsi-xmpp/xmpp_run b/app/jitsi/build/jitsi-xmpp/xmpp_run similarity index 100% rename from app/build/jitsi-xmpp/xmpp_run rename to app/jitsi/build/jitsi-xmpp/xmpp_run diff --git a/app/config/configuration/jitsi/global_env.tpl b/app/jitsi/config/global_env.tpl similarity index 100% rename from app/config/configuration/jitsi/global_env.tpl rename to app/jitsi/config/global_env.tpl diff --git a/app/deployment/jitsi.hcl b/app/jitsi/deploy/jitsi.hcl similarity index 92% rename from app/deployment/jitsi.hcl rename to app/jitsi/deploy/jitsi.hcl index 9c1edd5..852e1e6 100644 --- a/app/deployment/jitsi.hcl +++ b/app/jitsi/deploy/jitsi.hcl @@ -27,7 +27,7 @@ job "jitsi" { } template { - data = file("../config/configuration/jitsi/global_env.tpl") + data = file("../config/global_env.tpl") destination = "secrets/global_env" env = true } @@ -94,13 +94,13 @@ job "jitsi" { task "front" { driver = "docker" config { - image = "superboum/amd64_jitsi_meet:v2" + image = "superboum/amd64_jitsi_meet:v3" network_mode = "host" ports = [ "https_port" ] } template { - data = file("../config/configuration/jitsi/global_env.tpl") + data = file("../config/global_env.tpl") destination = "secrets/global_env" env = true } @@ -148,12 +148,12 @@ job "jitsi" { task "jicofo" { driver = "docker" config { - image = "superboum/amd64_jitsi_conference_focus:v5" + image = "superboum/amd64_jitsi_conference_focus:v6" network_mode = "host" } template { - data = file("../config/configuration/jitsi/global_env.tpl") + data = file("../config/global_env.tpl") destination = "secrets/global_env" env = true } @@ -178,7 +178,7 @@ job "jitsi" { task "videobridge" { driver = "docker" config { - image = "superboum/amd64_jitsi_videobridge:v15" + image = "superboum/amd64_jitsi_videobridge:v16" network_mode = "host" ports = [ "video1_port", "video2_port" ] ulimit { @@ -194,7 +194,7 @@ job "jitsi" { } template { - data = file("../config/configuration/jitsi/global_env.tpl") + data = file("../config/global_env.tpl") destination = "secrets/global_env" env = true } diff --git a/app/integration/jitsi/01_gen_certs.yml b/app/jitsi/integratio/01_gen_certs.yml similarity index 100% rename from app/integration/jitsi/01_gen_certs.yml rename to app/jitsi/integratio/01_gen_certs.yml diff --git a/app/integration/jitsi/02_run.yml b/app/jitsi/integratio/02_run.yml similarity index 100% rename from app/integration/jitsi/02_run.yml rename to app/jitsi/integratio/02_run.yml diff --git a/app/integration/jitsi/README.md b/app/jitsi/integratio/README.md similarity index 100% rename from app/integration/jitsi/README.md rename to app/jitsi/integratio/README.md diff --git a/app/integration/jitsi/dev.env b/app/jitsi/integratio/dev.env similarity index 100% rename from app/integration/jitsi/dev.env rename to app/jitsi/integratio/dev.env diff --git a/app/integration/jitsi/jitsi-certs/.gitignore b/app/jitsi/integratio/jitsi-certs/.gitignore similarity index 100% rename from app/integration/jitsi/jitsi-certs/.gitignore rename to app/jitsi/integratio/jitsi-certs/.gitignore diff --git a/app/jitsi/secrets/jitsi/auth.jitsi.deuxfleurs.fr.crt b/app/jitsi/secrets/jitsi/auth.jitsi.deuxfleurs.fr.crt new file mode 100644 index 0000000..f2c4d4b --- /dev/null +++ b/app/jitsi/secrets/jitsi/auth.jitsi.deuxfleurs.fr.crt @@ -0,0 +1 @@ +SSL_CERT jitsi_auth autj.jitsi.deuxfleurs.fr diff --git a/app/jitsi/secrets/jitsi/auth.jitsi.deuxfleurs.fr.key b/app/jitsi/secrets/jitsi/auth.jitsi.deuxfleurs.fr.key new file mode 100644 index 0000000..4a332f8 --- /dev/null +++ b/app/jitsi/secrets/jitsi/auth.jitsi.deuxfleurs.fr.key @@ -0,0 +1 @@ +SSL_KEY jitsi_auth autj.jitsi.deuxfleurs.fr diff --git a/app/jitsi/secrets/jitsi/jitsi.deuxfleurs.fr.crt b/app/jitsi/secrets/jitsi/jitsi.deuxfleurs.fr.crt new file mode 100644 index 0000000..32750d3 --- /dev/null +++ b/app/jitsi/secrets/jitsi/jitsi.deuxfleurs.fr.crt @@ -0,0 +1 @@ +SSL_CERT jitsi jitsi.deuxfleurs.fr diff --git a/app/jitsi/secrets/jitsi/jitsi.deuxfleurs.fr.key b/app/jitsi/secrets/jitsi/jitsi.deuxfleurs.fr.key new file mode 100644 index 0000000..7676132 --- /dev/null +++ b/app/jitsi/secrets/jitsi/jitsi.deuxfleurs.fr.key @@ -0,0 +1 @@ +SSL_KEY jitsi diff --git a/app/build/nextcloud/Dockerfile b/app/nextcloud/build/nextcloud/Dockerfile similarity index 100% rename from app/build/nextcloud/Dockerfile rename to app/nextcloud/build/nextcloud/Dockerfile diff --git a/app/build/nextcloud/container-setup.sh b/app/nextcloud/build/nextcloud/container-setup.sh similarity index 100% rename from app/build/nextcloud/container-setup.sh rename to app/nextcloud/build/nextcloud/container-setup.sh diff --git a/app/build/nextcloud/entrypoint.sh b/app/nextcloud/build/nextcloud/entrypoint.sh similarity index 100% rename from app/build/nextcloud/entrypoint.sh rename to app/nextcloud/build/nextcloud/entrypoint.sh diff --git a/app/config/configuration/nextcloud/config.php.tpl b/app/nextcloud/config/config.php.tpl similarity index 100% rename from app/config/configuration/nextcloud/config.php.tpl rename to app/nextcloud/config/config.php.tpl diff --git a/app/deployment/nextcloud.hcl b/app/nextcloud/deploy/nextcloud.hcl similarity index 94% rename from app/deployment/nextcloud.hcl rename to app/nextcloud/deploy/nextcloud.hcl index 8eed7d9..8852787 100644 --- a/app/deployment/nextcloud.hcl +++ b/app/nextcloud/deploy/nextcloud.hcl @@ -28,7 +28,7 @@ job "nextcloud" { } template { - data = file("../config/configuration/nextcloud/config.php.tpl") + data = file("../config/config.php.tpl") destination = "secrets/config.php" } diff --git a/app/nextcloud/integration/README.md b/app/nextcloud/integration/README.md new file mode 100644 index 0000000..3d49768 --- /dev/null +++ b/app/nextcloud/integration/README.md @@ -0,0 +1,20 @@ +Install Owncloud CLI: + +php ./occ \ + --no-interaction \ + --verbose \ + maintenance:install \ + --database pgsql \ + --database-name nextcloud \ + --database-host postgres \ + --database-user nextcloud \ + --database-pass nextcloud \ + --admin-user nextcloud \ + --admin-pass nextcloud \ + --admin-email coucou@deuxfleurs.fr + +Official image entrypoint: + +https://github.com/nextcloud/docker/blob/master/20.0/fpm/entrypoint.sh + + diff --git a/app/integration/plume/bottin.json b/app/nextcloud/integration/bottin.json similarity index 100% rename from app/integration/plume/bottin.json rename to app/nextcloud/integration/bottin.json diff --git a/app/nextcloud/integration/docker-compose.yml b/app/nextcloud/integration/docker-compose.yml new file mode 100644 index 0000000..7ba090b --- /dev/null +++ b/app/nextcloud/integration/docker-compose.yml @@ -0,0 +1,27 @@ +version: '3.4' +services: + php: + image: lxpz/deuxfleurs_nextcloud_amd64:8 + depends_on: + - bottin + - postgres + ports: + - "80:80" + + postgres: + image: postgres:9.6.19 + environment: + - POSTGRES_DB=nextcloud + - POSTGRES_USER=nextcloud + - POSTGRES_PASSWORD=nextcloud + + bottin: + image: lxpz/bottin_amd64:14 + depends_on: + - consul + volumes: + - ./bottin.json:/config.json + + consul: + image: consul:1.8.4 + diff --git a/app/deployment/platoo.hcl b/app/platoo/deploy/platoo.hcl similarity index 100% rename from app/deployment/platoo.hcl rename to app/platoo/deploy/platoo.hcl diff --git a/app/platoo/secrets/platoo/bddpw b/app/platoo/secrets/platoo/bddpw new file mode 100644 index 0000000..1c9d86e --- /dev/null +++ b/app/platoo/secrets/platoo/bddpw @@ -0,0 +1 @@ +SERVICE_PASSWORD platoo diff --git a/app/build/plume/Dockerfile b/app/plume/build/plume/Dockerfile similarity index 94% rename from app/build/plume/Dockerfile rename to app/plume/build/plume/Dockerfile index 9199e1e..4e05424 100644 --- a/app/build/plume/Dockerfile +++ b/app/plume/build/plume/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update && \ ARG VERSION WORKDIR /opt -RUN git clone -n https://git.deuxfleurs.fr/Deuxfleurs/plume.git +RUN git clone -n https://git.joinplu.me/Plume/Plume.git plume WORKDIR /opt/plume RUN git checkout ${VERSION} @@ -52,5 +52,3 @@ COPY --from=builder /usr/local/cargo/bin/plume /usr/local/bin/ COPY plm-start /usr/local/bin/ CMD ["plm-start"] - -EXPOSE 7878 diff --git a/app/build/plume/README.md b/app/plume/build/plume/README.md similarity index 100% rename from app/build/plume/README.md rename to app/plume/build/plume/README.md diff --git a/app/build/plume/plm-start b/app/plume/build/plume/plm-start similarity index 100% rename from app/build/plume/plm-start rename to app/plume/build/plume/plm-start diff --git a/app/config/configuration/plume/app.env b/app/plume/config/app.env similarity index 100% rename from app/config/configuration/plume/app.env rename to app/plume/config/app.env diff --git a/app/deployment/plume.hcl b/app/plume/deploy/plume.hcl similarity index 93% rename from app/deployment/plume.hcl rename to app/plume/deploy/plume.hcl index 59ec28a..0a82c57 100644 --- a/app/deployment/plume.hcl +++ b/app/plume/deploy/plume.hcl @@ -17,7 +17,7 @@ job "plume" { task "plume" { driver = "docker" config { - image = "superboum/plume:v1" + image = "superboum/plume:v2" network_mode = "host" ports = [ "web_port" ] #command = "cat" @@ -29,7 +29,7 @@ job "plume" { } template { - data = file("../config/configuration/plume/app.env") + data = file("../config/app.env") destination = "secrets/app.env" env = true } diff --git a/app/plume/integration/bottin.json b/app/plume/integration/bottin.json new file mode 100644 index 0000000..a970762 --- /dev/null +++ b/app/plume/integration/bottin.json @@ -0,0 +1,31 @@ +{ + "suffix": "dc=deuxfleurs,dc=fr", + "bind": "0.0.0.0:389", + "consul_host": "http://consul:8500", + "log_level": "debug", + "acl": [ + "*,dc=deuxfleurs,dc=fr::read:*:* !userpassword", + "*::read modify:SELF:*", + "ANONYMOUS::bind:*,ou=users,dc=deuxfleurs,dc=fr:", + "ANONYMOUS::bind:cn=admin,dc=deuxfleurs,dc=fr:", + "*,ou=services,ou=users,dc=deuxfleurs,dc=fr::bind:*,ou=users,dc=deuxfleurs,dc=fr:*", + "*,ou=services,ou=users,dc=deuxfleurs,dc=fr::read:*:*", + + "*:cn=asso_deuxfleurs,ou=groups,dc=deuxfleurs,dc=fr:add:*,ou=invitations,dc=deuxfleurs,dc=fr:*", + "ANONYMOUS::bind:*,ou=invitations,dc=deuxfleurs,dc=fr:", + "*,ou=invitations,dc=deuxfleurs,dc=fr::delete:SELF:*", + + "*:cn=asso_deuxfleurs,ou=groups,dc=deuxfleurs,dc=fr:add:*,ou=users,dc=deuxfleurs,dc=fr:*", + "*,ou=invitations,dc=deuxfleurs,dc=fr::add:*,ou=users,dc=deuxfleurs,dc=fr:*", + + "*:cn=asso_deuxfleurs,ou=groups,dc=deuxfleurs,dc=fr:modifyAdd:cn=email,ou=groups,dc=deuxfleurs,dc=fr:*", + "*,ou=invitations,dc=deuxfleurs,dc=fr::modifyAdd:cn=email,ou=groups,dc=deuxfleurs,dc=fr:*", + "*:cn=asso_deuxfleurs,ou=groups,dc=deuxfleurs,dc=fr:modifyAdd:cn=seafile,ou=groups,dc=deuxfleurs,dc=fr:*", + "*,ou=invitations,dc=deuxfleurs,dc=fr::modifyAdd:cn=seafile,ou=groups,dc=deuxfleurs,dc=fr:*", + "*:cn=asso_deuxfleurs,ou=groups,dc=deuxfleurs,dc=fr:modifyAdd:cn=nextcloud,ou=groups,dc=deuxfleurs,dc=fr:*", + "*,ou=invitations,dc=deuxfleurs,dc=fr::modifyAdd:cn=seafile,ou=nextcloud,dc=deuxfleurs,dc=fr:*", + + "cn=admin,dc=deuxfleurs,dc=fr::read add modify delete:*:*", + "*:cn=admin,ou=groups,dc=deuxfleurs,dc=fr:read add modify delete:*:*" + ] +} diff --git a/app/integration/plume/docker-compose.yml b/app/plume/integration/docker-compose.yml similarity index 100% rename from app/integration/plume/docker-compose.yml rename to app/plume/integration/docker-compose.yml diff --git a/app/integration/plume/plume.env b/app/plume/integration/plume.env similarity index 100% rename from app/integration/plume/plume.env rename to app/plume/integration/plume.env diff --git a/app/plume/secrets/plume/pgsql_pw b/app/plume/secrets/plume/pgsql_pw new file mode 100644 index 0000000..978be54 --- /dev/null +++ b/app/plume/secrets/plume/pgsql_pw @@ -0,0 +1 @@ +CMD openssl rand -base64 32 diff --git a/app/plume/secrets/plume/secret_key b/app/plume/secrets/plume/secret_key new file mode 100644 index 0000000..978be54 --- /dev/null +++ b/app/plume/secrets/plume/secret_key @@ -0,0 +1 @@ +CMD openssl rand -base64 32 diff --git a/app/build/postgres/Dockerfile b/app/postgres/build/postgres/Dockerfile similarity index 100% rename from app/build/postgres/Dockerfile rename to app/postgres/build/postgres/Dockerfile diff --git a/app/build/postgres/README.md b/app/postgres/build/postgres/README.md similarity index 100% rename from app/build/postgres/README.md rename to app/postgres/build/postgres/README.md diff --git a/app/build/postgres/postgresql.conf b/app/postgres/build/postgres/postgresql.conf similarity index 100% rename from app/build/postgres/postgresql.conf rename to app/postgres/build/postgres/postgresql.conf diff --git a/app/build/postgres/start.sh b/app/postgres/build/postgres/start.sh similarity index 100% rename from app/build/postgres/start.sh rename to app/postgres/build/postgres/start.sh diff --git a/app/config/configuration/postgres/keeper/env.tpl b/app/postgres/config/keeper/env.tpl similarity index 100% rename from app/config/configuration/postgres/keeper/env.tpl rename to app/postgres/config/keeper/env.tpl diff --git a/app/deployment/postgres.hcl b/app/postgres/deploy/postgres.hcl similarity index 97% rename from app/deployment/postgres.hcl rename to app/postgres/deploy/postgres.hcl index 388c65e..f5eec51 100644 --- a/app/deployment/postgres.hcl +++ b/app/postgres/deploy/postgres.hcl @@ -101,7 +101,7 @@ job "postgres" { } template { - data = file("../config/configuration/postgres/keeper/env.tpl") + data = file("../config/keeper/env.tpl") destination = "secrets/env" env = true } diff --git a/app/postgres/secrets/postgres/keeper/pg_repl_pwd b/app/postgres/secrets/postgres/keeper/pg_repl_pwd new file mode 100644 index 0000000..ae0c229 --- /dev/null +++ b/app/postgres/secrets/postgres/keeper/pg_repl_pwd @@ -0,0 +1 @@ +SERVICE_PASSWORD replicator diff --git a/app/postgres/secrets/postgres/keeper/pg_repl_username b/app/postgres/secrets/postgres/keeper/pg_repl_username new file mode 100644 index 0000000..58e6e46 --- /dev/null +++ b/app/postgres/secrets/postgres/keeper/pg_repl_username @@ -0,0 +1 @@ +CONST replicator diff --git a/app/postgres/secrets/postgres/keeper/pg_su_pwd b/app/postgres/secrets/postgres/keeper/pg_su_pwd new file mode 100644 index 0000000..907e2b8 --- /dev/null +++ b/app/postgres/secrets/postgres/keeper/pg_su_pwd @@ -0,0 +1 @@ +USER postgres superuser password diff --git a/app/requirements.txt b/app/requirements.txt new file mode 100644 index 0000000..7874d93 --- /dev/null +++ b/app/requirements.txt @@ -0,0 +1,3 @@ +python-consul==1.1.0 +python-ldap==3.3.1 +passlib==1.7.4 diff --git a/app/deployment/science.hcl b/app/science/deploy/science.hcl similarity index 100% rename from app/deployment/science.hcl rename to app/science/deploy/science.hcl diff --git a/app/build/mariadb/60-disable-dialog.cnf b/app/seafile/build/mariadb/60-disable-dialog.cnf similarity index 100% rename from app/build/mariadb/60-disable-dialog.cnf rename to app/seafile/build/mariadb/60-disable-dialog.cnf diff --git a/app/build/mariadb/60-ldap.cnf b/app/seafile/build/mariadb/60-ldap.cnf similarity index 100% rename from app/build/mariadb/60-ldap.cnf rename to app/seafile/build/mariadb/60-ldap.cnf diff --git a/app/build/mariadb/60-remote.cnf b/app/seafile/build/mariadb/60-remote.cnf similarity index 100% rename from app/build/mariadb/60-remote.cnf rename to app/seafile/build/mariadb/60-remote.cnf diff --git a/app/build/mariadb/Dockerfile b/app/seafile/build/mariadb/Dockerfile similarity index 100% rename from app/build/mariadb/Dockerfile rename to app/seafile/build/mariadb/Dockerfile diff --git a/app/build/mariadb/README.md b/app/seafile/build/mariadb/README.md similarity index 100% rename from app/build/mariadb/README.md rename to app/seafile/build/mariadb/README.md diff --git a/app/build/mariadb/entrypoint.sh b/app/seafile/build/mariadb/entrypoint.sh similarity index 100% rename from app/build/mariadb/entrypoint.sh rename to app/seafile/build/mariadb/entrypoint.sh diff --git a/app/build/mariadb/nsswitch.conf b/app/seafile/build/mariadb/nsswitch.conf similarity index 100% rename from app/build/mariadb/nsswitch.conf rename to app/seafile/build/mariadb/nsswitch.conf diff --git a/app/build/mariadb/pam-mariadb b/app/seafile/build/mariadb/pam-mariadb similarity index 100% rename from app/build/mariadb/pam-mariadb rename to app/seafile/build/mariadb/pam-mariadb diff --git a/app/build/seafile/Dockerfile b/app/seafile/build/seafile/Dockerfile similarity index 100% rename from app/build/seafile/Dockerfile rename to app/seafile/build/seafile/Dockerfile diff --git a/app/build/seafile/README.md b/app/seafile/build/seafile/README.md similarity index 100% rename from app/build/seafile/README.md rename to app/seafile/build/seafile/README.md diff --git a/app/build/seafile/seadocker b/app/seafile/build/seafile/seadocker similarity index 100% rename from app/build/seafile/seadocker rename to app/seafile/build/seafile/seadocker diff --git a/app/build/seafile/seaenv b/app/seafile/build/seafile/seaenv similarity index 100% rename from app/build/seafile/seaenv rename to app/seafile/build/seafile/seaenv diff --git a/app/config/configuration/seafile/ccnet/seafile.ini b/app/seafile/config/ccnet/seafile.ini similarity index 100% rename from app/config/configuration/seafile/ccnet/seafile.ini rename to app/seafile/config/ccnet/seafile.ini diff --git a/app/config/configuration/seafile/conf/ccnet.conf.tpl b/app/seafile/config/conf/ccnet.conf.tpl similarity index 100% rename from app/config/configuration/seafile/conf/ccnet.conf.tpl rename to app/seafile/config/conf/ccnet.conf.tpl diff --git a/app/config/configuration/seafile/conf/gunicorn.conf b/app/seafile/config/conf/gunicorn.conf similarity index 100% rename from app/config/configuration/seafile/conf/gunicorn.conf rename to app/seafile/config/conf/gunicorn.conf diff --git a/app/config/configuration/seafile/conf/seafdav.conf b/app/seafile/config/conf/seafdav.conf similarity index 100% rename from app/config/configuration/seafile/conf/seafdav.conf rename to app/seafile/config/conf/seafdav.conf diff --git a/app/config/configuration/seafile/conf/seafile.conf.tpl b/app/seafile/config/conf/seafile.conf.tpl similarity index 100% rename from app/config/configuration/seafile/conf/seafile.conf.tpl rename to app/seafile/config/conf/seafile.conf.tpl diff --git a/app/config/configuration/seafile/conf/seahub_settings.py.tpl b/app/seafile/config/conf/seahub_settings.py.tpl similarity index 100% rename from app/config/configuration/seafile/conf/seahub_settings.py.tpl rename to app/seafile/config/conf/seahub_settings.py.tpl diff --git a/app/config/configuration/mariadb/main/env.tpl b/app/seafile/config/mariadb/main/env.tpl similarity index 100% rename from app/config/configuration/mariadb/main/env.tpl rename to app/seafile/config/mariadb/main/env.tpl diff --git a/app/deployment/seafile.hcl b/app/seafile/deploy/seafile.hcl similarity index 90% rename from app/deployment/seafile.hcl rename to app/seafile/deploy/seafile.hcl index 3af7db3..d8488d2 100644 --- a/app/deployment/seafile.hcl +++ b/app/seafile/deploy/seafile.hcl @@ -35,7 +35,7 @@ job "seafile" { } template { - data = file("../config/configuration/mariadb/main/env.tpl") + data = file("../config/mariadb/main/env.tpl") destination = "secrets/env" env = true } @@ -179,30 +179,30 @@ job "seafile" { } template { - data = file("../config/configuration/seafile/conf/ccnet.conf.tpl") + data = file("../config/conf/ccnet.conf.tpl") destination = "secrets/conf/ccnet.conf" } template { - data = file("../config/configuration/seafile/conf/seafile.conf.tpl") + data = file("../config/conf/seafile.conf.tpl") destination = "secrets/conf/seafile.conf" } template { - data = file("../config/configuration/seafile/conf/seahub_settings.py.tpl") + data = file("../config/conf/seahub_settings.py.tpl") destination = "secrets/conf/seahub_settings.py" } template { - data = file("../config/configuration/seafile/ccnet/seafile.ini") + data = file("../config/ccnet/seafile.ini") destination = "secrets/ccnet/seafile.ini" } template { - data = file("../config/configuration/seafile/conf/seafdav.conf") + data = file("../config/conf/seafdav.conf") destination = "secrets/conf/seafdav.conf" } template { - data = file("../config/configuration/seafile/conf/gunicorn.conf") + data = file("../config/conf/gunicorn.conf") destination = "secrets/conf/gunicorn.conf" } diff --git a/app/seafile/secrets/mariadb/main/ldap_binddn b/app/seafile/secrets/mariadb/main/ldap_binddn new file mode 100644 index 0000000..e77ff39 --- /dev/null +++ b/app/seafile/secrets/mariadb/main/ldap_binddn @@ -0,0 +1 @@ +SERVICE_DN mysql MySQL/MariaDB database diff --git a/app/seafile/secrets/mariadb/main/ldap_bindpwd b/app/seafile/secrets/mariadb/main/ldap_bindpwd new file mode 100644 index 0000000..c29f983 --- /dev/null +++ b/app/seafile/secrets/mariadb/main/ldap_bindpwd @@ -0,0 +1 @@ +SERVICE_PASSWORD mysql diff --git a/app/seafile/secrets/mariadb/main/mysql_pwd b/app/seafile/secrets/mariadb/main/mysql_pwd new file mode 100644 index 0000000..ae7fd75 --- /dev/null +++ b/app/seafile/secrets/mariadb/main/mysql_pwd @@ -0,0 +1 @@ +USER mysql_pwd (what is this?) diff --git a/app/seafile/secrets/seafile/conf/mykey.peer b/app/seafile/secrets/seafile/conf/mykey.peer new file mode 100644 index 0000000..12f0e5f --- /dev/null +++ b/app/seafile/secrets/seafile/conf/mykey.peer @@ -0,0 +1 @@ +USER Seafile peer key diff --git a/app/secretmgr.py b/app/secretmgr.py new file mode 100755 index 0000000..6af6d13 --- /dev/null +++ b/app/secretmgr.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 + +# DEPENDENCY: python-consul +import consul + +# DEPENDENCY: python-ldap +import ldap + +# DEPENDENCY: passlib +from passlib.hash import ldap_salted_sha1 + +import os +import sys +import glob +import subprocess +import getpass +import base64 +from secrets import token_bytes + + +""" +TODO: this will be a utility to handle secrets in the Consul database +for the various components of the Deuxfleurs infrastructure + +Functionnalities: +- check that secrets are correctly configured +- help user fill in secrets +- create LDAP service users and fill in corresponding secrets +- maybe one day: manage SSL certificates and keys + +It uses files placed in /secrets/* to know what secrets +it should handle. These secret files contain directives for what to do +about these secrets. + +Example directives: + +USER +(a secret that must be filled in by the user) + +USER_LONG +(the same, indicates that the secret fits on several lines) + +CMD +(a secret that is generated by running this command) + +CONST +(the secret has a constant value set here) + +CONST_LONG + +(same) + +SERVICE_DN +(the LDAP DN of a service user) + +SERVICE_PASSWORD +(the LDAP password for the corresponding service user) + +SSL_CERT +(a SSL domain for the given domains) + +SSL_KEY +(the SSL key going with corresponding certificate) + +RSA_PUBLIC_KEY +(a public RSA key) + +RSA_PRIVATE_KEY +(the corresponding private RSA key) +""" + + +# Parameters +LDAP_URL = "ldap://localhost:1389" +SERVICE_DN_SUFFIX = "ou=services,ou=users,dc=deuxfleurs,dc=fr" +consul_server = consul.Consul() + + +# ---- + +USER = "USER" +USER_LONG = "USER_LONG" +CMD = "CMD" +CONST = "CONST" +CONST_LONG = "CONST_LONG" +SERVICE_DN = "SERVICE_DN" +SERVICE_PASSWORD = "SERVICE_PASSWORD" +SSL_CERT = "SSL_CERT" +SSL_KEY = "SSL_KEY" +RSA_PUBLIC_KEY = "RSA_PUBLIC_KEY" +RSA_PRIVATE_KEY = "RSA_PRIVATE_KEY" + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + +def read_secret(key, file_path): + lines = [l.strip() for l in open(file_path, "r")] + l0 = lines[0].split(" ") + stype = l0[0] + secret = {"type": stype, "key": key} + if stype in [USER, USER_LONG]: + secret["desc"] = " ".join(l0[1:]) + elif stype == CMD: + secret["cmd"] = " ".join(l0[1:]) + elif stype == CONST: + secret["value"] = " ".join(l0[1:]) + elif stype == CONST_LONG: + secret["value"] = "\n".join(lines[1:]) + elif stype in [SERVICE_DN, SERVICE_PASSWORD]: + secret["service"] = l0[1] + if stype == SERVICE_DN: + secret["service_desc"] = " ".join(l0[2:]) + elif stype in [SSL_CERT, SSL_KEY]: + secret["cert_name"] = l0[1] + if stype == SSL_CERT: + secret["cert_domains"] = l0[2:] + elif stype in [RSA_PUBLIC_KEY, RSA_PRIVATE_KEY]: + secret["key_name"] = l0[1] + if stype == RSA_PUBLIC_KEY: + secret["key_desc"] = " ".join(l0[2:]) + else: + print(bcolors.FAIL, "ERROR:", bcolors.ENDC, "Invalid secret type", stype, "in", file_path) + sys.exit(-1) + + return secret + +def read_secrets(module_list): + secrets = {} + for mod in module_list: + for file_path in glob.glob(mod.strip('/') + "/secrets/**", recursive=True): + if os.path.isfile(file_path): + key = '/'.join(file_path.split("/")[1:]) + secrets[key] = read_secret(key, file_path) + return secrets + +def get_secrets_services(secrets): + services = {} + for key, secret in secrets.items(): + if secret["type"] not in [SERVICE_DN, SERVICE_PASSWORD]: + continue + svc = secret["service"] + print(svc, "@", key, bcolors.OKCYAN, "...", bcolors.ENDC) + if svc not in services: + services[svc] = { + "dn": "cn=%s,%s"%(svc, SERVICE_DN_SUFFIX), + "pass": None, + "dn_at": [], + "pass_at": [], + } + if secret["type"] == SERVICE_DN: + services[svc]["dn_at"].append(key) + services[svc]["desc"] = secret["service_desc"] + + if secret["type"] == SERVICE_PASSWORD: + services[svc]["pass_at"].append(key) + _, data = consul_server.kv.get(key) + if data is not None: + if services[svc]["pass"] is None: + services[svc]["pass"] = data["Value"].decode('ascii').strip() + + return services + +ldap_admin_conn = None +def get_ldap_admin_conn(): + global ldap_admin_conn + if ldap_admin_conn is None: + ldap_admin_conn = ldap.initialize(LDAP_URL) + ldap_user = input("LDAP admin user (full DN, please!): ") + ldap_pass = getpass.getpass("LDAP admin password: ") + ldap_admin_conn.simple_bind_s(ldap_user, ldap_pass) + return ldap_admin_conn + +# ---- CHECK COMMAND ---- + +def check_secrets(module_list): + secrets = read_secrets(module_list) + print("Found", len(secrets), "secrets to check") + print() + + check_secrets_presence(secrets) + check_secrets_services(secrets) + +def check_secrets_presence(secrets): + print("Checking secrets presence...") + for key in secrets.keys(): + _, data = consul_server.kv.get(key) + if data is None: + print(key, bcolors.FAIL, "x", bcolors.ENDC) + else: + print(key, bcolors.OKGREEN, "✓", bcolors.ENDC) + print() + +def check_secrets_services(secrets): + print("Checking secrets for LDAP service users...") + services = get_secrets_services(secrets) + + for svc_name, svc in services.items(): + for dn_key in svc["dn_at"]: + _, data = consul_server.kv.get(dn_key) + if data is not None: + got_val = data["Value"].decode('ascii').strip() + if got_val != svc["dn"]: + print(svc_name, "wrong DN at", dn_key, bcolors.FAIL, "x", bcolors.ENDC) + print("got:", got_val, "instead of:", svc["dn"]) + + if svc["pass"] is None: + print(svc_name, bcolors.FAIL, "no password stored", bcolors.ENDC) + else: + for pass_key in svc["pass_at"]: + _, data = consul_server.kv.get(pass_key) + if data is not None: + got_val = data["Value"].decode('ascii').strip() + if got_val != svc["pass"]: + print(svc_name, "wrong pass at", dn_key, bcolors.FAIL, "x", bcolors.ENDC) + + l = ldap.initialize(LDAP_URL) + try: + l.simple_bind_s(svc["dn"], svc["pass"]) + print(svc_name, bcolors.OKGREEN, "✓", bcolors.ENDC) + except Exception as e: + print(svc_name, bcolors.FAIL, e, bcolors.ENDC) + print() + + +# ---- GEN COMMAND ---- + +def gen_secrets(module_list, regen): + secrets = read_secrets(module_list) + print("Found", len(secrets), "secrets to check and maybe generate") + print() + + gen_secrets_base(secrets, regen) + gen_secrets_services(secrets, regen) + + check_secrets_presence(secrets) + check_secrets_services(secrets) + +def gen_secrets_base(secrets, regen): + print("Filling in user secrets and cmd secrets...") + + for key, secret in secrets.items(): + _, data = consul_server.kv.get(key) + if data is not None and not regen: + continue + + if secret["type"] == USER: + print("----") + print(key) + print("Description:", secret["desc"]) + print("Enter value for secret, or ^C to skip:") + try: + val = input().strip() + consul_server.kv.put(key, val) + print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) + except KeyboardInterrupt: + print(bcolors.WARNING, "Skipped.", bcolors.ENDC) + + if secret["type"] == USER_LONG: + print("----") + print(key) + print("Description:", secret["desc"]) + print("Enter value for secret, or ^C to skip:") + print("THIS IS A LONG VALUE, ENTER SEVERAL LINES AND FINISH WITH A LINE CONTAINING A SINGLE .") + try: + lines = [] + while True: + line = input().strip() + if line == ".": + break + vals.append(line) + val = "\n".join(lines) + consul_server.kv.put(key, val) + print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) + except KeyboardInterrupt: + print(bcolors.WARNING, "Skipped.", bcolors.ENDC) + + if secret["type"] in [CONST, CONST_LONG]: + print("----") + print(key) + print("Resetting to constant value.") + consul_server.kv.put(key, secret["value"]) + print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) + + if secret["type"] == CMD: + print("----") + print(key) + print("Executing command:", secret["cmd"]) + val = subprocess.check_output(["sh", "-c", secret["cmd"]]) + consul_server.kv.put(key, val) + print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) + + print() + +def gen_secrets_services(secrets, regen): + print("Generating LDAP service accounts...") + services = get_secrets_services(secrets) + + for svc_name, svc in services.items(): + print("----") + print("Service:", svc_name) + print("Description:", svc["desc"]) + + for dn_key in svc["dn_at"]: + _, data = consul_server.kv.get(dn_key) + if data is None or data["Value"].decode('ascii').strip() != svc["dn"]: + print(bcolors.OKCYAN, "Setting DN", bcolors.ENDC, "at", dn_key) + consul_server.kv.put(dn_key, svc["dn"]) + + if svc["pass"] is None or regen: + print(bcolors.OKCYAN, "Generating new password", bcolors.ENDC) + svc["pass"] = base64.urlsafe_b64encode(token_bytes(12)).decode('ascii') + + l = ldap.initialize(LDAP_URL) + try: + l.simple_bind_s(svc["dn"], svc["pass"]) + except: + fix_service_user(svc) + + for pass_key in svc["pass_at"]: + _, data = consul_server.kv.get(pass_key) + if data is None or data["Value"].decode('ascii').strip() != svc["pass"]: + print(bcolors.OKCYAN, "Setting password", bcolors.ENDC, "at", pass_key) + consul_server.kv.put(pass_key, svc["pass"]) + + print() + +def fix_service_user(svc): + print("Fixing service user", svc["dn"], "...") + l = get_ldap_admin_conn() + res = l.search_s(svc["dn"], ldap.SCOPE_BASE, "objectclass=*") + pass_crypt = ldap_salted_sha1.hash(svc["pass"]) + if res is None or len(res) == 0: + print(bcolors.OKCYAN, "Creating entity...", bcolors.ENDC) + l.add_s(svc["dn"], + [ + ("objectclass", [b"person", b"top"]), + ("displayname", [svc["desc"].encode('ascii')]), + ("userpassword", [pass_crypt.encode('ascii')]), + ]) + else: + print(bcolors.OKCYAN, "Resetting entity password", bcolors.ENDC) + l.modify_s(svc["dn"], + [ + (ldap.MOD_REPLACE, "userpassword", [pass_crypt.encode('ascii')]) + ]) + +# ---- MAIN ---- + +if __name__ == "__main__": + for i, val in enumerate(sys.argv): + if val == "check": + check_secrets(sys.argv[i+1:]) + break + elif val == "gen": + gen_secrets(sys.argv[i+1:], False) + break + elif val == "regen": + gen_secrets(sys.argv[i+1:], True) + break + + diff --git a/app/config/configuration/traefik/traefik.toml b/app/traefik/config/traefik.toml similarity index 100% rename from app/config/configuration/traefik/traefik.toml rename to app/traefik/config/traefik.toml diff --git a/app/deployment/traefik.hcl b/app/traefik/deploy/traefik.hcl similarity index 95% rename from app/deployment/traefik.hcl rename to app/traefik/deploy/traefik.hcl index d0dc129..bcfa95b 100644 --- a/app/deployment/traefik.hcl +++ b/app/traefik/deploy/traefik.hcl @@ -29,7 +29,7 @@ job "frontend" { } template { - data = file("../config/configuration/traefik/traefik.toml") + data = file("../config/traefik.toml") destination = "secrets/traefik.toml" } diff --git a/os/config/roles/consul/tasks/main.yml b/os/config/roles/consul/tasks/main.yml index 761c1f8..994ecd7 100644 --- a/os/config/roles/consul/tasks/main.yml +++ b/os/config/roles/consul/tasks/main.yml @@ -1,6 +1,6 @@ - name: "Set consul version" set_fact: - consul_version: 1.8.4 + consul_version: 1.9.0 - name: "Download and install Consul for x86_64" unarchive: diff --git a/os/config/roles/nomad/tasks/main.yml b/os/config/roles/nomad/tasks/main.yml index c79013e..625d7b7 100644 --- a/os/config/roles/nomad/tasks/main.yml +++ b/os/config/roles/nomad/tasks/main.yml @@ -1,6 +1,6 @@ - name: "Set nomad version" set_fact: - nomad_version: 0.12.9 + nomad_version: 1.0.1 - name: "Download and install Nomad for x86_64" unarchive: