Merge pull request 'Reorganize app/ and add script for secret management' (#29) from test_reorganize into master

Reviewed-on: Deuxfleurs/infrastructure#29

README.md

@@ -82,6 +82,7 @@ alias bind_df="ssh \
   -L 8500:127.0.0.1:8500 \
   -L 8082:traefik-admin.service.2.cluster.deuxfleurs.fr:8082 \
   -L 5432:psql-proxy.service.2.cluster.deuxfleurs.fr:5432 \
+  -L 1389:bottin2.service.2.cluster.deuxfleurs.fr:389 \
   <a server from the cluster>"
 ```
 

app/.gitignore

@@ -0,0 +1 @@
+__pycache__

app/README.md

@@ -0,0 +1,52 @@
+## Understand this folder hierarchy
+
+This folder contains the following hierarchy:
+
+- `<module>/build/<image_name>/`: folders with dockerfiles and other necessary resources for building container images
+- `<module>/config/`: folder containing configuration files, referenced by deployment file
+- `<module>/secrets/`: folder containing secrets, which can be synchronized with Consul using `secretmgr.py`
+- `<module>/deploy/`: folder containing the HCL file(s) necessary for deploying the module
+- `<module>/integration/`: folder containing files for integration testing using docker-compose
+
+## How to install `secretmgr.py` dependencies
+
+How to install its dependencies:
+
+```bash
+# on fedora:
+dnf install -y openldap-devel
+# on ubuntu:
+apt-get install -y libldap2-dev
+
+# for eveyrone:
+pip3 install --user --requirement requirements.txt
+```
+
+## How to use `secretmgr.py`
+
+Check that all secrets are correctly deployed for app `dummy`:
+
+```bash
+./secretmgr.py check dummy
+```
+
+Generate secrets for app `dummy` if they don't already exist:
+
+```bash
+./secretmgr.py gen dummy
+```
+
+Rotate secrets for app `dummy`, overwriting existing ones (be careful, this is dangerous!):
+
+```bash
+./secretmgr.py regen dummy
+```
+
+## How to upgrade our packaged apps to a new version?
+
+ 1. Edit `docker-compose.yml`
+ 2. Change the `VERSION` variable to the desired version
+ 3. Increment the docker image tag by 1 (eg: superboum/riot:v13 -> superboum/riot:v14)
+ 4. Run `docker-compose build`
+ 5. Run `docker-compose push`
+ 6. Done

app/build/README.md

@@ -1,8 +0,0 @@
-## How to upgrade our packaged apps to a new version?
-
- 1. Edit `docker-compose.yml`
- 2. Change the `VERSION` variable to the desired version
- 3. Increment the docker image tag by 1 (eg: superboum/riot:v13 -> superboum/riot:v14)
- 4. Run `docker-compose build`
- 5. Run `docker-compose push`
- 6. Done

app/build/blog-quentin/Dockerfile

@@ -1,16 +0,0 @@
-FROM amd64/debian:stretch as builder
-
-COPY ./quentin.dufour.io/Gemfile /root/quentin.dufour.io/Gemfile
-
-WORKDIR /root/quentin.dufour.io
-
-RUN apt-get update && \
-    apt-get install -y ruby-dev gem build-essential bundler zlib1g-dev libxml2-dev && \
-    bundle install
-
-COPY ./quentin.dufour.io/ /root/quentin.dufour.io/
-RUN bundle exec jekyll build
-
-FROM superboum/amd64_webserver:v2
-COPY --from=builder /root/quentin.dufour.io/_site /srv/http
-

app/build/blog-quentin/README.md

@@ -1 +0,0 @@
-sudo docker build -t superboum/amd64_blog:v19 .

app/build/coturn/Dockerfile

@@ -1,8 +0,0 @@
-FROM amd64/debian:buster
-
-RUN apt-get update && \
-    apt-get dist-upgrade -y && \
-    apt-get install -y \
-      coturn
-
-CMD ["/usr/bin/turnserver"]

app/build/coturn/README.md

@@ -1,17 +0,0 @@
-
-## Génère l'image
-```
-sudo docker build -t registry.gitlab.com/superboum/ankh-morpork/amd64_coturn:v1 .
-```
-
-## Run bash dans le container
-```
-sudo docker run --rm -t -i registry.gitlab.com/superboum/ankh-morpork/amd64_coturn:v1 bash
-sudo docker run --rm -t -i -p 3478:3478/udp -p 3479:3479/udp -p 3478:3478/tcp -p 3479:3479/tcp registry.gitlab.com/superboum/ankh-morpork/amd64_coturn:v1 
-```
-
-## Used ports
-- udp/tcp 3478 3479 
-
-## Publish
-sudo docker push registry.gitlab.com/superboum/ankh-morpork/amd64_coturn:v1

app/build/landing/README.md

@@ -1,3 +0,0 @@
-```
-docker build -t superboum/amd64_landing:v8 .
-```

app/build/static/Dockerfile

@@ -1,9 +0,0 @@
-FROM golang:1.11.1-stretch as builder
-
-COPY ./goStatic /goStatic
-WORKDIR /goStatic
-RUN CGO_ENABLED=0 go build -a -o web-server .
-
-FROM scratch
-COPY --from=builder /goStatic/web-server /
-ENTRYPOINT ["/web-server"]

app/build/static/README.md

@@ -1,5 +0,0 @@
-
-```
-sudo docker build -t superboum/amd64_webserver:v3 .
-sudo docker push superboum/amd64_webserver:v3
-``` 

app/build/static/goStatic

@@ -1 +0,0 @@
-Subproject commit 3f97f57aaee09a142afe3ca0f1a5d51acd856436

app/config/secrets/.gitignore

@@ -1,11 +0,0 @@
-# Blacklist everything cleverly
-*
-!*/
-
-# Whitelist some patterns
-!*.sample
-!*.gen
-!*.sh
-!.gitignore
-
-# Whitelist specific files

app/config/secrets/chat/fb2mx/db_url.sample

@@ -1 +0,0 @@
-postgres://username:password@hostname/dbname

app/config/secrets/plume/pgsql_pw.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-openssl rand -base64 32 > pgsql_pw

app/config/secrets/plume/secret_key.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-openssl rand -base64 32 > secret_key

app/directory/deploy/directory.hcl

@@ -35,7 +35,7 @@ job "directory" {
       }
 
       template {
-        data = file("../config/configuration/directory/bottin/config.json")
+        data = file("../config/bottin/config.json")
         destination = "secrets/config.json"
       }
 
@@ -78,7 +78,7 @@ job "directory" {
       }
 
       template {
-        data = file("../config/configuration/directory/guichet/config.json.tpl")
+        data = file("../config/guichet/config.json.tpl")
         destination = "secrets/config.json"
       }
 

app/docker-compose.yml

@@ -3,7 +3,7 @@ services:
 
   mariadb:
     build:
-      context: ./mariadb
+      context: ./seafile/build/mariadb
       args:
         VERSION: 4 # fake for now
     image: superboum/amd64_mariadb:v4 
@@ -11,7 +11,7 @@ services:
   # Instant Messaging
   riot:
     build:
-      context: ./riotweb
+      context: ./im/build/riotweb
       args:
         # https://github.com/vector-im/riot-web/releases
         VERSION: 1.7.16
@@ -19,7 +19,7 @@ services:
 
   synapse:
     build: 
-      context: ./matrix-synapse
+      context: ./im/build/matrix-synapse
       args:
         # https://github.com/matrix-org/synapse/releases
         VERSION: 1.25.0
@@ -28,7 +28,7 @@ services:
   # Email
   sogo:
     build:
-      context: ./sogo
+      context: ./email/build/sogo
       args:
         # fake for now
         VERSION: 5.0.0
@@ -36,7 +36,7 @@ services:
 
   alps:
     build:
-      context: ./alps
+      context: ./email/build/alps
       args:
         VERSION: 5cef0aaff2b8b6ee3e00b566123517e241d8cfb8
     image: superboum/amd64_alps:v1
@@ -44,7 +44,7 @@ services:
   # VoIP
   jitsi-meet:
     build:
-      context: ./jitsi-meet
+      context: ./jitsi/build/jitsi-meet
       args:
         # https://github.com/jitsi/jitsi-meet
         PREFIXV: stable/jitsi-meet_
@@ -53,7 +53,7 @@ services:
 
   jitsi-conference-focus:
     build:
-      context: ./jitsi-conference-focus
+      context: ./jitsi/build/jitsi-conference-focus
       args:
         # https://github.com/jitsi/jicofo
         PREFIXV: stable/jitsi-meet_
@@ -62,7 +62,7 @@ services:
 
   jitsi-videobridge:
     build:
-      context: ./jitsi-videobridge
+      context: ./jitsi/build/jitsi-videobridge
       args:
         # https://github.com/jitsi/jitsi-videobridge
         PREFIXV: stable/jitsi-meet_
@@ -71,21 +71,21 @@ services:
 
   jitsi-xmpp:
     build:
-      context: ./jitsi-xmpp
+      context: ./jitsi/build/jitsi-xmpp
       args:
         VERSION: 0.11.2-1
     image: superboum/amd64_jitsi_xmpp:v8
 
   plume:
     build:
-      context: ./plume
+      context: ./plume/build/plume
       args:
         VERSION: 0.6.0
     image: superboum/plume:v2
 
   postfix:
     build:
-      context: ./postfix
+      context: ./email/build/postfix
       args:
         # https://packages.debian.org/fr/buster/postfix
         VERSION: 3.4.14-0+deb10u1

app/dummy/secrets/dummy/test_cmd

@@ -0,0 +1 @@
+CMD head -c 10 /dev/urandom | base64

app/dummy/secrets/dummy/test_const

@@ -0,0 +1 @@
+CONST this is a constant

app/dummy/secrets/dummy/test_const_long

@@ -0,0 +1,5 @@
+CONST_LONG
+this is a 
+constant
+on several
+lines

app/dummy/secrets/dummy/test_service_dn

@@ -0,0 +1 @@
+SERVICE_DN dummy Dummy service for testing secretmgr.py

app/dummy/secrets/dummy/test_service_password

@@ -0,0 +1 @@
+SERVICE_PASSWORD dummy

app/dummy/secrets/dummy/test_user

@@ -0,0 +1 @@
+USER Test user value