From 3baa511fce2e5731b97cf27a18b18bd2c1af18bc Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Thu, 27 Jan 2022 16:32:57 +0100
Subject: [PATCH] Plume backup + WIP consul

---
 app/backup/deploy/backup-daily.hcl            | 159 +++++++++++++++---
 app/backup/deploy/backup-matrix.hcl           |  62 -------
 .../email/dovecot/backup_aws_access_key_id    |   0
 .../dovecot/backup_aws_secret_access_key      |   0
 .../email/dovecot/backup_restic_password      |   0
 .../email/dovecot/backup_restic_repository    |   0
 app/plume/secrets/plume/aws_access_key_id     |   1 -
 app/plume/secrets/plume/aws_endpoint          |   1 -
 app/plume/secrets/plume/aws_secret_access_key |   1 -
 .../secrets/plume/backup_aws_access_key_id    |   1 +
 .../plume/backup_aws_secret_access_key        |   1 +
 .../secrets/plume/backup_restic_password      |   1 +
 .../secrets/plume/backup_restic_repository    |   1 +
 op_guide/backup_minio/README.md               |  56 ++++--
 14 files changed, 187 insertions(+), 97 deletions(-)
 delete mode 100644 app/backup/deploy/backup-matrix.hcl
 rename app/{backup => email}/secrets/email/dovecot/backup_aws_access_key_id (100%)
 rename app/{backup => email}/secrets/email/dovecot/backup_aws_secret_access_key (100%)
 rename app/{backup => email}/secrets/email/dovecot/backup_restic_password (100%)
 rename app/{backup => email}/secrets/email/dovecot/backup_restic_repository (100%)
 delete mode 100644 app/plume/secrets/plume/aws_access_key_id
 delete mode 100644 app/plume/secrets/plume/aws_endpoint
 delete mode 100644 app/plume/secrets/plume/aws_secret_access_key
 create mode 100644 app/plume/secrets/plume/backup_aws_access_key_id
 create mode 100644 app/plume/secrets/plume/backup_aws_secret_access_key
 create mode 100644 app/plume/secrets/plume/backup_restic_password
 create mode 100644 app/plume/secrets/plume/backup_restic_repository

diff --git a/app/backup/deploy/backup-daily.hcl b/app/backup/deploy/backup-daily.hcl
index a1c2f89..ef9b219 100644
--- a/app/backup/deploy/backup-daily.hcl
+++ b/app/backup/deploy/backup-daily.hcl
@@ -1,52 +1,167 @@
 job "backup_daily" {
   datacenters = ["dc1"]
   type = "batch"
+
+  priority = "60"
+
   periodic {
     cron = "@daily"
     // Do not allow overlapping runs.
     prohibit_overlap = true
   }
 
-  task "backup-dovecot" {
+  group "backup-dovecot" {
     constraint {
       attribute = "${attr.unique.hostname}"
       operator = "="
       value = "digitale"
     }
 
-    driver = "docker"
+    task "main" {
+      driver = "docker"
 
-    config {
-      image = "restic/restic:0.12.1"
-      entrypoint = [ "/bin/sh", "-c" ]
-      args = [ "restic backup /mail && restic forget --keep-within 1m1d --keep-within-weekly 3m --keep-within-monthly 1y ; restic prune --max-unused 50% --max-repack-size 2G ; restic check" ]
-      volumes = [
-        "/mnt/ssd/mail:/mail"
-      ]
-    }
+      config {
+        image = "restic/restic:0.12.1"
+        entrypoint = [ "/bin/sh", "-c" ]
+        args = [ "restic backup /mail && restic forget --keep-within 1m1d --keep-within-weekly 3m --keep-within-monthly 1y ; restic prune --max-unused 50% --max-repack-size 2G ; restic check" ]
+        volumes = [
+          "/mnt/ssd/mail:/mail"
+        ]
+      }
 
-
-    template {
-      data = <<EOH
+      template {
+        data = <<EOH
 AWS_ACCESS_KEY_ID={{ key "secrets/email/dovecot/backup_aws_access_key_id" }}
 AWS_SECRET_ACCESS_KEY={{ key "secrets/email/dovecot/backup_aws_secret_access_key" }}
 RESTIC_REPOSITORY={{ key "secrets/email/dovecot/backup_restic_repository" }}
 RESTIC_PASSWORD={{ key "secrets/email/dovecot/backup_restic_password" }}
 EOH
 
-       destination = "secrets/env_vars"
-       env = true
+         destination = "secrets/env_vars"
+         env = true
+      }
+
+      resources {
+        cpu = 500
+        memory = 200
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
+    }
+  }
+
+  group "backup-plume" {
+    constraint {
+      attribute = "${attr.unique.hostname}"
+      operator = "="
+      value = "digitale"
     }
 
-    resources {
-      memory = 200
+    task "main" {
+      driver = "docker"
+
+      config {
+        image = "restic/restic:0.12.1"
+        entrypoint = [ "/bin/sh", "-c" ]
+        args = [ "restic backup /plume && restic forget --keep-within 1m1d --keep-within-weekly 3m --keep-within-monthly 1y ; restic prune --max-unused 50% --max-repack-size 2G ; restic check" ]
+        volumes = [
+          "/mnt/ssd/plume/media:/plume"
+        ]
+      }
+
+      template {
+        data = <<EOH
+AWS_ACCESS_KEY_ID={{ key "secrets/plume/backup_aws_access_key_id" }}
+AWS_SECRET_ACCESS_KEY={{ key "secrets/plume/backup_aws_secret_access_key" }}
+RESTIC_REPOSITORY={{ key "secrets/plume/backup_restic_repository" }}
+RESTIC_PASSWORD={{ key "secrets/plume/backup_restic_password" }}
+EOH
+
+         destination = "secrets/env_vars"
+         env = true
+      }
+
+      resources {
+        cpu = 500
+        memory = 200
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
+    }
+  }
+
+  group "backup-consul" {
+    task "export-kv-store" {
+      driver = "docker"
+
+      lifecycle {      
+        hook = "prestart"      
+        sidecar = false    
+      }
+
+      config {
+        image = "consul:1.11.2"
+        entrypoint = [ "/bin/sh", "-c" ]
+        NETWORK MODE HOST
+        args = [ "/bin/consul kv export > $NOMAD_ALLOC_DIR/consul.json" ]
+      }
+
+      resources {
+        cpu = 200
+        memory = 200
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
     }
 
-    restart {
-      attempts = 2
-      interval = "30m"
-      delay = "15s"
-      mode = "fail"
+    task "upload-kv-store" {
+      driver = "docker"
+
+      config {
+        image = "restic/restic:0.12.1"
+        entrypoint = [ "/bin/sh", "-c" ]
+        args = [ "restic backup $NOMAD_ALLOC_DIR && restic forget --keep-within 1m1d --keep-within-weekly 3m --keep-within-monthly 1y ; restic prune --max-unused 50% --max-repack-size 2G ; restic check" ]
+      }
+
+
+      template {
+        data = <<EOH
+AWS_ACCESS_KEY_ID={{ key "secrets/plume/backup_aws_access_key_id" }}
+AWS_SECRET_ACCESS_KEY={{ key "secrets/plume/backup_aws_secret_access_key" }}
+RESTIC_REPOSITORY={{ key "secrets/plume/backup_restic_repository" }}
+RESTIC_PASSWORD={{ key "secrets/plume/backup_restic_password" }}
+EOH
+
+         destination = "secrets/env_vars"
+         env = true
+      }
+
+      resources {
+        cpu = 200
+        memory = 200
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
     }
   }
 }
diff --git a/app/backup/deploy/backup-matrix.hcl b/app/backup/deploy/backup-matrix.hcl
deleted file mode 100644
index cb28c90..0000000
--- a/app/backup/deploy/backup-matrix.hcl
+++ /dev/null
@@ -1,62 +0,0 @@
-job "backup_manual_matrix" {
-	datacenters = ["dc1"]
-
-	type = "batch"
-
-	task "backup-matrix" {
-		driver = "docker"
-
-			config {
-				image = "superboum/backup_matrix:4"
-				volumes = [
-					"secrets/id_ed25519:/root/.ssh/id_ed25519",
-					"secrets/id_ed25519.pub:/root/.ssh/id_ed25519.pub",
-					"secrets/known_hosts:/root/.ssh/known_hosts",
-					"/mnt/glusterfs/chat/matrix/synapse/media:/mnt/glusterfs/chat/matrix/synapse/media"
-				]
-				network_mode = "host"
-			}
-
-		env {
-			CONSUL_HTTP_ADDR = "http://consul.service.2.cluster.deuxfleurs.fr:8500"
-		}
-
-		template {
-			data = <<EOH
-TARGET_SSH_USER={{ key "secrets/backup/target_ssh_user" }}
-TARGET_SSH_PORT={{ key "secrets/backup/target_ssh_port" }}
-TARGET_SSH_HOST={{ key "secrets/backup/target_ssh_host" }}
-TARGET_SSH_DIR={{ key "secrets/backup/target_ssh_dir" }}
-REPL_PSQL_USER={{ key "secrets/postgres/keeper/pg_repl_username" }}
-REPL_PSQL_PWD={{ key "secrets/postgres/keeper/pg_repl_pwd" }}
-EOH
-
-			destination = "secrets/env_vars"
-			env = true
-		}
-
-		template {
-			data = "{{ key \"secrets/backup/id_ed25519\" }}"
-			destination = "secrets/id_ed25519"
-		}
-		template {
-			data = "{{ key \"secrets/backup/id_ed25519.pub\" }}"
-			destination = "secrets/id_ed25519.pub"
-		}
-		template {
-			data = "{{ key \"secrets/backup/target_ssh_fingerprint\" }}"
-			destination = "secrets/known_hosts"
-		}
-
-		resources {
-			memory = 200
-		}
-
-		restart {
-			attempts = 2
-			interval = "30m"
-			delay = "15s"
-			mode = "fail"
-		}
-	}
-}
diff --git a/app/backup/secrets/email/dovecot/backup_aws_access_key_id b/app/email/secrets/email/dovecot/backup_aws_access_key_id
similarity index 100%
rename from app/backup/secrets/email/dovecot/backup_aws_access_key_id
rename to app/email/secrets/email/dovecot/backup_aws_access_key_id
diff --git a/app/backup/secrets/email/dovecot/backup_aws_secret_access_key b/app/email/secrets/email/dovecot/backup_aws_secret_access_key
similarity index 100%
rename from app/backup/secrets/email/dovecot/backup_aws_secret_access_key
rename to app/email/secrets/email/dovecot/backup_aws_secret_access_key
diff --git a/app/backup/secrets/email/dovecot/backup_restic_password b/app/email/secrets/email/dovecot/backup_restic_password
similarity index 100%
rename from app/backup/secrets/email/dovecot/backup_restic_password
rename to app/email/secrets/email/dovecot/backup_restic_password
diff --git a/app/backup/secrets/email/dovecot/backup_restic_repository b/app/email/secrets/email/dovecot/backup_restic_repository
similarity index 100%
rename from app/backup/secrets/email/dovecot/backup_restic_repository
rename to app/email/secrets/email/dovecot/backup_restic_repository
diff --git a/app/plume/secrets/plume/aws_access_key_id b/app/plume/secrets/plume/aws_access_key_id
deleted file mode 100644
index 8129980..0000000
--- a/app/plume/secrets/plume/aws_access_key_id
+++ /dev/null
@@ -1 +0,0 @@
-USER AWS access key ID, eg. GKxxxx
diff --git a/app/plume/secrets/plume/aws_endpoint b/app/plume/secrets/plume/aws_endpoint
deleted file mode 100644
index 9868a5a..0000000
--- a/app/plume/secrets/plume/aws_endpoint
+++ /dev/null
@@ -1 +0,0 @@
-USER AWS Endpoint, eg. s3.garage.tld
diff --git a/app/plume/secrets/plume/aws_secret_access_key b/app/plume/secrets/plume/aws_secret_access_key
deleted file mode 100644
index 03a5225..0000000
--- a/app/plume/secrets/plume/aws_secret_access_key
+++ /dev/null
@@ -1 +0,0 @@
-USER AWS Secret Access Key
diff --git a/app/plume/secrets/plume/backup_aws_access_key_id b/app/plume/secrets/plume/backup_aws_access_key_id
new file mode 100644
index 0000000..9235e53
--- /dev/null
+++ b/app/plume/secrets/plume/backup_aws_access_key_id
@@ -0,0 +1 @@
+USER Backup AWS access key ID
diff --git a/app/plume/secrets/plume/backup_aws_secret_access_key b/app/plume/secrets/plume/backup_aws_secret_access_key
new file mode 100644
index 0000000..f34677e
--- /dev/null
+++ b/app/plume/secrets/plume/backup_aws_secret_access_key
@@ -0,0 +1 @@
+USER Backup AWS secret access key
diff --git a/app/plume/secrets/plume/backup_restic_password b/app/plume/secrets/plume/backup_restic_password
new file mode 100644
index 0000000..fbaa5fa
--- /dev/null
+++ b/app/plume/secrets/plume/backup_restic_password
@@ -0,0 +1 @@
+USER Restic password to encrypt backups
diff --git a/app/plume/secrets/plume/backup_restic_repository b/app/plume/secrets/plume/backup_restic_repository
new file mode 100644
index 0000000..3f6cb93
--- /dev/null
+++ b/app/plume/secrets/plume/backup_restic_repository
@@ -0,0 +1 @@
+USER Restic repository, eg. s3:https://s3.garage.tld
diff --git a/op_guide/backup_minio/README.md b/op_guide/backup_minio/README.md
index 31194e5..b67e42a 100644
--- a/op_guide/backup_minio/README.md
+++ b/op_guide/backup_minio/README.md
@@ -3,9 +3,12 @@ Add the admin account as `deuxfleurs` to your `~/.mc/config` file
 You need to choose some names/identifiers:
 
 ```bash
-export BUCKET_NAME=example
-export NEW_ACCESS_KEY_ID=hello
+export ENDPOINT="https://s3.garage.tld"
+export SERVICE_NAME="example"
 
+
+export BUCKET_NAME="backups-${SERVICE_NAME}"
+export NEW_ACCESS_KEY_ID="key-${SERVICE_NAME}"
 export NEW_SECRET_ACCESS_KEY=$(openssl rand -base64 32)
 export POLICY_NAME="policy-$BUCKET_NAME"
 ```
@@ -22,7 +25,19 @@ Create a new user:
 mc admin user add deuxfleurs $NEW_ACCESS_KEY_ID $NEW_SECRET_ACCESS_KEY
 ```
 
-Add this new user to your `~/.mc/config.json` file, as `backup-user` for example.
+Add this new user to your `~/.mc/config.json`, run this command before to generate the snippet to copy/paste:
+
+```
+cat > /dev/stdout <<EOF
+"$NEW_ACCESS_KEY_ID": {
+	"url": "$ENDPOINT",
+	"accessKey": "$NEW_ACCESS_KEY_ID",
+	"secretKey": "$NEW_SECRET_ACCESS_KEY",
+	"api": "S3v4",
+	"path": "auto"
+},
+EOF
+```
 
 ---
 
@@ -79,8 +94,6 @@ mc ls backup-user/
 Now we need to initialize the repository with restic.
 
 ```bash
-export ENDPOINT="https://garage.tld"
-
 export AWS_ACCESS_KEY_ID=$NEW_ACCESS_KEY_ID
 export AWS_SECRET_ACCESS_KEY=$NEW_SECRET_ACCESS_KEY
 export RESTIC_REPOSITORY="s3:$ENDPOINT/$BUCKET_NAME"
@@ -101,16 +114,39 @@ See your snapshots with:
 restic snapshots
 ```
 
+Check also these useful commands:
+
+```
+restic ls
+restic diff
+restic help
+```
+
 ---
 
 Add the secrets to Consul, near your service secrets.
 The idea is that the backuping service is a component of the global running service.
-You must add:
-  - `backup_aws_access_key_id`
-  - `backup_aws_secret_access_key`
-  - `backup_restic_repository`
-  - `backup_restic_password`
+You must run in `app/<name>/secrets/<subpath>`:
 
+```bash
+echo "USER Backup AWS access key ID" > backup_aws_access_key_id
+echo "USER Backup AWS secret access key" > backup_aws_secret_access_key
+echo "USER Restic repository, eg. s3:https://s3.garage.tld" > backup_restic_repository
+echo "USER Restic password to encrypt backups" > backup_restic_password
+```
+
+Then run secretmgr:
+
+```bash
+# Spawning a nix shell is an easy way to get all the dependencies you need
+nix-shell
+
+# Check that secretmgr works for you
+python3 secretmgr.py check <name>
+
+# Now interactively feed the secrets
+python3 secretmgr.py gen <name>
+```
 
 ---