Merge branch 'main' into wgautomesh

cluster/prod/app/backup/build/backup-garage/Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine:3.17
+
+RUN apk add rclone btrfs-progs curl bash jq
+
+COPY do-backup.sh /do-backup.sh
+
+CMD bash /do-backup.sh

cluster/prod/app/backup/build/backup-garage/do-backup.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# DEPENDENCIES: btrfs-progs curl rclone jq
+
+# PARAMETERS (environmenet variables)
+#       $BACKUP_BASEDIR         => where to store backups and btrfs snapshots
+#       $GARAGE_ADMIN_TOKEN     => Garage administration access token
+#       $GARAGE_ACCESS_KEY      => Garage access key
+#       $GARAGE_SECRET_KEY      => Garage secret key
+
+if [ -z "$BACKUP_BASEDIR" -o -z "$GARAGE_ACCESS_KEY" -o -z "$GARAGE_ADMIN_TOKEN" ]; then
+        echo "Missing parameters"
+fi
+
+if [ ! -d "$BACKUP_BASEDIR/buckets" ]; then
+        btrfs subvolume create "$BACKUP_BASEDIR/buckets"
+fi
+
+
+function gcurl {
+        curl -s -H "Authorization: Bearer $GARAGE_ADMIN_TOKEN" $@
+}
+
+BUCKETS=$(gcurl "http://localhost:3903/v0/bucket" | jq -r '.[].id')
+
+for BUCKET in $BUCKETS; do
+        echo "==== BUCKET $BUCKET ===="
+
+        gcurl "http://localhost:3903/v0/bucket?id=$BUCKET" > "$BACKUP_BASEDIR/buckets/$BUCKET.json"
+
+        ALIASES=$(jq -r '.globalAliases[]' < "$BACKUP_BASEDIR/buckets/$BUCKET.json")
+        echo "(aka. $ALIASES)"
+
+        case $ALIASES in
+                *backup*)
+                        echo "Skipping $BUCKET (not doing backup of backup)"
+                        ;;
+                *cache*)
+                        echo "Skipping $BUCKET (not doing backup of cache)"
+                        ;;
+                *)
+                        echo "Backing up $BUCKET"
+
+                        if [ ! -d "$BACKUP_BASEDIR/buckets/$BUCKET" ]; then
+                                mkdir "$BACKUP_BASEDIR/buckets/$BUCKET"
+                        fi
+
+                        gcurl -X POST -H "Content-Type: application/json"  --data @- "http://localhost:3903/v0/bucket/allow" >/dev/null <<EOF
+                                {
+                                        "bucketId": "$BUCKET",
+                                        "accessKeyId": "$GARAGE_ACCESS_KEY",
+                                        "permissions": {"read": true}
+                                }
+EOF
+
+                        rclone sync --s3-endpoint http://localhost:3900 \
+                                --s3-access-key-id $GARAGE_ACCESS_KEY \
+                                --s3-secret-access-key $GARAGE_SECRET_KEY \
+                                --s3-region garage \
+                                --s3-force-path-style \
+                                --transfers 32 \
+                                --fast-list \
+                                --stats-one-line \
+                                --stats 10s \
+                                --stats-log-level NOTICE \
+                                ":s3:$BUCKET" "$BACKUP_BASEDIR/buckets/$BUCKET" 2>&1
+                        ;;
+        esac
+done
+
+echo "========= DONE SYNCHRONIZING =========="
+
+if [ ! -d "$BACKUP_BASEDIR/snapshots" ]; then
+        mkdir "$BACKUP_BASEDIR/snapshots"
+fi
+
+SNAPSHOT="$BACKUP_BASEDIR/snapshots/buckets-$(date +%F)"
+if [ ! -e "$SNAPSHOT" ]; then
+        echo "Making snapshot: $SNAPSHOT"
+        btrfs subvolume snapshot "$BACKUP_BASEDIR/buckets" "$SNAPSHOT"
+        btrfs prop set "$SNAPSHOT" ro true
+fi
+
+

cluster/prod/app/backup/deploy/backup-daily.hcl

@@ -1,5 +1,5 @@
 job "backup_daily" {
-  datacenters = ["orion", "neptune"]
+  datacenters = ["orion", "neptune", "scorpio"]
   type = "batch"
 
   priority = "60"
@@ -44,7 +44,7 @@ EOH
       resources {
         cpu = 500
         memory = 100
-	memory_max = 300
+		memory_max = 1000
       }
 
       restart {
@@ -90,7 +90,7 @@ EOH
       resources {
         cpu = 500
         memory = 100
-	memory_max = 300
+		memory_max = 1000
       }
 
       restart {
@@ -225,9 +225,54 @@ EOH
          env = true
       }
 
+      resources {
+        cpu = 500
+        memory = 100
+		memory_max = 1000
+      }
+
+      restart {
+        attempts = 2
+        interval = "30m"
+        delay = "15s"
+        mode = "fail"
+      }
+    }
+  }
+
+  group "backup-garage" {
+    constraint {
+      attribute = "${attr.unique.hostname}"
+      operator = "="
+      value = "abricot"
+    }
+
+    task "main" {
+      driver = "docker"
+
+      config {
+        image = "lxpz/backup_garage:4"
+		network_mode = "host"
+        volumes = [
+          "/mnt/storage/backup/garage.deuxfleurs.fr:/backup"
+        ]
+      }
+
+      template {
+        data = <<EOH
+BACKUP_BASEDIR=/backup
+GARAGE_ADMIN_TOKEN={{ key "secrets/garage/admin_token" }}
+GARAGE_ACCESS_KEY={{ key "secrets/backup/garage/s3_access_key_id" }}
+GARAGE_SECRET_KEY={{ key "secrets/backup/garage/s3_secret_access_key" }}
+EOH
+         destination = "secrets/env_vars"
+         env = true
+      }
+
       resources {
         cpu = 500
         memory = 200
+		memory_max = 4000
       }
 
       restart {

cluster/prod/app/email/deploy/email-android7.hcl

@@ -0,0 +1,126 @@
+job "email-android7" {
+  datacenters = ["neptune"]
+  type = "service"
+  priority = 100
+
+  group "rsa-ecc-proxy" {
+    network {
+      port "smtps" {
+        static = 465
+        to = 465
+      }
+      port "imaps" {
+        static = 993
+        to = 993
+      }
+    } 
+    task "imaps-proxy" {
+      driver = "docker"
+      config {
+        image = "alpine/socat:1.7.4.4"
+        readonly_rootfs = true
+        ports = [ "imaps" ]
+        network_mode = "host"
+        args = [
+          "openssl-listen:993,reuseaddr,fork,verify=0,bind=0.0.0.0,cert=/var/secrets/rsa.crt,key=/var/secrets/rsa.key",
+          "openssl:imap.deuxfleurs.fr:993,verify=0",
+        ]
+        volumes = [
+          "secrets/certs:/var/secrets"
+        ]
+      }
+
+      template {
+        data = "{{ key \"secrets/email/tls-tls-proxy/rsa.crt\" }}"
+        destination = "secrets/certs/rsa.crt"
+      }
+      template {
+        data = "{{ key \"secrets/email/tls-tls-proxy/rsa.key\" }}"
+        destination = "secrets/certs/rsa.key"
+      }
+
+      resources {
+        cpu = 50
+        memory = 50
+      }
+
+      service {
+        name = "imap-android7"
+        port = "imaps"
+        address_mode = "host"
+        tags = [
+          "rsa-ecc-proxy",
+          "(diplonat (tcp_port 993))",
+          "d53-a imap-android7.deuxfleurs.fr",
+          # ipv6 is commented for now as socat does not listen on ipv6 now
+          # "d53-aaaa imap-android7.deuxfleurs.fr"
+        ]
+        check {
+          type = "tcp"
+          port = "imaps"
+          interval = "60s"
+          timeout = "5s"
+          check_restart {
+            limit = 3
+            grace = "90s"
+            ignore_warnings = false
+          }
+        }
+      }
+    }
+    task "smtps-proxy" {
+      driver = "docker"
+      config {
+        image = "alpine/socat:1.7.4.4"
+        readonly_rootfs = true
+        network_mode = "host"
+        ports = [ "smtps" ]
+        args = [
+          "openssl-listen:465,reuseaddr,fork,verify=0,bind=0.0.0.0,cert=/var/secrets/rsa.crt,key=/var/secrets/rsa.key",
+          "openssl:smtp.deuxfleurs.fr:465,verify=0",
+        ]
+        volumes = [
+          "secrets/certs:/var/secrets"
+        ]
+      }
+
+      template {
+        data = "{{ key \"secrets/email/tls-tls-proxy/rsa.crt\" }}"
+        destination = "secrets/certs/rsa.crt"
+      }
+      template {
+        data = "{{ key \"secrets/email/tls-tls-proxy/rsa.key\" }}"
+        destination = "secrets/certs/rsa.key"
+      }
+
+      resources {
+        cpu = 50
+        memory = 50
+      }
+
+      service {
+        name = "smtp-android7"
+        port = "smtps"
+        address_mode = "host"
+        tags = [
+          "rsa-ecc-proxy",
+          "(diplonat (tcp_port 465))",
+          "d53-a smtp-android7.deuxfleurs.fr",
+          # ipv6 is commented for now as socat does not listen on ipv6 now
+          # "d53-aaaa smtp-android7.deuxfleurs.fr"
+        ]
+        check {
+          type = "tcp"
+          port = "smtps"
+          interval = "60s"
+          timeout = "5s"
+          check_restart {
+            limit = 3
+            grace = "90s"
+            ignore_warnings = false
+          }
+        }
+      }
+    }
+  }
+}

cluster/prod/app/email/integration/README.md

@@ -0,0 +1,23 @@
+# Email
+
+## TLS TLS Proxy
+
+Required for Android 7.0 that does not support elliptic curves.
+
+Generate a key:
+
+```bash
+openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout rsa.key -out rsa.crt -subj "/CN=imap.deuxfleurs.fr" -addext "subjectAltName=DNS:smtp.deuxfleurs.fr"
+```
+
+Run the command:
+
+```bash
+./integration/proxy.sh imap.deuxfleurs.fr:993 1993
+```
+
+Test it:
+
+```bash
+openssl s_client localhost:1993
+```

cluster/prod/app/email/integration/tls-tls-proxy.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+UPSTREAM=$1
+PROXY_PORT=$2
+socat -dd \
+"openssl-listen:${PROXY_PORT},\
+reuseaddr,\
+fork,\
+cert=/tmp/tls-tls-proxy/rsa.crt,\
+key=/tmp/tls-tls-proxy/rsa.key,\
+verify=0,\
+bind=0.0.0.0" \
+"openssl:${UPSTREAM},\
+verify=0"

cluster/prod/app/email/secrets.toml

@@ -21,3 +21,12 @@ password_secret = "email/sogo/ldap_bindpw"
 type = 'user'
 description = 'SoGo postgres auth (format: sogo:<password>) (TODO: replace this with two separate files and change template)'
 
+# ---- TLS TLS PROXY ---
+
+[secrets."email/tls-tls-proxy/rsa.crt"]
+type="user"
+description="PEM encoded file containing the RSA certificate"
+
+[secrets."email/tls-tls-proxy/rsa.key"]
+type="user"
+description="PEM encoded file containing the RSA key"

cluster/prod/app/matrix/build/docker-compose.yml

@@ -6,18 +6,18 @@ services:
       context: ./riotweb
       args:
         # https://github.com/vector-im/riot-web/releases
-        VERSION: 1.11.12
+        VERSION: 1.11.25
-    image: superboum/amd64_riotweb:v32
+    image: superboum/amd64_riotweb:v33
 
   synapse:
     build:
       context: ./matrix-synapse
       args:
         # https://github.com/matrix-org/synapse/releases
-        VERSION: 1.70.0
+        VERSION: 1.79.0
         # https://github.com/matrix-org/synapse-s3-storage-provider/commits/main
         # Update with the latest commit on main each time you update the synapse version
         # otherwise synapse may fail to launch due to incompatibility issues
         # see this issue for an example: https://github.com/matrix-org/synapse-s3-storage-provider/issues/64
-        S3_VERSION: 40c6a5599b2d8176318003f84b167ea545cebba3
+        S3_VERSION: v1.2.0
-    image: superboum/amd64_synapse:v55
+    image: superboum/amd64_synapse:v56

cluster/prod/app/matrix/config/synapse/homeserver.yaml

@@ -450,7 +450,7 @@ presence:
   enabled: false
 limit_remote_rooms:
   enabled: true
-  complexity: 3.0
+  complexity: 10.0
   complexity_error: "Ce salon de discussion a trop d'activité, le serveur n'est pas assez puissant pour le rejoindre. N'hésitez pas à remonter l'information à l'équipe technique, nous pourrons ajuster la limitation au besoin."
   admins_can_join: false
 retention:

cluster/prod/app/matrix/deploy/im.hcl

@@ -14,7 +14,7 @@ job "matrix" {
       driver = "docker"
 
       config {
-        image = "superboum/amd64_synapse:v55"
+        image = "superboum/amd64_synapse:v56"
         network_mode = "host"
         readonly_rootfs = true
         ports = [ "api_port" ]
@@ -98,7 +98,7 @@ job "matrix" {
       driver = "docker"
 
       config {
-        image = "superboum/amd64_synapse:v55"
+        image = "superboum/amd64_synapse:v56"
         readonly_rootfs = true
         command = "/usr/local/bin/matrix-s3-async"
         work_dir = "/tmp"
@@ -142,7 +142,7 @@ EOH
     task "server" {
       driver = "docker"
       config {
-        image = "superboum/amd64_riotweb:v32"
+        image = "superboum/amd64_riotweb:v33"
         ports = [ "web_port" ]
         volumes = [
           "secrets/config.json:/srv/http/config.json"

cluster/staging/app/core/deploy/core-system.hcl

@@ -121,7 +121,7 @@ EOH
         data = <<EOH
 TRICOT_NODE_NAME={{ env "attr.unique.consul.name" }}
 TRICOT_LETSENCRYPT_EMAIL=alex@adnab.me
-TRICOT_ENABLE_COMPRESSION=true
+#TRICOT_ENABLE_COMPRESSION=true
 TRICOT_CONSUL_HOST=https://localhost:8501
 TRICOT_CONSUL_CA_CERT=/etc/tricot/consul-ca.crt
 TRICOT_CONSUL_CLIENT_CERT=/etc/tricot/consul-client.crt