From 71062a2d40961c396aba93e5e61f3bd59c8872b9 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Mon, 10 Jan 2022 21:36:27 +0100
Subject: [PATCH] Matrix media natively on S3

---
 app/docker-compose.yml                        | 16 ++++++
 app/im/build/matrix-synapse/Dockerfile        | 54 +++++++++++++++++++
 app/im/build/matrix-synapse/entrypoint.sh     |  3 ++
 app/im/build/matrix-synapse/matrix-s3-async   | 16 ++++++
 .../matrix-synapse/matrix-s3-async-sqlite     | 13 +++++
 app/im/config/homeserver.yaml                 | 16 +++++-
 app/im/deploy/im.hcl                          | 34 ++++++++++--
 configuration.nix                             |  2 +-
 8 files changed, 149 insertions(+), 5 deletions(-)
 create mode 100644 app/docker-compose.yml
 create mode 100644 app/im/build/matrix-synapse/Dockerfile
 create mode 100755 app/im/build/matrix-synapse/entrypoint.sh
 create mode 100755 app/im/build/matrix-synapse/matrix-s3-async
 create mode 100755 app/im/build/matrix-synapse/matrix-s3-async-sqlite

diff --git a/app/docker-compose.yml b/app/docker-compose.yml
new file mode 100644
index 0000000..40091c9
--- /dev/null
+++ b/app/docker-compose.yml
@@ -0,0 +1,16 @@
+version: '3.4'
+services:
+
+  synapse:
+    build: 
+      context: ./im/build/matrix-synapse
+      args:
+        # https://github.com/matrix-org/synapse/releases
+        VERSION: 1.48.0
+        # https://github.com/matrix-org/synapse-s3-storage-provider/commits/main
+        # Update with the latest commit on main each time you update the synapse version
+        # otherwise synapse may fail to launch due to incompatibility issues
+        # see this issue for an example: https://github.com/matrix-org/synapse-s3-storage-provider/issues/64
+        S3_VERSION: 41f5bf83485c61efb61c9e77f5e93cbeb969f517
+    image: lxpz/amd64_synapse:test7
+
diff --git a/app/im/build/matrix-synapse/Dockerfile b/app/im/build/matrix-synapse/Dockerfile
new file mode 100644
index 0000000..4388d88
--- /dev/null
+++ b/app/im/build/matrix-synapse/Dockerfile
@@ -0,0 +1,54 @@
+FROM amd64/debian:buster as builder
+
+ARG VERSION
+ARG S3_VERSION
+RUN apt-get update && \
+    apt-get -qq -y full-upgrade && \
+    apt-get install -y \
+      python3 \
+      python3-pip \
+      python3-dev \
+      python3-setuptools \
+      libffi-dev \
+      build-essential \
+      libssl-dev \
+      libjpeg-dev \
+      libjpeg62-turbo-dev \
+      libxml2-dev \
+      zlib1g-dev \
+      # postgresql-dev \
+      libpq-dev \
+      virtualenv \
+      libxslt1-dev \
+      git
+
+RUN virtualenv /root/matrix-env -p /usr/bin/python3
+RUN . /root/matrix-env/bin/activate && \
+    pip3 install \
+      https://github.com/matrix-org/synapse/archive/v${VERSION}.tar.gz#egg=matrix-synapse[matrix-synapse-ldap3,postgres,resources.consent,saml2,url_preview] && \
+    pip3 install \
+      git+https://github.com/Alexis211/synapse-s3-storage-provider.git@${S3_VERSION}
+
+FROM amd64/debian:buster
+
+RUN apt-get update && \
+    apt-get -qq -y full-upgrade && \
+    apt-get install -y \
+      python3 \
+      python3-distutils \
+      libffi6 \
+      libjpeg62-turbo \
+      libssl1.1 \
+      libxslt1.1 \
+      libpq5 \
+      zlib1g \
+      libjemalloc2 \
+      ca-certificates
+
+ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libjemalloc.so.2
+COPY --from=builder /root/matrix-env /root/matrix-env
+COPY matrix-s3-async /usr/local/bin/matrix-s3-async
+COPY matrix-s3-async-sqlite /usr/local/bin/matrix-s3-async-sqlite
+COPY entrypoint.sh /usr/local/bin/entrypoint
+
+ENTRYPOINT ["/usr/local/bin/entrypoint"]
diff --git a/app/im/build/matrix-synapse/entrypoint.sh b/app/im/build/matrix-synapse/entrypoint.sh
new file mode 100755
index 0000000..b93a702
--- /dev/null
+++ b/app/im/build/matrix-synapse/entrypoint.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+. /root/matrix-env/bin/activate
+exec "$@"
diff --git a/app/im/build/matrix-synapse/matrix-s3-async b/app/im/build/matrix-synapse/matrix-s3-async
new file mode 100755
index 0000000..e435144
--- /dev/null
+++ b/app/im/build/matrix-synapse/matrix-s3-async
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cat > database.yaml <<EOF
+user: $PG_USER
+password: $PG_PASS
+database: $PG_DB
+host: $PG_HOST
+port: $PG_PORT
+EOF
+
+while true; do
+  /root/matrix-env/bin/s3_media_upload update-db 0d
+  /root/matrix-env/bin/s3_media_upload --no-progress check-deleted /var/lib/matrix-synapse/media
+  /root/matrix-env/bin/s3_media_upload --no-progress upload /var/lib/matrix-synapse/media matrix --delete --endpoint-url https://garage.deuxfleurs.fr 
+  sleep 600
+done
diff --git a/app/im/build/matrix-synapse/matrix-s3-async-sqlite b/app/im/build/matrix-synapse/matrix-s3-async-sqlite
new file mode 100755
index 0000000..b9eb73a
--- /dev/null
+++ b/app/im/build/matrix-synapse/matrix-s3-async-sqlite
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+cat > database.yaml <<EOF
+sqlite:
+  database: /ephemeral/homeserver.db
+EOF
+
+while true; do
+  /root/matrix-env/bin/s3_media_upload update-db 0d
+  /root/matrix-env/bin/s3_media_upload --no-progress check-deleted /ephemeral/media_store
+  /root/matrix-env/bin/s3_media_upload --no-progress upload /ephemeral/media_store matrix --delete --endpoint-url https://garage-staging.home.adnab.me
+  sleep 600
+done
diff --git a/app/im/config/homeserver.yaml b/app/im/config/homeserver.yaml
index c54bc4e..4a7e862 100644
--- a/app/im/config/homeserver.yaml
+++ b/app/im/config/homeserver.yaml
@@ -930,7 +930,7 @@ log_config: "/etc/matrix-synapse/synapse.log.config.yaml"
 
 # Directory where uploaded images and attachments are stored.
 #
-media_store_path: "/data/media_store"
+media_store_path: "/ephemeral/media_store"
 
 # Media storage providers allow media to be stored in different
 # locations.
@@ -946,6 +946,20 @@ media_store_path: "/data/media_store"
 #    config:
 #       directory: /mnt/some/other/directory
 
+media_storage_providers:
+- module: s3_storage_provider.S3StorageProviderBackend
+  store_local: True
+  store_remote: True
+  store_synchronous: True
+  config:
+    bucket: synapse-data
+    # All of the below options are optional, for use with non-AWS S3-like
+    # services, or to specify access tokens here instead of some external method.
+    region_name: garage-staging
+    endpoint_url: https://garage-staging.home.adnab.me
+    access_key_id: {{ key "secrets/synapse/s3_access_key" | trimSpace }}
+    secret_access_key: {{ key "secrets/synapse/s3_secret_key" | trimSpace }}
+
 # The largest allowed upload size in bytes
 #
 # If you are using a reverse proxy you may also need to set this value in
diff --git a/app/im/deploy/im.hcl b/app/im/deploy/im.hcl
index fd97c7b..1902978 100644
--- a/app/im/deploy/im.hcl
+++ b/app/im/deploy/im.hcl
@@ -52,10 +52,10 @@ job "im" {
     task "synapse" {
       driver = "docker"
       config {
-        image = "matrixdotorg/synapse:v1.48.0"
+        image = "lxpz/amd64_synapse:test7"
         ports = [ "http" ]
 
-        entrypoint = [ "/usr/local/bin/python" ]
+        command = "python"
         args = [
           "-m", "synapse.app.homeserver",
           "-n",
@@ -65,7 +65,6 @@ job "im" {
         volumes = [
           "secrets:/etc/matrix-synapse",
           "../alloc/data:/ephemeral",
-          "/mnt/garage-staging/synapse-data:/data",
         ]
       }
 
@@ -105,6 +104,35 @@ job "im" {
       }
     }
 
+    task "media-async-upload" {
+      driver = "docker"
+
+      config {
+        image = "lxpz/amd64_synapse:test7"
+        readonly_rootfs = true
+        command = "/usr/local/bin/matrix-s3-async-sqlite"
+        work_dir = "/ephemeral"
+        volumes = [
+          "../alloc/data:/ephemeral",
+        ]
+      }
+
+      resources {
+        cpu = 100
+        memory = 200
+      }
+
+      template {
+        data = <<EOH
+AWS_ACCESS_KEY_ID={{ key "secrets/synapse/s3_access_key" | trimSpace }}
+AWS_SECRET_ACCESS_KEY={{ key "secrets/synapse/s3_secret_key" | trimSpace }}
+AWS_DEFAULT_REGION=garage-staging
+EOH
+        destination = "secrets/env"
+        env = true
+      }
+    }
+
     task "replicate-db" {
       driver = "docker"
       config {
diff --git a/configuration.nix b/configuration.nix
index c1fa60e..2eefbc4 100644
--- a/configuration.nix
+++ b/configuration.nix
@@ -268,7 +268,7 @@ in
 
   # Mount Garage using Rclone
   systemd.services.mountgarage = {
-    enable = true;
+    enable = false;
     description = "Mount the Garage data store";
     path = [
       pkgs.fuse