Matrix media natively on S3

This commit is contained in:
Alex 2022-01-10 21:36:27 +01:00
parent 4927080443
commit 71062a2d40
No known key found for this signature in database
GPG key ID: EDABF9711E244EB1
8 changed files with 149 additions and 5 deletions

16
app/docker-compose.yml Normal file
View file

@ -0,0 +1,16 @@
version: '3.4'
services:
synapse:
build:
context: ./im/build/matrix-synapse
args:
# https://github.com/matrix-org/synapse/releases
VERSION: 1.48.0
# https://github.com/matrix-org/synapse-s3-storage-provider/commits/main
# Update with the latest commit on main each time you update the synapse version
# otherwise synapse may fail to launch due to incompatibility issues
# see this issue for an example: https://github.com/matrix-org/synapse-s3-storage-provider/issues/64
S3_VERSION: 41f5bf83485c61efb61c9e77f5e93cbeb969f517
image: lxpz/amd64_synapse:test7

View file

@ -0,0 +1,54 @@
FROM amd64/debian:buster as builder
ARG VERSION
ARG S3_VERSION
RUN apt-get update && \
apt-get -qq -y full-upgrade && \
apt-get install -y \
python3 \
python3-pip \
python3-dev \
python3-setuptools \
libffi-dev \
build-essential \
libssl-dev \
libjpeg-dev \
libjpeg62-turbo-dev \
libxml2-dev \
zlib1g-dev \
# postgresql-dev \
libpq-dev \
virtualenv \
libxslt1-dev \
git
RUN virtualenv /root/matrix-env -p /usr/bin/python3
RUN . /root/matrix-env/bin/activate && \
pip3 install \
https://github.com/matrix-org/synapse/archive/v${VERSION}.tar.gz#egg=matrix-synapse[matrix-synapse-ldap3,postgres,resources.consent,saml2,url_preview] && \
pip3 install \
git+https://github.com/Alexis211/synapse-s3-storage-provider.git@${S3_VERSION}
FROM amd64/debian:buster
RUN apt-get update && \
apt-get -qq -y full-upgrade && \
apt-get install -y \
python3 \
python3-distutils \
libffi6 \
libjpeg62-turbo \
libssl1.1 \
libxslt1.1 \
libpq5 \
zlib1g \
libjemalloc2 \
ca-certificates
ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libjemalloc.so.2
COPY --from=builder /root/matrix-env /root/matrix-env
COPY matrix-s3-async /usr/local/bin/matrix-s3-async
COPY matrix-s3-async-sqlite /usr/local/bin/matrix-s3-async-sqlite
COPY entrypoint.sh /usr/local/bin/entrypoint
ENTRYPOINT ["/usr/local/bin/entrypoint"]

View file

@ -0,0 +1,3 @@
#!/bin/sh
. /root/matrix-env/bin/activate
exec "$@"

View file

@ -0,0 +1,16 @@
#!/bin/bash
cat > database.yaml <<EOF
user: $PG_USER
password: $PG_PASS
database: $PG_DB
host: $PG_HOST
port: $PG_PORT
EOF
while true; do
/root/matrix-env/bin/s3_media_upload update-db 0d
/root/matrix-env/bin/s3_media_upload --no-progress check-deleted /var/lib/matrix-synapse/media
/root/matrix-env/bin/s3_media_upload --no-progress upload /var/lib/matrix-synapse/media matrix --delete --endpoint-url https://garage.deuxfleurs.fr
sleep 600
done

View file

@ -0,0 +1,13 @@
#!/bin/bash
cat > database.yaml <<EOF
sqlite:
database: /ephemeral/homeserver.db
EOF
while true; do
/root/matrix-env/bin/s3_media_upload update-db 0d
/root/matrix-env/bin/s3_media_upload --no-progress check-deleted /ephemeral/media_store
/root/matrix-env/bin/s3_media_upload --no-progress upload /ephemeral/media_store matrix --delete --endpoint-url https://garage-staging.home.adnab.me
sleep 600
done

View file

@ -930,7 +930,7 @@ log_config: "/etc/matrix-synapse/synapse.log.config.yaml"
# Directory where uploaded images and attachments are stored. # Directory where uploaded images and attachments are stored.
# #
media_store_path: "/data/media_store" media_store_path: "/ephemeral/media_store"
# Media storage providers allow media to be stored in different # Media storage providers allow media to be stored in different
# locations. # locations.
@ -946,6 +946,20 @@ media_store_path: "/data/media_store"
# config: # config:
# directory: /mnt/some/other/directory # directory: /mnt/some/other/directory
media_storage_providers:
- module: s3_storage_provider.S3StorageProviderBackend
store_local: True
store_remote: True
store_synchronous: True
config:
bucket: synapse-data
# All of the below options are optional, for use with non-AWS S3-like
# services, or to specify access tokens here instead of some external method.
region_name: garage-staging
endpoint_url: https://garage-staging.home.adnab.me
access_key_id: {{ key "secrets/synapse/s3_access_key" | trimSpace }}
secret_access_key: {{ key "secrets/synapse/s3_secret_key" | trimSpace }}
# The largest allowed upload size in bytes # The largest allowed upload size in bytes
# #
# If you are using a reverse proxy you may also need to set this value in # If you are using a reverse proxy you may also need to set this value in

View file

@ -52,10 +52,10 @@ job "im" {
task "synapse" { task "synapse" {
driver = "docker" driver = "docker"
config { config {
image = "matrixdotorg/synapse:v1.48.0" image = "lxpz/amd64_synapse:test7"
ports = [ "http" ] ports = [ "http" ]
entrypoint = [ "/usr/local/bin/python" ] command = "python"
args = [ args = [
"-m", "synapse.app.homeserver", "-m", "synapse.app.homeserver",
"-n", "-n",
@ -65,7 +65,6 @@ job "im" {
volumes = [ volumes = [
"secrets:/etc/matrix-synapse", "secrets:/etc/matrix-synapse",
"../alloc/data:/ephemeral", "../alloc/data:/ephemeral",
"/mnt/garage-staging/synapse-data:/data",
] ]
} }
@ -105,6 +104,35 @@ job "im" {
} }
} }
task "media-async-upload" {
driver = "docker"
config {
image = "lxpz/amd64_synapse:test7"
readonly_rootfs = true
command = "/usr/local/bin/matrix-s3-async-sqlite"
work_dir = "/ephemeral"
volumes = [
"../alloc/data:/ephemeral",
]
}
resources {
cpu = 100
memory = 200
}
template {
data = <<EOH
AWS_ACCESS_KEY_ID={{ key "secrets/synapse/s3_access_key" | trimSpace }}
AWS_SECRET_ACCESS_KEY={{ key "secrets/synapse/s3_secret_key" | trimSpace }}
AWS_DEFAULT_REGION=garage-staging
EOH
destination = "secrets/env"
env = true
}
}
task "replicate-db" { task "replicate-db" {
driver = "docker" driver = "docker"
config { config {

View file

@ -268,7 +268,7 @@ in
# Mount Garage using Rclone # Mount Garage using Rclone
systemd.services.mountgarage = { systemd.services.mountgarage = {
enable = true; enable = false;
description = "Mount the Garage data store"; description = "Mount the Garage data store";
path = [ path = [
pkgs.fuse pkgs.fuse