From 5c7a8c72d80a2c818e41f5d6aafb4acc70867f1c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 12 May 2023 18:38:48 +0200 Subject: [PATCH] first plume on staging with S3 backend --- .../app/plume/build/docker-compose.yml | 8 + .../staging/app/plume/build/plume/Dockerfile | 61 ++++++++ .../staging/app/plume/build/plume/README.md | 3 + cluster/staging/app/plume/config/app.env | 38 +++++ .../staging/app/plume/config/litestream.yml | 11 ++ cluster/staging/app/plume/deploy/plume.hcl | 139 ++++++++++++++++++ .../staging/app/plume/integration/bottin.json | 31 ++++ .../app/plume/integration/docker-compose.yml | 28 ++++ .../staging/app/plume/integration/plume.env | 31 ++++ cluster/staging/app/plume/secrets.toml | 10 ++ 10 files changed, 360 insertions(+) create mode 100644 cluster/staging/app/plume/build/docker-compose.yml create mode 100644 cluster/staging/app/plume/build/plume/Dockerfile create mode 100644 cluster/staging/app/plume/build/plume/README.md create mode 100644 cluster/staging/app/plume/config/app.env create mode 100644 cluster/staging/app/plume/config/litestream.yml create mode 100644 cluster/staging/app/plume/deploy/plume.hcl create mode 100644 cluster/staging/app/plume/integration/bottin.json create mode 100644 cluster/staging/app/plume/integration/docker-compose.yml create mode 100644 cluster/staging/app/plume/integration/plume.env create mode 100644 cluster/staging/app/plume/secrets.toml diff --git a/cluster/staging/app/plume/build/docker-compose.yml b/cluster/staging/app/plume/build/docker-compose.yml new file mode 100644 index 0000000..db2be83 --- /dev/null +++ b/cluster/staging/app/plume/build/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3.4' +services: + plume: + build: + context: ./plume + args: + VERSION: 24d3b289da085261966fb338113610905dfca8c9 + image: lxpz/plume_dev:v1 diff --git a/cluster/staging/app/plume/build/plume/Dockerfile b/cluster/staging/app/plume/build/plume/Dockerfile new file mode 100644 index 0000000..b7bb862 --- /dev/null +++ b/cluster/staging/app/plume/build/plume/Dockerfile @@ -0,0 +1,61 @@ +#FROM rust:1.69-bullseye as builder +FROM debian:bullseye-slim as builder + +RUN apt-get update && \ + apt-get install -y \ + pkg-config \ + git \ + curl \ + postgresql \ + postgresql-contrib \ + libpq-dev \ + gettext \ + git \ + python \ + curl \ + gcc \ + make \ + openssl \ + libssl-dev \ + libclang-dev \ + libsqlite3-dev + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain nightly -y +ENV PATH=/root/.cargo/bin:$PATH +RUN cargo install wasm-pack + +ARG PLUME_VERSION +WORKDIR /opt +RUN git clone -n https://git.joinplu.me/lx/Plume.git plume + +WORKDIR /opt/plume +RUN git checkout ${PLUME_VERSION} +RUN rm rust-toolchain + +WORKDIR /opt/plume/script +RUN chmod a+x ./wasm-deps.sh && ./wasm-deps.sh + +WORKDIR /opt/plume +RUN chmod a+x ./script/plume-front.sh && ./script/plume-front.sh +RUN cargo install --path ./ --force --no-default-features --features sqlite,s3 +RUN cargo install --path plume-cli --force --no-default-features --features sqlite,s3 +RUN cargo clean + +#----------------------------- +FROM debian:bullseye-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + libpq5 \ + libssl1.1 \ + rclone \ + fuse \ + sqlite3 + +WORKDIR /app + +COPY --from=builder /opt/plume /app +COPY --from=builder /root/.cargo/bin/plm /usr/local/bin/ +COPY --from=builder /root/.cargo/bin/plume /usr/local/bin/ + +CMD ["plume"] diff --git a/cluster/staging/app/plume/build/plume/README.md b/cluster/staging/app/plume/build/plume/README.md new file mode 100644 index 0000000..6d86d81 --- /dev/null +++ b/cluster/staging/app/plume/build/plume/README.md @@ -0,0 +1,3 @@ +Try build: + +sudo docker build -t superboum/plume:v1 --build-arg VERSION=003dcf861a9f55720b03d52f2f95f5f59e338809 . diff --git a/cluster/staging/app/plume/config/app.env b/cluster/staging/app/plume/config/app.env new file mode 100644 index 0000000..6950736 --- /dev/null +++ b/cluster/staging/app/plume/config/app.env @@ -0,0 +1,38 @@ +BASE_URL=plume.staging.deuxfleurs.org +# generate one with openssl rand -base64 32 +ROCKET_SECRET_KEY={{ key "secrets/plume/secret_key" | trimSpace }} + +# Mail settings +#MAIL_SERVER=smtp.example.org +#MAIL_USER=example +#MAIL_PASSWORD=123456 +#MAIL_HELO_NAME=example.org + +# S3 settings +S3_BUCKET=plume +AWS_ACCESS_KEY_ID={{ key "secrets/plume/s3_access_key" | trimSpace }} +AWS_SECRET_ACCESS_KEY={{ key "secrets/plume/s3_secret_key" | trimSpace }} +S3_REGION=garage-staging +S3_HOSTNAME={{ env "attr.unique.network.ip-address" }}:3990 +S3_PROTOCOL=http +S3_PATH_STYLE=true + +# DATABASE SETUP +DATABASE_URL=/ephemeral/plume.db +MIGRATION_DIRECTORY=migrations/sqlite + +USE_HTTPS=0 +ROCKET_ADDRESS=:: +ROCKET_PORT={{ env "NOMAD_PORT_web_port" }} + +MEDIA_UPLOAD_DIRECTORY=/app/static/media +SEARCH_INDEX=/app/search_index + +LDAP_ADDR=ldap://bottin.service.staging.consul:389 +LDAP_BASE_DN=ou=users,dc=staging,dc=deuxfleurs,dc=org +LDAP_USER_NAME_ATTR=cn +LDAP_USER_MAIL_ATTR=mail +LDAP_TLS=false + +RUST_BACKTRACE=1 +RUST_LOG=debug diff --git a/cluster/staging/app/plume/config/litestream.yml b/cluster/staging/app/plume/config/litestream.yml new file mode 100644 index 0000000..35d7ba9 --- /dev/null +++ b/cluster/staging/app/plume/config/litestream.yml @@ -0,0 +1,11 @@ + +dbs: + - path: /ephemeral/plume.db + replicas: + - url: s3://plume/plume.db + region: garage-staging + endpoint: http://{{ env "attr.unique.network.ip-address" }}:3990 + access-key-id: {{ key "secrets/plume/s3_access_key" | trimSpace }} + secret-access-key: {{ key "secrets/plume/s3_secret_key" | trimSpace }} + force-path-style: true + sync-interval: 60s diff --git a/cluster/staging/app/plume/deploy/plume.hcl b/cluster/staging/app/plume/deploy/plume.hcl new file mode 100644 index 0000000..483828d --- /dev/null +++ b/cluster/staging/app/plume/deploy/plume.hcl @@ -0,0 +1,139 @@ +job "plume-blog" { + datacenters = ["neptune"] + type = "service" + + constraint { + attribute = "${attr.cpu.arch}" + value = "amd64" + } + + group "plume" { + count = 1 + + network { + port "web_port" { } + } + + task "restore-db" { + lifecycle { + hook = "prestart" + sidecar = false + } + + driver = "docker" + config { + image = "litestream/litestream:0.3.7" + args = [ + "restore", "-config", "/etc/litestream.yml", "/ephemeral/plume.db" + ] + volumes = [ + "../alloc/data:/ephemeral", + "secrets/litestream.yml:/etc/litestream.yml" + ] + } + user = "0" + + template { + data = file("../config/litestream.yml") + destination = "secrets/litestream.yml" + } + + resources { + memory = 100 + memory_max = 1000 + cpu = 1000 + } + } + + task "plume" { + constraint { + attribute = "${attr.unique.hostname}" + operator = "=" + value = "carcajou" + } + + driver = "docker" + config { + image = "lxpz/devplume:v3" + network_mode = "host" + ports = [ "web_port" ] + command = "sh" + args = [ "-c", "plm search init; plm search refill; plume" ] + volumes = [ + "/mnt/ssd/plume/search_index:/app/search_index", + "../alloc/data:/ephemeral" + ] + } + user = "0" + + template { + data = file("../config/app.env") + destination = "secrets/app.env" + env = true + } + + resources { + memory = 200 + memory_max = 800 + cpu = 100 + } + + service { + name = "plume" + tags = [ + "plume", + "tricot plume.staging.deuxfleurs.org", + "d53-cname plume.staging.deuxfleurs.org", + ] + port = "web_port" + address_mode = "host" + check { + type = "http" + protocol = "http" + port = "web_port" + path = "/" + interval = "60s" + timeout = "5s" + check_restart { + limit = 3 + grace = "600s" + ignore_warnings = false + } + } + } + restart { + interval = "30m" + attempts = 20 + delay = "15s" + mode = "delay" + } + } + + task "replicate-db" { + driver = "docker" + config { + image = "litestream/litestream:0.3.7" + args = [ + "replicate", "-config", "/etc/litestream.yml" + ] + volumes = [ + "../alloc/data:/ephemeral", + "secrets/litestream.yml:/etc/litestream.yml" + ] + } + user = "0" + + template { + data = file("../config/litestream.yml") + destination = "secrets/litestream.yml" + } + + resources { + memory = 200 + memory_max = 1000 + cpu = 100 + } + } + } +} + diff --git a/cluster/staging/app/plume/integration/bottin.json b/cluster/staging/app/plume/integration/bottin.json new file mode 100644 index 0000000..a970762 --- /dev/null +++ b/cluster/staging/app/plume/integration/bottin.json @@ -0,0 +1,31 @@ +{ + "suffix": "dc=deuxfleurs,dc=fr", + "bind": "0.0.0.0:389", + "consul_host": "http://consul:8500", + "log_level": "debug", + "acl": [ + "*,dc=deuxfleurs,dc=fr::read:*:* !userpassword", + "*::read modify:SELF:*", + "ANONYMOUS::bind:*,ou=users,dc=deuxfleurs,dc=fr:", + "ANONYMOUS::bind:cn=admin,dc=deuxfleurs,dc=fr:", + "*,ou=services,ou=users,dc=deuxfleurs,dc=fr::bind:*,ou=users,dc=deuxfleurs,dc=fr:*", + "*,ou=services,ou=users,dc=deuxfleurs,dc=fr::read:*:*", + + "*:cn=asso_deuxfleurs,ou=groups,dc=deuxfleurs,dc=fr:add:*,ou=invitations,dc=deuxfleurs,dc=fr:*", + "ANONYMOUS::bind:*,ou=invitations,dc=deuxfleurs,dc=fr:", + "*,ou=invitations,dc=deuxfleurs,dc=fr::delete:SELF:*", + + "*:cn=asso_deuxfleurs,ou=groups,dc=deuxfleurs,dc=fr:add:*,ou=users,dc=deuxfleurs,dc=fr:*", + "*,ou=invitations,dc=deuxfleurs,dc=fr::add:*,ou=users,dc=deuxfleurs,dc=fr:*", + + "*:cn=asso_deuxfleurs,ou=groups,dc=deuxfleurs,dc=fr:modifyAdd:cn=email,ou=groups,dc=deuxfleurs,dc=fr:*", + "*,ou=invitations,dc=deuxfleurs,dc=fr::modifyAdd:cn=email,ou=groups,dc=deuxfleurs,dc=fr:*", + "*:cn=asso_deuxfleurs,ou=groups,dc=deuxfleurs,dc=fr:modifyAdd:cn=seafile,ou=groups,dc=deuxfleurs,dc=fr:*", + "*,ou=invitations,dc=deuxfleurs,dc=fr::modifyAdd:cn=seafile,ou=groups,dc=deuxfleurs,dc=fr:*", + "*:cn=asso_deuxfleurs,ou=groups,dc=deuxfleurs,dc=fr:modifyAdd:cn=nextcloud,ou=groups,dc=deuxfleurs,dc=fr:*", + "*,ou=invitations,dc=deuxfleurs,dc=fr::modifyAdd:cn=seafile,ou=nextcloud,dc=deuxfleurs,dc=fr:*", + + "cn=admin,dc=deuxfleurs,dc=fr::read add modify delete:*:*", + "*:cn=admin,ou=groups,dc=deuxfleurs,dc=fr:read add modify delete:*:*" + ] +} diff --git a/cluster/staging/app/plume/integration/docker-compose.yml b/cluster/staging/app/plume/integration/docker-compose.yml new file mode 100644 index 0000000..b88de8a --- /dev/null +++ b/cluster/staging/app/plume/integration/docker-compose.yml @@ -0,0 +1,28 @@ +version: '3.4' +services: + plume: + image: superboum/plume:v1 + env_file: + - plume.env + depends_on: + - consul + - postgres + ports: + - "7878:7878" + + postgres: + image: postgres:9.6.19 + environment: + - POSTGRES_DB=plume + - POSTGRES_USER=plume + - POSTGRES_PASSWORD=plume + + bottin: + image: lxpz/bottin_amd64:14 + depends_on: + - consul + volumes: + - ./bottin.json:/config.json + + consul: + image: consul:1.8.4 diff --git a/cluster/staging/app/plume/integration/plume.env b/cluster/staging/app/plume/integration/plume.env new file mode 100644 index 0000000..88c62dc --- /dev/null +++ b/cluster/staging/app/plume/integration/plume.env @@ -0,0 +1,31 @@ +BASE_URL=integration.env +# generate one with openssl rand -base64 32 +ROCKET_SECRET_KEY=cXZbKoxWIBo0wdaD8tbA1B3BlH2LBSUmgzdyZZr8QxI= + +# Mail settings +#MAIL_SERVER=smtp.example.org +#MAIL_USER=example +#MAIL_PASSWORD=123456 +#MAIL_HELO_NAME=example.org + +# DATABASE SETUP +POSTGRES_PASSWORD=plume +POSTGRES_USER=plume +POSTGRES_DB=plume +DATABASE_URL=postgres://plume:plume@postgres:5432/plume +MIGRATION_DIRECTORY=migrations/postgres + +USE_HTTPS=0 +ROCKET_ADDRESS=0.0.0.0 +ROCKET_PORT=7878 + +MEDIA_UPLOAD_DIRECTORY=/app/static/media +SEARCH_INDEX=/app/search_index +DOMAIN_NAME="integration.env" +INSTANCE_NAME="Integration Instance" + +LDAP_ADDR=ldap://bottin:389 +LDAP_BASE_DN=ou=users,dc=deuxfleurs,dc=fr +LDAP_USER_NAME_ATTR=cn +LDAP_USER_MAIL_ATTR=mail +LDAP_TLS=false diff --git a/cluster/staging/app/plume/secrets.toml b/cluster/staging/app/plume/secrets.toml new file mode 100644 index 0000000..4d68a5c --- /dev/null +++ b/cluster/staging/app/plume/secrets.toml @@ -0,0 +1,10 @@ +[service_user."plume"] +password_secret = "plume/pgsql_pw" + + +[secrets."plume/secret_key"] +type = 'command' +rotate = true +command = 'openssl rand -base64 32' + +