Deuxfleurs/nixcfg
17
6
Fork 2
Code Issues 15 Pull requests 2 Projects Releases Wiki Activity

Merge branch 'main' into wgautomesh

Browse source
This commit is contained in:
Alex 2023-03-24 11:29:14 +01:00
commit 5cd69a9ba1
11 changed files with 320 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cluster/prod/app/backup/build/backup-garage/Dockerfile Normal file
View file

@ -0,0 +1,7 @@
FROM alpine:3.17
RUN apk add rclone btrfs-progs curl bash jq
COPY do-backup.sh /do-backup.sh
CMD bash /do-backup.sh

84
cluster/prod/app/backup/build/backup-garage/do-backup.sh Normal file
View file

@ -0,0 +1,84 @@
#!/usr/bin/env bash
# DEPENDENCIES: btrfs-progs curl rclone jq
# PARAMETERS (environmenet variables)
# $BACKUP_BASEDIR => where to store backups and btrfs snapshots
# $GARAGE_ADMIN_TOKEN => Garage administration access token
# $GARAGE_ACCESS_KEY => Garage access key
# $GARAGE_SECRET_KEY => Garage secret key
if [ -z "$BACKUP_BASEDIR" -o -z "$GARAGE_ACCESS_KEY" -o -z "$GARAGE_ADMIN_TOKEN" ]; then
echo "Missing parameters"
fi
if [ ! -d "$BACKUP_BASEDIR/buckets" ]; then
btrfs subvolume create "$BACKUP_BASEDIR/buckets"
fi
function gcurl {
curl -s -H "Authorization: Bearer $GARAGE_ADMIN_TOKEN" $@
}
BUCKETS=$(gcurl "http://localhost:3903/v0/bucket" | jq -r '.[].id')
for BUCKET in $BUCKETS; do
echo "==== BUCKET $BUCKET ===="
gcurl "http://localhost:3903/v0/bucket?id=$BUCKET" > "$BACKUP_BASEDIR/buckets/$BUCKET.json"
ALIASES=$(jq -r '.globalAliases[]' < "$BACKUP_BASEDIR/buckets/$BUCKET.json")
echo "(aka. $ALIASES)"
case $ALIASES in
*backup*)
echo "Skipping $BUCKET (not doing backup of backup)"
;;
*cache*)
echo "Skipping $BUCKET (not doing backup of cache)"
;;
*)
echo "Backing up $BUCKET"
if [ ! -d "$BACKUP_BASEDIR/buckets/$BUCKET" ]; then
mkdir "$BACKUP_BASEDIR/buckets/$BUCKET"
fi
gcurl -X POST -H "Content-Type: application/json" --data @- "http://localhost:3903/v0/bucket/allow" >/dev/null <<EOF
{
"bucketId": "$BUCKET",
"accessKeyId": "$GARAGE_ACCESS_KEY",
"permissions": {"read": true}
}
EOF
rclone sync --s3-endpoint http://localhost:3900 \
--s3-access-key-id $GARAGE_ACCESS_KEY \
--s3-secret-access-key $GARAGE_SECRET_KEY \
--s3-region garage \
--s3-force-path-style \
--transfers 32 \
--fast-list \
--stats-one-line \
--stats 10s \
--stats-log-level NOTICE \
":s3:$BUCKET" "$BACKUP_BASEDIR/buckets/$BUCKET" 2>&1
;;
esac
done
echo "========= DONE SYNCHRONIZING =========="
if [ ! -d "$BACKUP_BASEDIR/snapshots" ]; then
mkdir "$BACKUP_BASEDIR/snapshots"
fi
SNAPSHOT="$BACKUP_BASEDIR/snapshots/buckets-$(date +%F)"
if [ ! -e "$SNAPSHOT" ]; then
echo "Making snapshot: $SNAPSHOT"
btrfs subvolume snapshot "$BACKUP_BASEDIR/buckets" "$SNAPSHOT"
btrfs prop set "$SNAPSHOT" ro true
fi

51
cluster/prod/app/backup/deploy/backup-daily.hcl
View file

@ -1,5 +1,5 @@
job "backup_daily" {
datacenters = ["orion", "neptune"]
datacenters = ["orion", "neptune", "scorpio"]
type = "batch"
priority = "60"
@ -44,7 +44,7 @@ EOH
resources {
cpu = 500
memory = 100
memory_max = 300
memory_max = 1000
}
restart {
@ -90,7 +90,7 @@ EOH
resources {
cpu = 500
memory = 100
memory_max = 300
memory_max = 1000
}
restart {
@ -225,9 +225,54 @@ EOH
env = true
}
resources {
cpu = 500
memory = 100
memory_max = 1000
}
restart {
attempts = 2
interval = "30m"
delay = "15s"
mode = "fail"
}
}
}
group "backup-garage" {
constraint {
attribute = "${attr.unique.hostname}"
operator = "="
value = "abricot"
}
task "main" {
driver = "docker"
config {
image = "lxpz/backup_garage:4"
network_mode = "host"
volumes = [
"/mnt/storage/backup/garage.deuxfleurs.fr:/backup"
]
}
template {
data = <<EOH
BACKUP_BASEDIR=/backup
GARAGE_ADMIN_TOKEN={{ key "secrets/garage/admin_token" }}
GARAGE_ACCESS_KEY={{ key "secrets/backup/garage/s3_access_key_id" }}
GARAGE_SECRET_KEY={{ key "secrets/backup/garage/s3_secret_access_key" }}
EOH
destination = "secrets/env_vars"
env = true
}
resources {
cpu = 500
memory = 200
memory_max = 4000
}
restart {

126
cluster/prod/app/email/deploy/email-android7.hcl Normal file
View file

@ -0,0 +1,126 @@
job "email-android7" {
datacenters = ["neptune"]
type = "service"
priority = 100
group "rsa-ecc-proxy" {
network {
port "smtps" {
static = 465
to = 465
}
port "imaps" {
static = 993
to = 993
}
}
task "imaps-proxy" {
driver = "docker"
config {
image = "alpine/socat:1.7.4.4"
readonly_rootfs = true
ports = [ "imaps" ]
network_mode = "host"
args = [
"openssl-listen:993,reuseaddr,fork,verify=0,bind=0.0.0.0,cert=/var/secrets/rsa.crt,key=/var/secrets/rsa.key",
"openssl:imap.deuxfleurs.fr:993,verify=0",
]
volumes = [
"secrets/certs:/var/secrets"
]
}
template {
data = "{{ key \"secrets/email/tls-tls-proxy/rsa.crt\" }}"
destination = "secrets/certs/rsa.crt"
}
template {
data = "{{ key \"secrets/email/tls-tls-proxy/rsa.key\" }}"
destination = "secrets/certs/rsa.key"
}
resources {
cpu = 50
memory = 50
}
service {
name = "imap-android7"
port = "imaps"
address_mode = "host"
tags = [
"rsa-ecc-proxy",
"(diplonat (tcp_port 993))",
"d53-a imap-android7.deuxfleurs.fr",
# ipv6 is commented for now as socat does not listen on ipv6 now
# "d53-aaaa imap-android7.deuxfleurs.fr"
]
check {
type = "tcp"
port = "imaps"
interval = "60s"
timeout = "5s"
check_restart {
limit = 3
grace = "90s"
ignore_warnings = false
}
}
}
}
task "smtps-proxy" {
driver = "docker"
config {
image = "alpine/socat:1.7.4.4"
readonly_rootfs = true
network_mode = "host"
ports = [ "smtps" ]
args = [
"openssl-listen:465,reuseaddr,fork,verify=0,bind=0.0.0.0,cert=/var/secrets/rsa.crt,key=/var/secrets/rsa.key",
"openssl:smtp.deuxfleurs.fr:465,verify=0",
]
volumes = [
"secrets/certs:/var/secrets"
]
}
template {
data = "{{ key \"secrets/email/tls-tls-proxy/rsa.crt\" }}"
destination = "secrets/certs/rsa.crt"
}
template {
data = "{{ key \"secrets/email/tls-tls-proxy/rsa.key\" }}"
destination = "secrets/certs/rsa.key"
}
resources {
cpu = 50
memory = 50
}
service {
name = "smtp-android7"
port = "smtps"
address_mode = "host"
tags = [
"rsa-ecc-proxy",
"(diplonat (tcp_port 465))",
"d53-a smtp-android7.deuxfleurs.fr",
# ipv6 is commented for now as socat does not listen on ipv6 now
# "d53-aaaa smtp-android7.deuxfleurs.fr"
]
check {
type = "tcp"
port = "smtps"
interval = "60s"
timeout = "5s"
check_restart {
limit = 3
grace = "90s"
ignore_warnings = false
}
}
}
}
}
}

23
cluster/prod/app/email/integration/README.md Normal file
View file

@ -0,0 +1,23 @@
# Email
## TLS TLS Proxy
Required for Android 7.0 that does not support elliptic curves.
Generate a key:
```bash
openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout rsa.key -out rsa.crt -subj "/CN=imap.deuxfleurs.fr" -addext "subjectAltName=DNS:smtp.deuxfleurs.fr"
```
Run the command:
```bash
./integration/proxy.sh imap.deuxfleurs.fr:993 1993
```
Test it:
```bash
openssl s_client localhost:1993
```

13
cluster/prod/app/email/integration/tls-tls-proxy.sh Normal file
View file

@ -0,0 +1,13 @@
#!/usr/bin/env bash
UPSTREAM=$1
PROXY_PORT=$2
socat -dd \
"openssl-listen:${PROXY_PORT},\
reuseaddr,\
fork,\
cert=/tmp/tls-tls-proxy/rsa.crt,\
key=/tmp/tls-tls-proxy/rsa.key,\
verify=0,\
bind=0.0.0.0" \
"openssl:${UPSTREAM},\
verify=0"

9
cluster/prod/app/email/secrets.toml
View file

@ -21,3 +21,12 @@ password_secret = "email/sogo/ldap_bindpw"
type = 'user'
description = 'SoGo postgres auth (format: sogo:<password>) (TODO: replace this with two separate files and change template)'
# ---- TLS TLS PROXY ---
[secrets."email/tls-tls-proxy/rsa.crt"]
type="user"
description="PEM encoded file containing the RSA certificate"
[secrets."email/tls-tls-proxy/rsa.key"]
type="user"
description="PEM encoded file containing the RSA key"

10
cluster/prod/app/matrix/build/docker-compose.yml
View file

@ -6,18 +6,18 @@ services:
context: ./riotweb
args:
# https://github.com/vector-im/riot-web/releases
VERSION: 1.11.12
image: superboum/amd64_riotweb:v32
VERSION: 1.11.25
image: superboum/amd64_riotweb:v33
synapse:
build:
context: ./matrix-synapse
args:
# https://github.com/matrix-org/synapse/releases
VERSION: 1.70.0
VERSION: 1.79.0
# https://github.com/matrix-org/synapse-s3-storage-provider/commits/main
# Update with the latest commit on main each time you update the synapse version
# otherwise synapse may fail to launch due to incompatibility issues
# see this issue for an example: https://github.com/matrix-org/synapse-s3-storage-provider/issues/64
S3_VERSION: 40c6a5599b2d8176318003f84b167ea545cebba3
image: superboum/amd64_synapse:v55
S3_VERSION: v1.2.0
image: superboum/amd64_synapse:v56

2
cluster/prod/app/matrix/config/synapse/homeserver.yaml
View file

@ -450,7 +450,7 @@ presence:
enabled: false
limit_remote_rooms:
enabled: true
complexity: 3.0
complexity: 10.0
complexity_error: "Ce salon de discussion a trop d'activité, le serveur n'est pas assez puissant pour le rejoindre. N'hésitez pas à remonter l'information à l'équipe technique, nous pourrons ajuster la limitation au besoin."
admins_can_join: false
retention:

6
cluster/prod/app/matrix/deploy/im.hcl
View file

@ -14,7 +14,7 @@ job "matrix" {
driver = "docker"
config {
image = "superboum/amd64_synapse:v55"
image = "superboum/amd64_synapse:v56"
network_mode = "host"
readonly_rootfs = true
ports = [ "api_port" ]
@ -98,7 +98,7 @@ job "matrix" {
driver = "docker"
config {
image = "superboum/amd64_synapse:v55"
image = "superboum/amd64_synapse:v56"
readonly_rootfs = true
command = "/usr/local/bin/matrix-s3-async"
work_dir = "/tmp"
@ -142,7 +142,7 @@ EOH
task "server" {
driver = "docker"
config {
image = "superboum/amd64_riotweb:v32"
image = "superboum/amd64_riotweb:v33"
ports = [ "web_port" ]
volumes = [
"secrets/config.json:/srv/http/config.json"

2
cluster/staging/app/core/deploy/core-system.hcl
View file

@ -121,7 +121,7 @@ EOH
data = <<EOH
TRICOT_NODE_NAME={{ env "attr.unique.consul.name" }}
TRICOT_LETSENCRYPT_EMAIL=alex@adnab.me
TRICOT_ENABLE_COMPRESSION=true
#TRICOT_ENABLE_COMPRESSION=true
TRICOT_CONSUL_HOST=https://localhost:8501
TRICOT_CONSUL_CA_CERT=/etc/tricot/consul-ca.crt
TRICOT_CONSUL_CLIENT_CERT=/etc/tricot/consul-client.crt