From 3befdea20669f5b6e09cadb20f91d33a3cded459 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Fri, 28 Apr 2023 09:26:32 +0200 Subject: [PATCH 1/6] nix: allow wireguard + logs --- nix/deuxfleurs.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index f7b70d7..ad5249d 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -201,10 +201,15 @@ in domain-insecure = [ "consul." ]; local-zone = [ "consul. nodefault" ]; log-servfail = true; + verbosity = 1; + log-queries = true; + use-syslog = false; + logfile = "/dev/stdout"; access-control = [ "127.0.0.0/8 allow" "${cfg.lan_ip}/${toString cfg.lan_ip_prefix_length} allow" "172.17.0.0/16 allow" + "10.83.0.0/16 allow" ]; }; forward-zone = [ From e23b52346742ebd9cb039e3af705ba150492cd1e Mon Sep 17 00:00:00 2001 From: Baptiste Jonglez Date: Wed, 3 May 2023 08:53:59 +0200 Subject: [PATCH 2/6] Add infinite restart policy for postgresql --- cluster/prod/app/postgres/deploy/postgres.hcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cluster/prod/app/postgres/deploy/postgres.hcl b/cluster/prod/app/postgres/deploy/postgres.hcl index 9bad079..2aa2d07 100644 --- a/cluster/prod/app/postgres/deploy/postgres.hcl +++ b/cluster/prod/app/postgres/deploy/postgres.hcl @@ -16,6 +16,13 @@ job "postgres14" { port "psql_port" { static = 5433 } } + restart { + interval = "10m" + attempts = 10 + delay = "15s" + mode = "delay" + } + task "sentinel" { driver = "docker" From f3cd2e98b4bd26294464c5152b7082fc8a6b9ff2 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 4 May 2023 16:39:25 +0200 Subject: [PATCH 3/6] multisite postgres, orient plume to correct db --- cluster/prod/app/plume/config/app.env | 2 +- cluster/prod/app/postgres/deploy/postgres.hcl | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cluster/prod/app/plume/config/app.env b/cluster/prod/app/plume/config/app.env index 5c9ede6..b751bd6 100644 --- a/cluster/prod/app/plume/config/app.env +++ b/cluster/prod/app/plume/config/app.env @@ -12,7 +12,7 @@ ROCKET_SECRET_KEY={{ key "secrets/plume/secret_key" | trimSpace }} POSTGRES_PASSWORD={{ key "secrets/plume/pgsql_pw" | trimSpace }} POSTGRES_USER=plume POSTGRES_DB=plume -DATABASE_URL=postgres://plume:{{ key "secrets/plume/pgsql_pw" | trimSpace }}@psql-proxy.service.prod.consul:5432/plume +DATABASE_URL=postgres://plume:{{ key "secrets/plume/pgsql_pw" | trimSpace }}@{{ env "meta.site" }}.psql-proxy.service.prod.consul:5432/plume MIGRATION_DIRECTORY=migrations/postgres USE_HTTPS=0 diff --git a/cluster/prod/app/postgres/deploy/postgres.hcl b/cluster/prod/app/postgres/deploy/postgres.hcl index 2aa2d07..e8825a1 100644 --- a/cluster/prod/app/postgres/deploy/postgres.hcl +++ b/cluster/prod/app/postgres/deploy/postgres.hcl @@ -1,5 +1,5 @@ job "postgres14" { - datacenters = ["orion"] + datacenters = ["orion", "neptune", "bespin"] type = "system" priority = 90 @@ -16,6 +16,13 @@ job "postgres14" { port "psql_port" { static = 5433 } } + constraint { + attribute = "${attr.unique.hostname}" + operator = "set_contains_any" + # target: courgette,df-ymf,abricot (or ananas) + value = "diplotaxis,courgette,concombre,df-ymf" + } + restart { interval = "10m" attempts = 10 @@ -106,7 +113,7 @@ job "postgres14" { } service { - tags = ["sql"] + tags = ["sql", "${meta.site}"] port = "psql_proxy_port" address_mode = "host" name = "psql-proxy" @@ -186,7 +193,7 @@ job "postgres14" { } service { - tags = ["sql"] + tags = ["sql", "${meta.site}"] port = "psql_port" address_mode = "host" name = "psql-keeper" From e375304c388948efcb8cfcb45d8a8288a635a766 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 4 May 2023 16:48:22 +0200 Subject: [PATCH 4/6] orient SoGo and Synapse to closest psql-proxy; psql backup anywhere --- cluster/prod/app/backup/deploy/backup-weekly.hcl | 4 ++-- cluster/prod/app/email/config/sogo/sogo.conf.tpl | 14 +++++++------- .../prod/app/matrix/config/synapse/homeserver.yaml | 2 +- cluster/prod/app/matrix/deploy/im.hcl | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cluster/prod/app/backup/deploy/backup-weekly.hcl b/cluster/prod/app/backup/deploy/backup-weekly.hcl index 36a507a..6a00507 100644 --- a/cluster/prod/app/backup/deploy/backup-weekly.hcl +++ b/cluster/prod/app/backup/deploy/backup-weekly.hcl @@ -1,5 +1,5 @@ job "backup_weekly" { - datacenters = ["orion"] + datacenters = ["orion", "neptune", "bespin"] type = "batch" priority = "60" @@ -30,7 +30,7 @@ AWS_ENDPOINT=s3.deuxfleurs.shirokumo.net AWS_ACCESS_KEY_ID={{ key "secrets/postgres/backup/aws_access_key_id" }} AWS_SECRET_ACCESS_KEY={{ key "secrets/postgres/backup/aws_secret_access_key" }} CRYPT_PUBLIC_KEY={{ key "secrets/postgres/backup/crypt_public_key" }} -PSQL_HOST=psql-proxy.service.prod.consul +PSQL_HOST={{ env "meta.site" }}.psql-proxy.service.prod.consul PSQL_USER={{ key "secrets/postgres/keeper/pg_repl_username" }} PGPASSWORD={{ key "secrets/postgres/keeper/pg_repl_pwd" }} EOH diff --git a/cluster/prod/app/email/config/sogo/sogo.conf.tpl b/cluster/prod/app/email/config/sogo/sogo.conf.tpl index d6094bf..bb87f14 100644 --- a/cluster/prod/app/email/config/sogo/sogo.conf.tpl +++ b/cluster/prod/app/email/config/sogo/sogo.conf.tpl @@ -3,13 +3,13 @@ WOWorkersCount = 3; SxVMemLimit = 300; WOPort = "127.0.0.1:20000"; - SOGoProfileURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@psql-proxy.service.prod.consul:5432/sogo/sogo_user_profile"; - OCSFolderInfoURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@psql-proxy.service.prod.consul:5432/sogo/sogo_folder_info"; - OCSSessionsFolderURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@psql-proxy.service.prod.consul:5432/sogo/sogo_sessions_folder"; - OCSEMailAlarmsFolderURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@psql-proxy.service.prod.consul:5432/sogo/sogo_alarms_folder"; - OCSStoreURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@psql-proxy.service.prod.consul:5432/sogo/sogo_store"; - OCSAclURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@psql-proxy.service.prod.consul:5432/sogo/sogo_acl"; - OCSCacheFolderURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@psql-proxy.service.prod.consul:5432/sogo/sogo_cache_folder"; + SOGoProfileURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@{{ env "meta.site" }}.psql-proxy.service.prod.consul:5432/sogo/sogo_user_profile"; + OCSFolderInfoURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@{{ env "meta.site" }}.psql-proxy.service.prod.consul:5432/sogo/sogo_folder_info"; + OCSSessionsFolderURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@{{ env "meta.site" }}.psql-proxy.service.prod.consul:5432/sogo/sogo_sessions_folder"; + OCSEMailAlarmsFolderURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@{{ env "meta.site" }}.psql-proxy.service.prod.consul:5432/sogo/sogo_alarms_folder"; + OCSStoreURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@{{ env "meta.site" }}.psql-proxy.service.prod.consul:5432/sogo/sogo_store"; + OCSAclURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@{{ env "meta.site" }}.psql-proxy.service.prod.consul:5432/sogo/sogo_acl"; + OCSCacheFolderURL = "postgresql://{{ key "secrets/email/sogo/postgre_auth" | trimSpace }}@{{ env "meta.site" }}.psql-proxy.service.prod.consul:5432/sogo/sogo_cache_folder"; SOGoTimeZone = "Europe/Paris"; SOGoMailDomain = "deuxfleurs.fr"; SOGoLanguage = French; diff --git a/cluster/prod/app/matrix/config/synapse/homeserver.yaml b/cluster/prod/app/matrix/config/synapse/homeserver.yaml index aac8709..ecdf1cd 100644 --- a/cluster/prod/app/matrix/config/synapse/homeserver.yaml +++ b/cluster/prod/app/matrix/config/synapse/homeserver.yaml @@ -61,7 +61,7 @@ database: user: {{ key "secrets/chat/synapse/postgres_user" | trimSpace }} password: {{ key "secrets/chat/synapse/postgres_pwd" | trimSpace }} database: {{ key "secrets/chat/synapse/postgres_db" | trimSpace }} - host: psql-proxy.service.prod.consul + host: {{ env "meta.site" }}.psql-proxy.service.prod.consul port: 5432 cp_min: 5 cp_max: 10 diff --git a/cluster/prod/app/matrix/deploy/im.hcl b/cluster/prod/app/matrix/deploy/im.hcl index ed05ffc..6b9690d 100644 --- a/cluster/prod/app/matrix/deploy/im.hcl +++ b/cluster/prod/app/matrix/deploy/im.hcl @@ -123,7 +123,7 @@ AWS_DEFAULT_REGION=garage PG_USER={{ key "secrets/chat/synapse/postgres_user" | trimSpace }} PG_PASS={{ key "secrets/chat/synapse/postgres_pwd" | trimSpace }} PG_DB={{ key "secrets/chat/synapse/postgres_db" | trimSpace }} -PG_HOST=psql-proxy.service.2.cluster.deuxfleurs.fr +PG_HOST={{ env "meta.site" }}.psql-proxy.service.2.cluster.deuxfleurs.fr PG_PORT=5432 EOH destination = "secrets/env" From b73c39c7c17d06468f5efc6c7a9c99bf09ec4f1f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 4 May 2023 17:00:31 +0200 Subject: [PATCH 5/6] multi-zone matrix --- cluster/prod/app/matrix/deploy/im.hcl | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/cluster/prod/app/matrix/deploy/im.hcl b/cluster/prod/app/matrix/deploy/im.hcl index 6b9690d..324c3d9 100644 --- a/cluster/prod/app/matrix/deploy/im.hcl +++ b/cluster/prod/app/matrix/deploy/im.hcl @@ -1,5 +1,5 @@ job "matrix" { - datacenters = ["orion"] + datacenters = ["orion", "neptune"] type = "service" priority = 40 @@ -8,6 +8,7 @@ job "matrix" { network { port "api_port" { static = 8008 } + port "web_port" { to = 8043 } } task "synapse" { @@ -79,6 +80,7 @@ job "matrix" { "tricot im.deuxfleurs.fr:443/_matrix 100", "tricot im.deuxfleurs.fr/_synapse 100", "tricot-add-header Access-Control-Allow-Origin *", + "d53-cname im.deuxfleurs.fr", ] check { type = "tcp" @@ -130,17 +132,8 @@ EOH env = true } } - } - - group "riotweb" { - count = 1 - - network { - port "web_port" { to = 8043 } - } - - task "server" { + task "riotweb" { driver = "docker" config { image = "superboum/amd64_riotweb:v33" @@ -164,6 +157,7 @@ EOH "webstatic", "tricot im.deuxfleurs.fr 10", "tricot riot.deuxfleurs.fr 10", + "d53-cname riot.deuxfleurs.fr", ] port = "web_port" address_mode = "host" From 24192cc61a982402e201d6dde4fa5ac2994e025f Mon Sep 17 00:00:00 2001 From: Maximilien Richer Date: Sun, 7 May 2023 23:46:48 +0200 Subject: [PATCH 6/6] Update telemetry stack apps --- cluster/prod/app/telemetry/deploy/telemetry-service.hcl | 2 +- cluster/prod/app/telemetry/deploy/telemetry-storage.hcl | 2 +- cluster/prod/app/telemetry/deploy/telemetry-system.hcl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/prod/app/telemetry/deploy/telemetry-service.hcl b/cluster/prod/app/telemetry/deploy/telemetry-service.hcl index afa8a8d..9ec43ae 100644 --- a/cluster/prod/app/telemetry/deploy/telemetry-service.hcl +++ b/cluster/prod/app/telemetry/deploy/telemetry-service.hcl @@ -45,7 +45,7 @@ job "telemetry-service" { task "grafana" { driver = "docker" config { - image = "grafana/grafana:9.3.2" + image = "grafana/grafana:9.5.1" network_mode = "host" ports = [ "grafana" ] volumes = [ diff --git a/cluster/prod/app/telemetry/deploy/telemetry-storage.hcl b/cluster/prod/app/telemetry/deploy/telemetry-storage.hcl index d4667fa..b012e3f 100644 --- a/cluster/prod/app/telemetry/deploy/telemetry-storage.hcl +++ b/cluster/prod/app/telemetry/deploy/telemetry-storage.hcl @@ -20,7 +20,7 @@ job "telemetry-storage" { task "prometheus" { driver = "docker" config { - image = "prom/prometheus:v2.41.0" + image = "prom/prometheus:v2.43.1" network_mode = "host" ports = [ "prometheus" ] args = [ diff --git a/cluster/prod/app/telemetry/deploy/telemetry-system.hcl b/cluster/prod/app/telemetry/deploy/telemetry-system.hcl index ae9ff72..a861c61 100644 --- a/cluster/prod/app/telemetry/deploy/telemetry-system.hcl +++ b/cluster/prod/app/telemetry/deploy/telemetry-system.hcl @@ -12,7 +12,7 @@ job "telemetry-system" { driver = "docker" config { - image = "quay.io/prometheus/node-exporter:v1.4.0" + image = "quay.io/prometheus/node-exporter:v1.5.0" network_mode = "host" volumes = [ "/:/host:ro,rslave"