From 945dd4fa9ae898562648bc33b4798adaaca08be4 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Thu, 17 Nov 2022 00:17:56 +0100
Subject: [PATCH] Run Garage as a Nomad Nix job on staging cluster

---
 .../staging/app/garage/deploy/garage-nix.hcl  | 218 ++++++++++++++++++
 cluster/staging/cluster.nix                   |  20 ++
 cluster/staging/node/cariacou.nix             |  18 --
 3 files changed, 238 insertions(+), 18 deletions(-)
 create mode 100644 cluster/staging/app/garage/deploy/garage-nix.hcl

diff --git a/cluster/staging/app/garage/deploy/garage-nix.hcl b/cluster/staging/app/garage/deploy/garage-nix.hcl
new file mode 100644
index 0000000..373fd10
--- /dev/null
+++ b/cluster/staging/app/garage/deploy/garage-nix.hcl
@@ -0,0 +1,218 @@
+job "garage-staging" {
+  type = "system"
+  #datacenters = [ "neptune", "pluton" ]
+  datacenters = [ "neptune" ]
+
+  priority = 80
+
+  constraint {
+    attribute = "${attr.cpu.arch}"
+    value     = "amd64"
+  }
+
+  group "garage-staging" {
+    network {
+      port "s3" { static = 3990 }
+      port "rpc" { static = 3991 }
+      port "web" { static = 3992 }
+      port "k2v" { static = 3993 }
+      port "admin" { static = 3909 }
+    }
+
+    update {
+      max_parallel = 1
+      min_healthy_time = "30s"
+      healthy_deadline = "5m"
+    }
+
+    # task "repair" {
+    #   lifecycle {
+    #     hook = "prestart"
+    #     sidecar = false
+    #   }
+
+    #   driver = "docker"
+
+    #   config {
+    #     image = "dxflrs/amd64_garage:v0.7.99-k2v"
+    #     command = "/garage"
+    #     args = [ "offline-repair", "--yes", "object_counters" ]
+    #     network_mode = "host"
+    #     volumes = [
+    #       "/mnt/storage/garage-staging/data:/data",
+    #       "/mnt/ssd/garage-staging/meta:/meta",
+    #       "secrets/garage.toml:/etc/garage.toml",
+    #     ]
+    #   }
+
+    #   template {
+    #     data = file("../config/garage.toml")
+    #     destination = "secrets/garage.toml"
+    #   }
+
+    #   resources {
+    #     memory = 2000
+    #     cpu = 1000
+    #   }
+    # }
+
+    task "server" {
+      driver = "nix"
+
+      config {
+        packages = [
+          "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=nix-remove-system&rev=60c26fbc628d7b450ae39214b578ab6a30583d5c"
+        ]
+        command = [ "/bin/garage", "server" ]
+        bind = {
+          "/mnt/storage/garage-staging/data" = "/data",
+          "/mnt/ssd/garage-staging/meta" = "/meta",
+        }
+      }
+
+      template {
+        data = file("../config/garage.toml")
+        destination = "etc/garage.toml"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-ca.crt\" }}"
+        destination = "etc/garage/consul-ca.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-client.crt\" }}"
+        destination = "etc/garage/consul-client.crt"
+      }
+
+      template {
+        data = "{{ key \"secrets/consul/consul-client.key\" }}"
+        destination = "etc/garage/consul-client.key"
+      }
+
+      template {
+        data = <<EOH
+RUST_LOG=garage=debug
+EOH
+        destination = "secrets/env"
+        env = true
+      }
+
+      resources {
+        memory = 2000
+        memory_max = 3000
+        cpu = 500
+      }
+
+      kill_signal = "SIGINT"
+      kill_timeout = "20s"
+
+      service {
+        tags = [
+          "garage-staging-api",
+          "tricot garage-staging.home.adnab.me",
+          "tricot garage.staging.deuxfleurs.org",
+          "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = 3990
+        address_mode = "driver"
+        name = "garage-staging-api"
+        check {
+          type = "tcp"
+          port = 3990
+          address_mode = "driver"
+          interval = "60s"
+          timeout = "5s"
+          check_restart {
+            limit = 3
+            grace = "90s"
+            ignore_warnings = false
+          }
+        }
+      }
+
+      service {
+        tags = [
+          "garage-staging-k2v-api",
+          "tricot k2v.staging.deuxfleurs.org",
+          "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = 3993
+        address_mode = "driver"
+        name = "garage-staging-k2v-api"
+        # check {
+        #   type = "tcp"
+        #   port = 3993
+        #   address_mode = "driver"
+        #   interval = "60s"
+        #   timeout = "5s"
+        #   check_restart {
+        #     limit = 3
+        #     grace = "90s"
+        #     ignore_warnings = false
+        #   }
+        # }
+      }
+
+      service {
+        tags = ["garage-staging-rpc"]
+        port = 3991
+        address_mode = "driver"
+        name = "garage-staging-rpc"
+        check {
+          type = "tcp"
+          port = 3991
+          address_mode = "driver"
+          interval = "60s"
+          timeout = "5s"
+          check_restart {
+            limit = 3
+            grace = "90s"
+            ignore_warnings = false
+          }
+        }
+      }
+
+      service {
+        tags = [
+            "garage-staging-web",
+            "tricot *.web.staging.deuxfleurs.org",
+            "tricot staging.deuxfleurs.org",
+            "tricot matrix.home.adnab.me/.well-known/matrix/server",
+            "tricot-add-header Access-Control-Allow-Origin *",
+        ]
+        port = 3992
+        address_mode = "driver"
+        name = "garage-staging-web"
+        check {
+          type = "tcp"
+          port = 3992
+          address_mode = "driver"
+          interval = "60s"
+          timeout = "5s"
+          check_restart {
+            limit = 3
+            grace = "90s"
+            ignore_warnings = false
+          }
+        }
+      }
+
+      service {
+        tags = [
+          "garage-staging-admin",
+        ]
+        port = 3909
+        address_mode = "driver"
+        name = "garage-staging-admin"
+      }
+
+      restart { 
+        interval = "30m"  
+        attempts = 10  
+        delay    = "15s"  
+        mode     = "delay"
+      }
+    }
+  }
+}
diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix
index 6e0a589..f5cfdc4 100644
--- a/cluster/staging/cluster.nix
+++ b/cluster/staging/cluster.nix
@@ -68,4 +68,24 @@
 
   # For Garage ipv6 communication
   networking.firewall.allowedTCPPorts = [ 3991 ];
+
+  ## -----
+
+  ## EXPERIMENTAL ON STAGING: NIX NOMAD DRIVER
+  services.nomad.dropPrivileges = false;
+  services.nomad.extraPackages = [
+    pkgs.nix
+    pkgs.git
+  ];
+  services.nomad.extraSettingsPlugins =
+    let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
+    in [ nomad_driver_nix ];
+
+  # the nix driver requires flakes to be enabled
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+
+  # use our cache as additionnal substituer (we put precompiled packages there,
+  # like we used to do on the docker hub)
+  nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ];
+  nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ];
 }
diff --git a/cluster/staging/node/cariacou.nix b/cluster/staging/node/cariacou.nix
index d653979..61d3f28 100644
--- a/cluster/staging/node/cariacou.nix
+++ b/cluster/staging/node/cariacou.nix
@@ -21,22 +21,4 @@
 
   deuxfleurs.cluster_ip = "10.14.1.1";
   deuxfleurs.is_raft_server = true;
-
-  # ----
-
-  # experimental: config for nomad nix driver
-  services.nomad.dropPrivileges = false;
-  services.nomad.extraPackages = [
-    pkgs.nix
-    pkgs.git
-  ];
-  services.nomad.extraSettingsPlugins =
-    let nomad_driver_nix = import ./nomad-driver-nix.nix { inherit pkgs; };
-    in [ nomad_driver_nix ];
-  # the nix driver requires flakes to be enabled
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
-
-  # use our cache as additionnal substituer
-  nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ];
-  nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ];
 }