Allow grafana to move between nodes

app/telemetry/config/grafana-litestream.yml

@@ -0,0 +1,10 @@
+dbs:
+  - path: /ephemeral/grafana.db
+    replicas:
+      - url: s3://grafana-db/grafana.db
+        region: garage-staging
+        endpoint: http://{{ env "attr.unique.network.ip-address" }}:3990
+        access-key-id: {{ key "secrets/telemetry/grafana/s3_access_key" | trimSpace }}
+        secret-access-key:  {{ key "secrets/telemetry/grafana/s3_secret_key" | trimSpace }}
+        force-path-style: true
+        sync-interval: 60s

app/telemetry/deploy/telemetry-system.hcl

@@ -30,7 +30,7 @@ job "telemetry-system" {
       }
 
       resources {
-        memory = 2500
+        memory = 1500
         cpu = 500
       }
 

app/telemetry/deploy/telemetry.hcl

@@ -31,7 +31,7 @@ EOH
       }
 
       resources {
-        memory = 1500
+        memory = 1000
         cpu = 500
       }
 
@@ -62,18 +62,42 @@ EOH
   group "grafana" {
     count = 1
 
-    constraint {
-      attribute = "${attr.unique.hostname}"
-      operator = "="
-      value = "carcajou"
-    }
-
     network {
       port "grafana" {
         static = 3333
       }
     }
 
+    task "restore-db" {
+      lifecycle {
+        hook = "prestart"
+        sidecar = false
+      }
+
+      driver = "docker"
+      config {
+        image = "litestream/litestream:0.3.7"
+        args = [
+          "restore", "-config", "/etc/litestream.yml", "/ephemeral/grafana.db"
+        ]
+        volumes = [
+          "../alloc/data:/ephemeral",
+          "secrets/litestream.yml:/etc/litestream.yml"
+        ]
+      }
+      user = "472"
+
+      template {
+        data = file("../config/grafana-litestream.yml")
+        destination = "secrets/litestream.yml"
+      }
+
+      resources {
+        memory = 200
+        cpu = 1000
+      }
+    }
+
     task "grafana" {
       driver = "docker"
       config {
@@ -81,7 +105,7 @@ EOH
         network_mode = "host"
         ports = [ "grafana" ]
         volumes = [
-          "/mnt/ssd/telemetry/grafana:/var/lib/grafana",
+          "../alloc/data:/var/lib/grafana",
           "secrets/elastic.yaml:/etc/grafana/provisioning/datasources/elastic.yaml"
         ]
       }
@@ -104,7 +128,53 @@ EOH
         memory = 500
         cpu = 100
       }
+
+      service {
+        tags = [
+            "grafana",
+            "tricot grafana.home.adnab.me",
+        ]
+        port = 3333
+        address_mode = "driver"
+        name = "grafana"
+        check {
+          type = "tcp"
+          port = 3333
+          address_mode = "driver"
+          interval = "60s"
+          timeout = "5s"
+          check_restart {
+            limit = 3
+            grace = "90s"
+            ignore_warnings = false
+          }
+        }
+      }
     }
 
+    task "replicate-db" {
+      driver = "docker"
+      config {
+        image = "litestream/litestream:0.3.7"
+        args = [
+          "replicate", "-config", "/etc/litestream.yml"
+        ]
+        volumes = [
+          "../alloc/data:/ephemeral",
+          "secrets/litestream.yml:/etc/litestream.yml"
+        ]
+      }
+      user = "472"
+
+      template {
+        data = file("../config/grafana-litestream.yml")
+        destination = "secrets/litestream.yml"
+      }
+
+      resources {
+        memory = 200
+        cpu = 100
+      }
+    }
   }
 }