handle nix cache in albatros

hcl/builder.hcl

@@ -15,6 +15,46 @@ job "builder" {
     unlimited = false
   }
 
+  constraint {
+    distinct_hosts = true
+  }
+
+  group "nix" {
+  task "warmup-cache" {
+    driver = "docker"
+    config {
+      image = "nixpkgs/nix:nixos-22.11"
+      args = [ "/tmp/warmup.sh" ]
+      volumes = [ 
+        "local/warmup.sh:/tmp/warmup.sh",
+        "/var/cache/albatros/nix:/mnt/nix"
+      ]
+    }
+
+    lifecycle {
+      hook = "prestart"
+      sidecar = false
+    }
+
+    template {
+      data = <<EOH
+#!/usr/bin/env bash
+set -euxo pipefail
+
+if [[ ! -f /mnt/nix/nixos-22.11 ]]; then
+  rm -rf /mnt/nix/*
+  cp -r /nix/* /mnt/nix/
+  touch /mnt/nix/nixos-22.11
+  echo "initialized cache"
+else
+  echo "cache already initialized"
+fi
+      EOH
+      destination = "local/warmup.sh"
+      perms = "555"
+    }
+  }
+
   task "runner" {
     driver = "docker"
     config {
@@ -24,6 +64,7 @@ job "builder" {
         "local/builder.sh:/tmp/builder.sh",
         "local/nix.conf:/etc/nix/nix.conf",
         "local/secrets:/var/run/secrets/albatros",
+        "/var/cache/albatros/nix:/nix:ro"
       ]
     }
 
@@ -31,6 +72,14 @@ job "builder" {
       file = "secrets/secret.txt"
     }
 
+    env {
+      COMMIT = "${NOMAD_META_COMMIT}"
+      BRANCH = "${NOMAD_META_BRANCH}"
+      REPO_URL = "${NOMAD_META_REPO_URL}"
+      FLAVOR = "${NOMAD_META_FLAVOR}"
+      NIX_REMOTE = "daemon"     
+    }
+
     template {
       data = <<EOH
 #!/usr/bin/env bash
@@ -42,10 +91,6 @@ git init
 git remote add origin ${NOMAD_META_REPO_URL}
 git fetch origin ${NOMAD_META_COMMIT}
 git checkout ${NOMAD_META_COMMIT} -b ${NOMAD_META_BRANCH}
-export COMMIT=${NOMAD_META_COMMIT}
-export BRANCH=${NOMAD_META_BRANCH}
-export REPO_URL=${NOMAD_META_REPO_URL}
-export FLAVOR=${NOMAD_META_FLAVOR}
 if [[ -s /var/run/secrets/albatros/secret.txt ]]; then
   export SECRET_PATH=/var/run/secrets/albatros/secret.txt
 fi
@@ -76,10 +121,53 @@ experimental-features = nix-command flakes
       attempts = 0
     }
 
+    resources {
+        # actual work should be done in the nix daemon
+        cpu    = 100
+        memory = 500
+    }
+  }
+
+  task "daemon" {
+    driver = "docker"
+    config {
+      image = "nixpkgs/nix:nixos-22.11"
+      command = "nix-daemon"
+      volumes = [ 
+        "/var/cache/albatros/nix:/nix"
+      ]
+    }
+
+    template {
+      data = <<EOH
+substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
+trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
+max-jobs = auto
+cores = 0
+log-lines = 200
+filter-syscalls = false
+sandbox = false
+keep-outputs = true
+keep-derivations = true
+experimental-features = nix-command flakes
+      EOH
+      destination = "local/nix.conf"
+    }
+
+    restart {
+      attempts = 0
+    }
+
     resources {
         cpu    = 1000
         memory = 4000
     }
+
+    lifecycle {
+      hook = "poststart"
+      sidecar = true
+    }
+  }
   }
 }