working on better builder

.albatros

@@ -2,9 +2,9 @@
 set -euxo pipefail
 nix build
 
-if [[ $BRANCH == "main" && -f $SECRET_PATH ]]; then
-  mkdir -p /kaniko/.docker
-  cp $SECRET_PATH /kaniko/.docker/config.json
-  export HOME=/kaniko
-  nix develop --command sh -c "executor --force --destination dxflrs/albatros:${COMMIT} --context dir://`pwd` --verbosity=debug" 1>&2
-fi
+#if [[ $BRANCH == "main" && -f $SECRET_PATH ]]; then
+#  mkdir -p /kaniko/.docker
+#  cp $SECRET_PATH /kaniko/.docker/config.json
+#  export HOME=/kaniko
+#  nix develop --command sh -c "executor --force --destination dxflrs/albatros:${COMMIT} --context dir://`pwd` --verbosity=debug" 1>&2
+#fi

README.md

@@ -59,8 +59,20 @@ And... that's basically all you need.
 
 ### A NixOS builder with local cache
 
-Currently, we propose a single, nix-based builder.
-I am currently working on using proper caching on it.
+The nix builder will create a local cache on your node agent at `/var/cache/albatros`.
+It will create two folders: `/var/cache/albatros/nix` that contains a nix store,
+and `/var/cache/albatros/cacheroot` that contains nix roots, ie. the builds we must keep.
+
+Caching is handled thanks to these roots: roots that are older than 7 days will be removed,
+then a garbage collection will be triggered, and their associate content will be deleted.
+
+To keep your cache between builds, you must put your outlink in `/mnt/cacheroot` with a unique name.
+The git commit is a good candidate:
+
+```bash
+# create an output link at a persisted path
+nix build --out-link $GCROOT
+```
 
 ## Register a build
 

hcl/builder.hcl

@@ -37,19 +37,7 @@ job "builder" {
     }
 
     template {
-      data = <<EOH
-#!/usr/bin/env bash
-set -euxo pipefail
-
-if [[ ! -f /mnt/nix/nixos-22.11 ]]; then
-  rm -rf /mnt/nix/*
-  cp -r /nix/* /mnt/nix/
-  touch /mnt/nix/nixos-22.11
-  echo "initialized cache"
-else
-  echo "cache already initialized"
-fi
-      EOH
+      data = file("./warmup.sh")
       destination = "local/warmup.sh"
       perms = "555"
     }
@@ -65,7 +53,8 @@ fi
         "local/builder.sh:/tmp/builder.sh",
         "local/nix.conf:/etc/nix/nix.conf",
         "local/secrets:/var/run/secrets/albatros",
-        "/var/cache/albatros/nix:/nix:ro"
+        "/var/cache/albatros/nix:/nix:ro",
+        "/var/cache/albatros/cacheroot:/mnt/cacheroot"
       ]
     }
 
@@ -78,49 +67,18 @@ fi
       BRANCH = "${NOMAD_META_BRANCH}"
       REPO_URL = "${NOMAD_META_REPO_URL}"
       FLAVOR = "${NOMAD_META_FLAVOR}"
+      GCROOT = "/mnt/cacheroot/${NOMAD_META_COMMIT}"
       NIX_REMOTE = "daemon"     
     }
 
     template {
-      data = <<EOH
-#!/usr/bin/env bash
-set -euxo pipefail
-
-# setup workspace
-mkdir /workspace
-cd /workspace
-git init 
-git remote add origin ${NOMAD_META_REPO_URL}
-git fetch origin ${NOMAD_META_COMMIT}
-git checkout ${NOMAD_META_COMMIT} -b ${NOMAD_META_BRANCH}
-if [[ -s /var/run/secrets/albatros/secret.txt ]]; then
-  export SECRET_PATH=/var/run/secrets/albatros/secret.txt
-fi
-
-# wait for nix daemon during 120 seconds
-timeout -s 9 120 bash -c 'until nix store ping; do sleep 2; done'
-
-# launch user's build
-./.albatros
-
-      EOH
+      data = file("./user.sh")
       destination = "local/builder.sh"
       perms = "555"
     }
 
     template {
-      data = <<EOH
-substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
-trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
-max-jobs = auto
-cores = 0
-log-lines = 200
-filter-syscalls = false
-sandbox = false
-keep-outputs = true
-keep-derivations = true
-experimental-features = nix-command flakes
-      EOH
+      data = file("./nix.conf")
       destination = "local/nix.conf"
     }
 
@@ -141,26 +99,22 @@ experimental-features = nix-command flakes
       image = "nixpkgs/nix:nixos-22.11"
       command = "nix-daemon"
       volumes = [ 
-        "/var/cache/albatros/nix:/nix"
+        "/var/cache/albatros/nix:/nix",
+        "local/pin.sh:/usr/local/bin/pin.sh"
       ]
     }
 
     template {
-      data = <<EOH
-substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
-trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
-max-jobs = auto
-cores = 0
-log-lines = 200
-filter-syscalls = false
-sandbox = false
-keep-outputs = true
-keep-derivations = true
-experimental-features = nix-command flakes
-      EOH
+      data = file("./nix.conf")
       destination = "local/nix.conf"
     }
 
+    template {
+      data = file("./pin.sh")
+      destination = "local/pin.sh"
+      perms = "555"
+    }
+
     restart {
       attempts = 0
     }

hcl/nix.conf

@@ -0,0 +1,9 @@
+substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
+trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
+max-jobs = auto
+cores = 0
+log-lines = 200
+filter-syscalls = false
+sandbox = false
+experimental-features = nix-command flakes
+post-build-hook = /usr/local/bin/pin.sh

hcl/pin.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+echo "hook on $DRV_PATH"

hcl/user.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# setup workspace
+mkdir /workspace
+cd /workspace
+git init 
+git remote add origin ${NOMAD_META_REPO_URL}
+git fetch origin ${NOMAD_META_COMMIT}
+git checkout ${NOMAD_META_COMMIT} -b ${NOMAD_META_BRANCH}
+if [[ -s /var/run/secrets/albatros/secret.txt ]]; then
+  export SECRET_PATH=/var/run/secrets/albatros/secret.txt
+fi
+
+# wait for nix daemon during 120 seconds
+timeout -s 9 120 bash -c 'until nix store ping; do sleep 2; done'
+
+# launch user's build
+./.albatros

hcl/warmup.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+if [[ ! -f /mnt/nix/nixos-22.11 ]]; then
+  rm -rf /mnt/nix/*
+  cp -r /nix/* /mnt/nix/
+  touch /mnt/nix/nixos-22.11
+  echo "initialized cache"
+else
+  echo "cache already initialized"
+fi