working on better builder

This commit is contained in:
Quentin 2023-03-24 14:35:43 +01:00
parent 71421f812c
commit 4c8ea6cec9
Signed by: quentin
GPG key ID: E9602264D639FF68
7 changed files with 78 additions and 69 deletions

View file

@ -2,9 +2,9 @@
set -euxo pipefail
nix build
if [[ $BRANCH == "main" && -f $SECRET_PATH ]]; then
mkdir -p /kaniko/.docker
cp $SECRET_PATH /kaniko/.docker/config.json
export HOME=/kaniko
nix develop --command sh -c "executor --force --destination dxflrs/albatros:${COMMIT} --context dir://`pwd` --verbosity=debug" 1>&2
fi
#if [[ $BRANCH == "main" && -f $SECRET_PATH ]]; then
# mkdir -p /kaniko/.docker
# cp $SECRET_PATH /kaniko/.docker/config.json
# export HOME=/kaniko
# nix develop --command sh -c "executor --force --destination dxflrs/albatros:${COMMIT} --context dir://`pwd` --verbosity=debug" 1>&2
#fi

View file

@ -59,8 +59,20 @@ And... that's basically all you need.
### A NixOS builder with local cache
Currently, we propose a single, nix-based builder.
I am currently working on using proper caching on it.
The nix builder will create a local cache on your node agent at `/var/cache/albatros`.
It will create two folders: `/var/cache/albatros/nix` that contains a nix store,
and `/var/cache/albatros/cacheroot` that contains nix roots, ie. the builds we must keep.
Caching is handled thanks to these roots: roots that are older than 7 days will be removed,
then a garbage collection will be triggered, and their associate content will be deleted.
To keep your cache between builds, you must put your outlink in `/mnt/cacheroot` with a unique name.
The git commit is a good candidate:
```bash
# create an output link at a persisted path
nix build --out-link $GCROOT
```
## Register a build

View file

@ -37,19 +37,7 @@ job "builder" {
}
template {
data = <<EOH
#!/usr/bin/env bash
set -euxo pipefail
if [[ ! -f /mnt/nix/nixos-22.11 ]]; then
rm -rf /mnt/nix/*
cp -r /nix/* /mnt/nix/
touch /mnt/nix/nixos-22.11
echo "initialized cache"
else
echo "cache already initialized"
fi
EOH
data = file("./warmup.sh")
destination = "local/warmup.sh"
perms = "555"
}
@ -65,7 +53,8 @@ fi
"local/builder.sh:/tmp/builder.sh",
"local/nix.conf:/etc/nix/nix.conf",
"local/secrets:/var/run/secrets/albatros",
"/var/cache/albatros/nix:/nix:ro"
"/var/cache/albatros/nix:/nix:ro",
"/var/cache/albatros/cacheroot:/mnt/cacheroot"
]
}
@ -78,49 +67,18 @@ fi
BRANCH = "${NOMAD_META_BRANCH}"
REPO_URL = "${NOMAD_META_REPO_URL}"
FLAVOR = "${NOMAD_META_FLAVOR}"
GCROOT = "/mnt/cacheroot/${NOMAD_META_COMMIT}"
NIX_REMOTE = "daemon"
}
template {
data = <<EOH
#!/usr/bin/env bash
set -euxo pipefail
# setup workspace
mkdir /workspace
cd /workspace
git init
git remote add origin ${NOMAD_META_REPO_URL}
git fetch origin ${NOMAD_META_COMMIT}
git checkout ${NOMAD_META_COMMIT} -b ${NOMAD_META_BRANCH}
if [[ -s /var/run/secrets/albatros/secret.txt ]]; then
export SECRET_PATH=/var/run/secrets/albatros/secret.txt
fi
# wait for nix daemon during 120 seconds
timeout -s 9 120 bash -c 'until nix store ping; do sleep 2; done'
# launch user's build
./.albatros
EOH
data = file("./user.sh")
destination = "local/builder.sh"
perms = "555"
}
template {
data = <<EOH
substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
max-jobs = auto
cores = 0
log-lines = 200
filter-syscalls = false
sandbox = false
keep-outputs = true
keep-derivations = true
experimental-features = nix-command flakes
EOH
data = file("./nix.conf")
destination = "local/nix.conf"
}
@ -141,26 +99,22 @@ experimental-features = nix-command flakes
image = "nixpkgs/nix:nixos-22.11"
command = "nix-daemon"
volumes = [
"/var/cache/albatros/nix:/nix"
"/var/cache/albatros/nix:/nix",
"local/pin.sh:/usr/local/bin/pin.sh"
]
}
template {
data = <<EOH
substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
max-jobs = auto
cores = 0
log-lines = 200
filter-syscalls = false
sandbox = false
keep-outputs = true
keep-derivations = true
experimental-features = nix-command flakes
EOH
data = file("./nix.conf")
destination = "local/nix.conf"
}
template {
data = file("./pin.sh")
destination = "local/pin.sh"
perms = "555"
}
restart {
attempts = 0
}

9
hcl/nix.conf Normal file
View file

@ -0,0 +1,9 @@
substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
max-jobs = auto
cores = 0
log-lines = 200
filter-syscalls = false
sandbox = false
experimental-features = nix-command flakes
post-build-hook = /usr/local/bin/pin.sh

4
hcl/pin.sh Normal file
View file

@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -euxo pipefail
echo "hook on $DRV_PATH"

19
hcl/user.sh Normal file
View file

@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -euxo pipefail
# setup workspace
mkdir /workspace
cd /workspace
git init
git remote add origin ${NOMAD_META_REPO_URL}
git fetch origin ${NOMAD_META_COMMIT}
git checkout ${NOMAD_META_COMMIT} -b ${NOMAD_META_BRANCH}
if [[ -s /var/run/secrets/albatros/secret.txt ]]; then
export SECRET_PATH=/var/run/secrets/albatros/secret.txt
fi
# wait for nix daemon during 120 seconds
timeout -s 9 120 bash -c 'until nix store ping; do sleep 2; done'
# launch user's build
./.albatros

11
hcl/warmup.sh Normal file
View file

@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euxo pipefail
if [[ ! -f /mnt/nix/nixos-22.11 ]]; then
rm -rf /mnt/nix/*
cp -r /nix/* /mnt/nix/
touch /mnt/nix/nixos-22.11
echo "initialized cache"
else
echo "cache already initialized"
fi