Deuxfleurs/nixcfg
17
6
Fork 2
Code Issues 15 Pull requests 2 Projects Releases Wiki Activity

Staging: let nodes use each other as Nix caches (only inside same site)

Browse source
This commit is contained in:
Alex 2022-12-02 11:59:32 +01:00
parent 88ddfea4d5
commit a1a2a83727
Signed by: lx
GPG key ID: 0E496D15096376BE
1 changed files with 61 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
cluster/staging/cluster.nix
View file

@ -72,12 +72,15 @@
# For Garage ipv6 communication
networking.firewall.allowedTCPPorts = [ 3991 ];
## -----
## ===== EXPERIMENTAL SECTION FOR STAGING CLUSTER =====
## EXPERIMENTAL ON STAGING: NIX NOMAD JOBS
# We're doing lots of experiments so GC periodically is usefull.
nix.gc.automatic = true;
imports = [
## ---- Nix Nomad jobs using nomad-driver-nix2 ----
({ pkgs, ... }: {
services.nomad.dropPrivileges = false;
# ----- nomad-driver-nix & nomad-driver-nix2 -----
services.nomad.extraSettingsPlugins = [
(import ./nomad-driver-nix2.nix { inherit pkgs; })
];
@ -85,23 +88,58 @@
pkgs.nix
pkgs.git
];
# default config for the nix2 driver
services.nomad.settings.plugin = [
{
"nix2-driver" = [
{
config = [
{
# default_nixpkgs = "github:nixos/nixpkgs/nixos-22.11";
default_nixpkgs = "github:nixos/nixpkgs/nixos-22.11";
}
];
}
];
}
];
# use our cache as additionnal substituer (we put precompiled packages there,
# like we used to do on the docker hub)
})
## ---- Nix cache: use our cache on Garage (prod cluster) ----
# Use our cache as additionnal substituer (this acts the same way for
# our Nix packages than the Docker hub acts for our Docker images)
({ pkgs, ... }: {
nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ];
nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ];
})
## ---- Nix mutual cache ----
# Let nodes in a same site/zone copy from each other's Nix stores
# Note that nodes will only copy from one another packages that are
# signed by one of the trusted public keys, i.e. packages comming
# from cache.nixos.org and nix.web.deuxfleurs.fr.
# This is good as it kind of mitigates supply-chain attacks where
# one node's cache would become poisonned, although arguably when
# an attacker has gained root access on one node, it can easily
# become root on all the others through Nomad. Downsides include
# missed opportunities for not rebuilding stuff between machines
# (e.g. derivations that are built in the process of doing
# nixos-rebuild), and warnings appearing in the logs whenever such
# an opportunity was not taken due to missing signatures.
({ pkgs, config, ... }:
let substituter_port = 1728;
in
{
services.nix-serve = {
enable = true;
port = substituter_port;
openFirewall = false;
bindAddress = config.deuxfleurs.cluster_ip;
package = pkgs.haskellPackages.nix-serve-ng;
};
nix.settings.substituters = map
({ IP, ... }: "http://${IP}:${builtins.toString substituter_port}")
(builtins.filter
({ site_name, IP, ...}:
(IP != config.deuxfleurs.cluster_ip
&& site_name == config.deuxfleurs.site_name))
config.deuxfleurs.cluster_nodes);
})
];
}