From a1a2a837275afc546713852164ff9b29aab13881 Mon Sep 17 00:00:00 2001
From: Alex Auvolat <alex@adnab.me>
Date: Fri, 2 Dec 2022 11:59:32 +0100
Subject: [PATCH] Staging: let nodes use each other as Nix caches (only inside
 same site)

---
 cluster/staging/cluster.nix | 84 +++++++++++++++++++++++++++----------
 1 file changed, 61 insertions(+), 23 deletions(-)

diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix
index 7582992..c387a22 100644
--- a/cluster/staging/cluster.nix
+++ b/cluster/staging/cluster.nix
@@ -72,36 +72,74 @@
   # For Garage ipv6 communication
   networking.firewall.allowedTCPPorts = [ 3991 ];
 
-  ## -----
+  ## ===== EXPERIMENTAL SECTION FOR STAGING CLUSTER =====
 
-  ## EXPERIMENTAL ON STAGING: NIX NOMAD JOBS
-  services.nomad.dropPrivileges = false;
+  # We're doing lots of experiments so GC periodically is usefull.
+  nix.gc.automatic = true;
 
-  # ----- nomad-driver-nix & nomad-driver-nix2 -----
-  services.nomad.extraSettingsPlugins = [
-    (import ./nomad-driver-nix2.nix { inherit pkgs; })
-  ];
-  services.nomad.extraPackages = [
-    pkgs.nix
-    pkgs.git
-  ];
-  # default config for the nix2 driver
-  services.nomad.settings.plugin = [
-    {
-      "nix2-driver" = [
+  imports = [
+    ## ---- Nix Nomad jobs using nomad-driver-nix2 ----
+    ({ pkgs, ... }: {
+      services.nomad.dropPrivileges = false;
+      services.nomad.extraSettingsPlugins = [
+        (import ./nomad-driver-nix2.nix { inherit pkgs; })
+      ];
+      services.nomad.extraPackages = [
+        pkgs.nix
+        pkgs.git
+      ];
+      services.nomad.settings.plugin = [
         {
-          config = [
+          "nix2-driver" = [
             {
-              # default_nixpkgs = "github:nixos/nixpkgs/nixos-22.11";
+              config = [
+                {
+                  default_nixpkgs = "github:nixos/nixpkgs/nixos-22.11";
+                }
+              ];
             }
           ];
         }
       ];
-    }
+    })
+    ## ---- Nix cache: use our cache on Garage (prod cluster) ----
+    # Use our cache as additionnal substituer (this acts the same way for
+    # our Nix packages than the Docker hub acts for our Docker images)
+    ({ pkgs, ... }: {
+      nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ];
+      nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ];
+    })
+    ## ---- Nix mutual cache ----
+    # Let nodes in a same site/zone copy from each other's Nix stores
+    # Note that nodes will only copy from one another packages that are
+    # signed by one of the trusted public keys, i.e. packages comming
+    # from cache.nixos.org and nix.web.deuxfleurs.fr.
+    # This is good as it kind of mitigates supply-chain attacks where
+    # one node's cache would become poisonned, although arguably when
+    # an attacker has gained root access on one node, it can easily
+    # become root on all the others through Nomad. Downsides include
+    # missed opportunities for not rebuilding stuff between machines
+    # (e.g. derivations that are built in the process of doing
+    # nixos-rebuild), and warnings appearing in the logs whenever such
+    # an opportunity was not taken due to missing signatures.
+    ({ pkgs, config, ... }:
+      let substituter_port = 1728;
+      in
+      {
+        services.nix-serve = {
+          enable = true;
+          port = substituter_port;
+          openFirewall = false;
+          bindAddress = config.deuxfleurs.cluster_ip;
+          package = pkgs.haskellPackages.nix-serve-ng;
+        };
+        nix.settings.substituters = map
+            ({ IP, ... }: "http://${IP}:${builtins.toString substituter_port}")
+            (builtins.filter
+              ({ site_name, IP, ...}:
+                (IP != config.deuxfleurs.cluster_ip
+                && site_name == config.deuxfleurs.site_name))
+              config.deuxfleurs.cluster_nodes);
+      })
   ];
-
-  # use our cache as additionnal substituer (we put precompiled packages there,
-  # like we used to do on the docker hub)
-  nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ];
-  nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ];
 }