nixcfg/cluster/staging/cluster.nix

{ config, pkgs, ... } @ args:

{
  deuxfleurs.cluster_name = "staging";

  # The IP range to use for the Wireguard overlay of this cluster
  deuxfleurs.cluster_prefix = "10.14.0.0";
  deuxfleurs.cluster_prefix_length = 16;

  deuxfleurs.cluster_nodes = [
    {
      hostname = "carcajou";
      site_name = "neptune";
      publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
      IP = "10.14.1.2";
      endpoint = "77.207.15.215:33722";
    }
    {
      hostname = "caribou";
      site_name = "neptune";
      publicKey = "lABn/axzD1jkFulX8c+K3B3CbKXORlIMDDoe8sQVxhs=";
      IP = "10.14.1.3";
      endpoint = "77.207.15.215:33723";
    }
    {
      hostname = "origan";
      site_name = "jupiter";
      publicKey = "smBQYUS60JDkNoqkTT7TgbpqFiM43005fcrT6472llI=";
      IP = "10.14.2.33";
      endpoint = "82.64.238.84:33733";
    }
    {
      hostname = "piranha";
      site_name = "corrin";
      publicKey = "m9rLf+233X1VColmeVrM/xfDGro5W6Gk5N0zqcf32WY=";
      IP = "10.14.3.1";
      #endpoint = "82.120.233.78:33721";
    }
    {
      hostname = "df-pw5";
      site_name = "bespin";
      publicKey = "XLOYoMXF+PO4jcgfSVAk+thh4VmWx0wzWnb0xs08G1s=";
      IP = "10.14.4.1";
      endpoint = "bitfrost.fiber.shirokumo.net:33734";
    }
  ];
  services.wgautomesh.logLevel = "debug";

  # Bootstrap IPs for Consul cluster,
  # these are IPs on the Wireguard overlay
  services.consul.extraConfig.retry_join = [
    "10.14.1.1"  # cariacou
    "10.14.1.2" # carcajou
    "10.14.1.3"  # caribou
  ];

  deuxfleurs.admin_accounts = {
    lx = [
      # Keys for accessing nodes from outside
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy"
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIw+IIX8+lZX9RrHAbwi/bncLYStXpI4EmK3AUcqPY2O lx@kusanagi "
    ];
    quentin = [
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDT1+H08FdUSvdPpPKdcafq4+JRHvFVjfvG5Id97LAoROmFRUb/ZOMTLdNuD7FqvW0Da5CPxIMr8ZxfrFLtpGyuG7qdI030iIRZPlKpBh37epZHaV+l9F4ZwJQMIBO9cuyLPXgsyvM/s7tDtrdK1k7JTf2EVvoirrjSzBaMhAnhi7//to8zvujDtgDZzy6aby75bAaDetlYPBq2brWehtrf9yDDG9WAMYJqp//scje/WmhbRR6eSdim1HaUcWk5+4ZPt8sQJcy8iWxQ4jtgjqTvMOe5v8ZPkxJNBine/ZKoJsv7FzKem00xEH7opzktaGukyEqH0VwOwKhmBiqsX2yN quentin@dufour.io"
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBu+KUebaWwlugMC5fGbNhHc6IaQDAC6+1vMc4Ww7nVU1rs2nwI7L5qcWxOwNdhFaorZQZy/fJuCWdFbF61RCKGayBWPLZHGPsfqDuggYNEi1Qil1kpeCECfDQNjyMTK058ZBBhOWNMHBjlLWXUlRJDkRBBECY0vo4jRv22SvSaPUCAnkdJ9rbAp/kqb497PTIb2r1l1/ew8YdhINAlpYQFQezZVfkZdTKxt22n0QCjhupqjfh3gfNnbBX0z/iO+RvAOWRIZsjPFLC+jXl+n7cnu2cq1nvST5eHiYfXXeIgIwmeENLKqp+2Twr7PIdv22PnJkh6iR5kx7eTRxkNZdN quentin@deuxfleurs.fr"
    ];
    adrien = [
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINvUFN6HmZS5oxxOtmF6ug393m5NYbSbDI4G8pX6H9GZ adrien@pratchett"
    ];
    maximilien = [
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHMMR6zNzz8NQU80wFquhUCeiXJuGphjP+zNouKbn228GyESu8sfNBwnuZq86vblR11Lz8l2rtCM73GfAKg29qmUWUHRKWvRIYWv2vaUJcCdy0bAxIzcvCvjZX0SpnIKxe9y3Rp0LGO5WLYfw0ZFaavwFZP0Z8w1Kj9/zBmL2X2avbhkaYHi/C1yXhbvESYQysmqLa48EX/TS616MBrgR9zbI9AoTQ9NOHnR14Tve/AP/khcZoBJdm4hTttMbNkEc0wonzdylTDew263SPRs/uoqnQIpUtErdPHqU10Yup8HjXjEyFJsSwcZcM5sZOw5JKckKJwmcd0yjO/x/4/Mk5"
    ];
    trinity = [
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDWGWTRoF5MjQ5bmFdQENQlNdoYtA7Wd61GM0TMHZDki"
    ];
    kokakiwi = [
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPTsEgcOtb2bij+Ih8eg8ZqO7d3IMiWykv6deMzlSSS kokakiwi@kira"
    ];
    baptiste = [
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCnGkJZZrHIUp9q0DXmVLLuhCIe7Vu1J3j6dJ1z1BglqX7yOLdFQ6LhHXx65aND/KCOM1815tJSnaAyKWEj9qJ31RVUoRl42yBn54DvQumamJUaXAHqJrXhjwxfUkF9B73ZSUzHGADlQnxcBkmrjC5FkrpC/s4xr0o7/GIBkBdtZhX9YpxBfpH6wEcCruTOlm92E3HvvjpBb/wHsoxL1f2czvWe69021gqWEYRFjqtBwP36NYZnGOJZ0RrlP3wUrGCSHxOKW+2Su+tM6g07KPJn5l1wNJiOcyBQ0/Sv7ptCJ9+rTQNeVBMoXshaucYP/bKJbqH7dONrYDgz59C4+Kax"
    ];
  };

  # For Garage ipv6 communication
  networking.firewall.allowedTCPPorts = [ 3991 ];

  ## ===== EXPERIMENTAL SECTION FOR STAGING CLUSTER =====

  # We're doing lots of experiments so GC periodically is usefull.
  nix.gc.automatic = true;

  imports = [
    ## ---- Nix Nomad jobs using nomad-driver-nix2 ----
    ({ pkgs, ... }: {
      services.nomad.dropPrivileges = false;
      services.nomad.extraSettingsPlugins = [
        (import ./nomad-driver-nix2.nix { inherit pkgs; })
      ];
      services.nomad.extraPackages = [
        pkgs.nix
        pkgs.git
      ];
      services.nomad.settings.plugin = [
        {
          "nix2-driver" = [
            {
              config = [
                {
                  default_nixpkgs = "github:nixos/nixpkgs/nixos-22.11";
                }
              ];
            }
          ];
        }
      ];
    })
    ## ---- Nix cache: use our cache on Garage (prod cluster) ----
    # Use our cache as additionnal substituer (this acts the same way for
    # our Nix packages than the Docker hub acts for our Docker images)
    ({ pkgs, ... }: {
      nix.settings.substituters = [ "https://nix.web.deuxfleurs.fr" ];
      nix.settings.trusted-public-keys = [ "nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=" ];
    })
    ## ---- Nix mutual cache ----
    # Let nodes in a same site/zone copy from each other's Nix stores
    # Note that nodes will only copy from one another packages that are
    # signed by one of the trusted public keys, i.e. packages comming
    # from cache.nixos.org and nix.web.deuxfleurs.fr.
    # This is good as it kind of mitigates supply-chain attacks where
    # one node's cache would become poisonned, although arguably when
    # an attacker has gained root access on one node, it can easily
    # become root on all the others through Nomad. Downsides include
    # missed opportunities for not rebuilding stuff between machines
    # (e.g. derivations that are built in the process of doing
    # nixos-rebuild), and warnings appearing in the logs whenever such
    # an opportunity was not taken due to missing signatures.
    ({ pkgs, config, ... }:
      let substituter_port = 1728;
      in
      {
        services.nix-serve = {
          enable = true;
          port = substituter_port;
          openFirewall = false;
          bindAddress = config.deuxfleurs.cluster_ip;
          package = pkgs.haskellPackages.nix-serve-ng;
        };
        nix.settings.substituters = map
            ({ IP, ... }: "http://${IP}:${builtins.toString substituter_port}")
            (builtins.filter
              ({ site_name, IP, ...}:
                (IP != config.deuxfleurs.cluster_ip
                && site_name == config.deuxfleurs.site_name))
              config.deuxfleurs.cluster_nodes);
      })
  ];
}