cluster/staging/cluster.nix

@@ -1,49 +1,43 @@
 { config, pkgs, ... } @ args:
 
 {
-  deuxfleurs.cluster_name = "staging";
+  deuxfleurs.clusterName = "staging";
 
   # The IP range to use for the Wireguard overlay of this cluster
-  deuxfleurs.cluster_prefix = "10.14.0.0";
+  deuxfleurs.clusterPrefix = "10.14.0.0/16";
-  deuxfleurs.cluster_prefix_length = 16;
 
-  deuxfleurs.cluster_nodes = [
+  deuxfleurs.clusterNodes = {
-    {
+    "carcajou" = {
-      hostname = "carcajou";
+      siteName = "neptune";
-      site_name = "neptune";
       publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
-      IP = "10.14.1.2";
+      address = "10.14.1.2";
       endpoint = "77.207.15.215:33722";
-    }
+    };
-    {
+    "caribou" = {
-      hostname = "caribou";
+      siteName = "neptune";
-      site_name = "neptune";
       publicKey = "lABn/axzD1jkFulX8c+K3B3CbKXORlIMDDoe8sQVxhs=";
-      IP = "10.14.1.3";
+      address = "10.14.1.3";
       endpoint = "77.207.15.215:33723";
-    }
+    };
-    {
+    "origan" = {
-      hostname = "origan";
+      siteName = "jupiter";
-      site_name = "jupiter";
       publicKey = "smBQYUS60JDkNoqkTT7TgbpqFiM43005fcrT6472llI=";
-      IP = "10.14.2.33";
+      address = "10.14.2.33";
       endpoint = "82.64.238.84:33733";
-    }
+    };
-    {
+    "piranha" = {
-      hostname = "piranha";
+      siteName = "corrin";
-      site_name = "corrin";
       publicKey = "m9rLf+233X1VColmeVrM/xfDGro5W6Gk5N0zqcf32WY=";
-      IP = "10.14.3.1";
+      address = "10.14.3.1";
       #endpoint = "82.120.233.78:33721";
-    }
+    };
-    {
+    "df-pw5" = {
-      hostname = "df-pw5";
+      siteName = "bespin";
-      site_name = "bespin";
       publicKey = "XLOYoMXF+PO4jcgfSVAk+thh4VmWx0wzWnb0xs08G1s=";
-      IP = "10.14.4.1";
+      address = "10.14.4.1";
       endpoint = "bitfrost.fiber.shirokumo.net:33734";
-    }
+    };
-  ];
+  };
   services.wgautomesh.logLevel = "debug";
 
   # Bootstrap IPs for Consul cluster,
@@ -54,7 +48,7 @@
     "10.14.1.3"  # caribou
   ];
 
-  deuxfleurs.admin_accounts = {
+  deuxfleurs.adminAccounts = {
     lx = [
       # Keys for accessing nodes from outside
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy"
@@ -142,16 +136,16 @@
           enable = true;
           port = substituter_port;
           openFirewall = false;
-          bindAddress = config.deuxfleurs.cluster_ip;
+          bindAddress = "0.0.0.0";
           package = pkgs.haskellPackages.nix-serve-ng;
         };
         nix.settings.substituters = map
-            ({ IP, ... }: "http://${IP}:${builtins.toString substituter_port}")
+            ({ address, ... }: "http://${address}:${builtins.toString substituter_port}")
-            (builtins.filter
+            (builtins.attrValues (pkgs.lib.filterAttrs
-              ({ site_name, IP, ...}:
+              (hostname: { siteName, ...}:
-                (IP != config.deuxfleurs.cluster_ip
+                (hostname != config.deuxfleurs.hostName
-                && site_name == config.deuxfleurs.site_name))
+                && siteName == config.deuxfleurs.siteName))
-              config.deuxfleurs.cluster_nodes);
+              config.deuxfleurs.clusterNodes));
       })
   ];
 }

cluster/staging/node/carcajou.nix

@@ -19,11 +19,8 @@
   boot.loader.timeout = 20;
   boot.loader.efi.canTouchEfiVariables = true;
 
-  networking.hostName = "carcajou";
+  deuxfleurs.hostName = "carcajou";
-
+  deuxfleurs.ipv6Address = "2001:910:1204:1::22";
-  deuxfleurs.ipv6 = "2001:910:1204:1::22";
-
-  deuxfleurs.cluster_ip = "10.14.1.2";
 
   system.stateVersion = "21.05";
 }

cluster/staging/node/caribou.nix

@@ -8,12 +8,9 @@
   boot.loader.timeout = 20;
   boot.loader.efi.canTouchEfiVariables = true;
 
-  networking.hostName = "caribou";
+  deuxfleurs.hostName = "caribou";
-
+  deuxfleurs.ipv6Address = "2001:910:1204:1::23";
-  deuxfleurs.ipv6 = "2001:910:1204:1::23";
+  deuxfleurs.isRaftServer = true;
-
-  deuxfleurs.cluster_ip = "10.14.1.3";
-  deuxfleurs.is_raft_server = true;
 
   system.stateVersion = "21.05";
 }

cluster/staging/node/df-pw5.nix

@@ -9,13 +9,9 @@
   boot.loader.efi.efiSysMountPoint = "/boot";
   boot.loader.timeout = 20;
 
-  networking.hostName = "df-pw5";
+  deuxfleurs.hostName = "df-pw5";
-
   deuxfleurs.staticIPv4.address = "192.168.5.130/24";
-  deuxfleurs.ipv6 = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7";
+  deuxfleurs.ipv6Address = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7";
-
-  deuxfleurs.cluster_ip = "10.14.4.1";
-  deuxfleurs.is_raft_server = false;
 
   system.stateVersion = "22.11";
 }

cluster/staging/node/origan.nix

@@ -8,13 +8,10 @@
   boot.loader.timeout = 20;
   boot.loader.efi.canTouchEfiVariables = true;
 
-  networking.hostName = "origan";
+  deuxfleurs.hostName = "origan";
-
   deuxfleurs.staticIPv4.address = "192.168.1.33/24";
-  deuxfleurs.ipv6 = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec";
+  deuxfleurs.ipv6Address = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec";
-
+  deuxfleurs.isRaftServer = true;
-  deuxfleurs.cluster_ip = "10.14.2.33";
-  deuxfleurs.is_raft_server = true;
 
   system.stateVersion = "22.11";
 }

cluster/staging/node/piranha.nix

@@ -8,13 +8,10 @@
   boot.loader.timeout = 20;
   boot.loader.efi.canTouchEfiVariables = true;
 
-  networking.hostName = "piranha";
+  deuxfleurs.hostName = "piranha";
-
   deuxfleurs.staticIPv4.address = "192.168.1.25/24";
-  deuxfleurs.ipv6 = "2a01:cb05:8984:9900:223:24ff:feb0:ea82";
+  deuxfleurs.ipv6Address = "2a01:cb05:8984:9900:223:24ff:feb0:ea82";
-
+  deuxfleurs.isRaftServer = true;
-  deuxfleurs.cluster_ip = "10.14.3.1";
-  deuxfleurs.is_raft_server = true;
 
   system.stateVersion = "22.11";
 }

cluster/staging/site/bespin.nix

@@ -1,9 +1,9 @@
 { config, pkgs, ... }:
 
 {
-  deuxfleurs.site_name = "bespin";
+  deuxfleurs.siteName = "bespin";
   deuxfleurs.staticIPv4.defaultGateway = "192.168.5.254";
-  deuxfleurs.cname_target = "bespin.site.staging.deuxfleurs.org.";
+  deuxfleurs.cnameTarget = "bespin.site.staging.deuxfleurs.org.";
 
   networking.firewall.allowedTCPPorts = [ 80 443 ];
 }

cluster/staging/site/corrin.nix

@@ -1,10 +1,10 @@
 { config, pkgs, ... }:
 
 {
-  deuxfleurs.site_name = "corrin";
+  deuxfleurs.siteName = "corrin";
   deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1";
-  deuxfleurs.cname_target = "corrin.site.staging.deuxfleurs.org.";
+  deuxfleurs.cnameTarget = "corrin.site.staging.deuxfleurs.org.";
-  deuxfleurs.public_ipv4 = "82.120.233.78";
+  deuxfleurs.publicIPv4 = "82.120.233.78";
 
   networking.firewall.allowedTCPPorts = [ 80 443 ];
 }

cluster/staging/site/jupiter.nix

@@ -1,12 +1,9 @@
 { config, pkgs, ... }:
 
 {
-  deuxfleurs.site_name = "jupiter";
+  deuxfleurs.siteName = "jupiter";
   deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1";
-  deuxfleurs.cname_target = "jupiter.site.staging.deuxfleurs.org.";
+  deuxfleurs.cnameTarget = "jupiter.site.staging.deuxfleurs.org.";
-
-  # no public ipv4 is used for the staging cluster on Jupiter
-  # deuxfleurs.public_ipv4 = "???";
 
   networking.firewall.allowedTCPPorts = [ 80 443 ];
 }

cluster/staging/site/neptune.nix

@@ -1,12 +1,8 @@
 { config, pkgs, ... }:
 
 {
-  deuxfleurs.site_name = "neptune";
+  deuxfleurs.siteName = "neptune";
-  deuxfleurs.cname_target = "neptune.site.staging.deuxfleurs.org.";
+  deuxfleurs.cnameTarget = "neptune.site.staging.deuxfleurs.org.";
-
-  # no public ipv4 is used for the staging cluster on Neptune,
-  # because the Internet connection is already used for the prod cluster
-  # deuxfleurs.public_ipv4 = "77.207.15.215";
 
   networking.firewall.allowedTCPPorts = [ 80 443 ];
 }

nix/deuxfleurs.nix

@@ -6,110 +6,98 @@ in
   with builtins;
   with pkgs.lib;
 {
-  options.deuxfleurs =
+  options.deuxfleurs = with types; {
-    let wg_node = with types; submodule {
-      options = {
-        hostname = mkOption {
-          type = str;
-          description = "Host name";
-        };
-        site_name = mkOption {
-          type = nullOr str;
-          description = "Site where the node is located";
-          default = null;
-        };
-        IP = mkOption {
-          type = str;
-          description = "IP Address in the Wireguard network";
-        };
-        publicKey = mkOption {
-          type = str;
-          description = "Public key";
-        };
-        endpoint = mkOption {
-          type = nullOr str;
-          default = null;
-          description = "Wireguard endpoint on the public Internet";
-        };
-      };
-    };
-  in
-  {
     # Parameters for individual nodes
-    ipv6 = mkOption {
+    hostName = mkOption {
+      description = "Node name";
+      type = str;
+    };
+    ipv6Address = mkOption {
       description = "Static public IPv6 address of this node";
-      type = types.str;
+      type = str;
     };
     staticIPv4.address = mkOption {
       description = "IP address (with prefix length) of this node on the local network interface";
-      type = types.nullOr types.str;
+      type = nullOr str;
       default = null;
     };
 
-    cluster_ip = mkOption {
+    isRaftServer = mkOption {
-      description = "IP address of this node on the Wesher mesh network";
-      type = types.str;
-    };
-
-    is_raft_server = mkOption {
       description = "Make this node a RAFT server for the Nomad and Consul deployments";
-      type = types.bool;
+      type = bool;
       default = false;
     };
 
     # Parameters that generally vary between sites
-    site_name = mkOption {
+    siteName = mkOption {
       description = "Site (availability zone) on which this node is deployed";
-      type = types.str;
+      type = str;
     };
     staticIPv4.defaultGateway = mkOption {
       description = "IPv4 address of the default route on the local network interface";
-      type = types.nullOr types.str;
+      type = nullOr str;
       default = null;
     };
-    public_ipv4 = mkOption {
+
+    publicIPv4 = mkOption {
       description = "Public IPv4 through which this node is accessible (possibly after port opening using DiploNAT), for domain names that are updated by D53";
-      type = types.nullOr types.str;
+      type = nullOr str;
       default = null;
     };
-    cname_target = mkOption {
+    cnameTarget = mkOption {
       description = "DNS CNAME target to use for services hosted in this site, for domain names that are updated by D53";
-      type = types.nullOr types.str;
+      type = nullOr str;
       default = null;
     };
 
     # Parameters common to all nodes
-    cluster_name = mkOption {
+    clusterName = mkOption {
       description = "Name of this Deuxfleurs deployment";
-      type = types.str;
+      type = str;
     };
-    cluster_prefix = mkOption {
+    clusterPrefix = mkOption {
-      description = "IP address prefix for the Wireguard overlay network";
+      description = "IP address prefix (and length) for the Wireguard overlay network";
-      type = types.str;
+      type = str;
     };
-    cluster_prefix_length = mkOption {
+    clusterNodes = mkOption {
-      description = "IP address prefix length for the Wireguard overlay network";
-      type = types.int;
-      default = 16;
-    };
-    cluster_nodes = mkOption {
       description = "Nodes that are part of the cluster";
-      type = types.listOf wg_node;
+      type = attrsOf (submodule {
+        options = {
+          siteName = mkOption {
+            type = nullOr str;
+            description = "Site where the node is located";
+            default = null;
+          };
+          address = mkOption {
+            type = str;
+            description = "IP Address in the Wireguard network";
+          };
+          publicKey = mkOption {
+            type = str;
+            description = "Public key";
+          };
+          endpoint = mkOption {
+            type = nullOr str;
+            default = null;
+            description = "Wireguard endpoint on the public Internet";
+          };
+        };
+      });
     };
-    admin_accounts = mkOption {
+    adminAccounts = mkOption {
       description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
-      type = types.attrsOf (types.listOf types.str);
+      type = attrsOf (listOf str);
     };
     bootstrap = mkOption {
       description = "Whether to enable bootstrapping for Nomad and Consul";
-      type = types.bool;
+      type = bool;
       default = false;
     };
 
     # Options that generally stay to their default value
     wireguardPort = mkOption {
       description = "Port for incoming Wireguard VPN connections";
-      type = types.port;
+      type = port;
       default = 33799;
     };
   };
@@ -119,25 +107,29 @@ in
     ];
 
   config =
-    let node_meta = {
+    let
-        "site" = cfg.site_name;
+      clusterNodeCfg = getAttr cfg.hostName cfg.clusterNodes;
-        "public_ipv6" = cfg.ipv6;
+      clusterAddress = clusterNodeCfg.address;
+      node_meta = {
+        "site" = cfg.siteName;
+        "public_ipv6" = cfg.ipv6Address;
       } //
-      (if cfg.public_ipv4 != null
+      (if cfg.publicIPv4 != null
-        then { "public_ipv4" = cfg.public_ipv4; }
+        then { "public_ipv4" = cfg.publicIPv4; }
         else {}) //
-      (if cfg.cname_target != null
+      (if cfg.cnameTarget != null
-        then { "cname_target" = cfg.cname_target; }
+        then { "cname_target" = cfg.cnameTarget; }
         else {});
     in
   {
+    networking.hostName = cfg.hostName;
 
     # Configure admin accounts on all nodes
-    users.users = builtins.mapAttrs (name: publicKeys: {
+    users.users = mapAttrs (name: publicKeys: {
       isNormalUser = true;
       extraGroups = [ "wheel" ];
       openssh.authorizedKeys.keys = publicKeys;
-    }) cfg.admin_accounts;
+    }) cfg.adminAccounts;
 
     # Configure network interfaces
     networking.useDHCP = false;
@@ -148,7 +140,7 @@ in
           Name = "en* eth*";
         };
         ipv6AcceptRAConfig = {
-          Token = "static:${cfg.ipv6}";
+          Token = "static:${cfg.ipv6Address}";
           UseDNS = false;
         };
       } // (if cfg.staticIPv4.address == null || cfg.staticIPv4.defaultGateway == null then {
@@ -195,7 +187,7 @@ in
           # Forward .consul queries to Consul daemon
           {
             name = "consul.";
-            stub-addr = "${cfg.cluster_ip}@8600";
+            stub-addr = "${clusterAddress}@8600";
             stub-no-cache = true;
             stub-tcp-upstream = false;
             stub-tls-upstream = false;
@@ -208,7 +200,7 @@ in
 
     # Configure Wireguard VPN between all nodes
     networking.wireguard.interfaces.wg0 = {
-      ips = [ "${cfg.cluster_ip}/16" ];
+      ips = [ "${clusterAddress}/16" ];
       listenPort = cfg.wireguardPort;
       privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
       mtu = 1420;
@@ -220,28 +212,24 @@ in
       gossipSecretFile = "/var/lib/wgautomesh/gossip_secret";
       persistFile = "/var/lib/wgautomesh/state";
       upnpForwardPublicPort = 
-        let
+        if clusterNodeCfg.endpoint != null then
-          us = filter ({ hostname, ...}: hostname == config.networking.hostName) cfg.cluster_nodes;
+          strings.toInt (lists.last (split ":" clusterNodeCfg.endpoint))
-        in
-        if length us > 0 && (head us).endpoint != null then
-          strings.toInt (lists.last (split ":" (head us).endpoint))
         else null;
-      peers = map ({ publicKey, endpoint, IP, ... }: {
+      peers = attrValues (mapAttrs (hostname: { publicKey, endpoint, address, ... }: {
-        address = IP;
+        inherit address endpoint;
         pubkey = publicKey;
-        endpoint = endpoint;
+      }) cfg.clusterNodes);
-      }) cfg.cluster_nodes;
     };
     # Old code for wg-quick, we can use this as a fallback if we fail to make wgautomesh work
     # systemd.services."wg-quick-wg0".after = [ "unbound.service" ];
     # networking.wg-quick.interfaces.wg0 = {
-    #   address = [ "${cfg.cluster_ip}/16" ];
+    #   address = [ "${clusterAddress}/16" ];
     #   listenPort = cfg.wireguardPort;
     #   privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
     #   mtu = 1420;
-    #   peers = map ({ publicKey, endpoint, IP, ... }: {
+    #   peers = map ({ publicKey, endpoint, address, ... }: {
     #     inherit publicKey endpoint;
-    #     allowedIPs = [ "${IP}/32" ];
+    #     allowedIPs = [ "${address}/32" ];
     #     persistentKeepalive = 25;
     # };
 
@@ -255,25 +243,25 @@ in
     '';
 
     # Configure /etc/hosts to link all hostnames to their Wireguard IP
-    networking.extraHosts = builtins.concatStringsSep "\n" (map
+    networking.extraHosts = concatStringsSep "\n" (attrValues (mapAttrs
-      ({ hostname, IP, ...}: "${IP} ${hostname}")
+      (hostname: { address, ...}: "${address} ${hostname}")
-      cfg.cluster_nodes);
+      cfg.clusterNodes));
 
     # Enable Hashicorp Consul & Nomad
     services.consul.enable = true;
     systemd.services.consul.after = [ "wg-quick-wg0.service" ];
     services.consul.extraConfig =
-      (if cfg.is_raft_server
+      (if cfg.isRaftServer
       then { server = true; }
         // (if cfg.bootstrap then { bootstrap_expect = 3; } else {})
       else {}) //
     {
       inherit node_meta;
-      datacenter = cfg.cluster_name;
+      datacenter = cfg.clusterName;
       ui_config = {
         enabled = true;
       };
-      bind_addr = "${cfg.cluster_ip}";
+      bind_addr = "${clusterAddress}";
 
       addresses = {
         https = "0.0.0.0";
@@ -303,18 +291,18 @@ in
       pkgs.zstd
     ];
     services.nomad.settings =
-      (if cfg.is_raft_server
+      (if cfg.isRaftServer
       then {
           server = { enabled = true; }
             // (if cfg.bootstrap then { bootstrap_expect = 3; } else {});
       } else {}) //
     {
-      region = cfg.cluster_name;
+      region = cfg.clusterName;
-      datacenter = cfg.site_name;
+      datacenter = cfg.siteName;
       advertise = {
-        rpc = "${cfg.cluster_ip}";
+        rpc = "${clusterAddress}";
-        http = "${cfg.cluster_ip}";
+        http = "${clusterAddress}";
-        serf = "${cfg.cluster_ip}";
+        serf = "${clusterAddress}";
       };
       consul = {
         address = "localhost:8501";
@@ -367,7 +355,7 @@ in
 
       allowedTCPPorts = [
         # Allow anyone to connect on SSH port
-        (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
+        (head ({ openssh.ports = [22]; } // config.services).openssh.ports)
       ];
 
       allowedUDPPorts = [
@@ -385,14 +373,14 @@ in
         iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
 
         # Allow other nodes on VPN to access all ports
-        iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
+        iptables -A INPUT -s ${cfg.clusterPrefix} -j ACCEPT
       '';
 
       # When stopping firewall, delete all rules that were configured manually above
       extraStopCommands = ''
         iptables -D INPUT -s 192.168.0.0/16 -p udp --sport 1900 -j ACCEPT
         iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
-        iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
+        iptables -D INPUT -s ${cfg.clusterPrefix} -j ACCEPT
       '';
     };
   };

nix/remote-unlock.nix

@@ -28,7 +28,7 @@ in
     boot.initrd.network.ssh = {
       enable = true;
       port = 222;
-      authorizedKeys = concatLists (mapAttrsToList (name: user: user) config.deuxfleurs.admin_accounts);
+      authorizedKeys = concatLists (mapAttrsToList (name: user: user) config.deuxfleurs.adminAccounts);
       hostKeys = [ "/var/lib/deuxfleurs/remote-unlock/ssh_host_ed25519_key" ];
     };
     boot.initrd.network.postCommands = ''