diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix index cf30d6e..56ca904 100644 --- a/cluster/staging/cluster.nix +++ b/cluster/staging/cluster.nix @@ -1,49 +1,43 @@ { config, pkgs, ... } @ args: { - deuxfleurs.cluster_name = "staging"; + deuxfleurs.clusterName = "staging"; # The IP range to use for the Wireguard overlay of this cluster - deuxfleurs.cluster_prefix = "10.14.0.0"; - deuxfleurs.cluster_prefix_length = 16; + deuxfleurs.clusterPrefix = "10.14.0.0/16"; - deuxfleurs.cluster_nodes = [ - { - hostname = "carcajou"; - site_name = "neptune"; + deuxfleurs.clusterNodes = { + "carcajou" = { + siteName = "neptune"; publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk="; - IP = "10.14.1.2"; + address = "10.14.1.2"; endpoint = "77.207.15.215:33722"; - } - { - hostname = "caribou"; - site_name = "neptune"; + }; + "caribou" = { + siteName = "neptune"; publicKey = "lABn/axzD1jkFulX8c+K3B3CbKXORlIMDDoe8sQVxhs="; - IP = "10.14.1.3"; + address = "10.14.1.3"; endpoint = "77.207.15.215:33723"; - } - { - hostname = "origan"; - site_name = "jupiter"; + }; + "origan" = { + siteName = "jupiter"; publicKey = "smBQYUS60JDkNoqkTT7TgbpqFiM43005fcrT6472llI="; - IP = "10.14.2.33"; + address = "10.14.2.33"; endpoint = "82.64.238.84:33733"; - } - { - hostname = "piranha"; - site_name = "corrin"; + }; + "piranha" = { + siteName = "corrin"; publicKey = "m9rLf+233X1VColmeVrM/xfDGro5W6Gk5N0zqcf32WY="; - IP = "10.14.3.1"; + address = "10.14.3.1"; #endpoint = "82.120.233.78:33721"; - } - { - hostname = "df-pw5"; - site_name = "bespin"; + }; + "df-pw5" = { + siteName = "bespin"; publicKey = "XLOYoMXF+PO4jcgfSVAk+thh4VmWx0wzWnb0xs08G1s="; - IP = "10.14.4.1"; + address = "10.14.4.1"; endpoint = "bitfrost.fiber.shirokumo.net:33734"; - } - ]; + }; + }; services.wgautomesh.logLevel = "debug"; # Bootstrap IPs for Consul cluster, @@ -54,7 +48,7 @@ "10.14.1.3" # caribou ]; - deuxfleurs.admin_accounts = { + deuxfleurs.adminAccounts = { lx = [ # Keys for accessing nodes from outside "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy" @@ -142,16 +136,16 @@ enable = true; port = substituter_port; openFirewall = false; - bindAddress = config.deuxfleurs.cluster_ip; + bindAddress = "0.0.0.0"; package = pkgs.haskellPackages.nix-serve-ng; }; nix.settings.substituters = map - ({ IP, ... }: "http://${IP}:${builtins.toString substituter_port}") - (builtins.filter - ({ site_name, IP, ...}: - (IP != config.deuxfleurs.cluster_ip - && site_name == config.deuxfleurs.site_name)) - config.deuxfleurs.cluster_nodes); + ({ address, ... }: "http://${address}:${builtins.toString substituter_port}") + (builtins.attrValues (pkgs.lib.filterAttrs + (hostname: { siteName, ...}: + (hostname != config.deuxfleurs.hostName + && siteName == config.deuxfleurs.siteName)) + config.deuxfleurs.clusterNodes)); }) ]; } diff --git a/cluster/staging/node/carcajou.nix b/cluster/staging/node/carcajou.nix index 5822f49..e6c1653 100644 --- a/cluster/staging/node/carcajou.nix +++ b/cluster/staging/node/carcajou.nix @@ -19,11 +19,8 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "carcajou"; - - deuxfleurs.ipv6 = "2001:910:1204:1::22"; - - deuxfleurs.cluster_ip = "10.14.1.2"; + deuxfleurs.hostName = "carcajou"; + deuxfleurs.ipv6Address = "2001:910:1204:1::22"; system.stateVersion = "21.05"; } diff --git a/cluster/staging/node/caribou.nix b/cluster/staging/node/caribou.nix index 2e8691a..ad5a65d 100644 --- a/cluster/staging/node/caribou.nix +++ b/cluster/staging/node/caribou.nix @@ -8,12 +8,9 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "caribou"; - - deuxfleurs.ipv6 = "2001:910:1204:1::23"; - - deuxfleurs.cluster_ip = "10.14.1.3"; - deuxfleurs.is_raft_server = true; + deuxfleurs.hostName = "caribou"; + deuxfleurs.ipv6Address = "2001:910:1204:1::23"; + deuxfleurs.isRaftServer = true; system.stateVersion = "21.05"; } diff --git a/cluster/staging/node/df-pw5.nix b/cluster/staging/node/df-pw5.nix index 356a2ae..0e5be15 100644 --- a/cluster/staging/node/df-pw5.nix +++ b/cluster/staging/node/df-pw5.nix @@ -9,13 +9,9 @@ boot.loader.efi.efiSysMountPoint = "/boot"; boot.loader.timeout = 20; - networking.hostName = "df-pw5"; - + deuxfleurs.hostName = "df-pw5"; deuxfleurs.staticIPv4.address = "192.168.5.130/24"; - deuxfleurs.ipv6 = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7"; - - deuxfleurs.cluster_ip = "10.14.4.1"; - deuxfleurs.is_raft_server = false; + deuxfleurs.ipv6Address = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7"; system.stateVersion = "22.11"; } diff --git a/cluster/staging/node/origan.nix b/cluster/staging/node/origan.nix index 6db7f87..d900fd6 100644 --- a/cluster/staging/node/origan.nix +++ b/cluster/staging/node/origan.nix @@ -8,13 +8,10 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "origan"; - + deuxfleurs.hostName = "origan"; deuxfleurs.staticIPv4.address = "192.168.1.33/24"; - deuxfleurs.ipv6 = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec"; - - deuxfleurs.cluster_ip = "10.14.2.33"; - deuxfleurs.is_raft_server = true; + deuxfleurs.ipv6Address = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec"; + deuxfleurs.isRaftServer = true; system.stateVersion = "22.11"; } diff --git a/cluster/staging/node/piranha.nix b/cluster/staging/node/piranha.nix index 4873693..436965c 100644 --- a/cluster/staging/node/piranha.nix +++ b/cluster/staging/node/piranha.nix @@ -8,13 +8,10 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "piranha"; - + deuxfleurs.hostName = "piranha"; deuxfleurs.staticIPv4.address = "192.168.1.25/24"; - deuxfleurs.ipv6 = "2a01:cb05:8984:9900:223:24ff:feb0:ea82"; - - deuxfleurs.cluster_ip = "10.14.3.1"; - deuxfleurs.is_raft_server = true; + deuxfleurs.ipv6Address = "2a01:cb05:8984:9900:223:24ff:feb0:ea82"; + deuxfleurs.isRaftServer = true; system.stateVersion = "22.11"; } diff --git a/cluster/staging/site/bespin.nix b/cluster/staging/site/bespin.nix index 3fcefbb..22feb59 100644 --- a/cluster/staging/site/bespin.nix +++ b/cluster/staging/site/bespin.nix @@ -1,9 +1,9 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "bespin"; + deuxfleurs.siteName = "bespin"; deuxfleurs.staticIPv4.defaultGateway = "192.168.5.254"; - deuxfleurs.cname_target = "bespin.site.staging.deuxfleurs.org."; + deuxfleurs.cnameTarget = "bespin.site.staging.deuxfleurs.org."; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/staging/site/corrin.nix b/cluster/staging/site/corrin.nix index 0ff7b80..0083986 100644 --- a/cluster/staging/site/corrin.nix +++ b/cluster/staging/site/corrin.nix @@ -1,10 +1,10 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "corrin"; + deuxfleurs.siteName = "corrin"; deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; - deuxfleurs.cname_target = "corrin.site.staging.deuxfleurs.org."; - deuxfleurs.public_ipv4 = "82.120.233.78"; + deuxfleurs.cnameTarget = "corrin.site.staging.deuxfleurs.org."; + deuxfleurs.publicIPv4 = "82.120.233.78"; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/staging/site/jupiter.nix b/cluster/staging/site/jupiter.nix index 2269836..2d39f5a 100644 --- a/cluster/staging/site/jupiter.nix +++ b/cluster/staging/site/jupiter.nix @@ -1,12 +1,9 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "jupiter"; + deuxfleurs.siteName = "jupiter"; deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; - deuxfleurs.cname_target = "jupiter.site.staging.deuxfleurs.org."; - - # no public ipv4 is used for the staging cluster on Jupiter - # deuxfleurs.public_ipv4 = "???"; + deuxfleurs.cnameTarget = "jupiter.site.staging.deuxfleurs.org."; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/staging/site/neptune.nix b/cluster/staging/site/neptune.nix index 36d5957..f94d62f 100644 --- a/cluster/staging/site/neptune.nix +++ b/cluster/staging/site/neptune.nix @@ -1,12 +1,8 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "neptune"; - deuxfleurs.cname_target = "neptune.site.staging.deuxfleurs.org."; - - # no public ipv4 is used for the staging cluster on Neptune, - # because the Internet connection is already used for the prod cluster - # deuxfleurs.public_ipv4 = "77.207.15.215"; + deuxfleurs.siteName = "neptune"; + deuxfleurs.cnameTarget = "neptune.site.staging.deuxfleurs.org."; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 6d27d5c..7632486 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -6,110 +6,98 @@ in with builtins; with pkgs.lib; { - options.deuxfleurs = - let wg_node = with types; submodule { - options = { - hostname = mkOption { - type = str; - description = "Host name"; - }; - site_name = mkOption { - type = nullOr str; - description = "Site where the node is located"; - default = null; - }; - IP = mkOption { - type = str; - description = "IP Address in the Wireguard network"; - }; - publicKey = mkOption { - type = str; - description = "Public key"; - }; - endpoint = mkOption { - type = nullOr str; - default = null; - description = "Wireguard endpoint on the public Internet"; - }; - }; - }; - in - { + options.deuxfleurs = with types; { # Parameters for individual nodes - ipv6 = mkOption { + hostName = mkOption { + description = "Node name"; + type = str; + }; + ipv6Address = mkOption { description = "Static public IPv6 address of this node"; - type = types.str; + type = str; }; staticIPv4.address = mkOption { description = "IP address (with prefix length) of this node on the local network interface"; - type = types.nullOr types.str; + type = nullOr str; default = null; }; - cluster_ip = mkOption { - description = "IP address of this node on the Wesher mesh network"; - type = types.str; - }; - - is_raft_server = mkOption { + isRaftServer = mkOption { description = "Make this node a RAFT server for the Nomad and Consul deployments"; - type = types.bool; + type = bool; default = false; }; # Parameters that generally vary between sites - site_name = mkOption { + siteName = mkOption { description = "Site (availability zone) on which this node is deployed"; - type = types.str; + type = str; }; staticIPv4.defaultGateway = mkOption { description = "IPv4 address of the default route on the local network interface"; - type = types.nullOr types.str; + type = nullOr str; default = null; }; - public_ipv4 = mkOption { + + publicIPv4 = mkOption { description = "Public IPv4 through which this node is accessible (possibly after port opening using DiploNAT), for domain names that are updated by D53"; - type = types.nullOr types.str; + type = nullOr str; default = null; }; - cname_target = mkOption { + cnameTarget = mkOption { description = "DNS CNAME target to use for services hosted in this site, for domain names that are updated by D53"; - type = types.nullOr types.str; + type = nullOr str; default = null; }; # Parameters common to all nodes - cluster_name = mkOption { + clusterName = mkOption { description = "Name of this Deuxfleurs deployment"; - type = types.str; + type = str; }; - cluster_prefix = mkOption { - description = "IP address prefix for the Wireguard overlay network"; - type = types.str; + clusterPrefix = mkOption { + description = "IP address prefix (and length) for the Wireguard overlay network"; + type = str; }; - cluster_prefix_length = mkOption { - description = "IP address prefix length for the Wireguard overlay network"; - type = types.int; - default = 16; - }; - cluster_nodes = mkOption { + clusterNodes = mkOption { description = "Nodes that are part of the cluster"; - type = types.listOf wg_node; + type = attrsOf (submodule { + options = { + siteName = mkOption { + type = nullOr str; + description = "Site where the node is located"; + default = null; + }; + address = mkOption { + type = str; + description = "IP Address in the Wireguard network"; + }; + publicKey = mkOption { + type = str; + description = "Public key"; + }; + endpoint = mkOption { + type = nullOr str; + default = null; + description = "Wireguard endpoint on the public Internet"; + }; + }; + }); }; - admin_accounts = mkOption { + adminAccounts = mkOption { description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys"; - type = types.attrsOf (types.listOf types.str); + type = attrsOf (listOf str); }; bootstrap = mkOption { description = "Whether to enable bootstrapping for Nomad and Consul"; - type = types.bool; + type = bool; default = false; }; # Options that generally stay to their default value wireguardPort = mkOption { description = "Port for incoming Wireguard VPN connections"; - type = types.port; + type = port; default = 33799; }; }; @@ -119,25 +107,29 @@ in ]; config = - let node_meta = { - "site" = cfg.site_name; - "public_ipv6" = cfg.ipv6; + let + clusterNodeCfg = getAttr cfg.hostName cfg.clusterNodes; + clusterAddress = clusterNodeCfg.address; + node_meta = { + "site" = cfg.siteName; + "public_ipv6" = cfg.ipv6Address; } // - (if cfg.public_ipv4 != null - then { "public_ipv4" = cfg.public_ipv4; } + (if cfg.publicIPv4 != null + then { "public_ipv4" = cfg.publicIPv4; } else {}) // - (if cfg.cname_target != null - then { "cname_target" = cfg.cname_target; } + (if cfg.cnameTarget != null + then { "cname_target" = cfg.cnameTarget; } else {}); in { + networking.hostName = cfg.hostName; # Configure admin accounts on all nodes - users.users = builtins.mapAttrs (name: publicKeys: { + users.users = mapAttrs (name: publicKeys: { isNormalUser = true; extraGroups = [ "wheel" ]; openssh.authorizedKeys.keys = publicKeys; - }) cfg.admin_accounts; + }) cfg.adminAccounts; # Configure network interfaces networking.useDHCP = false; @@ -148,7 +140,7 @@ in Name = "en* eth*"; }; ipv6AcceptRAConfig = { - Token = "static:${cfg.ipv6}"; + Token = "static:${cfg.ipv6Address}"; UseDNS = false; }; } // (if cfg.staticIPv4.address == null || cfg.staticIPv4.defaultGateway == null then { @@ -195,7 +187,7 @@ in # Forward .consul queries to Consul daemon { name = "consul."; - stub-addr = "${cfg.cluster_ip}@8600"; + stub-addr = "${clusterAddress}@8600"; stub-no-cache = true; stub-tcp-upstream = false; stub-tls-upstream = false; @@ -208,7 +200,7 @@ in # Configure Wireguard VPN between all nodes networking.wireguard.interfaces.wg0 = { - ips = [ "${cfg.cluster_ip}/16" ]; + ips = [ "${clusterAddress}/16" ]; listenPort = cfg.wireguardPort; privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; mtu = 1420; @@ -220,28 +212,24 @@ in gossipSecretFile = "/var/lib/wgautomesh/gossip_secret"; persistFile = "/var/lib/wgautomesh/state"; upnpForwardPublicPort = - let - us = filter ({ hostname, ...}: hostname == config.networking.hostName) cfg.cluster_nodes; - in - if length us > 0 && (head us).endpoint != null then - strings.toInt (lists.last (split ":" (head us).endpoint)) + if clusterNodeCfg.endpoint != null then + strings.toInt (lists.last (split ":" clusterNodeCfg.endpoint)) else null; - peers = map ({ publicKey, endpoint, IP, ... }: { - address = IP; + peers = attrValues (mapAttrs (hostname: { publicKey, endpoint, address, ... }: { + inherit address endpoint; pubkey = publicKey; - endpoint = endpoint; - }) cfg.cluster_nodes; + }) cfg.clusterNodes); }; # Old code for wg-quick, we can use this as a fallback if we fail to make wgautomesh work # systemd.services."wg-quick-wg0".after = [ "unbound.service" ]; # networking.wg-quick.interfaces.wg0 = { - # address = [ "${cfg.cluster_ip}/16" ]; + # address = [ "${clusterAddress}/16" ]; # listenPort = cfg.wireguardPort; # privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; # mtu = 1420; - # peers = map ({ publicKey, endpoint, IP, ... }: { + # peers = map ({ publicKey, endpoint, address, ... }: { # inherit publicKey endpoint; - # allowedIPs = [ "${IP}/32" ]; + # allowedIPs = [ "${address}/32" ]; # persistentKeepalive = 25; # }; @@ -255,25 +243,25 @@ in ''; # Configure /etc/hosts to link all hostnames to their Wireguard IP - networking.extraHosts = builtins.concatStringsSep "\n" (map - ({ hostname, IP, ...}: "${IP} ${hostname}") - cfg.cluster_nodes); + networking.extraHosts = concatStringsSep "\n" (attrValues (mapAttrs + (hostname: { address, ...}: "${address} ${hostname}") + cfg.clusterNodes)); # Enable Hashicorp Consul & Nomad services.consul.enable = true; systemd.services.consul.after = [ "wg-quick-wg0.service" ]; services.consul.extraConfig = - (if cfg.is_raft_server + (if cfg.isRaftServer then { server = true; } // (if cfg.bootstrap then { bootstrap_expect = 3; } else {}) else {}) // { inherit node_meta; - datacenter = cfg.cluster_name; + datacenter = cfg.clusterName; ui_config = { enabled = true; }; - bind_addr = "${cfg.cluster_ip}"; + bind_addr = "${clusterAddress}"; addresses = { https = "0.0.0.0"; @@ -303,18 +291,18 @@ in pkgs.zstd ]; services.nomad.settings = - (if cfg.is_raft_server + (if cfg.isRaftServer then { server = { enabled = true; } // (if cfg.bootstrap then { bootstrap_expect = 3; } else {}); } else {}) // { - region = cfg.cluster_name; - datacenter = cfg.site_name; + region = cfg.clusterName; + datacenter = cfg.siteName; advertise = { - rpc = "${cfg.cluster_ip}"; - http = "${cfg.cluster_ip}"; - serf = "${cfg.cluster_ip}"; + rpc = "${clusterAddress}"; + http = "${clusterAddress}"; + serf = "${clusterAddress}"; }; consul = { address = "localhost:8501"; @@ -367,7 +355,7 @@ in allowedTCPPorts = [ # Allow anyone to connect on SSH port - (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports) + (head ({ openssh.ports = [22]; } // config.services).openssh.ports) ]; allowedUDPPorts = [ @@ -385,14 +373,14 @@ in iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # Allow other nodes on VPN to access all ports - iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT + iptables -A INPUT -s ${cfg.clusterPrefix} -j ACCEPT ''; # When stopping firewall, delete all rules that were configured manually above extraStopCommands = '' iptables -D INPUT -s 192.168.0.0/16 -p udp --sport 1900 -j ACCEPT iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT - iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT + iptables -D INPUT -s ${cfg.clusterPrefix} -j ACCEPT ''; }; }; diff --git a/nix/remote-unlock.nix b/nix/remote-unlock.nix index fdc5285..3c3e4c8 100644 --- a/nix/remote-unlock.nix +++ b/nix/remote-unlock.nix @@ -28,7 +28,7 @@ in boot.initrd.network.ssh = { enable = true; port = 222; - authorizedKeys = concatLists (mapAttrsToList (name: user: user) config.deuxfleurs.admin_accounts); + authorizedKeys = concatLists (mapAttrsToList (name: user: user) config.deuxfleurs.adminAccounts); hostKeys = [ "/var/lib/deuxfleurs/remote-unlock/ssh_host_ed25519_key" ]; }; boot.initrd.network.postCommands = ''