diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix index 79fb3fb..5007815 100644 --- a/cluster/staging/cluster.nix +++ b/cluster/staging/cluster.nix @@ -2,65 +2,26 @@ { deuxfleurs.cluster_name = "staging"; - deuxfleurs.cluster_nodes = [ - { - hostname = "spoutnik"; - site_name = "pluton"; - publicKey = "fO8qZOZmnug84cA8nvfjl5MUqyWljP0BAz/4tHRZyEg="; - IP = "10.42.2.2"; - endpoint = "77.141.67.109:42136"; - } - { - hostname = "cariacou"; - site_name = "neptune"; - publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA="; - IP = "10.42.2.21"; - endpoint = "82.66.112.151:33721"; - lan_endpoint = "192.168.1.21:33721"; - } - { - hostname = "carcajou"; - site_name = "neptune"; - publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk="; - IP = "10.42.2.22"; - endpoint = "82.66.112.151:33722"; - lan_endpoint = "192.168.1.22:33722"; - } - { - hostname = "caribou"; - site_name = "neptune"; - publicKey = "g6ZED/wPn5MPfytJKwPI19808CXtEad0IJUkEAAzwyY="; - IP = "10.42.2.23"; - endpoint = "82.66.112.151:33723"; - lan_endpoint = "192.168.1.23:33723"; - } + + # Bootstrap nodes for Wesher overlay network + services.wesher.join = [ + "2a01:e0a:c:a720::21" # cariacou + "2a01:e0a:c:a720::22" # carcajou + "2a01:e0a:c:a720::23" # caribou ]; - deuxfleurs.admin_nodes = [ - { - hostname = "hammerhead"; - publicKey = "b5hF+GSTgg3oM6wnjL7jRbfyf1jtsWdVptPPbAh3Qic="; - IP = "10.42.0.1"; - endpoint = "5.135.179.11:51349"; - } - { - hostname = "robinson"; - publicKey = "ETaZFil3mFXlJ0LaJZyWqJVLV2IZUF5PB/8M7WbQSTg="; - IP = "10.42.0.42"; - endpoint = "77.141.67.109:33742"; - } - { - hostname = "shiki"; - publicKey = "QUiUNMk70TEQ75Ut7Uqikr5uGVSXmx8EGNkGM6tANlg="; - IP = "10.42.0.206"; - endpoint = "37.187.118.206:51820"; - } - { - hostname = "lindy"; - publicKey = "wen9GnZy2iLT6RyHfn7ydS/wvdvow1XPmhZxIkrDbks="; - IP = "10.42.0.66"; - endpoint = "82.66.112.151:33766"; - } + + # The IP range to use for the Wesher overlay of this cluster + deuxfleurs.wesher_cluster_prefix = "10.14.0.0"; + deuxfleurs.wesher_cluster_prefix_length = 16; + + # Bootstrap IPs for Consul cluster, + # these are IPs on the Wesher overlay + services.consul.extraConfig.retry_join = [ + "10.14.181.82" # caribou + "10.14.179.56" # cariacou + "10.14.252.121" # carcajou ]; + deuxfleurs.admin_accounts = { lx = [ # Keys for accessing nodes from outside diff --git a/cluster/staging/node/carcajou.nix b/cluster/staging/node/carcajou.nix index 304f358..dbcc5ec 100644 --- a/cluster/staging/node/carcajou.nix +++ b/cluster/staging/node/carcajou.nix @@ -17,16 +17,9 @@ deuxfleurs.network_interface = "eno1"; deuxfleurs.lan_ip = "192.168.1.22"; + deuxfleurs.ipv6 = "2a01:e0a:c:a720::22"; - networking.interfaces.eno1.ipv6.addresses = [ - { - address = "2a01:e0a:c:a720::22"; - prefixLength = 64; - } - ]; - - deuxfleurs.vpn_ip = "10.42.2.22"; - deuxfleurs.vpn_listen_port = 33722; + deuxfleurs.cluster_ip = "10.14.252.121"; deuxfleurs.is_raft_server = true; # Enable netdata monitoring diff --git a/cluster/staging/node/cariacou.nix b/cluster/staging/node/cariacou.nix index 5f57fbd..14d1842 100644 --- a/cluster/staging/node/cariacou.nix +++ b/cluster/staging/node/cariacou.nix @@ -17,16 +17,9 @@ deuxfleurs.network_interface = "eno1"; deuxfleurs.lan_ip = "192.168.1.21"; + deuxfleurs.ipv6 = "2a01:e0a:c:a720::21"; - networking.interfaces.eno1.ipv6.addresses = [ - { - address = "2a01:e0a:c:a720::21"; - prefixLength = 64; - } - ]; - - deuxfleurs.vpn_ip = "10.42.2.21"; - deuxfleurs.vpn_listen_port = 33721; + deuxfleurs.cluster_ip = "10.14.179.56"; deuxfleurs.is_raft_server = true; # Enable netdata monitoring diff --git a/cluster/staging/node/caribou.nix b/cluster/staging/node/caribou.nix index 7b785ef..3b41972 100644 --- a/cluster/staging/node/caribou.nix +++ b/cluster/staging/node/caribou.nix @@ -17,16 +17,9 @@ deuxfleurs.network_interface = "eno1"; deuxfleurs.lan_ip = "192.168.1.23"; + deuxfleurs.ipv6 = "2a01:e0a:c:a720::23"; - networking.interfaces.eno1.ipv6.addresses = [ - { - address = "2a01:e0a:c:a720::23"; - prefixLength = 64; - } - ]; - - deuxfleurs.vpn_ip = "10.42.2.23"; - deuxfleurs.vpn_listen_port = 33723; + deuxfleurs.cluster_ip = "10.14.181.82"; deuxfleurs.is_raft_server = true; # Enable netdata monitoring diff --git a/cluster/staging/site/neptune.nix b/cluster/staging/site/neptune.nix index 1b80144..38a4bab 100644 --- a/cluster/staging/site/neptune.nix +++ b/cluster/staging/site/neptune.nix @@ -4,6 +4,7 @@ deuxfleurs.site_name = "neptune"; deuxfleurs.lan_default_gateway = "192.168.1.254"; deuxfleurs.lan_ip_prefix_length = 24; + deuxfleurs.ipv6_prefix_length = 64; networking.nameservers = [ "192.168.1.254" ]; diff --git a/nix/configuration.nix b/nix/configuration.nix index 3f3aa49..984307c 100644 --- a/nix/configuration.nix +++ b/nix/configuration.nix @@ -69,6 +69,7 @@ SystemMaxUse=1G rclone docker docker-compose + wireguard wesher ]; @@ -81,43 +82,6 @@ SystemMaxUse=1G services.openssh.enable = true; services.openssh.passwordAuthentication = false; - services.wesher = { - enable = true; - join = [ "192.168.1.22" "192.168.1.23" ]; - bindAddr = config.deuxfleurs.lan_ip; # for now - overlayNet = "10.14.0.0/16"; - }; - - # ---- CONFIG FOR DEUXFLEURS CLUSTER ---- - - # Open ports in the firewall. - networking.firewall = { - enable = true; - - # Allow anyone to connect on SSH port - allowedTCPPorts = [ - (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports) - ]; - - # Allow specific hosts access to specific things in the cluster - extraCommands = '' - # Allow everything from router (usefull for UPnP/IGD) - iptables -A INPUT -s 192.168.1.254 -j ACCEPT - - # Allow docker containers to access all ports - iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT - - # Allow other nodes on VPN to access all ports - iptables -A INPUT -s 10.42.0.0/16 -j ACCEPT - ''; - - # When stopping firewall, delete all rules that were configured manually above - extraStopCommands = '' - iptables -D INPUT -s 192.168.1.254 -j ACCEPT - iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT - iptables -D INPUT -s 10.42.0.0/16 -j ACCEPT - ''; - }; # This value determines the NixOS release from which the default # settings for stateful data, like file locations and database versions diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 7772f0e..2050776 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -7,37 +7,6 @@ in with pkgs.lib; { options.deuxfleurs = - let wg_node = with types; submodule { - options = { - hostname = mkOption { - type = str; - description = "Host name"; - }; - site_name = mkOption { - type = nullOr str; - description = "Site where the node is located"; - default = null; - }; - IP = mkOption { - type = str; - description = "IP Address"; - }; - publicKey = mkOption { - type = str; - description = "Public key"; - }; - endpoint = mkOption { - type = nullOr str; - description = "Wireguard endpoint on the public Internet"; - }; - lan_endpoint = mkOption { - type = nullOr str; - description = "Wireguard endpoint for nodes in the same site"; - default = null; - }; - }; - }; - in { # Parameters for individual nodes network_interface = mkOption { @@ -52,14 +21,28 @@ in description = "Prefix length associated with lan_ip"; type = types.int; }; - - vpn_ip = mkOption { - description = "IP address of this node on the Wireguard VPN"; + ipv6 = mkOption { + description = "Public IPv6 address of this node"; type = types.str; }; - vpn_listen_port = mkOption { - description = "Port for incoming Wireguard VPN connections"; - type = types.port; + ipv6_prefix_length = mkOption { + description = "Prefix length associated with ipv6 ip"; + type = types.int; + }; + + wesher_cluster_prefix = mkOption { + description = "IP address prefix for the Wesher overlay network"; + type = types.str; + }; + wesher_cluster_prefix_length = mkOption { + description = "IP address prefix length for the Wesher overlay network"; + type = types.int; + default = 16; + }; + + cluster_ip = mkOption { + description = "IP address of this node on the Wesher mesh network"; + type = types.str; }; is_raft_server = mkOption { description = "Make this node a RAFT server for the Nomad and Consul deployments"; @@ -83,14 +66,6 @@ in description = "Name of this Deuxfleurs deployment"; type = types.str; }; - cluster_nodes = mkOption { - description = "Nodes that are part of the cluster"; - type = types.listOf wg_node; - }; - admin_nodes = mkOption { - description = "Machines that are part of the Wireguard VPN for administration purposes"; - type = types.listOf wg_node; - }; admin_accounts = mkOption { description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys"; type = types.attrsOf (types.listOf types.str); @@ -114,32 +89,31 @@ in prefixLength = cfg.lan_ip_prefix_length; } ]; + ipv6.addresses = [ + { + address = cfg.ipv6; + prefixLength = cfg.ipv6_prefix_length; + } + ]; }; networking.defaultGateway = { address = cfg.lan_default_gateway; interface = cfg.network_interface; }; - # Configure Wireguard VPN between all nodes - networking.wireguard.interfaces.wg0 = { - ips = [ "${cfg.vpn_ip}/16" ]; - listenPort = cfg.vpn_listen_port; - privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; - peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: { - publicKey = publicKey; - allowedIPs = [ "${IP}/32" ]; - endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null - then lan_endpoint else endpoint; - persistentKeepalive = 25; - }) (cfg.cluster_nodes ++ cfg.admin_nodes); + # wesher overlay network + services.wesher = { + enable = true; + bindAddr = cfg.ipv6; + overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}"; + interface = "wg0"; + logLevel = "debug"; }; - networking.firewall.allowedUDPPorts = [ cfg.vpn_listen_port ]; - # Configure /etc/hosts to link all hostnames to their Wireguard IP - networking.extraHosts = builtins.concatStringsSep "\n" (map - ({ hostname, IP, ...}: "${IP} ${hostname}") - (cfg.cluster_nodes ++ cfg.admin_nodes)); + #networking.extraHosts = builtins.concatStringsSep "\n" (map + # ({ hostname, IP, ...}: "${IP} ${hostname}") + # (cfg.cluster_nodes ++ cfg.admin_nodes)); # Enable Hashicorp Consul & Nomad services.consul.enable = true; @@ -156,14 +130,12 @@ in "site" = cfg.site_name; }; ui = true; - bind_addr = cfg.vpn_ip; + bind_addr = "${cfg.cluster_ip}"; ports.http = -1; addresses.https = "0.0.0.0"; ports.https = 8501; - retry_join = map (node_info: node_info.IP) cfg.cluster_nodes; - ca_file = "/var/lib/consul/pki/consul-ca.crt"; cert_file = "/var/lib/consul/pki/consul2022.crt"; key_file = "/var/lib/consul/pki/consul2022.key"; @@ -185,9 +157,9 @@ in region = cfg.cluster_name; datacenter = cfg.site_name; advertise = { - rpc = cfg.vpn_ip; - http = cfg.vpn_ip; - serf = cfg.vpn_ip; + rpc = "${cfg.cluster_ip}"; + http = "${cfg.cluster_ip}"; + serf = "${cfg.cluster_ip}"; }; consul = { address = "localhost:8501"; @@ -227,5 +199,36 @@ in } ]; }; + + # ---- Firewall config ---- + + # Open ports in the firewall. + networking.firewall = { + enable = true; + + # Allow anyone to connect on SSH port + allowedTCPPorts = [ + (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports) + ]; + + # Allow specific hosts access to specific things in the cluster + extraCommands = '' + # Allow everything from router (usefull for UPnP/IGD) + iptables -A INPUT -s 192.168.1.254 -j ACCEPT + + # Allow docker containers to access all ports + iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT + + # Allow other nodes on VPN to access all ports + iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT + ''; + + # When stopping firewall, delete all rules that were configured manually above + extraStopCommands = '' + iptables -D INPUT -s 192.168.1.254 -j ACCEPT + iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT + iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT + ''; + }; }; }