Replace ad-hoc wireguard by wesher on staging cluster

cluster/staging/cluster.nix

@@ -2,65 +2,26 @@
 
 {
   deuxfleurs.cluster_name = "staging";
-  deuxfleurs.cluster_nodes = [
+
-    {
+  # Bootstrap nodes for Wesher overlay network
-      hostname = "spoutnik";
+  services.wesher.join = [
-      site_name = "pluton";
+	"2a01:e0a:c:a720::21"  # cariacou
-      publicKey = "fO8qZOZmnug84cA8nvfjl5MUqyWljP0BAz/4tHRZyEg=";
+	"2a01:e0a:c:a720::22"  # carcajou
-      IP = "10.42.2.2";
+	"2a01:e0a:c:a720::23"  # caribou
-      endpoint = "77.141.67.109:42136";
-    }
-    { 
-      hostname = "cariacou";
-      site_name = "neptune";
-      publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
-      IP = "10.42.2.21";
-      endpoint = "82.66.112.151:33721";
-      lan_endpoint = "192.168.1.21:33721";
-    }
-    { 
-      hostname = "carcajou";
-      site_name = "neptune";
-      publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
-      IP = "10.42.2.22";
-      endpoint = "82.66.112.151:33722";
-      lan_endpoint = "192.168.1.22:33722";
-    }
-    { 
-      hostname = "caribou";
-      site_name = "neptune";
-      publicKey = "g6ZED/wPn5MPfytJKwPI19808CXtEad0IJUkEAAzwyY=";
-      IP = "10.42.2.23";
-      endpoint = "82.66.112.151:33723";
-      lan_endpoint = "192.168.1.23:33723";
-    }
   ];
-  deuxfleurs.admin_nodes = [
+
-    {
+  # The IP range to use for the Wesher overlay of this cluster
-      hostname = "hammerhead";
+  deuxfleurs.wesher_cluster_prefix = "10.14.0.0";
-      publicKey = "b5hF+GSTgg3oM6wnjL7jRbfyf1jtsWdVptPPbAh3Qic=";
+  deuxfleurs.wesher_cluster_prefix_length = 16;
-      IP = "10.42.0.1";
+
-      endpoint = "5.135.179.11:51349";
+  # Bootstrap IPs for Consul cluster,
-    }
+  # these are IPs on the Wesher overlay
-    {
+  services.consul.extraConfig.retry_join = [
-      hostname = "robinson";
+    "10.14.181.82"  # caribou
-      publicKey = "ETaZFil3mFXlJ0LaJZyWqJVLV2IZUF5PB/8M7WbQSTg=";
+    "10.14.179.56"  # cariacou
-      IP = "10.42.0.42";
+    "10.14.252.121" # carcajou
-      endpoint = "77.141.67.109:33742";
-    }
-    {
-      hostname = "shiki";
-      publicKey = "QUiUNMk70TEQ75Ut7Uqikr5uGVSXmx8EGNkGM6tANlg=";
-      IP = "10.42.0.206";
-      endpoint = "37.187.118.206:51820";
-    }
-    {
-      hostname = "lindy";
-      publicKey = "wen9GnZy2iLT6RyHfn7ydS/wvdvow1XPmhZxIkrDbks=";
-      IP = "10.42.0.66";
-      endpoint = "82.66.112.151:33766";
-    }
   ];
+
   deuxfleurs.admin_accounts = {
     lx = [
       # Keys for accessing nodes from outside

cluster/staging/node/carcajou.nix

@@ -17,16 +17,9 @@
 
   deuxfleurs.network_interface = "eno1";
   deuxfleurs.lan_ip = "192.168.1.22";
+  deuxfleurs.ipv6 = "2a01:e0a:c:a720::22";
 
-  networking.interfaces.eno1.ipv6.addresses = [
+  deuxfleurs.cluster_ip = "10.14.252.121";
-    {
-      address = "2a01:e0a:c:a720::22";
-      prefixLength = 64;
-    }
-  ];
-
-  deuxfleurs.vpn_ip = "10.42.2.22";
-  deuxfleurs.vpn_listen_port = 33722;
   deuxfleurs.is_raft_server = true;
 
   # Enable netdata monitoring

cluster/staging/node/cariacou.nix

@@ -17,16 +17,9 @@
 
   deuxfleurs.network_interface = "eno1";
   deuxfleurs.lan_ip = "192.168.1.21";
+  deuxfleurs.ipv6 = "2a01:e0a:c:a720::21";
 
-  networking.interfaces.eno1.ipv6.addresses = [
+  deuxfleurs.cluster_ip = "10.14.179.56";
-    {
-      address = "2a01:e0a:c:a720::21";
-      prefixLength = 64;
-    }
-  ];
-
-  deuxfleurs.vpn_ip = "10.42.2.21";
-  deuxfleurs.vpn_listen_port = 33721;
   deuxfleurs.is_raft_server = true;
 
   # Enable netdata monitoring

cluster/staging/node/caribou.nix

@@ -17,16 +17,9 @@
 
   deuxfleurs.network_interface = "eno1";
   deuxfleurs.lan_ip = "192.168.1.23";
+  deuxfleurs.ipv6 = "2a01:e0a:c:a720::23";
 
-  networking.interfaces.eno1.ipv6.addresses = [
+  deuxfleurs.cluster_ip = "10.14.181.82";
-    {
-      address = "2a01:e0a:c:a720::23";
-      prefixLength = 64;
-    }
-  ];
-
-  deuxfleurs.vpn_ip = "10.42.2.23";
-  deuxfleurs.vpn_listen_port = 33723;
   deuxfleurs.is_raft_server = true;
 
   # Enable netdata monitoring

cluster/staging/site/neptune.nix

@@ -4,6 +4,7 @@
   deuxfleurs.site_name = "neptune";
   deuxfleurs.lan_default_gateway = "192.168.1.254";
   deuxfleurs.lan_ip_prefix_length = 24;
+  deuxfleurs.ipv6_prefix_length = 64;
 
   networking.nameservers = [ "192.168.1.254" ];
 

nix/configuration.nix

@@ -69,6 +69,7 @@ SystemMaxUse=1G
     rclone
     docker
     docker-compose
+    wireguard
     wesher
   ];
 
@@ -81,43 +82,6 @@ SystemMaxUse=1G
   services.openssh.enable = true;
   services.openssh.passwordAuthentication = false;
 
-  services.wesher = {
-    enable = true;
-    join = [ "192.168.1.22" "192.168.1.23" ];
-    bindAddr = config.deuxfleurs.lan_ip;  # for now
-    overlayNet = "10.14.0.0/16";
-  };
-
-  # ---- CONFIG FOR DEUXFLEURS CLUSTER ----
-
-  # Open ports in the firewall.
-  networking.firewall = {
-    enable = true;
-
-    # Allow anyone to connect on SSH port
-    allowedTCPPorts = [
-      (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
-    ];
-
-    # Allow specific hosts access to specific things in the cluster
-    extraCommands = ''
-      # Allow everything from router (usefull for UPnP/IGD)
-      iptables -A INPUT -s 192.168.1.254 -j ACCEPT
-
-      # Allow docker containers to access all ports
-      iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
-
-      # Allow other nodes on VPN to access all ports
-      iptables -A INPUT -s 10.42.0.0/16 -j ACCEPT
-    '';
-
-    # When stopping firewall, delete all rules that were configured manually above
-    extraStopCommands = ''
-      iptables -D INPUT -s 192.168.1.254 -j ACCEPT
-      iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
-      iptables -D INPUT -s 10.42.0.0/16 -j ACCEPT
-    '';
-  };
 
   # This value determines the NixOS release from which the default
   # settings for stateful data, like file locations and database versions

nix/deuxfleurs.nix

@@ -7,37 +7,6 @@ in
   with pkgs.lib;
 {
   options.deuxfleurs = 
-    let wg_node = with types; submodule {
-      options = {
-        hostname = mkOption {
-          type = str;
-          description = "Host name";
-        };
-        site_name = mkOption {
-          type = nullOr str;
-          description = "Site where the node is located";
-          default = null;
-        };
-        IP = mkOption {
-          type = str;
-          description = "IP Address";
-        };
-        publicKey = mkOption {
-          type = str;
-          description = "Public key";
-        };
-        endpoint = mkOption {
-          type = nullOr str;
-          description = "Wireguard endpoint on the public Internet";
-        };
-        lan_endpoint = mkOption {
-          type = nullOr str;
-          description = "Wireguard endpoint for nodes in the same site";
-          default = null;
-        };
-      };
-    };
-  in
   {
     # Parameters for individual nodes
     network_interface = mkOption {
@@ -52,14 +21,28 @@ in
       description = "Prefix length associated with lan_ip";
       type = types.int;
     };
-
+    ipv6 = mkOption {
-    vpn_ip = mkOption {
+      description = "Public IPv6 address of this node";
-      description = "IP address of this node on the Wireguard VPN";
       type = types.str;
     };
-    vpn_listen_port = mkOption {
+    ipv6_prefix_length = mkOption {
-      description = "Port for incoming Wireguard VPN connections";
+      description = "Prefix length associated with ipv6 ip";
-      type = types.port;
+      type = types.int;
+    };
+
+    wesher_cluster_prefix = mkOption {
+      description = "IP address prefix for the Wesher overlay network";
+      type = types.str;
+    };
+    wesher_cluster_prefix_length = mkOption {
+      description = "IP address prefix length for the Wesher overlay network";
+      type = types.int;
+      default = 16;
+    };
+
+    cluster_ip = mkOption {
+      description = "IP address of this node on the Wesher mesh network";
+      type = types.str;
     };
     is_raft_server = mkOption {
       description = "Make this node a RAFT server for the Nomad and Consul deployments";
@@ -83,14 +66,6 @@ in
       description = "Name of this Deuxfleurs deployment";
       type = types.str;
     };
-    cluster_nodes = mkOption {
-      description = "Nodes that are part of the cluster";
-      type = types.listOf wg_node;
-    };
-    admin_nodes = mkOption {
-      description = "Machines that are part of the Wireguard VPN for administration purposes";
-      type = types.listOf wg_node;
-    };
     admin_accounts = mkOption {
       description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
       type = types.attrsOf (types.listOf types.str);
@@ -114,32 +89,31 @@ in
           prefixLength = cfg.lan_ip_prefix_length;
         }
       ];
+      ipv6.addresses = [
+        {
+          address = cfg.ipv6;
+          prefixLength = cfg.ipv6_prefix_length;
+        }
+      ];
     };
     networking.defaultGateway = {
       address = cfg.lan_default_gateway;
       interface = cfg.network_interface;
     };
 
-    # Configure Wireguard VPN between all nodes
+    # wesher overlay network
-    networking.wireguard.interfaces.wg0 = {
+    services.wesher = {
-      ips = [ "${cfg.vpn_ip}/16" ];
+      enable = true;
-      listenPort = cfg.vpn_listen_port;
+      bindAddr = cfg.ipv6;
-      privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
+      overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
-      peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
+      interface = "wg0";
-        publicKey = publicKey;
+      logLevel = "debug";
-        allowedIPs = [ "${IP}/32" ];
-        endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
-                   then lan_endpoint else endpoint;
-        persistentKeepalive = 25;
-      }) (cfg.cluster_nodes ++ cfg.admin_nodes);
     };
 
-    networking.firewall.allowedUDPPorts = [ cfg.vpn_listen_port ];
-
     # Configure /etc/hosts to link all hostnames to their Wireguard IP
-    networking.extraHosts = builtins.concatStringsSep "\n" (map
+    #networking.extraHosts = builtins.concatStringsSep "\n" (map
-      ({ hostname, IP, ...}: "${IP} ${hostname}")
+    #  ({ hostname, IP, ...}: "${IP} ${hostname}")
-      (cfg.cluster_nodes ++ cfg.admin_nodes));
+    #  (cfg.cluster_nodes ++ cfg.admin_nodes));
 
     # Enable Hashicorp Consul & Nomad
     services.consul.enable = true;
@@ -156,14 +130,12 @@ in
         "site" = cfg.site_name;
       };
       ui = true;
-      bind_addr = cfg.vpn_ip;
+      bind_addr = "${cfg.cluster_ip}";
 
       ports.http = -1;
       addresses.https = "0.0.0.0";
       ports.https = 8501;
 
-      retry_join = map (node_info: node_info.IP) cfg.cluster_nodes;
-
       ca_file = "/var/lib/consul/pki/consul-ca.crt";
       cert_file = "/var/lib/consul/pki/consul2022.crt";
       key_file = "/var/lib/consul/pki/consul2022.key";
@@ -185,9 +157,9 @@ in
       region = cfg.cluster_name;
       datacenter = cfg.site_name;
       advertise = {
-        rpc = cfg.vpn_ip;
+        rpc = "${cfg.cluster_ip}";
-        http = cfg.vpn_ip;
+        http = "${cfg.cluster_ip}";
-        serf = cfg.vpn_ip;
+        serf = "${cfg.cluster_ip}";
       };
       consul = {
         address = "localhost:8501";
@@ -227,5 +199,36 @@ in
         }
       ];
     };
+
+    # ---- Firewall config ----
+
+    # Open ports in the firewall.
+    networking.firewall = {
+      enable = true;
+
+      # Allow anyone to connect on SSH port
+      allowedTCPPorts = [
+        (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
+      ];
+
+      # Allow specific hosts access to specific things in the cluster
+      extraCommands = ''
+        # Allow everything from router (usefull for UPnP/IGD)
+        iptables -A INPUT -s 192.168.1.254 -j ACCEPT
+
+        # Allow docker containers to access all ports
+        iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
+
+        # Allow other nodes on VPN to access all ports
+        iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+      '';
+
+      # When stopping firewall, delete all rules that were configured manually above
+      extraStopCommands = ''
+        iptables -D INPUT -s 192.168.1.254 -j ACCEPT
+        iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
+        iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+      '';
+    };
   };
 }