Remove wesher, reconfigure staging without it

cluster/staging/cluster.nix

@@ -3,16 +3,36 @@
 {
   deuxfleurs.cluster_name = "staging";
 
-  # Bootstrap nodes for Wesher overlay network
+  # The IP range to use for the Wireguard overlay of this cluster
-  services.wesher.join = [
+  deuxfleurs.cluster_prefix = "10.14.0.0";
-	"192.168.1.21"  # cariacou
+  deuxfleurs.cluster_prefix_length = 16;
-	"192.168.1.22"  # carcajou
-	"192.168.1.23"  # caribou
-  ];
 
-  # The IP range to use for the Wesher overlay of this cluster
+  deuxfleurs.cluster_nodes = [
-  deuxfleurs.wesher_cluster_prefix = "10.14.0.0";
+    {
-  deuxfleurs.wesher_cluster_prefix_length = 16;
+      hostname = "cariacou";
+      site_name = "neptune";
+      publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
+      IP = "10.14.179.56";
+      lan_endpoint = "192.168.1.21:33799";
+      endpoint = "192.168.1.21:33799"; ## TODO nat
+    }
+    {
+      hostname = "carcajou";
+      site_name = "neptune";
+      publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
+      IP = "10.14.252.121";
+      lan_endpoint = "192.168.1.22:33799";
+      endpoint = "192.168.1.22:33799";	 ## TODO nat
+    }
+    {
+      hostname = "caribou";
+      site_name = "neptune";
+      publicKey = "X1OlfgjZDdJw3yTg0ytGFlJZXzqT0yHJmYP15R9IpD8=";
+      IP = "10.14.181.82";
+      lan_endpoint = "192.168.1.23:33799";
+      endpoint = "192.168.1.23:33799";	 ## TODO nat
+    }
+  ];
 
   # Bootstrap IPs for Consul cluster,
   # these are IPs on the Wesher overlay
@@ -22,13 +42,6 @@
     "10.14.252.121" # carcajou
   ];
 
-  # Add wesher hosts to /etc/hosts, needed for elasticsearch
-  networking.extraHosts = ''
-10.14.181.82 caribou
-10.14.179.56 cariacou
-10.14.242.121 carcajou
-    '';
-
   deuxfleurs.admin_accounts = {
     lx = [
       # Keys for accessing nodes from outside

deploy_nixos

@@ -3,10 +3,12 @@
 copy nix/configuration.nix /etc/nixos/configuration.nix
 copy nix/deuxfleurs.nix /etc/nixos/deuxfleurs.nix
 copy nix/remote-unlock.nix /etc/nixos/remote-unlock.nix
-copy nix/wesher.nix /etc/nixos/wesher.nix
-copy nix/wesher_service.nix /etc/nixos/wesher_service.nix
 copy cluster/$CLUSTER/cluster.nix /etc/nixos/cluster.nix
 copy cluster/$CLUSTER/node/$NIXHOST.nix /etc/nixos/node.nix
 copy cluster/$CLUSTER/node/$NIXHOST.site.nix /etc/nixos/site.nix
 
+cmd 'mkdir -p /var/lib/deuxfleurs/wireguard-keys'
+cmd 'test -f /var/lib/deuxfleurs/wireguard-keys/private || (wg genkey > /var/lib/deuxfleurs/wireguard-keys/private; chmod 600 /var/lib/deuxfleurs/wireguard-keys/private)'
+cmd 'echo "Public key: $(wg pubkey < /var/lib/deuxfleurs/wireguard-keys/private)"'
+
 cmd nixos-rebuild switch --show-trace

deploy_wesher_key

@@ -1,4 +0,0 @@
-#!/usr/bin/env ./sshtool
-
-write_pass deuxfleurs/cluster/$CLUSTER/wesher_key /var/lib/wesher/secrets
-cmd systemctl restart wesher

gen_wesher_key

@@ -1,17 +0,0 @@
-#!/usr/bin/env sh
-
-cd $(dirname $0)
-
-CLUSTER="$1"
-if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
-	echo "Usage: $0 <cluster name>"
-	echo "The cluster name must be the name of a subdirectory of cluster/"
-	exit 1
-fi
-
-K=deuxfleurs/cluster/$CLUSTER/wesher_key
-if ! pass $K >/dev/null; then
-	pass insert -m $K <<EOF
-WESHER_CLUSTER_KEY=$(head -c 32 /dev/urandom | base64)
-EOF
-fi

nix/configuration.nix

@@ -11,8 +11,6 @@
       ./hardware-configuration.nix
       # Include generic Deuxfleurs module
       ./deuxfleurs.nix
-      # Wesher module
-      ./wesher_service.nix
       # Configuration for this deployment (a cluster)
       ./cluster.nix
       # Configuration local for this Deuxfleurs site (set of nodes)
@@ -21,10 +19,6 @@
       ./node.nix
     ];
 
-  nixpkgs.overlays = [
-    (import ./wesher.nix)
-  ];
-
   # The global useDHCP flag is deprecated, therefore explicitly set to false here.
   # Per-interface useDHCP will be mandatory in the future, so this generated config
   # replicates the default behaviour.
@@ -70,7 +64,6 @@ SystemMaxUse=1G
     docker
     docker-compose
     wireguard-tools
-    wesher
     #qemu
     #qemu_kvm
   ];

nix/deuxfleurs.nix

@@ -7,6 +7,37 @@ in
   with pkgs.lib;
 {
   options.deuxfleurs =
+    let wg_node = with types; submodule {
+      options = {
+        hostname = mkOption {
+          type = str;
+          description = "Host name";
+        };
+        site_name = mkOption {
+          type = nullOr str;
+          description = "Site where the node is located";
+          default = null;
+        };
+        IP = mkOption {
+          type = str;
+          description = "IP Address in the Wireguard network";
+        };
+        publicKey = mkOption {
+          type = str;
+          description = "Public key";
+        };
+        endpoint = mkOption {
+          type = nullOr str;
+          description = "Wireguard endpoint on the public Internet";
+        };
+        lan_endpoint = mkOption {
+          type = nullOr str;
+          description = "Wireguard endpoint for nodes in the same site";
+          default = null;
+        };
+      };
+    };
+  in
   {
     # Parameters for individual nodes
     network_interface = mkOption {
@@ -30,27 +61,22 @@ in
       type = types.int;
     };
 
-    wesher_cluster_prefix = mkOption {
-      description = "IP address prefix for the Wesher overlay network";
-      type = types.str;
-    };
-    wesher_cluster_prefix_length = mkOption {
-      description = "IP address prefix length for the Wesher overlay network";
-      type = types.int;
-      default = 16;
-    };
-
     cluster_ip = mkOption {
       description = "IP address of this node on the Wesher mesh network";
       type = types.str;
     };
+    wireguard_port = mkOption {
+      description = "Port for incoming Wireguard VPN connections";
+      type = types.port;
+      default = 33799;
+    };
+
     is_raft_server = mkOption {
       description = "Make this node a RAFT server for the Nomad and Consul deployments";
       type = types.bool;
       default = false;
     };
 
-
     # Parameters that generally vary between sites
     lan_default_gateway = mkOption {
       description = "IP address of the default route on the locak network interface";
@@ -70,6 +96,19 @@ in
       description = "Name of this Deuxfleurs deployment";
       type = types.str;
     };
+    cluster_prefix = mkOption {
+      description = "IP address prefix for the Wireguard overlay network";
+      type = types.str;
+    };
+    cluster_prefix_length = mkOption {
+      description = "IP address prefix length for the Wireguard overlay network";
+      type = types.int;
+      default = 16;
+    };
+    cluster_nodes = mkOption {
+      description = "Nodes that are part of the cluster";
+      type = types.listOf wg_node;
+    };
     admin_accounts = mkOption {
       description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
       type = types.attrsOf (types.listOf types.str);
@@ -116,19 +155,24 @@ in
       cfg.lan_ip
     ] ++ cfg.nameservers;
 
-    # wesher overlay network
+    # Configure Wireguard VPN between all nodes
-    services.wesher = {
+    networking.wireguard.interfaces.wg0 = {
-      enable = true;
+      ips = [ "${cfg.cluster_ip}/16" ];
-      bindIface = cfg.network_interface;
+      listenPort = cfg.wireguard_port;
-      overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
+      privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
-      interface = "wg0";
+      peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
-      logLevel = "debug";
+        publicKey = publicKey;
+        allowedIPs = [ "${IP}/32" ];
+        endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
+                   then lan_endpoint else endpoint;
+        persistentKeepalive = 25;
+      }) cfg.cluster_nodes;
     };
 
     # Configure /etc/hosts to link all hostnames to their Wireguard IP
-    #networking.extraHosts = builtins.concatStringsSep "\n" (map
+    networking.extraHosts = builtins.concatStringsSep "\n" (map
-    #  ({ hostname, IP, ...}: "${IP} ${hostname}")
+      ({ hostname, IP, ...}: "${IP} ${hostname}")
-    #  (cfg.cluster_nodes ++ cfg.admin_nodes));
+      cfg.cluster_nodes);
 
     # Enable Hashicorp Consul & Nomad
     services.consul.enable = true;
@@ -245,11 +289,16 @@ in
     networking.firewall = {
       enable = true;
 
-      # Allow anyone to connect on SSH port
       allowedTCPPorts = [
+        # Allow anyone to connect on SSH port
         (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
       ];
 
+      allowedUDPPorts = [
+        # Allow peers to connect to Wireguard
+        cfg.wireguard_port
+      ];
+
       # Allow specific hosts access to specific things in the cluster
       extraCommands = ''
         # Allow everything from router (usefull for UPnP/IGD)
@@ -259,14 +308,14 @@ in
         iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
 
         # Allow other nodes on VPN to access all ports
-        iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+        iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
       '';
 
       # When stopping firewall, delete all rules that were configured manually above
       extraStopCommands = ''
         iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
         iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
-        iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
+        iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
       '';
     };
   };

nix/wesher.nix

@@ -1,25 +0,0 @@
-self: super:
-{
-  wesher = super.buildGoModule rec {
-    pname = "wesher";
-    version = "0.2.6";
-
-    src = super.fetchFromGitHub {
-      owner = "costela";
-      repo = "wesher";
-      rev = "v${version}";
-      sha256 = "1iagmnw2yf15r0fpikk610w0lm0gcxw83lcwfjyr2jv1q2ys71hh";
-    };
-
-    vendorSha256 = "0nyg0wzn8d4rzjs8yrxxj3gha94043ll80s1ql0fml025q2f3705";
-
-    checkPhase = "true";
-
-    meta = with super.lib; {
-      description = "wireguard overlay mesh network manager";
-      homepage = "https://github.com/costela/wesher";
-      license = licenses.gpl3Plus;
-      platforms = platforms.linux;
-    };
-  };
-}

nix/wesher_service.nix

@@ -1,137 +0,0 @@
-{ config, lib, pkgs, ... }:
-with lib;
-let
-  keysPath = "/var/lib/wesher/secrets";
-  cfg = config.services.wesher;
-in {
-  options = with types; {
-    services.wesher = {
-      enable = mkEnableOption "wesher wireguard overlay mesh network manager";
-
-      package = mkOption {
-        type = package;
-        default = pkgs.wesher;
-        defaultText = literalExpression "pkgs.wesher";
-        description = "Wesher package to use.";
-      };
-
-      clusterKey = mkOption {
-        type = nullOr str;
-        default = null;
-        description = "shared key for cluster membership to use on first initialization, if no key was previously used by Wesher. Must be 32 bytes base64 encoded; will be generated if not provided. Setting this parameter value will not overwrite an existing cluster key; to do so please delete ${keysPath}";
-      };
-
-      bindAddr = mkOption {
-        type = nullOr str;
-        default = null;
-        description = "IP address to bind to for cluster membership (cannot be used with --bind-iface)";
-      };
-
-      bindIface = mkOption {
-        type = nullOr str;
-        default = null;
-        description = "Interface to bind to for cluster membership (cannot be used with --bind-addr)";
-      };
-
-      join = mkOption {
-        type = listOf str;
-        default = [];
-        description = "list of hostnames or IP addresses to existing cluster members; if not provided, will attempt resuming any known state or otherwise wait for further members";
-      };
-
-      clusterPort = mkOption {
-        type = port;
-        default = 7946;
-        description = "port used for membership gossip traffic (both TCP and UDP); must be the same accross cluster";
-      };
-
-      wireguardPort = mkOption {
-        type = port;
-        default = 51820;
-        description = "port used for wireguard traffic (UDP); must be the same accross cluster";
-      };
-
-      overlayNet = mkOption {
-        type = str;
-        default = "10.0.0.0/8";
-        description = "the network in which to allocate addresses for the overlay mesh network (CIDR format); smaller networks increase the chance of IP collision";
-      };
-
-      interface = mkOption {
-        type = str;
-        default = "wgoverlay";
-        description = "name of the wireguard interface to create and manage";
-      };
-
-      logLevel = mkOption {
-        type = str;
-        default = "warn";
-        description = "set the verbosity (one of debug/info/warn/error)";
-      };
-      
-    };
-  };
-
-  config = mkIf cfg.enable (let binWesher = cfg.package + "/bin/wesher";
-  in {
-    system.activationScripts.wesher = if (cfg.clusterKey != null) then ''
-      if [ ! -e ${keysPath} ]
-      then
-        mkdir --mode=700 -p ${builtins.dirOf keysPath}
-        echo "WESHER_CLUSTER_KEY=${cfg.clusterKey}" > ${keysPath}
-      fi
-    '' else ''
-      if [ ! -e ${keysPath} ]
-      then
-        mkdir --mode=700 -p ${builtins.dirOf keysPath}
-        echo "WESHER_CLUSTER_KEY=$(head -c 32 /dev/urandom | base64)" > ${keysPath}
-      fi
-    '';
-
-    systemd.services.wesher = {
-      description = "wesher wireguard overlay mesh network manager";
-      bindsTo = [ "network-online.target" ];
-      after = [ "network-online.target" ];
-      wantedBy = [ "multi-user.target" ];
-
-      environment = {
-        WESHER_JOIN = builtins.concatStringsSep "," cfg.join;
-        WESHER_CLUSTER_PORT = builtins.toString cfg.clusterPort;
-        WESHER_WIREGUARD_PORT = builtins.toString cfg.wireguardPort;
-        WESHER_OVERLAY_NET = cfg.overlayNet;
-        WESHER_INTERFACE = cfg.interface;
-        WESHER_LOG_LEVEL = cfg.logLevel;
-        WESHER_NO_ETC_HOSTS = "true";
-      } 
-      // (if (cfg.bindAddr != null) then { WESHER_BIND_ADDR = cfg.bindAddr; } else {})
-      // (if (cfg.bindIface != null) then { WESHER_BIND_IFACE = cfg.bindIface; } else {})
-      ;
-
-      serviceConfig = {
-        ExecStart = "${binWesher}";
-        Restart = "always";
-
-        EnvironmentFile = keysPath;
-
-        User = "wesher";
-        DynamicUser = true;
-        StateDirectory = "wesher";
-
-        AmbientCapabilities = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE";
-        CapabilityBoundingSet = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE";
-        MemoryDenyWriteExecute = true;
-        ProtectControlGroups = true;
-        ProtectKernelModules = true;
-        ProtectKernelTunables = true;
-        RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6 AF_NETLINK";
-        RestrictNamespaces = true;
-        RestrictRealtime = true;
-        SystemCallArchitectures = "native";
-        SystemCallFilter = "~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @resources";
-      };
-    };
-
-    networking.firewall.allowedUDPPorts = mkIf cfg.enable [ cfg.clusterPort cfg.wireguardPort ];
-    networking.firewall.allowedTCPPorts = mkIf cfg.enable [ cfg.clusterPort ];
-  });
-}