Replace ad-hoc wireguard by wesher on staging cluster

This commit is contained in:
Alex 2022-04-20 18:04:57 +02:00
parent 04f2bd48bb
commit 10d370491e
Signed by untrusted user: lx
GPG key ID: 0E496D15096376BE
7 changed files with 97 additions and 189 deletions

View file

@ -2,65 +2,26 @@
{ {
deuxfleurs.cluster_name = "staging"; deuxfleurs.cluster_name = "staging";
deuxfleurs.cluster_nodes = [
{ # Bootstrap nodes for Wesher overlay network
hostname = "spoutnik"; services.wesher.join = [
site_name = "pluton"; "2a01:e0a:c:a720::21" # cariacou
publicKey = "fO8qZOZmnug84cA8nvfjl5MUqyWljP0BAz/4tHRZyEg="; "2a01:e0a:c:a720::22" # carcajou
IP = "10.42.2.2"; "2a01:e0a:c:a720::23" # caribou
endpoint = "77.141.67.109:42136";
}
{
hostname = "cariacou";
site_name = "neptune";
publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
IP = "10.42.2.21";
endpoint = "82.66.112.151:33721";
lan_endpoint = "192.168.1.21:33721";
}
{
hostname = "carcajou";
site_name = "neptune";
publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
IP = "10.42.2.22";
endpoint = "82.66.112.151:33722";
lan_endpoint = "192.168.1.22:33722";
}
{
hostname = "caribou";
site_name = "neptune";
publicKey = "g6ZED/wPn5MPfytJKwPI19808CXtEad0IJUkEAAzwyY=";
IP = "10.42.2.23";
endpoint = "82.66.112.151:33723";
lan_endpoint = "192.168.1.23:33723";
}
]; ];
deuxfleurs.admin_nodes = [
{ # The IP range to use for the Wesher overlay of this cluster
hostname = "hammerhead"; deuxfleurs.wesher_cluster_prefix = "10.14.0.0";
publicKey = "b5hF+GSTgg3oM6wnjL7jRbfyf1jtsWdVptPPbAh3Qic="; deuxfleurs.wesher_cluster_prefix_length = 16;
IP = "10.42.0.1";
endpoint = "5.135.179.11:51349"; # Bootstrap IPs for Consul cluster,
} # these are IPs on the Wesher overlay
{ services.consul.extraConfig.retry_join = [
hostname = "robinson"; "10.14.181.82" # caribou
publicKey = "ETaZFil3mFXlJ0LaJZyWqJVLV2IZUF5PB/8M7WbQSTg="; "10.14.179.56" # cariacou
IP = "10.42.0.42"; "10.14.252.121" # carcajou
endpoint = "77.141.67.109:33742";
}
{
hostname = "shiki";
publicKey = "QUiUNMk70TEQ75Ut7Uqikr5uGVSXmx8EGNkGM6tANlg=";
IP = "10.42.0.206";
endpoint = "37.187.118.206:51820";
}
{
hostname = "lindy";
publicKey = "wen9GnZy2iLT6RyHfn7ydS/wvdvow1XPmhZxIkrDbks=";
IP = "10.42.0.66";
endpoint = "82.66.112.151:33766";
}
]; ];
deuxfleurs.admin_accounts = { deuxfleurs.admin_accounts = {
lx = [ lx = [
# Keys for accessing nodes from outside # Keys for accessing nodes from outside

View file

@ -17,16 +17,9 @@
deuxfleurs.network_interface = "eno1"; deuxfleurs.network_interface = "eno1";
deuxfleurs.lan_ip = "192.168.1.22"; deuxfleurs.lan_ip = "192.168.1.22";
deuxfleurs.ipv6 = "2a01:e0a:c:a720::22";
networking.interfaces.eno1.ipv6.addresses = [ deuxfleurs.cluster_ip = "10.14.252.121";
{
address = "2a01:e0a:c:a720::22";
prefixLength = 64;
}
];
deuxfleurs.vpn_ip = "10.42.2.22";
deuxfleurs.vpn_listen_port = 33722;
deuxfleurs.is_raft_server = true; deuxfleurs.is_raft_server = true;
# Enable netdata monitoring # Enable netdata monitoring

View file

@ -17,16 +17,9 @@
deuxfleurs.network_interface = "eno1"; deuxfleurs.network_interface = "eno1";
deuxfleurs.lan_ip = "192.168.1.21"; deuxfleurs.lan_ip = "192.168.1.21";
deuxfleurs.ipv6 = "2a01:e0a:c:a720::21";
networking.interfaces.eno1.ipv6.addresses = [ deuxfleurs.cluster_ip = "10.14.179.56";
{
address = "2a01:e0a:c:a720::21";
prefixLength = 64;
}
];
deuxfleurs.vpn_ip = "10.42.2.21";
deuxfleurs.vpn_listen_port = 33721;
deuxfleurs.is_raft_server = true; deuxfleurs.is_raft_server = true;
# Enable netdata monitoring # Enable netdata monitoring

View file

@ -17,16 +17,9 @@
deuxfleurs.network_interface = "eno1"; deuxfleurs.network_interface = "eno1";
deuxfleurs.lan_ip = "192.168.1.23"; deuxfleurs.lan_ip = "192.168.1.23";
deuxfleurs.ipv6 = "2a01:e0a:c:a720::23";
networking.interfaces.eno1.ipv6.addresses = [ deuxfleurs.cluster_ip = "10.14.181.82";
{
address = "2a01:e0a:c:a720::23";
prefixLength = 64;
}
];
deuxfleurs.vpn_ip = "10.42.2.23";
deuxfleurs.vpn_listen_port = 33723;
deuxfleurs.is_raft_server = true; deuxfleurs.is_raft_server = true;
# Enable netdata monitoring # Enable netdata monitoring

View file

@ -4,6 +4,7 @@
deuxfleurs.site_name = "neptune"; deuxfleurs.site_name = "neptune";
deuxfleurs.lan_default_gateway = "192.168.1.254"; deuxfleurs.lan_default_gateway = "192.168.1.254";
deuxfleurs.lan_ip_prefix_length = 24; deuxfleurs.lan_ip_prefix_length = 24;
deuxfleurs.ipv6_prefix_length = 64;
networking.nameservers = [ "192.168.1.254" ]; networking.nameservers = [ "192.168.1.254" ];

View file

@ -69,6 +69,7 @@ SystemMaxUse=1G
rclone rclone
docker docker
docker-compose docker-compose
wireguard
wesher wesher
]; ];
@ -81,43 +82,6 @@ SystemMaxUse=1G
services.openssh.enable = true; services.openssh.enable = true;
services.openssh.passwordAuthentication = false; services.openssh.passwordAuthentication = false;
services.wesher = {
enable = true;
join = [ "192.168.1.22" "192.168.1.23" ];
bindAddr = config.deuxfleurs.lan_ip; # for now
overlayNet = "10.14.0.0/16";
};
# ---- CONFIG FOR DEUXFLEURS CLUSTER ----
# Open ports in the firewall.
networking.firewall = {
enable = true;
# Allow anyone to connect on SSH port
allowedTCPPorts = [
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
];
# Allow specific hosts access to specific things in the cluster
extraCommands = ''
# Allow everything from router (usefull for UPnP/IGD)
iptables -A INPUT -s 192.168.1.254 -j ACCEPT
# Allow docker containers to access all ports
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
# Allow other nodes on VPN to access all ports
iptables -A INPUT -s 10.42.0.0/16 -j ACCEPT
'';
# When stopping firewall, delete all rules that were configured manually above
extraStopCommands = ''
iptables -D INPUT -s 192.168.1.254 -j ACCEPT
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
iptables -D INPUT -s 10.42.0.0/16 -j ACCEPT
'';
};
# This value determines the NixOS release from which the default # This value determines the NixOS release from which the default
# settings for stateful data, like file locations and database versions # settings for stateful data, like file locations and database versions

View file

@ -7,37 +7,6 @@ in
with pkgs.lib; with pkgs.lib;
{ {
options.deuxfleurs = options.deuxfleurs =
let wg_node = with types; submodule {
options = {
hostname = mkOption {
type = str;
description = "Host name";
};
site_name = mkOption {
type = nullOr str;
description = "Site where the node is located";
default = null;
};
IP = mkOption {
type = str;
description = "IP Address";
};
publicKey = mkOption {
type = str;
description = "Public key";
};
endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint on the public Internet";
};
lan_endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint for nodes in the same site";
default = null;
};
};
};
in
{ {
# Parameters for individual nodes # Parameters for individual nodes
network_interface = mkOption { network_interface = mkOption {
@ -52,14 +21,28 @@ in
description = "Prefix length associated with lan_ip"; description = "Prefix length associated with lan_ip";
type = types.int; type = types.int;
}; };
ipv6 = mkOption {
vpn_ip = mkOption { description = "Public IPv6 address of this node";
description = "IP address of this node on the Wireguard VPN";
type = types.str; type = types.str;
}; };
vpn_listen_port = mkOption { ipv6_prefix_length = mkOption {
description = "Port for incoming Wireguard VPN connections"; description = "Prefix length associated with ipv6 ip";
type = types.port; type = types.int;
};
wesher_cluster_prefix = mkOption {
description = "IP address prefix for the Wesher overlay network";
type = types.str;
};
wesher_cluster_prefix_length = mkOption {
description = "IP address prefix length for the Wesher overlay network";
type = types.int;
default = 16;
};
cluster_ip = mkOption {
description = "IP address of this node on the Wesher mesh network";
type = types.str;
}; };
is_raft_server = mkOption { is_raft_server = mkOption {
description = "Make this node a RAFT server for the Nomad and Consul deployments"; description = "Make this node a RAFT server for the Nomad and Consul deployments";
@ -83,14 +66,6 @@ in
description = "Name of this Deuxfleurs deployment"; description = "Name of this Deuxfleurs deployment";
type = types.str; type = types.str;
}; };
cluster_nodes = mkOption {
description = "Nodes that are part of the cluster";
type = types.listOf wg_node;
};
admin_nodes = mkOption {
description = "Machines that are part of the Wireguard VPN for administration purposes";
type = types.listOf wg_node;
};
admin_accounts = mkOption { admin_accounts = mkOption {
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys"; description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
type = types.attrsOf (types.listOf types.str); type = types.attrsOf (types.listOf types.str);
@ -114,32 +89,31 @@ in
prefixLength = cfg.lan_ip_prefix_length; prefixLength = cfg.lan_ip_prefix_length;
} }
]; ];
ipv6.addresses = [
{
address = cfg.ipv6;
prefixLength = cfg.ipv6_prefix_length;
}
];
}; };
networking.defaultGateway = { networking.defaultGateway = {
address = cfg.lan_default_gateway; address = cfg.lan_default_gateway;
interface = cfg.network_interface; interface = cfg.network_interface;
}; };
# Configure Wireguard VPN between all nodes # wesher overlay network
networking.wireguard.interfaces.wg0 = { services.wesher = {
ips = [ "${cfg.vpn_ip}/16" ]; enable = true;
listenPort = cfg.vpn_listen_port; bindAddr = cfg.ipv6;
privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: { interface = "wg0";
publicKey = publicKey; logLevel = "debug";
allowedIPs = [ "${IP}/32" ];
endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
then lan_endpoint else endpoint;
persistentKeepalive = 25;
}) (cfg.cluster_nodes ++ cfg.admin_nodes);
}; };
networking.firewall.allowedUDPPorts = [ cfg.vpn_listen_port ];
# Configure /etc/hosts to link all hostnames to their Wireguard IP # Configure /etc/hosts to link all hostnames to their Wireguard IP
networking.extraHosts = builtins.concatStringsSep "\n" (map #networking.extraHosts = builtins.concatStringsSep "\n" (map
({ hostname, IP, ...}: "${IP} ${hostname}") # ({ hostname, IP, ...}: "${IP} ${hostname}")
(cfg.cluster_nodes ++ cfg.admin_nodes)); # (cfg.cluster_nodes ++ cfg.admin_nodes));
# Enable Hashicorp Consul & Nomad # Enable Hashicorp Consul & Nomad
services.consul.enable = true; services.consul.enable = true;
@ -156,14 +130,12 @@ in
"site" = cfg.site_name; "site" = cfg.site_name;
}; };
ui = true; ui = true;
bind_addr = cfg.vpn_ip; bind_addr = "${cfg.cluster_ip}";
ports.http = -1; ports.http = -1;
addresses.https = "0.0.0.0"; addresses.https = "0.0.0.0";
ports.https = 8501; ports.https = 8501;
retry_join = map (node_info: node_info.IP) cfg.cluster_nodes;
ca_file = "/var/lib/consul/pki/consul-ca.crt"; ca_file = "/var/lib/consul/pki/consul-ca.crt";
cert_file = "/var/lib/consul/pki/consul2022.crt"; cert_file = "/var/lib/consul/pki/consul2022.crt";
key_file = "/var/lib/consul/pki/consul2022.key"; key_file = "/var/lib/consul/pki/consul2022.key";
@ -185,9 +157,9 @@ in
region = cfg.cluster_name; region = cfg.cluster_name;
datacenter = cfg.site_name; datacenter = cfg.site_name;
advertise = { advertise = {
rpc = cfg.vpn_ip; rpc = "${cfg.cluster_ip}";
http = cfg.vpn_ip; http = "${cfg.cluster_ip}";
serf = cfg.vpn_ip; serf = "${cfg.cluster_ip}";
}; };
consul = { consul = {
address = "localhost:8501"; address = "localhost:8501";
@ -227,5 +199,36 @@ in
} }
]; ];
}; };
# ---- Firewall config ----
# Open ports in the firewall.
networking.firewall = {
enable = true;
# Allow anyone to connect on SSH port
allowedTCPPorts = [
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
];
# Allow specific hosts access to specific things in the cluster
extraCommands = ''
# Allow everything from router (usefull for UPnP/IGD)
iptables -A INPUT -s 192.168.1.254 -j ACCEPT
# Allow docker containers to access all ports
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
# Allow other nodes on VPN to access all ports
iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
'';
# When stopping firewall, delete all rules that were configured manually above
extraStopCommands = ''
iptables -D INPUT -s 192.168.1.254 -j ACCEPT
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
'';
};
}; };
} }