Replace ad-hoc wireguard by wesher on staging cluster

This commit is contained in:
Alex 2022-04-20 18:04:57 +02:00
parent 04f2bd48bb
commit 10d370491e
Signed by untrusted user: lx
GPG key ID: 0E496D15096376BE
7 changed files with 97 additions and 189 deletions

View file

@ -2,65 +2,26 @@
{
deuxfleurs.cluster_name = "staging";
deuxfleurs.cluster_nodes = [
{
hostname = "spoutnik";
site_name = "pluton";
publicKey = "fO8qZOZmnug84cA8nvfjl5MUqyWljP0BAz/4tHRZyEg=";
IP = "10.42.2.2";
endpoint = "77.141.67.109:42136";
}
{
hostname = "cariacou";
site_name = "neptune";
publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
IP = "10.42.2.21";
endpoint = "82.66.112.151:33721";
lan_endpoint = "192.168.1.21:33721";
}
{
hostname = "carcajou";
site_name = "neptune";
publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
IP = "10.42.2.22";
endpoint = "82.66.112.151:33722";
lan_endpoint = "192.168.1.22:33722";
}
{
hostname = "caribou";
site_name = "neptune";
publicKey = "g6ZED/wPn5MPfytJKwPI19808CXtEad0IJUkEAAzwyY=";
IP = "10.42.2.23";
endpoint = "82.66.112.151:33723";
lan_endpoint = "192.168.1.23:33723";
}
# Bootstrap nodes for Wesher overlay network
services.wesher.join = [
"2a01:e0a:c:a720::21" # cariacou
"2a01:e0a:c:a720::22" # carcajou
"2a01:e0a:c:a720::23" # caribou
];
deuxfleurs.admin_nodes = [
{
hostname = "hammerhead";
publicKey = "b5hF+GSTgg3oM6wnjL7jRbfyf1jtsWdVptPPbAh3Qic=";
IP = "10.42.0.1";
endpoint = "5.135.179.11:51349";
}
{
hostname = "robinson";
publicKey = "ETaZFil3mFXlJ0LaJZyWqJVLV2IZUF5PB/8M7WbQSTg=";
IP = "10.42.0.42";
endpoint = "77.141.67.109:33742";
}
{
hostname = "shiki";
publicKey = "QUiUNMk70TEQ75Ut7Uqikr5uGVSXmx8EGNkGM6tANlg=";
IP = "10.42.0.206";
endpoint = "37.187.118.206:51820";
}
{
hostname = "lindy";
publicKey = "wen9GnZy2iLT6RyHfn7ydS/wvdvow1XPmhZxIkrDbks=";
IP = "10.42.0.66";
endpoint = "82.66.112.151:33766";
}
# The IP range to use for the Wesher overlay of this cluster
deuxfleurs.wesher_cluster_prefix = "10.14.0.0";
deuxfleurs.wesher_cluster_prefix_length = 16;
# Bootstrap IPs for Consul cluster,
# these are IPs on the Wesher overlay
services.consul.extraConfig.retry_join = [
"10.14.181.82" # caribou
"10.14.179.56" # cariacou
"10.14.252.121" # carcajou
];
deuxfleurs.admin_accounts = {
lx = [
# Keys for accessing nodes from outside

View file

@ -17,16 +17,9 @@
deuxfleurs.network_interface = "eno1";
deuxfleurs.lan_ip = "192.168.1.22";
deuxfleurs.ipv6 = "2a01:e0a:c:a720::22";
networking.interfaces.eno1.ipv6.addresses = [
{
address = "2a01:e0a:c:a720::22";
prefixLength = 64;
}
];
deuxfleurs.vpn_ip = "10.42.2.22";
deuxfleurs.vpn_listen_port = 33722;
deuxfleurs.cluster_ip = "10.14.252.121";
deuxfleurs.is_raft_server = true;
# Enable netdata monitoring

View file

@ -17,16 +17,9 @@
deuxfleurs.network_interface = "eno1";
deuxfleurs.lan_ip = "192.168.1.21";
deuxfleurs.ipv6 = "2a01:e0a:c:a720::21";
networking.interfaces.eno1.ipv6.addresses = [
{
address = "2a01:e0a:c:a720::21";
prefixLength = 64;
}
];
deuxfleurs.vpn_ip = "10.42.2.21";
deuxfleurs.vpn_listen_port = 33721;
deuxfleurs.cluster_ip = "10.14.179.56";
deuxfleurs.is_raft_server = true;
# Enable netdata monitoring

View file

@ -17,16 +17,9 @@
deuxfleurs.network_interface = "eno1";
deuxfleurs.lan_ip = "192.168.1.23";
deuxfleurs.ipv6 = "2a01:e0a:c:a720::23";
networking.interfaces.eno1.ipv6.addresses = [
{
address = "2a01:e0a:c:a720::23";
prefixLength = 64;
}
];
deuxfleurs.vpn_ip = "10.42.2.23";
deuxfleurs.vpn_listen_port = 33723;
deuxfleurs.cluster_ip = "10.14.181.82";
deuxfleurs.is_raft_server = true;
# Enable netdata monitoring

View file

@ -4,6 +4,7 @@
deuxfleurs.site_name = "neptune";
deuxfleurs.lan_default_gateway = "192.168.1.254";
deuxfleurs.lan_ip_prefix_length = 24;
deuxfleurs.ipv6_prefix_length = 64;
networking.nameservers = [ "192.168.1.254" ];

View file

@ -69,6 +69,7 @@ SystemMaxUse=1G
rclone
docker
docker-compose
wireguard
wesher
];
@ -81,43 +82,6 @@ SystemMaxUse=1G
services.openssh.enable = true;
services.openssh.passwordAuthentication = false;
services.wesher = {
enable = true;
join = [ "192.168.1.22" "192.168.1.23" ];
bindAddr = config.deuxfleurs.lan_ip; # for now
overlayNet = "10.14.0.0/16";
};
# ---- CONFIG FOR DEUXFLEURS CLUSTER ----
# Open ports in the firewall.
networking.firewall = {
enable = true;
# Allow anyone to connect on SSH port
allowedTCPPorts = [
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
];
# Allow specific hosts access to specific things in the cluster
extraCommands = ''
# Allow everything from router (usefull for UPnP/IGD)
iptables -A INPUT -s 192.168.1.254 -j ACCEPT
# Allow docker containers to access all ports
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
# Allow other nodes on VPN to access all ports
iptables -A INPUT -s 10.42.0.0/16 -j ACCEPT
'';
# When stopping firewall, delete all rules that were configured manually above
extraStopCommands = ''
iptables -D INPUT -s 192.168.1.254 -j ACCEPT
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
iptables -D INPUT -s 10.42.0.0/16 -j ACCEPT
'';
};
# This value determines the NixOS release from which the default
# settings for stateful data, like file locations and database versions

View file

@ -7,37 +7,6 @@ in
with pkgs.lib;
{
options.deuxfleurs =
let wg_node = with types; submodule {
options = {
hostname = mkOption {
type = str;
description = "Host name";
};
site_name = mkOption {
type = nullOr str;
description = "Site where the node is located";
default = null;
};
IP = mkOption {
type = str;
description = "IP Address";
};
publicKey = mkOption {
type = str;
description = "Public key";
};
endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint on the public Internet";
};
lan_endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint for nodes in the same site";
default = null;
};
};
};
in
{
# Parameters for individual nodes
network_interface = mkOption {
@ -52,14 +21,28 @@ in
description = "Prefix length associated with lan_ip";
type = types.int;
};
vpn_ip = mkOption {
description = "IP address of this node on the Wireguard VPN";
ipv6 = mkOption {
description = "Public IPv6 address of this node";
type = types.str;
};
vpn_listen_port = mkOption {
description = "Port for incoming Wireguard VPN connections";
type = types.port;
ipv6_prefix_length = mkOption {
description = "Prefix length associated with ipv6 ip";
type = types.int;
};
wesher_cluster_prefix = mkOption {
description = "IP address prefix for the Wesher overlay network";
type = types.str;
};
wesher_cluster_prefix_length = mkOption {
description = "IP address prefix length for the Wesher overlay network";
type = types.int;
default = 16;
};
cluster_ip = mkOption {
description = "IP address of this node on the Wesher mesh network";
type = types.str;
};
is_raft_server = mkOption {
description = "Make this node a RAFT server for the Nomad and Consul deployments";
@ -83,14 +66,6 @@ in
description = "Name of this Deuxfleurs deployment";
type = types.str;
};
cluster_nodes = mkOption {
description = "Nodes that are part of the cluster";
type = types.listOf wg_node;
};
admin_nodes = mkOption {
description = "Machines that are part of the Wireguard VPN for administration purposes";
type = types.listOf wg_node;
};
admin_accounts = mkOption {
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
type = types.attrsOf (types.listOf types.str);
@ -114,32 +89,31 @@ in
prefixLength = cfg.lan_ip_prefix_length;
}
];
ipv6.addresses = [
{
address = cfg.ipv6;
prefixLength = cfg.ipv6_prefix_length;
}
];
};
networking.defaultGateway = {
address = cfg.lan_default_gateway;
interface = cfg.network_interface;
};
# Configure Wireguard VPN between all nodes
networking.wireguard.interfaces.wg0 = {
ips = [ "${cfg.vpn_ip}/16" ];
listenPort = cfg.vpn_listen_port;
privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
publicKey = publicKey;
allowedIPs = [ "${IP}/32" ];
endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
then lan_endpoint else endpoint;
persistentKeepalive = 25;
}) (cfg.cluster_nodes ++ cfg.admin_nodes);
# wesher overlay network
services.wesher = {
enable = true;
bindAddr = cfg.ipv6;
overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
interface = "wg0";
logLevel = "debug";
};
networking.firewall.allowedUDPPorts = [ cfg.vpn_listen_port ];
# Configure /etc/hosts to link all hostnames to their Wireguard IP
networking.extraHosts = builtins.concatStringsSep "\n" (map
({ hostname, IP, ...}: "${IP} ${hostname}")
(cfg.cluster_nodes ++ cfg.admin_nodes));
#networking.extraHosts = builtins.concatStringsSep "\n" (map
# ({ hostname, IP, ...}: "${IP} ${hostname}")
# (cfg.cluster_nodes ++ cfg.admin_nodes));
# Enable Hashicorp Consul & Nomad
services.consul.enable = true;
@ -156,14 +130,12 @@ in
"site" = cfg.site_name;
};
ui = true;
bind_addr = cfg.vpn_ip;
bind_addr = "${cfg.cluster_ip}";
ports.http = -1;
addresses.https = "0.0.0.0";
ports.https = 8501;
retry_join = map (node_info: node_info.IP) cfg.cluster_nodes;
ca_file = "/var/lib/consul/pki/consul-ca.crt";
cert_file = "/var/lib/consul/pki/consul2022.crt";
key_file = "/var/lib/consul/pki/consul2022.key";
@ -185,9 +157,9 @@ in
region = cfg.cluster_name;
datacenter = cfg.site_name;
advertise = {
rpc = cfg.vpn_ip;
http = cfg.vpn_ip;
serf = cfg.vpn_ip;
rpc = "${cfg.cluster_ip}";
http = "${cfg.cluster_ip}";
serf = "${cfg.cluster_ip}";
};
consul = {
address = "localhost:8501";
@ -227,5 +199,36 @@ in
}
];
};
# ---- Firewall config ----
# Open ports in the firewall.
networking.firewall = {
enable = true;
# Allow anyone to connect on SSH port
allowedTCPPorts = [
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
];
# Allow specific hosts access to specific things in the cluster
extraCommands = ''
# Allow everything from router (usefull for UPnP/IGD)
iptables -A INPUT -s 192.168.1.254 -j ACCEPT
# Allow docker containers to access all ports
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
# Allow other nodes on VPN to access all ports
iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
'';
# When stopping firewall, delete all rules that were configured manually above
extraStopCommands = ''
iptables -D INPUT -s 192.168.1.254 -j ACCEPT
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
'';
};
};
}