Replace ad-hoc wireguard by wesher on staging cluster
This commit is contained in:
parent
04f2bd48bb
commit
10d370491e
GPG key ID:
0E496D15096376BE
7 changed files with 97 additions and 189 deletions
|
|
@ -2,65 +2,26 @@
|
|||
|
||||
{
|
||||
deuxfleurs.cluster_name = "staging";
|
||||
deuxfleurs.cluster_nodes = [
|
||||
{
|
||||
hostname = "spoutnik";
|
||||
site_name = "pluton";
|
||||
publicKey = "fO8qZOZmnug84cA8nvfjl5MUqyWljP0BAz/4tHRZyEg=";
|
||||
IP = "10.42.2.2";
|
||||
endpoint = "77.141.67.109:42136";
|
||||
}
|
||||
{
|
||||
hostname = "cariacou";
|
||||
site_name = "neptune";
|
||||
publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
|
||||
IP = "10.42.2.21";
|
||||
endpoint = "82.66.112.151:33721";
|
||||
lan_endpoint = "192.168.1.21:33721";
|
||||
}
|
||||
{
|
||||
hostname = "carcajou";
|
||||
site_name = "neptune";
|
||||
publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
|
||||
IP = "10.42.2.22";
|
||||
endpoint = "82.66.112.151:33722";
|
||||
lan_endpoint = "192.168.1.22:33722";
|
||||
}
|
||||
{
|
||||
hostname = "caribou";
|
||||
site_name = "neptune";
|
||||
publicKey = "g6ZED/wPn5MPfytJKwPI19808CXtEad0IJUkEAAzwyY=";
|
||||
IP = "10.42.2.23";
|
||||
endpoint = "82.66.112.151:33723";
|
||||
lan_endpoint = "192.168.1.23:33723";
|
||||
}
|
||||
|
||||
# Bootstrap nodes for Wesher overlay network
|
||||
services.wesher.join = [
|
||||
"2a01:e0a:c:a720::21" # cariacou
|
||||
"2a01:e0a:c:a720::22" # carcajou
|
||||
"2a01:e0a:c:a720::23" # caribou
|
||||
];
|
||||
deuxfleurs.admin_nodes = [
|
||||
{
|
||||
hostname = "hammerhead";
|
||||
publicKey = "b5hF+GSTgg3oM6wnjL7jRbfyf1jtsWdVptPPbAh3Qic=";
|
||||
IP = "10.42.0.1";
|
||||
endpoint = "5.135.179.11:51349";
|
||||
}
|
||||
{
|
||||
hostname = "robinson";
|
||||
publicKey = "ETaZFil3mFXlJ0LaJZyWqJVLV2IZUF5PB/8M7WbQSTg=";
|
||||
IP = "10.42.0.42";
|
||||
endpoint = "77.141.67.109:33742";
|
||||
}
|
||||
{
|
||||
hostname = "shiki";
|
||||
publicKey = "QUiUNMk70TEQ75Ut7Uqikr5uGVSXmx8EGNkGM6tANlg=";
|
||||
IP = "10.42.0.206";
|
||||
endpoint = "37.187.118.206:51820";
|
||||
}
|
||||
{
|
||||
hostname = "lindy";
|
||||
publicKey = "wen9GnZy2iLT6RyHfn7ydS/wvdvow1XPmhZxIkrDbks=";
|
||||
IP = "10.42.0.66";
|
||||
endpoint = "82.66.112.151:33766";
|
||||
}
|
||||
|
||||
# The IP range to use for the Wesher overlay of this cluster
|
||||
deuxfleurs.wesher_cluster_prefix = "10.14.0.0";
|
||||
deuxfleurs.wesher_cluster_prefix_length = 16;
|
||||
|
||||
# Bootstrap IPs for Consul cluster,
|
||||
# these are IPs on the Wesher overlay
|
||||
services.consul.extraConfig.retry_join = [
|
||||
"10.14.181.82" # caribou
|
||||
"10.14.179.56" # cariacou
|
||||
"10.14.252.121" # carcajou
|
||||
];
|
||||
|
||||
deuxfleurs.admin_accounts = {
|
||||
lx = [
|
||||
# Keys for accessing nodes from outside
|
||||
|
|
|
|||
|
|
@ -17,16 +17,9 @@
|
|||
|
||||
deuxfleurs.network_interface = "eno1";
|
||||
deuxfleurs.lan_ip = "192.168.1.22";
|
||||
deuxfleurs.ipv6 = "2a01:e0a:c:a720::22";
|
||||
|
||||
networking.interfaces.eno1.ipv6.addresses = [
|
||||
{
|
||||
address = "2a01:e0a:c:a720::22";
|
||||
prefixLength = 64;
|
||||
}
|
||||
];
|
||||
|
||||
deuxfleurs.vpn_ip = "10.42.2.22";
|
||||
deuxfleurs.vpn_listen_port = 33722;
|
||||
deuxfleurs.cluster_ip = "10.14.252.121";
|
||||
deuxfleurs.is_raft_server = true;
|
||||
|
||||
# Enable netdata monitoring
|
||||
|
|
|
|||
|
|
@ -17,16 +17,9 @@
|
|||
|
||||
deuxfleurs.network_interface = "eno1";
|
||||
deuxfleurs.lan_ip = "192.168.1.21";
|
||||
deuxfleurs.ipv6 = "2a01:e0a:c:a720::21";
|
||||
|
||||
networking.interfaces.eno1.ipv6.addresses = [
|
||||
{
|
||||
address = "2a01:e0a:c:a720::21";
|
||||
prefixLength = 64;
|
||||
}
|
||||
];
|
||||
|
||||
deuxfleurs.vpn_ip = "10.42.2.21";
|
||||
deuxfleurs.vpn_listen_port = 33721;
|
||||
deuxfleurs.cluster_ip = "10.14.179.56";
|
||||
deuxfleurs.is_raft_server = true;
|
||||
|
||||
# Enable netdata monitoring
|
||||
|
|
|
|||
|
|
@ -17,16 +17,9 @@
|
|||
|
||||
deuxfleurs.network_interface = "eno1";
|
||||
deuxfleurs.lan_ip = "192.168.1.23";
|
||||
deuxfleurs.ipv6 = "2a01:e0a:c:a720::23";
|
||||
|
||||
networking.interfaces.eno1.ipv6.addresses = [
|
||||
{
|
||||
address = "2a01:e0a:c:a720::23";
|
||||
prefixLength = 64;
|
||||
}
|
||||
];
|
||||
|
||||
deuxfleurs.vpn_ip = "10.42.2.23";
|
||||
deuxfleurs.vpn_listen_port = 33723;
|
||||
deuxfleurs.cluster_ip = "10.14.181.82";
|
||||
deuxfleurs.is_raft_server = true;
|
||||
|
||||
# Enable netdata monitoring
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
deuxfleurs.site_name = "neptune";
|
||||
deuxfleurs.lan_default_gateway = "192.168.1.254";
|
||||
deuxfleurs.lan_ip_prefix_length = 24;
|
||||
deuxfleurs.ipv6_prefix_length = 64;
|
||||
|
||||
networking.nameservers = [ "192.168.1.254" ];
|
||||
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ SystemMaxUse=1G
|
|||
rclone
|
||||
docker
|
||||
docker-compose
|
||||
wireguard
|
||||
wesher
|
||||
];
|
||||
|
||||
|
|
@ -81,43 +82,6 @@ SystemMaxUse=1G
|
|||
services.openssh.enable = true;
|
||||
services.openssh.passwordAuthentication = false;
|
||||
|
||||
services.wesher = {
|
||||
enable = true;
|
||||
join = [ "192.168.1.22" "192.168.1.23" ];
|
||||
bindAddr = config.deuxfleurs.lan_ip; # for now
|
||||
overlayNet = "10.14.0.0/16";
|
||||
};
|
||||
|
||||
# ---- CONFIG FOR DEUXFLEURS CLUSTER ----
|
||||
|
||||
# Open ports in the firewall.
|
||||
networking.firewall = {
|
||||
enable = true;
|
||||
|
||||
# Allow anyone to connect on SSH port
|
||||
allowedTCPPorts = [
|
||||
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
|
||||
];
|
||||
|
||||
# Allow specific hosts access to specific things in the cluster
|
||||
extraCommands = ''
|
||||
# Allow everything from router (usefull for UPnP/IGD)
|
||||
iptables -A INPUT -s 192.168.1.254 -j ACCEPT
|
||||
|
||||
# Allow docker containers to access all ports
|
||||
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
|
||||
|
||||
# Allow other nodes on VPN to access all ports
|
||||
iptables -A INPUT -s 10.42.0.0/16 -j ACCEPT
|
||||
'';
|
||||
|
||||
# When stopping firewall, delete all rules that were configured manually above
|
||||
extraStopCommands = ''
|
||||
iptables -D INPUT -s 192.168.1.254 -j ACCEPT
|
||||
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
|
||||
iptables -D INPUT -s 10.42.0.0/16 -j ACCEPT
|
||||
'';
|
||||
};
|
||||
|
||||
# This value determines the NixOS release from which the default
|
||||
# settings for stateful data, like file locations and database versions
|
||||
|
|
|
|||
|
|
@ -7,37 +7,6 @@ in
|
|||
with pkgs.lib;
|
||||
{
|
||||
options.deuxfleurs =
|
||||
let wg_node = with types; submodule {
|
||||
options = {
|
||||
hostname = mkOption {
|
||||
type = str;
|
||||
description = "Host name";
|
||||
};
|
||||
site_name = mkOption {
|
||||
type = nullOr str;
|
||||
description = "Site where the node is located";
|
||||
default = null;
|
||||
};
|
||||
IP = mkOption {
|
||||
type = str;
|
||||
description = "IP Address";
|
||||
};
|
||||
publicKey = mkOption {
|
||||
type = str;
|
||||
description = "Public key";
|
||||
};
|
||||
endpoint = mkOption {
|
||||
type = nullOr str;
|
||||
description = "Wireguard endpoint on the public Internet";
|
||||
};
|
||||
lan_endpoint = mkOption {
|
||||
type = nullOr str;
|
||||
description = "Wireguard endpoint for nodes in the same site";
|
||||
default = null;
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
# Parameters for individual nodes
|
||||
network_interface = mkOption {
|
||||
|
|
@ -52,14 +21,28 @@ in
|
|||
description = "Prefix length associated with lan_ip";
|
||||
type = types.int;
|
||||
};
|
||||
|
||||
vpn_ip = mkOption {
|
||||
description = "IP address of this node on the Wireguard VPN";
|
||||
ipv6 = mkOption {
|
||||
description = "Public IPv6 address of this node";
|
||||
type = types.str;
|
||||
};
|
||||
vpn_listen_port = mkOption {
|
||||
description = "Port for incoming Wireguard VPN connections";
|
||||
type = types.port;
|
||||
ipv6_prefix_length = mkOption {
|
||||
description = "Prefix length associated with ipv6 ip";
|
||||
type = types.int;
|
||||
};
|
||||
|
||||
wesher_cluster_prefix = mkOption {
|
||||
description = "IP address prefix for the Wesher overlay network";
|
||||
type = types.str;
|
||||
};
|
||||
wesher_cluster_prefix_length = mkOption {
|
||||
description = "IP address prefix length for the Wesher overlay network";
|
||||
type = types.int;
|
||||
default = 16;
|
||||
};
|
||||
|
||||
cluster_ip = mkOption {
|
||||
description = "IP address of this node on the Wesher mesh network";
|
||||
type = types.str;
|
||||
};
|
||||
is_raft_server = mkOption {
|
||||
description = "Make this node a RAFT server for the Nomad and Consul deployments";
|
||||
|
|
@ -83,14 +66,6 @@ in
|
|||
description = "Name of this Deuxfleurs deployment";
|
||||
type = types.str;
|
||||
};
|
||||
cluster_nodes = mkOption {
|
||||
description = "Nodes that are part of the cluster";
|
||||
type = types.listOf wg_node;
|
||||
};
|
||||
admin_nodes = mkOption {
|
||||
description = "Machines that are part of the Wireguard VPN for administration purposes";
|
||||
type = types.listOf wg_node;
|
||||
};
|
||||
admin_accounts = mkOption {
|
||||
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
|
||||
type = types.attrsOf (types.listOf types.str);
|
||||
|
|
@ -114,32 +89,31 @@ in
|
|||
prefixLength = cfg.lan_ip_prefix_length;
|
||||
}
|
||||
];
|
||||
ipv6.addresses = [
|
||||
{
|
||||
address = cfg.ipv6;
|
||||
prefixLength = cfg.ipv6_prefix_length;
|
||||
}
|
||||
];
|
||||
};
|
||||
networking.defaultGateway = {
|
||||
address = cfg.lan_default_gateway;
|
||||
interface = cfg.network_interface;
|
||||
};
|
||||
|
||||
# Configure Wireguard VPN between all nodes
|
||||
networking.wireguard.interfaces.wg0 = {
|
||||
ips = [ "${cfg.vpn_ip}/16" ];
|
||||
listenPort = cfg.vpn_listen_port;
|
||||
privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
|
||||
peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
|
||||
publicKey = publicKey;
|
||||
allowedIPs = [ "${IP}/32" ];
|
||||
endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
|
||||
then lan_endpoint else endpoint;
|
||||
persistentKeepalive = 25;
|
||||
}) (cfg.cluster_nodes ++ cfg.admin_nodes);
|
||||
# wesher overlay network
|
||||
services.wesher = {
|
||||
enable = true;
|
||||
bindAddr = cfg.ipv6;
|
||||
overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
|
||||
interface = "wg0";
|
||||
logLevel = "debug";
|
||||
};
|
||||
|
||||
networking.firewall.allowedUDPPorts = [ cfg.vpn_listen_port ];
|
||||
|
||||
# Configure /etc/hosts to link all hostnames to their Wireguard IP
|
||||
networking.extraHosts = builtins.concatStringsSep "\n" (map
|
||||
({ hostname, IP, ...}: "${IP} ${hostname}")
|
||||
(cfg.cluster_nodes ++ cfg.admin_nodes));
|
||||
#networking.extraHosts = builtins.concatStringsSep "\n" (map
|
||||
# ({ hostname, IP, ...}: "${IP} ${hostname}")
|
||||
# (cfg.cluster_nodes ++ cfg.admin_nodes));
|
||||
|
||||
# Enable Hashicorp Consul & Nomad
|
||||
services.consul.enable = true;
|
||||
|
|
@ -156,14 +130,12 @@ in
|
|||
"site" = cfg.site_name;
|
||||
};
|
||||
ui = true;
|
||||
bind_addr = cfg.vpn_ip;
|
||||
bind_addr = "${cfg.cluster_ip}";
|
||||
|
||||
ports.http = -1;
|
||||
addresses.https = "0.0.0.0";
|
||||
ports.https = 8501;
|
||||
|
||||
retry_join = map (node_info: node_info.IP) cfg.cluster_nodes;
|
||||
|
||||
ca_file = "/var/lib/consul/pki/consul-ca.crt";
|
||||
cert_file = "/var/lib/consul/pki/consul2022.crt";
|
||||
key_file = "/var/lib/consul/pki/consul2022.key";
|
||||
|
|
@ -185,9 +157,9 @@ in
|
|||
region = cfg.cluster_name;
|
||||
datacenter = cfg.site_name;
|
||||
advertise = {
|
||||
rpc = cfg.vpn_ip;
|
||||
http = cfg.vpn_ip;
|
||||
serf = cfg.vpn_ip;
|
||||
rpc = "${cfg.cluster_ip}";
|
||||
http = "${cfg.cluster_ip}";
|
||||
serf = "${cfg.cluster_ip}";
|
||||
};
|
||||
consul = {
|
||||
address = "localhost:8501";
|
||||
|
|
@ -227,5 +199,36 @@ in
|
|||
}
|
||||
];
|
||||
};
|
||||
|
||||
# ---- Firewall config ----
|
||||
|
||||
# Open ports in the firewall.
|
||||
networking.firewall = {
|
||||
enable = true;
|
||||
|
||||
# Allow anyone to connect on SSH port
|
||||
allowedTCPPorts = [
|
||||
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
|
||||
];
|
||||
|
||||
# Allow specific hosts access to specific things in the cluster
|
||||
extraCommands = ''
|
||||
# Allow everything from router (usefull for UPnP/IGD)
|
||||
iptables -A INPUT -s 192.168.1.254 -j ACCEPT
|
||||
|
||||
# Allow docker containers to access all ports
|
||||
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
|
||||
|
||||
# Allow other nodes on VPN to access all ports
|
||||
iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
|
||||
'';
|
||||
|
||||
# When stopping firewall, delete all rules that were configured manually above
|
||||
extraStopCommands = ''
|
||||
iptables -D INPUT -s 192.168.1.254 -j ACCEPT
|
||||
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
|
||||
iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue