Deuxfleurs/nixcfg
17
6
Fork 2
Code Issues 15 Pull requests 2 Projects Releases Wiki Activity

Simplify network configuration #11

Merged
lx merged 25 commits from simplify-network-config into main 2023-05-16 13:19:33 +00:00
Conversation 0 Commits 25 Files changed 43 +144 -185
12 changed files with 144 additions and 185 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 96566ae523 - Show all commits

70
cluster/staging/cluster.nix
View file

@ -1,49 +1,43 @@
{ config, pkgs, ... } @ args:
{
deuxfleurs.cluster_name = "staging";
deuxfleurs.clusterName = "staging";
# The IP range to use for the Wireguard overlay of this cluster
deuxfleurs.cluster_prefix = "10.14.0.0";
deuxfleurs.cluster_prefix_length = 16;
deuxfleurs.clusterPrefix = "10.14.0.0/16";
deuxfleurs.cluster_nodes = [
{
hostname = "carcajou";
site_name = "neptune";
deuxfleurs.clusterNodes = {
"carcajou" = {
siteName = "neptune";
publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
IP = "10.14.1.2";
address = "10.14.1.2";
endpoint = "77.207.15.215:33722";
}
{
hostname = "caribou";
site_name = "neptune";
};
"caribou" = {
siteName = "neptune";
publicKey = "lABn/axzD1jkFulX8c+K3B3CbKXORlIMDDoe8sQVxhs=";
IP = "10.14.1.3";
address = "10.14.1.3";
endpoint = "77.207.15.215:33723";
}
{
hostname = "origan";
site_name = "jupiter";
};
"origan" = {
siteName = "jupiter";
publicKey = "smBQYUS60JDkNoqkTT7TgbpqFiM43005fcrT6472llI=";
IP = "10.14.2.33";
address = "10.14.2.33";
endpoint = "82.64.238.84:33733";
}
{
hostname = "piranha";
site_name = "corrin";
};
"piranha" = {
siteName = "corrin";
publicKey = "m9rLf+233X1VColmeVrM/xfDGro5W6Gk5N0zqcf32WY=";
IP = "10.14.3.1";
address = "10.14.3.1";
#endpoint = "82.120.233.78:33721";
}
{
hostname = "df-pw5";
site_name = "bespin";
};
"df-pw5" = {
siteName = "bespin";
publicKey = "XLOYoMXF+PO4jcgfSVAk+thh4VmWx0wzWnb0xs08G1s=";
IP = "10.14.4.1";
address = "10.14.4.1";
endpoint = "bitfrost.fiber.shirokumo.net:33734";
}
];
};
};
services.wgautomesh.logLevel = "debug";
# Bootstrap IPs for Consul cluster,
@ -54,7 +48,7 @@
"10.14.1.3" # caribou
];
deuxfleurs.admin_accounts = {
deuxfleurs.adminAccounts = {
lx = [
# Keys for accessing nodes from outside
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy"
@ -142,16 +136,16 @@
enable = true;
port = substituter_port;
openFirewall = false;
bindAddress = config.deuxfleurs.cluster_ip;
bindAddress = "0.0.0.0";
package = pkgs.haskellPackages.nix-serve-ng;
};
nix.settings.substituters = map
({ IP, ... }: "http://${IP}:${builtins.toString substituter_port}")
(builtins.filter
({ site_name, IP, ...}:
(IP != config.deuxfleurs.cluster_ip
&& site_name == config.deuxfleurs.site_name))
config.deuxfleurs.cluster_nodes);
({ address, ... }: "http://${address}:${builtins.toString substituter_port}")
(builtins.attrValues (pkgs.lib.filterAttrs
(hostname: { siteName, ...}:
(hostname != config.deuxfleurs.hostName
&& siteName == config.deuxfleurs.siteName))
config.deuxfleurs.clusterNodes));
})
];
}

7
cluster/staging/node/carcajou.nix
View file

@ -19,11 +19,8 @@
boot.loader.timeout = 20;
boot.loader.efi.canTouchEfiVariables = true;
networking.hostName = "carcajou";
deuxfleurs.ipv6 = "2001:910:1204:1::22";
deuxfleurs.cluster_ip = "10.14.1.2";
deuxfleurs.hostName = "carcajou";
deuxfleurs.ipv6Address = "2001:910:1204:1::22";
system.stateVersion = "21.05";
}

9
cluster/staging/node/caribou.nix
View file

@ -8,12 +8,9 @@
boot.loader.timeout = 20;
boot.loader.efi.canTouchEfiVariables = true;
networking.hostName = "caribou";
deuxfleurs.ipv6 = "2001:910:1204:1::23";
deuxfleurs.cluster_ip = "10.14.1.3";
deuxfleurs.is_raft_server = true;
deuxfleurs.hostName = "caribou";
deuxfleurs.ipv6Address = "2001:910:1204:1::23";
deuxfleurs.isRaftServer = true;
system.stateVersion = "21.05";
}

8
cluster/staging/node/df-pw5.nix
View file

@ -9,13 +9,9 @@
boot.loader.efi.efiSysMountPoint = "/boot";
boot.loader.timeout = 20;
networking.hostName = "df-pw5";
deuxfleurs.hostName = "df-pw5";
deuxfleurs.staticIPv4.address = "192.168.5.130/24";
deuxfleurs.ipv6 = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7";
deuxfleurs.cluster_ip = "10.14.4.1";
deuxfleurs.is_raft_server = false;
deuxfleurs.ipv6Address = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7";
system.stateVersion = "22.11";
}

9
cluster/staging/node/origan.nix
View file

@ -8,13 +8,10 @@
boot.loader.timeout = 20;
boot.loader.efi.canTouchEfiVariables = true;
networking.hostName = "origan";
deuxfleurs.hostName = "origan";
deuxfleurs.staticIPv4.address = "192.168.1.33/24";
deuxfleurs.ipv6 = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec";
deuxfleurs.cluster_ip = "10.14.2.33";
deuxfleurs.is_raft_server = true;
deuxfleurs.ipv6Address = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec";
deuxfleurs.isRaftServer = true;
system.stateVersion = "22.11";
}

9
cluster/staging/node/piranha.nix
View file

@ -8,13 +8,10 @@
boot.loader.timeout = 20;
boot.loader.efi.canTouchEfiVariables = true;
networking.hostName = "piranha";
deuxfleurs.hostName = "piranha";
deuxfleurs.staticIPv4.address = "192.168.1.25/24";
deuxfleurs.ipv6 = "2a01:cb05:8984:9900:223:24ff:feb0:ea82";
deuxfleurs.cluster_ip = "10.14.3.1";
deuxfleurs.is_raft_server = true;
deuxfleurs.ipv6Address = "2a01:cb05:8984:9900:223:24ff:feb0:ea82";
deuxfleurs.isRaftServer = true;
system.stateVersion = "22.11";
}

4
cluster/staging/site/bespin.nix
View file

@ -1,9 +1,9 @@
{ config, pkgs, ... }:
{
deuxfleurs.site_name = "bespin";
deuxfleurs.siteName = "bespin";
deuxfleurs.staticIPv4.defaultGateway = "192.168.5.254";
deuxfleurs.cname_target = "bespin.site.staging.deuxfleurs.org.";
deuxfleurs.cnameTarget = "bespin.site.staging.deuxfleurs.org.";
networking.firewall.allowedTCPPorts = [ 80 443 ];
}

6
cluster/staging/site/corrin.nix
View file

@ -1,10 +1,10 @@
{ config, pkgs, ... }:
{
deuxfleurs.site_name = "corrin";
deuxfleurs.siteName = "corrin";
deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1";
deuxfleurs.cname_target = "corrin.site.staging.deuxfleurs.org.";
deuxfleurs.public_ipv4 = "82.120.233.78";
deuxfleurs.cnameTarget = "corrin.site.staging.deuxfleurs.org.";
deuxfleurs.publicIPv4 = "82.120.233.78";
networking.firewall.allowedTCPPorts = [ 80 443 ];
}

7
cluster/staging/site/jupiter.nix
View file

@ -1,12 +1,9 @@
{ config, pkgs, ... }:
{
deuxfleurs.site_name = "jupiter";
deuxfleurs.siteName = "jupiter";
deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1";
deuxfleurs.cname_target = "jupiter.site.staging.deuxfleurs.org.";
# no public ipv4 is used for the staging cluster on Jupiter
# deuxfleurs.public_ipv4 = "???";
deuxfleurs.cnameTarget = "jupiter.site.staging.deuxfleurs.org.";
networking.firewall.allowedTCPPorts = [ 80 443 ];
}

8
cluster/staging/site/neptune.nix
View file

@ -1,12 +1,8 @@
{ config, pkgs, ... }:
{
deuxfleurs.site_name = "neptune";
deuxfleurs.cname_target = "neptune.site.staging.deuxfleurs.org.";
# no public ipv4 is used for the staging cluster on Neptune,
# because the Internet connection is already used for the prod cluster
# deuxfleurs.public_ipv4 = "77.207.15.215";
deuxfleurs.siteName = "neptune";
deuxfleurs.cnameTarget = "neptune.site.staging.deuxfleurs.org.";
networking.firewall.allowedTCPPorts = [ 80 443 ];
}

190
nix/deuxfleurs.nix
View file

@ -6,110 +6,98 @@ in
with builtins;
with pkgs.lib;
{
options.deuxfleurs =
let wg_node = with types; submodule {
options = {
hostname = mkOption {
type = str;
description = "Host name";
};
site_name = mkOption {
type = nullOr str;
description = "Site where the node is located";
default = null;
};
IP = mkOption {
type = str;
description = "IP Address in the Wireguard network";
};
publicKey = mkOption {
type = str;
description = "Public key";
};
endpoint = mkOption {
type = nullOr str;
default = null;
description = "Wireguard endpoint on the public Internet";
};
};
};
in
{
options.deuxfleurs = with types; {
# Parameters for individual nodes
ipv6 = mkOption {
hostName = mkOption {
description = "Node name";
type = str;
};
ipv6Address = mkOption {
description = "Static public IPv6 address of this node";
type = types.str;
type = str;
};
staticIPv4.address = mkOption {
description = "IP address (with prefix length) of this node on the local network interface";
type = types.nullOr types.str;
type = nullOr str;
default = null;
};
cluster_ip = mkOption {
description = "IP address of this node on the Wesher mesh network";
type = types.str;
};
is_raft_server = mkOption {
isRaftServer = mkOption {
description = "Make this node a RAFT server for the Nomad and Consul deployments";
type = types.bool;
type = bool;
default = false;
};
# Parameters that generally vary between sites
site_name = mkOption {
siteName = mkOption {
description = "Site (availability zone) on which this node is deployed";
type = types.str;
type = str;
};
staticIPv4.defaultGateway = mkOption {
description = "IPv4 address of the default route on the local network interface";
type = types.nullOr types.str;
type = nullOr str;
default = null;
};
public_ipv4 = mkOption {
publicIPv4 = mkOption {
description = "Public IPv4 through which this node is accessible (possibly after port opening using DiploNAT), for domain names that are updated by D53";
type = types.nullOr types.str;
type = nullOr str;
default = null;
};
cname_target = mkOption {
cnameTarget = mkOption {
description = "DNS CNAME target to use for services hosted in this site, for domain names that are updated by D53";
type = types.nullOr types.str;
type = nullOr str;
default = null;
};
# Parameters common to all nodes
cluster_name = mkOption {
clusterName = mkOption {
description = "Name of this Deuxfleurs deployment";
type = types.str;
type = str;
};
cluster_prefix = mkOption {
description = "IP address prefix for the Wireguard overlay network";
type = types.str;
clusterPrefix = mkOption {
description = "IP address prefix (and length) for the Wireguard overlay network";
type = str;
};
cluster_prefix_length = mkOption {
description = "IP address prefix length for the Wireguard overlay network";
type = types.int;
default = 16;
};
cluster_nodes = mkOption {
clusterNodes = mkOption {
description = "Nodes that are part of the cluster";
type = types.listOf wg_node;
type = attrsOf (submodule {
options = {
siteName = mkOption {
type = nullOr str;
description = "Site where the node is located";
default = null;
};
address = mkOption {
type = str;
description = "IP Address in the Wireguard network";
};
publicKey = mkOption {
type = str;
description = "Public key";
};
endpoint = mkOption {
type = nullOr str;
default = null;
description = "Wireguard endpoint on the public Internet";
};
};
});
};
admin_accounts = mkOption {
adminAccounts = mkOption {
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
type = types.attrsOf (types.listOf types.str);
type = attrsOf (listOf str);
};
bootstrap = mkOption {
description = "Whether to enable bootstrapping for Nomad and Consul";
type = types.bool;
type = bool;
default = false;
};
# Options that generally stay to their default value
wireguardPort = mkOption {
description = "Port for incoming Wireguard VPN connections";
type = types.port;
type = port;
default = 33799;
};
};
@ -119,25 +107,29 @@ in
];
config =
let node_meta = {
"site" = cfg.site_name;
"public_ipv6" = cfg.ipv6;
let
clusterNodeCfg = getAttr cfg.hostName cfg.clusterNodes;
clusterAddress = clusterNodeCfg.address;
node_meta = {
"site" = cfg.siteName;
"public_ipv6" = cfg.ipv6Address;
} //
(if cfg.public_ipv4 != null
then { "public_ipv4" = cfg.public_ipv4; }
(if cfg.publicIPv4 != null
then { "public_ipv4" = cfg.publicIPv4; }
else {}) //
(if cfg.cname_target != null
then { "cname_target" = cfg.cname_target; }
(if cfg.cnameTarget != null
then { "cname_target" = cfg.cnameTarget; }
else {});
in
{
networking.hostName = cfg.hostName;
# Configure admin accounts on all nodes
users.users = builtins.mapAttrs (name: publicKeys: {
users.users = mapAttrs (name: publicKeys: {
isNormalUser = true;
extraGroups = [ "wheel" ];
openssh.authorizedKeys.keys = publicKeys;
}) cfg.admin_accounts;
}) cfg.adminAccounts;
# Configure network interfaces
networking.useDHCP = false;
@ -148,7 +140,7 @@ in
Name = "en* eth*";
};
ipv6AcceptRAConfig = {
Token = "static:${cfg.ipv6}";
Token = "static:${cfg.ipv6Address}";
UseDNS = false;
};
} // (if cfg.staticIPv4.address == null || cfg.staticIPv4.defaultGateway == null then {
@ -195,7 +187,7 @@ in
# Forward .consul queries to Consul daemon
{
name = "consul.";
stub-addr = "${cfg.cluster_ip}@8600";
stub-addr = "${clusterAddress}@8600";
stub-no-cache = true;
stub-tcp-upstream = false;
stub-tls-upstream = false;
@ -208,7 +200,7 @@ in
# Configure Wireguard VPN between all nodes
networking.wireguard.interfaces.wg0 = {
ips = [ "${cfg.cluster_ip}/16" ];
ips = [ "${clusterAddress}/16" ];
listenPort = cfg.wireguardPort;
privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
mtu = 1420;
@ -220,28 +212,24 @@ in
gossipSecretFile = "/var/lib/wgautomesh/gossip_secret";
persistFile = "/var/lib/wgautomesh/state";
upnpForwardPublicPort =
let
us = filter ({ hostname, ...}: hostname == config.networking.hostName) cfg.cluster_nodes;
in
if length us > 0 && (head us).endpoint != null then
strings.toInt (lists.last (split ":" (head us).endpoint))
if clusterNodeCfg.endpoint != null then
strings.toInt (lists.last (split ":" clusterNodeCfg.endpoint))
else null;
peers = map ({ publicKey, endpoint, IP, ... }: {
address = IP;
peers = attrValues (mapAttrs (hostname: { publicKey, endpoint, address, ... }: {
inherit address endpoint;
pubkey = publicKey;
endpoint = endpoint;
}) cfg.cluster_nodes;
}) cfg.clusterNodes);
};
# Old code for wg-quick, we can use this as a fallback if we fail to make wgautomesh work
# systemd.services."wg-quick-wg0".after = [ "unbound.service" ];
# networking.wg-quick.interfaces.wg0 = {
# address = [ "${cfg.cluster_ip}/16" ];
# address = [ "${clusterAddress}/16" ];
# listenPort = cfg.wireguardPort;
# privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
# mtu = 1420;
# peers = map ({ publicKey, endpoint, IP, ... }: {
# peers = map ({ publicKey, endpoint, address, ... }: {
# inherit publicKey endpoint;
# allowedIPs = [ "${IP}/32" ];
# allowedIPs = [ "${address}/32" ];
# persistentKeepalive = 25;
# };
@ -255,25 +243,25 @@ in
'';
# Configure /etc/hosts to link all hostnames to their Wireguard IP
networking.extraHosts = builtins.concatStringsSep "\n" (map
({ hostname, IP, ...}: "${IP} ${hostname}")
cfg.cluster_nodes);
networking.extraHosts = concatStringsSep "\n" (attrValues (mapAttrs
(hostname: { address, ...}: "${address} ${hostname}")
cfg.clusterNodes));
# Enable Hashicorp Consul & Nomad
services.consul.enable = true;
systemd.services.consul.after = [ "wg-quick-wg0.service" ];
services.consul.extraConfig =
(if cfg.is_raft_server
(if cfg.isRaftServer
then { server = true; }
// (if cfg.bootstrap then { bootstrap_expect = 3; } else {})
else {}) //
{
inherit node_meta;
datacenter = cfg.cluster_name;
datacenter = cfg.clusterName;
ui_config = {
enabled = true;
};
bind_addr = "${cfg.cluster_ip}";
bind_addr = "${clusterAddress}";
addresses = {
https = "0.0.0.0";
@ -303,18 +291,18 @@ in
pkgs.zstd
];
services.nomad.settings =
(if cfg.is_raft_server
(if cfg.isRaftServer
then {
server = { enabled = true; }
// (if cfg.bootstrap then { bootstrap_expect = 3; } else {});
} else {}) //
{
region = cfg.cluster_name;
datacenter = cfg.site_name;
region = cfg.clusterName;
datacenter = cfg.siteName;
advertise = {
rpc = "${cfg.cluster_ip}";
http = "${cfg.cluster_ip}";
serf = "${cfg.cluster_ip}";
rpc = "${clusterAddress}";
http = "${clusterAddress}";
serf = "${clusterAddress}";
};
consul = {
address = "localhost:8501";
@ -367,7 +355,7 @@ in
allowedTCPPorts = [
# Allow anyone to connect on SSH port
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
(head ({ openssh.ports = [22]; } // config.services).openssh.ports)
];
allowedUDPPorts = [
@ -385,14 +373,14 @@ in
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
# Allow other nodes on VPN to access all ports
iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
iptables -A INPUT -s ${cfg.clusterPrefix} -j ACCEPT
'';
# When stopping firewall, delete all rules that were configured manually above
extraStopCommands = ''
iptables -D INPUT -s 192.168.0.0/16 -p udp --sport 1900 -j ACCEPT
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
iptables -D INPUT -s ${cfg.clusterPrefix} -j ACCEPT
'';
};
};

2
nix/remote-unlock.nix
View file

@ -28,7 +28,7 @@ in
boot.initrd.network.ssh = {
enable = true;
port = 222;
authorizedKeys = concatLists (mapAttrsToList (name: user: user) config.deuxfleurs.admin_accounts);
authorizedKeys = concatLists (mapAttrsToList (name: user: user) config.deuxfleurs.adminAccounts);
hostKeys = [ "/var/lib/deuxfleurs/remote-unlock/ssh_host_ed25519_key" ];
};
boot.initrd.network.postCommands = ''