Merge pull request 'Passer wgautomesh en prod' (#9) from wgautomesh into main

Reviewed-on: Deuxfleurs/nixcfg#9
This commit is contained in:
Alex 2023-03-24 11:05:29 +00:00
commit 76c8e8f0b0
9 changed files with 185 additions and 32 deletions

View file

@ -13,7 +13,6 @@
site_name = "neptune"; site_name = "neptune";
publicKey = "VvXT0fPDfWsHxumZqVShpS33dJQAdpJ1E79ZbCBJP34="; publicKey = "VvXT0fPDfWsHxumZqVShpS33dJQAdpJ1E79ZbCBJP34=";
IP = "10.83.1.1"; IP = "10.83.1.1";
lan_endpoint = "192.168.1.31:33799";
endpoint = "77.207.15.215:33731"; endpoint = "77.207.15.215:33731";
} }
{ {
@ -21,7 +20,6 @@
site_name = "neptune"; site_name = "neptune";
publicKey = "goTkBJGmzrGDOAjUcdH9G0JekipqSMoaYQdB6IHnzi0="; publicKey = "goTkBJGmzrGDOAjUcdH9G0JekipqSMoaYQdB6IHnzi0=";
IP = "10.83.1.2"; IP = "10.83.1.2";
lan_endpoint = "192.168.1.32:33799";
endpoint = "77.207.15.215:33732"; endpoint = "77.207.15.215:33732";
} }
{ {
@ -29,7 +27,6 @@
site_name = "neptune"; site_name = "neptune";
publicKey = "oZDAb8LoLW87ktUHyFFec0VaIar97bqq47mGbdVqJ0U="; publicKey = "oZDAb8LoLW87ktUHyFFec0VaIar97bqq47mGbdVqJ0U=";
IP = "10.83.1.3"; IP = "10.83.1.3";
lan_endpoint = "192.168.1.33:33799";
endpoint = "77.207.15.215:33733"; endpoint = "77.207.15.215:33733";
} }
{ {
@ -37,7 +34,6 @@
site_name = "orion"; site_name = "orion";
publicKey = "EtRoWBYCdjqgXX0L+uWLg8KxNfIK8k9OTh30tL19bXU="; publicKey = "EtRoWBYCdjqgXX0L+uWLg8KxNfIK8k9OTh30tL19bXU=";
IP = "10.83.2.1"; IP = "10.83.2.1";
lan_endpoint = "192.168.1.11:33799";
endpoint = "82.66.80.201:33731"; endpoint = "82.66.80.201:33731";
} }
{ {
@ -45,7 +41,6 @@
site_name = "orion"; site_name = "orion";
publicKey = "HbLC938mysadMSOxWgq8+qrv+dBKzPP/43OMJp/3phA="; publicKey = "HbLC938mysadMSOxWgq8+qrv+dBKzPP/43OMJp/3phA=";
IP = "10.83.2.2"; IP = "10.83.2.2";
lan_endpoint = "192.168.1.12:33799";
endpoint = "82.66.80.201:33732"; endpoint = "82.66.80.201:33732";
} }
{ {
@ -53,7 +48,6 @@
site_name = "orion"; site_name = "orion";
publicKey = "e1C8jgTj9eD20ywG08G1FQZ+Js3wMK/msDUE1wO3l1Y="; publicKey = "e1C8jgTj9eD20ywG08G1FQZ+Js3wMK/msDUE1wO3l1Y=";
IP = "10.83.2.3"; IP = "10.83.2.3";
lan_endpoint = "192.168.1.13:33799";
endpoint = "82.66.80.201:33733"; endpoint = "82.66.80.201:33733";
} }
{ {
@ -61,7 +55,6 @@
site_name = "bespin"; site_name = "bespin";
publicKey = "bIjxey/VhBgVrLa0FxN/KISOt2XFmQeSh1MPivUq9gg="; publicKey = "bIjxey/VhBgVrLa0FxN/KISOt2XFmQeSh1MPivUq9gg=";
IP = "10.83.3.1"; IP = "10.83.3.1";
lan_endpoint = "192.168.5.117:33799";
endpoint = "109.136.55.235:33731"; endpoint = "109.136.55.235:33731";
} }
{ {
@ -69,7 +62,6 @@
site_name = "bespin"; site_name = "bespin";
publicKey = "pUIKv8UBl586O7DBrHBsb9BgNU7WlYQ2r2RSNkD+JAQ="; publicKey = "pUIKv8UBl586O7DBrHBsb9BgNU7WlYQ2r2RSNkD+JAQ=";
IP = "10.83.3.2"; IP = "10.83.3.2";
lan_endpoint = "192.168.5.134:33799";
endpoint = "109.136.55.235:33732"; endpoint = "109.136.55.235:33732";
} }
{ {
@ -77,7 +69,6 @@
site_name = "bespin"; site_name = "bespin";
publicKey = "VBmpo15iIJP7250NAsF+ryhZc3j+8TZFnE1Djvn5TXI="; publicKey = "VBmpo15iIJP7250NAsF+ryhZc3j+8TZFnE1Djvn5TXI=";
IP = "10.83.3.3"; IP = "10.83.3.3";
lan_endpoint = "192.168.5.116:33799";
endpoint = "109.136.55.235:33733"; endpoint = "109.136.55.235:33733";
} }
{ {
@ -85,7 +76,6 @@
site_name = "scorpio"; site_name = "scorpio";
publicKey = "Sm9cmNZ/BfWVPFflMO+fuyiera4r203b/dKhHTQmBFg="; publicKey = "Sm9cmNZ/BfWVPFflMO+fuyiera4r203b/dKhHTQmBFg=";
IP = "10.83.4.1"; IP = "10.83.4.1";
lan_endpoint = "192.168.1.41:33799";
endpoint = "82.65.41.110:33741"; endpoint = "82.65.41.110:33741";
} }
]; ];

View file

@ -13,7 +13,6 @@
site_name = "neptune"; site_name = "neptune";
publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk="; publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
IP = "10.14.1.2"; IP = "10.14.1.2";
lan_endpoint = "192.168.1.22:33799";
endpoint = "77.207.15.215:33722"; endpoint = "77.207.15.215:33722";
} }
{ {
@ -21,7 +20,6 @@
site_name = "neptune"; site_name = "neptune";
publicKey = "lABn/axzD1jkFulX8c+K3B3CbKXORlIMDDoe8sQVxhs="; publicKey = "lABn/axzD1jkFulX8c+K3B3CbKXORlIMDDoe8sQVxhs=";
IP = "10.14.1.3"; IP = "10.14.1.3";
lan_endpoint = "192.168.1.23:33799";
endpoint = "77.207.15.215:33723"; endpoint = "77.207.15.215:33723";
} }
{ {
@ -29,7 +27,6 @@
site_name = "jupiter"; site_name = "jupiter";
publicKey = "smBQYUS60JDkNoqkTT7TgbpqFiM43005fcrT6472llI="; publicKey = "smBQYUS60JDkNoqkTT7TgbpqFiM43005fcrT6472llI=";
IP = "10.14.2.33"; IP = "10.14.2.33";
lan_endpoint = "192.168.1.33:33799";
endpoint = "82.64.238.84:33733"; endpoint = "82.64.238.84:33733";
} }
{ {
@ -37,18 +34,17 @@
site_name = "corrin"; site_name = "corrin";
publicKey = "m9rLf+233X1VColmeVrM/xfDGro5W6Gk5N0zqcf32WY="; publicKey = "m9rLf+233X1VColmeVrM/xfDGro5W6Gk5N0zqcf32WY=";
IP = "10.14.3.1"; IP = "10.14.3.1";
lan_endpoint = "192.168.1.25:33799"; #endpoint = "82.120.233.78:33721";
endpoint = "82.120.233.78:33721";
} }
{ {
hostname = "df-pw5"; hostname = "df-pw5";
site_name = "bespin"; site_name = "bespin";
publicKey = "XLOYoMXF+PO4jcgfSVAk+thh4VmWx0wzWnb0xs08G1s="; publicKey = "XLOYoMXF+PO4jcgfSVAk+thh4VmWx0wzWnb0xs08G1s=";
IP = "10.14.4.1"; IP = "10.14.4.1";
lan_endpoint = "192.168.5.130:33799";
endpoint = "bitfrost.fiber.shirokumo.net:33734"; endpoint = "bitfrost.fiber.shirokumo.net:33734";
} }
]; ];
services.wgautomesh.logLevel = "debug";
# Bootstrap IPs for Consul cluster, # Bootstrap IPs for Consul cluster,
# these are IPs on the Wireguard overlay # these are IPs on the Wireguard overlay

View file

@ -8,3 +8,4 @@ piranha.polyno.me,2a01:cb05:8984:3c00:223:24ff:feb0:ea82 ssh-ed25519 AAAAC3NzaC1
2001:910:1204:1::21 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPXTUrXRFhudJBESCqjHCOttzqYPyIzpPOMkI8+SwLRx 2001:910:1204:1::21 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPXTUrXRFhudJBESCqjHCOttzqYPyIzpPOMkI8+SwLRx
2a01:e0a:5e4:1d0:223:24ff:feaf:fdec ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsZas74RT6lCZwuUOPR23nPdbSdpWORyAmRgjoiMVHK 2a01:e0a:5e4:1d0:223:24ff:feaf:fdec ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsZas74RT6lCZwuUOPR23nPdbSdpWORyAmRgjoiMVHK
df-pw5.machine.deuxfleurs.fr ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK/dJIxioCkfeehxeGiZR7qquYGoqEH/YrRJ/ukEcaLH df-pw5.machine.deuxfleurs.fr ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK/dJIxioCkfeehxeGiZR7qquYGoqEH/YrRJ/ukEcaLH
10.14.3.1 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJnpO6zpLWsyyugOoOj+2bUow9TUrcWgURFGGaoyu+co

View file

@ -10,7 +10,9 @@ Host origan
HostName origan.df.trinity.fr.eu.org HostName origan.df.trinity.fr.eu.org
Host piranha Host piranha
HostName piranha.polyno.me ProxyJump caribou.machine.deuxfleurs.fr
HostName 10.14.3.1
#HostName piranha.polyno.me
Host df-pw5 Host df-pw5
HostName df-pw5.machine.deuxfleurs.fr HostName df-pw5.machine.deuxfleurs.fr

View file

@ -7,6 +7,10 @@ copy cluster/$CLUSTER/cluster.nix /etc/nixos/cluster.nix
copy cluster/$CLUSTER/node/$NIXHOST.nix /etc/nixos/node.nix copy cluster/$CLUSTER/node/$NIXHOST.nix /etc/nixos/node.nix
copy cluster/$CLUSTER/node/$NIXHOST.site.nix /etc/nixos/site.nix copy cluster/$CLUSTER/node/$NIXHOST.site.nix /etc/nixos/site.nix
cmd mkdir -p /var/lib/wgautomesh
write_pass deuxfleurs/cluster/$CLUSTER/wgautomesh_gossip_secret /var/lib/wgautomesh/gossip_secret
copy nix/wgautomesh.nix /etc/nixos/wgautomesh.nix
if [ "$CLUSTER" = "staging" ]; then if [ "$CLUSTER" = "staging" ]; then
copy nix/nomad-driver-nix2.nix /etc/nixos/nomad-driver-nix2.nix copy nix/nomad-driver-nix2.nix /etc/nixos/nomad-driver-nix2.nix
fi fi
@ -14,6 +18,9 @@ fi
if [ "$CLUSTER" = "prod" ]; then if [ "$CLUSTER" = "prod" ]; then
cmd nixos-rebuild boot cmd nixos-rebuild boot
message "-------------------------------------------------------------------------------------" message "-------------------------------------------------------------------------------------"
message "NIXOS CHANGES THAT WOULD BE APPLIED UPON SWITCH OR REBOOT:"
cmd nixos-rebuild dry-activate
message "-------------------------------------------------------------------------------------"
message "New NixOS configuration hasn't been applied, to avoid disturbing production services." message "New NixOS configuration hasn't been applied, to avoid disturbing production services."
message "Please apply the following procedure to node '$NIXHOST':" message "Please apply the following procedure to node '$NIXHOST':"
message "1. Drain node in Nomad so that all jobs are relocated elsewhere" message "1. Drain node in Nomad so that all jobs are relocated elsewhere"

View file

@ -16,6 +16,7 @@ ports so that we can avoid conflicts when adding services.
587 postfix submission 587 postfix submission
993 dovecot imaps 993 dovecot imaps
1337 dovecot zauthoentication 1337 dovecot zauthoentication
1666 wgautomesh
1728 nix-serve 1728 nix-serve
3719 grafana 3719 grafana
3900 garage S3 (prod) 3900 garage S3 (prod)

View file

@ -84,5 +84,8 @@ SystemMaxUse=1G
dns = [ "172.17.0.1" ]; dns = [ "172.17.0.1" ];
})}"; })}";
}; };
nix.gc.automatic = true;
nix.gc.options = "--delete-older-than 30d";
} }

View file

@ -28,12 +28,8 @@ in
}; };
endpoint = mkOption { endpoint = mkOption {
type = nullOr str; type = nullOr str;
description = "Wireguard endpoint on the public Internet";
};
lan_endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint for nodes in the same site";
default = null; default = null;
description = "Wireguard endpoint on the public Internet";
}; };
}; };
}; };
@ -134,6 +130,10 @@ in
}; };
}; };
imports = [
./wgautomesh.nix
];
config = config =
let node_meta = { let node_meta = {
"site" = cfg.site_name; "site" = cfg.site_name;
@ -147,6 +147,7 @@ in
else {}); else {});
in in
{ {
# Configure admin accounts on all nodes # Configure admin accounts on all nodes
users.users = builtins.mapAttrs (name: publicKeys: { users.users = builtins.mapAttrs (name: publicKeys: {
isNormalUser = true; isNormalUser = true;
@ -233,20 +234,43 @@ in
services.resolved.enable = false; services.resolved.enable = false;
# Configure Wireguard VPN between all nodes # Configure Wireguard VPN between all nodes
systemd.services."wg-quick-wg0".after = [ "unbound.service" ]; networking.wireguard.interfaces.wg0 = {
networking.wg-quick.interfaces.wg0 = { ips = [ "${cfg.cluster_ip}/16" ];
address = [ "${cfg.cluster_ip}/16" ];
listenPort = cfg.wireguard_port; listenPort = cfg.wireguard_port;
privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
mtu = 1420; mtu = 1420;
peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: { };
publicKey = publicKey; services.wgautomesh = {
allowedIPs = [ "${IP}/32" ]; enable = true;
endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null interface = "wg0";
then lan_endpoint else endpoint; gossipPort = 1666;
persistentKeepalive = 25; gossipSecretFile = "/var/lib/wgautomesh/gossip_secret";
persistFile = "/var/lib/wgautomesh/state";
upnpForwardPublicPort =
let
us = filter ({ hostname, ...}: hostname == config.networking.hostName) cfg.cluster_nodes;
in
if length us > 0 && (head us).endpoint != null then
strings.toInt (lists.last (split ":" (head us).endpoint))
else null;
peers = map ({ publicKey, endpoint, IP, ... }: {
address = IP;
pubkey = publicKey;
endpoint = endpoint;
}) cfg.cluster_nodes; }) cfg.cluster_nodes;
}; };
# Old code for wg-quick, we can use this as a fallback if we fail to make wgautomesh work
# systemd.services."wg-quick-wg0".after = [ "unbound.service" ];
# networking.wg-quick.interfaces.wg0 = {
# address = [ "${cfg.cluster_ip}/16" ];
# listenPort = cfg.wireguard_port;
# privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
# mtu = 1420;
# peers = map ({ publicKey, endpoint, IP, ... }: {
# inherit publicKey endpoint;
# allowedIPs = [ "${IP}/32" ];
# persistentKeepalive = 25;
# };
system.activationScripts.generate_df_wg_key = '' system.activationScripts.generate_df_wg_key = ''
if [ ! -f /var/lib/deuxfleurs/wireguard-keys/private ]; then if [ ! -f /var/lib/deuxfleurs/wireguard-keys/private ]; then
@ -303,7 +327,7 @@ in
services.nomad.enable = true; services.nomad.enable = true;
systemd.services.nomad.after = [ "wg-quick-wg0.service" ]; systemd.services.nomad.after = [ "wg-quick-wg0.service" ];
services.nomad.package = pkgs.nomad_1_3; services.nomad.package = pkgs.nomad_1_4;
services.nomad.extraPackages = [ services.nomad.extraPackages = [
pkgs.glibc pkgs.glibc
pkgs.zstd pkgs.zstd

129
nix/wgautomesh.nix Normal file
View file

@ -0,0 +1,129 @@
let
wgautomesh = builtins.fetchTarball {
url = "https://git.deuxfleurs.fr/attachments/ce203833-1ae7-43d4-9bf4-b49b560c9f4b";
sha256 = "sha256:1kc990s7xkwff53vs6c3slg7ljsyr9xz1i13j61ivfj6djyh8rmj";
};
in
{ lib, config, pkgs, ... }:
with lib;
let
cfg = config.services.wgautomesh;
in
with builtins;
{
options.services.wgautomesh = {
enable = mkEnableOption "wgautomesh";
logLevel = mkOption {
type = types.enum [ "trace" "debug" "info" "warn" "error" ];
default = "info";
description = "wgautomesh log level (trace/debug/info/warn/error)";
};
interface = mkOption {
type = types.str;
description = "Wireguard interface to manage";
};
gossipPort = mkOption {
type = types.port;
description = "wgautomesh gossip port";
};
gossipSecretFile = mkOption {
type = types.nullOr types.str;
description = "File containing the gossip secret encryption key";
};
persistFile = mkOption {
type = types.nullOr types.str;
description = "Path where to persist known peer addresses";
};
lanDiscovery = mkOption {
type = types.bool;
default = true;
description = "Enable discovery using LAN broadcast";
};
openFirewall = mkOption {
type = types.bool;
default = true;
description = "Automatically open gossip port in firewall";
};
upnpForwardPublicPort = mkOption {
type = types.nullOr types.port;
default = null;
description = "Public port number to try to redirect to this machine using UPnP IGD";
};
peers = mkOption {
type = types.listOf (types.submodule {
options = {
pubkey = mkOption {
type = types.str;
description = "Wireguard public key";
};
address = mkOption {
type = types.str;
description = "Wireguard peer address";
};
endpoint = mkOption {
type = types.nullOr types.str;
description = "bootstrap endpoint";
};
};
});
description = "wgautomesh peer list";
};
};
config = mkIf cfg.enable (
let
peerDefs = map (peer:
let endpointDef = if peer.endpoint == null then ""
else ''endpoint = "${peer.endpoint}"'';
in
''
[[peers]]
pubkey = "${peer.pubkey}"
address = "${peer.address}"
${endpointDef}
'') cfg.peers;
extraDefs = (if cfg.lanDiscovery then ["lan_discovery = true"] else [])
++ (if (cfg.gossipSecretFile != null)
then [''gossip_secret_file = "${cfg.gossipSecretFile}"''] else [])
++ (if (cfg.persistFile != null)
then [''persist_file = "${cfg.persistFile}"''] else [])
++ (if (cfg.upnpForwardPublicPort != null)
then [''upnp_forward_external_port = ${toString cfg.upnpForwardPublicPort}''] else []);
configfile = pkgs.writeText "wgautomesh.toml" ''
interface = "${cfg.interface}"
gossip_port = ${toString cfg.gossipPort}
${concatStringsSep "\n" extraDefs}
${concatStringsSep "\n" peerDefs}
'';
in {
systemd.services.wgautomesh = {
enable = true;
path = [ pkgs.wireguard-tools ];
environment = {
RUST_LOG = "wgautomesh=${cfg.logLevel}";
};
description = "wgautomesh";
serviceConfig = {
Type = "simple";
ExecStart = "${wgautomesh}/bin/wgautomesh ${configfile}";
Restart = "always";
RestartSec = "30";
ExecStartPre = [ "+${pkgs.coreutils}/bin/chown wgautomesh /var/lib/wgautomesh/gossip_secret" ];
DynamicUser = true;
User = "wgautomesh";
StateDirectory = "wgautomesh";
StateDirectoryMode = "0700";
AmbientCapabilities = "CAP_NET_ADMIN";
CapabilityBoundingSet = "CAP_NET_ADMIN";
};
wantedBy = [ "multi-user.target" ];
};
networking.firewall.allowedUDPPorts = mkIf cfg.openFirewall [ cfg.gossipPort ];
});
}