From a0db30ca26ee0ca8c8efbabd76ba584331b5337c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 24 Mar 2023 12:58:44 +0100 Subject: [PATCH 01/20] Sanitize DNS configuration - get rid of outside nameserver, unbound does the recursive resolving itself (and it checks DNSSEC) - remove CAP_NET_BIND_SERVICE for Consul as it is no longer binding on port 53 (was already obsolete) - make unbound config independant of LAN IPv4 address --- cluster/staging/site/bespin.nix | 1 - cluster/staging/site/corrin.nix | 1 - cluster/staging/site/jupiter.nix | 1 - cluster/staging/site/neptune.nix | 1 - nix/deuxfleurs.nix | 38 +++++++++----------------------- 5 files changed, 10 insertions(+), 32 deletions(-) diff --git a/cluster/staging/site/bespin.nix b/cluster/staging/site/bespin.nix index 9401f74..1133603 100644 --- a/cluster/staging/site/bespin.nix +++ b/cluster/staging/site/bespin.nix @@ -6,7 +6,6 @@ deuxfleurs.ipv6_default_gateway = "2a02:a03f:6510:5102::1"; deuxfleurs.lan_ip_prefix_length = 24; deuxfleurs.ipv6_prefix_length = 64; - deuxfleurs.nameservers = [ "192.168.5.254" ]; deuxfleurs.cname_target = "bespin.site.staging.deuxfleurs.org."; networking.firewall.allowedTCPPorts = [ 80 443 ]; diff --git a/cluster/staging/site/corrin.nix b/cluster/staging/site/corrin.nix index ca2ae49..8bf8693 100644 --- a/cluster/staging/site/corrin.nix +++ b/cluster/staging/site/corrin.nix @@ -6,7 +6,6 @@ deuxfleurs.ipv6_default_gateway = "fe80::7ec1:77ff:fe3e:bb90"; deuxfleurs.lan_ip_prefix_length = 24; deuxfleurs.ipv6_prefix_length = 64; - deuxfleurs.nameservers = [ "192.168.1.1" ]; deuxfleurs.cname_target = "corrin.site.staging.deuxfleurs.org."; deuxfleurs.public_ipv4 = "82.120.233.78"; diff --git a/cluster/staging/site/jupiter.nix b/cluster/staging/site/jupiter.nix index 31b9f47..291e582 100644 --- a/cluster/staging/site/jupiter.nix +++ b/cluster/staging/site/jupiter.nix @@ -6,7 +6,6 @@ deuxfleurs.ipv6_default_gateway = "fe80::9038:202a:73a0:e73b"; deuxfleurs.lan_ip_prefix_length = 24; deuxfleurs.ipv6_prefix_length = 64; - deuxfleurs.nameservers = [ "192.168.1.1" ]; deuxfleurs.cname_target = "jupiter.site.staging.deuxfleurs.org."; # no public ipv4 is used for the staging cluster on Jupiter diff --git a/cluster/staging/site/neptune.nix b/cluster/staging/site/neptune.nix index 5399826..b030b46 100644 --- a/cluster/staging/site/neptune.nix +++ b/cluster/staging/site/neptune.nix @@ -6,7 +6,6 @@ deuxfleurs.ipv6_default_gateway = "2001:910:1204:1::1"; deuxfleurs.lan_ip_prefix_length = 24; deuxfleurs.ipv6_prefix_length = 64; - deuxfleurs.nameservers = [ "192.168.1.1" ]; deuxfleurs.cname_target = "neptune.site.staging.deuxfleurs.org."; # no public ipv4 is used for the staging cluster on Neptune, diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index f7b70d7..245e12c 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -96,10 +96,6 @@ in type = types.nullOr types.str; default = null; }; - nameservers = mkOption { - description = "External DNS servers to use"; - type = types.listOf types.str; - }; # Parameters common to all nodes cluster_name = mkOption { @@ -190,47 +186,36 @@ in }; }; - # Configure Unbound DNS to redirect to Consul queries under .consul - # and to pass directly to public DNS resolver all others + # Configure Unbound as a central DNS server for everything + # - is its own recursor (applies DNSSec) for everything, + # no need to declare an outside nameserver + # - redirects to Consul queries under .consul services.unbound = { enable = true; - enableRootTrustAnchor = false; # disable DNSSEC as it causes issues settings = { server = { - interface = [ "127.0.0.1" "${cfg.lan_ip}" "172.17.0.1" ]; + interface = [ "127.0.0.1" "172.17.0.1" ]; domain-insecure = [ "consul." ]; local-zone = [ "consul. nodefault" ]; log-servfail = true; access-control = [ "127.0.0.0/8 allow" - "${cfg.lan_ip}/${toString cfg.lan_ip_prefix_length} allow" "172.17.0.0/16 allow" ]; }; - forward-zone = [ + stub-zone = [ # Forward .consul queries to Consul daemon { name = "consul."; - forward-addr = "${cfg.lan_ip}@8600"; - forward-no-cache = true; - forward-tcp-upstream = false; - forward-tls-upstream = false; - } - # Forward all queries to our ISP's nameserver - { - name = "."; - forward-addr = cfg.nameservers; - forward-first = true; + stub-addr = "${cfg.cluster_ip}@8600"; + stub-no-cache = true; + stub-tcp-upstream = false; + stub-tls-upstream = false; } ]; }; resolveLocalQueries = true; }; - # Reach Unbound through the IP of our LAN interface, - # instead of 127.0.0.1 (this will also work in Docker containers) - networking.nameservers = [ # TODO remove this ? - cfg.lan_ip - ]; services.resolved.enable = false; # Configure Wireguard VPN between all nodes @@ -321,9 +306,6 @@ in verify_outgoing = true; verify_server_hostname = true; }; - systemd.services.consul.serviceConfig = { # TODO remove this ? - AmbientCapabilities = "CAP_NET_BIND_SERVICE"; - }; services.nomad.enable = true; systemd.services.nomad.after = [ "wg-quick-wg0.service" ]; From e2aea648cf2e6c8b11d53d6149f9e9d161da6d3e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 24 Mar 2023 14:32:39 +0100 Subject: [PATCH 02/20] greatly simplify ipv4 and ipv6 configuration --- cluster/staging/known_hosts | 1 + cluster/staging/node/carcajou.nix | 8 ++- cluster/staging/node/caribou.nix | 2 - cluster/staging/node/df-pw5.nix | 5 +- cluster/staging/node/origan.nix | 3 +- cluster/staging/node/piranha.nix | 3 +- cluster/staging/site/bespin.nix | 5 +- cluster/staging/site/corrin.nix | 5 +- cluster/staging/site/jupiter.nix | 5 +- cluster/staging/site/neptune.nix | 4 -- nix/deuxfleurs.nix | 87 ++++++++++++++----------------- nix/remote-unlock.nix | 27 ++++++++-- 12 files changed, 76 insertions(+), 79 deletions(-) diff --git a/cluster/staging/known_hosts b/cluster/staging/known_hosts index 0cb04f2..d721c27 100644 --- a/cluster/staging/known_hosts +++ b/cluster/staging/known_hosts @@ -9,3 +9,4 @@ piranha.polyno.me,2a01:cb05:8984:3c00:223:24ff:feb0:ea82 ssh-ed25519 AAAAC3NzaC1 2a01:e0a:5e4:1d0:223:24ff:feaf:fdec ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsZas74RT6lCZwuUOPR23nPdbSdpWORyAmRgjoiMVHK df-pw5.machine.deuxfleurs.fr ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK/dJIxioCkfeehxeGiZR7qquYGoqEH/YrRJ/ukEcaLH 10.14.3.1 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJnpO6zpLWsyyugOoOj+2bUow9TUrcWgURFGGaoyu+co +192.168.1.22 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMf/ioVSSb19Slu+HZLgKt4f1/XsL+K9uMxazSWb/+nQ diff --git a/cluster/staging/node/carcajou.nix b/cluster/staging/node/carcajou.nix index e1bd3a6..5822f49 100644 --- a/cluster/staging/node/carcajou.nix +++ b/cluster/staging/node/carcajou.nix @@ -8,6 +8,12 @@ ./remote-unlock.nix ]; + deuxfleurs.remoteUnlock = { + networkInterface = "eno1"; + staticIP = "192.168.1.22/24"; + defaultGateway = "192.168.1.1"; + }; + # Use the systemd-boot EFI boot loader. boot.loader.systemd-boot.enable = true; boot.loader.timeout = 20; @@ -15,8 +21,6 @@ networking.hostName = "carcajou"; - deuxfleurs.network_interface = "eno1"; - deuxfleurs.lan_ip = "192.168.1.22"; deuxfleurs.ipv6 = "2001:910:1204:1::22"; deuxfleurs.cluster_ip = "10.14.1.2"; diff --git a/cluster/staging/node/caribou.nix b/cluster/staging/node/caribou.nix index 02cb16d..2e8691a 100644 --- a/cluster/staging/node/caribou.nix +++ b/cluster/staging/node/caribou.nix @@ -10,8 +10,6 @@ networking.hostName = "caribou"; - deuxfleurs.network_interface = "eno1"; - deuxfleurs.lan_ip = "192.168.1.23"; deuxfleurs.ipv6 = "2001:910:1204:1::23"; deuxfleurs.cluster_ip = "10.14.1.3"; diff --git a/cluster/staging/node/df-pw5.nix b/cluster/staging/node/df-pw5.nix index 33888d6..356a2ae 100644 --- a/cluster/staging/node/df-pw5.nix +++ b/cluster/staging/node/df-pw5.nix @@ -11,10 +11,11 @@ networking.hostName = "df-pw5"; - deuxfleurs.network_interface = "eno1"; - deuxfleurs.lan_ip = "192.168.5.130"; + deuxfleurs.staticIPv4.address = "192.168.5.130/24"; deuxfleurs.ipv6 = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7"; deuxfleurs.cluster_ip = "10.14.4.1"; deuxfleurs.is_raft_server = false; + + system.stateVersion = "22.11"; } diff --git a/cluster/staging/node/origan.nix b/cluster/staging/node/origan.nix index 50bce58..6db7f87 100644 --- a/cluster/staging/node/origan.nix +++ b/cluster/staging/node/origan.nix @@ -10,8 +10,7 @@ networking.hostName = "origan"; - deuxfleurs.network_interface = "eno1"; - deuxfleurs.lan_ip = "192.168.1.33"; + deuxfleurs.staticIPv4.address = "192.168.1.33/24"; deuxfleurs.ipv6 = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec"; deuxfleurs.cluster_ip = "10.14.2.33"; diff --git a/cluster/staging/node/piranha.nix b/cluster/staging/node/piranha.nix index 9ac2a07..4873693 100644 --- a/cluster/staging/node/piranha.nix +++ b/cluster/staging/node/piranha.nix @@ -10,8 +10,7 @@ networking.hostName = "piranha"; - deuxfleurs.network_interface = "eno1"; - deuxfleurs.lan_ip = "192.168.1.25"; + deuxfleurs.staticIPv4.address = "192.168.1.25/24"; deuxfleurs.ipv6 = "2a01:cb05:8984:9900:223:24ff:feb0:ea82"; deuxfleurs.cluster_ip = "10.14.3.1"; diff --git a/cluster/staging/site/bespin.nix b/cluster/staging/site/bespin.nix index 1133603..3fcefbb 100644 --- a/cluster/staging/site/bespin.nix +++ b/cluster/staging/site/bespin.nix @@ -2,10 +2,7 @@ { deuxfleurs.site_name = "bespin"; - deuxfleurs.lan_default_gateway = "192.168.5.254"; - deuxfleurs.ipv6_default_gateway = "2a02:a03f:6510:5102::1"; - deuxfleurs.lan_ip_prefix_length = 24; - deuxfleurs.ipv6_prefix_length = 64; + deuxfleurs.staticIPv4.defaultGateway = "192.168.5.254"; deuxfleurs.cname_target = "bespin.site.staging.deuxfleurs.org."; networking.firewall.allowedTCPPorts = [ 80 443 ]; diff --git a/cluster/staging/site/corrin.nix b/cluster/staging/site/corrin.nix index 8bf8693..0ff7b80 100644 --- a/cluster/staging/site/corrin.nix +++ b/cluster/staging/site/corrin.nix @@ -2,10 +2,7 @@ { deuxfleurs.site_name = "corrin"; - deuxfleurs.lan_default_gateway = "192.168.1.1"; - deuxfleurs.ipv6_default_gateway = "fe80::7ec1:77ff:fe3e:bb90"; - deuxfleurs.lan_ip_prefix_length = 24; - deuxfleurs.ipv6_prefix_length = 64; + deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; deuxfleurs.cname_target = "corrin.site.staging.deuxfleurs.org."; deuxfleurs.public_ipv4 = "82.120.233.78"; diff --git a/cluster/staging/site/jupiter.nix b/cluster/staging/site/jupiter.nix index 291e582..2269836 100644 --- a/cluster/staging/site/jupiter.nix +++ b/cluster/staging/site/jupiter.nix @@ -2,10 +2,7 @@ { deuxfleurs.site_name = "jupiter"; - deuxfleurs.lan_default_gateway = "192.168.1.1"; - deuxfleurs.ipv6_default_gateway = "fe80::9038:202a:73a0:e73b"; - deuxfleurs.lan_ip_prefix_length = 24; - deuxfleurs.ipv6_prefix_length = 64; + deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; deuxfleurs.cname_target = "jupiter.site.staging.deuxfleurs.org."; # no public ipv4 is used for the staging cluster on Jupiter diff --git a/cluster/staging/site/neptune.nix b/cluster/staging/site/neptune.nix index b030b46..36d5957 100644 --- a/cluster/staging/site/neptune.nix +++ b/cluster/staging/site/neptune.nix @@ -2,10 +2,6 @@ { deuxfleurs.site_name = "neptune"; - deuxfleurs.lan_default_gateway = "192.168.1.1"; - deuxfleurs.ipv6_default_gateway = "2001:910:1204:1::1"; - deuxfleurs.lan_ip_prefix_length = 24; - deuxfleurs.ipv6_prefix_length = 64; deuxfleurs.cname_target = "neptune.site.staging.deuxfleurs.org."; # no public ipv4 is used for the staging cluster on Neptune, diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 245e12c..6d27d5c 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -36,36 +36,20 @@ in in { # Parameters for individual nodes - network_interface = mkOption { - description = "Network interface name to configure"; - type = types.str; - }; - lan_ip = mkOption { - description = "IP address of this node on the local network interface"; - type = types.str; - }; - lan_ip_prefix_length = mkOption { - description = "Prefix length associated with lan_ip"; - type = types.int; - }; ipv6 = mkOption { - description = "Public IPv6 address of this node"; + description = "Static public IPv6 address of this node"; type = types.str; }; - ipv6_prefix_length = mkOption { - description = "Prefix length associated with ipv6 ip"; - type = types.int; + staticIPv4.address = mkOption { + description = "IP address (with prefix length) of this node on the local network interface"; + type = types.nullOr types.str; + default = null; }; cluster_ip = mkOption { description = "IP address of this node on the Wesher mesh network"; type = types.str; }; - wireguard_port = mkOption { - description = "Port for incoming Wireguard VPN connections"; - type = types.port; - default = 33799; - }; is_raft_server = mkOption { description = "Make this node a RAFT server for the Nomad and Consul deployments"; @@ -74,18 +58,15 @@ in }; # Parameters that generally vary between sites - lan_default_gateway = mkOption { - description = "IPv4 address of the default route on the local network interface"; - type = types.str; - }; - ipv6_default_gateway = mkOption { - description = "IPv6 address of the default IPv6 gateway for the targeted net interface"; - type = types.str; - }; site_name = mkOption { description = "Site (availability zone) on which this node is deployed"; type = types.str; }; + staticIPv4.defaultGateway = mkOption { + description = "IPv4 address of the default route on the local network interface"; + type = types.nullOr types.str; + default = null; + }; public_ipv4 = mkOption { description = "Public IPv4 through which this node is accessible (possibly after port opening using DiploNAT), for domain names that are updated by D53"; type = types.nullOr types.str; @@ -124,6 +105,13 @@ in type = types.bool; default = false; }; + + # Options that generally stay to their default value + wireguardPort = mkOption { + description = "Port for incoming Wireguard VPN connections"; + type = types.port; + default = 33799; + }; }; imports = [ @@ -157,33 +145,33 @@ in systemd.network.networks = { "10-uplink" = { matchConfig = { - # We could preprend "en* eth*" to match all ethernet interfaces - Name = "${cfg.network_interface}"; + Name = "en* eth*"; }; + ipv6AcceptRAConfig = { + Token = "static:${cfg.ipv6}"; + UseDNS = false; + }; + } // (if cfg.staticIPv4.address == null || cfg.staticIPv4.defaultGateway == null then { networkConfig = { - IPv6AcceptRA = false; - LinkLocalAddressing = "no"; + DHCP = "ipv4"; }; + dhcpV4Config = { + UseDNS = false; + }; + } else { address = [ - "${cfg.lan_ip}/${toString cfg.lan_ip_prefix_length}" - "${cfg.ipv6}/${toString cfg.ipv6_prefix_length}" + "${cfg.staticIPv4.address}" ]; routes = [ { routeConfig = { - Gateway = cfg.lan_default_gateway; + Gateway = cfg.staticIPv4.defaultGateway; # GatewayOnLink - Takes a boolean. If set to true, the kernel does not have to check if the gateway is reachable directly by the current machine (i.e., attached to the local network), so that we can insert the route in the kernel table without it being complained about. Defaults to "no". GatewayOnLink = true; }; } - { - routeConfig = { - Gateway = cfg.ipv6_default_gateway; - GatewayOnLink = true; - }; - } ]; - }; + }); }; # Configure Unbound as a central DNS server for everything @@ -221,7 +209,7 @@ in # Configure Wireguard VPN between all nodes networking.wireguard.interfaces.wg0 = { ips = [ "${cfg.cluster_ip}/16" ]; - listenPort = cfg.wireguard_port; + listenPort = cfg.wireguardPort; privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; mtu = 1420; }; @@ -248,7 +236,7 @@ in # systemd.services."wg-quick-wg0".after = [ "unbound.service" ]; # networking.wg-quick.interfaces.wg0 = { # address = [ "${cfg.cluster_ip}/16" ]; - # listenPort = cfg.wireguard_port; + # listenPort = cfg.wireguardPort; # privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; # mtu = 1420; # peers = map ({ publicKey, endpoint, IP, ... }: { @@ -384,13 +372,14 @@ in allowedUDPPorts = [ # Allow peers to connect to Wireguard - cfg.wireguard_port + cfg.wireguardPort ]; # Allow specific hosts access to specific things in the cluster extraCommands = '' - # Allow everything from router (usefull for UPnP/IGD) - iptables -A INPUT -s ${cfg.lan_default_gateway} -j ACCEPT + # Allow UDP packets comming from port 1900 from a local address, + # these are necessary for UPnP/IGD + iptables -A INPUT -s 192.168.0.0/16 -p udp --sport 1900 -j ACCEPT # Allow docker containers to access all ports iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT @@ -401,7 +390,7 @@ in # When stopping firewall, delete all rules that were configured manually above extraStopCommands = '' - iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT + iptables -D INPUT -s 192.168.0.0/16 -p udp --sport 1900 -j ACCEPT iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT ''; diff --git a/nix/remote-unlock.nix b/nix/remote-unlock.nix index 2975a94..fdc5285 100644 --- a/nix/remote-unlock.nix +++ b/nix/remote-unlock.nix @@ -1,8 +1,27 @@ { config, pkgs, ... }: + +let + cfg = config.deuxfleurs.remoteUnlock; +in with builtins; with pkgs.lib; { + options.deuxfleurs.remoteUnlock = { + networkInterface = mkOption { + description = "Network interface to configure with static IP"; + type = types.str; + }; + staticIP = mkOption { + description = "IP address (with prefix length) of this node on the local network interface"; + type = types.str; + }; + defaultGateway = mkOption { + description = "IP address of default gateway"; + type = types.str; + }; + }; + config = { boot.initrd.availableKernelModules = [ "pps_core" "ptp" "e1000e" ]; boot.initrd.network.enable = true; @@ -13,12 +32,12 @@ hostKeys = [ "/var/lib/deuxfleurs/remote-unlock/ssh_host_ed25519_key" ]; }; boot.initrd.network.postCommands = '' - ip addr add ${config.deuxfleurs.lan_ip}/${toString config.deuxfleurs.lan_ip_prefix_length} dev ${config.deuxfleurs.network_interface} - ip link set dev ${config.deuxfleurs.network_interface} up - ip route add default via ${config.deuxfleurs.lan_default_gateway} dev ${config.deuxfleurs.network_interface} + ip addr add ${cfg.staticIP} dev ${cfg.networkInterface} + ip link set dev ${cfg.networkInterface} up + ip route add default via ${cfg.defaultGateway} dev ${cfg.networkInterface} ip a ip route - ping -c 4 ${config.deuxfleurs.lan_default_gateway} + ping -c 4 ${cfg.defaultGateway} echo 'echo run cryptsetup-askpass to unlock drives' >> /root/.profile ''; }; From 96566ae523934f5a37b8d7c2a9ef928cd5c0d098 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 24 Mar 2023 15:26:39 +0100 Subject: [PATCH 03/20] refactor configuration syntax --- cluster/staging/cluster.nix | 70 +++++------ cluster/staging/node/carcajou.nix | 7 +- cluster/staging/node/caribou.nix | 9 +- cluster/staging/node/df-pw5.nix | 8 +- cluster/staging/node/origan.nix | 9 +- cluster/staging/node/piranha.nix | 9 +- cluster/staging/site/bespin.nix | 4 +- cluster/staging/site/corrin.nix | 6 +- cluster/staging/site/jupiter.nix | 7 +- cluster/staging/site/neptune.nix | 8 +- nix/deuxfleurs.nix | 190 ++++++++++++++---------------- nix/remote-unlock.nix | 2 +- 12 files changed, 144 insertions(+), 185 deletions(-) diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix index cf30d6e..56ca904 100644 --- a/cluster/staging/cluster.nix +++ b/cluster/staging/cluster.nix @@ -1,49 +1,43 @@ { config, pkgs, ... } @ args: { - deuxfleurs.cluster_name = "staging"; + deuxfleurs.clusterName = "staging"; # The IP range to use for the Wireguard overlay of this cluster - deuxfleurs.cluster_prefix = "10.14.0.0"; - deuxfleurs.cluster_prefix_length = 16; + deuxfleurs.clusterPrefix = "10.14.0.0/16"; - deuxfleurs.cluster_nodes = [ - { - hostname = "carcajou"; - site_name = "neptune"; + deuxfleurs.clusterNodes = { + "carcajou" = { + siteName = "neptune"; publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk="; - IP = "10.14.1.2"; + address = "10.14.1.2"; endpoint = "77.207.15.215:33722"; - } - { - hostname = "caribou"; - site_name = "neptune"; + }; + "caribou" = { + siteName = "neptune"; publicKey = "lABn/axzD1jkFulX8c+K3B3CbKXORlIMDDoe8sQVxhs="; - IP = "10.14.1.3"; + address = "10.14.1.3"; endpoint = "77.207.15.215:33723"; - } - { - hostname = "origan"; - site_name = "jupiter"; + }; + "origan" = { + siteName = "jupiter"; publicKey = "smBQYUS60JDkNoqkTT7TgbpqFiM43005fcrT6472llI="; - IP = "10.14.2.33"; + address = "10.14.2.33"; endpoint = "82.64.238.84:33733"; - } - { - hostname = "piranha"; - site_name = "corrin"; + }; + "piranha" = { + siteName = "corrin"; publicKey = "m9rLf+233X1VColmeVrM/xfDGro5W6Gk5N0zqcf32WY="; - IP = "10.14.3.1"; + address = "10.14.3.1"; #endpoint = "82.120.233.78:33721"; - } - { - hostname = "df-pw5"; - site_name = "bespin"; + }; + "df-pw5" = { + siteName = "bespin"; publicKey = "XLOYoMXF+PO4jcgfSVAk+thh4VmWx0wzWnb0xs08G1s="; - IP = "10.14.4.1"; + address = "10.14.4.1"; endpoint = "bitfrost.fiber.shirokumo.net:33734"; - } - ]; + }; + }; services.wgautomesh.logLevel = "debug"; # Bootstrap IPs for Consul cluster, @@ -54,7 +48,7 @@ "10.14.1.3" # caribou ]; - deuxfleurs.admin_accounts = { + deuxfleurs.adminAccounts = { lx = [ # Keys for accessing nodes from outside "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy" @@ -142,16 +136,16 @@ enable = true; port = substituter_port; openFirewall = false; - bindAddress = config.deuxfleurs.cluster_ip; + bindAddress = "0.0.0.0"; package = pkgs.haskellPackages.nix-serve-ng; }; nix.settings.substituters = map - ({ IP, ... }: "http://${IP}:${builtins.toString substituter_port}") - (builtins.filter - ({ site_name, IP, ...}: - (IP != config.deuxfleurs.cluster_ip - && site_name == config.deuxfleurs.site_name)) - config.deuxfleurs.cluster_nodes); + ({ address, ... }: "http://${address}:${builtins.toString substituter_port}") + (builtins.attrValues (pkgs.lib.filterAttrs + (hostname: { siteName, ...}: + (hostname != config.deuxfleurs.hostName + && siteName == config.deuxfleurs.siteName)) + config.deuxfleurs.clusterNodes)); }) ]; } diff --git a/cluster/staging/node/carcajou.nix b/cluster/staging/node/carcajou.nix index 5822f49..e6c1653 100644 --- a/cluster/staging/node/carcajou.nix +++ b/cluster/staging/node/carcajou.nix @@ -19,11 +19,8 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "carcajou"; - - deuxfleurs.ipv6 = "2001:910:1204:1::22"; - - deuxfleurs.cluster_ip = "10.14.1.2"; + deuxfleurs.hostName = "carcajou"; + deuxfleurs.ipv6Address = "2001:910:1204:1::22"; system.stateVersion = "21.05"; } diff --git a/cluster/staging/node/caribou.nix b/cluster/staging/node/caribou.nix index 2e8691a..ad5a65d 100644 --- a/cluster/staging/node/caribou.nix +++ b/cluster/staging/node/caribou.nix @@ -8,12 +8,9 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "caribou"; - - deuxfleurs.ipv6 = "2001:910:1204:1::23"; - - deuxfleurs.cluster_ip = "10.14.1.3"; - deuxfleurs.is_raft_server = true; + deuxfleurs.hostName = "caribou"; + deuxfleurs.ipv6Address = "2001:910:1204:1::23"; + deuxfleurs.isRaftServer = true; system.stateVersion = "21.05"; } diff --git a/cluster/staging/node/df-pw5.nix b/cluster/staging/node/df-pw5.nix index 356a2ae..0e5be15 100644 --- a/cluster/staging/node/df-pw5.nix +++ b/cluster/staging/node/df-pw5.nix @@ -9,13 +9,9 @@ boot.loader.efi.efiSysMountPoint = "/boot"; boot.loader.timeout = 20; - networking.hostName = "df-pw5"; - + deuxfleurs.hostName = "df-pw5"; deuxfleurs.staticIPv4.address = "192.168.5.130/24"; - deuxfleurs.ipv6 = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7"; - - deuxfleurs.cluster_ip = "10.14.4.1"; - deuxfleurs.is_raft_server = false; + deuxfleurs.ipv6Address = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7"; system.stateVersion = "22.11"; } diff --git a/cluster/staging/node/origan.nix b/cluster/staging/node/origan.nix index 6db7f87..d900fd6 100644 --- a/cluster/staging/node/origan.nix +++ b/cluster/staging/node/origan.nix @@ -8,13 +8,10 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "origan"; - + deuxfleurs.hostName = "origan"; deuxfleurs.staticIPv4.address = "192.168.1.33/24"; - deuxfleurs.ipv6 = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec"; - - deuxfleurs.cluster_ip = "10.14.2.33"; - deuxfleurs.is_raft_server = true; + deuxfleurs.ipv6Address = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec"; + deuxfleurs.isRaftServer = true; system.stateVersion = "22.11"; } diff --git a/cluster/staging/node/piranha.nix b/cluster/staging/node/piranha.nix index 4873693..436965c 100644 --- a/cluster/staging/node/piranha.nix +++ b/cluster/staging/node/piranha.nix @@ -8,13 +8,10 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "piranha"; - + deuxfleurs.hostName = "piranha"; deuxfleurs.staticIPv4.address = "192.168.1.25/24"; - deuxfleurs.ipv6 = "2a01:cb05:8984:9900:223:24ff:feb0:ea82"; - - deuxfleurs.cluster_ip = "10.14.3.1"; - deuxfleurs.is_raft_server = true; + deuxfleurs.ipv6Address = "2a01:cb05:8984:9900:223:24ff:feb0:ea82"; + deuxfleurs.isRaftServer = true; system.stateVersion = "22.11"; } diff --git a/cluster/staging/site/bespin.nix b/cluster/staging/site/bespin.nix index 3fcefbb..22feb59 100644 --- a/cluster/staging/site/bespin.nix +++ b/cluster/staging/site/bespin.nix @@ -1,9 +1,9 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "bespin"; + deuxfleurs.siteName = "bespin"; deuxfleurs.staticIPv4.defaultGateway = "192.168.5.254"; - deuxfleurs.cname_target = "bespin.site.staging.deuxfleurs.org."; + deuxfleurs.cnameTarget = "bespin.site.staging.deuxfleurs.org."; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/staging/site/corrin.nix b/cluster/staging/site/corrin.nix index 0ff7b80..0083986 100644 --- a/cluster/staging/site/corrin.nix +++ b/cluster/staging/site/corrin.nix @@ -1,10 +1,10 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "corrin"; + deuxfleurs.siteName = "corrin"; deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; - deuxfleurs.cname_target = "corrin.site.staging.deuxfleurs.org."; - deuxfleurs.public_ipv4 = "82.120.233.78"; + deuxfleurs.cnameTarget = "corrin.site.staging.deuxfleurs.org."; + deuxfleurs.publicIPv4 = "82.120.233.78"; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/staging/site/jupiter.nix b/cluster/staging/site/jupiter.nix index 2269836..2d39f5a 100644 --- a/cluster/staging/site/jupiter.nix +++ b/cluster/staging/site/jupiter.nix @@ -1,12 +1,9 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "jupiter"; + deuxfleurs.siteName = "jupiter"; deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; - deuxfleurs.cname_target = "jupiter.site.staging.deuxfleurs.org."; - - # no public ipv4 is used for the staging cluster on Jupiter - # deuxfleurs.public_ipv4 = "???"; + deuxfleurs.cnameTarget = "jupiter.site.staging.deuxfleurs.org."; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/staging/site/neptune.nix b/cluster/staging/site/neptune.nix index 36d5957..f94d62f 100644 --- a/cluster/staging/site/neptune.nix +++ b/cluster/staging/site/neptune.nix @@ -1,12 +1,8 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "neptune"; - deuxfleurs.cname_target = "neptune.site.staging.deuxfleurs.org."; - - # no public ipv4 is used for the staging cluster on Neptune, - # because the Internet connection is already used for the prod cluster - # deuxfleurs.public_ipv4 = "77.207.15.215"; + deuxfleurs.siteName = "neptune"; + deuxfleurs.cnameTarget = "neptune.site.staging.deuxfleurs.org."; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 6d27d5c..7632486 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -6,110 +6,98 @@ in with builtins; with pkgs.lib; { - options.deuxfleurs = - let wg_node = with types; submodule { - options = { - hostname = mkOption { - type = str; - description = "Host name"; - }; - site_name = mkOption { - type = nullOr str; - description = "Site where the node is located"; - default = null; - }; - IP = mkOption { - type = str; - description = "IP Address in the Wireguard network"; - }; - publicKey = mkOption { - type = str; - description = "Public key"; - }; - endpoint = mkOption { - type = nullOr str; - default = null; - description = "Wireguard endpoint on the public Internet"; - }; - }; - }; - in - { + options.deuxfleurs = with types; { # Parameters for individual nodes - ipv6 = mkOption { + hostName = mkOption { + description = "Node name"; + type = str; + }; + ipv6Address = mkOption { description = "Static public IPv6 address of this node"; - type = types.str; + type = str; }; staticIPv4.address = mkOption { description = "IP address (with prefix length) of this node on the local network interface"; - type = types.nullOr types.str; + type = nullOr str; default = null; }; - cluster_ip = mkOption { - description = "IP address of this node on the Wesher mesh network"; - type = types.str; - }; - - is_raft_server = mkOption { + isRaftServer = mkOption { description = "Make this node a RAFT server for the Nomad and Consul deployments"; - type = types.bool; + type = bool; default = false; }; # Parameters that generally vary between sites - site_name = mkOption { + siteName = mkOption { description = "Site (availability zone) on which this node is deployed"; - type = types.str; + type = str; }; staticIPv4.defaultGateway = mkOption { description = "IPv4 address of the default route on the local network interface"; - type = types.nullOr types.str; + type = nullOr str; default = null; }; - public_ipv4 = mkOption { + + publicIPv4 = mkOption { description = "Public IPv4 through which this node is accessible (possibly after port opening using DiploNAT), for domain names that are updated by D53"; - type = types.nullOr types.str; + type = nullOr str; default = null; }; - cname_target = mkOption { + cnameTarget = mkOption { description = "DNS CNAME target to use for services hosted in this site, for domain names that are updated by D53"; - type = types.nullOr types.str; + type = nullOr str; default = null; }; # Parameters common to all nodes - cluster_name = mkOption { + clusterName = mkOption { description = "Name of this Deuxfleurs deployment"; - type = types.str; + type = str; }; - cluster_prefix = mkOption { - description = "IP address prefix for the Wireguard overlay network"; - type = types.str; + clusterPrefix = mkOption { + description = "IP address prefix (and length) for the Wireguard overlay network"; + type = str; }; - cluster_prefix_length = mkOption { - description = "IP address prefix length for the Wireguard overlay network"; - type = types.int; - default = 16; - }; - cluster_nodes = mkOption { + clusterNodes = mkOption { description = "Nodes that are part of the cluster"; - type = types.listOf wg_node; + type = attrsOf (submodule { + options = { + siteName = mkOption { + type = nullOr str; + description = "Site where the node is located"; + default = null; + }; + address = mkOption { + type = str; + description = "IP Address in the Wireguard network"; + }; + publicKey = mkOption { + type = str; + description = "Public key"; + }; + endpoint = mkOption { + type = nullOr str; + default = null; + description = "Wireguard endpoint on the public Internet"; + }; + }; + }); }; - admin_accounts = mkOption { + adminAccounts = mkOption { description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys"; - type = types.attrsOf (types.listOf types.str); + type = attrsOf (listOf str); }; bootstrap = mkOption { description = "Whether to enable bootstrapping for Nomad and Consul"; - type = types.bool; + type = bool; default = false; }; # Options that generally stay to their default value wireguardPort = mkOption { description = "Port for incoming Wireguard VPN connections"; - type = types.port; + type = port; default = 33799; }; }; @@ -119,25 +107,29 @@ in ]; config = - let node_meta = { - "site" = cfg.site_name; - "public_ipv6" = cfg.ipv6; + let + clusterNodeCfg = getAttr cfg.hostName cfg.clusterNodes; + clusterAddress = clusterNodeCfg.address; + node_meta = { + "site" = cfg.siteName; + "public_ipv6" = cfg.ipv6Address; } // - (if cfg.public_ipv4 != null - then { "public_ipv4" = cfg.public_ipv4; } + (if cfg.publicIPv4 != null + then { "public_ipv4" = cfg.publicIPv4; } else {}) // - (if cfg.cname_target != null - then { "cname_target" = cfg.cname_target; } + (if cfg.cnameTarget != null + then { "cname_target" = cfg.cnameTarget; } else {}); in { + networking.hostName = cfg.hostName; # Configure admin accounts on all nodes - users.users = builtins.mapAttrs (name: publicKeys: { + users.users = mapAttrs (name: publicKeys: { isNormalUser = true; extraGroups = [ "wheel" ]; openssh.authorizedKeys.keys = publicKeys; - }) cfg.admin_accounts; + }) cfg.adminAccounts; # Configure network interfaces networking.useDHCP = false; @@ -148,7 +140,7 @@ in Name = "en* eth*"; }; ipv6AcceptRAConfig = { - Token = "static:${cfg.ipv6}"; + Token = "static:${cfg.ipv6Address}"; UseDNS = false; }; } // (if cfg.staticIPv4.address == null || cfg.staticIPv4.defaultGateway == null then { @@ -195,7 +187,7 @@ in # Forward .consul queries to Consul daemon { name = "consul."; - stub-addr = "${cfg.cluster_ip}@8600"; + stub-addr = "${clusterAddress}@8600"; stub-no-cache = true; stub-tcp-upstream = false; stub-tls-upstream = false; @@ -208,7 +200,7 @@ in # Configure Wireguard VPN between all nodes networking.wireguard.interfaces.wg0 = { - ips = [ "${cfg.cluster_ip}/16" ]; + ips = [ "${clusterAddress}/16" ]; listenPort = cfg.wireguardPort; privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; mtu = 1420; @@ -220,28 +212,24 @@ in gossipSecretFile = "/var/lib/wgautomesh/gossip_secret"; persistFile = "/var/lib/wgautomesh/state"; upnpForwardPublicPort = - let - us = filter ({ hostname, ...}: hostname == config.networking.hostName) cfg.cluster_nodes; - in - if length us > 0 && (head us).endpoint != null then - strings.toInt (lists.last (split ":" (head us).endpoint)) + if clusterNodeCfg.endpoint != null then + strings.toInt (lists.last (split ":" clusterNodeCfg.endpoint)) else null; - peers = map ({ publicKey, endpoint, IP, ... }: { - address = IP; + peers = attrValues (mapAttrs (hostname: { publicKey, endpoint, address, ... }: { + inherit address endpoint; pubkey = publicKey; - endpoint = endpoint; - }) cfg.cluster_nodes; + }) cfg.clusterNodes); }; # Old code for wg-quick, we can use this as a fallback if we fail to make wgautomesh work # systemd.services."wg-quick-wg0".after = [ "unbound.service" ]; # networking.wg-quick.interfaces.wg0 = { - # address = [ "${cfg.cluster_ip}/16" ]; + # address = [ "${clusterAddress}/16" ]; # listenPort = cfg.wireguardPort; # privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; # mtu = 1420; - # peers = map ({ publicKey, endpoint, IP, ... }: { + # peers = map ({ publicKey, endpoint, address, ... }: { # inherit publicKey endpoint; - # allowedIPs = [ "${IP}/32" ]; + # allowedIPs = [ "${address}/32" ]; # persistentKeepalive = 25; # }; @@ -255,25 +243,25 @@ in ''; # Configure /etc/hosts to link all hostnames to their Wireguard IP - networking.extraHosts = builtins.concatStringsSep "\n" (map - ({ hostname, IP, ...}: "${IP} ${hostname}") - cfg.cluster_nodes); + networking.extraHosts = concatStringsSep "\n" (attrValues (mapAttrs + (hostname: { address, ...}: "${address} ${hostname}") + cfg.clusterNodes)); # Enable Hashicorp Consul & Nomad services.consul.enable = true; systemd.services.consul.after = [ "wg-quick-wg0.service" ]; services.consul.extraConfig = - (if cfg.is_raft_server + (if cfg.isRaftServer then { server = true; } // (if cfg.bootstrap then { bootstrap_expect = 3; } else {}) else {}) // { inherit node_meta; - datacenter = cfg.cluster_name; + datacenter = cfg.clusterName; ui_config = { enabled = true; }; - bind_addr = "${cfg.cluster_ip}"; + bind_addr = "${clusterAddress}"; addresses = { https = "0.0.0.0"; @@ -303,18 +291,18 @@ in pkgs.zstd ]; services.nomad.settings = - (if cfg.is_raft_server + (if cfg.isRaftServer then { server = { enabled = true; } // (if cfg.bootstrap then { bootstrap_expect = 3; } else {}); } else {}) // { - region = cfg.cluster_name; - datacenter = cfg.site_name; + region = cfg.clusterName; + datacenter = cfg.siteName; advertise = { - rpc = "${cfg.cluster_ip}"; - http = "${cfg.cluster_ip}"; - serf = "${cfg.cluster_ip}"; + rpc = "${clusterAddress}"; + http = "${clusterAddress}"; + serf = "${clusterAddress}"; }; consul = { address = "localhost:8501"; @@ -367,7 +355,7 @@ in allowedTCPPorts = [ # Allow anyone to connect on SSH port - (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports) + (head ({ openssh.ports = [22]; } // config.services).openssh.ports) ]; allowedUDPPorts = [ @@ -385,14 +373,14 @@ in iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # Allow other nodes on VPN to access all ports - iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT + iptables -A INPUT -s ${cfg.clusterPrefix} -j ACCEPT ''; # When stopping firewall, delete all rules that were configured manually above extraStopCommands = '' iptables -D INPUT -s 192.168.0.0/16 -p udp --sport 1900 -j ACCEPT iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT - iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT + iptables -D INPUT -s ${cfg.clusterPrefix} -j ACCEPT ''; }; }; diff --git a/nix/remote-unlock.nix b/nix/remote-unlock.nix index fdc5285..3c3e4c8 100644 --- a/nix/remote-unlock.nix +++ b/nix/remote-unlock.nix @@ -28,7 +28,7 @@ in boot.initrd.network.ssh = { enable = true; port = 222; - authorizedKeys = concatLists (mapAttrsToList (name: user: user) config.deuxfleurs.admin_accounts); + authorizedKeys = concatLists (mapAttrsToList (name: user: user) config.deuxfleurs.adminAccounts); hostKeys = [ "/var/lib/deuxfleurs/remote-unlock/ssh_host_ed25519_key" ]; }; boot.initrd.network.postCommands = '' From a31c6d109e95a33e165d43c4a47de54ea53ca436 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Mar 2023 16:27:08 +0200 Subject: [PATCH 04/20] remove obsolete directives --- nix/configuration.nix | 3 --- 1 file changed, 3 deletions(-) diff --git a/nix/configuration.nix b/nix/configuration.nix index 9d3169b..d88d6ef 100644 --- a/nix/configuration.nix +++ b/nix/configuration.nix @@ -36,9 +36,6 @@ boot.kernel.sysctl = { "vm.max_map_count" = 262144; - "net.ipv6.conf.all.accept_ra" = 0; - "net.ipv6.conf.all.autoconf" = 0; - "net.ipv6.conf.all.use_tempaddr" = 0; }; services.journald.extraConfig = '' From cb8d7e92d2aa2950fa403ff8e2ec2a9a31b48b32 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 5 Apr 2023 10:25:22 +0200 Subject: [PATCH 05/20] staging: ipv6-only diplonat for automatic address discovery --- cluster/staging/app/core/deploy/core-system.hcl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cluster/staging/app/core/deploy/core-system.hcl b/cluster/staging/app/core/deploy/core-system.hcl index 05fa0f2..dba6b95 100644 --- a/cluster/staging/app/core/deploy/core-system.hcl +++ b/cluster/staging/app/core/deploy/core-system.hcl @@ -13,7 +13,6 @@ job "core-system" { stagger = "1m" } -/* group "diplonat" { task "diplonat" { driver = "nix2" @@ -21,7 +20,7 @@ job "core-system" { config { packages = [ "#iptables", - "git+https://git.deuxfleurs.fr/Deuxfleurs/diplonat.git?ref=main&rev=f306e8dc8d0e93478353ce39b6064e8c06a8bca6" + "git+https://git.deuxfleurs.fr/Deuxfleurs/diplonat.git?ref=stun&rev=21ab77b8288630c5f39a30b098c6a3888df622a1" ] command = "diplonat" } @@ -53,6 +52,7 @@ job "core-system" { data = < Date: Wed, 5 Apr 2023 13:20:17 +0200 Subject: [PATCH 06/20] Allow for IPv6 with RA disabled by manually providing gateway --- cluster/staging/node/carcajou.nix | 2 +- cluster/staging/node/caribou.nix | 2 +- cluster/staging/node/df-pw5.nix | 2 +- cluster/staging/node/origan.nix | 2 +- cluster/staging/node/piranha.nix | 2 +- nix/deuxfleurs.nix | 87 +++++++++++++++++++------------ 6 files changed, 59 insertions(+), 38 deletions(-) diff --git a/cluster/staging/node/carcajou.nix b/cluster/staging/node/carcajou.nix index e6c1653..d5211f1 100644 --- a/cluster/staging/node/carcajou.nix +++ b/cluster/staging/node/carcajou.nix @@ -20,7 +20,7 @@ boot.loader.efi.canTouchEfiVariables = true; deuxfleurs.hostName = "carcajou"; - deuxfleurs.ipv6Address = "2001:910:1204:1::22"; + deuxfleurs.staticIPv6.address = "2001:910:1204:1::22"; system.stateVersion = "21.05"; } diff --git a/cluster/staging/node/caribou.nix b/cluster/staging/node/caribou.nix index ad5a65d..80293e9 100644 --- a/cluster/staging/node/caribou.nix +++ b/cluster/staging/node/caribou.nix @@ -9,7 +9,7 @@ boot.loader.efi.canTouchEfiVariables = true; deuxfleurs.hostName = "caribou"; - deuxfleurs.ipv6Address = "2001:910:1204:1::23"; + deuxfleurs.staticIPv6.address = "2001:910:1204:1::23"; deuxfleurs.isRaftServer = true; system.stateVersion = "21.05"; diff --git a/cluster/staging/node/df-pw5.nix b/cluster/staging/node/df-pw5.nix index 0e5be15..e1a132c 100644 --- a/cluster/staging/node/df-pw5.nix +++ b/cluster/staging/node/df-pw5.nix @@ -11,7 +11,7 @@ deuxfleurs.hostName = "df-pw5"; deuxfleurs.staticIPv4.address = "192.168.5.130/24"; - deuxfleurs.ipv6Address = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7"; + deuxfleurs.staticIPv6.address = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7"; system.stateVersion = "22.11"; } diff --git a/cluster/staging/node/origan.nix b/cluster/staging/node/origan.nix index d900fd6..6906f4d 100644 --- a/cluster/staging/node/origan.nix +++ b/cluster/staging/node/origan.nix @@ -10,7 +10,7 @@ deuxfleurs.hostName = "origan"; deuxfleurs.staticIPv4.address = "192.168.1.33/24"; - deuxfleurs.ipv6Address = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec"; + deuxfleurs.staticIPv6.address = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec"; deuxfleurs.isRaftServer = true; system.stateVersion = "22.11"; diff --git a/cluster/staging/node/piranha.nix b/cluster/staging/node/piranha.nix index 2b9907d..bc1aa03 100644 --- a/cluster/staging/node/piranha.nix +++ b/cluster/staging/node/piranha.nix @@ -10,7 +10,7 @@ deuxfleurs.hostName = "piranha"; deuxfleurs.staticIPv4.address = "192.168.1.25/24"; - deuxfleurs.ipv6Address = "2a01:cb05:9142:7400:223:24ff:feb0:ea82"; + deuxfleurs.staticIPv6.address = "2a01:cb05:9142:7400:223:24ff:feb0:ea82"; deuxfleurs.isRaftServer = true; system.stateVersion = "22.11"; diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 7632486..8b9a5ef 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -12,16 +12,15 @@ in description = "Node name"; type = str; }; - ipv6Address = mkOption { - description = "Static public IPv6 address of this node"; - type = str; - }; staticIPv4.address = mkOption { description = "IP address (with prefix length) of this node on the local network interface"; type = nullOr str; default = null; }; - + staticIPv6.address = mkOption { + description = "Static public IPv6 address of this node"; + type = str; + }; isRaftServer = mkOption { description = "Make this node a RAFT server for the Nomad and Consul deployments"; type = bool; @@ -38,6 +37,19 @@ in type = nullOr str; default = null; }; + staticIPv6.defaultGateway = mkOption { + description = '' + IPv6 address of the default route on the local network interface. + IPv6 Router Advertisements (RA) will be totally disabled. + ''; + type = nullOr str; + default = null; + }; + staticIPv6.prefixLength = mkOption { + description = "IPv6 prefix length"; + type = int; + default = 64; + }; publicIPv4 = mkOption { description = "Public IPv4 through which this node is accessible (possibly after port opening using DiploNAT), for domain names that are updated by D53"; @@ -112,7 +124,7 @@ in clusterAddress = clusterNodeCfg.address; node_meta = { "site" = cfg.siteName; - "public_ipv6" = cfg.ipv6Address; + "public_ipv6" = cfg.staticIPv6.address; } // (if cfg.publicIPv4 != null then { "public_ipv4" = cfg.publicIPv4; } @@ -134,37 +146,46 @@ in # Configure network interfaces networking.useDHCP = false; networking.useNetworkd = true; - systemd.network.networks = { - "10-uplink" = { - matchConfig = { - Name = "en* eth*"; - }; - ipv6AcceptRAConfig = { - Token = "static:${cfg.ipv6Address}"; - UseDNS = false; - }; - } // (if cfg.staticIPv4.address == null || cfg.staticIPv4.defaultGateway == null then { - networkConfig = { - DHCP = "ipv4"; - }; - dhcpV4Config = { - UseDNS = false; - }; - } else { - address = [ - "${cfg.staticIPv4.address}" - ]; - routes = [ - { + systemd.network.networks."10-uplink" = + let + dyn_v4 = cfg.staticIPv4.address == null || cfg.staticIPv4.defaultGateway == null; + dyn_v6 = cfg.staticIPv6.defaultGateway == null; + in + { + matchConfig.Name = "en* eth*"; + + address = + optional (!dyn_v4) "${cfg.staticIPv4.address}" + ++ optional (!dyn_v6) "${cfg.staticIPv6.address}/${toString cfg.staticIPv6.prefixLength}"; + + routes = + optional (!dyn_v4) { routeConfig = { Gateway = cfg.staticIPv4.defaultGateway; # GatewayOnLink - Takes a boolean. If set to true, the kernel does not have to check if the gateway is reachable directly by the current machine (i.e., attached to the local network), so that we can insert the route in the kernel table without it being complained about. Defaults to "no". GatewayOnLink = true; }; - } - ]; - }); - }; + } ++ optional (!dyn_v6) { + routeConfig = { + Gateway = cfg.staticIPv6.defaultGateway; + GatewayOnLink = true; + }; + }; + + # Dynamic IPv4: enable DHCP but not for DNS servers + networkConfig.DHCP = mkIf dyn_v4 "ipv4"; + dhcpV4Config.UseDNS = mkIf dyn_v4 false; + + # Dynamic IPv6: only fetch default route, use static + # address and no DNS servers + ipv6AcceptRAConfig.Token = mkIf dyn_v6 "static:${cfg.staticIPv6.address}"; + ipv6AcceptRAConfig.UseDNS = mkIf dyn_v6 false; + + # Static IPv6: disable all router advertisements and + # link-local addresses + networkConfig.IPv6AcceptRA = mkIf (!dyn_v6) false; + networkConfig.LinkLocalAddressing = mkIf (!dyn_v6) "no"; + }; # Configure Unbound as a central DNS server for everything # - is its own recursor (applies DNSSec) for everything, @@ -211,7 +232,7 @@ in gossipPort = 1666; gossipSecretFile = "/var/lib/wgautomesh/gossip_secret"; persistFile = "/var/lib/wgautomesh/state"; - upnpForwardPublicPort = + upnpForwardPublicPort = if clusterNodeCfg.endpoint != null then strings.toInt (lists.last (split ":" clusterNodeCfg.endpoint)) else null; From bb25797d2f164f3f0f3c81a492e8f6e7fe1fd232 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 5 Apr 2023 13:44:38 +0200 Subject: [PATCH 07/20] make script clearer and add documentation --- nix/deuxfleurs.nix | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 8b9a5ef..57486c2 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -40,13 +40,13 @@ in staticIPv6.defaultGateway = mkOption { description = '' IPv6 address of the default route on the local network interface. - IPv6 Router Advertisements (RA) will be totally disabled. + IPv6 Router Advertisements (RA) will be totally disabled if this is set. ''; type = nullOr str; default = null; }; staticIPv6.prefixLength = mkOption { - description = "IPv6 prefix length"; + description = "IPv6 prefix length, used only when router advertisements are disabled."; type = int; default = 64; }; @@ -148,24 +148,30 @@ in networking.useNetworkd = true; systemd.network.networks."10-uplink" = let - dyn_v4 = cfg.staticIPv4.address == null || cfg.staticIPv4.defaultGateway == null; - dyn_v6 = cfg.staticIPv6.defaultGateway == null; + # IPv4 configuration is obtained by DHCP by default, + # unless a static v4 address and default gateway are given + noDHCP = cfg.staticIPv4.address != null && cfg.staticIPv4.defaultGateway != null; + # IPv6 configuration is obtained through router advertisements + # (RA), using a static token to ensure a static IPv6, + # unless defaultGateway is specified, in which case RAs are + # disabled entirely + noRA = cfg.staticIPv6.defaultGateway != null; in { matchConfig.Name = "en* eth*"; address = - optional (!dyn_v4) "${cfg.staticIPv4.address}" - ++ optional (!dyn_v6) "${cfg.staticIPv6.address}/${toString cfg.staticIPv6.prefixLength}"; + optional noDHCP "${cfg.staticIPv4.address}" + ++ optional noRA "${cfg.staticIPv6.address}/${toString cfg.staticIPv6.prefixLength}"; routes = - optional (!dyn_v4) { + optional noDHCP { routeConfig = { Gateway = cfg.staticIPv4.defaultGateway; # GatewayOnLink - Takes a boolean. If set to true, the kernel does not have to check if the gateway is reachable directly by the current machine (i.e., attached to the local network), so that we can insert the route in the kernel table without it being complained about. Defaults to "no". GatewayOnLink = true; }; - } ++ optional (!dyn_v6) { + } ++ optional noRA { routeConfig = { Gateway = cfg.staticIPv6.defaultGateway; GatewayOnLink = true; @@ -173,18 +179,18 @@ in }; # Dynamic IPv4: enable DHCP but not for DNS servers - networkConfig.DHCP = mkIf dyn_v4 "ipv4"; - dhcpV4Config.UseDNS = mkIf dyn_v4 false; + networkConfig.DHCP = mkIf (!noDHCP) "ipv4"; + dhcpV4Config.UseDNS = mkIf (!noDHCP) false; # Dynamic IPv6: only fetch default route, use static # address and no DNS servers - ipv6AcceptRAConfig.Token = mkIf dyn_v6 "static:${cfg.staticIPv6.address}"; - ipv6AcceptRAConfig.UseDNS = mkIf dyn_v6 false; + ipv6AcceptRAConfig.Token = mkIf (!noRA) "static:${cfg.staticIPv6.address}"; + ipv6AcceptRAConfig.UseDNS = mkIf (!noRA) false; # Static IPv6: disable all router advertisements and # link-local addresses - networkConfig.IPv6AcceptRA = mkIf (!dyn_v6) false; - networkConfig.LinkLocalAddressing = mkIf (!dyn_v6) "no"; + networkConfig.IPv6AcceptRA = mkIf noRA false; + networkConfig.LinkLocalAddressing = mkIf noRA "no"; }; # Configure Unbound as a central DNS server for everything From 16422d280959808aff1670a066788e98049f42b7 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 5 Apr 2023 14:04:11 +0200 Subject: [PATCH 08/20] introduce back static ipv4 prefix lenght but with default value --- cluster/staging/node/df-pw5.nix | 2 +- cluster/staging/node/origan.nix | 2 +- cluster/staging/node/piranha.nix | 2 +- nix/deuxfleurs.nix | 7 ++++++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cluster/staging/node/df-pw5.nix b/cluster/staging/node/df-pw5.nix index e1a132c..2f20f1c 100644 --- a/cluster/staging/node/df-pw5.nix +++ b/cluster/staging/node/df-pw5.nix @@ -10,7 +10,7 @@ boot.loader.timeout = 20; deuxfleurs.hostName = "df-pw5"; - deuxfleurs.staticIPv4.address = "192.168.5.130/24"; + deuxfleurs.staticIPv4.address = "192.168.5.130"; deuxfleurs.staticIPv6.address = "2a02:a03f:6510:5102:223:24ff:feb0:e8a7"; system.stateVersion = "22.11"; diff --git a/cluster/staging/node/origan.nix b/cluster/staging/node/origan.nix index 6906f4d..49ecbbf 100644 --- a/cluster/staging/node/origan.nix +++ b/cluster/staging/node/origan.nix @@ -9,7 +9,7 @@ boot.loader.efi.canTouchEfiVariables = true; deuxfleurs.hostName = "origan"; - deuxfleurs.staticIPv4.address = "192.168.1.33/24"; + deuxfleurs.staticIPv4.address = "192.168.1.33"; deuxfleurs.staticIPv6.address = "2a01:e0a:5e4:1d0:223:24ff:feaf:fdec"; deuxfleurs.isRaftServer = true; diff --git a/cluster/staging/node/piranha.nix b/cluster/staging/node/piranha.nix index bc1aa03..896f169 100644 --- a/cluster/staging/node/piranha.nix +++ b/cluster/staging/node/piranha.nix @@ -9,7 +9,7 @@ boot.loader.efi.canTouchEfiVariables = true; deuxfleurs.hostName = "piranha"; - deuxfleurs.staticIPv4.address = "192.168.1.25/24"; + deuxfleurs.staticIPv4.address = "192.168.1.25"; deuxfleurs.staticIPv6.address = "2a01:cb05:9142:7400:223:24ff:feb0:ea82"; deuxfleurs.isRaftServer = true; diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 57486c2..74a5734 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -37,6 +37,11 @@ in type = nullOr str; default = null; }; + staticIPv4.prefixLength = mkOption { + description = "IPv4 prefix length for LAN addresses, only used with static configuration"; + type = int; + default = 24; + }; staticIPv6.defaultGateway = mkOption { description = '' IPv6 address of the default route on the local network interface. @@ -161,7 +166,7 @@ in matchConfig.Name = "en* eth*"; address = - optional noDHCP "${cfg.staticIPv4.address}" + optional noDHCP "${cfg.staticIPv4.address}/${toString cfg.staticIPv4.prefixLength}" ++ optional noRA "${cfg.staticIPv6.address}/${toString cfg.staticIPv6.prefixLength}"; routes = From c08bc17cc0ca37557a4b43f3ef2e1bcf8a1db2c0 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 5 Apr 2023 14:06:59 +0200 Subject: [PATCH 09/20] Adapt prod config to new parameters --- cluster/prod/cluster.nix | 101 ++++++++++++++----------------- cluster/prod/node/abricot.nix | 11 +--- cluster/prod/node/celeri.nix | 11 +--- cluster/prod/node/concombre.nix | 12 ++-- cluster/prod/node/courgette.nix | 11 +--- cluster/prod/node/dahlia.nix | 12 ++-- cluster/prod/node/df-ykl.nix | 12 ++-- cluster/prod/node/df-ymf.nix | 11 +--- cluster/prod/node/df-ymk.nix | 11 +--- cluster/prod/node/diplotaxis.nix | 11 +--- cluster/prod/node/doradille.nix | 11 +--- cluster/prod/site/bespin.nix | 10 +-- cluster/prod/site/neptune.nix | 12 ++-- cluster/prod/site/orion.nix | 17 +++--- cluster/prod/site/scorpio.nix | 12 ++-- 15 files changed, 98 insertions(+), 167 deletions(-) diff --git a/cluster/prod/cluster.nix b/cluster/prod/cluster.nix index 44c376e..cbeed8f 100644 --- a/cluster/prod/cluster.nix +++ b/cluster/prod/cluster.nix @@ -1,84 +1,73 @@ { config, pkgs, ... } @ args: { - deuxfleurs.cluster_name = "prod"; + deuxfleurs.clusterName = "prod"; # The IP range to use for the Wireguard overlay of this cluster - deuxfleurs.cluster_prefix = "10.83.0.0"; - deuxfleurs.cluster_prefix_length = 16; + deuxfleurs.clusterPrefix = "10.83.0.0/16"; - deuxfleurs.cluster_nodes = [ - { - hostname = "concombre"; - site_name = "neptune"; + deuxfleurs.cluster_nodes = { + "concombre" = { + siteName = "neptune"; publicKey = "VvXT0fPDfWsHxumZqVShpS33dJQAdpJ1E79ZbCBJP34="; - IP = "10.83.1.1"; + address = "10.83.1.1"; endpoint = "77.207.15.215:33731"; - } - { - hostname = "courgette"; - site_name = "neptune"; + }; + "courgette" = { + siteName = "neptune"; publicKey = "goTkBJGmzrGDOAjUcdH9G0JekipqSMoaYQdB6IHnzi0="; - IP = "10.83.1.2"; + address = "10.83.1.2"; endpoint = "77.207.15.215:33732"; - } - { - hostname = "celeri"; - site_name = "neptune"; + }; + "celeri" = { + siteName = "neptune"; publicKey = "oZDAb8LoLW87ktUHyFFec0VaIar97bqq47mGbdVqJ0U="; - IP = "10.83.1.3"; + address = "10.83.1.3"; endpoint = "77.207.15.215:33733"; - } - { - hostname = "dahlia"; - site_name = "orion"; + }; + "dahlia" = { + siteName = "orion"; publicKey = "EtRoWBYCdjqgXX0L+uWLg8KxNfIK8k9OTh30tL19bXU="; - IP = "10.83.2.1"; + address = "10.83.2.1"; endpoint = "82.66.80.201:33731"; - } - { - hostname = "diplotaxis"; - site_name = "orion"; + }; + "diplotaxis" = { + siteName = "orion"; publicKey = "HbLC938mysadMSOxWgq8+qrv+dBKzPP/43OMJp/3phA="; - IP = "10.83.2.2"; + address = "10.83.2.2"; endpoint = "82.66.80.201:33732"; - } - { - hostname = "doradille"; - site_name = "orion"; + }; + "doradille" = { + siteName = "orion"; publicKey = "e1C8jgTj9eD20ywG08G1FQZ+Js3wMK/msDUE1wO3l1Y="; - IP = "10.83.2.3"; + address = "10.83.2.3"; endpoint = "82.66.80.201:33733"; - } - { - hostname = "df-ykl"; - site_name = "bespin"; + }; + "df-ykl" = { + siteName = "bespin"; publicKey = "bIjxey/VhBgVrLa0FxN/KISOt2XFmQeSh1MPivUq9gg="; - IP = "10.83.3.1"; + address = "10.83.3.1"; endpoint = "109.136.55.235:33731"; - } - { - hostname = "df-ymf"; - site_name = "bespin"; + }; + "df-ymf" = { + siteName = "bespin"; publicKey = "pUIKv8UBl586O7DBrHBsb9BgNU7WlYQ2r2RSNkD+JAQ="; - IP = "10.83.3.2"; + address = "10.83.3.2"; endpoint = "109.136.55.235:33732"; - } - { - hostname = "df-ymk"; - site_name = "bespin"; + }; + "df-ymk" = { + siteName = "bespin"; publicKey = "VBmpo15iIJP7250NAsF+ryhZc3j+8TZFnE1Djvn5TXI="; - IP = "10.83.3.3"; + address = "10.83.3.3"; endpoint = "109.136.55.235:33733"; - } - { - hostname = "abricot"; - site_name = "scorpio"; + }; + "abricot" = { + siteName = "scorpio"; publicKey = "Sm9cmNZ/BfWVPFflMO+fuyiera4r203b/dKhHTQmBFg="; - IP = "10.83.4.1"; + address = "10.83.4.1"; endpoint = "82.65.41.110:33741"; - } - ]; + }; + }; # Bootstrap IPs for Consul cluster, # these are IPs on the Wireguard overlay @@ -88,7 +77,7 @@ "10.83.3.1" # df-ykl ]; - deuxfleurs.admin_accounts = { + deuxfleurs.adminAccounts = { lx = [ # Keys for accessing nodes from outside "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy" diff --git a/cluster/prod/node/abricot.nix b/cluster/prod/node/abricot.nix index b092fb2..69cc38d 100644 --- a/cluster/prod/node/abricot.nix +++ b/cluster/prod/node/abricot.nix @@ -8,12 +8,7 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "abricot"; - - deuxfleurs.network_interface = "eno1"; - deuxfleurs.lan_ip = "192.168.1.41"; - deuxfleurs.ipv6 = "2a01:e0a:e4:2dd0::41"; - - deuxfleurs.cluster_ip = "10.83.4.1"; - deuxfleurs.is_raft_server = false; + deuxfleurs.hostName = "abricot"; + deuxfleurs.staticIPv4.address = "192.168.1.41"; + deuxfleurs.staticIPv6.address = "2a01:e0a:e4:2dd0::41"; } diff --git a/cluster/prod/node/celeri.nix b/cluster/prod/node/celeri.nix index fdb88b9..45087f3 100644 --- a/cluster/prod/node/celeri.nix +++ b/cluster/prod/node/celeri.nix @@ -8,12 +8,7 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "celeri"; - - deuxfleurs.network_interface = "enp0s31f6"; - deuxfleurs.lan_ip = "192.168.1.33"; - deuxfleurs.ipv6 = "2001:910:1204:1::33"; - - deuxfleurs.cluster_ip = "10.83.1.3"; - deuxfleurs.is_raft_server = false; + deuxfleurs.hostName = "celeri"; + deuxfleurs.staticIPv4.address = "192.168.1.33"; + deuxfleurs.staticIPv6.address = "2001:910:1204:1::33"; } diff --git a/cluster/prod/node/concombre.nix b/cluster/prod/node/concombre.nix index eefc9e7..9a9e456 100644 --- a/cluster/prod/node/concombre.nix +++ b/cluster/prod/node/concombre.nix @@ -8,12 +8,8 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "concombre"; - - deuxfleurs.network_interface = "enp0s31f6"; - deuxfleurs.lan_ip = "192.168.1.31"; - deuxfleurs.ipv6 = "2001:910:1204:1::31"; - - deuxfleurs.cluster_ip = "10.83.1.1"; - deuxfleurs.is_raft_server = true; + deuxfleurs.hostName = "concombre"; + deuxfleurs.staticIPv4.address = "192.168.1.31"; + deuxfleurs.staticIPv6.address = "2001:910:1204:1::31"; + deuxfleurs.isRaftServer = true; } diff --git a/cluster/prod/node/courgette.nix b/cluster/prod/node/courgette.nix index c2bf0d8..081c297 100644 --- a/cluster/prod/node/courgette.nix +++ b/cluster/prod/node/courgette.nix @@ -8,12 +8,7 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "courgette"; - - deuxfleurs.network_interface = "enp0s31f6"; - deuxfleurs.lan_ip = "192.168.1.32"; - deuxfleurs.ipv6 = "2001:910:1204:1::32"; - - deuxfleurs.cluster_ip = "10.83.1.2"; - deuxfleurs.is_raft_server = false; + deuxfleurs.hostName = "courgette"; + deuxfleurs.staticIPv4.address = "192.168.1.32"; + deuxfleurs.staticIPv6.address = "2001:910:1204:1::32"; } diff --git a/cluster/prod/node/dahlia.nix b/cluster/prod/node/dahlia.nix index fc51ea8..ee9e7aa 100644 --- a/cluster/prod/node/dahlia.nix +++ b/cluster/prod/node/dahlia.nix @@ -7,12 +7,8 @@ boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "dahlia"; - - deuxfleurs.network_interface = "enp0s31f6"; - deuxfleurs.lan_ip = "192.168.1.11"; - deuxfleurs.ipv6 = "2a01:e0a:28f:5e60::11"; - - deuxfleurs.cluster_ip = "10.83.2.1"; - deuxfleurs.is_raft_server = true; + deuxfleurs.hostName = "dahlia"; + deuxfleurs.staticIPv4.address = "192.168.1.11"; + deuxfleurs.staticIPv6.address = "2a01:e0a:28f:5e60::11"; + deuxfleurs.isRaftServer = true; } diff --git a/cluster/prod/node/df-ykl.nix b/cluster/prod/node/df-ykl.nix index 04a2b35..843d322 100644 --- a/cluster/prod/node/df-ykl.nix +++ b/cluster/prod/node/df-ykl.nix @@ -7,14 +7,10 @@ boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "df-ykl"; - - deuxfleurs.network_interface = "enp0s31f6"; - deuxfleurs.lan_ip = "192.168.5.117"; - deuxfleurs.ipv6 = "2a02:a03f:6510:5102:6e4b:90ff:fe3b:e86c"; - - deuxfleurs.cluster_ip = "10.83.3.1"; - deuxfleurs.is_raft_server = true; + deuxfleurs.hostName = "df-ykl"; + deuxfleurs.staticIPv4.address = "192.168.5.117"; + deuxfleurs.staticIPv6.address = "2a02:a03f:6510:5102:6e4b:90ff:fe3b:e86c"; + deuxfleurs.isRaftServer = true; fileSystems."/mnt" = { device = "/dev/disk/by-uuid/f7aa396f-23d0-44d3-89cf-3cb00bbb6c3b"; diff --git a/cluster/prod/node/df-ymf.nix b/cluster/prod/node/df-ymf.nix index 15c5b1e..df2ebb3 100644 --- a/cluster/prod/node/df-ymf.nix +++ b/cluster/prod/node/df-ymf.nix @@ -7,14 +7,9 @@ boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "df-ymf"; - - deuxfleurs.network_interface = "enp0s31f6"; - deuxfleurs.lan_ip = "192.168.5.134"; - deuxfleurs.ipv6 = "2a02:a03f:6510:5102:6e4b:90ff:fe3a:6174"; - - deuxfleurs.cluster_ip = "10.83.3.2"; - deuxfleurs.is_raft_server = false; + deuxfleurs.hostName = "df-ymf"; + deuxfleurs.staticIPv4.address = "192.168.5.134"; + deuxfleurs.staticIPv6.address = "2a02:a03f:6510:5102:6e4b:90ff:fe3a:6174"; fileSystems."/mnt" = { device = "/dev/disk/by-uuid/fec20a7e-5019-4747-8f73-77f3f196c122"; diff --git a/cluster/prod/node/df-ymk.nix b/cluster/prod/node/df-ymk.nix index d7deb49..f98b576 100644 --- a/cluster/prod/node/df-ymk.nix +++ b/cluster/prod/node/df-ymk.nix @@ -7,14 +7,9 @@ boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "df-ymk"; - - deuxfleurs.network_interface = "enp0s31f6"; - deuxfleurs.lan_ip = "192.168.5.116"; - deuxfleurs.ipv6 = "2a02:a03f:6510:5102:6e4b:90ff:fe3b:e939"; - - deuxfleurs.cluster_ip = "10.83.3.3"; - deuxfleurs.is_raft_server = false; + deuxfleurs.hostName = "df-ymk"; + deuxfleurs.staticIPv4.address = "192.168.5.116"; + deuxfleurs.staticIPv6.address = "2a02:a03f:6510:5102:6e4b:90ff:fe3b:e939"; fileSystems."/mnt" = { device = "/dev/disk/by-uuid/51d95b17-0e06-4a73-9e4e-ae5363cc4015"; diff --git a/cluster/prod/node/diplotaxis.nix b/cluster/prod/node/diplotaxis.nix index c1ce4f9..f9c7faf 100644 --- a/cluster/prod/node/diplotaxis.nix +++ b/cluster/prod/node/diplotaxis.nix @@ -8,12 +8,7 @@ boot.loader.grub.version = 2; boot.loader.grub.device = "/dev/nvme0n1"; # or "nodev" for efi only - networking.hostName = "diplotaxis"; - - deuxfleurs.network_interface = "enp0s31f6"; - deuxfleurs.lan_ip = "192.168.1.12"; - deuxfleurs.ipv6 = "2a01:e0a:28f:5e60::12"; - - deuxfleurs.cluster_ip = "10.83.2.2"; - deuxfleurs.is_raft_server = false; + deuxfleurs.hostName = "diplotaxis"; + deuxfleurs.staticIPv4.address = "192.168.1.12"; + deuxfleurs.staticIPv6.address = "2a01:e0a:28f:5e60::12"; } diff --git a/cluster/prod/node/doradille.nix b/cluster/prod/node/doradille.nix index f1c6e57..a4dc691 100644 --- a/cluster/prod/node/doradille.nix +++ b/cluster/prod/node/doradille.nix @@ -8,12 +8,7 @@ boot.loader.grub.version = 2; boot.loader.grub.device = "/dev/nvme0n1"; # or "nodev" for efi only - networking.hostName = "doradille"; - - deuxfleurs.network_interface = "enp0s31f6"; - deuxfleurs.lan_ip = "192.168.1.13"; - deuxfleurs.ipv6 = "2a01:e0a:28f:5e60::13"; - - deuxfleurs.cluster_ip = "10.83.2.3"; - deuxfleurs.is_raft_server = false; + deuxfleurs.hostName = "doradille"; + deuxfleurs.staticIPv4.address = "192.168.1.13"; + deuxfleurs.staticIPv6.address = "2a01:e0a:28f:5e60::13"; } diff --git a/cluster/prod/site/bespin.nix b/cluster/prod/site/bespin.nix index de39f85..3c9a668 100644 --- a/cluster/prod/site/bespin.nix +++ b/cluster/prod/site/bespin.nix @@ -1,13 +1,9 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "bespin"; - deuxfleurs.lan_default_gateway = "192.168.5.254"; - deuxfleurs.ipv6_default_gateway = "2a02:a03f:6510:5102::1"; - deuxfleurs.lan_ip_prefix_length = 24; - deuxfleurs.ipv6_prefix_length = 64; - deuxfleurs.nameservers = [ "192.168.5.254" ]; - deuxfleurs.cname_target = "bespin.site.deuxfleurs.fr."; + deuxfleurs.siteName = "bespin"; + deuxfleurs.staticIPv4.defaultGateway = "192.168.5.254"; + deuxfleurs.cnameTarget = "bespin.site.deuxfleurs.fr."; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/prod/site/neptune.nix b/cluster/prod/site/neptune.nix index a4aac6d..81495c6 100644 --- a/cluster/prod/site/neptune.nix +++ b/cluster/prod/site/neptune.nix @@ -1,14 +1,10 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "neptune"; - deuxfleurs.lan_default_gateway = "192.168.1.1"; - deuxfleurs.ipv6_default_gateway = "2001:910:1204:1::1"; - deuxfleurs.lan_ip_prefix_length = 24; - deuxfleurs.ipv6_prefix_length = 64; - deuxfleurs.nameservers = [ "192.168.1.1" ]; - deuxfleurs.cname_target = "neptune.site.deuxfleurs.fr."; - deuxfleurs.public_ipv4 = "77.207.15.215"; + deuxfleurs.siteName = "neptune"; + deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; + deuxfleurs.cnameTarget = "neptune.site.deuxfleurs.fr."; + deuxfleurs.publicIPv4 = "77.207.15.215"; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/prod/site/orion.nix b/cluster/prod/site/orion.nix index fb4ba22..5f6c33e 100644 --- a/cluster/prod/site/orion.nix +++ b/cluster/prod/site/orion.nix @@ -1,14 +1,15 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "orion"; - deuxfleurs.lan_default_gateway = "192.168.1.254"; - deuxfleurs.ipv6_default_gateway = "2a01:e0a:28f:5e60::1"; - deuxfleurs.lan_ip_prefix_length = 24; - deuxfleurs.ipv6_prefix_length = 64; - deuxfleurs.nameservers = [ "192.168.1.254" ]; - deuxfleurs.cname_target = "orion.site.deuxfleurs.fr."; - deuxfleurs.public_ipv4 = "82.66.80.201"; + deuxfleurs.siteName = "orion"; + deuxfleurs.staticIPv4.defaultGateway = "192.168.1.254"; + # Setting an IPv6 default gateway will disable RA for now. + # Adding this for now as Orion has the mail servers and we are + # not yet confident we can disable this without getting ourselves + # banned by sending from unwanted IPs (although it should be ok). + deuxfleurs.staticIPv6.defaultGateway = "2a01:e0a:28f:5e60::1"; + deuxfleurs.cnameTarget = "orion.site.deuxfleurs.fr."; + deuxfleurs.publicIPv4 = "82.66.80.201"; networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/prod/site/scorpio.nix b/cluster/prod/site/scorpio.nix index b58e25c..b1e0f20 100644 --- a/cluster/prod/site/scorpio.nix +++ b/cluster/prod/site/scorpio.nix @@ -1,14 +1,10 @@ { config, pkgs, ... }: { - deuxfleurs.site_name = "scorpio"; - deuxfleurs.lan_default_gateway = "192.168.1.254"; - deuxfleurs.ipv6_default_gateway = "2a01:e0a:e4:2dd0::1"; - deuxfleurs.lan_ip_prefix_length = 24; - deuxfleurs.ipv6_prefix_length = 64; - deuxfleurs.nameservers = [ "192.168.1.254" ]; - deuxfleurs.cname_target = "scorpio.site.deuxfleurs.fr."; - deuxfleurs.public_ipv4 = "82.65.41.110"; + deuxfleurs.siteName = "scorpio"; + deuxfleurs.staticIPv4.defaultGateway = "192.168.1.254"; + deuxfleurs.cnameTarget = "scorpio.site.deuxfleurs.fr."; + deuxfleurs.publicIPv4 = "82.65.41.110"; networking.firewall.allowedTCPPorts = [ 80 443 ]; } From 07f50f297a8fcb3ccee167ca486084bafb2c1120 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 5 Apr 2023 16:30:28 +0200 Subject: [PATCH 10/20] D53 with addresses from DiploNAT autodiscovery; diplonat fw opening for tricot --- cluster/staging/app/core/deploy/core-service.hcl | 2 +- cluster/staging/app/core/deploy/core-system.hcl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/staging/app/core/deploy/core-service.hcl b/cluster/staging/app/core/deploy/core-service.hcl index 6799e01..bf835c7 100644 --- a/cluster/staging/app/core/deploy/core-service.hcl +++ b/cluster/staging/app/core/deploy/core-service.hcl @@ -11,7 +11,7 @@ job "core-service" { config { packages = [ - "git+https://git.deuxfleurs.fr/lx/D53.git?ref=main&rev=86c255dfeabc60b0ef46ff78bc487c61c9548c79" + "git+https://git.deuxfleurs.fr/lx/D53.git?ref=diplonat-autodiscovery&rev=d906a6ebb5d977f44340b157a520477849ced161" ] command = "d53" } diff --git a/cluster/staging/app/core/deploy/core-system.hcl b/cluster/staging/app/core/deploy/core-system.hcl index dba6b95..2ec9b58 100644 --- a/cluster/staging/app/core/deploy/core-system.hcl +++ b/cluster/staging/app/core/deploy/core-system.hcl @@ -140,7 +140,7 @@ EOH tags = [ "d53-aaaa ${meta.site}.site.staging.deuxfleurs.org", "d53-aaaa staging.deuxfleurs.org", - # "(diplonat (tcp_port 80))" + "(diplonat (tcp_port 80))" ] address_mode = "host" } @@ -149,7 +149,7 @@ EOH name = "tricot-https" port = "https_port" tags = [ - # "(diplonat (tcp_port 443))" + "(diplonat (tcp_port 443))" ] address_mode = "host" } From 0372df95b5689d5104131ccc75f05d791c6e6a23 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 19 Apr 2023 20:36:24 +0200 Subject: [PATCH 11/20] staging: fix consul server addresses --- cluster/staging/cluster.nix | 4 ++-- cluster/staging/ssh_config | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix index 56ca904..2fec94c 100644 --- a/cluster/staging/cluster.nix +++ b/cluster/staging/cluster.nix @@ -43,9 +43,9 @@ # Bootstrap IPs for Consul cluster, # these are IPs on the Wireguard overlay services.consul.extraConfig.retry_join = [ - "10.14.1.1" # cariacou - "10.14.1.2" # carcajou "10.14.1.3" # caribou + "10.14.2.33" # origan + "10.14.3.1" # piranha ]; deuxfleurs.adminAccounts = { diff --git a/cluster/staging/ssh_config b/cluster/staging/ssh_config index 3043207..08cac54 100644 --- a/cluster/staging/ssh_config +++ b/cluster/staging/ssh_config @@ -10,7 +10,7 @@ Host origan HostName origan.df.trinity.fr.eu.org Host piranha - ProxyJump caribou.machine.deuxfleurs.fr + ProxyJump carcajou.machine.deuxfleurs.fr HostName 10.14.3.1 #HostName piranha.polyno.me From e5f9f3c8495a8f0e6b9c46b56d207b57aad3ec3d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 19 Apr 2023 21:05:47 +0200 Subject: [PATCH 12/20] increase diplonat ram --- cluster/staging/app/core/deploy/core-system.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/staging/app/core/deploy/core-system.hcl b/cluster/staging/app/core/deploy/core-system.hcl index 2ec9b58..1d3eb84 100644 --- a/cluster/staging/app/core/deploy/core-system.hcl +++ b/cluster/staging/app/core/deploy/core-system.hcl @@ -65,7 +65,7 @@ EOH } resources { - memory = 40 + memory = 100 } } } From b4e82e37e4e1718dfffa70cd0c6222c1b34fc997 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 20 Apr 2023 15:13:13 +0200 Subject: [PATCH 13/20] diplonat with fixed iptables thing --- .../core/deploy/{core-service.hcl => d53.hcl} | 2 +- cluster/staging/app/core/deploy/diplonat.hcl | 75 +++++++++++++++++++ .../deploy/{core-system.hcl => tricot.hcl} | 60 +-------------- 3 files changed, 78 insertions(+), 59 deletions(-) rename cluster/staging/app/core/deploy/{core-service.hcl => d53.hcl} (98%) create mode 100644 cluster/staging/app/core/deploy/diplonat.hcl rename cluster/staging/app/core/deploy/{core-system.hcl => tricot.hcl} (64%) diff --git a/cluster/staging/app/core/deploy/core-service.hcl b/cluster/staging/app/core/deploy/d53.hcl similarity index 98% rename from cluster/staging/app/core/deploy/core-service.hcl rename to cluster/staging/app/core/deploy/d53.hcl index bf835c7..5d57eb3 100644 --- a/cluster/staging/app/core/deploy/core-service.hcl +++ b/cluster/staging/app/core/deploy/d53.hcl @@ -1,4 +1,4 @@ -job "core-service" { +job "core:d53" { datacenters = ["neptune", "jupiter", "corrin", "bespin"] type = "service" priority = 90 diff --git a/cluster/staging/app/core/deploy/diplonat.hcl b/cluster/staging/app/core/deploy/diplonat.hcl new file mode 100644 index 0000000..ba1e4b5 --- /dev/null +++ b/cluster/staging/app/core/deploy/diplonat.hcl @@ -0,0 +1,75 @@ +job "core:diplonat" { + datacenters = ["neptune", "jupiter", "corrin", "bespin"] + type = "system" + priority = 90 + + constraint { + attribute = "${attr.cpu.arch}" + value = "amd64" + } + + update { + max_parallel = 3 + stagger = "20s" + } + + group "diplonat" { + task "diplonat" { + driver = "nix2" + + config { + packages = [ + "#iptables", + "#bash", + "#coreutils", + "git+https://git.deuxfleurs.fr/Deuxfleurs/diplonat.git?ref=stun&rev=f5fc635b75dfa17b83a8db4893a7be206b4f9892" + ] + command = "diplonat" + } + user = "root" + + restart { + interval = "30m" + attempts = 2 + delay = "15s" + mode = "delay" + } + + template { + data = "{{ key \"secrets/consul/consul-ca.crt\" }}" + destination = "etc/diplonat/consul-ca.crt" + } + + template { + data = "{{ key \"secrets/consul/consul-client.crt\" }}" + destination = "etc/diplonat/consul-client.crt" + } + + template { + data = "{{ key \"secrets/consul/consul-client.key\" }}" + destination = "etc/diplonat/consul-client.key" + } + + template { + data = < Date: Fri, 21 Apr 2023 11:29:15 +0200 Subject: [PATCH 14/20] Fix unbound; remove Nixos firewall (use only diplonat) --- cluster/prod/cluster.nix | 2 +- cluster/prod/site/bespin.nix | 2 -- cluster/prod/site/neptune.nix | 2 -- cluster/prod/site/orion.nix | 2 -- cluster/prod/site/scorpio.nix | 2 -- cluster/staging/site/bespin.nix | 2 -- cluster/staging/site/corrin.nix | 2 -- cluster/staging/site/jupiter.nix | 2 -- cluster/staging/site/neptune.nix | 2 -- nix/deuxfleurs.nix | 2 ++ 10 files changed, 3 insertions(+), 17 deletions(-) diff --git a/cluster/prod/cluster.nix b/cluster/prod/cluster.nix index cbeed8f..ea3bdec 100644 --- a/cluster/prod/cluster.nix +++ b/cluster/prod/cluster.nix @@ -6,7 +6,7 @@ # The IP range to use for the Wireguard overlay of this cluster deuxfleurs.clusterPrefix = "10.83.0.0/16"; - deuxfleurs.cluster_nodes = { + deuxfleurs.clusterNodes = { "concombre" = { siteName = "neptune"; publicKey = "VvXT0fPDfWsHxumZqVShpS33dJQAdpJ1E79ZbCBJP34="; diff --git a/cluster/prod/site/bespin.nix b/cluster/prod/site/bespin.nix index 3c9a668..cdce53e 100644 --- a/cluster/prod/site/bespin.nix +++ b/cluster/prod/site/bespin.nix @@ -4,6 +4,4 @@ deuxfleurs.siteName = "bespin"; deuxfleurs.staticIPv4.defaultGateway = "192.168.5.254"; deuxfleurs.cnameTarget = "bespin.site.deuxfleurs.fr."; - - networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/prod/site/neptune.nix b/cluster/prod/site/neptune.nix index 81495c6..ab24f4a 100644 --- a/cluster/prod/site/neptune.nix +++ b/cluster/prod/site/neptune.nix @@ -5,6 +5,4 @@ deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; deuxfleurs.cnameTarget = "neptune.site.deuxfleurs.fr."; deuxfleurs.publicIPv4 = "77.207.15.215"; - - networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/prod/site/orion.nix b/cluster/prod/site/orion.nix index 5f6c33e..58c49ab 100644 --- a/cluster/prod/site/orion.nix +++ b/cluster/prod/site/orion.nix @@ -10,6 +10,4 @@ deuxfleurs.staticIPv6.defaultGateway = "2a01:e0a:28f:5e60::1"; deuxfleurs.cnameTarget = "orion.site.deuxfleurs.fr."; deuxfleurs.publicIPv4 = "82.66.80.201"; - - networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/prod/site/scorpio.nix b/cluster/prod/site/scorpio.nix index b1e0f20..e36dc1d 100644 --- a/cluster/prod/site/scorpio.nix +++ b/cluster/prod/site/scorpio.nix @@ -5,6 +5,4 @@ deuxfleurs.staticIPv4.defaultGateway = "192.168.1.254"; deuxfleurs.cnameTarget = "scorpio.site.deuxfleurs.fr."; deuxfleurs.publicIPv4 = "82.65.41.110"; - - networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/staging/site/bespin.nix b/cluster/staging/site/bespin.nix index 22feb59..2dbfbad 100644 --- a/cluster/staging/site/bespin.nix +++ b/cluster/staging/site/bespin.nix @@ -4,6 +4,4 @@ deuxfleurs.siteName = "bespin"; deuxfleurs.staticIPv4.defaultGateway = "192.168.5.254"; deuxfleurs.cnameTarget = "bespin.site.staging.deuxfleurs.org."; - - networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/staging/site/corrin.nix b/cluster/staging/site/corrin.nix index 6eb5239..027f6b3 100644 --- a/cluster/staging/site/corrin.nix +++ b/cluster/staging/site/corrin.nix @@ -5,6 +5,4 @@ deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; deuxfleurs.cnameTarget = "corrin.site.staging.deuxfleurs.org."; deuxfleurs.publicIPv4 = "2.13.96.213"; - - networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/staging/site/jupiter.nix b/cluster/staging/site/jupiter.nix index 2d39f5a..28ba297 100644 --- a/cluster/staging/site/jupiter.nix +++ b/cluster/staging/site/jupiter.nix @@ -4,6 +4,4 @@ deuxfleurs.siteName = "jupiter"; deuxfleurs.staticIPv4.defaultGateway = "192.168.1.1"; deuxfleurs.cnameTarget = "jupiter.site.staging.deuxfleurs.org."; - - networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/cluster/staging/site/neptune.nix b/cluster/staging/site/neptune.nix index f94d62f..86148f4 100644 --- a/cluster/staging/site/neptune.nix +++ b/cluster/staging/site/neptune.nix @@ -3,6 +3,4 @@ { deuxfleurs.siteName = "neptune"; deuxfleurs.cnameTarget = "neptune.site.staging.deuxfleurs.org."; - - networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 74a5734..4423318 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -213,6 +213,8 @@ in access-control = [ "127.0.0.0/8 allow" "172.17.0.0/16 allow" + "192.168.0.0/16 allow" + "${cfg.clusterPrefix} allow" ]; }; stub-zone = [ From 0b3332fd3234a13fc5d780f94a74133d1e7ba199 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 21 Apr 2023 11:55:24 +0200 Subject: [PATCH 15/20] break out core services into separate files --- cluster/prod/app/core/deploy/bottin.hcl | 100 +++++++ cluster/prod/app/core/deploy/core-system.hcl | 257 ------------------ .../core/deploy/{core-service.hcl => d53.hcl} | 2 +- cluster/prod/app/core/deploy/diplonat.hcl | 68 +++++ cluster/prod/app/core/deploy/tricot.hcl | 109 ++++++++ 5 files changed, 278 insertions(+), 258 deletions(-) create mode 100644 cluster/prod/app/core/deploy/bottin.hcl delete mode 100644 cluster/prod/app/core/deploy/core-system.hcl rename cluster/prod/app/core/deploy/{core-service.hcl => d53.hcl} (98%) create mode 100644 cluster/prod/app/core/deploy/diplonat.hcl create mode 100644 cluster/prod/app/core/deploy/tricot.hcl diff --git a/cluster/prod/app/core/deploy/bottin.hcl b/cluster/prod/app/core/deploy/bottin.hcl new file mode 100644 index 0000000..40bb5af --- /dev/null +++ b/cluster/prod/app/core/deploy/bottin.hcl @@ -0,0 +1,100 @@ +job "core:bottin" { + datacenters = ["orion", "neptune", "scorpio"] + type = "system" + priority = 90 + + update { + max_parallel = 1 + stagger = "1m" + } + + group "bottin" { + constraint { + distinct_property = "${meta.site}" + value = "1" + } + + network { + port "ldap_port" { + static = 389 + to = 389 + } + } + + task "bottin" { + driver = "docker" + config { + image = "dxflrs/bottin:7h18i30cckckaahv87d3c86pn4a7q41z" + network_mode = "host" + readonly_rootfs = true + ports = [ "ldap_port" ] + volumes = [ + "secrets/config.json:/config.json", + "secrets:/etc/bottin", + ] + } + + restart { + interval = "5m" + attempts = 10 + delay = "15s" + mode = "delay" + } + + resources { + memory = 100 + memory_max = 200 + } + + template { + data = file("../config/bottin/config.json.tpl") + destination = "secrets/config.json" + } + + template { + data = "{{ key \"secrets/consul/consul.crt\" }}" + destination = "secrets/consul.crt" + } + + template { + data = "{{ key \"secrets/consul/consul-client.crt\" }}" + destination = "secrets/consul-client.crt" + } + + template { + data = "{{ key \"secrets/consul/consul-client.key\" }}" + destination = "secrets/consul-client.key" + } + + template { + data = < Date: Fri, 21 Apr 2023 12:03:35 +0200 Subject: [PATCH 16/20] Diplonat on bespin, ipv6-only --- cluster/prod/app/core/deploy/diplonat.hcl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cluster/prod/app/core/deploy/diplonat.hcl b/cluster/prod/app/core/deploy/diplonat.hcl index bf56fd5..d6f8423 100644 --- a/cluster/prod/app/core/deploy/diplonat.hcl +++ b/cluster/prod/app/core/deploy/diplonat.hcl @@ -1,5 +1,5 @@ job "core:diplonat" { - datacenters = ["orion", "neptune", "scorpio"] + datacenters = ["orion", "neptune", "scorpio", "bespin"] type = "system" priority = 90 @@ -53,6 +53,9 @@ DIPLONAT_CONSUL_URL=https://consul.service.prod.consul:8501 DIPLONAT_CONSUL_TLS_SKIP_VERIFY=true DIPLONAT_CONSUL_CLIENT_CERT=/etc/diplonat/consul-client.crt DIPLONAT_CONSUL_CLIENT_KEY=/etc/diplonat/consul-client.key +{{ if env "meta.site" | eq "bespin" }} +DIPLONAT_IPV6_ONLY=true +{{ end }} RUST_LOG=debug EOH destination = "secrets/env" From 607add3161f9a465fc061cfd3a5434aa4dbd4796 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 21 Apr 2023 14:36:10 +0200 Subject: [PATCH 17/20] make specifying an ipv6 fully optionnal --- cluster/prod/app/garage/config/garage.toml | 6 ++++-- cluster/staging/app/core/deploy/d53.hcl | 4 ++-- cluster/staging/app/garage/config/garage.toml | 6 ++++-- cluster/staging/app/garage/deploy/garage.hcl | 1 + nix/deuxfleurs.nix | 19 +++++++++++-------- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cluster/prod/app/garage/config/garage.toml b/cluster/prod/app/garage/config/garage.toml index 36daa5d..5b10707 100644 --- a/cluster/prod/app/garage/config/garage.toml +++ b/cluster/prod/app/garage/config/garage.toml @@ -6,8 +6,10 @@ db_engine = "lmdb" replication_mode = "3" -rpc_bind_addr = "[{{ env "meta.public_ipv6" }}]:3901" -rpc_public_addr = "[{{ env "meta.public_ipv6" }}]:3901" +{{ with $a := env "attr.unique.hostname" | printf "diplonat/autodiscovery/ipv6/%s" | key | parseJSON }} +rpc_bind_addr = "[{{ $a.address }}]:3901" +rpc_public_addr = "[{{ $a.address }}]:3901" +{{ end }} rpc_secret = "{{ key "secrets/garage/rpc_secret" | trimSpace }}" [consul_discovery] diff --git a/cluster/staging/app/core/deploy/d53.hcl b/cluster/staging/app/core/deploy/d53.hcl index 5d57eb3..fb1c1bf 100644 --- a/cluster/staging/app/core/deploy/d53.hcl +++ b/cluster/staging/app/core/deploy/d53.hcl @@ -11,7 +11,7 @@ job "core:d53" { config { packages = [ - "git+https://git.deuxfleurs.fr/lx/D53.git?ref=diplonat-autodiscovery&rev=d906a6ebb5d977f44340b157a520477849ced161" + "git+https://git.deuxfleurs.fr/lx/D53.git?ref=diplonat-autodiscovery&rev=49d94dae1d753c1f3349be7ea9bc7e7978c0af15" ] command = "d53" } @@ -52,7 +52,7 @@ D53_CONSUL_CLIENT_KEY=/etc/tricot/consul-client.key D53_PROVIDERS=deuxfleurs.org:gandi D53_GANDI_API_KEY={{ key "secrets/d53/gandi_api_key" }} D53_ALLOWED_DOMAINS=staging.deuxfleurs.org -RUST_LOG=d53=info +RUST_LOG=d53=debug EOH destination = "secrets/env" env = true diff --git a/cluster/staging/app/garage/config/garage.toml b/cluster/staging/app/garage/config/garage.toml index f14a602..26e0361 100644 --- a/cluster/staging/app/garage/config/garage.toml +++ b/cluster/staging/app/garage/config/garage.toml @@ -6,8 +6,10 @@ db_engine = "lmdb" replication_mode = "3" -rpc_bind_addr = "[{{ env "meta.public_ipv6" }}]:3991" -rpc_public_addr = "[{{ env "meta.public_ipv6" }}]:3991" +{{ with $a := env "attr.unique.hostname" | printf "diplonat/autodiscovery/ipv6/%s" | key | parseJSON }} +rpc_bind_addr = "[{{ $a.address }}]:3991" +rpc_public_addr = "[{{ $a.address }}]:3991" +{{ end }} rpc_secret = "{{ key "secrets/garage-staging/rpc_secret" | trimSpace }}" bootstrap_peers = [] diff --git a/cluster/staging/app/garage/deploy/garage.hcl b/cluster/staging/app/garage/deploy/garage.hcl index 6e37d82..1fc969b 100644 --- a/cluster/staging/app/garage/deploy/garage.hcl +++ b/cluster/staging/app/garage/deploy/garage.hcl @@ -25,6 +25,7 @@ job "garage-staging" { config { packages = [ "#bash", # so that we can enter a shell inside container + "#coreutils", "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?ref=main&rev=0d0906b066eb76111f3b427dce1c50eac083366c", ] command = "garage" diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 4423318..0f5c185 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -19,7 +19,7 @@ in }; staticIPv6.address = mkOption { description = "Static public IPv6 address of this node"; - type = str; + type = nullOr str; }; isRaftServer = mkOption { description = "Make this node a RAFT server for the Nomad and Consul deployments"; @@ -129,8 +129,10 @@ in clusterAddress = clusterNodeCfg.address; node_meta = { "site" = cfg.siteName; - "public_ipv6" = cfg.staticIPv6.address; } // + (if cfg.staticIPv6.address != null + then { "public_ipv6" = cfg.staticIPv6.address; } + else {}) // (if cfg.publicIPv4 != null then { "public_ipv4" = cfg.publicIPv4; } else {}) // @@ -156,11 +158,12 @@ in # IPv4 configuration is obtained by DHCP by default, # unless a static v4 address and default gateway are given noDHCP = cfg.staticIPv4.address != null && cfg.staticIPv4.defaultGateway != null; - # IPv6 configuration is obtained through router advertisements - # (RA), using a static token to ensure a static IPv6, - # unless defaultGateway is specified, in which case RAs are - # disabled entirely - noRA = cfg.staticIPv6.defaultGateway != null; + # IPv6 configuration is obtained through router advertisements (RA), + # possibly using a static token to ensure a static IPv6, + # unless a static v6 address and default gateway are given, + # in which case RAs are disabled entirely + noRA = cfg.staticIPv6.address != null && cfg.staticIPv6.defaultGateway != null; + staticV6 = cfg.staticIPv6.address != null; in { matchConfig.Name = "en* eth*"; @@ -189,7 +192,7 @@ in # Dynamic IPv6: only fetch default route, use static # address and no DNS servers - ipv6AcceptRAConfig.Token = mkIf (!noRA) "static:${cfg.staticIPv6.address}"; + ipv6AcceptRAConfig.Token = mkIf (!noRA && staticV6) "static:${cfg.staticIPv6.address}"; ipv6AcceptRAConfig.UseDNS = mkIf (!noRA) false; # Static IPv6: disable all router advertisements and From 6c07a429781d4a26a546e3f3049b41e0b968b033 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 4 May 2023 13:39:33 +0200 Subject: [PATCH 18/20] different wgautomesh gossip ports for prod and staging --- cluster/staging/cluster.nix | 2 ++ nix/deuxfleurs.nix | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cluster/staging/cluster.nix b/cluster/staging/cluster.nix index 2fec94c..362724b 100644 --- a/cluster/staging/cluster.nix +++ b/cluster/staging/cluster.nix @@ -38,6 +38,8 @@ endpoint = "bitfrost.fiber.shirokumo.net:33734"; }; }; + + deuxfleurs.wgautomeshPort = 1667; services.wgautomesh.logLevel = "debug"; # Bootstrap IPs for Consul cluster, diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 0f5c185..9ef2b2a 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -117,6 +117,11 @@ in type = port; default = 33799; }; + wgautomeshPort = mkOption { + description = "Gossip port for wgautomesh"; + type = port; + default = 1666; + }; }; imports = [ @@ -245,7 +250,7 @@ in services.wgautomesh = { enable = true; interface = "wg0"; - gossipPort = 1666; + gossipPort = cfg.wgautomeshPort; gossipSecretFile = "/var/lib/wgautomesh/gossip_secret"; persistFile = "/var/lib/wgautomesh/state"; upnpForwardPublicPort = From 258d27c566c78dfc714079bf921270a71fdc9535 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 9 May 2023 15:12:03 +0200 Subject: [PATCH 19/20] deploy tricot at bespin, register gitea (not accessed yet) --- cluster/prod/app/core/deploy/tricot.hcl | 2 +- cluster/prod/register_external_services.sh | 41 ++++++++++++++++++++++ cluster/prod/register_personal_services.sh | 17 --------- 3 files changed, 42 insertions(+), 18 deletions(-) create mode 100755 cluster/prod/register_external_services.sh delete mode 100644 cluster/prod/register_personal_services.sh diff --git a/cluster/prod/app/core/deploy/tricot.hcl b/cluster/prod/app/core/deploy/tricot.hcl index 7c955d2..7c3bada 100644 --- a/cluster/prod/app/core/deploy/tricot.hcl +++ b/cluster/prod/app/core/deploy/tricot.hcl @@ -1,5 +1,5 @@ job "core:tricot" { - datacenters = ["orion", "neptune", "scorpio"] + datacenters = ["orion", "neptune", "scorpio", "bespin"] type = "system" priority = 90 diff --git a/cluster/prod/register_external_services.sh b/cluster/prod/register_external_services.sh new file mode 100755 index 0000000..9c00216 --- /dev/null +++ b/cluster/prod/register_external_services.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Bruxelles (bespin): git forge at git.deuxfleurs.fr + +curl -vv -X PUT http://localhost:8500/v1/catalog/register -H "Content-Type: application/json" --data @- < Date: Tue, 16 May 2023 14:14:27 +0200 Subject: [PATCH 20/20] use RA on orion as well --- cluster/prod/site/orion.nix | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cluster/prod/site/orion.nix b/cluster/prod/site/orion.nix index 58c49ab..dd8e208 100644 --- a/cluster/prod/site/orion.nix +++ b/cluster/prod/site/orion.nix @@ -3,11 +3,6 @@ { deuxfleurs.siteName = "orion"; deuxfleurs.staticIPv4.defaultGateway = "192.168.1.254"; - # Setting an IPv6 default gateway will disable RA for now. - # Adding this for now as Orion has the mail servers and we are - # not yet confident we can disable this without getting ourselves - # banned by sending from unwanted IPs (although it should be ok). - deuxfleurs.staticIPv6.defaultGateway = "2a01:e0a:28f:5e60::1"; deuxfleurs.cnameTarget = "orion.site.deuxfleurs.fr."; deuxfleurs.publicIPv4 = "82.66.80.201"; }