diff --git a/app/garage-staging/deploy/garage.hcl b/app/garage-staging/deploy/garage.hcl index 671ab97..3b13e6a 100644 --- a/app/garage-staging/deploy/garage.hcl +++ b/app/garage-staging/deploy/garage.hcl @@ -28,7 +28,7 @@ job "garage-staging" { driver = "docker" config { - image = "dxflrs/amd64_garage:884e7f0f04fc04bdf453dade98f31a1511c9a214" + image = "dxflrs/amd64_garage:5afab390d80c57a275a52752c10a0a3c29a3b233" command = "/garage" args = [ "server" ] network_mode = "host" diff --git a/cluster/prod/node/celeri.nix b/cluster/prod/node/celeri.nix index 09de302..02a33c9 100644 --- a/cluster/prod/node/celeri.nix +++ b/cluster/prod/node/celeri.nix @@ -10,13 +10,9 @@ networking.hostName = "celeri"; - networking.interfaces.eno1.useDHCP = false; - networking.interfaces.eno1.ipv4.addresses = [ - { - address = "192.168.1.33"; - prefixLength = 24; - } - ]; + deuxfleurs.network_interface = "eno1"; + deuxfleurs.lan_ip = "192.168.1.33"; + networking.interfaces.eno1.ipv6.addresses = [ { address = "2a01:e0a:c:a720::33"; diff --git a/cluster/prod/node/concombre.nix b/cluster/prod/node/concombre.nix index f06e387..517dcf8 100644 --- a/cluster/prod/node/concombre.nix +++ b/cluster/prod/node/concombre.nix @@ -10,13 +10,9 @@ networking.hostName = "concombre"; - networking.interfaces.eno1.useDHCP = false; - networking.interfaces.eno1.ipv4.addresses = [ - { - address = "192.168.1.31"; - prefixLength = 24; - } - ]; + deuxfleurs.network_interface = "eno1"; + deuxfleurs.lan_ip = "192.168.1.31"; + networking.interfaces.eno1.ipv6.addresses = [ { address = "2a01:e0a:c:a720::31"; diff --git a/cluster/prod/node/courgette.nix b/cluster/prod/node/courgette.nix index e540de1..d34e7a5 100644 --- a/cluster/prod/node/courgette.nix +++ b/cluster/prod/node/courgette.nix @@ -10,13 +10,9 @@ networking.hostName = "courgette"; - networking.interfaces.eno1.useDHCP = false; - networking.interfaces.eno1.ipv4.addresses = [ - { - address = "192.168.1.32"; - prefixLength = 24; - } - ]; + deuxfleurs.network_interface = "eno1"; + deuxfleurs.lan_ip = "192.168.1.32"; + networking.interfaces.eno1.ipv6.addresses = [ { address = "2a01:e0a:c:a720::32"; diff --git a/cluster/prod/site/neptune.nix b/cluster/prod/site/neptune.nix index c0eb39e..900ddf0 100644 --- a/cluster/prod/site/neptune.nix +++ b/cluster/prod/site/neptune.nix @@ -1,11 +1,9 @@ { config, pkgs, ... }: { - networking.defaultGateway = { - address = "192.168.1.254"; - interface = "eno1"; - }; - networking.nameservers = [ "192.168.1.254" ]; - deuxfleurs.site_name = "neptune"; + deuxfleurs.lan_default_gateway = "192.168.1.254"; + deuxfleurs.lan_ip_prefix_length = 24; + + networking.nameservers = [ "192.168.1.254" ]; } diff --git a/cluster/staging/node/carcajou.nix b/cluster/staging/node/carcajou.nix index 82cd8f9..beff6a7 100644 --- a/cluster/staging/node/carcajou.nix +++ b/cluster/staging/node/carcajou.nix @@ -10,13 +10,9 @@ networking.hostName = "carcajou"; - networking.interfaces.eno1.useDHCP = false; - networking.interfaces.eno1.ipv4.addresses = [ - { - address = "192.168.1.22"; - prefixLength = 24; - } - ]; + deuxfleurs.network_interface = "eno1"; + deuxfleurs.lan_ip = "192.168.1.22"; + networking.interfaces.eno1.ipv6.addresses = [ { address = "2a01:e0a:c:a720::22"; diff --git a/cluster/staging/node/cariacou.nix b/cluster/staging/node/cariacou.nix index d9dbc93..430996c 100644 --- a/cluster/staging/node/cariacou.nix +++ b/cluster/staging/node/cariacou.nix @@ -10,13 +10,9 @@ networking.hostName = "cariacou"; - networking.interfaces.eno1.useDHCP = false; - networking.interfaces.eno1.ipv4.addresses = [ - { - address = "192.168.1.21"; - prefixLength = 24; - } - ]; + deuxfleurs.network_interface = "eno1"; + deuxfleurs.lan_ip = "192.168.1.21"; + networking.interfaces.eno1.ipv6.addresses = [ { address = "2a01:e0a:c:a720::21"; diff --git a/cluster/staging/node/caribou.nix b/cluster/staging/node/caribou.nix index 3be3cdc..7b785ef 100644 --- a/cluster/staging/node/caribou.nix +++ b/cluster/staging/node/caribou.nix @@ -3,6 +3,11 @@ { config, pkgs, ... }: { + imports = + [ + ./remote-unlock.nix + ]; + # Use the systemd-boot EFI boot loader. boot.loader.systemd-boot.enable = true; boot.loader.timeout = 20; @@ -10,13 +15,9 @@ networking.hostName = "caribou"; - networking.interfaces.eno1.useDHCP = false; - networking.interfaces.eno1.ipv4.addresses = [ - { - address = "192.168.1.23"; - prefixLength = 24; - } - ]; + deuxfleurs.network_interface = "eno1"; + deuxfleurs.lan_ip = "192.168.1.23"; + networking.interfaces.eno1.ipv6.addresses = [ { address = "2a01:e0a:c:a720::23"; diff --git a/cluster/staging/site/neptune.nix b/cluster/staging/site/neptune.nix index ec8b052..5fed713 100644 --- a/cluster/staging/site/neptune.nix +++ b/cluster/staging/site/neptune.nix @@ -1,13 +1,11 @@ { config, pkgs, ... }: { - networking.defaultGateway = { - address = "192.168.1.254"; - interface = "eno1"; - }; - networking.nameservers = [ "192.168.1.254" ]; - deuxfleurs.site_name = "neptune"; + deuxfleurs.lan_default_gateway = "192.168.1.254"; + deuxfleurs.lan_ip_prefix_length = 24; + + networking.nameservers = [ "192.168.1.254" ]; networking.firewall.allowedTCPPorts = [ 80 443 ]; diff --git a/configuration.nix b/configuration.nix index 8feb850..278e841 100644 --- a/configuration.nix +++ b/configuration.nix @@ -40,6 +40,7 @@ nmap bind inetutils + pciutils vim tmux ncdu diff --git a/deploy.sh b/deploy.sh index af11604..7527f7f 100755 --- a/deploy.sh +++ b/deploy.sh @@ -40,6 +40,7 @@ for NIXHOST in $NIXHOSTLIST; do ssh -F $SSH_CONFIG $SSH_DEST mkdir -p $TMP_PATH $TMP_PATH/pki cat configuration.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/configuration.nix > /dev/null cat nix/deuxfleurs.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/deuxfleurs.nix > /dev/null + cat nix/remote-unlock.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/remote-unlock.nix > /dev/null cat cluster/$CLUSTER/cluster.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/cluster.nix > /dev/null cat cluster/$CLUSTER/node/$NIXHOST.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/node.nix > /dev/null cat cluster/$CLUSTER/node/$NIXHOST.site.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/site.nix > /dev/null @@ -57,7 +58,9 @@ for NIXHOST in $NIXHOSTLIST; do set -ex cd $TMP_PATH -mv deuxfleurs.nix configuration.nix cluster.nix node.nix site.nix /etc/nixos +mv deuxfleurs.nix remote-unlock.nix configuration.nix cluster.nix node.nix site.nix /etc/nixos + +nixos-rebuild switch test -f rclone.conf && (mv rclone.conf /root; chmod 600 /root/rclone.conf) @@ -73,8 +76,6 @@ if [ -f pki/nomad-ca.crt ]; then mv pki/nomad* /var/lib/nomad/pki fi -nixos-rebuild switch - # Save up-to-date Consul client certificates in Consul itself export CONSUL_HTTP_ADDR=https://localhost:8501 export CONSUL_CACERT=/var/lib/consul/pki/consul-ca.crt diff --git a/doc/example-hardware-configuration.nix b/doc/example-hardware-configuration.nix new file mode 100644 index 0000000..0a72bd1 --- /dev/null +++ b/doc/example-hardware-configuration.nix @@ -0,0 +1,38 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" ]; + boot.initrd.kernelModules = [ "dm-snapshot" ]; + boot.kernelModules = [ "kvm-intel" ]; + boot.extraModulePackages = [ ]; + + boot.initrd.luks.devices."cryptlvm" = { + device = "/dev/disk/by-uuid/5d4fcef7-433d-43a9-be26-be940ce291c0"; + allowDiscards = true; + }; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/5bad79e1-fdbf-48f3-861f-6810adc76195"; + fsType = "ext4"; + options = [ "relatime" "discard" ]; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/07E8-5958"; + fsType = "vfat"; + }; + + swapDevices = + [ { device = "/dev/disk/by-uuid/a3e0bae8-8b07-4e66-a4a7-6955639f2155"; } + ]; + + hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} + diff --git a/nix/deuxfleurs.nix b/nix/deuxfleurs.nix index 54af8ff..fc39071 100644 --- a/nix/deuxfleurs.nix +++ b/nix/deuxfleurs.nix @@ -29,11 +29,20 @@ in }; in { - # Parameters that may vary between nodes - site_name = mkOption { - description = "Site (availability zone) on which this node is deployed"; + # Parameters for individual nodes + network_interface = mkOption { + description = "Network interface name to configure"; type = types.str; }; + lan_ip = mkOption { + description = "IP address of this node on the local network interface"; + type = types.str; + }; + lan_ip_prefix_length = mkOption { + description = "Prefix length associated with lan_ip"; + type = types.int; + }; + vpn_ip = mkOption { description = "IP address of this node on the Wireguard VPN"; type = types.str; @@ -48,6 +57,17 @@ in default = false; }; + + # Parameters that generally vary between sites + lan_default_gateway = mkOption { + description = "IP address of the default route on the locak network interface"; + type = types.str; + }; + site_name = mkOption { + description = "Site (availability zone) on which this node is deployed"; + type = types.str; + }; + # Parameters common to all nodes cluster_name = mkOption { description = "Name of this Deuxfleurs deployment"; @@ -75,6 +95,21 @@ in openssh.authorizedKeys.keys = publicKeys; }) cfg.admin_accounts; + # Configure network interfaces + networking.interfaces = attrsets.setAttrByPath [ config.deuxfleurs.network_interface ] { + useDHCP = false; + ipv4.addresses = [ + { + address = config.deuxfleurs.lan_ip; + prefixLength = config.deuxfleurs.lan_ip_prefix_length; + } + ]; + }; + networking.defaultGateway = { + address = config.deuxfleurs.lan_default_gateway; + interface = config.deuxfleurs.network_interface; + }; + # Configure Wireguard VPN between all nodes networking.wireguard.interfaces.wg0 = { ips = [ "${cfg.vpn_ip}/16" ]; diff --git a/nix/remote-unlock.nix b/nix/remote-unlock.nix new file mode 100644 index 0000000..669f578 --- /dev/null +++ b/nix/remote-unlock.nix @@ -0,0 +1,26 @@ +{ config, pkgs, ... }: + + with builtins; + with pkgs.lib; +{ + config = { + boot.initrd.availableKernelModules = [ "pps_core" "ptp" "e1000e" ]; + boot.initrd.network.enable = true; + boot.initrd.network.ssh = { + enable = true; + port = 2222; + authorizedKeys = concatLists (mapAttrsToList (name: user: user) config.deuxfleurs.admin_accounts); + hostKeys = [ "/var/lib/deuxfleurs/remote-unlock/ssh_host_ed25519_key" ]; + }; + boot.initrd.network.postCommands = '' + ip addr add ${config.deuxfleurs.lan_ip}/${toString config.deuxfleurs.lan_ip_prefix_length} dev ${config.deuxfleurs.network_interface} + ip link set dev ${config.deuxfleurs.network_interface} up + ip route add default via ${config.deuxfleurs.lan_default_gateway} dev ${config.deuxfleurs.network_interface} + ip a + ip route + ping -c 4 ${config.deuxfleurs.lan_default_gateway} + echo 'echo run cryptsetup-askpass to unlock drives' >> /root/.profile + ''; + }; +} + diff --git a/ssh_known_hosts b/ssh_known_hosts index 5c5dd66..71f0390 100644 --- a/ssh_known_hosts +++ b/ssh_known_hosts @@ -1,14 +1,11 @@ 10.42.0.22 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHXyY9yZdq/VYpg3K1plBzFvim41tWlu+Dmov3BNSm39 10.42.0.21 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILYIRbHPU1w1xWRpm2+u3QsXOKfMjv1EXrpYbHT+epds -10.42.0.23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICNRilSqrgqdZlLdhN2eOY5ZMbzqpeqAdIf6QuChhXU+ 10.42.0.2 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIORTjNdFaDbhto8kuMPB3hYkfnzdnceZVIELApUe5PPL 10.42.1.33 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOuY1CvhxBP9BtKkTlmOUu6Hhy8OQTB3R8OCFXbHA/RA 10.42.1.31 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL3N0QOFNGkCpVLuOHFdpnBaxIFH925KpdIHV/3F9+BR 10.42.1.32 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPCXJeo6yeQeTN7D7OZwLd8zbyU1jWywlhQ29yyk7x+G -192.168.1.22 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHXyY9yZdq/VYpg3K1plBzFvim41tWlu+Dmov3BNSm39 -192.168.1.21 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILYIRbHPU1w1xWRpm2+u3QsXOKfMjv1EXrpYbHT+epds -192.168.1.23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICNRilSqrgqdZlLdhN2eOY5ZMbzqpeqAdIf6QuChhXU+ 10.42.2.2 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIORTjNdFaDbhto8kuMPB3hYkfnzdnceZVIELApUe5PPL 10.42.2.22 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHXyY9yZdq/VYpg3K1plBzFvim41tWlu+Dmov3BNSm39 10.42.2.21 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILYIRbHPU1w1xWRpm2+u3QsXOKfMjv1EXrpYbHT+epds -10.42.2.23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICNRilSqrgqdZlLdhN2eOY5ZMbzqpeqAdIf6QuChhXU+ +192.168.1.23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDsYD1gNmGyb6c9wjGR6tC69fHP6+FpPHTBT6laPTHeD +10.42.2.23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDsYD1gNmGyb6c9wjGR6tC69fHP6+FpPHTBT6laPTHeD