{ config, pkgs, ... }: let cfg = config.deuxfleurs; in with builtins; with pkgs.lib; { options.deuxfleurs = let wg_node = with types; submodule { options = { hostname = mkOption { type = str; description = "Host name"; }; site_name = mkOption { type = nullOr str; description = "Site where the node is located"; default = null; }; IP = mkOption { type = str; description = "IP Address"; }; publicKey = mkOption { type = str; description = "Public key"; }; endpoint = mkOption { type = nullOr str; description = "Wireguard endpoint on the public Internet"; }; lan_endpoint = mkOption { type = nullOr str; description = "Wireguard endpoint for nodes in the same site"; default = null; }; }; }; in { # Parameters for individual nodes network_interface = mkOption { description = "Network interface name to configure"; type = types.str; }; lan_ip = mkOption { description = "IP address of this node on the local network interface"; type = types.str; }; lan_ip_prefix_length = mkOption { description = "Prefix length associated with lan_ip"; type = types.int; }; vpn_ip = mkOption { description = "IP address of this node on the Wireguard VPN"; type = types.str; }; vpn_listen_port = mkOption { description = "Port for incoming Wireguard VPN connections"; type = types.port; }; is_raft_server = mkOption { description = "Make this node a RAFT server for the Nomad and Consul deployments"; type = types.bool; default = false; }; # Parameters that generally vary between sites lan_default_gateway = mkOption { description = "IP address of the default route on the locak network interface"; type = types.str; }; site_name = mkOption { description = "Site (availability zone) on which this node is deployed"; type = types.str; }; # Parameters common to all nodes cluster_name = mkOption { description = "Name of this Deuxfleurs deployment"; type = types.str; }; cluster_nodes = mkOption { description = "Nodes that are part of the cluster"; type = types.listOf wg_node; }; admin_nodes = mkOption { description = "Machines that are part of the Wireguard VPN for administration purposes"; type = types.listOf wg_node; }; admin_accounts = mkOption { description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys"; type = types.attrsOf (types.listOf types.str); }; }; config = { # Configure admin accounts on all nodes users.users = builtins.mapAttrs (name: publicKeys: { isNormalUser = true; extraGroups = [ "wheel" ]; openssh.authorizedKeys.keys = publicKeys; }) cfg.admin_accounts; # Configure network interfaces networking.interfaces = attrsets.setAttrByPath [ cfg.network_interface ] { useDHCP = false; ipv4.addresses = [ { address = cfg.lan_ip; prefixLength = cfg.lan_ip_prefix_length; } ]; }; networking.defaultGateway = { address = cfg.lan_default_gateway; interface = cfg.network_interface; }; # Configure Wireguard VPN between all nodes networking.wireguard.interfaces.wg0 = { ips = [ "${cfg.vpn_ip}/16" ]; listenPort = cfg.vpn_listen_port; privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private"; peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: { publicKey = publicKey; allowedIPs = [ "${IP}/32" ]; endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null then lan_endpoint else endpoint; persistentKeepalive = 25; }) (cfg.cluster_nodes ++ cfg.admin_nodes); }; networking.firewall.allowedUDPPorts = [ cfg.vpn_listen_port ]; # Configure /etc/hosts to link all hostnames to their Wireguard IP networking.extraHosts = builtins.concatStringsSep "\n" (map ({ hostname, IP, ...}: "${IP} ${hostname}") (cfg.cluster_nodes ++ cfg.admin_nodes)); # Enable Hashicorp Consul & Nomad services.consul.enable = true; services.consul.extraConfig = (if cfg.is_raft_server then { server = true; bootstrap_expect = 3; } else {}) // { datacenter = cfg.cluster_name; node_meta = { "site" = cfg.site_name; }; ui = true; bind_addr = cfg.vpn_ip; ports.http = -1; addresses.https = "0.0.0.0"; ports.https = 8501; retry_join = map (node_info: node_info.IP) cfg.cluster_nodes; ca_file = "/var/lib/consul/pki/consul-ca.crt"; cert_file = "/var/lib/consul/pki/consul2022.crt"; key_file = "/var/lib/consul/pki/consul2022.key"; verify_incoming = true; verify_outgoing = true; verify_server_hostname = true; }; services.nomad.enable = true; services.nomad.package = pkgs.nomad_1_1; services.nomad.settings = (if cfg.is_raft_server then { server = { enabled = true; bootstrap_expect = 3; }; } else {}) // { region = cfg.cluster_name; datacenter = cfg.site_name; advertise = { rpc = cfg.vpn_ip; http = cfg.vpn_ip; serf = cfg.vpn_ip; }; consul = { address = "localhost:8501"; ca_file = "/var/lib/nomad/pki/consul2022.crt"; cert_file = "/var/lib/nomad/pki/consul2022-client.crt"; key_file = "/var/lib/nomad/pki/consul2022-client.key"; ssl = true; }; client = { enabled = true; network_interface = "wg0"; meta = { "site" = cfg.site_name; }; }; tls = { http = true; rpc = true; ca_file = "/var/lib/nomad/pki/nomad-ca.crt"; cert_file = "/var/lib/nomad/pki/nomad2022.crt"; key_file = "/var/lib/nomad/pki/nomad2022.key"; verify_server_hostname = true; verify_https_client = true; }; plugin = [ { docker = [ { config = [ { volumes.enabled = true; allow_privileged = true; } ]; } ]; } ]; }; }; }