nixcfg/nix/deuxfleurs.nix

{ config, pkgs, ... }:

let
  cfg = config.deuxfleurs;
in
  with builtins;
  with pkgs.lib;
{
  options.deuxfleurs =
    let wg_node = with types; submodule {
      options = {
        hostname = mkOption {
          type = str;
          description = "Host name";
        };
        site_name = mkOption {
          type = nullOr str;
          description = "Site where the node is located";
          default = null;
        };
        IP = mkOption {
          type = str;
          description = "IP Address in the Wireguard network";
        };
        publicKey = mkOption {
          type = str;
          description = "Public key";
        };
        endpoint = mkOption {
          type = nullOr str;
          description = "Wireguard endpoint on the public Internet";
        };
        lan_endpoint = mkOption {
          type = nullOr str;
          description = "Wireguard endpoint for nodes in the same site";
          default = null;
        };
      };
    };
  in
  {
    # Parameters for individual nodes
    network_interface = mkOption {
      description = "Network interface name to configure";
      type = types.str;
    };
    lan_ip = mkOption {
      description = "IP address of this node on the local network interface";
      type = types.str;
    };
    lan_ip_prefix_length = mkOption {
      description = "Prefix length associated with lan_ip";
      type = types.int;
    };
    ipv6 = mkOption {
      description = "Public IPv6 address of this node";
      type = types.str;
    };
    ipv6_prefix_length = mkOption {
      description = "Prefix length associated with ipv6 ip";
      type = types.int;
    };

    cluster_ip = mkOption {
      description = "IP address of this node on the Wesher mesh network";
      type = types.str;
    };
    wireguard_port = mkOption {
      description = "Port for incoming Wireguard VPN connections";
      type = types.port;
      default = 33799;
    };

    is_raft_server = mkOption {
      description = "Make this node a RAFT server for the Nomad and Consul deployments";
      type = types.bool;
      default = false;
    };

    # Parameters that generally vary between sites
    lan_default_gateway = mkOption {
      description = "IPv4 address of the default route on the local network interface";
      type = types.str;
    };
    ipv6_default_gateway = mkOption {
      description = "IPv6 address of the default IPv6 gateway for the targeted net interface";
      type = types.str;
    };
    site_name = mkOption {
      description = "Site (availability zone) on which this node is deployed";
      type = types.str;
    };
    nameservers = mkOption {
      description = "External DNS servers to use";
      type = types.listOf types.str;
    };

    # Parameters common to all nodes
    cluster_name = mkOption {
      description = "Name of this Deuxfleurs deployment";
      type = types.str;
    };
    cluster_prefix = mkOption {
      description = "IP address prefix for the Wireguard overlay network";
      type = types.str;
    };
    cluster_prefix_length = mkOption {
      description = "IP address prefix length for the Wireguard overlay network";
      type = types.int;
      default = 16;
    };
    cluster_nodes = mkOption {
      description = "Nodes that are part of the cluster";
      type = types.listOf wg_node;
    };
    admin_accounts = mkOption {
      description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
      type = types.attrsOf (types.listOf types.str);
    };
    bootstrap = mkOption {
      description = "Whether to enable bootstrapping for Nomad and Consul";
      type = types.bool;
      default = false;
    };
  };

  config = {
    # Configure admin accounts on all nodes
    users.users = builtins.mapAttrs (name: publicKeys: {
      isNormalUser = true;
      extraGroups = [ "wheel" ];
      openssh.authorizedKeys.keys = publicKeys;
    }) cfg.admin_accounts;

    # Configure network interfaces
    networking.interfaces =
      let ip4config = {
          useDHCP = false;
          ipv4.addresses = [
            {
              address = cfg.lan_ip;
              prefixLength = cfg.lan_ip_prefix_length;
            }
          ];
        };
      ip6config = {
        tempAddress = "disabled";
        ipv6.addresses = [
          {
            address = cfg.ipv6;
            prefixLength = cfg.ipv6_prefix_length;
          }
        ];
      };
    in
      (attrsets.setAttrByPath [ cfg.network_interface ] (ip4config // ip6config));

    networking.defaultGateway = {
      address = cfg.lan_default_gateway;
      interface = cfg.network_interface;
    };

    networking.defaultGateway6 = {
      address = cfg.ipv6_default_gateway;
      interface = cfg.network_interface;
    };

    networking.nameservers = [
      cfg.lan_ip
    ] ++ cfg.nameservers;

    # Configure Wireguard VPN between all nodes
    networking.wireguard.interfaces.wg0 = {
      ips = [ "${cfg.cluster_ip}/16" ];
      listenPort = cfg.wireguard_port;
      privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
      peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
        publicKey = publicKey;
        allowedIPs = [ "${IP}/32" ];
        endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
                   then lan_endpoint else endpoint;
        persistentKeepalive = 25;
      }) cfg.cluster_nodes;
    };

    # Configure /etc/hosts to link all hostnames to their Wireguard IP
    networking.extraHosts = builtins.concatStringsSep "\n" (map
      ({ hostname, IP, ...}: "${IP} ${hostname}")
      cfg.cluster_nodes);

    # Enable Hashicorp Consul & Nomad
    services.consul.enable = true;
    services.consul.extraConfig =
      (if cfg.is_raft_server
      then { server = true; }
        // (if cfg.bootstrap then { bootstrap_expect = 3; } else {})
      else {}) //
    {
      datacenter = cfg.cluster_name;
      node_meta = {
        "site" = cfg.site_name;
      };
      ui_config = {
        enabled = true;
      };
      bind_addr = "${cfg.cluster_ip}";

      addresses = {
        https = "0.0.0.0";
        dns = "0.0.0.0";
      };
      ports = {
        http = -1;
        https = 8501;
        dns = 53;
      };

      recursors = [ cfg.nameservers ];

      ca_file = "/var/lib/consul/pki/consul-ca.crt";
      cert_file = "/var/lib/consul/pki/consul2022.crt";
      key_file = "/var/lib/consul/pki/consul2022.key";
      verify_incoming = true;
      verify_outgoing = true;
      verify_server_hostname = true;
    };
    systemd.services.consul.serviceConfig = {
      AmbientCapabilities = "CAP_NET_BIND_SERVICE";
    };

    services.nomad.enable = true;
    services.nomad.package = pkgs.nomad_1_3;
    services.nomad.extraPackages = [
      pkgs.glibc
      pkgs.zstd
      #pkgs.qemu
      #pkgs.qemu_kvm
    ];
    services.nomad.settings =
      (if cfg.is_raft_server
      then {
          server = { enabled = true; }
            // (if cfg.bootstrap then { bootstrap_expect = 3; } else {});
      } else {}) //
    {
      region = cfg.cluster_name;
      datacenter = cfg.site_name;
      advertise = {
        rpc = "${cfg.cluster_ip}";
        http = "${cfg.cluster_ip}";
        serf = "${cfg.cluster_ip}";
      };
      consul = {
        address = "consul.service.prod.consul:8501";
        ca_file = "/var/lib/nomad/pki/consul2022.crt";
        cert_file = "/var/lib/nomad/pki/consul2022-client.crt";
        key_file = "/var/lib/nomad/pki/consul2022-client.key";
        ssl = true;
      };
      client = {
        enabled = true;
        network_interface = "wg0";
        meta = {
          "site" = cfg.site_name;
        };
      };
      tls = {
        http = true;
        rpc = true;
        ca_file = "/var/lib/nomad/pki/nomad-ca.crt";
        cert_file = "/var/lib/nomad/pki/nomad2022.crt";
        key_file = "/var/lib/nomad/pki/nomad2022.key";
        verify_server_hostname = true;
        verify_https_client = true;
      };
      plugin = [
        {
          docker = [
            {
              config = [
                {
                  volumes.enabled = true;
                  allow_privileged = true;
                }
              ];
            }
          ];
          #qemu = [
          #  {
          #    enabled = true;
          #  }
          #];
        }
      ];
    };

    # ---- Firewall config ----

    # Open ports in the firewall.
    networking.firewall = {
      enable = true;

      allowedTCPPorts = [
        # Allow anyone to connect on SSH port
        (builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
      ];

      allowedUDPPorts = [
        # Allow peers to connect to Wireguard
        cfg.wireguard_port
      ];

      # Allow specific hosts access to specific things in the cluster
      extraCommands = ''
        # Allow everything from router (usefull for UPnP/IGD)
        iptables -A INPUT -s ${cfg.lan_default_gateway} -j ACCEPT

        # Allow docker containers to access all ports
        iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT

        # Allow other nodes on VPN to access all ports
        iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
      '';

      # When stopping firewall, delete all rules that were configured manually above
      extraStopCommands = ''
        iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
        iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
        iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
      '';
    };
  };
}