nixcfg/nix/deuxfleurs.nix

372 lines
11 KiB
Nix

{ config, pkgs, ... }:
let
cfg = config.deuxfleurs;
in
with builtins;
with pkgs.lib;
{
options.deuxfleurs =
let wg_node = with types; submodule {
options = {
hostname = mkOption {
type = str;
description = "Host name";
};
site_name = mkOption {
type = nullOr str;
description = "Site where the node is located";
default = null;
};
IP = mkOption {
type = str;
description = "IP Address in the Wireguard network";
};
publicKey = mkOption {
type = str;
description = "Public key";
};
endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint on the public Internet";
};
lan_endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint for nodes in the same site";
default = null;
};
};
};
in
{
# Parameters for individual nodes
network_interface = mkOption {
description = "Network interface name to configure";
type = types.str;
};
lan_ip = mkOption {
description = "IP address of this node on the local network interface";
type = types.str;
};
lan_ip_prefix_length = mkOption {
description = "Prefix length associated with lan_ip";
type = types.int;
};
ipv6 = mkOption {
description = "Public IPv6 address of this node";
type = types.str;
};
ipv6_prefix_length = mkOption {
description = "Prefix length associated with ipv6 ip";
type = types.int;
};
cluster_ip = mkOption {
description = "IP address of this node on the Wesher mesh network";
type = types.str;
};
wireguard_port = mkOption {
description = "Port for incoming Wireguard VPN connections";
type = types.port;
default = 33799;
};
is_raft_server = mkOption {
description = "Make this node a RAFT server for the Nomad and Consul deployments";
type = types.bool;
default = false;
};
# Parameters that generally vary between sites
lan_default_gateway = mkOption {
description = "IPv4 address of the default route on the local network interface";
type = types.str;
};
ipv6_default_gateway = mkOption {
description = "IPv6 address of the default IPv6 gateway for the targeted net interface";
type = types.str;
};
site_name = mkOption {
description = "Site (availability zone) on which this node is deployed";
type = types.str;
};
nameservers = mkOption {
description = "External DNS servers to use";
type = types.listOf types.str;
};
# Parameters common to all nodes
cluster_name = mkOption {
description = "Name of this Deuxfleurs deployment";
type = types.str;
};
cluster_prefix = mkOption {
description = "IP address prefix for the Wireguard overlay network";
type = types.str;
};
cluster_prefix_length = mkOption {
description = "IP address prefix length for the Wireguard overlay network";
type = types.int;
default = 16;
};
cluster_nodes = mkOption {
description = "Nodes that are part of the cluster";
type = types.listOf wg_node;
};
admin_accounts = mkOption {
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
type = types.attrsOf (types.listOf types.str);
};
bootstrap = mkOption {
description = "Whether to enable bootstrapping for Nomad and Consul";
type = types.bool;
default = false;
};
};
config = {
# Configure admin accounts on all nodes
users.users = builtins.mapAttrs (name: publicKeys: {
isNormalUser = true;
extraGroups = [ "wheel" ];
openssh.authorizedKeys.keys = publicKeys;
}) cfg.admin_accounts;
# Configure network interfaces
networking.interfaces =
let ip4config = {
useDHCP = false;
ipv4.addresses = [
{
address = cfg.lan_ip;
prefixLength = cfg.lan_ip_prefix_length;
}
];
};
ip6config = {
tempAddress = "disabled";
ipv6.addresses = [
{
address = cfg.ipv6;
prefixLength = cfg.ipv6_prefix_length;
}
];
};
in
(attrsets.setAttrByPath [ cfg.network_interface ] (ip4config // ip6config));
networking.defaultGateway = {
address = cfg.lan_default_gateway;
interface = cfg.network_interface;
};
networking.defaultGateway6 = {
address = cfg.ipv6_default_gateway;
interface = cfg.network_interface;
};
# Configure Unbound DNS to redirect to Consul queries under .consul
# and to pass directly to public DNS resolver all others
services.unbound = {
enable = true;
settings = {
server = {
interface = [ "127.0.0.1" "${cfg.lan_ip}" ];
domain-insecure = [ "consul." ];
local-zone = [ "consul. nodefault" ];
log-servfail = true;
access-control = [
"127.0.0.0/8 allow"
"${cfg.lan_ip}/${toString cfg.lan_ip_prefix_length} allow"
"172.17.0.0/16 allow"
];
};
forward-zone = [
# Forward .consul queries to Consul daemon
{
name = "consul.";
forward-addr = "${cfg.lan_ip}@8600";
forward-no-cache = true;
forward-tcp-upstream = false;
forward-tls-upstream = false;
}
# Forward all queries to our ISP's nameserver
{
name = ".";
forward-addr = cfg.nameservers;
forward-first = true;
}
];
};
resolveLocalQueries = false; # don't overwrite our resolv.conf
};
# Reach Unbound through the IP of our LAN interface,
# instead of 127.0.0.1 (this will also work in Docker containers)
networking.nameservers = [
cfg.lan_ip
];
# Configure Wireguard VPN between all nodes
networking.wireguard.interfaces.wg0 = {
ips = [ "${cfg.cluster_ip}/16" ];
listenPort = cfg.wireguard_port;
privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
publicKey = publicKey;
allowedIPs = [ "${IP}/32" ];
endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
then lan_endpoint else endpoint;
persistentKeepalive = 25;
}) cfg.cluster_nodes;
};
# Configure /etc/hosts to link all hostnames to their Wireguard IP
networking.extraHosts = builtins.concatStringsSep "\n" (map
({ hostname, IP, ...}: "${IP} ${hostname}")
cfg.cluster_nodes);
# Enable Hashicorp Consul & Nomad
services.consul.enable = true;
services.consul.extraConfig =
(if cfg.is_raft_server
then { server = true; }
// (if cfg.bootstrap then { bootstrap_expect = 3; } else {})
else {}) //
{
datacenter = cfg.cluster_name;
node_meta = {
"site" = cfg.site_name;
};
ui_config = {
enabled = true;
};
bind_addr = "${cfg.cluster_ip}";
addresses = {
https = "0.0.0.0";
dns = "0.0.0.0";
};
ports = {
http = -1;
https = 8501;
};
performance = {
rpc_hold_timeout = "70s";
};
ca_file = "/var/lib/consul/pki/consul-ca.crt";
cert_file = "/var/lib/consul/pki/consul2022.crt";
key_file = "/var/lib/consul/pki/consul2022.key";
verify_incoming = true;
verify_outgoing = true;
verify_server_hostname = true;
};
systemd.services.consul.serviceConfig = {
AmbientCapabilities = "CAP_NET_BIND_SERVICE";
};
services.nomad.enable = true;
services.nomad.package = pkgs.nomad_1_3;
services.nomad.extraPackages = [
pkgs.glibc
pkgs.zstd
#pkgs.qemu
#pkgs.qemu_kvm
];
services.nomad.settings =
(if cfg.is_raft_server
then {
server = { enabled = true; }
// (if cfg.bootstrap then { bootstrap_expect = 3; } else {});
} else {}) //
{
region = cfg.cluster_name;
datacenter = cfg.site_name;
advertise = {
rpc = "${cfg.cluster_ip}";
http = "${cfg.cluster_ip}";
serf = "${cfg.cluster_ip}";
};
consul = {
address = "localhost:8501";
ca_file = "/var/lib/nomad/pki/consul2022.crt";
cert_file = "/var/lib/nomad/pki/consul2022-client.crt";
key_file = "/var/lib/nomad/pki/consul2022-client.key";
ssl = true;
checks_use_advertise = true;
};
client = {
enabled = true;
network_interface = "wg0";
meta = {
"site" = cfg.site_name;
"public_ipv6" = cfg.ipv6;
};
};
tls = {
http = true;
rpc = true;
ca_file = "/var/lib/nomad/pki/nomad-ca.crt";
cert_file = "/var/lib/nomad/pki/nomad2022.crt";
key_file = "/var/lib/nomad/pki/nomad2022.key";
verify_server_hostname = true;
verify_https_client = true;
};
plugin = [
{
docker = [
{
config = [
{
volumes.enabled = true;
allow_privileged = true;
}
];
}
];
#qemu = [
# {
# enabled = true;
# }
#];
}
];
};
# ---- Firewall config ----
# Open ports in the firewall.
networking.firewall = {
enable = true;
allowedTCPPorts = [
# Allow anyone to connect on SSH port
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
];
allowedUDPPorts = [
# Allow peers to connect to Wireguard
cfg.wireguard_port
];
# Allow specific hosts access to specific things in the cluster
extraCommands = ''
# Allow everything from router (usefull for UPnP/IGD)
iptables -A INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
# Allow docker containers to access all ports
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
# Allow other nodes on VPN to access all ports
iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
'';
# When stopping firewall, delete all rules that were configured manually above
extraStopCommands = ''
iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
'';
};
};
}