nixcfg/nix/deuxfleurs.nix

334 lines
9.5 KiB
Nix
Raw Normal View History

{ config, pkgs, ... }:
let
cfg = config.deuxfleurs;
in
with builtins;
with pkgs.lib;
{
2022-05-08 22:20:02 +00:00
options.deuxfleurs =
let wg_node = with types; submodule {
options = {
hostname = mkOption {
type = str;
description = "Host name";
};
site_name = mkOption {
type = nullOr str;
description = "Site where the node is located";
default = null;
};
IP = mkOption {
type = str;
description = "IP Address in the Wireguard network";
};
publicKey = mkOption {
type = str;
description = "Public key";
};
endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint on the public Internet";
};
lan_endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint for nodes in the same site";
default = null;
};
};
};
in
{
# Parameters for individual nodes
network_interface = mkOption {
description = "Network interface name to configure";
type = types.str;
};
lan_ip = mkOption {
description = "IP address of this node on the local network interface";
type = types.str;
};
lan_ip_prefix_length = mkOption {
description = "Prefix length associated with lan_ip";
type = types.int;
};
ipv6 = mkOption {
description = "Public IPv6 address of this node";
type = types.str;
};
ipv6_prefix_length = mkOption {
description = "Prefix length associated with ipv6 ip";
type = types.int;
};
cluster_ip = mkOption {
description = "IP address of this node on the Wesher mesh network";
type = types.str;
};
wireguard_port = mkOption {
description = "Port for incoming Wireguard VPN connections";
type = types.port;
default = 33799;
};
is_raft_server = mkOption {
description = "Make this node a RAFT server for the Nomad and Consul deployments";
type = types.bool;
default = false;
};
# Parameters that generally vary between sites
lan_default_gateway = mkOption {
2022-08-24 09:06:55 +00:00
description = "IPv4 address of the default route on the local network interface";
type = types.str;
};
ipv6_default_gateway = mkOption {
description = "IPv6 address of the default IPv6 gateway for the targeted net interface";
type = types.str;
};
site_name = mkOption {
description = "Site (availability zone) on which this node is deployed";
type = types.str;
};
2022-06-01 12:46:00 +00:00
nameservers = mkOption {
description = "External DNS servers to use";
type = types.listOf types.str;
};
# Parameters common to all nodes
cluster_name = mkOption {
description = "Name of this Deuxfleurs deployment";
type = types.str;
};
cluster_prefix = mkOption {
description = "IP address prefix for the Wireguard overlay network";
type = types.str;
};
cluster_prefix_length = mkOption {
description = "IP address prefix length for the Wireguard overlay network";
type = types.int;
default = 16;
};
cluster_nodes = mkOption {
description = "Nodes that are part of the cluster";
type = types.listOf wg_node;
};
admin_accounts = mkOption {
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
type = types.attrsOf (types.listOf types.str);
};
bootstrap = mkOption {
description = "Whether to enable bootstrapping for Nomad and Consul";
type = types.bool;
default = false;
};
};
config = {
# Configure admin accounts on all nodes
users.users = builtins.mapAttrs (name: publicKeys: {
isNormalUser = true;
extraGroups = [ "wheel" ];
openssh.authorizedKeys.keys = publicKeys;
}) cfg.admin_accounts;
# Configure network interfaces
2022-05-08 22:20:02 +00:00
networking.interfaces =
let ip4config = {
useDHCP = false;
ipv4.addresses = [
{
address = cfg.lan_ip;
prefixLength = cfg.lan_ip_prefix_length;
}
];
};
ip6config = {
2022-08-23 11:12:07 +00:00
tempAddress = "disabled";
2022-05-08 22:20:02 +00:00
ipv6.addresses = [
{
address = cfg.ipv6;
prefixLength = cfg.ipv6_prefix_length;
}
];
};
in
(attrsets.setAttrByPath [ cfg.network_interface ] (ip4config // ip6config));
2022-06-01 12:46:00 +00:00
networking.defaultGateway = {
2022-04-19 15:41:18 +00:00
address = cfg.lan_default_gateway;
interface = cfg.network_interface;
};
2022-08-24 09:06:55 +00:00
networking.defaultGateway6 = {
address = cfg.ipv6_default_gateway;
interface = cfg.network_interface;
};
2022-06-01 12:46:00 +00:00
networking.nameservers = [
cfg.lan_ip
] ++ cfg.nameservers;
# Configure Wireguard VPN between all nodes
networking.wireguard.interfaces.wg0 = {
ips = [ "${cfg.cluster_ip}/16" ];
listenPort = cfg.wireguard_port;
privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
publicKey = publicKey;
allowedIPs = [ "${IP}/32" ];
endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
then lan_endpoint else endpoint;
persistentKeepalive = 25;
}) cfg.cluster_nodes;
};
# Configure /etc/hosts to link all hostnames to their Wireguard IP
networking.extraHosts = builtins.concatStringsSep "\n" (map
({ hostname, IP, ...}: "${IP} ${hostname}")
cfg.cluster_nodes);
# Enable Hashicorp Consul & Nomad
services.consul.enable = true;
services.consul.extraConfig =
(if cfg.is_raft_server
then { server = true; }
// (if cfg.bootstrap then { bootstrap_expect = 3; } else {})
else {}) //
{
datacenter = cfg.cluster_name;
node_meta = {
"site" = cfg.site_name;
};
2022-07-27 09:18:23 +00:00
ui_config = {
enabled = true;
};
bind_addr = "${cfg.cluster_ip}";
2022-06-01 12:46:00 +00:00
addresses = {
https = "0.0.0.0";
dns = "0.0.0.0";
};
ports = {
http = -1;
https = 8501;
dns = 53;
};
recursors = [ cfg.nameservers ];
ca_file = "/var/lib/consul/pki/consul-ca.crt";
cert_file = "/var/lib/consul/pki/consul2022.crt";
key_file = "/var/lib/consul/pki/consul2022.key";
verify_incoming = true;
verify_outgoing = true;
verify_server_hostname = true;
};
2022-06-01 12:46:00 +00:00
systemd.services.consul.serviceConfig = {
AmbientCapabilities = "CAP_NET_BIND_SERVICE";
};
services.nomad.enable = true;
2022-07-27 09:18:23 +00:00
services.nomad.package = pkgs.nomad_1_3;
2022-05-30 12:57:05 +00:00
services.nomad.extraPackages = [
pkgs.glibc
pkgs.zstd
2022-07-27 09:18:23 +00:00
#pkgs.qemu
#pkgs.qemu_kvm
2022-05-30 12:57:05 +00:00
];
services.nomad.settings =
(if cfg.is_raft_server
then {
server = { enabled = true; }
// (if cfg.bootstrap then { bootstrap_expect = 3; } else {});
} else {}) //
{
region = cfg.cluster_name;
datacenter = cfg.site_name;
advertise = {
rpc = "${cfg.cluster_ip}";
http = "${cfg.cluster_ip}";
serf = "${cfg.cluster_ip}";
};
consul = {
address = "consul.service.prod.consul:8501";
ca_file = "/var/lib/nomad/pki/consul2022.crt";
cert_file = "/var/lib/nomad/pki/consul2022-client.crt";
key_file = "/var/lib/nomad/pki/consul2022-client.key";
ssl = true;
};
client = {
enabled = true;
network_interface = "wg0";
meta = {
"site" = cfg.site_name;
};
};
tls = {
http = true;
rpc = true;
ca_file = "/var/lib/nomad/pki/nomad-ca.crt";
cert_file = "/var/lib/nomad/pki/nomad2022.crt";
key_file = "/var/lib/nomad/pki/nomad2022.key";
verify_server_hostname = true;
verify_https_client = true;
};
plugin = [
{
docker = [
{
config = [
{
volumes.enabled = true;
allow_privileged = true;
}
];
}
];
2022-07-27 09:18:23 +00:00
#qemu = [
# {
# enabled = true;
# }
#];
}
];
};
# ---- Firewall config ----
# Open ports in the firewall.
networking.firewall = {
enable = true;
allowedTCPPorts = [
# Allow anyone to connect on SSH port
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
];
allowedUDPPorts = [
# Allow peers to connect to Wireguard
cfg.wireguard_port
];
# Allow specific hosts access to specific things in the cluster
extraCommands = ''
# Allow everything from router (usefull for UPnP/IGD)
2022-05-08 22:29:17 +00:00
iptables -A INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
# Allow docker containers to access all ports
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
# Allow other nodes on VPN to access all ports
iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
'';
# When stopping firewall, delete all rules that were configured manually above
extraStopCommands = ''
2022-05-08 22:29:17 +00:00
iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
'';
};
};
}