323 lines
9.0 KiB
Nix
323 lines
9.0 KiB
Nix
{ config, pkgs, ... }:
|
|
|
|
let
|
|
cfg = config.deuxfleurs;
|
|
in
|
|
with builtins;
|
|
with pkgs.lib;
|
|
{
|
|
options.deuxfleurs =
|
|
let wg_node = with types; submodule {
|
|
options = {
|
|
hostname = mkOption {
|
|
type = str;
|
|
description = "Host name";
|
|
};
|
|
site_name = mkOption {
|
|
type = nullOr str;
|
|
description = "Site where the node is located";
|
|
default = null;
|
|
};
|
|
IP = mkOption {
|
|
type = str;
|
|
description = "IP Address in the Wireguard network";
|
|
};
|
|
publicKey = mkOption {
|
|
type = str;
|
|
description = "Public key";
|
|
};
|
|
endpoint = mkOption {
|
|
type = nullOr str;
|
|
description = "Wireguard endpoint on the public Internet";
|
|
};
|
|
lan_endpoint = mkOption {
|
|
type = nullOr str;
|
|
description = "Wireguard endpoint for nodes in the same site";
|
|
default = null;
|
|
};
|
|
};
|
|
};
|
|
in
|
|
{
|
|
# Parameters for individual nodes
|
|
network_interface = mkOption {
|
|
description = "Network interface name to configure";
|
|
type = types.str;
|
|
};
|
|
lan_ip = mkOption {
|
|
description = "IP address of this node on the local network interface";
|
|
type = types.str;
|
|
};
|
|
lan_ip_prefix_length = mkOption {
|
|
description = "Prefix length associated with lan_ip";
|
|
type = types.int;
|
|
};
|
|
ipv6 = mkOption {
|
|
description = "Public IPv6 address of this node";
|
|
type = types.str;
|
|
};
|
|
ipv6_prefix_length = mkOption {
|
|
description = "Prefix length associated with ipv6 ip";
|
|
type = types.int;
|
|
};
|
|
|
|
cluster_ip = mkOption {
|
|
description = "IP address of this node on the Wesher mesh network";
|
|
type = types.str;
|
|
};
|
|
wireguard_port = mkOption {
|
|
description = "Port for incoming Wireguard VPN connections";
|
|
type = types.port;
|
|
default = 33799;
|
|
};
|
|
|
|
is_raft_server = mkOption {
|
|
description = "Make this node a RAFT server for the Nomad and Consul deployments";
|
|
type = types.bool;
|
|
default = false;
|
|
};
|
|
|
|
# Parameters that generally vary between sites
|
|
lan_default_gateway = mkOption {
|
|
description = "IP address of the default route on the locak network interface";
|
|
type = types.str;
|
|
};
|
|
site_name = mkOption {
|
|
description = "Site (availability zone) on which this node is deployed";
|
|
type = types.str;
|
|
};
|
|
nameservers = mkOption {
|
|
description = "External DNS servers to use";
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
# Parameters common to all nodes
|
|
cluster_name = mkOption {
|
|
description = "Name of this Deuxfleurs deployment";
|
|
type = types.str;
|
|
};
|
|
cluster_prefix = mkOption {
|
|
description = "IP address prefix for the Wireguard overlay network";
|
|
type = types.str;
|
|
};
|
|
cluster_prefix_length = mkOption {
|
|
description = "IP address prefix length for the Wireguard overlay network";
|
|
type = types.int;
|
|
default = 16;
|
|
};
|
|
cluster_nodes = mkOption {
|
|
description = "Nodes that are part of the cluster";
|
|
type = types.listOf wg_node;
|
|
};
|
|
admin_accounts = mkOption {
|
|
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
|
|
type = types.attrsOf (types.listOf types.str);
|
|
};
|
|
};
|
|
|
|
config = {
|
|
# Configure admin accounts on all nodes
|
|
users.users = builtins.mapAttrs (name: publicKeys: {
|
|
isNormalUser = true;
|
|
extraGroups = [ "wheel" ];
|
|
openssh.authorizedKeys.keys = publicKeys;
|
|
}) cfg.admin_accounts;
|
|
|
|
# Configure network interfaces
|
|
networking.interfaces =
|
|
let ip4config = {
|
|
useDHCP = false;
|
|
ipv4.addresses = [
|
|
{
|
|
address = cfg.lan_ip;
|
|
prefixLength = cfg.lan_ip_prefix_length;
|
|
}
|
|
];
|
|
};
|
|
ip6config = {
|
|
tempAddress = "disabled";
|
|
ipv6.addresses = [
|
|
{
|
|
address = cfg.ipv6;
|
|
prefixLength = cfg.ipv6_prefix_length;
|
|
}
|
|
];
|
|
};
|
|
in
|
|
(attrsets.setAttrByPath [ cfg.network_interface ] (ip4config // ip6config));
|
|
|
|
networking.defaultGateway = {
|
|
address = cfg.lan_default_gateway;
|
|
interface = cfg.network_interface;
|
|
};
|
|
|
|
networking.nameservers = [
|
|
cfg.lan_ip
|
|
] ++ cfg.nameservers;
|
|
|
|
# Configure Wireguard VPN between all nodes
|
|
networking.wireguard.interfaces.wg0 = {
|
|
ips = [ "${cfg.cluster_ip}/16" ];
|
|
listenPort = cfg.wireguard_port;
|
|
privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
|
|
peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
|
|
publicKey = publicKey;
|
|
allowedIPs = [ "${IP}/32" ];
|
|
endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
|
|
then lan_endpoint else endpoint;
|
|
persistentKeepalive = 25;
|
|
}) cfg.cluster_nodes;
|
|
};
|
|
|
|
# Configure /etc/hosts to link all hostnames to their Wireguard IP
|
|
networking.extraHosts = builtins.concatStringsSep "\n" (map
|
|
({ hostname, IP, ...}: "${IP} ${hostname}")
|
|
cfg.cluster_nodes);
|
|
|
|
# Enable Hashicorp Consul & Nomad
|
|
services.consul.enable = true;
|
|
services.consul.extraConfig =
|
|
(if cfg.is_raft_server
|
|
then {
|
|
server = true;
|
|
bootstrap_expect = 3;
|
|
}
|
|
else {}) //
|
|
{
|
|
datacenter = cfg.cluster_name;
|
|
node_meta = {
|
|
"site" = cfg.site_name;
|
|
};
|
|
ui_config = {
|
|
enabled = true;
|
|
};
|
|
bind_addr = "${cfg.cluster_ip}";
|
|
|
|
addresses = {
|
|
https = "0.0.0.0";
|
|
dns = "0.0.0.0";
|
|
};
|
|
ports = {
|
|
http = -1;
|
|
https = 8501;
|
|
dns = 53;
|
|
};
|
|
|
|
recursors = [ cfg.nameservers ];
|
|
|
|
ca_file = "/var/lib/consul/pki/consul-ca.crt";
|
|
cert_file = "/var/lib/consul/pki/consul2022.crt";
|
|
key_file = "/var/lib/consul/pki/consul2022.key";
|
|
verify_incoming = true;
|
|
verify_outgoing = true;
|
|
verify_server_hostname = true;
|
|
};
|
|
systemd.services.consul.serviceConfig = {
|
|
AmbientCapabilities = "CAP_NET_BIND_SERVICE";
|
|
};
|
|
|
|
services.nomad.enable = true;
|
|
services.nomad.package = pkgs.nomad_1_3;
|
|
services.nomad.extraPackages = [
|
|
pkgs.glibc
|
|
pkgs.zstd
|
|
#pkgs.qemu
|
|
#pkgs.qemu_kvm
|
|
];
|
|
services.nomad.settings =
|
|
(if cfg.is_raft_server
|
|
then { server = {
|
|
enabled = true;
|
|
bootstrap_expect = 3;
|
|
}; }
|
|
else {}) //
|
|
{
|
|
region = cfg.cluster_name;
|
|
datacenter = cfg.site_name;
|
|
advertise = {
|
|
rpc = "${cfg.cluster_ip}";
|
|
http = "${cfg.cluster_ip}";
|
|
serf = "${cfg.cluster_ip}";
|
|
};
|
|
consul = {
|
|
address = "localhost:8501";
|
|
ca_file = "/var/lib/nomad/pki/consul2022.crt";
|
|
cert_file = "/var/lib/nomad/pki/consul2022-client.crt";
|
|
key_file = "/var/lib/nomad/pki/consul2022-client.key";
|
|
ssl = true;
|
|
};
|
|
client = {
|
|
enabled = true;
|
|
network_interface = "wg0";
|
|
meta = {
|
|
"site" = cfg.site_name;
|
|
};
|
|
};
|
|
tls = {
|
|
http = true;
|
|
rpc = true;
|
|
ca_file = "/var/lib/nomad/pki/nomad-ca.crt";
|
|
cert_file = "/var/lib/nomad/pki/nomad2022.crt";
|
|
key_file = "/var/lib/nomad/pki/nomad2022.key";
|
|
verify_server_hostname = true;
|
|
verify_https_client = true;
|
|
};
|
|
plugin = [
|
|
{
|
|
docker = [
|
|
{
|
|
config = [
|
|
{
|
|
volumes.enabled = true;
|
|
allow_privileged = true;
|
|
}
|
|
];
|
|
}
|
|
];
|
|
#qemu = [
|
|
# {
|
|
# enabled = true;
|
|
# }
|
|
#];
|
|
}
|
|
];
|
|
};
|
|
|
|
# ---- Firewall config ----
|
|
|
|
# Open ports in the firewall.
|
|
networking.firewall = {
|
|
enable = true;
|
|
|
|
allowedTCPPorts = [
|
|
# Allow anyone to connect on SSH port
|
|
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
|
|
];
|
|
|
|
allowedUDPPorts = [
|
|
# Allow peers to connect to Wireguard
|
|
cfg.wireguard_port
|
|
];
|
|
|
|
# Allow specific hosts access to specific things in the cluster
|
|
extraCommands = ''
|
|
# Allow everything from router (usefull for UPnP/IGD)
|
|
iptables -A INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
|
|
|
|
# Allow docker containers to access all ports
|
|
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
|
|
|
|
# Allow other nodes on VPN to access all ports
|
|
iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
|
|
'';
|
|
|
|
# When stopping firewall, delete all rules that were configured manually above
|
|
extraStopCommands = ''
|
|
iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
|
|
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
|
|
iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
|
|
'';
|
|
};
|
|
};
|
|
}
|