Deuxfleurs/nixcfg
17
6
Fork 2
Code Issues 15 Pull requests 2 Projects Releases Wiki Activity

Remove wesher, reconfigure staging without it

Browse source
This commit is contained in:
Alex 2022-08-23 23:55:15 +02:00
parent a7ac31cdf5
commit ab901fc81d
Signed by: lx
GPG key ID: 0E496D15096376BE
8 changed files with 106 additions and 232 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
cluster/staging/cluster.nix
View file

@ -3,16 +3,36 @@
{
deuxfleurs.cluster_name = "staging";
# Bootstrap nodes for Wesher overlay network
services.wesher.join = [
"192.168.1.21" # cariacou
"192.168.1.22" # carcajou
"192.168.1.23" # caribou
];
# The IP range to use for the Wireguard overlay of this cluster
deuxfleurs.cluster_prefix = "10.14.0.0";
deuxfleurs.cluster_prefix_length = 16;
# The IP range to use for the Wesher overlay of this cluster
deuxfleurs.wesher_cluster_prefix = "10.14.0.0";
deuxfleurs.wesher_cluster_prefix_length = 16;
deuxfleurs.cluster_nodes = [
{
hostname = "cariacou";
site_name = "neptune";
publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
IP = "10.14.179.56";
lan_endpoint = "192.168.1.21:33799";
endpoint = "192.168.1.21:33799"; ## TODO nat
}
{
hostname = "carcajou";
site_name = "neptune";
publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
IP = "10.14.252.121";
lan_endpoint = "192.168.1.22:33799";
endpoint = "192.168.1.22:33799"; ## TODO nat
}
{
hostname = "caribou";
site_name = "neptune";
publicKey = "X1OlfgjZDdJw3yTg0ytGFlJZXzqT0yHJmYP15R9IpD8=";
IP = "10.14.181.82";
lan_endpoint = "192.168.1.23:33799";
endpoint = "192.168.1.23:33799"; ## TODO nat
}
];
# Bootstrap IPs for Consul cluster,
# these are IPs on the Wesher overlay
@ -22,13 +42,6 @@
"10.14.252.121" # carcajou
];
# Add wesher hosts to /etc/hosts, needed for elasticsearch
networking.extraHosts = ''
10.14.181.82 caribou
10.14.179.56 cariacou
10.14.242.121 carcajou
'';
deuxfleurs.admin_accounts = {
lx = [
# Keys for accessing nodes from outside

6
deploy_nixos
View file

@ -3,10 +3,12 @@
copy nix/configuration.nix /etc/nixos/configuration.nix
copy nix/deuxfleurs.nix /etc/nixos/deuxfleurs.nix
copy nix/remote-unlock.nix /etc/nixos/remote-unlock.nix
copy nix/wesher.nix /etc/nixos/wesher.nix
copy nix/wesher_service.nix /etc/nixos/wesher_service.nix
copy cluster/$CLUSTER/cluster.nix /etc/nixos/cluster.nix
copy cluster/$CLUSTER/node/$NIXHOST.nix /etc/nixos/node.nix
copy cluster/$CLUSTER/node/$NIXHOST.site.nix /etc/nixos/site.nix
cmd 'mkdir -p /var/lib/deuxfleurs/wireguard-keys'
cmd 'test -f /var/lib/deuxfleurs/wireguard-keys/private || (wg genkey > /var/lib/deuxfleurs/wireguard-keys/private; chmod 600 /var/lib/deuxfleurs/wireguard-keys/private)'
cmd 'echo "Public key: $(wg pubkey < /var/lib/deuxfleurs/wireguard-keys/private)"'
cmd nixos-rebuild switch --show-trace

4
deploy_wesher_key
View file

@ -1,4 +0,0 @@
#!/usr/bin/env ./sshtool
write_pass deuxfleurs/cluster/$CLUSTER/wesher_key /var/lib/wesher/secrets
cmd systemctl restart wesher

17
gen_wesher_key
View file

@ -1,17 +0,0 @@
#!/usr/bin/env sh
cd $(dirname $0)
CLUSTER="$1"
if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
echo "Usage: $0 <cluster name>"
echo "The cluster name must be the name of a subdirectory of cluster/"
exit 1
fi
K=deuxfleurs/cluster/$CLUSTER/wesher_key
if ! pass $K >/dev/null; then
pass insert -m $K <<EOF
WESHER_CLUSTER_KEY=$(head -c 32 /dev/urandom | base64)
EOF
fi

7
nix/configuration.nix
View file

@ -11,8 +11,6 @@
./hardware-configuration.nix
# Include generic Deuxfleurs module
./deuxfleurs.nix
# Wesher module
./wesher_service.nix
# Configuration for this deployment (a cluster)
./cluster.nix
# Configuration local for this Deuxfleurs site (set of nodes)
@ -21,10 +19,6 @@
./node.nix
];
nixpkgs.overlays = [
(import ./wesher.nix)
];
# The global useDHCP flag is deprecated, therefore explicitly set to false here.
# Per-interface useDHCP will be mandatory in the future, so this generated config
# replicates the default behaviour.
@ -70,7 +64,6 @@ SystemMaxUse=1G
docker
docker-compose
wireguard-tools
wesher
#qemu
#qemu_kvm
];

97
nix/deuxfleurs.nix
View file

@ -7,6 +7,37 @@ in
with pkgs.lib;
{
options.deuxfleurs =
let wg_node = with types; submodule {
options = {
hostname = mkOption {
type = str;
description = "Host name";
};
site_name = mkOption {
type = nullOr str;
description = "Site where the node is located";
default = null;
};
IP = mkOption {
type = str;
description = "IP Address in the Wireguard network";
};
publicKey = mkOption {
type = str;
description = "Public key";
};
endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint on the public Internet";
};
lan_endpoint = mkOption {
type = nullOr str;
description = "Wireguard endpoint for nodes in the same site";
default = null;
};
};
};
in
{
# Parameters for individual nodes
network_interface = mkOption {
@ -30,27 +61,22 @@ in
type = types.int;
};
wesher_cluster_prefix = mkOption {
description = "IP address prefix for the Wesher overlay network";
type = types.str;
};
wesher_cluster_prefix_length = mkOption {
description = "IP address prefix length for the Wesher overlay network";
type = types.int;
default = 16;
};
cluster_ip = mkOption {
description = "IP address of this node on the Wesher mesh network";
type = types.str;
};
wireguard_port = mkOption {
description = "Port for incoming Wireguard VPN connections";
type = types.port;
default = 33799;
};
is_raft_server = mkOption {
description = "Make this node a RAFT server for the Nomad and Consul deployments";
type = types.bool;
default = false;
};
# Parameters that generally vary between sites
lan_default_gateway = mkOption {
description = "IP address of the default route on the locak network interface";
@ -70,6 +96,19 @@ in
description = "Name of this Deuxfleurs deployment";
type = types.str;
};
cluster_prefix = mkOption {
description = "IP address prefix for the Wireguard overlay network";
type = types.str;
};
cluster_prefix_length = mkOption {
description = "IP address prefix length for the Wireguard overlay network";
type = types.int;
default = 16;
};
cluster_nodes = mkOption {
description = "Nodes that are part of the cluster";
type = types.listOf wg_node;
};
admin_accounts = mkOption {
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
type = types.attrsOf (types.listOf types.str);
@ -116,19 +155,24 @@ in
cfg.lan_ip
] ++ cfg.nameservers;
# wesher overlay network
services.wesher = {
enable = true;
bindIface = cfg.network_interface;
overlayNet = "${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length}";
interface = "wg0";
logLevel = "debug";
# Configure Wireguard VPN between all nodes
networking.wireguard.interfaces.wg0 = {
ips = [ "${cfg.cluster_ip}/16" ];
listenPort = cfg.wireguard_port;
privateKeyFile = "/var/lib/deuxfleurs/wireguard-keys/private";
peers = map ({ publicKey, endpoint, IP, site_name, lan_endpoint, ... }: {
publicKey = publicKey;
allowedIPs = [ "${IP}/32" ];
endpoint = if site_name != null && site_name == cfg.site_name && lan_endpoint != null
then lan_endpoint else endpoint;
persistentKeepalive = 25;
}) cfg.cluster_nodes;
};
# Configure /etc/hosts to link all hostnames to their Wireguard IP
#networking.extraHosts = builtins.concatStringsSep "\n" (map
# ({ hostname, IP, ...}: "${IP} ${hostname}")
# (cfg.cluster_nodes ++ cfg.admin_nodes));
networking.extraHosts = builtins.concatStringsSep "\n" (map
({ hostname, IP, ...}: "${IP} ${hostname}")
cfg.cluster_nodes);
# Enable Hashicorp Consul & Nomad
services.consul.enable = true;
@ -245,11 +289,16 @@ in
networking.firewall = {
enable = true;
# Allow anyone to connect on SSH port
allowedTCPPorts = [
# Allow anyone to connect on SSH port
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
];
allowedUDPPorts = [
# Allow peers to connect to Wireguard
cfg.wireguard_port
];
# Allow specific hosts access to specific things in the cluster
extraCommands = ''
# Allow everything from router (usefull for UPnP/IGD)
@ -259,14 +308,14 @@ in
iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT
# Allow other nodes on VPN to access all ports
iptables -A INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
iptables -A INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
'';
# When stopping firewall, delete all rules that were configured manually above
extraStopCommands = ''
iptables -D INPUT -s ${cfg.lan_default_gateway} -j ACCEPT
iptables -D INPUT -s 172.17.0.0/16 -j ACCEPT
iptables -D INPUT -s ${cfg.wesher_cluster_prefix}/${toString cfg.wesher_cluster_prefix_length} -j ACCEPT
iptables -D INPUT -s ${cfg.cluster_prefix}/${toString cfg.cluster_prefix_length} -j ACCEPT
'';
};
};

25
nix/wesher.nix
View file

@ -1,25 +0,0 @@
self: super:
{
wesher = super.buildGoModule rec {
pname = "wesher";
version = "0.2.6";
src = super.fetchFromGitHub {
owner = "costela";
repo = "wesher";
rev = "v${version}";
sha256 = "1iagmnw2yf15r0fpikk610w0lm0gcxw83lcwfjyr2jv1q2ys71hh";
};
vendorSha256 = "0nyg0wzn8d4rzjs8yrxxj3gha94043ll80s1ql0fml025q2f3705";
checkPhase = "true";
meta = with super.lib; {
description = "wireguard overlay mesh network manager";
homepage = "https://github.com/costela/wesher";
license = licenses.gpl3Plus;
platforms = platforms.linux;
};
};
}

137
nix/wesher_service.nix
View file

@ -1,137 +0,0 @@
{ config, lib, pkgs, ... }:
with lib;
let
keysPath = "/var/lib/wesher/secrets";
cfg = config.services.wesher;
in {
options = with types; {
services.wesher = {
enable = mkEnableOption "wesher wireguard overlay mesh network manager";
package = mkOption {
type = package;
default = pkgs.wesher;
defaultText = literalExpression "pkgs.wesher";
description = "Wesher package to use.";
};
clusterKey = mkOption {
type = nullOr str;
default = null;
description = "shared key for cluster membership to use on first initialization, if no key was previously used by Wesher. Must be 32 bytes base64 encoded; will be generated if not provided. Setting this parameter value will not overwrite an existing cluster key; to do so please delete ${keysPath}";
};
bindAddr = mkOption {
type = nullOr str;
default = null;
description = "IP address to bind to for cluster membership (cannot be used with --bind-iface)";
};
bindIface = mkOption {
type = nullOr str;
default = null;
description = "Interface to bind to for cluster membership (cannot be used with --bind-addr)";
};
join = mkOption {
type = listOf str;
default = [];
description = "list of hostnames or IP addresses to existing cluster members; if not provided, will attempt resuming any known state or otherwise wait for further members";
};
clusterPort = mkOption {
type = port;
default = 7946;
description = "port used for membership gossip traffic (both TCP and UDP); must be the same accross cluster";
};
wireguardPort = mkOption {
type = port;
default = 51820;
description = "port used for wireguard traffic (UDP); must be the same accross cluster";
};
overlayNet = mkOption {
type = str;
default = "10.0.0.0/8";
description = "the network in which to allocate addresses for the overlay mesh network (CIDR format); smaller networks increase the chance of IP collision";
};
interface = mkOption {
type = str;
default = "wgoverlay";
description = "name of the wireguard interface to create and manage";
};
logLevel = mkOption {
type = str;
default = "warn";
description = "set the verbosity (one of debug/info/warn/error)";
};
};
};
config = mkIf cfg.enable (let binWesher = cfg.package + "/bin/wesher";
in {
system.activationScripts.wesher = if (cfg.clusterKey != null) then ''
if [ ! -e ${keysPath} ]
then
mkdir --mode=700 -p ${builtins.dirOf keysPath}
echo "WESHER_CLUSTER_KEY=${cfg.clusterKey}" > ${keysPath}
fi
'' else ''
if [ ! -e ${keysPath} ]
then
mkdir --mode=700 -p ${builtins.dirOf keysPath}
echo "WESHER_CLUSTER_KEY=$(head -c 32 /dev/urandom | base64)" > ${keysPath}
fi
'';
systemd.services.wesher = {
description = "wesher wireguard overlay mesh network manager";
bindsTo = [ "network-online.target" ];
after = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
environment = {
WESHER_JOIN = builtins.concatStringsSep "," cfg.join;
WESHER_CLUSTER_PORT = builtins.toString cfg.clusterPort;
WESHER_WIREGUARD_PORT = builtins.toString cfg.wireguardPort;
WESHER_OVERLAY_NET = cfg.overlayNet;
WESHER_INTERFACE = cfg.interface;
WESHER_LOG_LEVEL = cfg.logLevel;
WESHER_NO_ETC_HOSTS = "true";
}
// (if (cfg.bindAddr != null) then { WESHER_BIND_ADDR = cfg.bindAddr; } else {})
// (if (cfg.bindIface != null) then { WESHER_BIND_IFACE = cfg.bindIface; } else {})
;
serviceConfig = {
ExecStart = "${binWesher}";
Restart = "always";
EnvironmentFile = keysPath;
User = "wesher";
DynamicUser = true;
StateDirectory = "wesher";
AmbientCapabilities = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE";
CapabilityBoundingSet = "CAP_NET_ADMIN CAP_NET_BIND_SERVICE";
MemoryDenyWriteExecute = true;
ProtectControlGroups = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6 AF_NETLINK";
RestrictNamespaces = true;
RestrictRealtime = true;
SystemCallArchitectures = "native";
SystemCallFilter = "~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @resources";
};
};
networking.firewall.allowedUDPPorts = mkIf cfg.enable [ cfg.clusterPort cfg.wireguardPort ];
networking.firewall.allowedTCPPorts = mkIf cfg.enable [ cfg.clusterPort ];
});
}