forked from Deuxfleurs/nixcfg
Modularize and prepare to support multiple clusters
This commit is contained in:
parent
cce5cd17f5
commit
f03cafd49b
20 changed files with 356 additions and 252 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -1,3 +1,6 @@
|
|||
notes/
|
||||
secrets/*
|
||||
!secrets/*.sample
|
||||
cluster/*/secrets/*
|
||||
!cluster/*/secrets/*.sample
|
||||
|
||||
|
|
|
|||
77
cluster/staging/cluster.nix
Normal file
77
cluster/staging/cluster.nix
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
{ config, pkgs, ... } @ args:
|
||||
|
||||
{
|
||||
deuxfleurs.cluster_name = "staging";
|
||||
deuxfleurs.cluster_nodes = [
|
||||
{
|
||||
hostname = "spoutnik";
|
||||
publicKey = "fO8qZOZmnug84cA8nvfjl5MUqyWljP0BAz/4tHRZyEg=";
|
||||
IP = "10.42.0.2";
|
||||
endpoint = "77.141.67.109:42136";
|
||||
}
|
||||
{
|
||||
hostname = "cariacou";
|
||||
publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
|
||||
IP = "10.42.0.21";
|
||||
endpoint = "82.66.112.151:33721";
|
||||
}
|
||||
{
|
||||
hostname = "carcajou";
|
||||
publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
|
||||
IP = "10.42.0.22";
|
||||
endpoint = "82.66.112.151:33722";
|
||||
}
|
||||
{
|
||||
hostname = "caribou";
|
||||
publicKey = "g6ZED/wPn5MPfytJKwPI19808CXtEad0IJUkEAAzwyY=";
|
||||
IP = "10.42.0.23";
|
||||
endpoint = "82.66.112.151:33723";
|
||||
}
|
||||
];
|
||||
deuxfleurs.admin_nodes = [
|
||||
{
|
||||
hostname = "hammerhead";
|
||||
publicKey = "b5hF+GSTgg3oM6wnjL7jRbfyf1jtsWdVptPPbAh3Qic=";
|
||||
IP = "10.42.0.1";
|
||||
endpoint = "5.135.179.11:51349";
|
||||
}
|
||||
{
|
||||
hostname = "robinson";
|
||||
publicKey = "ETaZFil3mFXlJ0LaJZyWqJVLV2IZUF5PB/8M7WbQSTg=";
|
||||
IP = "10.42.0.42";
|
||||
endpoint = "77.141.67.109:33742";
|
||||
}
|
||||
{
|
||||
hostname = "shiki";
|
||||
publicKey = "QUiUNMk70TEQ75Ut7Uqikr5uGVSXmx8EGNkGM6tANlg=";
|
||||
IP = "10.42.0.206";
|
||||
endpoint = "37.187.118.206:51820";
|
||||
}
|
||||
{
|
||||
hostname = "lindy";
|
||||
publicKey = "wen9GnZy2iLT6RyHfn7ydS/wvdvow1XPmhZxIkrDbks=";
|
||||
IP = "10.42.0.66";
|
||||
endpoint = "82.66.112.151:33766";
|
||||
}
|
||||
];
|
||||
deuxfleurs.admin_accounts = {
|
||||
lx = [
|
||||
# Keys for accessing nodes from outside
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy"
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDIDdVbA9fEdqSr5UJ77NnoIqDTVp8ca5kHExhZYI4ecBExFJfonJllXMBN9KdC4ukxtY8Ug47PcMOfMaTBZQc+e+KpvDWpkBt15Xpem3RCxmMBES79sLL7LgtAdBXc5mNaCX8EOEVixWKdarjvxRyf6py6the51G5muaiMpoj5fae4ZpRGjhGTPefzc7y7zRWBUUZ8pYHW774BIaK6XT9gn3hyHV+Occjl/UODXvodktk55YtnuPi8adXTYEsHrVVz8AkFhx+cr0U/U8vtQnsTrZG+JmgQLqpXVs0RDw5bE1RefEbMuYNKxutYKUe3L+ZJtDe0M0MqOFI8a4F5TxP5 katchup@konata"
|
||||
];
|
||||
quentin = [
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDT1+H08FdUSvdPpPKdcafq4+JRHvFVjfvG5Id97LAoROmFRUb/ZOMTLdNuD7FqvW0Da5CPxIMr8ZxfrFLtpGyuG7qdI030iIRZPlKpBh37epZHaV+l9F4ZwJQMIBO9cuyLPXgsyvM/s7tDtrdK1k7JTf2EVvoirrjSzBaMhAnhi7//to8zvujDtgDZzy6aby75bAaDetlYPBq2brWehtrf9yDDG9WAMYJqp//scje/WmhbRR6eSdim1HaUcWk5+4ZPt8sQJcy8iWxQ4jtgjqTvMOe5v8ZPkxJNBine/ZKoJsv7FzKem00xEH7opzktaGukyEqH0VwOwKhmBiqsX2yN quentin@dufour.io"
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBu+KUebaWwlugMC5fGbNhHc6IaQDAC6+1vMc4Ww7nVU1rs2nwI7L5qcWxOwNdhFaorZQZy/fJuCWdFbF61RCKGayBWPLZHGPsfqDuggYNEi1Qil1kpeCECfDQNjyMTK058ZBBhOWNMHBjlLWXUlRJDkRBBECY0vo4jRv22SvSaPUCAnkdJ9rbAp/kqb497PTIb2r1l1/ew8YdhINAlpYQFQezZVfkZdTKxt22n0QCjhupqjfh3gfNnbBX0z/iO+RvAOWRIZsjPFLC+jXl+n7cnu2cq1nvST5eHiYfXXeIgIwmeENLKqp+2Twr7PIdv22PnJkh6iR5kx7eTRxkNZdN quentin@deuxfleurs.fr"
|
||||
];
|
||||
adrien = [
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBfVX+iQSHl3V0el3/y2Rtl9Q/nrmLoTE3oXnR+16yX7g8HvzU871q89jbE/UWvNRvO4hirTcKF8yojuq8ZRCoUcQO+6/YlPrY/2G8kFhPTlUGDQ+mLT+ancZsom4mkg3I9oQjKZ9qxMD1GuU8Ydz4eXjhJ8OGFZhBpEgnrLmdA53Y5d2fCbaZN5EYD4sWEFYN7xBLxTGNwv0gygiPs967Z4/ZfHngTvqVoS9wnQThSCIoXPTWFAJCkN8dC5tPZwnbOT1bGcYUF0VTrcaD6cU6Q1ZRrtyqXxnnyxpQCAoe2hgdIm+LnDsBx9trfPauqi0dXi36X8pLmudW1f1RmKWT adrien@bacigalupi"
|
||||
];
|
||||
maximilien = [
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHMMR6zNzz8NQU80wFquhUCeiXJuGphjP+zNouKbn228GyESu8sfNBwnuZq86vblR11Lz8l2rtCM73GfAKg29qmUWUHRKWvRIYWv2vaUJcCdy0bAxIzcvCvjZX0SpnIKxe9y3Rp0LGO5WLYfw0ZFaavwFZP0Z8w1Kj9/zBmL2X2avbhkaYHi/C1yXhbvESYQysmqLa48EX/TS616MBrgR9zbI9AoTQ9NOHnR14Tve/AP/khcZoBJdm4hTttMbNkEc0wonzdylTDew263SPRs/uoqnQIpUtErdPHqU10Yup8HjXjEyFJsSwcZcM5sZOw5JKckKJwmcd0yjO/x/4/Mk5"
|
||||
];
|
||||
kokakiwi = [
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPTsEgcOtb2bij+Ih8eg8ZqO7d3IMiWykv6deMzlSSS kokakiwi@kira"
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
@ -24,15 +24,10 @@
|
|||
}
|
||||
];
|
||||
|
||||
networking.wireguard.interfaces.wg0 = {
|
||||
ips = [ "10.42.0.22/16" ];
|
||||
listenPort = 33722;
|
||||
};
|
||||
deuxfleurs.vpn_ip = "10.42.0.22";
|
||||
deuxfleurs.vpn_listen_port = 33722;
|
||||
deuxfleurs.is_raft_server = true;
|
||||
|
||||
# Enable netdata monitoring
|
||||
services.netdata.enable = true;
|
||||
|
||||
# Activate as Nomad and Consul server node
|
||||
services.nomad.settings.server.enabled = true;
|
||||
services.consul.extraConfig.server = true;
|
||||
}
|
||||
|
|
@ -24,10 +24,9 @@
|
|||
}
|
||||
];
|
||||
|
||||
networking.wireguard.interfaces.wg0 = {
|
||||
ips = [ "10.42.0.21/16" ];
|
||||
listenPort = 33721;
|
||||
};
|
||||
deuxfleurs.vpn_ip = "10.42.0.21";
|
||||
deuxfleurs.vpn_listen_port = 33721;
|
||||
deuxfleurs.is_raft_server = true;
|
||||
|
||||
# Enable netdata monitoring
|
||||
services.netdata.enable = true;
|
||||
|
|
@ -24,15 +24,10 @@
|
|||
}
|
||||
];
|
||||
|
||||
networking.wireguard.interfaces.wg0 = {
|
||||
ips = [ "10.42.0.23/16" ];
|
||||
listenPort = 33723;
|
||||
};
|
||||
deuxfleurs.vpn_ip = "10.42.0.23";
|
||||
deuxfleurs.vpn_listen_port = 33723;
|
||||
deuxfleurs.is_raft_server = true;
|
||||
|
||||
# Enable netdata monitoring
|
||||
services.netdata.enable = true;
|
||||
|
||||
# Activate as Nomad and Consul server node
|
||||
services.nomad.settings.server.enabled = true;
|
||||
services.consul.extraConfig.server = true;
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
interface = "eno1";
|
||||
};
|
||||
|
||||
services.nomad.settings.datacenter = "neptune";
|
||||
deuxfleurs.site_name = "neptune";
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 80 443 ];
|
||||
|
||||
|
|
@ -7,7 +7,7 @@
|
|||
};
|
||||
networking.nameservers = [ "213.186.33.99" "172.104.136.243" ];
|
||||
|
||||
services.nomad.settings.datacenter = "pluton";
|
||||
deuxfleurs.site_name = "pluton";
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 80 443 ];
|
||||
}
|
||||
|
|
@ -5,17 +5,18 @@
|
|||
{ config, pkgs, ... } @ args:
|
||||
|
||||
# Configuration local for this cluster node (hostname, IP, etc)
|
||||
let node_config = import ./node.nix args;
|
||||
site_config = import ./site.nix args;
|
||||
in
|
||||
{
|
||||
imports =
|
||||
[ # Include the results of the hardware scan.
|
||||
./hardware-configuration.nix
|
||||
# Configuration local for this cluster node (hostname, IP, etc)
|
||||
./node.nix
|
||||
# Include generic Deuxfleurs module
|
||||
./deuxfleurs.nix
|
||||
# Configuration for this deployment (a cluster)
|
||||
./cluster.nix
|
||||
# Configuration local for this Deuxfleurs site (set of nodes)
|
||||
./site.nix
|
||||
# Configuration local for this cluster node (hostname, IP, etc)
|
||||
./node.nix
|
||||
];
|
||||
|
||||
# The global useDHCP flag is deprecated, therefore explicitly set to false here.
|
||||
|
|
@ -27,77 +28,9 @@ in
|
|||
# Networking configuration (static IPs for each node is defined in node/*.nix)
|
||||
networking.nameservers = [ "9.9.9.9" ];
|
||||
|
||||
# Wireguard VPN configuration
|
||||
networking.wireguard.interfaces.wg0 = {
|
||||
privateKeyFile = "/root/wireguard-keys/private";
|
||||
peers = [
|
||||
{ # Hammerhead
|
||||
publicKey = "b5hF+GSTgg3oM6wnjL7jRbfyf1jtsWdVptPPbAh3Qic=";
|
||||
allowedIPs = [ "10.42.0.1/32" ];
|
||||
endpoint = "5.135.179.11:51349";
|
||||
persistentKeepalive = 25;
|
||||
}
|
||||
{ # Spoutnik
|
||||
publicKey = "fO8qZOZmnug84cA8nvfjl5MUqyWljP0BAz/4tHRZyEg=";
|
||||
allowedIPs = [ "10.42.0.2/32" ];
|
||||
endpoint = "77.141.67.109:42136";
|
||||
persistentKeepalive = 25;
|
||||
}
|
||||
{ # Robinson
|
||||
publicKey = "ETaZFil3mFXlJ0LaJZyWqJVLV2IZUF5PB/8M7WbQSTg=";
|
||||
allowedIPs = [ "10.42.0.42/32" ];
|
||||
endpoint = "77.141.67.109:33742";
|
||||
persistentKeepalive = 25;
|
||||
}
|
||||
{ # Shiki
|
||||
publicKey = "QUiUNMk70TEQ75Ut7Uqikr5uGVSXmx8EGNkGM6tANlg=";
|
||||
allowedIPs = [ "10.42.0.206/32" ];
|
||||
endpoint = "37.187.118.206:51820";
|
||||
persistentKeepalive = 25;
|
||||
}
|
||||
{ # Lindy
|
||||
publicKey = "wen9GnZy2iLT6RyHfn7ydS/wvdvow1XPmhZxIkrDbks=";
|
||||
allowedIPs = [ "10.42.0.66/32" ];
|
||||
endpoint = "82.66.112.151:33766";
|
||||
persistentKeepalive = 25;
|
||||
}
|
||||
{ # Carcajou
|
||||
publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
|
||||
allowedIPs = [ "10.42.0.21/32" ];
|
||||
endpoint = "82.66.112.151:33721";
|
||||
persistentKeepalive = 25;
|
||||
}
|
||||
{ # Carcajou
|
||||
publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
|
||||
allowedIPs = [ "10.42.0.22/32" ];
|
||||
endpoint = "82.66.112.151:33722";
|
||||
persistentKeepalive = 25;
|
||||
}
|
||||
{ # Caribou
|
||||
publicKey = "g6ZED/wPn5MPfytJKwPI19808CXtEad0IJUkEAAzwyY=";
|
||||
allowedIPs = [ "10.42.0.23/32" ];
|
||||
endpoint = "82.66.112.151:33723";
|
||||
persistentKeepalive = 25;
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
# Set your time zone.
|
||||
time.timeZone = "Europe/Paris";
|
||||
|
||||
networking.extraHosts = ''
|
||||
192.168.1.21 cariacou.lan
|
||||
192.168.1.22 carcajou.lan
|
||||
192.168.1.23 caribou.lan
|
||||
10.42.0.1 hammerhead
|
||||
10.42.0.2 spoutnik
|
||||
10.42.0.21 cariacou
|
||||
10.42.0.22 carcajou
|
||||
10.42.0.23 caribou
|
||||
10.42.0.66 lindy
|
||||
10.42.0.206 shiki
|
||||
'';
|
||||
|
||||
# Select internationalisation properties.
|
||||
# i18n.defaultLocale = "en_US.UTF-8";
|
||||
console = {
|
||||
|
|
@ -105,57 +38,6 @@ in
|
|||
keyMap = "fr";
|
||||
};
|
||||
|
||||
# Enable sound.
|
||||
# sound.enable = true;
|
||||
# hardware.pulseaudio.enable = true;
|
||||
|
||||
# Define user accounts
|
||||
users.users.lx = {
|
||||
isNormalUser = true;
|
||||
extraGroups = [
|
||||
"wheel" # Enable ‘sudo’ for the user.
|
||||
"video" # Having fun with links -g
|
||||
];
|
||||
openssh.authorizedKeys.keys = [
|
||||
# Keys for accessing nodes from outside
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy"
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDIDdVbA9fEdqSr5UJ77NnoIqDTVp8ca5kHExhZYI4ecBExFJfonJllXMBN9KdC4ukxtY8Ug47PcMOfMaTBZQc+e+KpvDWpkBt15Xpem3RCxmMBES79sLL7LgtAdBXc5mNaCX8EOEVixWKdarjvxRyf6py6the51G5muaiMpoj5fae4ZpRGjhGTPefzc7y7zRWBUUZ8pYHW774BIaK6XT9gn3hyHV+Occjl/UODXvodktk55YtnuPi8adXTYEsHrVVz8AkFhx+cr0U/U8vtQnsTrZG+JmgQLqpXVs0RDw5bE1RefEbMuYNKxutYKUe3L+ZJtDe0M0MqOFI8a4F5TxP5 katchup@konata"
|
||||
];
|
||||
};
|
||||
|
||||
users.users.quentin = {
|
||||
isNormalUser = true;
|
||||
extraGroups = [ "wheel" ];
|
||||
openssh.authorizedKeys.keys = [
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDT1+H08FdUSvdPpPKdcafq4+JRHvFVjfvG5Id97LAoROmFRUb/ZOMTLdNuD7FqvW0Da5CPxIMr8ZxfrFLtpGyuG7qdI030iIRZPlKpBh37epZHaV+l9F4ZwJQMIBO9cuyLPXgsyvM/s7tDtrdK1k7JTf2EVvoirrjSzBaMhAnhi7//to8zvujDtgDZzy6aby75bAaDetlYPBq2brWehtrf9yDDG9WAMYJqp//scje/WmhbRR6eSdim1HaUcWk5+4ZPt8sQJcy8iWxQ4jtgjqTvMOe5v8ZPkxJNBine/ZKoJsv7FzKem00xEH7opzktaGukyEqH0VwOwKhmBiqsX2yN quentin@dufour.io"
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBu+KUebaWwlugMC5fGbNhHc6IaQDAC6+1vMc4Ww7nVU1rs2nwI7L5qcWxOwNdhFaorZQZy/fJuCWdFbF61RCKGayBWPLZHGPsfqDuggYNEi1Qil1kpeCECfDQNjyMTK058ZBBhOWNMHBjlLWXUlRJDkRBBECY0vo4jRv22SvSaPUCAnkdJ9rbAp/kqb497PTIb2r1l1/ew8YdhINAlpYQFQezZVfkZdTKxt22n0QCjhupqjfh3gfNnbBX0z/iO+RvAOWRIZsjPFLC+jXl+n7cnu2cq1nvST5eHiYfXXeIgIwmeENLKqp+2Twr7PIdv22PnJkh6iR5kx7eTRxkNZdN quentin@deuxfleurs.fr"
|
||||
];
|
||||
};
|
||||
|
||||
users.users.adrien = {
|
||||
isNormalUser = true;
|
||||
extraGroups = [ "wheel" ];
|
||||
openssh.authorizedKeys.keys = [
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBfVX+iQSHl3V0el3/y2Rtl9Q/nrmLoTE3oXnR+16yX7g8HvzU871q89jbE/UWvNRvO4hirTcKF8yojuq8ZRCoUcQO+6/YlPrY/2G8kFhPTlUGDQ+mLT+ancZsom4mkg3I9oQjKZ9qxMD1GuU8Ydz4eXjhJ8OGFZhBpEgnrLmdA53Y5d2fCbaZN5EYD4sWEFYN7xBLxTGNwv0gygiPs967Z4/ZfHngTvqVoS9wnQThSCIoXPTWFAJCkN8dC5tPZwnbOT1bGcYUF0VTrcaD6cU6Q1ZRrtyqXxnnyxpQCAoe2hgdIm+LnDsBx9trfPauqi0dXi36X8pLmudW1f1RmKWT adrien@bacigalupi"
|
||||
];
|
||||
};
|
||||
|
||||
users.users.maximilien = {
|
||||
isNormalUser = true;
|
||||
extraGroups = [ "wheel" ];
|
||||
openssh.authorizedKeys.keys = [
|
||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHMMR6zNzz8NQU80wFquhUCeiXJuGphjP+zNouKbn228GyESu8sfNBwnuZq86vblR11Lz8l2rtCM73GfAKg29qmUWUHRKWvRIYWv2vaUJcCdy0bAxIzcvCvjZX0SpnIKxe9y3Rp0LGO5WLYfw0ZFaavwFZP0Z8w1Kj9/zBmL2X2avbhkaYHi/C1yXhbvESYQysmqLa48EX/TS616MBrgR9zbI9AoTQ9NOHnR14Tve/AP/khcZoBJdm4hTttMbNkEc0wonzdylTDew263SPRs/uoqnQIpUtErdPHqU10Yup8HjXjEyFJsSwcZcM5sZOw5JKckKJwmcd0yjO/x/4/Mk5"
|
||||
];
|
||||
};
|
||||
|
||||
users.users.kokakiwi = {
|
||||
isNormalUser = true;
|
||||
extraGroups = [ "wheel" ];
|
||||
openssh.authorizedKeys.keys = [
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPTsEgcOtb2bij+Ih8eg8ZqO7d3IMiWykv6deMzlSSS kokakiwi@kira"
|
||||
];
|
||||
};
|
||||
|
||||
# List packages installed in system profile. To search, run:
|
||||
# $ nix search wget
|
||||
environment.systemPackages = with pkgs; [
|
||||
|
|
@ -188,90 +70,6 @@ in
|
|||
|
||||
# ---- CONFIG FOR DEUXFLEURS CLUSTER ----
|
||||
|
||||
# Enable Hashicorp Consul & Nomad
|
||||
services.consul.enable = true;
|
||||
services.consul.extraConfig =
|
||||
let public_ip = (builtins.head (builtins.split "/" (builtins.head node_config.networking.wireguard.interfaces.wg0.ips)));
|
||||
in
|
||||
(if node_config.services.consul.extraConfig.server or false
|
||||
then { bootstrap_expect = 3; }
|
||||
else {}) //
|
||||
{
|
||||
datacenter = "staging";
|
||||
node_meta = {
|
||||
"site" = site_config.services.nomad.settings.datacenter;
|
||||
};
|
||||
ui = true;
|
||||
bind_addr = public_ip;
|
||||
|
||||
ports.http = -1;
|
||||
addresses.https = "0.0.0.0";
|
||||
ports.https = 8501;
|
||||
|
||||
retry_join = [ "10.42.0.2" "10.42.0.21" "10.42.0.22" "10.42.0.23" ];
|
||||
|
||||
ca_file = "/var/lib/consul/pki/consul-ca.crt";
|
||||
cert_file = "/var/lib/consul/pki/consul2022.crt";
|
||||
key_file = "/var/lib/consul/pki/consul2022.key";
|
||||
verify_incoming = true;
|
||||
verify_outgoing = true;
|
||||
verify_server_hostname = true;
|
||||
};
|
||||
|
||||
services.nomad.enable = true;
|
||||
services.nomad.package = pkgs.nomad_1_1;
|
||||
services.nomad.settings =
|
||||
let public_ip = (builtins.head (builtins.split "/" (builtins.head node_config.networking.wireguard.interfaces.wg0.ips)));
|
||||
in
|
||||
(if node_config.services.nomad.settings.server.enabled or false
|
||||
then { server = { bootstrap_expect = 3; }; }
|
||||
else {}) //
|
||||
{
|
||||
region = "staging";
|
||||
advertise = {
|
||||
rpc = public_ip;
|
||||
http = public_ip;
|
||||
serf = public_ip;
|
||||
};
|
||||
consul = {
|
||||
address = "localhost:8501";
|
||||
ca_file = "/var/lib/nomad/pki/consul2022.crt";
|
||||
cert_file = "/var/lib/nomad/pki/consul2022-client.crt";
|
||||
key_file = "/var/lib/nomad/pki/consul2022-client.key";
|
||||
ssl = true;
|
||||
};
|
||||
client = {
|
||||
enabled = true;
|
||||
network_interface = "wg0";
|
||||
meta = {
|
||||
"site" = site_config.services.nomad.settings.datacenter;
|
||||
};
|
||||
};
|
||||
tls = {
|
||||
http = true;
|
||||
rpc = true;
|
||||
ca_file = "/var/lib/nomad/pki/nomad-ca.crt";
|
||||
cert_file = "/var/lib/nomad/pki/nomad2022.crt";
|
||||
key_file = "/var/lib/nomad/pki/nomad2022.key";
|
||||
verify_server_hostname = true;
|
||||
verify_https_client = true;
|
||||
};
|
||||
plugin = [
|
||||
{
|
||||
docker = [
|
||||
{
|
||||
config = [
|
||||
{
|
||||
volumes.enabled = true;
|
||||
allow_privileged = true;
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
# Mount Garage using Rclone
|
||||
systemd.services.mountgarage = {
|
||||
enable = false;
|
||||
|
|
@ -296,12 +94,7 @@ in
|
|||
|
||||
# Allow anyone to connect on SSH port
|
||||
allowedTCPPorts = [
|
||||
(builtins.head ({ openssh.ports = [22]; } // node_config.services).openssh.ports)
|
||||
];
|
||||
|
||||
# Allow anyone to contact Wireguard VPN server
|
||||
allowedUDPPorts = [
|
||||
node_config.networking.wireguard.interfaces.wg0.listenPort
|
||||
(builtins.head ({ openssh.ports = [22]; } // config.services).openssh.ports)
|
||||
];
|
||||
|
||||
# Allow specific hosts access to specific things in the cluster
|
||||
|
|
|
|||
36
deploy.sh
36
deploy.sh
|
|
@ -1,15 +1,27 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Get cluster subdirectory name
|
||||
|
||||
cd $(dirname $0)
|
||||
|
||||
CLUSTER="$1"
|
||||
if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
|
||||
echo "Usage: $0 <cluster name>"
|
||||
echo "The cluster name must be the name of a subdirectory of cluster/"
|
||||
exit 1
|
||||
fi
|
||||
shift 1
|
||||
|
||||
# Do actual stuff
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
NIXHOSTLIST=$(ls node | grep -v '\.site\.')
|
||||
NIXHOSTLIST=$(ls cluster/$CLUSTER/node | grep -v '\.site\.')
|
||||
else
|
||||
NIXHOSTLIST="$@"
|
||||
fi
|
||||
|
||||
TMP_PATH=/tmp/tmp-deploy-$(date +%s)
|
||||
|
||||
SSH_CONFIG=cluster/$CLUSTER/ssh_config
|
||||
YEAR=$(date +%Y)
|
||||
|
||||
for NIXHOST in $NIXHOSTLIST; do
|
||||
|
|
@ -25,25 +37,27 @@ for NIXHOST in $NIXHOSTLIST; do
|
|||
|
||||
echo "Sending NixOS config files"
|
||||
|
||||
ssh -F ssh_config $SSH_DEST mkdir -p $TMP_PATH $TMP_PATH/pki
|
||||
cat configuration.nix | ssh -F ssh_config $SSH_DEST tee $TMP_PATH/configuration.nix > /dev/null
|
||||
cat node/$NIXHOST.nix | ssh -F ssh_config $SSH_DEST tee $TMP_PATH/node.nix > /dev/null
|
||||
cat node/$NIXHOST.site.nix | ssh -F ssh_config $SSH_DEST tee $TMP_PATH/site.nix > /dev/null
|
||||
ssh -F $SSH_CONFIG $SSH_DEST mkdir -p $TMP_PATH $TMP_PATH/pki
|
||||
cat configuration.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/configuration.nix > /dev/null
|
||||
cat nix/deuxfleurs.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/deuxfleurs.nix > /dev/null
|
||||
cat cluster/$CLUSTER/cluster.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/cluster.nix > /dev/null
|
||||
cat cluster/$CLUSTER/node/$NIXHOST.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/node.nix > /dev/null
|
||||
cat cluster/$CLUSTER/node/$NIXHOST.site.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/site.nix > /dev/null
|
||||
|
||||
echo "Sending secret files"
|
||||
for SECRET in rclone.conf \
|
||||
pki/consul-ca.crt pki/consul$YEAR.crt pki/consul$YEAR.key pki/consul$YEAR-client.crt pki/consul$YEAR-client.key \
|
||||
pki/nomad-ca.crt pki/nomad$YEAR.crt pki/nomad$YEAR.key; do
|
||||
test -f secrets/$SECRET && (cat secrets/$SECRET | ssh -F ssh_config $SSH_DEST tee $TMP_PATH/$SECRET > /dev/null)
|
||||
test -f cluster/$CLUSTER/secrets/$SECRET && (cat cluster/$CLUSTER/secrets/$SECRET | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/$SECRET > /dev/null)
|
||||
done
|
||||
|
||||
echo "Rebuilding NixOS"
|
||||
|
||||
ssh -F ssh_config $SSH_DEST tee $TMP_PATH/deploy.sh > /dev/null <<EOF
|
||||
ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/deploy.sh > /dev/null <<EOF
|
||||
set -ex
|
||||
|
||||
cd $TMP_PATH
|
||||
mv configuration.nix node.nix site.nix /etc/nixos
|
||||
mv deuxfleurs.nix configuration.nix cluster.nix node.nix site.nix /etc/nixos
|
||||
|
||||
test -f rclone.conf && (mv rclone.conf /root; chmod 600 /root/rclone.conf)
|
||||
|
||||
|
|
@ -71,6 +85,6 @@ consul kv put secrets/consul/consul-client.crt - < /var/lib/consul/pki/consul$YE
|
|||
consul kv put secrets/consul/consul-client.key - < /var/lib/consul/pki/consul$YEAR-client.key
|
||||
EOF
|
||||
|
||||
ssh -t -F ssh_config $SSH_DEST sudo sh $TMP_PATH/deploy.sh
|
||||
ssh -F ssh_config $SSH_DEST rm -rv '/tmp/tmp-deploy-*'
|
||||
ssh -t -F $SSH_CONFIG $SSH_DEST sudo sh $TMP_PATH/deploy.sh
|
||||
ssh -F $SSH_CONFIG $SSH_DEST rm -rv '/tmp/tmp-deploy-*'
|
||||
done
|
||||
|
|
|
|||
13
genpki.sh
13
genpki.sh
|
|
@ -2,11 +2,24 @@
|
|||
|
||||
set -xe
|
||||
|
||||
# Enter proper cluster subdirectory
|
||||
|
||||
cd $(dirname $0)
|
||||
|
||||
CLUSTER="$1"
|
||||
if [ ! -d "cluster/$CLUSTER" ]; then
|
||||
echo "Usage: $0 <cluster name>"
|
||||
echo "The cluster name must be the name of a subdirectory of cluster/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd cluster/$CLUSTER
|
||||
|
||||
mkdir -p secrets/pki
|
||||
cd secrets/pki
|
||||
|
||||
# Do actual stuff
|
||||
|
||||
YEAR=$(date +%Y)
|
||||
for APP in consul nomad; do
|
||||
# 1. Create certificate authority
|
||||
|
|
|
|||
185
nix/deuxfleurs.nix
Normal file
185
nix/deuxfleurs.nix
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
{ config, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.deuxfleurs;
|
||||
in
|
||||
with builtins;
|
||||
with pkgs.lib;
|
||||
{
|
||||
options.deuxfleurs =
|
||||
let wg_node = with types; submodule {
|
||||
options = {
|
||||
hostname = mkOption {
|
||||
type = str;
|
||||
description = "Host name";
|
||||
};
|
||||
IP = mkOption {
|
||||
type = str;
|
||||
description = "IP Address";
|
||||
};
|
||||
publicKey = mkOption {
|
||||
type = str;
|
||||
description = "Public key";
|
||||
};
|
||||
endpoint = mkOption {
|
||||
type = nullOr str;
|
||||
description = "Wireguard endpoint on the public Internet";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
# Parameters that may vary between nodes
|
||||
site_name = mkOption {
|
||||
description = "Site (availability zone) on which this node is deployed";
|
||||
type = types.str;
|
||||
};
|
||||
vpn_ip = mkOption {
|
||||
description = "IP address of this node on the Wireguard VPN";
|
||||
type = types.str;
|
||||
};
|
||||
vpn_listen_port = mkOption {
|
||||
description = "Port for incoming Wireguard VPN connections";
|
||||
type = types.port;
|
||||
};
|
||||
is_raft_server = mkOption {
|
||||
description = "Make this node a RAFT server for the Nomad and Consul deployments";
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
|
||||
# Parameters common to all nodes
|
||||
cluster_name = mkOption {
|
||||
description = "Name of this Deuxfleurs deployment";
|
||||
type = types.str;
|
||||
};
|
||||
cluster_nodes = mkOption {
|
||||
description = "Nodes that are part of the cluster";
|
||||
type = types.listOf wg_node;
|
||||
};
|
||||
admin_nodes = mkOption {
|
||||
description = "Machines that are part of the Wireguard VPN for administration purposes";
|
||||
type = types.listOf wg_node;
|
||||
};
|
||||
admin_accounts = mkOption {
|
||||
description = "List of users having an admin account on cluster nodes, maps user names to a list of authorized SSH keys";
|
||||
type = types.attrsOf (types.listOf types.str);
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
# Configure admin accounts on all nodes
|
||||
users.users = builtins.mapAttrs (name: publicKeys: {
|
||||
isNormalUser = true;
|
||||
extraGroups = [ "wheel" ];
|
||||
openssh.authorizedKeys.keys = publicKeys;
|
||||
}) cfg.admin_accounts;
|
||||
|
||||
# Configure Wireguard VPN between all nodes
|
||||
networking.wireguard.interfaces.wg0 = {
|
||||
ips = [ "${cfg.vpn_ip}/16" ];
|
||||
listenPort = cfg.vpn_listen_port;
|
||||
privateKeyFile = "/root/wireguard-keys/private";
|
||||
peers = map ({ publicKey, endpoint, IP, ... }: {
|
||||
publicKey = publicKey;
|
||||
allowedIPs = [ "${IP}/32" ];
|
||||
endpoint = endpoint;
|
||||
persistentKeepalive = 25;
|
||||
}) (cfg.cluster_nodes ++ cfg.admin_nodes);
|
||||
};
|
||||
|
||||
networking.firewall.allowedUDPPorts = [ cfg.vpn_listen_port ];
|
||||
|
||||
# Configure /etc/hosts to link all hostnames to their Wireguard IP
|
||||
networking.extraHosts = builtins.concatStringsSep "\n" (map
|
||||
({ hostname, IP, ...}: "${IP} ${hostname}")
|
||||
(cfg.cluster_nodes ++ cfg.admin_nodes));
|
||||
|
||||
# Enable Hashicorp Consul & Nomad
|
||||
services.consul.enable = true;
|
||||
services.consul.extraConfig =
|
||||
(if cfg.is_raft_server
|
||||
then {
|
||||
server = true;
|
||||
bootstrap_expect = 3;
|
||||
}
|
||||
else {}) //
|
||||
{
|
||||
datacenter = cfg.cluster_name;
|
||||
node_meta = {
|
||||
"site" = cfg.site_name;
|
||||
};
|
||||
ui = true;
|
||||
bind_addr = cfg.vpn_ip;
|
||||
|
||||
ports.http = -1;
|
||||
addresses.https = "0.0.0.0";
|
||||
ports.https = 8501;
|
||||
|
||||
retry_join = map (node_info: node_info.IP) cfg.cluster_nodes;
|
||||
|
||||
ca_file = "/var/lib/consul/pki/consul-ca.crt";
|
||||
cert_file = "/var/lib/consul/pki/consul2022.crt";
|
||||
key_file = "/var/lib/consul/pki/consul2022.key";
|
||||
verify_incoming = true;
|
||||
verify_outgoing = true;
|
||||
verify_server_hostname = true;
|
||||
};
|
||||
|
||||
services.nomad.enable = true;
|
||||
services.nomad.package = pkgs.nomad_1_1;
|
||||
services.nomad.settings =
|
||||
(if cfg.is_raft_server
|
||||
then { server = {
|
||||
enabled = true;
|
||||
bootstrap_expect = 3;
|
||||
}; }
|
||||
else {}) //
|
||||
{
|
||||
region = cfg.cluster_name;
|
||||
datacenter = cfg.site_name;
|
||||
advertise = {
|
||||
rpc = cfg.vpn_ip;
|
||||
http = cfg.vpn_ip;
|
||||
serf = cfg.vpn_ip;
|
||||
};
|
||||
consul = {
|
||||
address = "localhost:8501";
|
||||
ca_file = "/var/lib/nomad/pki/consul2022.crt";
|
||||
cert_file = "/var/lib/nomad/pki/consul2022-client.crt";
|
||||
key_file = "/var/lib/nomad/pki/consul2022-client.key";
|
||||
ssl = true;
|
||||
};
|
||||
client = {
|
||||
enabled = true;
|
||||
network_interface = "wg0";
|
||||
meta = {
|
||||
"site" = cfg.site_name;
|
||||
};
|
||||
};
|
||||
tls = {
|
||||
http = true;
|
||||
rpc = true;
|
||||
ca_file = "/var/lib/nomad/pki/nomad-ca.crt";
|
||||
cert_file = "/var/lib/nomad/pki/nomad2022.crt";
|
||||
key_file = "/var/lib/nomad/pki/nomad2022.key";
|
||||
verify_server_hostname = true;
|
||||
verify_https_client = true;
|
||||
};
|
||||
plugin = [
|
||||
{
|
||||
docker = [
|
||||
{
|
||||
config = [
|
||||
{
|
||||
volumes.enabled = true;
|
||||
allow_privileged = true;
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
17
tlsproxy.sh
17
tlsproxy.sh
|
|
@ -1,5 +1,22 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -xe
|
||||
|
||||
# Enter proper cluster subdirectory
|
||||
|
||||
cd $(dirname $0)
|
||||
|
||||
CLUSTER="$1"
|
||||
if [ ! -d "cluster/$CLUSTER" ]; then
|
||||
echo "Usage: $0 <cluster name>"
|
||||
echo "The cluster name must be the name of a subdirectory of cluster/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd cluster/$CLUSTER
|
||||
|
||||
# Do actual stuff
|
||||
|
||||
YEAR=$(date +%Y)
|
||||
|
||||
_int() {
|
||||
|
|
|
|||
23
upgrade.sh
23
upgrade.sh
|
|
@ -1,14 +1,27 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Get cluster subdirectory name
|
||||
|
||||
cd $(dirname $0)
|
||||
|
||||
CLUSTER="$1"
|
||||
if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
|
||||
echo "Usage: $0 <cluster name>"
|
||||
echo "The cluster name must be the name of a subdirectory of cluster/"
|
||||
exit 1
|
||||
fi
|
||||
shift 1
|
||||
|
||||
# Do actual stuff
|
||||
|
||||
if [ -z "$@" ]; then
|
||||
NIXHOSTLIST=$(ls node | grep -v '\.site\.')
|
||||
NIXHOSTLIST=$(ls cluster/$CLUSTER/node | grep -v '\.site\.')
|
||||
else
|
||||
NIXHOSTLIST="$@"
|
||||
fi
|
||||
|
||||
TMP_SCRIPT=/tmp/tmp-upgrade-$(date +%s).sh
|
||||
SSH_CONFIG=cluster/$CLUSTER/ssh_config
|
||||
|
||||
for NIXHOST in $NIXHOSTLIST; do
|
||||
NIXHOST=${NIXHOST%.*}
|
||||
|
|
@ -21,7 +34,7 @@ for NIXHOST in $NIXHOSTLIST; do
|
|||
|
||||
echo "==== DOING $NIXHOST ===="
|
||||
|
||||
ssh -F ssh_config $SSH_DEST tee $TMP_SCRIPT > /dev/null <<EOF
|
||||
ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_SCRIPT > /dev/null <<EOF
|
||||
set -ex
|
||||
|
||||
nix-channel --add https://nixos.org/channels/nixos-21.11 nixos
|
||||
|
|
@ -30,9 +43,9 @@ nixos-rebuild boot
|
|||
EOF
|
||||
|
||||
read -p "Press Enter to continue (run upgrade on $NIXHOST)..."
|
||||
ssh -t -F ssh_config $SSH_DEST sudo sh $TMP_SCRIPT
|
||||
ssh -F ssh_config $SSH_DEST rm -v $TMP_SCRIPT
|
||||
ssh -t -F $SSH_CONFIG $SSH_DEST sudo sh $TMP_SCRIPT
|
||||
ssh -F $SSH_CONFIG $SSH_DEST rm -v $TMP_SCRIPT
|
||||
|
||||
read -p "Press Enter to continue (reboot $NIXHOST)..."
|
||||
ssh -t -F ssh_config $SSH_DEST sudo reboot
|
||||
ssh -t -F $SSH_CONFIG $SSH_DEST sudo reboot
|
||||
done
|
||||
|
|
|
|||
Loading…
Reference in a new issue