From 2f6d64a1a81de511c0b62dc2241dc08dc75e5283 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 26 Dec 2021 13:23:01 +0100 Subject: [PATCH] Cleanup --- app/csi-s3/deploy/csi-s3.hcl | 39 ++++++++++++ app/dummy/deploy/.gitignore | 1 + configuration.nix | 112 +++++++++++++++-------------------- node/carcajou.nix | 3 - node/caribou.nix | 39 +----------- site/neptune.nix | 12 ---- site/pluton.nix | 1 + upgrade.sh | 6 +- 8 files changed, 96 insertions(+), 117 deletions(-) create mode 100644 app/csi-s3/deploy/csi-s3.hcl create mode 100644 app/dummy/deploy/.gitignore diff --git a/app/csi-s3/deploy/csi-s3.hcl b/app/csi-s3/deploy/csi-s3.hcl new file mode 100644 index 0000000..8e70c6a --- /dev/null +++ b/app/csi-s3/deploy/csi-s3.hcl @@ -0,0 +1,39 @@ +job "plugin-csi-s3-nodes" { + datacenters = ["neptune", "pluton"] + + # you can run node plugins as service jobs as well, but this ensures + # that all nodes in the DC have a copy. + type = "system" + + group "nodes" { + task "plugin" { + driver = "docker" + + config { + image = "ctrox/csi-s3:v1.2.0-rc.1" + + args = [ + "--endpoint=unix://csi/csi.sock", + "--nodeid=${node.unique.id}", + "--logtostderr", + "--v=5", + ] + + # node plugins must run as privileged jobs because they + # mount disks to the host + privileged = true + } + + csi_plugin { + id = "csi-s3" + type = "node" + mount_dir = "/csi" + } + + resources { + cpu = 500 + memory = 256 + } + } + } +} diff --git a/app/dummy/deploy/.gitignore b/app/dummy/deploy/.gitignore new file mode 100644 index 0000000..3af34ab --- /dev/null +++ b/app/dummy/deploy/.gitignore @@ -0,0 +1 @@ +dummy-volume.hcl diff --git a/configuration.nix b/configuration.nix index 726e425..c9fecd0 100644 --- a/configuration.nix +++ b/configuration.nix @@ -25,10 +25,9 @@ in # networking.wireless.enable = true; # Enables wireless support via wpa_supplicant. # Networking configuration (static IPs for each node is defined in node/*.nix) - networking.nameservers = [ "9.9.9.9" "213.186.33.99" "172.104.136.243" ]; + networking.nameservers = [ "9.9.9.9" ]; # Wireguard VPN configuration - # TODO: Max dit qu'on peut monter persistentKeepalive à 25s car les NAT ne mettent pas de tiemout inférieur à 30s networking.wireguard.interfaces.wg0 = { privateKeyFile = "/root/wireguard-keys/private"; peers = [ @@ -36,49 +35,49 @@ in publicKey = "b5hF+GSTgg3oM6wnjL7jRbfyf1jtsWdVptPPbAh3Qic="; allowedIPs = [ "10.42.0.1/32" ]; endpoint = "5.135.179.11:51349"; - persistentKeepalive = 10; + persistentKeepalive = 25; } { # Spoutnik publicKey = "fO8qZOZmnug84cA8nvfjl5MUqyWljP0BAz/4tHRZyEg="; allowedIPs = [ "10.42.0.2/32" ]; endpoint = "77.141.67.109:42136"; - persistentKeepalive = 10; + persistentKeepalive = 25; } { # Robinson publicKey = "ETaZFil3mFXlJ0LaJZyWqJVLV2IZUF5PB/8M7WbQSTg="; allowedIPs = [ "10.42.0.42/32" ]; endpoint = "77.141.67.109:33742"; - persistentKeepalive = 10; + persistentKeepalive = 25; } { # Shiki publicKey = "QUiUNMk70TEQ75Ut7Uqikr5uGVSXmx8EGNkGM6tANlg="; allowedIPs = [ "10.42.0.206/32" ]; endpoint = "37.187.118.206:51820"; - persistentKeepalive = 10; + persistentKeepalive = 25; } { # Lindy publicKey = "wen9GnZy2iLT6RyHfn7ydS/wvdvow1XPmhZxIkrDbks="; allowedIPs = [ "10.42.0.66/32" ]; endpoint = "82.66.112.151:33766"; - persistentKeepalive = 10; + persistentKeepalive = 25; } { # Carcajou publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA="; allowedIPs = [ "10.42.0.21/32" ]; endpoint = "82.66.112.151:33721"; - persistentKeepalive = 10; + persistentKeepalive = 25; } { # Carcajou publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk="; allowedIPs = [ "10.42.0.22/32" ]; endpoint = "82.66.112.151:33722"; - persistentKeepalive = 10; + persistentKeepalive = 25; } { # Caribou publicKey = "g6ZED/wPn5MPfytJKwPI19808CXtEad0IJUkEAAzwyY="; allowedIPs = [ "10.42.0.23/32" ]; endpoint = "82.66.112.151:33723"; - persistentKeepalive = 10; + persistentKeepalive = 25; } ]; }; @@ -90,8 +89,6 @@ in 192.168.1.21 cariacou.lan 192.168.1.22 carcajou.lan 192.168.1.23 caribou.lan -192.168.1.23 binarycache -192.168.1.23 binarycache.home.adnab.me 10.42.0.1 hammerhead 10.42.0.2 spoutnik 10.42.0.21 cariacou @@ -101,10 +98,6 @@ in 10.42.0.206 shiki ''; - # Configure network proxy if necessary - # networking.proxy.default = "http://user:password@proxy:port/"; - # networking.proxy.noProxy = "127.0.0.1,localhost,internal.domain"; - # Select internationalisation properties. # i18n.defaultLocale = "en_US.UTF-8"; console = { @@ -112,24 +105,11 @@ in keyMap = "fr"; }; - # Enable the X11 windowing system. - # services.xserver.enable = true; - - # Configure keymap in X11 - # services.xserver.layout = "us"; - # services.xserver.xkbOptions = "eurosign:e"; - - # Enable CUPS to print documents. - # services.printing.enable = true; - # Enable sound. # sound.enable = true; # hardware.pulseaudio.enable = true; - # Enable touchpad support (enabled default in most desktopManager). - # services.xserver.libinput.enable = true; - - # Define a user account. Don't forget to set a password with ‘passwd’. + # Define user accounts users.users.lx = { isNormalUser = true; extraGroups = [ @@ -183,6 +163,9 @@ in # List packages installed in system profile. To search, run: # $ nix search wget environment.systemPackages = with pkgs; [ + nmap + bind + inetutils vim tmux ncdu @@ -199,21 +182,14 @@ in programs.vim.defaultEditor = true; - # Some programs need SUID wrappers, can be configured further or are - # started in user sessions. - # programs.mtr.enable = true; - # programs.gnupg.agent = { - # enable = true; - # enableSSHSupport = true; - # }; - - # List services that you want to enable: - # Enable network time services.ntp.enable = true; - # Enable the OpenSSH daemon. + # Enable the OpenSSH daemon and disable password login. services.openssh.enable = true; + services.openssh.passwordAuthentication = false; + + # ---- CONFIG FOR DEUXFLEURS CLUSTER ---- # Enable Hashicorp Consul & Nomad services.consul.enable = true; @@ -232,6 +208,7 @@ in }; services.nomad.enable = true; + services.nomad.package = pkgs.nomad_1_1; services.nomad.settings = let public_ip = (builtins.head (builtins.split "/" (builtins.head node_config.networking.wireguard.interfaces.wg0.ips))); in @@ -257,7 +234,7 @@ in config = [ { volumes.enabled = true; - #allow_privileged = true; + allow_privileged = true; } ]; } @@ -268,60 +245,65 @@ in # Open ports in the firewall. networking.firewall = { + enable = true; + + # Allow anyone to connect on SSH port allowedTCPPorts = [ (builtins.head ({ openssh.ports = [22]; } // node_config.services).openssh.ports) ]; + + # Allow anyone to contact Wireguard VPN server allowedUDPPorts = [ node_config.networking.wireguard.interfaces.wg0.listenPort ]; - # Authorize nodes also on the Wireguard VPN to access services running here + # Allow specific hosts access to specific things in the cluster extraCommands = '' # Allow everything from router (usefull for UPnP/IGD) iptables -A INPUT -s 192.168.1.254 -j ACCEPT + # Allow Docker containers to access a few things + iptables -N CONTAINERS + iptables -A INPUT -s 172.17.0.0/16 -j CONTAINERS + + # Yugabyte YSQL + iptables -A CONTAINERS -p tcp --dport 5433 -j ACCEPT + + # Specific rules for VPN nodes iptables -N VPN iptables -A INPUT -s 10.42.0.0/16 -j VPN - # Nomad + # Allow server nodes to communicate between themselves on all ports + iptables -A VPN -s 10.42.0.2 -j ACCEPT + iptables -A VPN -s 10.42.0.21 -j ACCEPT + iptables -A VPN -s 10.42.0.22 -j ACCEPT + iptables -A VPN -s 10.42.0.23 -j ACCEPT + + # Allow all VPN users to access Nomad API iptables -A VPN -p tcp --dport 4646 -j ACCEPT - iptables -A VPN -p tcp --dport 4647 -j ACCEPT - iptables -A VPN -p tcp --dport 4648 -j ACCEPT - iptables -A VPN -p udp --dport 4648 -j ACCEPT - # Consul + # Same for Consul API iptables -A VPN -p tcp --dport 8500 -j ACCEPT - iptables -A VPN -p tcp --dport 8300 -j ACCEPT - iptables -A VPN -p tcp --dport 8301 -j ACCEPT - iptables -A VPN -p tcp --dport 8302 -j ACCEPT - iptables -A VPN -p udp --dport 8301 -j ACCEPT - iptables -A VPN -p udp --dport 8302 -j ACCEPT - # Garage - iptables -A VPN -p tcp --dport 3990 -j ACCEPT - iptables -A VPN -p tcp --dport 3991 -j ACCEPT - iptables -A VPN -p tcp --dport 3992 -j ACCEPT - - # Yugabyte DB + # Same for YugabyteDB YSQL and Admin ports iptables -A VPN -p tcp --dport 5433 -j ACCEPT iptables -A VPN -p tcp --dport 7000 -j ACCEPT - iptables -A VPN -p tcp --dport 7100 -j ACCEPT - iptables -A VPN -p tcp --dport 9100 -j ACCEPT - # Netdata monitoring + # Same for Netdata monitoring iptables -A VPN -p tcp --dport 19999 -j ACCEPT ''; - # When stopping firewall, delete filtering VPN chain + # When stopping firewall, delete all rules that were configured manually above extraStopCommands = '' iptables -D INPUT -s 192.168.1.254 -j ACCEPT iptables -D INPUT -s 10.42.0.0/16 -j VPN iptables -F VPN iptables -X VPN + iptables -D INPUT -s 172.17.0.0/16 -j CONTAINERS + iptables -F CONTAINERS + iptables -X CONTAINERS ''; }; - # Or disable the firewall altogether. - # networking.firewall.enable = false; # This value determines the NixOS release from which the default # settings for stateful data, like file locations and database versions diff --git a/node/carcajou.nix b/node/carcajou.nix index 0698663..59c2008 100644 --- a/node/carcajou.nix +++ b/node/carcajou.nix @@ -35,7 +35,4 @@ # Activate as Nomad and Consul server node services.nomad.settings.server.enabled = true; services.consul.extraConfig.server = true; - - # Use this node as entrypoint to cluster (Diplonat not working for now) - networking.firewall.allowedTCPPorts = [ 80 443 ]; } diff --git a/node/caribou.nix b/node/caribou.nix index 77fc35b..dc5e917 100644 --- a/node/caribou.nix +++ b/node/caribou.nix @@ -8,7 +8,7 @@ boot.loader.timeout = 20; boot.loader.efi.canTouchEfiVariables = true; - networking.hostName = "caribou"; # Define your hostname. + networking.hostName = "caribou"; networking.interfaces.eno1.useDHCP = false; networking.interfaces.eno1.ipv4.addresses = [ @@ -29,43 +29,10 @@ listenPort = 33723; }; - # OR use USB modem plugged in here - #networking.interfaces.enp0s20u1.useDHCP = true; + # Enable netdata monitoring + services.netdata.enable = true; # Activate as Nomad and Consul server node services.nomad.settings.server.enabled = true; services.consul.extraConfig.server = true; - - # Enable netdata monitoring - services.netdata.enable = true; - - # ---- - - # Enable nix-serve - services.nix-serve = { - enable = true; - secretKeyFile = "/var/cache-priv-key.pem"; - }; - - # Configure a Nginx web server to serve NixOS cache - services.nginx = { - enable = true; - virtualHosts = { - "binarycache.home.adnab.me" = { - serverAliases = [ "binarycache" ]; - listen = [ { - addr = "0.0.0.0"; - port = 7980; - } ]; - locations."/".extraConfig = '' - proxy_pass http://localhost:${toString config.services.nix-serve.port}; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - ''; - }; - }; - }; - - networking.firewall.allowedTCPPorts = [ 7980 ]; } diff --git a/site/neptune.nix b/site/neptune.nix index f640fc1..781e512 100644 --- a/site/neptune.nix +++ b/site/neptune.nix @@ -8,17 +8,5 @@ services.nomad.settings.datacenter = "neptune"; - # Allow router to reach nodes in this site networking.firewall.allowedTCPPorts = [ 80 443 ]; - - # ---- - - nix = { - binaryCaches = [ - "http://binarycache.home.adnab.me:7980" - ]; - binaryCachePublicKeys = [ - "binarycache.home.adnab.me:ErR6pMnewf9oVyZJd5uC2nI4EZF49c7Mh86eDZWYZaw=" - ]; - }; } diff --git a/site/pluton.nix b/site/pluton.nix index 22bea08..6b57129 100644 --- a/site/pluton.nix +++ b/site/pluton.nix @@ -5,6 +5,7 @@ address = "192.168.0.1"; interface = "enp0s25"; }; + networking.nameservers = [ "213.186.33.99" "172.104.136.243" ]; services.nomad.settings.datacenter = "pluton"; diff --git a/upgrade.sh b/upgrade.sh index 4134bbd..17d61ce 100755 --- a/upgrade.sh +++ b/upgrade.sh @@ -19,6 +19,10 @@ for NIXHOST in $NIXHOSTLIST; do echo "==== DOING $NIXHOST ====" + ssh -F ssh_config $SSH_DEST sudo nix-channel --add https://nixos.org/channels/nixos-21.11 nixos ssh -F ssh_config $SSH_DEST sudo nix-channel --update - ssh -F ssh_config $SSH_DEST sudo nixos-rebuild switch + ssh -F ssh_config $SSH_DEST sudo nixos-rebuild boot + + echo "Please reboot node manually to activate upgraded system:" + echo "$ ssh -F ssh_config $SSH_DEST sudo reboot" done