Cleanup

app/csi-s3/deploy/csi-s3.hcl

@@ -0,0 +1,39 @@
+job "plugin-csi-s3-nodes" {
+  datacenters = ["neptune", "pluton"]
+
+  # you can run node plugins as service jobs as well, but this ensures
+  # that all nodes in the DC have a copy.
+  type = "system"
+
+  group "nodes" {
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image = "ctrox/csi-s3:v1.2.0-rc.1"
+
+        args = [
+          "--endpoint=unix://csi/csi.sock",
+          "--nodeid=${node.unique.id}",
+          "--logtostderr",
+          "--v=5",
+        ]
+
+        # node plugins must run as privileged jobs because they
+        # mount disks to the host
+        privileged = true
+      }
+
+      csi_plugin {
+        id        = "csi-s3"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+}

app/dummy/deploy/.gitignore

@@ -0,0 +1 @@
+dummy-volume.hcl

configuration.nix

@@ -25,10 +25,9 @@ in
   # networking.wireless.enable = true;  # Enables wireless support via wpa_supplicant.
 
   # Networking configuration (static IPs for each node is defined in node/*.nix)
-  networking.nameservers = [ "9.9.9.9" "213.186.33.99" "172.104.136.243" ];
+  networking.nameservers = [ "9.9.9.9" ];
 
   # Wireguard VPN configuration
-  # TODO: Max dit qu'on peut monter persistentKeepalive à 25s car les NAT ne mettent pas de tiemout inférieur à 30s
   networking.wireguard.interfaces.wg0 = {
     privateKeyFile = "/root/wireguard-keys/private";
     peers = [
@@ -36,49 +35,49 @@ in
         publicKey = "b5hF+GSTgg3oM6wnjL7jRbfyf1jtsWdVptPPbAh3Qic=";
         allowedIPs = [ "10.42.0.1/32" ];
         endpoint = "5.135.179.11:51349";
-        persistentKeepalive = 10;
+        persistentKeepalive = 25;
       }
       { # Spoutnik
         publicKey = "fO8qZOZmnug84cA8nvfjl5MUqyWljP0BAz/4tHRZyEg=";
         allowedIPs = [ "10.42.0.2/32" ];
         endpoint = "77.141.67.109:42136";
-        persistentKeepalive = 10;
+        persistentKeepalive = 25;
       }
       { # Robinson
         publicKey = "ETaZFil3mFXlJ0LaJZyWqJVLV2IZUF5PB/8M7WbQSTg=";
         allowedIPs = [ "10.42.0.42/32" ];
         endpoint = "77.141.67.109:33742";
-        persistentKeepalive = 10;
+        persistentKeepalive = 25;
       }
       { # Shiki
         publicKey = "QUiUNMk70TEQ75Ut7Uqikr5uGVSXmx8EGNkGM6tANlg=";
         allowedIPs = [ "10.42.0.206/32" ];
         endpoint = "37.187.118.206:51820";
-        persistentKeepalive = 10;
+        persistentKeepalive = 25;
       }
       { # Lindy
         publicKey = "wen9GnZy2iLT6RyHfn7ydS/wvdvow1XPmhZxIkrDbks=";
         allowedIPs = [ "10.42.0.66/32" ];
         endpoint = "82.66.112.151:33766";
-        persistentKeepalive = 10;
+        persistentKeepalive = 25;
       }
       { # Carcajou
         publicKey = "qxrtfn2zRVnN52Y5NYumyU3/FcRMnh3kJ2C37JfrczA=";
         allowedIPs = [ "10.42.0.21/32" ];
         endpoint = "82.66.112.151:33721";
-        persistentKeepalive = 10;
+        persistentKeepalive = 25;
       }
       { # Carcajou
         publicKey = "7Nm7pMmyS7Nts1MB+loyD8u84ODxHPTkDu+uqQR6yDk=";
         allowedIPs = [ "10.42.0.22/32" ];
         endpoint = "82.66.112.151:33722";
-        persistentKeepalive = 10;
+        persistentKeepalive = 25;
       }
       { # Caribou
         publicKey = "g6ZED/wPn5MPfytJKwPI19808CXtEad0IJUkEAAzwyY=";
         allowedIPs = [ "10.42.0.23/32" ];
         endpoint = "82.66.112.151:33723";
-        persistentKeepalive = 10;
+        persistentKeepalive = 25;
       }
     ];
   };
@@ -90,8 +89,6 @@ in
 192.168.1.21 cariacou.lan
 192.168.1.22 carcajou.lan
 192.168.1.23 caribou.lan
-192.168.1.23 binarycache
-192.168.1.23 binarycache.home.adnab.me
 10.42.0.1 hammerhead
 10.42.0.2 spoutnik
 10.42.0.21 cariacou
@@ -101,10 +98,6 @@ in
 10.42.0.206 shiki
   '';
 
-  # Configure network proxy if necessary
-  # networking.proxy.default = "http://user:password@proxy:port/";
-  # networking.proxy.noProxy = "127.0.0.1,localhost,internal.domain";
-
   # Select internationalisation properties.
   # i18n.defaultLocale = "en_US.UTF-8";
   console = {
@@ -112,24 +105,11 @@ in
     keyMap = "fr";
   };
 
-  # Enable the X11 windowing system.
-  # services.xserver.enable = true;
-
-  # Configure keymap in X11
-  # services.xserver.layout = "us";
-  # services.xserver.xkbOptions = "eurosign:e";
-
-  # Enable CUPS to print documents.
-  # services.printing.enable = true;
-
   # Enable sound.
   # sound.enable = true;
   # hardware.pulseaudio.enable = true;
 
-  # Enable touchpad support (enabled default in most desktopManager).
+  # Define user accounts
-  # services.xserver.libinput.enable = true;
-
-  # Define a user account. Don't forget to set a password with ‘passwd’.
   users.users.lx = {
     isNormalUser = true;
     extraGroups = [
@@ -183,6 +163,9 @@ in
   # List packages installed in system profile. To search, run:
   # $ nix search wget
   environment.systemPackages = with pkgs; [
+    nmap
+    bind
+    inetutils
     vim
     tmux
     ncdu
@@ -199,21 +182,14 @@ in
 
   programs.vim.defaultEditor = true;
 
-  # Some programs need SUID wrappers, can be configured further or are
-  # started in user sessions.
-  # programs.mtr.enable = true;
-  # programs.gnupg.agent = {
-  #   enable = true;
-  #   enableSSHSupport = true;
-  # };
-
-  # List services that you want to enable:
-
   # Enable network time
   services.ntp.enable = true;
 
-  # Enable the OpenSSH daemon.
+  # Enable the OpenSSH daemon and disable password login.
   services.openssh.enable = true;
+  services.openssh.passwordAuthentication = false;
+
+  # ---- CONFIG FOR DEUXFLEURS CLUSTER ----
 
   # Enable Hashicorp Consul & Nomad
   services.consul.enable = true;
@@ -232,6 +208,7 @@ in
   };
 
   services.nomad.enable = true;
+  services.nomad.package = pkgs.nomad_1_1;
   services.nomad.settings =
     let public_ip = (builtins.head (builtins.split "/" (builtins.head node_config.networking.wireguard.interfaces.wg0.ips)));
     in
@@ -257,7 +234,7 @@ in
             config = [
               {
                 volumes.enabled = true;
-                #allow_privileged = true;
+                allow_privileged = true;
               }
             ];
           }
@@ -268,60 +245,65 @@ in
 
   # Open ports in the firewall.
   networking.firewall = {
+    enable = true;
+
+    # Allow anyone to connect on SSH port
     allowedTCPPorts = [
       (builtins.head ({ openssh.ports = [22]; } // node_config.services).openssh.ports)
     ];
+
+    # Allow anyone to contact Wireguard VPN server
     allowedUDPPorts = [
       node_config.networking.wireguard.interfaces.wg0.listenPort
     ];
 
-    # Authorize nodes also on the Wireguard VPN to access services running here
+    # Allow specific hosts access to specific things in the cluster
     extraCommands = ''
       # Allow everything from router (usefull for UPnP/IGD)
       iptables -A INPUT -s 192.168.1.254 -j ACCEPT
 
+      # Allow Docker containers to access a few things
+      iptables -N CONTAINERS
+      iptables -A INPUT -s 172.17.0.0/16 -j CONTAINERS
+
+      # Yugabyte YSQL
+      iptables -A CONTAINERS -p tcp --dport 5433 -j ACCEPT
+
+      # Specific rules for VPN nodes
       iptables -N VPN
       iptables -A INPUT -s 10.42.0.0/16 -j VPN
 
-      # Nomad
+      # Allow server nodes to communicate between themselves on all ports
+      iptables -A VPN -s 10.42.0.2 -j ACCEPT
+      iptables -A VPN -s 10.42.0.21 -j ACCEPT
+      iptables -A VPN -s 10.42.0.22 -j ACCEPT
+      iptables -A VPN -s 10.42.0.23 -j ACCEPT
+
+      # Allow all VPN users to access Nomad API
       iptables -A VPN -p tcp --dport 4646 -j ACCEPT
-      iptables -A VPN -p tcp --dport 4647 -j ACCEPT
-      iptables -A VPN -p tcp --dport 4648 -j ACCEPT
-      iptables -A VPN -p udp --dport 4648 -j ACCEPT
 
-      # Consul
+      # Same for Consul API
       iptables -A VPN -p tcp --dport 8500 -j ACCEPT
-      iptables -A VPN -p tcp --dport 8300 -j ACCEPT
-      iptables -A VPN -p tcp --dport 8301 -j ACCEPT
-      iptables -A VPN -p tcp --dport 8302 -j ACCEPT
-      iptables -A VPN -p udp --dport 8301 -j ACCEPT
-      iptables -A VPN -p udp --dport 8302 -j ACCEPT
 
-      # Garage
+      # Same for YugabyteDB YSQL and Admin ports
-      iptables -A VPN -p tcp --dport 3990 -j ACCEPT
-      iptables -A VPN -p tcp --dport 3991 -j ACCEPT
-      iptables -A VPN -p tcp --dport 3992 -j ACCEPT
-
-      # Yugabyte DB
       iptables -A VPN -p tcp --dport 5433 -j ACCEPT
       iptables -A VPN -p tcp --dport 7000 -j ACCEPT
-      iptables -A VPN -p tcp --dport 7100 -j ACCEPT
-      iptables -A VPN -p tcp --dport 9100 -j ACCEPT
 
-      # Netdata monitoring
+      # Same for Netdata monitoring
       iptables -A VPN -p tcp --dport 19999 -j ACCEPT
     '';
 
-    # When stopping firewall, delete filtering VPN chain
+    # When stopping firewall, delete all rules that were configured manually above
     extraStopCommands = ''
       iptables -D INPUT -s 192.168.1.254 -j ACCEPT
       iptables -D INPUT -s 10.42.0.0/16 -j VPN
       iptables -F VPN
       iptables -X VPN
+      iptables -D INPUT -s 172.17.0.0/16 -j CONTAINERS
+      iptables -F CONTAINERS
+      iptables -X CONTAINERS
     '';
   };
-  # Or disable the firewall altogether.
-  # networking.firewall.enable = false;
 
   # This value determines the NixOS release from which the default
   # settings for stateful data, like file locations and database versions

node/carcajou.nix

@@ -35,7 +35,4 @@
   # Activate as Nomad and Consul server node
   services.nomad.settings.server.enabled = true;
   services.consul.extraConfig.server = true;
-
-  # Use this node as entrypoint to cluster (Diplonat not working for now)
-  networking.firewall.allowedTCPPorts = [ 80 443 ];
 }

node/caribou.nix

@@ -8,7 +8,7 @@
   boot.loader.timeout = 20;
   boot.loader.efi.canTouchEfiVariables = true;
 
-  networking.hostName = "caribou"; # Define your hostname.
+  networking.hostName = "caribou";
 
   networking.interfaces.eno1.useDHCP = false;
   networking.interfaces.eno1.ipv4.addresses = [
@@ -29,43 +29,10 @@
     listenPort = 33723;
   };
 
-  # OR use USB modem plugged in here
+  # Enable netdata monitoring
-  #networking.interfaces.enp0s20u1.useDHCP = true;
+  services.netdata.enable = true;
 
   # Activate as Nomad and Consul server node
   services.nomad.settings.server.enabled = true;
   services.consul.extraConfig.server = true;
-
-  # Enable netdata monitoring
-  services.netdata.enable = true;
-
-  # ----
-
-  # Enable nix-serve
-  services.nix-serve = {
-    enable = true;
-    secretKeyFile = "/var/cache-priv-key.pem";
-  };
-
-  # Configure a Nginx web server to serve NixOS cache
-  services.nginx = {
-    enable = true;
-    virtualHosts = {
-      "binarycache.home.adnab.me" = {
-        serverAliases = [ "binarycache" ];
-        listen = [ {
-          addr = "0.0.0.0";
-          port = 7980;
-        } ];
-        locations."/".extraConfig = ''
-          proxy_pass http://localhost:${toString config.services.nix-serve.port};
-          proxy_set_header Host $host;
-          proxy_set_header X-Real-IP $remote_addr;
-          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        '';
-       };
-    };
-  };
-
-  networking.firewall.allowedTCPPorts = [ 7980 ];
 }

site/neptune.nix

@@ -8,17 +8,5 @@
 
   services.nomad.settings.datacenter = "neptune";
 
-  # Allow router to reach nodes in this site
   networking.firewall.allowedTCPPorts = [ 80 443 ];
-
-  # ----
-
-  nix = {
-    binaryCaches = [
-      "http://binarycache.home.adnab.me:7980"
-    ];
-    binaryCachePublicKeys = [
-      "binarycache.home.adnab.me:ErR6pMnewf9oVyZJd5uC2nI4EZF49c7Mh86eDZWYZaw="
-    ];
-  };
 }

site/pluton.nix

@@ -5,6 +5,7 @@
     address = "192.168.0.1";
     interface = "enp0s25";
   };
+  networking.nameservers = [ "213.186.33.99" "172.104.136.243" ];
 
   services.nomad.settings.datacenter = "pluton";
 

upgrade.sh

@@ -19,6 +19,10 @@ for NIXHOST in $NIXHOSTLIST; do
 
 	echo "==== DOING $NIXHOST ===="
 
+	ssh -F ssh_config $SSH_DEST sudo nix-channel --add https://nixos.org/channels/nixos-21.11 nixos
 	ssh -F ssh_config $SSH_DEST sudo nix-channel --update
-	ssh -F ssh_config $SSH_DEST sudo nixos-rebuild switch
+	ssh -F ssh_config $SSH_DEST sudo nixos-rebuild boot
+
+	echo "Please reboot node manually to activate upgraded system:"
+	echo "$ ssh -F ssh_config $SSH_DEST sudo reboot"
 done