Refactor deployment scripts

README.md

@@ -10,9 +10,10 @@ It sets up the following:
 
 The following scripts are available here:
 
+- `deploy_nixos`, the main script that updates the NixOS config
 - `genpki.sh`, a script to generate Consul and Nomad's TLS PKI (run this once only)
-- `deploy.sh`, the main script that updates the NixOS config and sets up all of the TLS secrets
+- `deploy_pki`, a script that sets up all of the TLS secrets
-- `upgrade.sh`, a script to upgrade NixOS
+- `upgrade_nixos`, a script to upgrade NixOS
 - `tlsproxy.sh`, a script that allows non-TLS access to the TLS-secured Consul and Nomad, by running a simple local proxy with socat
 - `tlsenv.sh`, a script to be sourced (`source tlsenv.sh`) that configures the correct environment variables to use the Nomad and Consul CLI tools with TLS
 

deploy.sh

@@ -1,91 +0,0 @@
-#!/usr/bin/env bash
-
-# Get cluster subdirectory name
-
-cd $(dirname $0)
-
-CLUSTER="$1"
-if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
-	echo "Usage: $0 <cluster name>"
-	echo "The cluster name must be the name of a subdirectory of cluster/"
-	exit 1
-fi
-shift 1
-
-# Do actual stuff
-
-if [ -z "$1" ]; then
-	NIXHOSTLIST=$(ls cluster/$CLUSTER/node | grep -v '\.site\.')
-else
-	NIXHOSTLIST="$@"
-fi
-
-TMP_PATH=/tmp/tmp-deploy-$(date +%s)
-SSH_CONFIG=cluster/$CLUSTER/ssh_config
-YEAR=$(date +%Y)
-
-for NIXHOST in $NIXHOSTLIST; do
-	NIXHOST=${NIXHOST%.*}
-
-	if [ -z "$SSH_USER" ]; then
-		SSH_DEST=$NIXHOST
-	else
-		SSH_DEST=$SSH_USER@$NIXHOST
-	fi
-
-	echo "==== DOING $NIXHOST ===="
-
-	echo "Sending NixOS config files"
-
-	ssh -F $SSH_CONFIG $SSH_DEST mkdir -p $TMP_PATH $TMP_PATH/pki
-	cat nix/configuration.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/configuration.nix > /dev/null
-	cat nix/deuxfleurs.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/deuxfleurs.nix > /dev/null
-	cat nix/remote-unlock.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/remote-unlock.nix > /dev/null
-	cat nix/wesher.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/wesher.nix > /dev/null
-	cat nix/wesher_service.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/wesher_service.nix > /dev/null
-	cat cluster/$CLUSTER/cluster.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/cluster.nix > /dev/null
-	cat cluster/$CLUSTER/node/$NIXHOST.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/node.nix > /dev/null
-	cat cluster/$CLUSTER/node/$NIXHOST.site.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/site.nix > /dev/null
-
-	echo "Sending secret files"
-	for SECRET in pki/consul-ca.crt pki/consul$YEAR.crt pki/consul$YEAR.key \
-		pki/consul$YEAR-client.crt pki/consul$YEAR-client.key \
-		pki/nomad-ca.crt pki/nomad$YEAR.crt pki/nomad$YEAR.key; do
-		test -f cluster/$CLUSTER/secrets/$SECRET && (cat cluster/$CLUSTER/secrets/$SECRET | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/$SECRET > /dev/null)
-	done
-
-	echo "Rebuilding NixOS"
-
-	ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/deploy.sh > /dev/null <<EOF
-set -ex
-
-cd $TMP_PATH
-mv deuxfleurs.nix remote-unlock.nix wesher.nix wesher_service.nix configuration.nix cluster.nix node.nix site.nix /etc/nixos
-
-nixos-rebuild switch
-
-mkdir -p /var/lib/nomad/pki /var/lib/consul/pki
-
-if [ -f pki/consul-ca.crt ]; then
-	cp pki/consul* /var/lib/nomad/pki
-	mv pki/consul* /var/lib/consul/pki
-	chown -R consul:root /var/lib/consul/pki
-fi
-
-if [ -f pki/nomad-ca.crt ]; then
-	mv pki/nomad* /var/lib/nomad/pki
-fi
-
-# Save up-to-date Consul client certificates in Consul itself
-export CONSUL_HTTP_ADDR=https://localhost:8501
-export CONSUL_CACERT=/var/lib/consul/pki/consul-ca.crt
-export CONSUL_CLIENT_CERT=/var/lib/consul/pki/consul$YEAR-client.crt
-export CONSUL_CLIENT_KEY=/var/lib/consul/pki/consul$YEAR-client.key
-consul kv put secrets/consul/consul-ca.crt - < /var/lib/consul/pki/consul-ca.crt
-consul kv put secrets/consul/consul-client.crt - < /var/lib/consul/pki/consul$YEAR-client.crt
-consul kv put secrets/consul/consul-client.key - < /var/lib/consul/pki/consul$YEAR-client.key
-EOF
-
-	ssh -t -F $SSH_CONFIG $SSH_DEST sudo sh $TMP_PATH/deploy.sh
-	ssh -F $SSH_CONFIG $SSH_DEST rm -rv '/tmp/tmp-deploy-*'
-done

deploy_nixos

@@ -0,0 +1,12 @@
+#!/usr/bin/env ./sshtool
+
+copy nix/configuration.nix /etc/nixos/configuration.nix
+copy nix/deuxfleurs.nix /etc/nixos/deuxfleurs.nix
+copy nix/remote-unlock.nix /etc/nixos/remote-unlock.nix
+copy nix/wesher.nix /etc/nixos/wesher.nix
+copy nix/wesher_service.nix /etc/nixos/wesher_service.nix
+copy cluster/$CLUSTER/cluster.nix /etc/nixos/cluster.nix
+copy cluster/$CLUSTER/node/$NIXHOST.nix /etc/nixos/node.nix
+copy cluster/$CLUSTER/node/$NIXHOST.site.nix /etc/nixos/site.nix
+
+cmd nixos-rebuild switch

deploy_pki

@@ -0,0 +1,34 @@
+#!/usr/bin/env ./sshtool
+
+PKI=cluster/$CLUSTER/secrets/pki
+YEAR=$(date +%Y)
+
+cmd mkdir -p /var/lib/nomad/pki /var/lib/consul/pki
+
+for file in consul-ca.crt consul$YEAR.crt consul$YEAR.key consul$YEAR-client.crt consul$YEAR-client.key; do
+	if [ -f "$PKI/$file" ]; then
+		copy $PKI/$file /var/lib/consul/pki/$file
+		cmd chown consul:root /var/lib/consul/pki/$file
+		cmd chmod 0400 /var/lib/consul/pki/$file
+	fi
+done
+
+cmd systemctl restart consul
+cmd sleep 10
+
+for file in nomad-ca.crt nomad$YEAR.crt nomad$YER.key; do
+	if [ -f "$PKI/$file" ]; then
+		copy $PKI/$file /var/lib/nomad/pki/$file
+	fi
+done
+
+cmd systemctl restart nomad
+
+set_env CONSUL_HTTP_ADDR=https://localhost:8501
+set_env CONSUL_CACERT=/var/lib/consul/pki/consul-ca.crt
+set_env CONSUL_CLIENT_CERT=/var/lib/consul/pki/consul$YEAR-client.crt
+set_env CONSUL_CLIENT_KEY=/var/lib/consul/pki/consul$YEAR-client.key
+
+cmd "consul kv put secrets/consul/consul-ca.crt - < /var/lib/consul/pki/consul-ca.crt"
+cmd "consul kv put secrets/consul/consul-client.crt - < /var/lib/consul/pki/consul$YEAR-client.crt"
+cmd "consul kv put secrets/consul/consul-client.key - < /var/lib/consul/pki/consul$YEAR-client.key"

sshtool

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+cd $(dirname $0)
+
+CMDFILE="$1"
+shift 1
+
+CLUSTER="$1"
+if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
+	echo "Usage: $CMDFILE <cluster name>"
+	echo "The cluster name must be the name of a subdirectory of cluster/"
+	exit 1
+fi
+shift 1
+
+if [ -z "$1" ]; then
+	NIXHOSTLIST=$(ls cluster/$CLUSTER/node | grep -v '\.site\.')
+else
+	NIXHOSTLIST="$@"
+fi
+
+if [ -z "$ROOT_PASS" ]; then
+	read -s -p "Enter remote root password: " ROOT_PASS
+	echo
+fi
+
+SSH_CONFIG=cluster/$CLUSTER/ssh_config
+
+function header {
+	cat <<EOF
+export DEPLOYTOOL_ROOT_PASSWORD=$ROOT_PASS
+cat > /tmp/deploytool_askpass <<EOG
+#!/usr/bin/env sh
+echo \$DEPLOYTOOL_ROOT_PASSWORD
+EOG
+chmod +x /tmp/deploytool_askpass
+export SUDO_ASKPASS=/tmp/deploytool_askpass
+sudo -A sh - <<EOEVERYTHING
+EOF
+}
+
+function footer {
+	echo EOEVERYTHING
+}
+
+function message {
+	echo "echo '$@'"
+}
+
+function cmd {
+	echo "echo '- run $@'"
+	echo "$@"
+}
+
+function set_env {
+	echo "echo '- set $@'"
+	echo "export $@"
+}
+
+function copy {
+	local FROM=$1
+	local TO=$2
+	cat <<EOF
+echo '- write $TO from $FROM'
+base64 -d <<EOG | tee $TO > /dev/null
+$(base64 <$FROM)
+EOG
+EOF
+}
+
+for NIXHOST in $NIXHOSTLIST; do
+	NIXHOST=${NIXHOST%.*}
+
+	if [ -z "$SSH_USER" ]; then
+		SSH_DEST=$NIXHOST
+	else
+		SSH_DEST=$SSH_USER@$NIXHOST
+	fi
+
+	echo "==== DOING $NIXHOST ===="
+
+	(header; . $CMDFILE; footer) | ssh -F $SSH_CONFIG $SSH_DEST sh -
+done

upgrade.sh

@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-
-# Get cluster subdirectory name
-
-cd $(dirname $0)
-
-CLUSTER="$1"
-if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
-	echo "Usage: $0 <cluster name>"
-	echo "The cluster name must be the name of a subdirectory of cluster/"
-	exit 1
-fi
-shift 1
-
-# Do actual stuff
-
-if [ -z "$@" ]; then
-	NIXHOSTLIST=$(ls cluster/$CLUSTER/node | grep -v '\.site\.')
-else
-	NIXHOSTLIST="$@"
-fi
-
-TMP_SCRIPT=/tmp/tmp-upgrade-$(date +%s).sh
-SSH_CONFIG=cluster/$CLUSTER/ssh_config
-
-for NIXHOST in $NIXHOSTLIST; do
-	NIXHOST=${NIXHOST%.*}
-
-	if [ -z "$SSH_USER" ]; then
-		SSH_DEST=$NIXHOST
-	else
-		SSH_DEST=$SSH_USER@$NIXHOST
-	fi
-
-	echo "==== DOING $NIXHOST ===="
-
-	ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_SCRIPT > /dev/null <<EOF
-set -ex
-
-nix-channel --add https://nixos.org/channels/nixos-21.11 nixos
-nix-channel --update
-nixos-rebuild boot
-EOF
-
-	read -p "Press Enter to continue (run upgrade on $NIXHOST)..."
-	ssh -t -F $SSH_CONFIG $SSH_DEST sudo sh $TMP_SCRIPT
-	ssh -F $SSH_CONFIG $SSH_DEST rm -v $TMP_SCRIPT
-
-	read -p "Press Enter to continue (reboot $NIXHOST)..."
-	ssh -t -F $SSH_CONFIG $SSH_DEST sudo reboot
-done

upgrade_nixos

@@ -0,0 +1,11 @@
+#!/usr/bin/env ./sshtool
+
+cmd nix-channel --add https://nixos.org/channels/nixos-21.11 nixos
+cmd nix-channel --update
+cmd nixos-rebuild boot
+
+if [ "$REBOOT_NODES" = "yes" ]; then
+	cmd reboot
+else
+	message "Node will not reboot, use \"REBOOT_NODES=yes $CMDFILE\" to reboot nodes when they finish upgrading."
+fi