Refactor deployment scripts

This commit is contained in:
Alex 2022-04-20 13:01:51 +02:00
parent 50e9f0b589
commit 9c9c776213
Signed by: lx
GPG key ID: 0E496D15096376BE
7 changed files with 143 additions and 144 deletions

View file

@ -10,9 +10,10 @@ It sets up the following:
The following scripts are available here:
- `deploy_nixos`, the main script that updates the NixOS config
- `genpki.sh`, a script to generate Consul and Nomad's TLS PKI (run this once only)
- `deploy.sh`, the main script that updates the NixOS config and sets up all of the TLS secrets
- `upgrade.sh`, a script to upgrade NixOS
- `deploy_pki`, a script that sets up all of the TLS secrets
- `upgrade_nixos`, a script to upgrade NixOS
- `tlsproxy.sh`, a script that allows non-TLS access to the TLS-secured Consul and Nomad, by running a simple local proxy with socat
- `tlsenv.sh`, a script to be sourced (`source tlsenv.sh`) that configures the correct environment variables to use the Nomad and Consul CLI tools with TLS

View file

@ -1,91 +0,0 @@
#!/usr/bin/env bash
# Get cluster subdirectory name
cd $(dirname $0)
CLUSTER="$1"
if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
echo "Usage: $0 <cluster name>"
echo "The cluster name must be the name of a subdirectory of cluster/"
exit 1
fi
shift 1
# Do actual stuff
if [ -z "$1" ]; then
NIXHOSTLIST=$(ls cluster/$CLUSTER/node | grep -v '\.site\.')
else
NIXHOSTLIST="$@"
fi
TMP_PATH=/tmp/tmp-deploy-$(date +%s)
SSH_CONFIG=cluster/$CLUSTER/ssh_config
YEAR=$(date +%Y)
for NIXHOST in $NIXHOSTLIST; do
NIXHOST=${NIXHOST%.*}
if [ -z "$SSH_USER" ]; then
SSH_DEST=$NIXHOST
else
SSH_DEST=$SSH_USER@$NIXHOST
fi
echo "==== DOING $NIXHOST ===="
echo "Sending NixOS config files"
ssh -F $SSH_CONFIG $SSH_DEST mkdir -p $TMP_PATH $TMP_PATH/pki
cat nix/configuration.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/configuration.nix > /dev/null
cat nix/deuxfleurs.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/deuxfleurs.nix > /dev/null
cat nix/remote-unlock.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/remote-unlock.nix > /dev/null
cat nix/wesher.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/wesher.nix > /dev/null
cat nix/wesher_service.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/wesher_service.nix > /dev/null
cat cluster/$CLUSTER/cluster.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/cluster.nix > /dev/null
cat cluster/$CLUSTER/node/$NIXHOST.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/node.nix > /dev/null
cat cluster/$CLUSTER/node/$NIXHOST.site.nix | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/site.nix > /dev/null
echo "Sending secret files"
for SECRET in pki/consul-ca.crt pki/consul$YEAR.crt pki/consul$YEAR.key \
pki/consul$YEAR-client.crt pki/consul$YEAR-client.key \
pki/nomad-ca.crt pki/nomad$YEAR.crt pki/nomad$YEAR.key; do
test -f cluster/$CLUSTER/secrets/$SECRET && (cat cluster/$CLUSTER/secrets/$SECRET | ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/$SECRET > /dev/null)
done
echo "Rebuilding NixOS"
ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_PATH/deploy.sh > /dev/null <<EOF
set -ex
cd $TMP_PATH
mv deuxfleurs.nix remote-unlock.nix wesher.nix wesher_service.nix configuration.nix cluster.nix node.nix site.nix /etc/nixos
nixos-rebuild switch
mkdir -p /var/lib/nomad/pki /var/lib/consul/pki
if [ -f pki/consul-ca.crt ]; then
cp pki/consul* /var/lib/nomad/pki
mv pki/consul* /var/lib/consul/pki
chown -R consul:root /var/lib/consul/pki
fi
if [ -f pki/nomad-ca.crt ]; then
mv pki/nomad* /var/lib/nomad/pki
fi
# Save up-to-date Consul client certificates in Consul itself
export CONSUL_HTTP_ADDR=https://localhost:8501
export CONSUL_CACERT=/var/lib/consul/pki/consul-ca.crt
export CONSUL_CLIENT_CERT=/var/lib/consul/pki/consul$YEAR-client.crt
export CONSUL_CLIENT_KEY=/var/lib/consul/pki/consul$YEAR-client.key
consul kv put secrets/consul/consul-ca.crt - < /var/lib/consul/pki/consul-ca.crt
consul kv put secrets/consul/consul-client.crt - < /var/lib/consul/pki/consul$YEAR-client.crt
consul kv put secrets/consul/consul-client.key - < /var/lib/consul/pki/consul$YEAR-client.key
EOF
ssh -t -F $SSH_CONFIG $SSH_DEST sudo sh $TMP_PATH/deploy.sh
ssh -F $SSH_CONFIG $SSH_DEST rm -rv '/tmp/tmp-deploy-*'
done

12
deploy_nixos Executable file
View file

@ -0,0 +1,12 @@
#!/usr/bin/env ./sshtool
copy nix/configuration.nix /etc/nixos/configuration.nix
copy nix/deuxfleurs.nix /etc/nixos/deuxfleurs.nix
copy nix/remote-unlock.nix /etc/nixos/remote-unlock.nix
copy nix/wesher.nix /etc/nixos/wesher.nix
copy nix/wesher_service.nix /etc/nixos/wesher_service.nix
copy cluster/$CLUSTER/cluster.nix /etc/nixos/cluster.nix
copy cluster/$CLUSTER/node/$NIXHOST.nix /etc/nixos/node.nix
copy cluster/$CLUSTER/node/$NIXHOST.site.nix /etc/nixos/site.nix
cmd nixos-rebuild switch

34
deploy_pki Executable file
View file

@ -0,0 +1,34 @@
#!/usr/bin/env ./sshtool
PKI=cluster/$CLUSTER/secrets/pki
YEAR=$(date +%Y)
cmd mkdir -p /var/lib/nomad/pki /var/lib/consul/pki
for file in consul-ca.crt consul$YEAR.crt consul$YEAR.key consul$YEAR-client.crt consul$YEAR-client.key; do
if [ -f "$PKI/$file" ]; then
copy $PKI/$file /var/lib/consul/pki/$file
cmd chown consul:root /var/lib/consul/pki/$file
cmd chmod 0400 /var/lib/consul/pki/$file
fi
done
cmd systemctl restart consul
cmd sleep 10
for file in nomad-ca.crt nomad$YEAR.crt nomad$YER.key; do
if [ -f "$PKI/$file" ]; then
copy $PKI/$file /var/lib/nomad/pki/$file
fi
done
cmd systemctl restart nomad
set_env CONSUL_HTTP_ADDR=https://localhost:8501
set_env CONSUL_CACERT=/var/lib/consul/pki/consul-ca.crt
set_env CONSUL_CLIENT_CERT=/var/lib/consul/pki/consul$YEAR-client.crt
set_env CONSUL_CLIENT_KEY=/var/lib/consul/pki/consul$YEAR-client.key
cmd "consul kv put secrets/consul/consul-ca.crt - < /var/lib/consul/pki/consul-ca.crt"
cmd "consul kv put secrets/consul/consul-client.crt - < /var/lib/consul/pki/consul$YEAR-client.crt"
cmd "consul kv put secrets/consul/consul-client.key - < /var/lib/consul/pki/consul$YEAR-client.key"

83
sshtool Executable file
View file

@ -0,0 +1,83 @@
#!/usr/bin/env bash
cd $(dirname $0)
CMDFILE="$1"
shift 1
CLUSTER="$1"
if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
echo "Usage: $CMDFILE <cluster name>"
echo "The cluster name must be the name of a subdirectory of cluster/"
exit 1
fi
shift 1
if [ -z "$1" ]; then
NIXHOSTLIST=$(ls cluster/$CLUSTER/node | grep -v '\.site\.')
else
NIXHOSTLIST="$@"
fi
if [ -z "$ROOT_PASS" ]; then
read -s -p "Enter remote root password: " ROOT_PASS
echo
fi
SSH_CONFIG=cluster/$CLUSTER/ssh_config
function header {
cat <<EOF
export DEPLOYTOOL_ROOT_PASSWORD=$ROOT_PASS
cat > /tmp/deploytool_askpass <<EOG
#!/usr/bin/env sh
echo \$DEPLOYTOOL_ROOT_PASSWORD
EOG
chmod +x /tmp/deploytool_askpass
export SUDO_ASKPASS=/tmp/deploytool_askpass
sudo -A sh - <<EOEVERYTHING
EOF
}
function footer {
echo EOEVERYTHING
}
function message {
echo "echo '$@'"
}
function cmd {
echo "echo '- run $@'"
echo "$@"
}
function set_env {
echo "echo '- set $@'"
echo "export $@"
}
function copy {
local FROM=$1
local TO=$2
cat <<EOF
echo '- write $TO from $FROM'
base64 -d <<EOG | tee $TO > /dev/null
$(base64 <$FROM)
EOG
EOF
}
for NIXHOST in $NIXHOSTLIST; do
NIXHOST=${NIXHOST%.*}
if [ -z "$SSH_USER" ]; then
SSH_DEST=$NIXHOST
else
SSH_DEST=$SSH_USER@$NIXHOST
fi
echo "==== DOING $NIXHOST ===="
(header; . $CMDFILE; footer) | ssh -F $SSH_CONFIG $SSH_DEST sh -
done

View file

@ -1,51 +0,0 @@
#!/usr/bin/env bash
# Get cluster subdirectory name
cd $(dirname $0)
CLUSTER="$1"
if [ -z "$CLUSTER" ] || [ ! -d "cluster/$CLUSTER" ]; then
echo "Usage: $0 <cluster name>"
echo "The cluster name must be the name of a subdirectory of cluster/"
exit 1
fi
shift 1
# Do actual stuff
if [ -z "$@" ]; then
NIXHOSTLIST=$(ls cluster/$CLUSTER/node | grep -v '\.site\.')
else
NIXHOSTLIST="$@"
fi
TMP_SCRIPT=/tmp/tmp-upgrade-$(date +%s).sh
SSH_CONFIG=cluster/$CLUSTER/ssh_config
for NIXHOST in $NIXHOSTLIST; do
NIXHOST=${NIXHOST%.*}
if [ -z "$SSH_USER" ]; then
SSH_DEST=$NIXHOST
else
SSH_DEST=$SSH_USER@$NIXHOST
fi
echo "==== DOING $NIXHOST ===="
ssh -F $SSH_CONFIG $SSH_DEST tee $TMP_SCRIPT > /dev/null <<EOF
set -ex
nix-channel --add https://nixos.org/channels/nixos-21.11 nixos
nix-channel --update
nixos-rebuild boot
EOF
read -p "Press Enter to continue (run upgrade on $NIXHOST)..."
ssh -t -F $SSH_CONFIG $SSH_DEST sudo sh $TMP_SCRIPT
ssh -F $SSH_CONFIG $SSH_DEST rm -v $TMP_SCRIPT
read -p "Press Enter to continue (reboot $NIXHOST)..."
ssh -t -F $SSH_CONFIG $SSH_DEST sudo reboot
done

11
upgrade_nixos Executable file
View file

@ -0,0 +1,11 @@
#!/usr/bin/env ./sshtool
cmd nix-channel --add https://nixos.org/channels/nixos-21.11 nixos
cmd nix-channel --update
cmd nixos-rebuild boot
if [ "$REBOOT_NODES" = "yes" ]; then
cmd reboot
else
message "Node will not reboot, use \"REBOOT_NODES=yes $CMDFILE\" to reboot nodes when they finish upgrading."
fi