scripts: Add VM host maintenance scripts

The `migrate-all.sh` script is used to migrate one or more VMs (default:
all) from one VM host to the other on demand.

The `shutdown-vmhost.sh` script prepares a VM host to shut down by
evicting Kubernetes Pods from the Nodes running on that host and then
shutting them down, followed by migrating the rest of the running VMs to
the other host.

scripts/migrate-all.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+# vim: set sw=4 ts=4 sts=4 et :
+
+list_vms() {
+    if [ $# -gt 0 ]; then
+        echo "$@" | xargs -n1
+    else
+        virsh list --name
+    fi
+}
+
+src=$1
+shift 2>&-
+dest=$1
+shift 2>&-
+
+if [ -z "${src}" ] || [ -z "${dest}" ]; then
+    printf 'usage: %s source_uri dest_uri [name [...]]\n' "${0##*/}"
+    exit 2
+fi
+
+export LIBVIRT_DEFAULT_URI="${src}"
+
+list_vms "$@" | while read name; do
+    [ -n "${name}" ] || continue
+    printf 'Migrating %s ...\n' "${name}"
+    virsh migrate "${name}" "${dest}" --live --persistent --undefinesource --verbose
+done

scripts/shutdown-vmhost.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+# vim: set sw=4 ts=4 sts=4 et :
+
+cordon() {
+    printf 'Cordoning node %s ... ' "$1" >&2
+    if kubectl cordon "$1"; then
+        :
+    else
+        r=$?
+        echo FAILED >&2
+        return $r
+    fi
+}
+
+drain() {
+    printf 'Draining node %s ... ' "$1"
+    kubectl drain \
+        --ignore-daemonsets \
+        --pod-selector 'app!=csi-attacher,app!=csi-provisioner' \
+        --delete-emptydir-data \
+        "$1"
+}
+
+migrate() {
+    printf 'Migrating %s to %s ... \n' "$1" "$2" >&2
+    virsh migrate "$1" "$2" --live --persistent --undefinesource --verbose
+}
+
+shutdown() {
+    if [ "$(virsh domstate "$1")" = 'shut off' ]; then
+        printf 'VM %s is already shut off\n' "$1" >&2
+        return 0
+    fi
+    virsh shutdown "$1" || return
+    printf 'Waiting for %s to shut down ... ' "$1" >&2
+    if virsh event --event lifecycle "$1"; then
+        echo OK >&2
+    else
+        r=$?
+        echo FAILED >&2
+        return $r
+    fi
+}
+
+host="$1"
+export LIBVIRT_DEFAULT_URI=qemu+ssh://${host}/system
+
+case "${host}" in
+vmhost0.pyrocufflink.blue)
+    dest=qemu+ssh://vmhost1.pyrocufflink.blue/system
+    ;;
+vmhost1.pyrocufflink.blue)
+    dest=qemu+ssh://vmhost0.pyrocufflink.blue/system
+    ;;
+*)
+    printf 'Unknown VM host: %s\n' "${host}" >&2
+    exit 1
+    ;;
+esac
+
+printf 'Checking libvirt connection ... '
+virsh uri || exit
+
+set --
+nodes=$(
+    kubectl get node -o name -l '!node-role.kubernetes.io/control-plane' \
+    | sed s@node/@@
+)
+for node in ${nodes}; do
+    if virsh domuuid "${node%%.*}" >/dev/null 2>&1; then
+        set -- "$@" "${node}"
+        cordon "${node}" || exit
+    fi
+done
+
+for node; do
+    drain "${node}" || exit
+done
+
+for node; do
+    shutdown "${node%%.*}" || exit
+done
+
+virsh list --name | while read -r vm_name; do
+    [ -n "${vm_name}" ] || continue
+    if [ "$(virsh domstate "${vm_name}")" = running ]; then
+        migrate "${vm_name}" "${dest}" || exit
+    fi
+done