setup: Use kickstart instead of Ansible

Kubernetes, or rather mostly Calico, does not play well on a machine with an immutable root filesyste. Specifically, Calico needs write access to a couple of paths on the root filesystem, such as `/etc/cni/net.d`, `/opt/cni/bin`, and `/usr/libexec/kubernetes/kubelet-plugins/volume`. Some of those paths can be configured, but doing so is quite cumbersome. While these paths could be made writable, e.g. using symlinks or bind mounts, it would add a lot of complexity to the *kubelet* Ansible role. After considering the options for a while, I decided that the best approach was probably to mount specific filesystems at these paths. Instead of using small LVM logical volumes for each one, I thought it would be better to use a single *btrfs* filesystem for all the mutable storage locations. This way, if I discover more paths that need to be writable, I can create subvolumes for them, without having to try to move or resize the existing volumes. Now that the Kubernetes nodes need their own special kickstart file for the disk layout, it also makes sense to handle the rest of the machine setup there, too. This eliminates the need for the *kubelet* Ansible role altogether. Any machine provisioned with this kickstart configuration is immediately ready to become a Kubernetes control plane or worker node.
2022-07-26 22:29:50 -05:00 · 2022-07-26 22:29:50 -05:00 · 8b8ae3df04
parent 70cb9186a6
commit 8b8ae3df04
2 changed files with 151 additions and 6 deletions
--- a/setup/README.md
+++ b/setup/README.md
@ -4,6 +4,13 @@
 * Fedora Kubernetes packages 1.22


+## Installation
+
+Use the [`fedora-k8s.ks`][0] kickstart file
+
+[0]: fedora-k8s.ks
+
+
 ## Machine Setup

 Add to *pyrocufflink.blue* domain:
@ -19,12 +26,6 @@ ansible-playbook \
    -e @join.creds
 ```

-Set up Kubernetes agent (`kubelet`):
-
-```sh
-ansible-playbook -l k8s-amd64-ctrl0.pyrocufflink.blue kubelet.yml -b
-```
-

 ## Initialize cluster

--- a/setup/fedora-k8s.ks
+++ b/setup/fedora-k8s.ks
@ -0,0 +1,144 @@
+# vim: set ft=sh :
+text
+url --metalink https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch
+repo --name=updates --metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f$releasever&arch=$basearch
+repo --name=fedora-modular --metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-modular-$releasever&arch=$basearch
+repo --name=updates-modular --metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-modular-f$releasever&arch=$basearch
+lang en_US.UTF-8
+keyboard us
+timezone --utc UTC
+rootpw --lock
+reboot
+
+bootloader --location mbr
+clearpart --all --initlabel
+reqpart
+part /boot --fstype ext4 --size=1024
+part btrfs.0 --fstype btrfs --size 4096
+part btrfs.1 --fstype btrfs --grow
+btrfs none --label fedora btrfs.0
+btrfs none --label data btrfs.1
+btrfs / --subvol --name root LABEL=fedora
+btrfs /home --subvol --name home LABEL=data
+btrfs /var --subvol --name var LABEL=data
+btrfs /etc/cni/net.d --subvol --name cni-net LABEL=data
+btrfs /usr/libexec/kubernetes/kubelet-plugins --subvol --name kubelet-plugins LABEL=data
+btrfs /opt --subvol --name opt LABEL=data
+
+%pre
+echo '%packages' > /tmp/packages.ks
+sys_vendor=$(tr A-Z a-z < /sys/devices/virtual/dmi/id/sys_vendor)
+case "${sys_vendor}" in
+kvm|bochs|qemu)
+    install_qga=1
+    ;;
+esac
+if [ ${install_qga:-0} -eq 1 ]; then
+    echo 'qemu-guest-agent' >> /tmp/packages.ks
+fi
+echo '%end' >> /tmp/packages.ks
+%end
+%include /tmp/packages.ks
+
+module --name cri-o --stream 1.22
+
+%packages --exclude-weakdeps
+-NetworkManager
+-authconfig
+-dhcp-client
+-dnf-plugins-core
+-dnf-yum
+-dracut-config-rescue
+-e2fsprogs
+-firewalld
+-man-db
+-openssh-clients
+-parted
+-plymouth
+-sssd-common
+-sssd-kcm
+-sudo
+-yum
+-zram-generator
+-zram-generator-defaults
+chrony
+cri-o
+cri-tools
+dnf
+dnf-command(system-upgrade)
+ethtool
+grubby
+iproute-tc
+iptables-nft
+kitty-terminfo
+kubernetes-client
+kubernetes-kubeadm
+kubernetes-node
+openssh-server
+rng-tools
+selinux-policy-targeted
+systemd-networkd
+%end
+
+services --enabled crio,kubelet,systemd-networkd,systemd-resolved
+
+%addon com_redhat_kdump --disable
+%end
+
+%post --erroronfail
+echo 'install_weak_deps=0' >> /etc/dnf/dnf.conf
+echo 'deltarpm=0' >> /etc/dnf/dnf.conf
+echo '%_excludedocs 1' >> /etc/rpm/macros
+
+systemctl mask systemd-journald-audit.socket
+
+sed -i \
+    -e 's:.*AuthorizedKeysCommand .*:AuthorizedKeysCommand /usr/local/libexec/ssh-authorized-keys %u %t:' \
+    -e 's:.*AuthorizedKeysCommandUser .*:AuthorizedKeysCommandUser nobody:' \
+    /etc/ssh/sshd_config
+cat > /usr/local/libexec/ssh-authorized-keys <<"EOF"
+#!/bin/sh
+
+USER="${1}"
+KEYTYPE="${2}"
+
+curl -gs http://sshkeys.pyrocufflink.blue/"${USER}"/"${KEYTYPE}".pub
+EOF
+chmod +x /usr/local/libexec/ssh-authorized-keys
+chcon -t bin_t /usr/local/libexec/ssh-authorized-keys
+setsebool -NP authlogin_yubikey on
+
+rm -rf /etc/sysconfig/network-scripts /etc/sysconfig/network
+
+cat > /etc/systemd/network/99-default.network <<EOF
+[Match]
+Name=en*
+Type=ether
+
+[Network]
+DHCP=true
+EOF
+ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
+
+# Generate SSH host keys before first boot, since / will be read-only then
+/usr/libexec/openssh/sshd-keygen ecdsa
+/usr/libexec/openssh/sshd-keygen ed25519
+/usr/libexec/openssh/sshd-keygen rsa
+
+cat > /etc/modules-load.d/k8s.conf <<'EOF'
+br_netfilter
+EOF
+
+cat > /etc/sysctl.d/k8s.conf <<'EOF'
+# Required for Kubernetes
+net.bridge.bridge-nf-call-iptables = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+net.ipv4.ip_forward = 1
+EOF
+
+sed -i 's/^driver = .*/driver = "btrfs"/' /etc/containers/storage.conf
+
+# Enable read-only rootfs. This cannot be done with part/logvol, as that would
+# make Anaconda mount it read-only befor the installation starts.
+sed -i -r '/\S+\s+\/\s+/s/subvol=root/ro,&/' /etc/fstab
+%end