kubernetes: Manage worker nodes

So far, I have been managing Kubernetes worker nodes with Fedora CoreOS
Ignition, but I have decided to move everything back to Fedora and
Ansible.  I like the idea of an immutable operating system, but the FCOS
implementation is not really what I want.  I like the automated updates,
but that can be accomplished with _dnf-automatic_.  I do _not_ like
giving up control of when to upgrade to the next Fedora release.
Mostly, I never did come up with a good way to manage application-level
configuration on FCOS machines.  None of my experiments (Cue+tmpl,
KCL+etcd+Luci) were successful, which mostly resulted in my manually
managing configuration on nodes individually.  Managing OS-level
configuration is also rather cumbersome, since it requires redeploying
the machine entirely.  Altogether, I just don't think FCOS fits with my
model of managing systems.

This commit introduces a new playbook, `kubernetes.yml`, and a handful of
new roles to manage Kubernetes worker nodes running Fedora Linux.  It
also adds two new deploy scripts, `k8s-worker.sh` and `k8s-longhorn.sh`,
which fully automate the process of bringing up worker nodes.

deploy.sh

@@ -12,5 +12,8 @@ set +a
 virsh uri || exit
 rbw unlock || exit
 sshca-cli user login || exit
+kubectl get node > /dev/null || exit
 
-. deploy/"${1}".sh
+script=deploy/"${1}".sh
+shift
+. "${script}"

deploy/k8s-longhorn.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+# vim: set ts=4 sw=4 noet :
+
+name=${1:-stor-$(diceware -n1 --no-caps)}
+hostname=${name}.k8s.pyrocufflink.black
+
+if ! virsh list --all --name | grep -qF ${name}; then
+	./newvm.sh ${name} \
+		--fedora 40 \
+		--memory 4096 \
+		--vcpus 4 \
+		--no-console \
+		--network network=kube \
+		-- \
+		--disk pool=default,size=4,cache=none \
+		--disk pool=default,size=8,cache=none \
+		--disk pool=default,size=512,cache=none \
+		|| exit
+	sleep 15
+fi
+
+if ! grep -q "${hostname}" hosts; then
+	sed -i '/\[k8s-longhorn\]/a'"${hostname}" hosts
+fi
+
+ansible-playbook \
+	-l ${hostname} \
+	wait-for-host.yml \
+	|| exit
+
+printf 'Waiting for SSH host certificate to be signed ... '
+until ssh-keyscan -c ${hostname} 2>/dev/null | grep -q cert; do
+	sleep 1
+done
+echo done
+
+ansible-playbook \
+	-l ${hostname} \
+	bootstrap.yml \
+	datavol.yml \
+	kubernetes.yml \
+	collectd.yml \
+	btop.yml \
+	-u root \
+	|| exit

deploy/k8s-node.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+# vim: set ts=4 sw=4 noet :
+
+name=${1:-node-$(diceware -n1 --no-caps)}
+hostname=${name}.k8s.pyrocufflink.black
+
+if ! virsh list --all --name | grep -qF ${name}; then
+	./newvm.sh ${name} \
+		--domain k8s.pyrocufflink.black \
+		--fedora 40 \
+		--memory 16384 \
+		--vcpus 8 \
+		--no-console \
+		--network network=kube \
+		-- \
+		--network network=storage \
+		--disk pool=default,size=32,cache=none \
+		--disk pool=default,size=32,cache=none \
+		|| exit
+	sleep 15
+fi
+
+if ! grep -q "${hostname}" hosts; then
+	sed -i '/\[k8s-node\]/a'"${hostname}" hosts
+fi
+
+ansible-playbook \
+	-l ${hostname} \
+	wait-for-host.yml \
+	|| exit
+
+printf 'Waiting for SSH host certificate to be signed ... '
+until ssh-keyscan -c ${hostname} 2>/dev/null | grep -q cert; do
+	sleep 1 || exit
+done
+echo done
+
+ansible-playbook \
+	-l ${hostname} \
+	bootstrap.yml \
+	datavol.yml \
+	users.yml \
+	kubernetes.yml \
+	collectd.yml \
+	btop.yml \
+	-u root \
+	|| exit

group_vars/k8s-longhorn.yml

@@ -0,0 +1,18 @@
+data_volumes:
+- dev: /dev/vdb
+  fstype: ext4
+  mountpoint: /var/lib/containers
+- dev: /dev/vdc
+  fstype: ext4
+  mountpoint: /var/lib/kubelet
+- dev: /dev/vdd
+  fstype: ext4
+  mountpoint: /var/lib/longhorn
+
+k8s_node_labels:
+  node-role.kubernetes.io/longhorn: ''
+  node.longhorn.io/create-default-disk: 'true'
+
+k8s_node_taints:
+- key: node-role.kubernetes.io/longhorn
+  effect: NoSchedule

group_vars/k8s-node.yml

@@ -1 +1,12 @@
 collectd_processes: '{{ collectd_processes_kubelet }}'
+
+data_volumes:
+- dev: /dev/vdb
+  fstype: ext4
+  mountpoint: /var/lib/containers
+- dev: /dev/vdc
+  fstype: ext4
+  mountpoint: /var/lib/kubelet
+
+k8s_node_labels:
+  network.du5t1n.me/storage: 'true'

hosts

@@ -11,6 +11,9 @@ bw0.pyrocufflink.blue
 [btop]
 chromie.pyrocufflink.blue
 
+[btop:children]
+kubelet
+
 [burp-client]
 bw0.pyrocufflink.blue
 cloud0.pyrocufflink.blue
@@ -24,6 +27,7 @@ git0.pyrocufflink.blue
 [collectd]
 
 [collectd:children]
+kubelet
 pyrocufflink
 
 [collectd-prometheus:children]
@@ -84,8 +88,13 @@ burp-server
 [k8s-controller]
 k8s-ctrl0.pyrocufflink.blue
 
+[k8s-longhorn]
+
 [k8s-node]
 
+[k8s-node:children]
+k8s-longhorn
+
 [kubelet:children]
 k8s-controller
 k8s-node
@@ -105,6 +114,9 @@ minio-backups
 [nfs-client:children]
 k8s-node
 
+[no-firewalld:children]
+kubelet
+
 [needproxy]
 nvr2.pyrocufflink.blue
 

kubernetes.yml

@@ -0,0 +1,19 @@
+- hosts: k8s-node
+  roles:
+  - role: k8s-worker
+    tags:
+    - k8s-worker
+  tasks:
+  - name: ensure kubernetes node labels and taints are set
+    delegate_to: localhost
+    kubernetes.core.k8s:
+      definition:
+        apiVersion: v1
+        kind: Node
+        metadata:
+          name: '{{ inventory_hostname }}'
+          labels: '{{ k8s_node_labels }}'
+        spec:
+          taints: '{{ k8s_node_taints | d(omit) }}'
+    tags:
+    - node-labels

roles/cri-o/files/crun.conf

@@ -0,0 +1,7 @@
+[crio.runtime]
+default_runtime = "crun"
+
+[crio.runtime.runtimes.crun]
+runtime_path = "/usr/bin/crun"
+runtime_type = "oci"
+runtime_root = "/run/crun"

roles/cri-o/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: restart crio
+  service:
+    name: crio
+    state: restarted

roles/cri-o/tasks/main.yml

@@ -0,0 +1,47 @@
+- name: ensure cri-o is installed
+  package:
+    name:
+    - container-selinux
+    - cri-o
+    - crun
+    state: present
+  tags:
+  - install
+
+- name: ensure crio configuration drop-in directory exists
+  file:
+    path: /etc/crio/crio.conf.d
+    owner: root
+    group: root
+    mode: u=rwx,go=rx
+    state: directory
+  tags:
+  - config
+
+- name: ensure cri-o is configured to use crun
+  copy:
+    src: crun.conf
+    dest: /etc/crio/crio.conf.d/10-crio-crun.conf
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - restart crio
+  tags:
+  - config
+
+- name: flush handlers
+  meta: flush_handlers
+
+- name: ensure cri-o service is enabled
+  service:
+    name: crio
+    enabled: true
+  tags:
+  - service
+- name: ensure cri-o service is running
+  service:
+    name: crio
+    state: started
+  tags:
+  - service

roles/k8s-worker/meta/main.yml

@@ -0,0 +1,4 @@
+dependencies:
+- role: kubelet
+  tags:
+  - kubelet

roles/k8s-worker/tasks/main.yml

@@ -0,0 +1,30 @@
+- name: stat /var/lib/kubelet/config.yaml
+  stat:
+    path: /var/lib/kubelet/config.yaml
+  ignore_errors: true
+  register: stat_kublet_config
+  tags:
+  - kubeadm-join
+
+- name: generate bootstrap token
+  delegate_to: '{{ groups["k8s-controller"][0] }}'
+  command:
+    kubeadm token create
+    --kubeconfig /etc/kubernetes/admin.conf
+    --ttl 1h
+    --print-join-command
+  when:
+    not stat_kublet_config.stat.exists
+  changed_when: true
+  register: kubeadm_token_create
+  tags:
+  - bootstrap-token
+  - kubeadm-join
+- name: join the kubernetes cluster
+  command: >-
+    {{ kubeadm_token_create.stdout }}
+  when:
+    not stat_kublet_config.stat.exists
+  changed_when: true
+  tags:
+  - kubeadm-join

roles/kubelet/files/sysctl.conf

@@ -0,0 +1,4 @@
+# Required for Kubernetes
+net.bridge.bridge-nf-call-iptables = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+net.ipv4.ip_forward = 1

roles/kubelet/handlers/main.yml

@@ -0,0 +1,11 @@
+- name: load kernel modules
+  command:
+    /usr/lib/systemd/systemd-modules-load
+
+- name: set kernel tunables
+  command:
+    sysctl --system
+
+- name: swapoff -a
+  command:
+    swapoff -a

roles/kubelet/meta/main.yml

@@ -0,0 +1,5 @@
+dependencies:
+- systemd-base
+- role: cri-o
+  tags:
+  - cri-o

roles/kubelet/tasks/main.yml

@@ -0,0 +1,92 @@
+- name: load os-specific values
+  include_vars: '{{ item }}'
+  with_first_found:
+  - '{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml'
+  - '{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml'
+  - '{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml'
+  - '{{ ansible_distribution }}.yml'
+  - '{{ ansible_os_family }}.yml'
+  - defaults.yml
+  tags:
+  - always
+
+- name: ensure required packages are installed
+  dnf:
+    name: '{{ kubernetes_packages }}'
+    install_weak_deps: false
+    state: present
+  tags:
+  - install
+
+- name: ensure firewalld service is stopped
+  service:
+    name: firewalld
+    state: stopped
+    enabled: false
+  ignore_errors: true
+  tags:
+  - firewalld
+
+- name: ensure kernel modules-load is configured for kubernetes
+  copy:
+    content: |+
+      {{ kubernetes_kernel_modules | join('\n') }}
+    dest: /etc/modules-load.d/k8s.conf
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - load kernel modules
+  tags:
+  - kmod
+
+- name: ensure kernel tunables are set for kubernetes
+  copy:
+    src: sysctl.conf
+    dest: /etc/sysctl.d/60-k8s.conf
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - set kernel tunables
+  tags:
+  - sysctl
+
+- name: ensure zram generator defaults are disabled
+  copy:
+    content: ''
+    dest: /etc/systemd/zram-generator.conf
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - reload systemd
+  tags:
+  - zram-generator
+
+- name: ensure zram0 is stopped
+  systemd:
+    name: systemd-zram-setup@zram0
+    state: stopped
+  ignore_errors: true
+  notify:
+  - swapoff -a
+  tags:
+  - zram-generator
+
+- name: ensure unneeded cni configuration files are removed
+  file:
+    path: /etc/cni/net.d/{{ item }}
+    state: absent
+  loop:
+  - 100-crio-bridge.conflist
+  - 200-loopback.conflist
+  tags:
+  - cni
+
+- name: ensure kubelet service is enabled
+  service:
+    name: kubelet
+    enabled: true
+  tags:
+  - service

roles/kubelet/vars/Fedora-40.yml

@@ -0,0 +1,6 @@
+kubernetes_distro_packages:
+- kubernetes
+- kubernetes-client
+- kubernetes-kubeadm
+
+kubernetes_packages: '{{ kubernetes_common_packages + kubernetes_distro_packages }}'

roles/kubelet/vars/main.yml

@@ -0,0 +1,11 @@
+kubernetes_kernel_modules:
+- br_netfilter
+
+kubernetes_common_packages:
+- ebtables
+- ethtool
+- iproute-tc
+# Required for Longhorn RWO volumes
+- iscsi-initiator-utils
+# Required for Longhorn RWX volumes
+- nfs-utils