kubernetes: Manage worker nodes

So far, I have been managing Kubernetes worker nodes with Fedora CoreOS
Ignition, but I have decided to move everything back to Fedora and
Ansible.  I like the idea of an immutable operating system, but the FCOS
implementation is not really what I want.  I like the automated updates,
but that can be accomplished with _dnf-automatic_.  I do _not_ like
giving up control of when to upgrade to the next Fedora release.
Mostly, I never did come up with a good way to manage application-level
configuration on FCOS machines.  None of my experiments (Cue+tmpl,
KCL+etcd+Luci) were successful, which mostly resulted in my manually
managing configuration on nodes individually.  Managing OS-level
configuration is also rather cumbersome, since it requires redeploying
the machine entirely.  Altogether, I just don't think FCOS fits with my
model of managing systems.

This commit introduces a new playbook, `kubernetes.yml`, and a handful of
new roles to manage Kubernetes worker nodes running Fedora Linux.  It
also adds two new deploy scripts, `k8s-worker.sh` and `k8s-longhorn.sh`,
which fully automate the process of bringing up worker nodes.

roles/cri-o/files/crun.conf

@@ -0,0 +1,7 @@
+[crio.runtime]
+default_runtime = "crun"
+
+[crio.runtime.runtimes.crun]
+runtime_path = "/usr/bin/crun"
+runtime_type = "oci"
+runtime_root = "/run/crun"

roles/cri-o/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: restart crio
+  service:
+    name: crio
+    state: restarted

roles/cri-o/tasks/main.yml

@@ -0,0 +1,47 @@
+- name: ensure cri-o is installed
+  package:
+    name:
+    - container-selinux
+    - cri-o
+    - crun
+    state: present
+  tags:
+  - install
+
+- name: ensure crio configuration drop-in directory exists
+  file:
+    path: /etc/crio/crio.conf.d
+    owner: root
+    group: root
+    mode: u=rwx,go=rx
+    state: directory
+  tags:
+  - config
+
+- name: ensure cri-o is configured to use crun
+  copy:
+    src: crun.conf
+    dest: /etc/crio/crio.conf.d/10-crio-crun.conf
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - restart crio
+  tags:
+  - config
+
+- name: flush handlers
+  meta: flush_handlers
+
+- name: ensure cri-o service is enabled
+  service:
+    name: crio
+    enabled: true
+  tags:
+  - service
+- name: ensure cri-o service is running
+  service:
+    name: crio
+    state: started
+  tags:
+  - service

roles/k8s-worker/meta/main.yml

@@ -0,0 +1,4 @@
+dependencies:
+- role: kubelet
+  tags:
+  - kubelet

roles/k8s-worker/tasks/main.yml

@@ -0,0 +1,30 @@
+- name: stat /var/lib/kubelet/config.yaml
+  stat:
+    path: /var/lib/kubelet/config.yaml
+  ignore_errors: true
+  register: stat_kublet_config
+  tags:
+  - kubeadm-join
+
+- name: generate bootstrap token
+  delegate_to: '{{ groups["k8s-controller"][0] }}'
+  command:
+    kubeadm token create
+    --kubeconfig /etc/kubernetes/admin.conf
+    --ttl 1h
+    --print-join-command
+  when:
+    not stat_kublet_config.stat.exists
+  changed_when: true
+  register: kubeadm_token_create
+  tags:
+  - bootstrap-token
+  - kubeadm-join
+- name: join the kubernetes cluster
+  command: >-
+    {{ kubeadm_token_create.stdout }}
+  when:
+    not stat_kublet_config.stat.exists
+  changed_when: true
+  tags:
+  - kubeadm-join

roles/kubelet/files/sysctl.conf

@@ -0,0 +1,4 @@
+# Required for Kubernetes
+net.bridge.bridge-nf-call-iptables = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+net.ipv4.ip_forward = 1

roles/kubelet/handlers/main.yml

@@ -0,0 +1,11 @@
+- name: load kernel modules
+  command:
+    /usr/lib/systemd/systemd-modules-load
+
+- name: set kernel tunables
+  command:
+    sysctl --system
+
+- name: swapoff -a
+  command:
+    swapoff -a

roles/kubelet/meta/main.yml

@@ -0,0 +1,5 @@
+dependencies:
+- systemd-base
+- role: cri-o
+  tags:
+  - cri-o

roles/kubelet/tasks/main.yml

@@ -0,0 +1,92 @@
+- name: load os-specific values
+  include_vars: '{{ item }}'
+  with_first_found:
+  - '{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml'
+  - '{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml'
+  - '{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml'
+  - '{{ ansible_distribution }}.yml'
+  - '{{ ansible_os_family }}.yml'
+  - defaults.yml
+  tags:
+  - always
+
+- name: ensure required packages are installed
+  dnf:
+    name: '{{ kubernetes_packages }}'
+    install_weak_deps: false
+    state: present
+  tags:
+  - install
+
+- name: ensure firewalld service is stopped
+  service:
+    name: firewalld
+    state: stopped
+    enabled: false
+  ignore_errors: true
+  tags:
+  - firewalld
+
+- name: ensure kernel modules-load is configured for kubernetes
+  copy:
+    content: |+
+      {{ kubernetes_kernel_modules | join('\n') }}
+    dest: /etc/modules-load.d/k8s.conf
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - load kernel modules
+  tags:
+  - kmod
+
+- name: ensure kernel tunables are set for kubernetes
+  copy:
+    src: sysctl.conf
+    dest: /etc/sysctl.d/60-k8s.conf
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - set kernel tunables
+  tags:
+  - sysctl
+
+- name: ensure zram generator defaults are disabled
+  copy:
+    content: ''
+    dest: /etc/systemd/zram-generator.conf
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - reload systemd
+  tags:
+  - zram-generator
+
+- name: ensure zram0 is stopped
+  systemd:
+    name: systemd-zram-setup@zram0
+    state: stopped
+  ignore_errors: true
+  notify:
+  - swapoff -a
+  tags:
+  - zram-generator
+
+- name: ensure unneeded cni configuration files are removed
+  file:
+    path: /etc/cni/net.d/{{ item }}
+    state: absent
+  loop:
+  - 100-crio-bridge.conflist
+  - 200-loopback.conflist
+  tags:
+  - cni
+
+- name: ensure kubelet service is enabled
+  service:
+    name: kubelet
+    enabled: true
+  tags:
+  - service

roles/kubelet/vars/Fedora-40.yml

@@ -0,0 +1,6 @@
+kubernetes_distro_packages:
+- kubernetes
+- kubernetes-client
+- kubernetes-kubeadm
+
+kubernetes_packages: '{{ kubernetes_common_packages + kubernetes_distro_packages }}'

roles/kubelet/vars/main.yml

@@ -0,0 +1,11 @@
+kubernetes_kernel_modules:
+- br_netfilter
+
+kubernetes_common_packages:
+- ebtables
+- ethtool
+- iproute-tc
+# Required for Longhorn RWO volumes
+- iscsi-initiator-utils
+# Required for Longhorn RWX volumes
+- nfs-utils