mkvm: Add script to create FCOS VM

Fedora CoreOS can be provisioned on a QEMU virtual machine by providing
the Ignition configuration via `fw_cfg` value.  Unfortunately, the
`string` method does not work with JSON values, so we have to use
`file`.  The configuration file has to be uploaded via SFTP, rather than
`virsh vol-import`, since the latter would create the file with the
wrong permissions, and QEMU does not automatically adjust the
permissions of files used this way (like it does for disks).

Makefile

@@ -1,7 +1,8 @@
 .PHONY: \
 	all \
 	clean \
-	publish
+	publish \
+	vm
 
 .DEFAULT_GOAL := all
 
@@ -26,3 +27,7 @@ $(foreach t,$(wildcard *.yaml),$(eval $(call genrules,$(t))))
 publish: \
 		nvr1.ign
 	rsync -rti $^ files.pyrocufflink.blue:public_html/
+
+vm: \
+		$(VMNAME).ign
+	sh mkvm.sh $(VMNAME)

after-install.target

@@ -0,0 +1,2 @@
+[Unit]
+Description=Start services after package install

collectd.yaml

@@ -4,7 +4,7 @@ version: 1.4.0
 ignition:
   config:
     merge:
-    - local: packages.yaml
+    - local: packages.ign
 
 storage:
   files:
@@ -51,7 +51,35 @@ storage:
             Port 9103
         </Plugin>
 
+  - path: /etc/selinux/collectdlocal.cil
+    mode: 0644
+    contents:
+      inline: |
+        (typepermissive collectd_t)
+
+  - path: /etc/systemd/system/semodule-collectdlocal.service
+    mode: 0644
+    contents:
+      inline: |
+        [Unit]
+        ConditionPathExists=/etc/selinux/collectdlocal.cil
+        Before=collectd.service
+
+        [Service]
+        Type=oneshot
+        ExecStart=/usr/sbin/semodule -i /etc/selinux/collectdlocal.cil
+        ExecStartPost=/bin/rm -f /etc/selinux/collectdlocal.cil
+
+        [Install]
+        WantedBy=multi-user.target
+
+  links:
+  - path: /etc/systemd/system/after-install.target.wants/collectd.service
+    target: /usr/lib/systemd/system/collectd.service
+
 systemd:
   units:
   - name: collectd.service
     enabled: true
+  - name: semodule-collectdlocal.service
+    enabled: true

common.yaml

@@ -8,4 +8,4 @@ ignition:
     - local: collectd.ign
     - local: local_exporter.ign
     - local: notify-shutdown.ign
-    - local: step-ssh.ign
+    - local: ssh-host-certs.ign

datadisk-var.yaml

@@ -0,0 +1,11 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  filesystems:
+  - path: /var
+    device: /dev/vdb
+    format: ext4
+    wipe_filesystem: true
+    label: var
+    with_mount_unit: true

dch-repo.yaml

@@ -0,0 +1,9 @@
+variant: fcos
+version: 1.4.0
+
+storage:
+  files:
+  - path: /etc/yum.repos.d/dch.repo
+    mode: 0644
+    contents:
+      local: dch.repo

dch.repo

@@ -0,0 +1,6 @@
+[dch]
+name=DCH - Fedora $releasever
+baseurl=https://files.pyrocufflink.blue/yum/dch/fedora/$releasever
+gpgkey=https://files.pyrocufflink.blue/yum/dch/gnupg.pub
+gpgcheck=1
+skip_if_unavailable=true

install-kubelet.service

@@ -1,19 +0,0 @@
-# vim: set ft=systemd :
-[Unit]
-Description=Install Kubernetes/Kubelet
-After=network-online.target
-Wants=network-online.target
-Before=zincati.service
-ConditionPathExists=!/usr/bin/kubectl
-
-[Service]
-Type=oneshot
-RemainAfterExit=yes
-ExecStart=/usr/bin/rpm-ostree install --apply-live --allow-inactive \
-	cri-o \
-	iscsi-initiator-utils \
-	kubernetes-node \
-	kubernetes-kubeadm
-
-[Install]
-WantedBy=multi-user.target

install-packages.service

@@ -1,17 +1,19 @@
 # vim: set ft=systemd :
 [Unit]
-Description=Install collectd
+Description=Install additional packages
 After=network-online.target
 Wants=network-online.target
 Before=zincati.service
+Before=systemd-user-sessions.service
 ConditionPathExists=/etc/ignition/packages.d
-ConditionPathExists=/etc/ignition/packages.installed
+ConditionPathExists=!/etc/ignition/packages.installed
 
 [Service]
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=/bin/sh /etc/ignition/install-packages.sh
 ExecStartPost=/bin/touch /etc/ignition/packages.installed
+StandardOutput=journal+console
 
 [Install]
 WantedBy=multi-user.target

install-packages.sh

@@ -6,3 +6,6 @@ if [ ! -d /etc/ignition/packages.d ]; then
 fi
 
 cat /etc/ignition/packages.d/* | xargs rpm-ostree install --apply-live -y
+
+systemctl preset-all --preset-mode=enable-only
+systemctl start after-install.target

k8s-aarch64-n1.yaml

@@ -0,0 +1,14 @@
+variant: fcos
+version: 1.4.0
+ignition:
+  config:
+    merge:
+    - local: common.ign
+    - local: kubelet.ign
+
+storage:
+  files:
+  - path: /etc/hostname
+    contents:
+      inline: k8s-aarch64-n1.pyrocufflink.blue
+    mode: 0644

k8s-amd64-n3.yaml

@@ -0,0 +1,15 @@
+variant: fcos
+version: 1.4.0
+ignition:
+  config:
+    merge:
+    - local: common.ign
+    - local: datadisk-var.ign
+    - local: kubelet.ign
+
+storage:
+  files:
+  - path: /etc/hostname
+    contents:
+      inline: k8s-amd64-n3.pyrocufflink.blue
+    mode: 0644

kubelet.yaml

@@ -1,26 +1,43 @@
 variant: fcos
 version: 1.4.0
+ignition:
+  config:
+    merge:
+    - local: packages.ign
+
 storage:
   files:
-  - path: /etc/systemd/system/install-kubelet.service
+  - path: /etc/ignition/packages.d/kubelet
-    contents:
-      local: install-kubelet.service
     mode: 0644
+    contents:
+      inline: |
+        cri-o
+        kubernetes-node
+        kubernetes-kubeadm
+
   - path: /etc/modules-load.d/k8s.conf
     contents:
       inline: |+
         br_netfilter
+
   - path: /etc/sysctl.d/k8s.conf
     contents:
       inline: |+
         net.bridge.bridge-nf-call-iptables = 1
         net.bridge.bridge-nf-call-ip6tables = 1
         net.ipv4.ip_forward = 1
+
   links:
   - path: /etc/resolv.conf
     overwrite: true
     target: ../run/systemd/resolve/resolv.conf
+
+  - path: /etc/systemd/system/after-install.target.wants/crio.service
+    target: /usr/lib/systemd/system/crio.service
+
 systemd:
   units:
-  - name: install-kubelet.service
+  - name: crio.service
+    enabled: true
+  - name: kubelet.service
     enabled: true

local_exporter.container

@@ -1,6 +1,7 @@
 [Unit]
 Description=Bridge for local Prometheus metrics
-After=network.target
+After=network-online.target
+Wants=network-online.target
 
 [Container]
 Image=git.pyrocufflink.net/containerimages/local_exporter:latest

mkvm.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+# vim: set sw=4 ts=4 sts=4 et :
+
+: "${POOL:=default}"
+: "${VCPUS:=2}"
+: "${MEMORY:=2048}"
+: "${DISK_SIZE:=10}"
+: "${NETWORK=network=prod}"
+
+VMNAME="$1"
+
+pooldir=$(virsh pool-dumpxml "${POOL}" | xmllint --xpath '//path/text()' -)
+
+vmhost=$(virsh uri | cut -d/ -f3)
+ign="${pooldir}/${VMNAME}.ign"
+if [ -n "${vmhost}" ]; then
+    scp -s "${VMNAME}".ign "${vmhost}:${ign}"
+else
+    cp "${VMNAME}".ign "${ign}"
+fi
+
+image=$(virsh vol-list "${POOL}" \
+    | awk '/fedora-coreos-.*-qemu/{print $2}' \
+    | sort -V \
+    | tail -n1)
+
+virt-install \
+    --name "${VMNAME}" \
+    --vcpus "${VCPUS}" \
+    --memory "${MEMORY}" \
+    --os-variant fedora-coreos-stable \
+    --graphics none \
+    --sound none \
+    --disk size="${DISK_SIZE},backing_store=${image}" \
+    ${DISK:+--disk "${DISK}"} \
+    --network "${NETWORK}" \
+    --qemu-commandline="--fw_cfg name=opt/com.coreos/config,file=${ign}" \
+    --import
+

nginx.conf

@@ -1,7 +1,7 @@
 worker_processes  auto;
 
 error_log  /var/log/nginx/error.log notice;
-pid        /var/run/nginx.pid;
+pid        /var/cache/nginx/nginx.pid;
 
 
 events {

nginx.container

@@ -7,10 +7,12 @@ After=network.target
 Image=docker.io/library/nginx:1.25
 User=101
 Group=101
-Volume=%E/nginx:/etc/nginx:ro
+Volume=%E/nginx/nginx.conf:/etc/nginx/nginx.conf:ro,z
-Volume=%E/pki/nginx:/etc/pki/nginx:ro
+Volume=%E/nginx/conf.d:/etc/nginx/conf.d:ro,z
-Tmpfs=/var/cache/nginx
+Volume=%E/nginx/default.d:/etc/nginx/default.d:ro,z
-Tmpfs=/var/run/nginx
+Volume=%E/pki/nginx:/etc/pki/nginx:ro,z
+Mount=type=tmpfs,dst=/var/cache/nginx,chown=true
+VolatileTmp=yes
 ReadOnly=true
 AddCapability=CAP_NET_BIND_SERVICE
 Network=host

packages.yaml

@@ -14,3 +14,12 @@ storage:
     mode: 0644
     contents:
       local: install-packages.service
+  - path: /etc/systemd/system/after-install.target
+    mode: 0644
+    contents:
+      local: after-install.target
+
+systemd:
+  units:
+  - name: install-packages.service
+    enabled: true

ssh-bootstrap.service

@@ -1,13 +0,0 @@
-# vim: set ft=systemd :
-[Service]
-Description=Bootstrap SSH host certificates
-ConditionPathExistsGlob=!/etc/ssh/ssh_host_*_key-cert.pub
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=oneshot
-ExecStart=/bin/sh /etc/ssh/bootstrap.sh
-
-[Install]
-WantedBy=multi-user.target

ssh-bootstrap.sh

@@ -1,35 +0,0 @@
-#!/bin/sh
-# vim: set sw=4 ts=4 sts=4 et :
-
-gen_sshd_config() {
-    {
-    for x in ssh_host_*_key-cert.pub; do
-        printf 'HostCertificate /etc/ssh/%s\n' "${x}"
-    done
-    } > sshd_config.d/10-hostcertificate.conf
-}
-
-parse_response() {
-    jq -r '.certificates | to_entries | .[] | .key + " " + .value' \
-        | while read filename contents; do
-            [ -n "${filename}" ] || continue
-            echo "${contents}" > "${filename}" || return
-    done
-}
-
-request_sign() {
-    set -- \
-        https://bootstrap.pyrocufflink.blue/sshkeys/sign \
-        -H 'Accept: application/json' \
-        -F hostname=$(hostname -f)
-    for f in /etc/ssh/ssh_host_*_key.pub; do
-        set -- "$@" -F keys=@"${f}"
-    done
-    curl -fsSL "$@"
-}
-
-cd /etc/ssh || exit
-request_sign | parse_response
-gen_sshd_config
-
-systemctl reload sshd

ssh-host-certs.yaml

@@ -0,0 +1,40 @@
+variant: fcos
+version: 1.4.0
+
+ignition:
+  config:
+    merge:
+    - local: dch-repo.ign
+
+storage:
+  files:
+  - path: /etc/ignition/packages.d/sshca
+    mode: 0644
+    contents:
+      inline: |
+        sshca-cli-systemd
+
+  - path: /etc/ssh/sshd_config.d/10-hostcertificate.conf
+    mode: 0644
+    contents:
+      inline: |
+        HostCertificate /etc/ssh/ssh_host_ecdsa_key-cert.pub
+        HostCertificate /etc/ssh/ssh_host_ed25519_key-cert.pub
+        HostCertificate /etc/ssh/ssh_host_rsa_key-cert.pub
+
+  - path: /etc/sysconfig/ssh-host-cert-sign
+    mode: 0644
+    contents:
+      inline: |
+        SSHCA_SERVER=https://sshca.pyrocufflink.blue
+
+  links:
+  - path: /etc/systemd/system/after-install.target.wants/ssh-host-certs.target
+    target: /usr/lib/systemd/system/ssh-host-certs.target
+  - path: /etc/systemd/system/after-install.target.wants/ssh-host-certs-renew.timer
+    target: /usr/lib/systemd/system/ssh-host-certs-renew.timer
+
+systemd:
+  units:
+  - name: ssh-host-certs-renew.timer
+    enabled: true

step-ssh-renew.env

@@ -1,3 +0,0 @@
-STEP_CA_URL=https://ca.pyrocufflink.blue:32599
-STEP_ROOT=/etc/pki/ca-trust/source/anchors/dch-root-ca.crt
-STEP_PROVISIONER=sshpop

step-ssh-renew.target

@@ -1,6 +0,0 @@
-[Unit]
-Description=Renew SSH host certificates
-StopWhenUnneeded=yes
-Wants=step-ssh-renew@ed25519.service
-Wants=step-ssh-renew@ecdsa.service
-Wants=step-ssh-renew@rsa.service

step-ssh-renew.timer

@@ -1,11 +0,0 @@
-[Unit]
-Description=Periodically renew SSH host certificates
-
-[Timer]
-Unit=%N.target
-OnCalendar=Tue *-*-* 00:00:00
-RandomizedDelaySec=48h
-Persistent=yes
-
-[Install]
-WantedBy=timers.target

step-ssh-renew@.container

@@ -1,20 +0,0 @@
-[Unit]
-Description=Renew SSH host %I certificate
-After=network-online.target
-Wants=network-online.target
-ConditionPathExists=/etc/ssh/ssh_host_%I_key-cert.pub
-
-[Container]
-ContainerName=step-ssh-renew-%I
-Image=docker.io/smallstep/step-cli:0.25.0
-EnvironmentFile=/etc/sysconfig/step-ssh-renew
-Exec=step ssh renew -f /etc/ssh/ssh_host_%I_key-cert.pub /etc/ssh/ssh_host_%I_key
-Volume=/etc/ssh:/etc/ssh:rw
-Volume=/etc/pki:/etc/pki:ro
-# Required in order to be able to write to /etc/ssh
-SecurityLabelDisable=true
-User=0
-Group=0
-
-[Service]
-Type=oneshot

step-ssh.yaml

@@ -1,29 +0,0 @@
-variant: fcos
-version: 1.4.0
-
-storage:
-  files:
-  - path: /etc/ssh/bootstrap.sh
-    mode: 0755
-    contents:
-      local: ssh-bootstrap.sh
-  - path: /etc/containers/systemd/step-ssh-renew@.container
-    mode: 0644
-    contents:
-      local: step-ssh-renew@.container
-  - path: /etc/sysconfig/step-ssh-renew
-    mode: 0600
-    contents:
-      local: step-ssh-renew.env
-  - path: /etc/systemd/system/ssh-bootstrap.service
-    mode: 0644
-    contents:
-      local: ssh-bootstrap.service
-  - path: /etc/systemd/system/step-ssh-renew.target
-    mode: 0644
-    contents:
-      local: step-ssh-renew.target
-  - path: /etc/systemd/system/step-ssh-renew.timer
-    mode: 0644
-    contents:
-      local: step-ssh-renew.timer