From c59e9de714c4514d8d6c09a1d9ce41ee39f80dce Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 12 Mar 2022 16:41:38 -0600 Subject: [PATCH] rootfs: Persistent storage for Jenkins, Docker Docker and Jenkins need more storage than is available in the *tmpfs* filesystem mounted at `/var`. We can use NBD to provide additional storage volumes for these paths. The `nbd@.service` systemd unit, which is included in the *nbd* package but not actually installed by it, starts an NBD client, using the configuration in `/etc/nbdtab` for the specified block device. To avoid hard-coding the NBD server name here, the file included in the rootfs image is actually a template. This template is rendered at boot time, using the same server name specified on the kernel command line for the rootfs device. Obviously, this implies that the same server has to host both the rootfs image and the persistent storage, but that is not likely to be a problem for this project. To allow multiple devices to use the same rootfs image but separate persistent storage devices, the `nbdtab` template can include a placeholder for the device's serial number. This only works for Raspberry Pi devices, of course, but the concept could be extended to other device types by adding more placeholders for other machine-specific properties (e.g. DMI UUID, etc.) Since `/var/lib/jenkins` is the home directory for the *jenkins* user, and thus the location of its `~/.ssh/authorized_keys` file, we have to make sure the *fetch-sshkeys@jenkins.service* unit does not start until the persistent storage volume is mounted. We also need a service unit that ensures the permissions of the mount point are correct, particularly for the first boot when the filesystem is initially created. Similarly, we have to ensure that the Docker daemon does not start until `/var/lib/docker` is mounted. Adding a `RequiresMountsFor=` property to the *docker.service* unit generates the necessary dependencies to accomplish this. --- configs/jenkinsagent_defconfig | 4 +++ rootfs/overlay/etc/fstab | 2 ++ rootfs/overlay/etc/nbdtab | 2 ++ .../systemd/system/chown-jenkins-home.service | 12 ++++++++ .../docker.service.d/requires-mounts.conf | 2 ++ .../lib/systemd/system/fetch-sshkeys@.service | 1 + .../systemd/system/nbdtab-generator.service | 12 ++++++++ .../overlay/usr/libexec/nbdtab-generator.sh | 29 +++++++++++++++++++ rootfs/post-build.sh | 4 +++ 9 files changed, 68 insertions(+) create mode 100644 rootfs/overlay/etc/nbdtab create mode 100644 rootfs/overlay/usr/lib/systemd/system/chown-jenkins-home.service create mode 100644 rootfs/overlay/usr/lib/systemd/system/docker.service.d/requires-mounts.conf create mode 100644 rootfs/overlay/usr/lib/systemd/system/nbdtab-generator.service create mode 100755 rootfs/overlay/usr/libexec/nbdtab-generator.sh create mode 100755 rootfs/post-build.sh diff --git a/configs/jenkinsagent_defconfig b/configs/jenkinsagent_defconfig index 9bc2288..a0d1aab 100644 --- a/configs/jenkinsagent_defconfig +++ b/configs/jenkinsagent_defconfig @@ -10,6 +10,7 @@ BR2_INIT_SYSTEMD=y BR2_SYSTEM_DEFAULT_PATH="/bin:/sbin:/usr/bin:/usr/sbin" BR2_ROOTFS_USERS_TABLES="$(BR2_EXTERNAL_jenkinsagent_PATH)/rootfs/users" BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_jenkinsagent_PATH)/rootfs/overlay" +BR2_ROOTFS_POST_BUILD_SCRIPT="$(BR2_EXTERNAL_jenkinsagent_PATH)/rootfs/post-build.sh" BR2_ROOTFS_POST_FAKEROOT_SCRIPT="$(BR2_EXTERNAL_jenkinsagent_PATH)/rootfs/post-fakeroot.sh" BR2_ROOTFS_POST_IMAGE_SCRIPT="$(BR2_EXTERNAL_jenkinsagent_PATH)/rootfs/post-image.sh" BR2_LINUX_KERNEL=y @@ -22,6 +23,8 @@ BR2_LINUX_KERNEL_INTREE_DTS_NAME="broadcom/bcm2711-rpi-4-b" BR2_LINUX_KERNEL_NEEDS_HOST_OPENSSL=y # BR2_PACKAGE_BUSYBOX is not set BR2_PACKAGE_GIT=y +BR2_PACKAGE_SED=y +BR2_PACKAGE_BTRFS_PROGS=y BR2_PACKAGE_XORG7=y BR2_PACKAGE_RPI_FIRMWARE=y BR2_PACKAGE_RPI_FIRMWARE_VARIANT_PI4=y @@ -35,6 +38,7 @@ BR2_PACKAGE_CHRONY=y BR2_PACKAGE_DHCPCD=y # BR2_PACKAGE_IFUPDOWN_SCRIPTS is not set BR2_PACKAGE_IPROUTE2=y +BR2_PACKAGE_NBD=y BR2_PACKAGE_OPENSSH=y BR2_PACKAGE_COREUTILS=y BR2_PACKAGE_COREUTILS_INDIVIDUAL_BINARIES=y diff --git a/rootfs/overlay/etc/fstab b/rootfs/overlay/etc/fstab index 10b4323..2c2fc39 100644 --- a/rootfs/overlay/etc/fstab +++ b/rootfs/overlay/etc/fstab @@ -3,3 +3,5 @@ tmpfs /root tmpfs size=1M,mode=550 0 0 /dev/mmcblk0 /run/storage ext4 ro,noexec,nosuid,nodev 0 2 overlay /etc/ssh overlay ro,lowerdir=/etc/ssh:/run/storage/ssh,noexec,nodev,nosuid,x-systemd.requires-mounts-for=/run/storage 0 0 /run/storage/docker/key.json /etc/docker/key.json none bind 0 0 +/dev/nbd1 /var/lib/docker btrfs _netdev,x-systemd.makefs,x-systemd.requires=nbd@nbd1.service 0 2 +/dev/nbd2 /var/lib/jenkins btrfs _netdev,x-systemd.makefs,x-systemd.requires=nbd@nbd2.service 0 2 diff --git a/rootfs/overlay/etc/nbdtab b/rootfs/overlay/etc/nbdtab new file mode 100644 index 0000000..33e804c --- /dev/null +++ b/rootfs/overlay/etc/nbdtab @@ -0,0 +1,2 @@ +nbd1 @NBDHOST@ docker-@SERIAL@ persist +nbd2 @NBDHOST@ jenkins-@SERIAL@ persist diff --git a/rootfs/overlay/usr/lib/systemd/system/chown-jenkins-home.service b/rootfs/overlay/usr/lib/systemd/system/chown-jenkins-home.service new file mode 100644 index 0000000..050f2f9 --- /dev/null +++ b/rootfs/overlay/usr/lib/systemd/system/chown-jenkins-home.service @@ -0,0 +1,12 @@ +[Unit] +Description=Set Jenkins home directory permissions +After=var-lib-jenkins.mount +Before=remote-fs.target + +[Service] +Type=oneshot +ExecStart=/usr/bin/chown jenkins:jenkins /var/lib/jenkins +ExecStart=/usr/bin/chmod u=rwx,go=rx /var/lib/jenkins + +[Install] +WantedBy=var-lib-jenkins.mount diff --git a/rootfs/overlay/usr/lib/systemd/system/docker.service.d/requires-mounts.conf b/rootfs/overlay/usr/lib/systemd/system/docker.service.d/requires-mounts.conf new file mode 100644 index 0000000..ee4f9cc --- /dev/null +++ b/rootfs/overlay/usr/lib/systemd/system/docker.service.d/requires-mounts.conf @@ -0,0 +1,2 @@ +[Unit] +RequiresMountsFor=/var/lib/docker diff --git a/rootfs/overlay/usr/lib/systemd/system/fetch-sshkeys@.service b/rootfs/overlay/usr/lib/systemd/system/fetch-sshkeys@.service index 3a80435..7f36107 100644 --- a/rootfs/overlay/usr/lib/systemd/system/fetch-sshkeys@.service +++ b/rootfs/overlay/usr/lib/systemd/system/fetch-sshkeys@.service @@ -2,6 +2,7 @@ Description=Fetch SSH authorized_keys for %I Wants=network-online.target After=network-online.target +After=remote-fs.target [Service] Type=oneshot diff --git a/rootfs/overlay/usr/lib/systemd/system/nbdtab-generator.service b/rootfs/overlay/usr/lib/systemd/system/nbdtab-generator.service new file mode 100644 index 0000000..884030a --- /dev/null +++ b/rootfs/overlay/usr/lib/systemd/system/nbdtab-generator.service @@ -0,0 +1,12 @@ +[Unit] +Description=Generate nbdtab +DefaultDependencies=no +Before=sysinit.target +After=local-fs.target + +[Service] +Type=oneshot +ExecStart=/usr/libexec/nbdtab-generator.sh + +[Install] +WantedBy=sysinit.target diff --git a/rootfs/overlay/usr/libexec/nbdtab-generator.sh b/rootfs/overlay/usr/libexec/nbdtab-generator.sh new file mode 100755 index 0000000..ebe0be9 --- /dev/null +++ b/rootfs/overlay/usr/libexec/nbdtab-generator.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +[ -f /etc/nbdtab ] || exit 0 + +# shellcheck disable=SC2046 +set -- $(cat /proc/cmdline) +while [ $# -ge 1 ]; do + case "$1" in + root=nbd:*) + arg=${1#*:} + host=${arg%:*} + ;; + esac + shift +done + +[ -n "${host}" ] || exit 0 + +serial=$(sed -nr '/Serial/s/.*([0-9a-f]{8})/\1/p' /proc/cpuinfo) +if [ $? -ne 0 ]; then + serial=UNKNOWN-SERIAL +fi +sed \ + -e s/@NBDHOST@/"${host}"/ \ + -e s/@SERIAL@/"${serial}"/ \ + /etc/nbdtab \ + > /run/nbdtab + +mount -o bind /run/nbdtab /etc/nbdtab diff --git a/rootfs/post-build.sh b/rootfs/post-build.sh new file mode 100755 index 0000000..c372ca7 --- /dev/null +++ b/rootfs/post-build.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cp -puv "${BUILD_DIR}"/nbd-*/systemd/nbd@.service \ + "${TARGET_DIR}"/usr/lib/systemd/system/