From ebce1073a6df652427abd7a360b864b7884221b0 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Thu, 1 Dec 2022 22:22:46 -0600 Subject: [PATCH 1/5] config: Fix syntax highlighting setting Apparently, Zola moved the `highlight_code` and `highlight_theme` settings into a new `markdown` section. --- config.toml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/config.toml b/config.toml index b78ea0a..88ec080 100644 --- a/config.toml +++ b/config.toml @@ -4,16 +4,17 @@ base_url = "https://dustin.hatch.name/" # Whether to automatically compile all Sass files in the sass directory compile_sass = true -# Whether to do syntax highlighting -# Theme can be customised by setting the `highlight_theme` variable to a theme supported by Zola -highlight_code = true -highlight_theme = 'axar' - # Whether to build a search index to be used later on by a JavaScript library build_search_index = false generate_feed = true feed_filename = 'atom.xml' +[markdown] +# Whether to do syntax highlighting +# Theme can be customised by setting the `highlight_theme` variable to a theme supported by Zola +highlight_code = true +highlight_theme = 'axar' + [extra] # Put all your custom variables here -- 2.51.0 From c08ff6c268697a4a934a489694a355a7aa615c21 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Thu, 1 Dec 2022 22:24:43 -0600 Subject: [PATCH 2/5] style: Increase paragraph line spacing This dramatically improves the readability of larget text blocks. --- sass/style.scss | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sass/style.scss b/sass/style.scss index 3594efa..b88cf0b 100644 --- a/sass/style.scss +++ b/sass/style.scss @@ -332,6 +332,15 @@ article.post .post-date { overflow: auto; } +.post p { + line-height: 1.4em; +} + +.post li { + line-height: 1.4em; + margin-bottom: 1em; +} + /* CV */ .cv.panel { -- 2.51.0 From 12c99fb5f0f158110a2b95d3ecff013a945257e5 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Thu, 1 Dec 2022 22:26:36 -0600 Subject: [PATCH 3/5] blog: Speed Up Jenkins Startup Time in Kubernetes --- content/blog/speed-up-jenkins-k8s-startup.md | 96 ++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 content/blog/speed-up-jenkins-k8s-startup.md diff --git a/content/blog/speed-up-jenkins-k8s-startup.md b/content/blog/speed-up-jenkins-k8s-startup.md new file mode 100644 index 0000000..3d8fc25 --- /dev/null +++ b/content/blog/speed-up-jenkins-k8s-startup.md @@ -0,0 +1,96 @@ ++++ +title = 'Speed Up Jenkins Startup Time in Kubernetes' +date = 2022-12-01T21:40:17-06:00 ++++ + +I recently migrated my Jenkins server at home to run inside my Kubernetes +cluster. I am very happy with it overall; upgrades are a lot simpler, and +Longhorn volume snapshots make rolling back bad plugin updates a breeze. One +issue that troubled me for a while, though, was that it took a *really* long +time for the Jenkins server container to start. Kubernetes would list the pod +in `ContainerCreating` state for several minutes, and then in +`ContainerCreateError` for a while, before finally starting the process. It +turns out this was because of the huge number of files in the Jenkins home +directory. When the container starts up, the container runtime has to go +through every file in the persistent volume and fix its permissions. My +Jenkins instance has over 1.5 million files, so scanning and modifying them all +takes a very long time. + +I was finally able to fix this issue today, after messing with it for a week or +so. There are two changes the container runtime has to make to every file in +the persistent volume: + +1. The group ownership/GID +2. The SELinux label + +Fixing the first problem is straightforward: set +`securityContext.fsGroupChangePolicy` on the pod or container to +`OnRootMismatch`. The container runtime will check the GID of the root +directory of the persistent volume, and if it is correct, skip checking any of +the rest of the files and directories. + +The second problem was quite a bit trickier, but still fixable. It took me a +bit longer to get the solution right, but with the help of a [cri-o GitHub +issue][0], I finally managed. The key is to configure the container to have a +static SELinux context; by default, the container runtime will assign a random +category when the container starts. Naturally, this means the context labels +of all the files in the persistent volume have to be changed every time, to +match the new category. Fortunately, the +`securityContext.seLinuxOptions.level` setting on the pod/container is +available. I looked at the category of the Jenkins current process and set +`level` to that: + +```sh +ps Z -p $(pgrep -f 'jenkins\.war') +``` + +``` +LABEL PID TTY STAT TIME COMMAND +system_u:system_r:container_t:s0:c525,c600 196790 ? Sl 0:50 java -Duser.home=/var/jenkins_home -Djenkins.model.Jenkins.slaveAgentPort=50000 -Dhudson.lifecycle=hudson.lifecycle.ExitLifecycle -jar /usr/share/jenkins/jenkins.war +``` + +The *level* field is the final two parts of the process's label and includes +the context's category. + +```yaml +spec: + containers: + - securityContext: + seLinuxOptions: + level: s0:c525,c600 +``` + +With this setting in place, the container will start with the same SELinux +context every time, so if the files are already labelled correctly, they do not +have to be changed. Unfortunately, by default, CRI-O, still walks the whole +directory tree to make sure. It can be configured to skip that step, though, +similar to the `fsGroupChangePolicy`. The pod needs a special annotation: + +```yaml +metadata: + annotations: + io.kubernetes.cri-o.TrySkipVolumeSELinuxLabel: 'true' +``` + +CRI-O itself also has to be configured to respect that annotation. CRI-O's +configuration is not well documented, but I was able to determine that these +two lines need to be added to `/etc/crio/crio.conf`: + +```toml +[crio.runtime.runtimes.runc] +allowed_annotations = ["io.kubernetes.cri-o.TrySkipVolumeSELinuxLabel"] +``` + +In summary, there were four steps to configure the container runtime not to +scan and touch every file in the persistent volume when starting the Jenkins +container: + +1. Set `securityContext.fsGroupChangePolicy` to `OnRootMismatch` +2. Set `securityContext.seLinuxOptions.level` to a static value +3. Add the `io.kubernetes.cri-o.TrySkipVolumeSELinuxLabel` annotation +4. Configure CRI-O to respect said annotation + +After completing all four steps, the Jenkins container starts up in seconds +instead of minutes. + +[0]: https://github.com/cri-o/cri-o/issues/6185 -- 2.51.0 From fab714379c5b139d6ff6c6efc5ae10ec7f786e2c Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Fri, 2 Dec 2022 21:36:03 -0600 Subject: [PATCH 4/5] ci: Build in Kubernetes Can't use the official Zola container image because it doesn't have a shell, so Jenkins can't run commands in it. --- ci/Containerfile | 15 --------------- ci/Jenkinsfile | 22 ++++++++++++++-------- ci/build.sh | 3 +-- ci/podTemplate.yaml | 20 ++++++++++++++++++++ 4 files changed, 35 insertions(+), 25 deletions(-) delete mode 100644 ci/Containerfile create mode 100644 ci/podTemplate.yaml diff --git a/ci/Containerfile b/ci/Containerfile deleted file mode 100644 index f7a735b..0000000 --- a/ci/Containerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM alpine - -RUN echo jenkins:*:3000018:3000017::/var/lib/jenkins:/bin/bash >> /etc/passwd - -RUN apk update && \ - apk add zola --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ && \ - apk add \ - openssh-client-default \ - python3 \ - py3-ruamel.yaml \ - rsync \ - && \ - rm -rf /var/cache/apk/* - -COPY ssh_known_hosts /etc/ssh/ssh_known_hosts diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 8184b16..a70b166 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -2,9 +2,8 @@ pipeline { agent { - dockerfile { - dir 'ci' - filename 'Containerfile' + kubernetes { + yamlFile 'ci/podTemplate.yaml' } } @@ -12,22 +11,29 @@ pipeline { disableConcurrentBuilds() } - triggers { - pollSCM '' + environment { + HOME = "${env.WORKSPACE}" } stages { stage('Build') { steps { - sh '. ci/build.sh' + container('zola') { + sh 'zola build --base-url /' + } + container('python') { + sh '. ci/build.sh' + } } } stage('Publish') { steps { - sshagent(['jenkins-web']) { - sh '. ci/publish.sh' + container('rsync') { + sshagent(['jenkins-web']) { + sh '. ci/publish.sh' + } } } } diff --git a/ci/build.sh b/ci/build.sh index df57139..734e263 100644 --- a/ci/build.sh +++ b/ci/build.sh @@ -1,5 +1,4 @@ -zola build --base-url / - +python3 -m pip install --user ruamel.yaml python3 /dev/fd/3 < songquotes.yml > public/songquotes.json 3< Date: Fri, 2 Dec 2022 22:07:09 -0600 Subject: [PATCH 5/5] ci: publish: Run a dry run for PR builds Only actually publish when building the *master* branch. --- ci/publish.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ci/publish.sh b/ci/publish.sh index 990775b..06f60cf 100644 --- a/ci/publish.sh +++ b/ci/publish.sh @@ -2,4 +2,12 @@ PUBLISH_HOST=web0.pyrocufflink.blue PUBLISH_USER=webapp.dchwww PUBLISH_PATH=htdocs/ -rsync -aP public/ ${PUBLISH_USER}@${PUBLISH_HOST}:${PUBLISH_PATH} +case "${BRANCH_NAME}" in +master) + ;; +*) + DRY_RUN=1 + ;; +esac + +rsync -aP public/ ${PUBLISH_USER}@${PUBLISH_HOST}:${PUBLISH_PATH} ${DRY_RUN:+-n} -- 2.51.0