wip: jenkins: Add iSCSI PV for airplaypi builds

wip: jenkins: buildroot iscsi pvc
v-m/alerts: Add alert for Internet down
2025-08-17 22:21:49 -05:00 · 2025-08-03 11:30:28 -05:00 · 2025-08-03 11:29:41 -05:00 · 2025-07-29 21:56:18 -05:00 · 2025-07-29 21:39:21 -05:00 · 2025-07-29 21:38:06 -05:00
39 changed files with 611 additions and 124 deletions
--- a/20125/config.yml
+++ b/20125/config.yml
@@ -32,56 +32,56 @@ applications:
  - instance: homeassistant.pyrocufflink.blue

 - name: Nextcloud
-  url: &url https://nextcloud.pyrocufflink.net/index.php
+  url: &url0 https://nextcloud.pyrocufflink.net/index.php
  icon:
    url: icons/nextcloud.png
  alerts:
-  - instance: *url
+  - instance: *url0
  - instance: cloud0.pyrocufflink.blue

 - name: Invoice Ninja
-  url: &url https://invoiceninja.pyrocufflink.net/
+  url: &url1 https://invoiceninja.pyrocufflink.net/
  icon:
    url: icons/invoiceninja.svg
    class: light-bg
  alerts:
-  - instance: *url
+  - instance: *url1

 - name: Jellyfin
-  url: &url https://jellyfin.pyrocufflink.net/
+  url: https://jellyfin.pyrocufflink.net/
  icon:
    url: icons/jellyfin.svg
  alerts:
-  - instance: *url
+  - job: jellyfin

 - name: Vaultwarden
-  url: &url https://bitwarden.pyrocufflink.net/
+  url: &url2 https://bitwarden.pyrocufflink.net/
  icon:
    url: icons/vaultwarden.svg
    class: light-bg
  alerts:
-  - instance: *url
+  - instance: *url2
  - alertgroup: Bitwarden

 - name: Paperless-ngx
-  url: &url https://paperless.pyrocufflink.blue/
+  url: &url3 https://paperless.pyrocufflink.blue/
  icon:
    url: icons/paperless-ngx.svg
  alerts:
-  - instance: *url
+  - instance: *url3
  - alertgroup: Paperless-ngx
  - job: paperless-ngx

 - name: Firefly III
-  url: &url https://firefly.pyrocufflink.blue/
+  url: &url4 https://firefly.pyrocufflink.blue/
  icon:
    url: icons/firefly-iii.svg
  alerts:
-  - instance: *url
+  - instance: *url4

 - name: Receipts
-  url: &url https://receipts.pyrocufflink.blue/
+  url: &url5 https://receipts.pyrocufflink.blue/
  icon:
    url: https://receipts.pyrocufflink.blue/static/icons/icon-512.png
  alerts:
-  - instance: *url
+  - instance: *url5
--- a/20125/status-server.yaml
+++ b/20125/status-server.yaml
@@ -33,11 +33,16 @@ spec:
      - name: status-server
        image: git.pyrocufflink.net/packages/20125.home
        imagePullPolicy: Always
+        env:
+        - name: RUST_LOG
+          value: info,status_server=debug
        volumeMounts:
        - mountPath: /usr/local/share/20125.home/config.yml
          name: config
          subPath: config.yml
          readOnly: True
+      nodeSelector:
+        kubernetes.io/arch: amd64
      imagePullSecrets:
      - name: imagepull-gitea
      volumes:
--- a/ansible/ara.yaml
+++ b/ansible/ara.yaml
@@ -32,6 +32,7 @@ spec:
      containers:
      - name: ara-api
        image: quay.io/recordsansible/ara-api
+        imagePullPolicy: IfNotPresent
        env:
        - name: ARA_BASE_DIR
          value: /etc/ara
--- a/ansible/rbac.yaml
+++ b/ansible/rbac.yaml
@@ -54,6 +54,7 @@ rules:
  - get
  resourceNames:
  - cluster-info
+  - kube-root-ca.crt

 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -132,3 +133,38 @@ roleRef:
 subjects:
 - kind: ServiceAccount
  name: host-provisioner
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: host-provisioner
+  namespace: victoria-metrics
+  annotations:
+    kubernetes.io/description: >-
+      Allows the host-provisioner to update the scrape-collectd
+      ConfigMap when adding new hosts.
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - configmaps
+  verbs:
+  - patch
+  - get
+  resourceNames:
+  - scrape-collectd
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: host-provisioner
+  namespace: victoria-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: host-provisioner
+subjects:
+- kind: ServiceAccount
+  name: host-provisioner
--- a/argocd/kustomization.yaml
+++ b/argocd/kustomization.yaml
@@ -24,6 +24,66 @@ configMapGenerator:
  - policy.csv

 patches:
+- patch: |-
+    apiVersion: apps/v1
+    kind: StatefulSet
+    metadata:
+      name: argocd-application-controller
+    spec:
+      template:
+        spec:
+          containers:
+          - name: argocd-application-controller
+            imagePullPolicy: IfNotPresent
+
+- patch: |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: argocd-notifications-controller
+    spec:
+      template:
+        spec:
+          containers:
+          - name: argocd-notifications-controller
+            imagePullPolicy: IfNotPresent
+
+- patch: |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: argocd-redis
+    spec:
+      template:
+        spec:
+          containers:
+          - name: redis
+            imagePullPolicy: IfNotPresent
+
+- patch: |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: argocd-repo-server
+    spec:
+      template:
+        spec:
+          containers:
+          - name: argocd-repo-server
+            imagePullPolicy: IfNotPresent
+
+- patch: |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: argocd-server
+    spec:
+      template:
+        spec:
+          containers:
+          - name: argocd-server
+            imagePullPolicy: IfNotPresent
+
 - patch: |-
    $patch: delete
    apiVersion: apiextensions.k8s.io/v1
--- a/authelia/configuration.yml
+++ b/authelia/configuration.yml
@@ -104,6 +104,8 @@ identity_providers:
      - profile
      - email
      - offline_access
+      - address
+      - phone
      authorization_policy: one_factor
      pre_configured_consent_duration: 8h
      token_endpoint_auth_method: client_secret_post
--- a/authelia/kustomization.yaml
+++ b/authelia/kustomization.yaml
@@ -37,6 +37,7 @@ patches:
        spec:
          containers:
          - name: authelia
+            imagePullPolicy: IfNotPresent
            env:
            - name: AUTHELIA_STORAGE_POSTGRES_TLS_CERTIFICATE_CHAIN_FILE
              value: /run/authelia/certs/postgresql/tls.crt
--- a/autoscaler/kustomization.yaml
+++ b/autoscaler/kustomization.yaml
@@ -22,6 +22,7 @@ patches:
        spec:
          containers:
          - name: cluster-autoscaler
+            imagePullPolicy: IfNotPresent
            command:
            - ./cluster-autoscaler
            - --v=4
--- a/cert-manager/cert-exporter.config.yml
+++ b/cert-manager/cert-exporter.config.yml
@@ -9,21 +9,6 @@ certs:
  namespace: default
  key: acme.sh/dustin.hatch.name/dustin.hatch.name.key
  cert: acme.sh/dustin.hatch.name/fullchain.cer
- name: hatchchat-cert
-  namespace: default
-  key: certificates/hatch.chat.key
-  cert: certificates/hatch.chat.crt
-  bundle: certificates/hatch.chat.pem
- name: tabitha-cert
-  namespace: default
-  key: certificates/tabitha.biz.key
-  cert: certificates/tabitha.biz.crt
-  bundle: certificates/tabitha.biz.pem
- name: chmod777-cert
-  namespace: default
-  key: certificates/chmod777.sh.key
-  cert: certificates/chmod777.sh.crt
-  bundle: certificates/chmod777.sh.pem
 - name: dustinandtabitha-cert
  namespace: default
  key: certificates/dustinandtabitha.com.key
@@ -34,8 +19,3 @@ certs:
  key: certificates/hatchlearningcenter.org.key
  cert: certificates/hatchlearningcenter.org.crt
  bundle: certificates/hatchlearningcenter.org.pem
- name: appsxyz-cert
-  namespace: default
-  key: certificates/apps.du5t1n.xyz.key
-  cert: certificates/apps.du5t1n.xyz.crt
-  bundle: certificates/apps.du5t1n.xyz.pem
--- a/cert-manager/cert-exporter.yaml
+++ b/cert-manager/cert-exporter.yaml
@@ -19,12 +19,8 @@ rules:
  resourceNames:
  - pyrocufflink-cert
  - dustinhatchname-cert
-  - hatchchat-cert
-  - tabitha-cert
-  - chmod777-cert
  - dustinandtabitha-cert
  - hlc-cert
-  - appsxyz-cert

 ---
 apiVersion: rbac.authorization.k8s.io/v1
--- a/cert-manager/certificates.yaml
+++ b/cert-manager/certificates.yaml
@@ -35,60 +35,6 @@ spec:
    algorithm: ECDSA
    rotationPolicy: Always

---
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: hatchchat-cert
-spec:
-  secretName: hatchchat-cert
-  dnsNames:
-  - hatch.chat
-  - '*.hatch.chat'
-  issuerRef:
-    group: cert-manager.io
-    kind: ClusterIssuer
-    name: zerossl
-  privateKey:
-    algorithm: ECDSA
-    rotationPolicy: Always
-
---
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: tabitha-cert
-spec:
-  secretName: tabitha-cert
-  dnsNames:
-  - tabitha.biz
-  - '*.tabitha.biz'
-  issuerRef:
-    group: cert-manager.io
-    kind: ClusterIssuer
-    name: zerossl
-  privateKey:
-    algorithm: ECDSA
-    rotationPolicy: Always
-
---
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: chmod777-cert
-spec:
-  secretName: chmod777-cert
-  dnsNames:
-  - chmod777.sh
-  - '*.chmod777.sh'
-  issuerRef:
-    group: cert-manager.io
-    kind: ClusterIssuer
-    name: zerossl
-  privateKey:
-    algorithm: ECDSA
-    rotationPolicy: Always
-
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
@@ -136,20 +82,3 @@ spec:
  privateKey:
    algorithm: ECDSA
    rotationPolicy: Always
-
---
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: appsxyz-cert
-spec:
-  secretName: appsxyz-cert
-  dnsNames:
-  - apps.du5t1n.xyz
-  issuerRef:
-    group: cert-manager.io
-    kind: ClusterIssuer
-    name: zerossl
-  privateKey:
-    algorithm: ECDSA
-    rotationPolicy: Always
--- a/cert-manager/jenkins.yaml
+++ b/cert-manager/jenkins.yaml
@@ -0,0 +1,30 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: jenkins
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - secrets
+  verbs:
+  - get
+  resourceNames:
+  - pyrocufflink-cert
+  - dustinhatchname-cert
+  - dustinandtabitha-cert
+  - hlc-cert
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: jenkins
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: jenkins
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: jenkins-jobs
--- a/cert-manager/kustomization.yaml
+++ b/cert-manager/kustomization.yaml
@@ -8,6 +8,7 @@ resources:
 - cert-exporter.yaml
 - dch-ca-issuer.yaml
 - secrets.yaml
+- jenkins.yaml

 configMapGenerator:
 - name: cert-exporter
--- a/firefly-iii/firefly-iii.yaml
+++ b/firefly-iii/firefly-iii.yaml
@@ -66,6 +66,7 @@ spec:
      containers:
      - name: firefly-iii
        image: docker.io/fireflyiii/core:version-6.0.19
+        imagePullPolicy: IfNotPresent
        envFrom:
        - configMapRef:
            name: firefly-iii
@@ -127,6 +128,7 @@ spec:
        spec:
          containers:
          - image: docker.io/library/busybox
+            imagePullPolicy: IfNotPresent
            name: wget
            command:
            - wget
--- a/home-assistant/home-assistant.yaml
+++ b/home-assistant/home-assistant.yaml
@@ -52,6 +52,16 @@ spec:
        app.kubernetes.io/name: home-assistant
        app.kubernetes.io/part-of: home-assistant
    spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: kubernetes.io/arch
+                operator: In
+                values:
+                - arm64
      containers:
      - name: home-assistant
        image: ghcr.io/home-assistant/home-assistant:2023.10.3
--- a/home-assistant/mosquitto.yaml
+++ b/home-assistant/mosquitto.yaml
@@ -55,6 +55,18 @@ spec:
        app.kubernetes.io/name: mosquitto
        app.kubernetes.io/part-of: home-assistant
    spec:
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - home-assistant
+              topologyKey: kubernetes.io/hostname
      containers:
      - name: mosquitto
        image: docker.io/library/eclipse-mosquitto:2.0.15
--- a/home-assistant/piper.yaml
+++ b/home-assistant/piper.yaml
@@ -36,6 +36,18 @@ spec:
        app.kubernetes.io/name: piper
        app.kubernetes.io/part-of: home-assistant
    spec:
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - home-assistant
+              topologyKey: kubernetes.io/hostname
      containers:
      - name: piper
        image: docker.io/rhasspy/wyoming-piper:1.3.2
--- a/home-assistant/whisper.yaml
+++ b/home-assistant/whisper.yaml
@@ -36,6 +36,18 @@ spec:
        app.kubernetes.io/name: whisper
        app.kubernetes.io/part-of: home-assistant
    spec:
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - home-assistant
+              topologyKey: kubernetes.io/hostname
      containers:
      - name: whisper
        image: docker.io/rhasspy/wyoming-whisper:1.0.0
--- a/home-assistant/zigbee2mqtt.yaml
+++ b/home-assistant/zigbee2mqtt.yaml
@@ -55,9 +55,10 @@ spec:
      nodeSelector:
        node-role.kubernetes.io/zigbee-ctrl: ''
      tolerations:
-      - key: du5t1n.me/machine
-        value: raspberrypi
-        effect: NoExecute
+      - key: node-role.kubernetes.io/zigbee-ctrl
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/zwave-ctrl
+        effect: NoSchedule
      containers:
      - name: zigbee2mqtt
        image: ghcr.io/koenkk/zigbee2mqtt:1.33.1
--- a/home-assistant/zwavejs2mqtt.yaml
+++ b/home-assistant/zwavejs2mqtt.yaml
@@ -57,9 +57,10 @@ spec:
      nodeSelector:
        node-role.kubernetes.io/zwave-ctrl: ''
      tolerations:
-      - key: du5t1n.me/machine
-        value: raspberrypi
-        effect: NoExecute
+      - key: node-role.kubernetes.io/zigbee-ctrl
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/zwave-ctrl
+        effect: NoSchedule
      containers:
      - name: zwavejs2mqtt
        image: ghcr.io/zwave-js/zwave-js-ui:9.1.2
--- a/jenkins/buildroot-iscsi.yaml
+++ b/jenkins/buildroot-iscsi.yaml
@@ -0,0 +1,98 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: buildroot-hudpi
+  namespace: jenkins-jobs
+  labels:
+    app.kubernetes.io/name: buildroot-hudpi
+    app.kubernetes.io/component: hudpi
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: ''
+  capacity:
+    storage: 64G
+  iscsi:
+    targetPortal: '[fd68:c2d2:500e:3ea3:8d42:e33e:264b:7c30]:3260'
+    iqn: iqn.2000-01.com.synology:storage0.Buildroot-hudpi.8181625090
+    lun: 1
+    chapAuthDiscovery: false
+    chapAuthSession: true
+    fsType: ext4
+    secretRef:
+      name: buildroot-hudpi-iscsi
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: network.du5t1n.me/storage
+          operator: In
+          values:
+          - 'true'
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: buildroot-hudpi
+  namespace: jenkins-jobs
+  labels:
+    app.kubernetes.io/name: buildroot-hudpi
+    app.kubernetes.io/component: hudpi
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: ''
+  resources:
+    requests:
+      storage: 64Gi
+
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: buildroot-airplaypi
+  namespace: jenkins-jobs
+  labels:
+    app.kubernetes.io/name: buildroot-airplaypi
+    app.kubernetes.io/component: airplaypi
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: ''
+  capacity:
+    storage: 32Gi
+  iscsi:
+    targetPortal: '[fd68:c2d2:500e:3ea3:8d42:e33e:264b:7c30]:3260'
+    iqn: iqn.2000-01.com.synology:storage0.Buildroot-airplaypi.8181625090
+    lun: 1
+    chapAuthDiscovery: false
+    chapAuthSession: true
+    fsType: ext4
+    secretRef:
+      name: buildroot-airplaypi-iscsi
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: network.du5t1n.me/storage
+          operator: In
+          values:
+          - 'true'
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: buildroot-airplaypi
+  namespace: jenkins-jobs
+  labels:
+    app.kubernetes.io/name: buildroot-airplaypi
+    app.kubernetes.io/component: airplaypi
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: ''
+  resources:
+    requests:
+      storage: 32Gi
--- a/jenkins/kustomization.yaml
+++ b/jenkins/kustomization.yaml
@@ -10,7 +10,8 @@ resources:
 - secrets.yaml
 - iscsi.yaml
 - gentoo-storage.yaml
- ../ssh-host-keys
+- ssh-host-keys
+- buildroot-iscsi.yaml

 patches:
 - patch: |
--- a/jenkins/secrets.yaml
+++ b/jenkins/secrets.yaml
@@ -73,3 +73,47 @@ spec:
      name: rpm-gpg-key-passphrase
      namespace: jenkins
    type: Opaque
+
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: buildroot-hudpi-iscsi
+  namespace: jenkins-jobs
+  labels: &labels
+    app.kubernetes.io/component: hudpi
+    app.kubernetes.io/name: buildroot-hudpi
+    app.kubernetes.io/part-of: buildroot
+spec:
+  encryptedData:
+    node.session.auth.password: AgB+7YE9JAI907M4S78W5ANGlEAj5/x9YMNvdd6KXPFAJAO3sKl1UrKVPKofQFiVHwLHpjqwjs1+g118jBO0wVQMomALuX8S5M9OziUkMwqjSraYRDnQiBk5Bs6P/RAa5UBCvDc28PDeFCdCYvZW8ZJx41rS1COkuZOcez5o1yMxjlEJcBJAZhFaAgmEfLSLOeMScwOxYIXnZIPL5hKrCKaGBdUZ5d2S5HmUjbdcXdS3AEznXu9RG5CSXKKYgBHU7mi6Z5DF3VuWwj+dwXpP0KBhjdJWBptdwWeWJ5Yhs02EWUVqMj9g6sZbK+R9QtWAflx3QqY89UJRaXHNT4Bs+uhIOxns/ptL9TAttZylzu5P8mTBXFPFSIlp3DJ8TORxKE3oRAk8gr9jIXfTzYuPyEb2trZIVXERVe5+k7w7QA4o3vt8SlJXyHKpRSQOBzYUZ0PTud/rF0gm9cFo29mA71MbR5OgRs9N59mrA6iejGHfXE+cpVs06AbPCkvdykHdB9mUqKQpYkdenRQLnOidxaAk9xgc744I4CvKxJ6gp1ub1xZNy7O4dg341Gsd/RBO49SVG+tlEwm6a9Fz3c8+9Gny8eayvBg287IRvYv5dEfm8JCkQ4dq4cqipPGhWjtTulsuN3mFw8hx0dLbzPI/3VDTc73si3flZ5QAkAXZp8shxcWxyYcuX04vjB+R72J8JvIHqlHcVK6nDN/7BSQ=
+    node.session.auth.password_in: AgBRLg+0Jm4XqBYlxNncpDT/7yJz0VJutAKwVposN1bNe3mPDGlTYScGq1u13qNG4xC7Yv41quM4vBrgPESRb+fF4h4iRBkURutW+BiHAg/p5BZKyyJlLe/9GU8WnFFCerQN7kFu8q7Nd78TgqdO5vo9/w1T5nPk87w7VD40JBAgkyihRk9L1ClXUC3gtqm3lm/r4+UutF2s3jRpCdZ8eZbr7Xuccbk6u/a+2DqQav5pNFdJLu3P5D5RrPO2GdwrLZDwQjZmTeDVwVD1lbTB0Jsbj77mE98PEN2DzY74EX+DuHvcprN8Tu9QAf1efe32xBnjJKt1r1n4OibVIivqVpnfO/x20G/gj1P5K8eHStAvnAYTHYfutOJsy/S9qrqMrX3J+kS4/OP9O+Hyb3JOXpRdXcwWGPaJl4C3MuHnwunjFlbjNJ9oeLYs7iqPtdHrEY09UWSj/VcNXwq0kTex5yWiqjj4pgZ0iUxB5RnbmJaEH1xpGhwkc1gvftCyA4CY/+iUIC0j2hV2WxcbOJGzZ+EKGq+mzXdWsTJGJvVdF7NFXYPADeJCQeG3MA3drjVZIu8fK8wOBqXlxvlRAHDUU7EkAoqaT51ezSl0x3wy6SeNFnengFGP0qcbLpgKF7oa/pa2mK++/VkfOi45NAi5LmP18kLlrJnQg9d6kzrHRThoQF5KOjt/WhgR0w8HKo5743NXlWT6YR4oZUFSSDk=
+    node.session.auth.username: AgCnSm21dt1x4kQWdEhLF4pV0Oz8lTpVZJoViF8T+KXbmhvgWIm6VkrD+A/C2e1/H1HXM9pSf3hkQerMl9DLSTTkV4FyLU3W6lrXxT/DoAVAl/Ji1uXtkdoANsq/jspj46QpqfIc6wlGQjSzMk1fysYBFQED3a8cqLFl6NA98VQT/EArpzu2Cmpi39Ras94wiiAxYKfoNs4oQ6ivaXjx+rbYa75cumRczFoanKALlAF/OZHQJ81AW7etlCcGPJKKz7EG1UZL43j4srrUoZK3r6b3fSXfPxpzJipRPisDyDhuvqS95elJLNnVVONsxCh5yVAw4LskCjaFEF8+lgHWkIAUN7fxMTTubF4deK1ihEoECYGaNvnasZIpgxCtX6LRtkEkXY53pcdOevEs+3G+LlYcq352G/gStkqdBkQV5fBa4w5y2jeruIfS+2FMmsk8G0GsV7XgceSL75Uh/wM6uQTBfqzNecdlNF5sfRzsWljVZWbWCp+f7MFut1kEwdsgwzWbX1cdkhASvSQwiZm9aS1yREYttM/xeNHpcxqgdEAirXAB8rRO3m3SJMQPJA9L7nm/NXgEVJ7S+oOz0tUPz4bUZVlKEY8Z3zQjHtLw13yqaxqHNB4gjrCJBCHA2+SDRQ3dHr27eQGUOlTqQZVa79S+qHy9Dy6EiLjSZYnNLWFZgZL9zUAeVGPviVm3Hei3JWONKGyE6uh5
+    node.session.auth.username_in: AgC0Bc3wzAeoK8hyglns7fpn7LAwkNrNuo6RjSdbteKVePbUJclqS+BaDTjMyU/Rq/iNsUZQgI4DRJkiQCZTC33wBbHhHU67nAtYART7rPcSBHA8EaWkADFLQiaflcLx0IK673agmVO84210BDvCkZMf/dSj6Kl2hiwqnGkx5ZQWvO+BbEQeOsD3Mia3DM3fnVcB7QHIsEJI+2QodIm6LVNIMJOGb/5+Ia8M38EVyys+QEEEFsLuGzDqruu0PeMz/hlHSMbjU+c7dieD2UPIttbmIdB8YK7MQV+IwhuOOgqucwYwK+aNpWFwK9+7kOVJRv/bkVIjwv80VuHC8/j87RjyoW51yMYKvovTrNnVJTgf1pHYutKctlJafKRYleEQ+ms6X+hptefxDsStzDDLeuB0ipVpu7R1b/KelgNySH0Z7CRZX7lWE7OMFdAquMKSBmyT4MGtiNYGPWzVC1SE1eI/nB7tpDUz+V77ai+zy3e1Hr3lyWzw+lhc/kJwN498+tPMzMeGqH2AGqA0QvtPo+8CDGz/rDbubNT8ZYrgfU7WrlR/LCyAy0B14wOAJ5IhnXN8TYgi2LKq6yJ1RnOyktOQPrwIKfgH8fGvx9Jne5StThbGRMc0QMKh9qhdhI5kvfnMuoLNQtsgii9EuXOBVKgI9+echEg+2N134HluTyFQV1gaUciT2kJ237az60jCFpgn9vX3E7GgHQ==
+  template:
+    metadata:
+      labels: *labels
+      name: buildroot-hudpi-iscsi
+      namespace: jenkins-jobs
+
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: buildroot-airplaypi-iscsi
+  namespace: jenkins-jobs
+  labels: &labels
+    app.kubernetes.io/component: airplaypi
+    app.kubernetes.io/name: buildroot-airplaypi
+    app.kubernetes.io/part-of: buildroot
+spec:
+  encryptedData:
+    node.session.auth.password: AgAAAH0omRLHXikdZ6mGXPxfQJv59KAgcDpGREhmT2EG3oNrt9Ow3kDQua3oiPdZcxwG5mimJ6nBPEOFRLPYaj46BUWlZWfvDGOVPwaIXOuO6oRvkwXHu7zcu8qIgh2hNcJNNjrRhwxMVa/IYJpQSZDotv0FIw+RQCY58SvuB/viyVjG5EcZYm69dDn9SQD8lIJvtHXRaezKOvSwQmnPsEYbqCnobsTKTciVbRXBkODOAzayZug4UdyeVrexgyqE9Uym/dPLRdnIIuW3mf3z3QVvKfYC89ETa9Rr4q34pb/2b1cuaHjmK40i2HOnLAgkdmnUdsm+0ulqxMTjhlXsAjZcR5qH5TqHnB/lFJxlJfoQsMFV3lcqq909xO4a70/AnTHF+unxzlrQXBQZ0ojO2iEPU2LNniSv12Mq2S6hx2riGD0PvfxmzkdbH6q2tXCn+7Tgwkx10QsSGk01Q/OCvrPKtCuSABfh+ODGJ4kcRFhF9nIi0AaYQytPNeRLgKpcgN64zqbsP1zolhimh/U6RHDEQabufl32Nn7GblaO+eiu9+jK2QpTE5mNg05rA8IFACb+jNdiWViaEIUXSvUjmPj5BhNwqe3AknqdrPOsVq+RYlmRkbiOyJxlVYfjUS/Ps8LGySUtb6tMYhFXqffbmxFY9dVTIhN6U2z6WSgxVzxEIFp5BHpoYOzTdLowOfleFFelmFCMDA6Ovsua3jI=
+    node.session.auth.password_in: AgCZ/LD9ejCea/udtBKSi1rm5RODKd92RE/m2Im9qJNwUlXgBDFFqKXMNf8FperHzZLJYqTzvBZEJcOgI6FdvY5oi+T2cJa10R+V7RM7YFR0Z6ey/JOsUJkf10CdMOWK1UTH8URhcKkaQhKqA956Ew/JZJoWvEnj967hzIkkqrz9SmbaJ1k8Pm0p4SpL9Jmz9rp6KT4bZUZmqHek7HrcFmO+LKtGLDKLIQEMvClZ6xFYG2bTxWhr/tjA2MolZdDZOsqrtSwSrge6e9Ptvk1ZxaO56O7VM2H3MC+s4DwvP7ibFk6/GFGg2P1QTwe1on/KOqZjXsYx4xTzbn+YY9gT0exNgAHtek1h42wOp98oLia3WWaVX0diHnMitXNEuBeK81aJcSjJg/MaHGVDc8yNa5UYBVTO/tYtTiN8FlXLob6moshKxblsSy4DB5RAqhYpZ2NnwHch9E41W1lHbWyGmbUanCP0F5C5CO7TQ9FMUwnFAfJ1NSLT9HzWIG5DPvgBOeUd9BtTuQGxc9qQBmqSPRklQrHycVgpB1KzBZ8qvDzS2+zKOXeuxG+xegR7CEBmLWkCh9WoLXpCp+GYUdY7oC5t+qS0tYaop1Vz70hlyHb9KVVGTwtkqZEyr/Y/Yk5ZWPk0TdgXe/F6awjhTcC54MAJjBaTHbkOSBLtBfvE7ixwMFnqX0HsYTz+nsfWE17GZRW5P+eMWUhysrTSTrw=
+    node.session.auth.username: AgAQhzmnkbpMVTzIKXs/NqoBl3BGCTJNbAcu/nXNkACNbA4oxe5OS8uErOMI4vGeqPGAdgSPsANugu0FTbprVC+7+K499mxddzUFE6JyV/spbA13eMo7at0d9Gwpv3i4hmxBFslED9B2/DPscXK4TOsgvidd+9K/n0RNWz9tRRsl8+vjTM4Hpkh1TrfqCtJvUSnHvhl+j+YVYMTfJrkCfjkxNSNYx1f8ipx3blwvqceZvht8iq0pVLQoj3lA06DdeQii+AGSpGBl9QXsUdxgsoQ93uYh5rZSfDPxNyNr7NADBumkNO5JlJMDgqHNP/vwF54jrE1MnPB0kfDtvRyXpl2GckpqK0CuHhqaLS0+iqymcxcCE0OKpU2Wd+tq3t0CB7IuJ3d+800NtRhAb7qwhrTME5J9yEpWp1ifu/piIaAQRmGm2MUHeDA+pCzY2Xx0S5rW3uC6/2/gvNIVyiaUzTos53PdfdYJKWffbSeBJhkhSMsIpkGASQ1wyGgX5gAzKDWDUyeBK/qwnEdS0EasDA39mGiem5w8t1gi2021SMpH8M81oZ7YeSV2Wu6aZCsJuWYcdRqrM+PGuhgkKNYnelRt1FjAREcCthPFNGquGbPHBzv7wynD4u/En5IzsdsAzec/EJfBxvbGCcRmVaQcxO0DXmk0S7Tl+a+3/E9Dckev1Xd+SV3EOii/1S7Ij4HymJcIMSFO3CGG
+    node.session.auth.username_in: AgAfO2gNFWu82gIk6E0J4rMeRBOTE025fwxwoNx6nMiLvIsyH6FEAqhYAWRgJv68X+u3fYIqfsQ2M2i31G/qvoKhp81hgGNTmOfbfdj3UAafCEotAug7EYjkZWPtr+rHLXaqKheR1FpRQD4Ukbet+JbT9mG3WzVAqPEtQ5xHCXZn6bgBkNdRQgNHnCK/EYjekHNs7qDa7ez4pNPzpOeRR3gffWD5E3q5x/PYBNL+/S8q1n3k+JUVnVdnWhju9BmncnOR5j+1v8IEhvMyZJAlXtl0i3tL/IfOVF2co5i3vffiC607AyhbD/B0SsNThlF63qFm4qmaUbjAMLZfveL52gGjiOXRHOVTiirzWDxqBFm5sz2hOpakK4qgtlahseN7eUfi7hrO3s/DkSYlj2Zd/X2tNUnugLXjl3FONxOp8HLi7muz1Dzyi6iLiHapyIlQZbAhkPbHTUbxaPXWlheDkAfCXBI54G2DZcI5xGgQq3Du2nVurKPk52ro3FO8VXAh0SUr6VhN9MuOcWzUWn/AHztTb6C6ABMBpkAw/mYmo5VBKUiUo+c6RpcD0a7kIKx8iELdp/fgsCaCWiQfIl0HV1Dx08FmTvA/h0/idW6SCGsBD5h7O8t/y2QUbhytln0L+TUOimseeCtu+N0TzYKU4+tmWPITWGojmw6panOkI5MwM4CbWLHI9K57l+v24pRS2XyQepaM6iTYzg==
+  template:
+    metadata:
+      name: buildroot-airplaypi-iscsi
+      namespace: jenkins-jobs
+      labels: *labels
--- a/jenkins/ssh-host-keys/kustomization.yaml
+++ b/jenkins/ssh-host-keys/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: jenkins-jobs
+
+resources:
+- ../../ssh-host-keys
--- a/kitchen/secrets.yaml
+++ b/kitchen/secrets.yaml
@@ -73,13 +73,13 @@ spec:
        weather:
          metrics:
            temperature: >-
-              homeassistant_sensor_temperature_celsius{entity="sensor.outdoor_temperature"}
+              round(homeassistant_sensor_temperature_celsius{entity="sensor.outdoor_temperature"}, 0.1)
            humidity: >-
-              homeassistant_sensor_humidity_percent{entity="sensor.outdoor_humidity"}
+              round(homeassistant_sensor_humidity_percent{entity="sensor.outdoor_humidity"}, 0.1)
            wind_speed: >-
-              homeassistant_sensor_unit_m_per_s{entity="sensor.wind_speed"}
+              round(homeassistant_sensor_unit_m_per_s{entity="sensor.wind_speed"}, 0.1)
            pool: >-
-              homeassistant_sensor_temperature_celsius{entity="sensor.pool_sensor_temperature"}
+              round(homeassistant_sensor_temperature_celsius{entity="sensor.pool_sensor_temperature"}, 0.1)

        homeassistant:
          url: wss://homeassistant.pyrocufflink.blue/api/websocket
--- a/kubelet-csr-approver/clusterrole.yaml
+++ b/kubelet-csr-approver/clusterrole.yaml
@@ -0,0 +1,42 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubelet-csr-approver
+rules:
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests/approval
+  verbs:
+  - update
+- apiGroups:
+  - certificates.k8s.io
+  resourceNames:
+  - kubernetes.io/kubelet-serving
+  resources:
+  - signers
+  verbs:
+  - approve
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
--- a/kubelet-csr-approver/deployment.yaml
+++ b/kubelet-csr-approver/deployment.yaml
@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kubelet-csr-approver
+  namespace: kube-system
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: kubelet-csr-approver
+
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: '8080'
+        prometheus.io/scrape: 'true'
+      labels:
+        app: kubelet-csr-approver
+
+    spec:
+      serviceAccountName: kubelet-csr-approver
+      containers:
+        - name: kubelet-csr-approver
+          image: postfinance/kubelet-csr-approver:latest
+          resources:
+            limits:
+              memory: "128Mi"
+              cpu: "500m"
+
+          args:
+            - -metrics-bind-address
+            - ":8080"
+            - -health-probe-bind-address
+            - ":8081"
+            - -leader-election
+
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8081
+
+          env:
+            - name: PROVIDER_REGEX
+              value: ^[abcdef]\.test\.ch$
+            - name: PROVIDER_IP_PREFIXES
+              value: "0.0.0.0/0,::/0"
+            - name: MAX_EXPIRATION_SEC
+              value: "31622400" # 366 days
+
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+          operator: Equal
--- a/kubelet-csr-approver/kustomization.yaml
+++ b/kubelet-csr-approver/kustomization.yaml
@@ -0,0 +1,42 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+labels:
+- pairs:
+    app.kubernetes.io/instance: kubelet-csr-approver
+
+resources:
+- clusterrole.yaml
+- deployment.yaml
+- rolebinding.yaml
+- serviceaccount.yaml
+
+patches:
+- patch: |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: kubelet-csr-approver
+      namespace: kube-system
+    spec:
+      template:
+        spec:
+          containers:
+          - name: kubelet-csr-approver
+            imagePullPolicy: IfNotPresent
+            env:
+            - name: PROVIDER_REGEX
+              value: ^(i-[a-z0-9]+\.[a-z0-9-]+\.compute\.internal|k8s-[a-z0-9-]+\.pyrocufflink\.blue|[a-z0-9-]+\.k8s\.pyrocufflink\.black)$
+            - name: PROVIDER_IP_PREFIXES
+              value: 172.30.0.0/16
+            - name: BYPASS_DNS_RESOLUTION
+              value: 'true'
+
+replicas:
+- name: kubelet-csr-approver
+  count: 1
+
+images:
+- name: postfinance/kubelet-csr-approver
+  newName: ghcr.io/postfinance/kubelet-csr-approver
+  newTag: v1.2.10
--- a/kubelet-csr-approver/rolebinding.yaml
+++ b/kubelet-csr-approver/rolebinding.yaml
@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubelet-csr-approver
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubelet-csr-approver
+subjects:
+- kind: ServiceAccount
+  name: kubelet-csr-approver
+  namespace: kube-system
--- a/kubelet-csr-approver/serviceaccount.yaml
+++ b/kubelet-csr-approver/serviceaccount.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubelet-csr-approver
+  namespace: kube-system
--- a/ntfy/kustomization.yaml
+++ b/ntfy/kustomization.yaml
@@ -20,4 +20,4 @@ configMapGenerator:

 images:
 - name: docker.io/binwiederhier/ntfy
-  newTag: v2.12.0
+  newTag: v2.13.0
--- a/ntfy/ntfy.yaml
+++ b/ntfy/ntfy.yaml
@@ -54,6 +54,7 @@ spec:
      containers:
      - name: ntfy
        image: docker.io/binwiederhier/ntfy:v2.5.0
+        imagePullPolicy: IfNotPresent
        args:
        - serve
        ports:
--- a/paperless-ngx/kustomization.yaml
+++ b/paperless-ngx/kustomization.yaml
@@ -49,4 +49,4 @@ images:
 - name: docker.io/gotenberg/gotenberg
  newTag: 8.21.1
 - name: docker.io/apache/tika
-  newTag: 3.2.0.0
+  newTag: 3.2.1.0
--- a/restic/kustomization.yaml
+++ b/restic/kustomization.yaml
@@ -36,6 +36,7 @@ patches:
            spec:
              containers:
              - name: restic-prune
+                imagePullPolicy: IfNotPresent
                env:
                - name: RESTIC_CACERT
                  value: /run/dch-ca/dch-root-ca.crt
@@ -48,3 +49,6 @@ patches:
                configMap:
                  name: dch-root-ca

+images:
+- name: ghcr.io/restic/restic
+  newTag: 0.18.0
--- a/ssh-host-keys/kustomization.yaml
+++ b/ssh-host-keys/kustomization.yaml
@@ -3,7 +3,6 @@ kind: Kustomization

 configMapGenerator:
 - name: ssh-known-hosts
-  namespace: jenkins-jobs
  files:
  - ssh_known_hosts
  options:
--- a/sshca/secrets.yaml
+++ b/sshca/secrets.yaml
@@ -59,7 +59,7 @@ metadata:
  namespace: sshca
 spec:
  encryptedData:
-    machine-ids.json: AgC41ts59Z0QpUXLn10Ecp59YrJbT2K7SsTpgVWQd/t3u6+Ds620hHlXZD9ewBSaKoaCtehw3Rr+TIR5h374aXjLO5uDMMJdfo0Bog3Zp9v0U9/4oC1AdGRrR9FP7Jdm9I5KH/IeQUBjlj5pAEPuyOBP9sa2ga1/fhMzmUTwl6y0LtV+33chVTpPd/xpIU6Q8MzFJeiCSiVGugpkDZeSL5Ij1zqX40ey/ApHnUq0rDMsvfJk/JCPVMg774F2GLvdXvQUHix2xZPwleF0y8gbmQ8OJAgZYt4QUxZSBD2z/uBb/1sfCxTBfNMqdt3Pr9uQg3P2ll9ndAGBx9ZxYeU2y/pNpVqy3hnIqPLg4CAen2hMVMMy0x/FG7s5d8aofXV2uw/wSXDC3ppWXYE4g4oAuvbFZFiUEO6aayUYlM/KDusgE4cR+I2jisqk0ELAZkK18drRS6Ipx9gN2BNqPFVaXqL18P2LF4YePlCEOLbsIOa64U3N6Z2zAA/Cf2VZmPWlAkIg2a1pbKa0kQNaI+EFlWLoMod5IJoWFSWUvoUoNHerEE+JFcxXOQLps8Bkpw87gMnwCwp8AEx+AxoTO00dRmD6/+UkWzL+Gq8gwwX7FEIWbIKabZz+sccxQu0lzzmQADJI4kPqDJqFWGuj2k1diMp1oAr2qVzpLmt2NumBCfDjA/T2sUSqpGdfp5+X0wC2OrTBrsOeQsVHABm2+kCS1lmPnfZqehR6VAcNAMbDjgD+joNesaAPk9xMcU34oIssR7UvmQmAJdsb2G4recT0WalGVKEP3OVORswUWpXEK+RaowmqsQEoCi0PolvtohXdwh9dsWjWwPZXI2YgVOqmKy2pHr+GTTSnuGFVUhKhEF+BvCL7th0TOLFa97Ob2N7Kpfi15QJaQ8TDNVfAwT7Asi2Md5Dy4Dt4FKYclojUJFvgtGV86U0zPmPyBrdNPIc4wmZMlhKR2/WIG6JSFofN9669ZWUSNt06DQan+xdrFkGeFnezq01Dch+4fA5fMwwc6x3XicKk9t+Wpd3MgskRm6z93sQ/lpBHfGw9WgyH7NvUWdOMIU5TZo1GZIHd4fl5zyalqL+CFSXXbKJjWqRL/96tIq6dvA8Gl1Ono1WmTlMUgiIkGg7LRx7WiTANOFUFJBtYgC+3vk5XxkUOjU10zADavBdJi94RhXeF/yCF9zbE9xYVn8MhcooXOJHTBJlOp0V3L1lWVg296BZWiiljhTOUQGt8aQD3QLuDCZPfVbUCadhm5oeAzj0vxzYbvoIRfoc5ljbKSbNiR/VfYv6oLj+/qzG23wDjbKC8/D/3W/Uj1gMapoTBzgK/Df9DDiy53Sv3L8tl5lGQRqzXJOFkAlf68Xq7BT1kqh2nbG5XX42IHvZ+VuvgYSqyVazIFALApCi2M8EFcdC9aZRVMnzRrO0K/SqZnfkekb5wt0duWGsyKVd0I9xFxWhPs4kNwJaYqyTRTvDHjAe37KSdbAMIHdhxw0NN30shCl74MqCCJArKMHf13oNDaeDIOB4cajEXjkf/9K9OZZJSgGTJv1BpMM52qDsCw8HYvvTUrmW5GA0pDoBiotFTWRPmVW+nEiTNF2dVzFq+M8Hc9tzYwPXCRPoskfllNZFqOs1HoqrAGmmPJc/3Fpn+9nIj+XNC8gTRmLItTgGARpHruaNVox9SsokMFUVYIKzk1D1YDOPb1jTOHhZV3V11o0VSxE2m5jjinbmmf8i0WNVQ6VFT3H66e/bAXlzpRLjzv20zCy+I7Sj2ymux6kaZ2k06vnUI+IFIotdmiXE5vocCzKkpG5ZC3ffAO1yemkBvJv5kdF6oCeukVSA4IpuLnAtOBIA=
+    machine-ids.json: AgBTuo9NXudBX1rgt2BAvnrSXNzx20Hu5Dk3KTgayyovGoTVSy4FqILx4T8nAO+Si7qFc8jPI9Z2JKYI7FdVh7UQGPOLXGQ9ucOAWhKM0oCOERE34C7bCBdBxtmdrMtxMx1RoDjI0hpY7TyG4s0Ol/btjsZ4BHaLfACRYLfhFapR1OR3zvgTRcVs2jslUnEGzdPj58Q0Bk6NMHS0ZzFiHcoF4GdkAwAlmSVdxkzYjetmwxPcgifj3fdpA9PZROfuf2Xp54fl2334TpMnaiEVxnNWKDEksczUxWtclGWz/HM5/lZu6rkztECBViwAhbZOnOtMGDqrH+kxk6WAUTbF74NIhrtG1kQTl802vKR+avjaTablUj6/f84v3YA4sF4gwAcN8QvhkUBtmA+Y6O3Eh20xxIUQOAy3ppEAVXgFlSNpubyRNe442A5HnVOqxqz517UmqVWt7ThhBBtRF/fHjjitPHny5DwrfGhKDzBnuE89wSj0orR+/uJwUHV+/rE6oyylNY2raaK7LXamO1ZTuuRPd71agclAQvXwSqan4QaAQdkviRk6UpYwBxX6vOg2zTcCGQtNWjiW7Rk9EWoaBGRLd6WxJwGinuNV2EspATwLQLAQ8Kf8X4sDyDH8xAXpiLXV3wOFOXK8nqQab+2CAvU8/VlH6Mwc8rNle0qQWwI0y12Ku7d2rERG/JhQUeE4ZfHa/qezLmb2S/Th7vZ1FKZQ+8F+DOW7CwWgJOSEM8UFih384IsTO7dZ3MT/HUIIDvQNYxcBDdLadQPugRjatAca3Rz+ST4FceXzazRPWq2AEDio2jfndOTjACvkTZdbPBbKJ8dAgi64uCZHPw8T5+WZb8n8vHk7md6gWL3lad6DdXntjhZ1MpsaE+8JFOlPGshY2JbAZ4+/dFpwEDLI36AEuoGjIhlUO1gJ9IxpYlYwGezLrgT0AovaIrRmar5Bk6uiqZhSQfFXMgXNq/pJ6ohM0IkQ4DtC/nFSHgmSgnWlEN5Z0CIU/lgUfyfzUjSmA9/CM6rw1anKFndnGRc4q+Qdwd08fRRvYf5zHF3a6am/V4wySlFPgUteGqyCwReKshDnHEU/5/kUwvfrqTx/etmCyA3bk4gHocEzGNC6iyL3GWjilywoZSUIOJYycbiY0CwMIuksJ9gyT68dA4tiWwVEt0VfmF6T1b5LTwAV7Mi3l08wGa0exbF6GolUeec3dwOMYH/BCVlYm40PWUKQ7wlYft+9Y0oIiCdBsHAxlK8tPoR7cPuCurZ12lLAErj1rw8720GCIdHEaUYS9UqR1xYdJ+WhqWOh4eZ3r6Y7pWm1vwPlc0hbszJsSivbvzexrHesf56gUeA5fveBSrztVUN2LnSqGWEL5i7/0V22RNCd1YOEfBGoHfekKsd6caH83RnBzca+ihalB22KDrY5yxU75DgJR4RMreAe78OCQZ9bT96+7sswzVNZOQ7NVTB7+J7eN3gLyXqTsEKVXu5a9B1TrAPba6F7VEppDCAzamPPEkuGGPC9DbfoEBl9mQYBenASUXFEYb51KrnrbUOLEGWq4WYnCv9mfedPtoT5bo32NQRb7WYmluHAsVglPfmrF7faWTdip/zWPKtMZ4eF4TUe3VPphuBxvjtTbpMFGHo7bebglbFAf0LaR+x4XIuYT6mvWq1YOnTIc6RgC66IyqRWIcfXZWBIAr9hUNX5mygNidfFLXDk8QhYZtu2YM+GXxNGkwH/E1g7zwg8iWTRugaIZz1qyTa1U7Exok4GBCHqkDhB92dVDcBM/wu/8xXngq9MyxposI1UYaqHeVUCZdsAUm5/iMtUHF3Jh6jTq3kpCzcYYirZbofd16cobPt5FZtGfsRohofjjOsGBa0vbC154lgicb0Kj98Freoz25fgfDMxP1vms/bb3tiayM7dwbO0UHKUNrX1EmshDcz5bESaXkxifOOKtXBc3dkfu9woBZ/gsOqEc2+waCWsE6hLvISpg464Fe1A+RgcymgOXIlimcOmLgra0h7lgO4tf0xkk9DV9pT6BZAYsej6/FTaFMNQqYEe/9+nqiORCCWfzql7lEirAkWLc0AjhFmPvGVZ/zRaDz/WHoSHydKAESZi21IfvRw9NLiO0XGgUTZU1wAmMdZh/jCiO4lL/05z3BSTy+ZVksHQ+o4tAHA=
  template:
    metadata:
      name: sshca-data
--- a/victoria-metrics/alerts.yml
+++ b/victoria-metrics/alerts.yml
@@ -42,6 +42,16 @@ groups:
    expr: >-
      absent(collectd_nut_percent)
    for: 10m
+  - alert: Internet is down
+    expr: >-
+      probe_success{job="blackbox"} == 0
+    for: 5m
+    annotations:
+      severity: critical
+      summary: The connection to the Internet is down.
+      description: >-
+        The Internet connection is down.  Try rebooting the ONT, or call
+        Everfast Fiber.

 - name: Bitwarden
  rules:
@@ -248,6 +258,13 @@ groups:

 - name: Paperless-ngx
  rules:
+  - alert: Paperless-ngx is down
+    expr: >-
+      up{job="paperless-ngx"} == 0 or absent(up{job="paperless-ngx"})
+    annotations:
+      summary: Paperless-ngx is down
+      description: >-
+        Paperless-ngx is offline.
  - alert: Celery tasks failed
    expr: >-
      max_over_time(
@@ -279,3 +296,15 @@ groups:
        Paperless-ngx uses a scheduled Celery task to periodically poll email
        mailboxes for new messages.  If this task does not start, new email
        messages will not be downloaded and imported into the document library.
+
+- name: Firefly III
+  rules:
+  - alert: Firefly III is down
+    expr: >-
+      probe_success{job="firefly-iii"} != 1
+
+- name: phpipam
+  rules:
+  - alert: phpipam is down
+    expr: >-
+      probe_success{job="phpipam"} != 1
--- a/victoria-metrics/scrape.yml
+++ b/victoria-metrics/scrape.yml
@@ -242,6 +242,22 @@ scrape_configs:
  - source_labels: [__address__]
    target_label: instance

+- job_name: victoria-logs
+  scheme: https
+  tls_config:
+    ca_file: /run/dch-ca/dch-root-ca.crt
+  dns_sd_configs:
+  - names:
+    - logs.pyrocufflink.blue
+    type: A
+    port: 443
+  relabel_configs:
+  - source_labels: [__meta_dns_name, __meta_dns_srv_record_port]
+    separator: ':'
+    target_label: __address__
+  - source_labels: [__address__]
+    target_label: instance
+
 - job_name: promtail
  static_configs:
  - targets:
@@ -473,3 +489,36 @@ scrape_configs:
    target_label: __address__
  - source_labels: [__address__]
    target_label: instance
+
+- job_name: firefly-iii
+  metrics_path: /probe
+  params:
+    module:
+    - http
+  static_configs:
+  - targets:
+    - https://firefly.pyrocufflink.blue/
+    - https://receipts.pyrocufflink.blue/
+  relabel_configs:
+  - source_labels: [__address__]
+    target_label: __param_target
+  - source_labels: [__param_target]
+    target_label: instance
+  - target_label: __address__
+    replacement: blackbox-exporter:9115
+
+- job_name: phpipam
+  metrics_path: /probe
+  params:
+    module:
+    - http
+  static_configs:
+  - targets:
+    - phpipam.pyrocufflink.blue
+  relabel_configs:
+  - source_labels: [__address__]
+    target_label: __param_target
+  - source_labels: [__param_target]
+    target_label: instance
+  - target_label: __address__
+    replacement: blackbox-exporter:9115
--- a/xactmon/xactmon.yaml
+++ b/xactmon/xactmon.yaml
@@ -51,6 +51,8 @@ spec:
          subPath: tmp
      imagePullSecrets:
      - name: imagepull-gitea
+      nodeSelector:
+        kubernetes.io/arch: amd64
      securityContext:
        runAsUser: 251
        runAsGroup: 251
@@ -132,6 +134,8 @@ spec:
          subPath: tmp
      imagePullSecrets:
      - name: imagepull-gitea
+      nodeSelector:
+        kubernetes.io/arch: amd64
      securityContext:
        runAsUser: 251
        runAsGroup: 251
@@ -214,6 +218,8 @@ spec:
          subPath: tmp
      imagePullSecrets:
      - name: imagepull-gitea
+      nodeSelector:
+        kubernetes.io/arch: amd64
      securityContext:
        runAsUser: 251
        runAsGroup: 251
@@ -296,6 +302,8 @@ spec:
          subPath: tmp
      imagePullSecrets:
      - name: imagepull-gitea
+      nodeSelector:
+        kubernetes.io/arch: amd64
      securityContext:
        runAsUser: 251
        runAsGroup: 251
Author	SHA1	Message	Date
Dustin C. Hatch	b642d5374a	wip: jenkins: Add iSCSI PV for airplaypi builds	2025-08-17 22:21:49 -05:00
Dustin C. Hatch	ed44ecb34e	wip: jenkins: buildroot iscsi pvc	2025-08-03 11:30:28 -05:00
Dustin C. Hatch	1ec974fa2d	v-m/alerts: Add alert for Internet down	2025-08-03 11:29:41 -05:00
Dustin C. Hatch	024eaf241f	Merge remote-tracking branch 'refs/remotes/origin/master'	2025-07-29 21:56:18 -05:00
Dustin C. Hatch	a6618cac11	h-a: Update taints for Zigbee/Zwave controllers With the introduction of the two new Raspberry Pi nodes that I intend to be used for anything that supports running on aarch64, I'm eliminating the `du5t1n.me/machine=raspberrypi` taint. It no longer makes sense, as the only node that has it is the Zigbee/ZWave controller. Having dedicated taints for those roles is much more clear.	2025-07-29 21:39:21 -05:00
Dustin C. Hatch	8b492d059d	xactmon: Pin to x86_64 nodes There are no ARM builds of the `xactmon` components.	2025-07-29 21:38:06 -05:00
Dustin C. Hatch	812b09626f	cert-manager: Drop chmod777.sh certificate This site now obtains its own certificate using Apache _mod_md_.	2025-07-28 18:59:06 -05:00
Dustin C. Hatch	32666aa628	h-a: Schedule Piper, Whisper, Mosquitto with HA Using pod affinity rules, we can schedule the ancillary processes for Home Assistant to run on the same node as the main server.	2025-07-27 18:39:55 -05:00
Dustin C. Hatch	7b440c44ec	h-a: Prefer running on a Raspberry Pi Now that we have Raspberry Pi CM4 worker nodes, let's configure Home Assistant to run on one, since it's pretty much designed to.	2025-07-27 18:35:07 -05:00
Dustin C. Hatch	6d2aa9c391	20125: Set log level Only errors are logged by default, which is less than helpful when troubleshooting a running but apparently misbehaving application...	2025-07-27 18:20:27 -05:00
Dustin C. Hatch	b989a7898e	20125: Pin to amd64 nodes There is no ARM build of the 20125 `status-server`, so we have to pin the pod to amd64 nodes to prevent it from being scheduled on a Raspberry Pi.	2025-07-27 18:19:58 -05:00
Dustin C. Hatch	921fadc44b	20125: Fix website URL anchors As it turns out, it's not possible to reuse a YAML anchor. At least in Rust's `serde_yaml`, only the final definition is used. All references, even those that appear before the final definition, use the same definition. Thus, each application that refers to its own URL in its match criteria needs a unique anchor.	2025-07-27 18:16:30 -05:00
Dustin C. Hatch	4dc21e6179	sshca: Add machine IDs for CM4 cluster nodes * _ctrl-2ed83d.k8s.pyrocufflink.black_ * _node-6a3f8.k8s.pyrocufflink.black_ * _node-6ed191.k8s.pyrocufflink.black_	2025-07-27 17:42:43 -05:00
Dustin C. Hatch	972831d15f	20125: Fix alert selector for Jellyfin Jellyfin is not scraped by the Blackbox exporter, but rather exposes its own metrics.	2025-07-27 17:40:54 -05:00
Dustin C. Hatch	38ee60e099	v-m: Add alerts for Firefly, Paperless, phpipam _Firefly III_ and _phpipam_ don't export any Prometheus metrics, so we have to scrape them via the Blackbox Exporter. Paperless-ngx only exposes metrics via Flower, but since it runs in the same container as the main application, we can assume that if the former is unavailable, the latter is as well.	2025-07-27 17:39:28 -05:00
Dustin C. Hatch	fac4b92b71	cert-manager: Drop hatch.chat certificate The _hatch.chat_ Matrix server has been gone for quite some time.	2025-07-23 11:59:28 -05:00
Dustin C. Hatch	81f8c58816	cert-manager: Drop tabitha.biz certificate This site now obtains its own certificate using Apache _mod_md_.	2025-07-23 11:41:09 -05:00
Dustin C. Hatch	592ff3ce9e	cert-manager: Drop apps.d.x certificate This site now obtains its own certificate using Apache _mod_md_.	2025-07-23 11:29:34 -05:00
Dustin C. Hatch	36015084c8	ansible: Allow host-provisioner to read root CA The Kubernetes root CA certificate is stored in a ConfigMap named `kube-root-ca.crt` in every namespace. The _host-provisioner_ needs to be able to read this ConfigMap in order to prepare control plane nodes, as it is used by HAProxy to check the health of the API servers running on each node.	2025-07-23 10:50:24 -05:00
Dustin C. Hatch	484c17c1d5	authelia: Add address, phone scopes for Jenkins Not sure why suddenly these need to be granted, but without them, I cannot log in to Jenkins.	2025-07-22 15:26:29 -05:00
Dustin C. Hatch	e845e66262	restic: pin to 0.18.0 Let's keep the version of `restic` used by the prune job in sync with the latest version in Fedora.	2025-07-21 18:58:57 -05:00
Dustin C. Hatch	717f9244e7	kubelet-csr-approver: Initial commit The [kubelet-csr-approver][0] is a controller that automatically approves CSRs for Kublets that match certain criteria. I've had it deployed in the cluster for a while, but apparently never committed the resources. These manifest files are taken from the [k8s deployment example][1] in the upstream repository. [0]: https://github.com/postfinance/kubelet-csr-approver [1]: https://github.com/postfinance/kubelet-csr-approver/tree/v1.2.10/deploy/k8s	2025-07-21 18:49:44 -05:00
Dustin C. Hatch	da2b1e60cd	autoscaler: Set imagePullPolicy: IfNotPresent We don't want to pull public container images that already exist. This creates prevents pods from starting if there is any connectivity issue with the upstream registry.	2025-07-21 17:17:16 -05:00
Dustin C. Hatch	810134e9bc	authelia: Set imagePullPolicy: IfNotPresent We don't want to pull public container images that already exist. This creates prevents pods from starting if there is any connectivity issue with the upstream registry.	2025-07-21 17:16:32 -05:00
Dustin C. Hatch	7fd613ccaf	ara: Set imagePullPolicy: IfNotPresent We don't want to pull public container images that already exist. This creates prevents pods from starting if there is any connectivity issue with the upstream registry.	2025-07-21 17:14:06 -05:00
Dustin C. Hatch	68c7e0d6cc	argocd: Set imagePullPolicy: IfNotPresent We don't want to pull public container images that already exist. This creates prevents pods from starting if there is any connectivity issue with the upstream registry.	2025-07-21 15:07:01 -05:00
Dustin C. Hatch	5da80c6a55	ntfy: Set imagePullPolicy: IfNotPresent We don't want to pull public container images that already exist. This creates prevents pods from starting if there is any connectivity issue with the upstream registry.	2025-07-21 15:07:01 -05:00
Dustin C. Hatch	32132842be	firefly-iii: Set imagePullPolicy: IfNotPresent We don't want to pull public container images that already exist. This creates prevents pods from starting if there is any connectivity issue with the upstream registry.	2025-07-21 15:07:01 -05:00
Dustin C. Hatch	0822afe0b3	kitchen: Round weather metrics Home Assistant has started sending the full sensor values for weather metrics to Prometheus, even though their precision is way beyond their accuracy. We don't need to see 4+ decimal points for these on the Kitchen display, so let's round the values when we query.	2025-07-21 14:40:35 -05:00
Dustin C. Hatch	e51878fa92	ansible: Allow h-p to update scrape-collectd CM The `scrape-collectd` ConfigMap in the `default` namespace is used by Victoria Metrics to identif the hosts from which it should scrape collectd metrics. When deploying new machines that are _not_ part of the Kubernetes cluster, we need to explicitly add them to this list. The _host-provisioner_ can do this with an Ansible task, but it needs the appropriate permissions to do so.	2025-07-21 12:24:00 -05:00
Dustin C. Hatch	dbbe23aaa5	cert-manager: Add role for Jenkins to access certs Ansible playbook running as Jenkins jobs need to be able to access the Secret resources containing certificates issued by _cert-manager_ in order to install them on managed nodes. Although not all jobs do this yet, eventually, the _cert-exporter_ will no longer be necessary, as the _certs.git_ repository will not be used anymore.	2025-07-21 12:24:00 -05:00
Dustin C. Hatch	d48dabca5b	Merge remote-tracking branch 'refs/remotes/origin/master'	2025-07-21 12:02:44 -05:00
Dustin C. Hatch	16dec1cdec	ssh-host-keys: Do not specify a namespace We don't want to hard-code a namespace for the `ssh-known-hosts` ConfigMap because that makes it less useful for other projects besides Jenkins. Instead, we omit the namespace specification and allow consumers to specify their own. The _jenkins_ project doesn't have a default namespace, since it specifies resources in the `jenkins` and `jenkins-jobs` namespaces, we need to create a sub-project to set the namespace for the `ssh-known-hosts` ConfigMap.	2025-07-21 11:47:39 -05:00
Dustin	959959155c	Merge pull request 'home-assistant: Update to 2025.7.1' (#69 ) from updatebot/home-assistant into master Reviewed-on: #69	2025-07-16 21:55:57 +00:00
Dustin	b36c132364	Merge pull request 'ntfy: Update to 2.13.0' (#72 ) from updatebot/ntfy into master Reviewed-on: #72	2025-07-16 21:49:29 +00:00
Dustin	dc31ae1cae	Merge pull request 'tika: Update to 3.2.1.0' (#71 ) from updatebot/paperless-ngx into master Reviewed-on: #71	2025-07-16 21:45:03 +00:00
bot	05048cbaa1	ntfy: Update to 2.13.0	2025-07-12 11:32:13 +00:00
bot	434d420e28	tika: Update to 3.2.1.0	2025-07-12 11:32:11 +00:00
Dustin C. Hatch	093e909475	v-m/scrape: Scrape Victoria Logs	2025-07-06 15:20:16 -05:00