From ca02dfec62121da387481f6fd51d13bb0c51ab4d Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 22 Jan 2024 11:12:19 -0600 Subject: [PATCH 1/8] v-m: Add host labels to collectd-virt metrics The *virt* plugin for *collectd* sets `instance` to the name of the libvirt domain the metric refers to. This makes it so there is no label identifying which host the VM is running on. Thus, if we want to classify metrics by VM host, we need to add that label explicitly. Since the `__address__` label is not available during metric relabeling, we need to store it in a temporary label, which gets dropped at the end of the relabeling phase. We copy the value of that label into a new label, but only for metrics that match the desired metric name. --- victoria-metrics/scrape.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/victoria-metrics/scrape.yml b/victoria-metrics/scrape.yml index 21dde17..cd294aa 100644 --- a/victoria-metrics/scrape.yml +++ b/victoria-metrics/scrape.yml @@ -80,9 +80,21 @@ scrape_configs: - files: - /scrape/collectd/scrape-collectd.yml relabel_configs: + - source_labels: + - __address__ + target_label: host__tmp - source_labels: [__address__] target_label: __address__ replacement: '$1:9103' + metric_relabel_configs: + - source_labels: + - __name__ + - host__tmp + separator: ; + regex: collectd_virt.*;(.*) + target_label: host + - action: labeldrop + regex: host__tmp - job_name: sambadc scrape_interval: 1m From 54e7a25f93a52cc90d41b7922864e1e0dcd5379d Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 22 Jan 2024 16:43:46 -0600 Subject: [PATCH 2/8] v-m: vmstorage: Remove startup/ready probes Kubernetes will not start additional Pods in a StatefulSet until the existing ones are Ready. This means that if there is a problem bringing up, e.g. `vmstorage-0`, it will never start `vmstorage-1` or `vmstorage-2`. Since this pretty much defeats the purpose of having a multi-node `vmstorage` cluster, we have to remove the readiness probe, so the Pods will be Ready as soon as they start. If there is a problem with one of them, it will matter less, as the others can still run. --- victoria-metrics/vmstorage.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/victoria-metrics/vmstorage.yaml b/victoria-metrics/vmstorage.yaml index 50c2d4a..f9b094f 100644 --- a/victoria-metrics/vmstorage.yaml +++ b/victoria-metrics/vmstorage.yaml @@ -50,17 +50,6 @@ spec: name: vmselect - containerPort: 8482 name: http - readinessProbe: &probe - httpGet: - port: http - path: /health - periodSeconds: 60 - startupProbe: - <<: *probe - periodSeconds: 1 - successThreshold: 1 - failureThreshold: 30 - timeoutSeconds: 1 securityContext: runAsNonRoot: true readOnlyRootFilesystem: true From 9b441738d4a9dd3c17b33d01f3193094374c3f0c Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 22 Jan 2024 16:55:03 -0600 Subject: [PATCH 3/8] dch-webhooks: Disable HTTPS redirect The [Generic Event][0] plugin for Jenkins does not support HTTPS webhooks, only plain HTTP. [0]: https://plugins.jenkins.io/generic-event/ --- dch-webhooks/ingress.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dch-webhooks/ingress.yaml b/dch-webhooks/ingress.yaml index e14109b..e5d37de 100644 --- a/dch-webhooks/ingress.yaml +++ b/dch-webhooks/ingress.yaml @@ -2,6 +2,8 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" labels: app.kubernetes.io/name: dch-webhooks app.kubernetes.io/component: dch-webhooks From 990204b2cfee549bfda1231cdd4f06b9de1d4c29 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 22 Jan 2024 16:57:38 -0600 Subject: [PATCH 4/8] kitchen: Use Certifi TLS CA bundle for OpenSSL The MQTT client needs a trusted root CA bundle, which is not available in the container image used by the *kitchen* server (it's based on *pythonctnr* which literally *only* includes Python). Fortunately, as it uses OpenSSL under the hood, we can configure it to use the bundle included with the *certifi* Python package via an environment variable. --- kitchen/kitchen.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kitchen/kitchen.yaml b/kitchen/kitchen.yaml index 6e30e59..d238f5f 100644 --- a/kitchen/kitchen.yaml +++ b/kitchen/kitchen.yaml @@ -38,6 +38,8 @@ spec: env: - name: TZ value: America/Chicago + - name: SSL_CERT_FILE + value: /usr/lib/python3.10/site-packages/certifi/cacert.pem imagePullPolicy: Always ports: - containerPort: 8000 From a7450a8af2f0706a4f480b08c55bb47fb09032b6 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 22 Jan 2024 16:59:53 -0600 Subject: [PATCH 5/8] kitchen: Fix Jenkins deployment role Since Jenkins jobs run in Kubernetes now, they can authenticate to the Kubernetes API using a ServiceAccount and do not need a dedicated User. --- kitchen/kitchen.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kitchen/kitchen.yaml b/kitchen/kitchen.yaml index d238f5f..fc3c9bf 100644 --- a/kitchen/kitchen.yaml +++ b/kitchen/kitchen.yaml @@ -131,6 +131,6 @@ roleRef: kind: Role name: jenkins subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: jenkins +- kind: ServiceAccount + name: default + namespace: jenkins-jobs From 3d55d7aafa111e6a24a70bdb916fb47eee5f1eb0 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 22 Jan 2024 17:01:13 -0600 Subject: [PATCH 6/8] keyserv: Add age key for NUT/dustin This key is used to encrypt the password for the NUT user *dustin*, which I use to manually control the UPS. --- keyserv/.gitignore | 1 + ...9affempnzzh9q50rhc78guwljczagc0aa3z9zvrle5dhcg4qh9hanr | 8 ++++++++ ...w7xct9f50yurw7h5flck8jycv0t4m4qj72frep3z09344pus9x4nkc | 8 ++++++++ keyserv/key-map.yml | 8 +++++++- keyserv/kustomization.yaml | 1 + 5 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 keyserv/.gitignore create mode 100644 keyserv/age-keys/age18zz39affempnzzh9q50rhc78guwljczagc0aa3z9zvrle5dhcg4qh9hanr create mode 100644 keyserv/age-keys/age1y5cdw7xct9f50yurw7h5flck8jycv0t4m4qj72frep3z09344pus9x4nkc diff --git a/keyserv/.gitignore b/keyserv/.gitignore new file mode 100644 index 0000000..0d60524 --- /dev/null +++ b/keyserv/.gitignore @@ -0,0 +1 @@ +master.key diff --git a/keyserv/age-keys/age18zz39affempnzzh9q50rhc78guwljczagc0aa3z9zvrle5dhcg4qh9hanr b/keyserv/age-keys/age18zz39affempnzzh9q50rhc78guwljczagc0aa3z9zvrle5dhcg4qh9hanr new file mode 100644 index 0000000..04d8401 --- /dev/null +++ b/keyserv/age-keys/age18zz39affempnzzh9q50rhc78guwljczagc0aa3z9zvrle5dhcg4qh9hanr @@ -0,0 +1,8 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkWVoydG5KRGpwSlMrTEhn +bmI4VFJabFVlclhwSThLRmlFbEd5L2tvQ0E4CnZkUXRXWVNERXQwbC84ZHRwWS9I +ODQvM283RUZsakwxOXZnOXMyV0tsZ00KLS0tIFlVMmMwNFR4OWZ5c3UxQ0QvcHpW +ZEdzV2l6NERCdlhVS2M4alFqMWhBdzAKClTysPEo5cDC8ZRplM0VW3FDvvdRz+tv +/N7n0dspfmAl+kj5LfTg3Thb6a1kOW7j5AuST2uPrRVocWMCoVH9cls3eItxqy8X +kPwnpSHD7N2v6Rd4t7qDHp3MH7CedK6WWJk+jYc0Gpm4bso= +-----END AGE ENCRYPTED FILE----- diff --git a/keyserv/age-keys/age1y5cdw7xct9f50yurw7h5flck8jycv0t4m4qj72frep3z09344pus9x4nkc b/keyserv/age-keys/age1y5cdw7xct9f50yurw7h5flck8jycv0t4m4qj72frep3z09344pus9x4nkc new file mode 100644 index 0000000..ac72563 --- /dev/null +++ b/keyserv/age-keys/age1y5cdw7xct9f50yurw7h5flck8jycv0t4m4qj72frep3z09344pus9x4nkc @@ -0,0 +1,8 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTTks1S2V0U2twVy93Z2lV +ZlRyVDk2OTdBemIzU3JsVE1JUlZSTXZhTG53CnhqS0ZyVVJxT1NLSC9WR1dQYWh6 +SUd2M08wQi9TU2o3YkcxMW9JaUlHeGMKLS0tIEs4K0JvaVBvTG0wSDgyemZJV0lX +VXFrbmVDenVBRy9aZmFLUEJ0ZmRBY3MKpR8uXoUp6R5BTFLBSdWlpd8YNRpxdn9J +DcAIH9KecbAyaHVjJspIvcQQVpz6Cvh5O34spY4U9Gg4dCOsGD+qB88vmR+B/rsy +jfTFe+Us6G87fUZ6NvdFJ8K3HsVXvcTFMNijMHw2SWlyJ3I= +-----END AGE ENCRYPTED FILE----- diff --git a/keyserv/key-map.yml b/keyserv/key-map.yml index 66535d4..fe330cc 100644 --- a/keyserv/key-map.yml +++ b/keyserv/key-map.yml @@ -1,12 +1,17 @@ dustin@hatch.name: +- age1668cmw7jeyfawpdp7c6c79hdqdmvzjrkuszz4c96sfugkyjsr39qv4vsg7 - age197zq0l27nwxj74d4pmpat6kqqth235mdc0ggmfm3006v0fy7advsg9ljts - age1c6swn9tm0502jd3e0yszfd4qd7lgx2nd9uk0hruuckhx7zpn3utqhau7mz +- age1dcyvkqde4j43gz6pzk6u8g3ph85tj3qr0tucr9lkcy4sgyqshe8qzq7d20 - age1ez6hv5frke4k4esk4p3nyf7y4g5mjq953t8ctk45qxnpreeerdpsrqu2dd - age1fc96yyd7a7l3uc4jr8sk06h8al607gjxd89q435jlp6nsmrhqflq5dkhtq - age1hl8dfgh938092h32zuex7xnfmqer3peg5gl6d892aarsw0s6nptq5tysu9 - age1j63kzwldegazaaj4rm2ydzlm4wlh6z4cgm4s7g6pzysskh04duhslyc5yy +- age1kfqgu0ug40uwrsqx94azeflg58wp4ckx3xsm5l2y6zvw95zqygfsy8x69t +- age1xfmmwhutwr4cml4dlj6rq6r9mgjs3fake0q4wuly5z9r9mqgk4nsk53d5j +- age1y3hea7a4rpeyjhcrcg29lsfzg9guwqeqx6m6q6szt5wuc8guy3hsl6t33e - age1y4prxtunmkx0kwrtl5qkxvj0gzl8kuyp9seyptgy2rlvrqyysegq32srfn -- age1668cmw7jeyfawpdp7c6c79hdqdmvzjrkuszz4c96sfugkyjsr39qv4vsg7 +- age1y5cdw7xct9f50yurw7h5flck8jycv0t4m4qj72frep3z09344pus9x4nkc burp1.pyrocufflink.blue: - age1xfmmwhutwr4cml4dlj6rq6r9mgjs3fake0q4wuly5z9r9mqgk4nsk53d5j @@ -23,6 +28,7 @@ nut0.pyrocufflink.blue: - age1y3hea7a4rpeyjhcrcg29lsfzg9guwqeqx6m6q6szt5wuc8guy3hsl6t33e - age1kfqgu0ug40uwrsqx94azeflg58wp4ckx3xsm5l2y6zvw95zqygfsy8x69t - age1xfmmwhutwr4cml4dlj6rq6r9mgjs3fake0q4wuly5z9r9mqgk4nsk53d5j +- age1y5cdw7xct9f50yurw7h5flck8jycv0t4m4qj72frep3z09344pus9x4nkc nvr1.pyrocufflink.blue: - age1668cmw7jeyfawpdp7c6c79hdqdmvzjrkuszz4c96sfugkyjsr39qv4vsg7 diff --git a/keyserv/kustomization.yaml b/keyserv/kustomization.yaml index da06303..ef9cf6e 100644 --- a/keyserv/kustomization.yaml +++ b/keyserv/kustomization.yaml @@ -43,6 +43,7 @@ secretGenerator: - age-keys/age1y3hea7a4rpeyjhcrcg29lsfzg9guwqeqx6m6q6szt5wuc8guy3hsl6t33e - age-keys/age1kfqgu0ug40uwrsqx94azeflg58wp4ckx3xsm5l2y6zvw95zqygfsy8x69t - age-keys/age1xfmmwhutwr4cml4dlj6rq6r9mgjs3fake0q4wuly5z9r9mqgk4nsk53d5j + - age-keys/age1y5cdw7xct9f50yurw7h5flck8jycv0t4m4qj72frep3z09344pus9x4nkc options: disableNameSuffixHash: true labels: From 2f9d8ad6180ad5e80a4c07709119ea545fa57f9f Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 22 Jan 2024 17:02:00 -0600 Subject: [PATCH 7/8] jenkins: Add CA key to ssh_known_hosts Since (almost) all managed hosts have SSH certificates signed by SSHCA now, the need to maintain a pseudo-dynamic SSH key list is winding down. If we include the SSH CA key in the global known hosts file, and explicitly list the couple of hosts that do not have a certificate, we can let Ansible use that instead of fetching the host keys on each run. --- jenkins/ssh_known_hosts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins/ssh_known_hosts b/jenkins/ssh_known_hosts index 1f0bc20..bac68f7 100644 --- a/jenkins/ssh_known_hosts +++ b/jenkins/ssh_known_hosts @@ -1,3 +1,4 @@ +@cert-authority *.pyrocufflink.blue ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII24CZGosLMTny0a2eDB6KOG47FhlwVkTEFQNAYzKV0t files.pyrocufflink.blue ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH+S6aTqXJ15DV3NczbPXVQKXxbvMVtaHToShsrhxps1GGWcJU/pbZtpAQcN4OGth7DQ1Q/1RvrFS+Fd/5U4wv4= files.pyrocufflink.blue ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzOkLdjAJDPyja2o4+Km52VNM4t7jeYTyMVYl4gtudq files.pyrocufflink.blue ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDbgN04bblL95EStM+wpGF1asvEOL6vmH/oNTIBRd0HbTz8jRa3CMOGWWG7/xGIRjrXglAGURGZ/EOqkyGIsciVtC53lwLuyZT18sqHrmp8S5uq/rNaY3rSVfc7kW/fXsNksjtwnQ/sNtawSZ6UFv+p/X47qOGv0XPAwAzoXDwDpQ27wOz1YnbBa+5itThLh6QvxgM1DKnb78uZ1TBpaCCdtL2iH1IVo3FLmah9bNWvUU1QECKyOUDw3IiwIS6owtHIrpdCiZTlPSJhBLPvv7P/L9V0bTfREP+MMDBT1hhj2NUgmDxC4sDd8k1Qy/qxeyU/FA+7dn7K8YVIEe9rNbs/ @@ -7,3 +8,5 @@ git.pyrocufflink.blue ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDF1tB0KeyMStHKbnumin git.pyrocufflink.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJPLXOT4j+jYAIGfuGbtG8ea3oBZwtvOEYNzUHpsQBF9VO9E9nTQBswSRzc+otPzZhr5lJ+BlGo439hHGkbOIo8= git.pyrocufflink.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHEF/IXycjT/sSIpFLRDEVZUu95QA3i7d5LZvB/RncHN git.pyrocufflink.net ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDF1tB0KeyMStHKbnuminTQ2xwjjxyES/DBSlZZH0c7F25Lu6TfvxEEs/7YUtZKAnwFGLfuqW5gaS3eWV1eA3w7WtEGYoBAD6VFZUjN0vfmhNYWC85DMcY+gqkOkBQy1/SvyYSPHrtkgTJRPMG/MdWGaYEN+w/znr1HETPnj4qm7bFC8yxXKU2PXzKlUeY+ynY3hhlzAVwG4Ay7hgH+nO071eoAQtYq9zBdYTifXxoTzatLgU8ib3EYuC3/LDNXkFhYhxDhfp9iwPCNZRi49AccMlL323/Vp8x/Jy/iY5A60Fk8cIYtneEihRkIpzpyTudEK0MXyJ+FJ0vmjiEX5ZY9 +mtrcs0.pyrocufflink.blue ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIFklfgYwVlea/FbFNguKEY2hMXw9iOneNveLVws8dd9 +serial0.pyrocufflink.blue ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIABidV03uxUtikscJfA3qZ+mgXW9KP2QWJBLhlDOleHQ From 0e20952740ec699b03a32c7c4265a5798173b298 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 22 Jan 2024 17:16:46 -0600 Subject: [PATCH 8/8] xactfetch: Sync vault before running The Bitwarden vault needs to be synced before *xactfetch* runs, in case the password for a bank website has changed since it was first fetched. --- xactfetch/xactfetch.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/xactfetch/xactfetch.yaml b/xactfetch/xactfetch.yaml index 8511dbb..d641c94 100644 --- a/xactfetch/xactfetch.yaml +++ b/xactfetch/xactfetch.yaml @@ -33,6 +33,17 @@ spec: readOnlyRootFilesystem: true runAsGroup: 999 runAsUser: 999 + - name: sync + image: git.pyrocufflink.net/packages/xactfetch + command: + - rbw + - sync + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/xactfetch + name: xactfetch-data + subPath: data containers: - name: xactfetch image: git.pyrocufflink.net/packages/xactfetch