From e0c633c21ed045b4cb62a44b5a29696d264eb9d8 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sun, 17 Nov 2024 18:43:00 +0000 Subject: [PATCH 01/13] v-m: scrape: Fix Nextcloud URL Nextcloud uses a _client-side_ (Javascript) redirect to navigate the browser to its `index.php`. The page it serves with this redirect is static and will often load successfully, even if there is a problem with the application. This causes the Blackbox exporter to record the site as "up," even when it it definitely is not. To avoid this, we can scrape the `index.php` page explicitly, ensuring that the application is loaded. --- 20125/config.yml | 2 +- victoria-metrics/scrape.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/20125/config.yml b/20125/config.yml index 7a06670..64b9dc3 100644 --- a/20125/config.yml +++ b/20125/config.yml @@ -31,7 +31,7 @@ applications: - instance: homeassistant.pyrocufflink.blue - name: Nextcloud - url: &url https://nextcloud.pyrocufflink.net/ + url: &url https://nextcloud.pyrocufflink.net/index.php icon: url: icons/nextcloud.png alerts: diff --git a/victoria-metrics/scrape.yml b/victoria-metrics/scrape.yml index 2591f42..4c8b8ff 100644 --- a/victoria-metrics/scrape.yml +++ b/victoria-metrics/scrape.yml @@ -57,7 +57,7 @@ scrape_configs: - http://pyrocufflink.net/ - http://ebonfire.com/ - http://chmod777.sh/ - - https://nextcloud.pyrocufflink.net/ + - https://nextcloud.pyrocufflink.net/index.php - https://bitwarden.pyrocufflink.blue/ - https://git.pyrocufflink.blue/ - https://tabitha.biz/ From 72d3f222c5d171b646c4bbfcaae165d3667b69b0 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Tue, 26 Nov 2024 03:35:21 +0000 Subject: [PATCH 02/13] jenkins: Trust SSHCA for pyrocufflink.black The new machines have names in the _pyrocufflink.black_ zone. We need to trust the SSHCA certificate to sign keys for these names in order to connect to them and manage them with Ansible. --- jenkins/ssh_known_hosts | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/ssh_known_hosts b/jenkins/ssh_known_hosts index bac68f7..2d972f3 100644 --- a/jenkins/ssh_known_hosts +++ b/jenkins/ssh_known_hosts @@ -1,4 +1,5 @@ @cert-authority *.pyrocufflink.blue ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII24CZGosLMTny0a2eDB6KOG47FhlwVkTEFQNAYzKV0t +@cert-authority *.pyrocufflink.black ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII24CZGosLMTny0a2eDB6KOG47FhlwVkTEFQNAYzKV0t files.pyrocufflink.blue ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH+S6aTqXJ15DV3NczbPXVQKXxbvMVtaHToShsrhxps1GGWcJU/pbZtpAQcN4OGth7DQ1Q/1RvrFS+Fd/5U4wv4= files.pyrocufflink.blue ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzOkLdjAJDPyja2o4+Km52VNM4t7jeYTyMVYl4gtudq files.pyrocufflink.blue ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDbgN04bblL95EStM+wpGF1asvEOL6vmH/oNTIBRd0HbTz8jRa3CMOGWWG7/xGIRjrXglAGURGZ/EOqkyGIsciVtC53lwLuyZT18sqHrmp8S5uq/rNaY3rSVfc7kW/fXsNksjtwnQ/sNtawSZ6UFv+p/X47qOGv0XPAwAzoXDwDpQ27wOz1YnbBa+5itThLh6QvxgM1DKnb78uZ1TBpaCCdtL2iH1IVo3FLmah9bNWvUU1QECKyOUDw3IiwIS6owtHIrpdCiZTlPSJhBLPvv7P/L9V0bTfREP+MMDBT1hhj2NUgmDxC4sDd8k1Qy/qxeyU/FA+7dn7K8YVIEe9rNbs/ From 107852ad54e628894f8f81fa9e79a6562356fa94 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Wed, 27 Nov 2024 22:03:40 -0600 Subject: [PATCH 03/13] home-assistant: Eable auto-login for desk panel Home Assistant supports unauthenticated access for certain clients using its _trusted_network_ auth provider. With this configuration, we allow the desk panel to automatically sign in as the _kiosk_ user, but all other clients must authenticate normally. --- home-assistant/configuration.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/home-assistant/configuration.yaml b/home-assistant/configuration.yaml index 14870c9..e1cdd75 100644 --- a/home-assistant/configuration.yaml +++ b/home-assistant/configuration.yaml @@ -38,6 +38,15 @@ recorder: commit_interval: 0 homeassistant: + auth_providers: + - type: homeassistant + - type: trusted_networks + trusted_networks: + - 172.31.1.81/32 + trusted_users: + 172.31.1.81: + - 03a8b3528f1145ab908e20ed5687d893 + allow_bypass_login: true whitelist_external_dirs: - /config - /tmp From b269fa581229a592e70ecbf95d0753e112048c1a Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 2 Dec 2024 23:06:30 +0000 Subject: [PATCH 04/13] home-assistant: Add service to shut down desk panel Home Assistant can now SSH into the desk panel and shut it down. --- home-assistant/kustomization.yaml | 1 + home-assistant/shell-command.yaml | 3 +++ home-assistant/shutdown-kiosk.sh | 4 ++++ home-assistant/ssh_known_hosts | 1 + 4 files changed, 9 insertions(+) create mode 100644 home-assistant/shutdown-kiosk.sh diff --git a/home-assistant/kustomization.yaml b/home-assistant/kustomization.yaml index 495a452..d60a352 100644 --- a/home-assistant/kustomization.yaml +++ b/home-assistant/kustomization.yaml @@ -30,6 +30,7 @@ configMapGenerator: - restart-diddy-mopidy.sh - restart-kitchen-mqttmarionette.sh - shell-command.yaml + - shutdown-kiosk.sh - ssh_known_hosts - rest-command.yaml options: diff --git a/home-assistant/shell-command.yaml b/home-assistant/shell-command.yaml index 3d5ffbb..e2a52e1 100644 --- a/home-assistant/shell-command.yaml +++ b/home-assistant/shell-command.yaml @@ -6,3 +6,6 @@ restart_diddy_mopidy: >- restart_kitchen_mqttmarionette: >- sh /run/config/restart-kitchen-mqttmarionette.sh + +shutdown_kiosk: >- + sh /run/config/shutdown-kiosk.sh diff --git a/home-assistant/shutdown-kiosk.sh b/home-assistant/shutdown-kiosk.sh new file mode 100644 index 0000000..b4179d8 --- /dev/null +++ b/home-assistant/shutdown-kiosk.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +set -e +ssh -i /run/secrets/home-assistant/sshkey.pem -oUserKnownHostsFile=/run/config/ssh_known_hosts -oBatchMode=yes kiosk@deskpanel.pyrocufflink.red doas systemctl poweroff diff --git a/home-assistant/ssh_known_hosts b/home-assistant/ssh_known_hosts index 9eaa503..5ce5ecf 100644 --- a/home-assistant/ssh_known_hosts +++ b/home-assistant/ssh_known_hosts @@ -1,2 +1,3 @@ diddy.pyrocufflink.red ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILx6gRqlVnvdqTIJTH16NBLJ4ORfTsBaUIEpt5ZMkkNW kitchen.pyrocufflink.red ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBLzMLOlFXPiovBwYLmXCVV8Md/xR36zwPj6egT9V3O7 +deskpanel.pyrocufflink.red ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcvO0jsZ8U2mw/HHs0BHbbEI48W0fxti8f5DuNyFS2L From 1768778b44f26da93c1ac8b0d9701d65a72ad96e Mon Sep 17 00:00:00 2001 From: bot Date: Sat, 21 Dec 2024 12:32:03 +0000 Subject: [PATCH 05/13] home-assistant: Update to 2024.12.5 --- home-assistant/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/home-assistant/kustomization.yaml b/home-assistant/kustomization.yaml index 495a452..1045b4d 100644 --- a/home-assistant/kustomization.yaml +++ b/home-assistant/kustomization.yaml @@ -117,7 +117,7 @@ patches: name: dch-root-ca images: - name: ghcr.io/home-assistant/home-assistant - newTag: 2024.11.2 + newTag: 2024.12.5 - name: docker.io/rhasspy/wyoming-whisper newTag: 2.2.0 - name: docker.io/rhasspy/wyoming-piper From 387b7d120e052f3a4472590af2e37ddb8aaeaa2c Mon Sep 17 00:00:00 2001 From: bot Date: Sat, 21 Dec 2024 12:32:04 +0000 Subject: [PATCH 06/13] whisper: Update to 2.4.0 --- home-assistant/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/home-assistant/kustomization.yaml b/home-assistant/kustomization.yaml index 1045b4d..8679ec7 100644 --- a/home-assistant/kustomization.yaml +++ b/home-assistant/kustomization.yaml @@ -119,7 +119,7 @@ images: - name: ghcr.io/home-assistant/home-assistant newTag: 2024.12.5 - name: docker.io/rhasspy/wyoming-whisper - newTag: 2.2.0 + newTag: 2.4.0 - name: docker.io/rhasspy/wyoming-piper newTag: 1.5.0 - name: docker.io/koenkk/zigbee2mqtt From 50459e111e2b057c441fbaed41faab2cd52ca0c7 Mon Sep 17 00:00:00 2001 From: bot Date: Sat, 21 Dec 2024 12:32:04 +0000 Subject: [PATCH 07/13] zigbee2mqtt: Update to 1.42.0 --- home-assistant/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/home-assistant/kustomization.yaml b/home-assistant/kustomization.yaml index 8679ec7..c7c31a6 100644 --- a/home-assistant/kustomization.yaml +++ b/home-assistant/kustomization.yaml @@ -123,7 +123,7 @@ images: - name: docker.io/rhasspy/wyoming-piper newTag: 1.5.0 - name: docker.io/koenkk/zigbee2mqtt - newTag: 1.41.0 + newTag: 1.42.0 - name: docker.io/zwavejs/zwave-js-ui newTag: 9.27.2 - name: docker.io/library/eclipse-mosquitto From 2691b58c0536024fbc4550efadfe825e748cdbf7 Mon Sep 17 00:00:00 2001 From: bot Date: Sat, 21 Dec 2024 12:32:04 +0000 Subject: [PATCH 08/13] zwavejs2mqtt: Update to 9.29.0 --- home-assistant/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/home-assistant/kustomization.yaml b/home-assistant/kustomization.yaml index c7c31a6..4c7d5cf 100644 --- a/home-assistant/kustomization.yaml +++ b/home-assistant/kustomization.yaml @@ -125,6 +125,6 @@ images: - name: docker.io/koenkk/zigbee2mqtt newTag: 1.42.0 - name: docker.io/zwavejs/zwave-js-ui - newTag: 9.27.2 + newTag: 9.29.0 - name: docker.io/library/eclipse-mosquitto newTag: 2.0.20 From 1c4b5e19a47525933081f10917df97e5fdacdcb8 Mon Sep 17 00:00:00 2001 From: bot Date: Sat, 21 Dec 2024 12:32:08 +0000 Subject: [PATCH 09/13] firefly-iii: Update to 6.1.25 --- firefly-iii/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firefly-iii/kustomization.yaml b/firefly-iii/kustomization.yaml index 694db58..49798d8 100644 --- a/firefly-iii/kustomization.yaml +++ b/firefly-iii/kustomization.yaml @@ -55,4 +55,4 @@ patches: defaultMode: 0640 images: - name: docker.io/fireflyiii/core - newTag: version-6.1.22 + newTag: version-6.1.25 From a79668dcf1f482273d2a9dadbaabfa3acf30bff6 Mon Sep 17 00:00:00 2001 From: bot Date: Sat, 21 Dec 2024 12:32:10 +0000 Subject: [PATCH 10/13] gotenberg: Update to 8.14.1 --- paperless-ngx/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paperless-ngx/kustomization.yaml b/paperless-ngx/kustomization.yaml index 6006e6b..5a233ab 100644 --- a/paperless-ngx/kustomization.yaml +++ b/paperless-ngx/kustomization.yaml @@ -47,6 +47,6 @@ images: - name: ghcr.io/paperless-ngx/paperless-ngx newTag: 2.13.5 - name: docker.io/gotenberg/gotenberg - newTag: 8.13.0 + newTag: 8.14.1 - name: docker.io/apache/tika newTag: 3.0.0.0 From d04c18cfcda2a7e4be8707bb749d229486d3e576 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 21 Dec 2024 12:14:11 -0600 Subject: [PATCH 11/13] v-m/alerts: Remove 'no file changes' alert It turns out this alert is not very useful, and indeed quite annoying. Many servers can go for days or even weeks with no changes, which is completely normal. --- victoria-metrics/alerts.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/victoria-metrics/alerts.yml b/victoria-metrics/alerts.yml index fb7a334..7205472 100644 --- a/victoria-metrics/alerts.yml +++ b/victoria-metrics/alerts.yml @@ -232,25 +232,6 @@ groups: Clients are scheduled to back up every day, but at least one has not been backed up in at least 7 days. Check the Restic configuration on that system to ensure backups are running properly. - - alert: No File Changes - expr: >- - max_over_time( - abs( - delta( - sum(restic_backup_size_total{ - client_hostname!="pxe0.pyrocufflink.blue", - client_hostname!="web0.pyrocufflink.blue", - }) - by (client_hostname, client_username) - ) - )[7d] - ) == 0 - annotations: - summary: The size of the Restic backup has not changed - description: >- - The size of the Restic backup for a particular client has not changed - in at least 7 days. This may indicate that the backup configuration - is incorrect. - name: Paperless-ngx rules: From a03d63841dcf250d20d40ebd5a63dd388e1ebf71 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 21 Dec 2024 12:17:45 -0600 Subject: [PATCH 12/13] v-m/alerts: Fire paperless email alert after 12h We don't need a notification about paperless not scheduling email tasks every time there is a gap in the metric. This can happen in some innocuous situations like when the pod restarts or if there is a brief disruption of service. Using the `absent_over_time` function with a range vector, we can have the alert fire only if there have been no email tasks scheduled within the last 12 hours. --- victoria-metrics/alerts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/victoria-metrics/alerts.yml b/victoria-metrics/alerts.yml index 7205472..d8c37ab 100644 --- a/victoria-metrics/alerts.yml +++ b/victoria-metrics/alerts.yml @@ -254,11 +254,11 @@ groups: for details about the task failures. - alert: Paperless email task not running expr: >- - absent( + absent_over_time( flower_events_total{ type="task-started", task="paperless_mail.tasks.process_mail_accounts" - } + }[12h] ) annotations: summary: Paperless task to process mail accounts has not run recently From b9d69ec0a36ec73a6ea4b89c1fa128574065c6da Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 21 Dec 2024 12:23:19 -0600 Subject: [PATCH 13/13] v-m/alerts: Ignore missing backups from Toad, Luma Toad and Luma can go offline for several days at a time if I don't use them. I don't need an alert telling me this. --- victoria-metrics/alerts.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/victoria-metrics/alerts.yml b/victoria-metrics/alerts.yml index d8c37ab..94c4dda 100644 --- a/victoria-metrics/alerts.yml +++ b/victoria-metrics/alerts.yml @@ -225,7 +225,10 @@ groups: the issue as soon as possible. - alert: Last Backup Age expr: >- - time() - restic_backup_timestamp > 604800 + time() - restic_backup_timestamp{ + client_hostname!="luma.pyrocufflink.blue", + client_hostname!="toad.pyrocufflink.blue", + }> 604800 annotations: summary: A Restic client has not backed up recently description: >-