victoria-metrics: Send alerts via ntfy

I don't like having alerts sent by e-mail.  Since I don't get e-mail
notifications on my watch, I often do not see alerts for quite some
time.  They are also much harder to read in an e-mail client (Fastmail
web an K-9 Mail both display them poorly).  I would much rather have
them delivered via _ntfy_, just like all the rest of the ephemeral
notifications I receive.

Fortunately, it is easy enough to integrate Alertmanager and _ntfy_
using the webhook notifier in Alertmanager.  Since _ntfy_ does not
natively support the Alertmanager webhook API, though, a bridge is
necessary to translate from one data format to the other.  There are a
few options for this bridge, but I chose
[alexbakker/alertmanager-ntfy][0] because it looked the most complete
while also having the simplest configuration format.  Sadly, it does not
expose any Prometheus metrics itself, and since it's deployed in the
_victoria-metrics_ namespace, it needs to be explicitly excluded from
the VMAgent scrape configuration.

[0]: https://github.com/alexbakker/alertmanager-ntfy

victoria-metrics/alertmanager-ntfy.config.yml

@@ -0,0 +1,24 @@
+http:
+  addr: :8000
+ntfy:
+  baseurl: http://ntfy.ntfy:2586
+  notification:
+    topic: alerts
+    priority: |
+      status == "firing" ? "urgent" : "default"
+    tags:
+    - tag: heavy_check_mark
+      condition: status == "resolved"
+    - tag: rotating_light
+      condition: status == "firing"
+    templates:
+      title: >-
+        {{ if eq .Status "resolved" }}Resolved: {{ end -}}
+        {{ if index .Annotations "summary" -}}
+        {{ index .Annotations "summary" }}
+        {{ else -}}
+        {{ index .Labels "alertname" }}
+        {{ index .Labels "alertgroup" }}
+        {{ end -}}
+      description: |
+        {{ index .Annotations "description" }}

victoria-metrics/alertmanager-ntfy.yaml

@@ -0,0 +1,69 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: alertmanager-ntfy
+  labels:
+    app.kubernetes.io/name: alertmanager-ntfy
+    app.kubernetes.io/component: alertmanager-ntfy
+spec:
+  ports:
+  - port: 8000
+    name: alertmanager-ntfy
+  selector:
+    app.kubernetes.io/name: alertmanager-ntfy
+    app.kubernetes.io/component: alertmanager-ntfy
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: alertmanager-ntfy
+  labels:
+    app.kubernetes.io/name: alertmanager-ntfy
+    app.kubernetes.io/component: alertmanager-ntfy
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: alertmanager-ntfy
+      app.kubernetes.io/component: alertmanager-ntfy
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: alertmanager-ntfy
+        app.kubernetes.io/component: alertmanager-ntfy
+    spec:
+      containers:
+      - name: alertmanager-ntfy
+        image: git.pyrocufflink.net/containerimages/alertmanager-ntfy
+        args:
+        - --configs=/config/config.yml
+        ports:
+        - containerPort: 8000
+          name: http
+        readinessProbe: &probe
+          tcpSocket:
+            port: http
+          periodSeconds: 60
+        startupProbe:
+          <<: *probe
+          periodSeconds: 1
+          successThreshold: 1
+          failureThreshold: 30
+          timeoutSeconds: 1
+        securityContext:
+          runAsNonRoot: true
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /config
+          name: config
+          readOnly: true
+      securityContext:
+        fsGroup: 21447
+        runAsGroup: 21447
+        runAsNonRoot: true
+        runAsUser: 21447
+      volumes:
+      - name: config
+        configMap:
+          name: alertmanager-ntfy

victoria-metrics/alertmanager.config.yml

@@ -8,16 +8,11 @@ receivers:
   - send_resolved: true
     to: gyrfalcon@ebonfire.com
   name: default-email
+- name: ntfy
+  webhook_configs:
+  - url: http://alertmanager-ntfy:8000/hook
 
 route:
   group_by:
   - '...'
-  receiver: default-email
-  routes:
-  - group_by:
-    - alertname
-    group_wait: 1m
-    match:
-      job: homeassistant
-    receiver: default-email
-    repeat_interval: 120h
+  receiver: ntfy

victoria-metrics/kustomization.yaml

@@ -20,6 +20,7 @@ resources:
 - vmagent.yaml
 - vmalert.yaml
 - alertmanager.yaml
+- alertmanager-ntfy.yaml
 - blackbox-exporter.yaml
 - ingress.yaml
 - ../dch-root-ca
@@ -43,6 +44,10 @@ configMapGenerator:
   options:
     disableNameSuffixHash: true
 
+- name: alertmanager-ntfy
+  files:
+  - config.yml=alertmanager-ntfy.config.yml
+
 - name: blackbox
   files:
   - blackbox.yml

victoria-metrics/scrape.yml

@@ -14,6 +14,10 @@ scrape_configs:
     separator: ':'
     regex: 'vmstorage:(vminsert|vmselect)'
     action: drop
+  - source_labels:
+    - __meta_kubernetes_pod_label_app_kubernetes_io_component
+    regex: alertmanager-ntfy
+    action: drop
   - source_labels:
     - __meta_kubernetes_pod_container_name
     regex: vmalert