raid-array: Fix md re-add automation

Recent versions of `mdadm` stopped accepting `/dev/disk/by-id` symlinks
as the MD device:

> mdadm: Value "/dev/disk/by-id/md-name-backup5" cannot be set as devname. Reason: Cannot be started from '/' or '<'.

To work around this, we need a script to resolve the symlink and pass
the real block device name.

group_vars/all.yml

@@ -147,6 +147,34 @@ fluent_bit_filters:
 - name: grep
   match: host.fluent-bit.service
   exclude: message \[output:http:victorialogs\] .+, HTTP status=200$
+- name: rewrite_tag
+  alias: ntfy
+  match: host.*
+  rule: transport kernel ntfy true
+- name: grep
+  match: ntfy
+  alias: ntfy.filter
+  regex: message ^md
+- name: lua
+  alias: ntfy.populate
+  match: ntfy
+  code: |
+    function ntfy_transform(tag, timestamp, record)
+        record["topic"] = "alerts"
+        record["tags"] = {
+          record["hostname"],
+          record["syslog_identifier"],
+        }
+        return 1, timestamp, record
+    end
+  call: ntfy_transform
+- name: record_modifier
+  alias: ntfy.clean
+  match: ntfy
+  allowlist_key:
+  - message
+  - tags
+  - topic
 fluent_bit_outputs:
 - name: http
   alias: victorialogs
@@ -161,3 +189,16 @@ fluent_bit_outputs:
   format: json_lines
   json_date_format: iso8601
   log_response_payload: false
+- name: http
+  alias: ntfy
+  workers: 1
+  match: ntfy
+  host: ntfy.pyrocufflink.blue
+  port: 443
+  tls: true
+  tls.verify: true
+  tls.verify_hostname: true
+  uri: /
+  format: json_lines
+  json_date_key: false
+  log_response_payload: false

raid-array.yml

@@ -29,14 +29,35 @@
     tags:
     - mdadm
     - mdadm-create
+  - name: ensure md-auto-readd script is installed
+    copy:
+      content: |+
+        #!/bin/sh
+
+        main() {
+            md_dev=$(readlink -e /dev/disk/by-id/md-name-"$2")
+            if [ -z "${md_dev}" ]; then
+                printf 'Could not find block device for MD raid %s\n' "$2" >&2
+                exit 1
+            fi
+            exec mdadm --re-add "${md_dev}" "$1"
+        }
+
+        main "$@" 2>&1 | logger
+      dest: /usr/local/libexec/md-auto-readd
+      owner: root
+      group: root
+      mode: u=rwx,go=rx
+    tags:
+    - script
   - name: ensure raid auto re-add udev rule exists
     copy:
       content: >+
-        ENV{ID_FS_LABEL}=="{{ md_name }}",
+        ACTION=="add",
         ENV{ID_FS_TYPE}=="linux_raid_member",
         ENV{ID_FS_USAGE}=="raid",
-        RUN+="/usr/sbin/mdadm --re-add /dev/disk/by-id/md-name-{{ md_name }} $devnode"
-      dest: /etc/udev/rules.d/80-{{ md_name }}.rules
+        RUN+="/usr/local/libexec/md-auto-readd $devnode $env{ID_FS_LABEL}"
+      dest: /etc/udev/rules.d/80-md-auto-readd.rules
       mode: u=rw,go=r
       owner: root
       group: root