From 2cba5eb2e4434807da9372762247a97b40d27e6b Mon Sep 17 00:00:00 2001
From: "Dustin C. Hatch" <dustin@hatch.name>
Date: Mon, 15 Sep 2025 10:46:45 -0500
Subject: [PATCH] fluent-bit: Make ntfy pipeline steps optional

Most hosts will not need to send any messages to ntfy.  Let's define the
ntfy pipeline stages only for the machines that need them.  There are
currently two use cases for ntfy:

* MD RAID status messages (from Chromie and nvr2)
* WAN Link status messages (from gw1)

Breaking up the pipeline into smaller pieces allows both of these use
cases to define their appropriate filters while still sharing the common
steps.  The other machines that have no use for these steps now omit
them entirely.
---
 group_vars/all.yml                       | 38 +++++++++++-------------
 group_vars/md-ntfy.yml                   | 20 +++++++++++++
 host_vars/gw1.pyrocufflink.blue/main.yml | 21 +++++++++++++
 hosts                                    |  4 +++
 4 files changed, 63 insertions(+), 20 deletions(-)
 create mode 100644 group_vars/md-ntfy.yml

diff --git a/group_vars/all.yml b/group_vars/all.yml
index b7a8487..c76dfed 100644
--- a/group_vars/all.yml
+++ b/group_vars/all.yml
@@ -142,23 +142,7 @@ dnf_automatic_schedule: >-
      | string
   }} *-*-* 04:00:00 America/Chicago
 
-fluent_bit_default_filters:
-- name: record_modifier
-  match: '*'
-  record:
-  - hostname ${HOSTNAME}
-# Avoid log amplification from logging the result of sending logs!
-- name: grep
-  match: host.fluent-bit.service
-  exclude: message \[output:http:.+\] .+, HTTP status=200$
-- name: rewrite_tag
-  alias: ntfy
-  match: host.*
-  rule: transport kernel ntfy true
-- name: grep
-  match: ntfy
-  alias: ntfy.filter
-  regex: message ^md
+fluent_bit_ntfy_common_filters:
 - name: lua
   alias: ntfy.populate
   match: ntfy
@@ -180,6 +164,18 @@ fluent_bit_default_filters:
   - tags
   - topic
 
+fluent_bit_common_filters:
+- name: record_modifier
+  match: '*'
+  record:
+  - hostname ${HOSTNAME}
+# Avoid log amplification from logging the result of sending logs!
+- name: grep
+  match: host.fluent-bit.service
+  exclude: message \[output:http:.+\] .+, HTTP status=200$
+
+fluent_bit_default_filters: '{{ fluent_bit_common_filters }}'
+
 fluent_bit_filters: '{{ fluent_bit_default_filters }}'
 
 fluent_bit_output_template_victorialogs:
@@ -193,12 +189,15 @@ fluent_bit_output_template_victorialogs:
   json_date_format: iso8601
   log_response_payload: false
 
-fluent_bit_output_systemd:
+_fluent_bit_output_systemd:
   name: http
   alias: victorialogs
   match: host.*
   uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date
 
+fluent_bit_output_systemd: >-
+  {{ _fluent_bit_output_systemd | combine(fluent_bit_output_template_victorialogs) }}
+
 fluent_bit_output_ntfy:
   name: http
   alias: ntfy
@@ -215,7 +214,6 @@ fluent_bit_output_ntfy:
   log_response_payload: false
 
 fluent_bit_main_outputs:
-- '{{ fluent_bit_output_systemd | combine(fluent_bit_output_template_victorialogs) }}'
-- '{{ fluent_bit_output_ntfy }}'
+- '{{ fluent_bit_output_systemd }}'
 
 fluent_bit_outputs: '{{ fluent_bit_main_outputs }}'
diff --git a/group_vars/md-ntfy.yml b/group_vars/md-ntfy.yml
new file mode 100644
index 0000000..f5a0332
--- /dev/null
+++ b/group_vars/md-ntfy.yml
@@ -0,0 +1,20 @@
+fluent_bit_ntfy_filters:
+- name: rewrite_tag
+  alias: ntfy
+  match: host.*
+  rule: transport kernel ntfy true
+- name: grep
+  match: ntfy
+  alias: ntfy.filter
+  regex: message ^md
+
+fluent_bit_default_filters: >-
+  {{
+    fluent_bit_common_filters +
+    fluent_bit_ntfy_filters +
+    fluent_bit_ntfy_common_filters
+  }}
+
+fluent_bit_main_outputs:
+- '{{ fluent_bit_output_systemd }}'
+- '{{ fluent_bit_output_ntfy }}'
diff --git a/host_vars/gw1.pyrocufflink.blue/main.yml b/host_vars/gw1.pyrocufflink.blue/main.yml
index cdcfdf8..b1d1e85 100644
--- a/host_vars/gw1.pyrocufflink.blue/main.yml
+++ b/host_vars/gw1.pyrocufflink.blue/main.yml
@@ -60,3 +60,24 @@ chrony_allow:
 - 172.24.100.0/24
 - 192.168.1.0/24
 - fd68:c2d2:500e:3e00::/56
+
+fluent_bit_ntfy_filters:
+- name: rewrite_tag
+  alias: ntfy
+  match: host.*
+  rule: transport kernel ntfy true
+- name: grep
+  match: ntfy
+  alias: ntfy.filter
+  regex: message wan.+Link
+
+fluent_bit_default_filters: >-
+  {{
+    fluent_bit_common_filters +
+    fluent_bit_ntfy_filters +
+    fluent_bit_ntfy_common_filters
+  }}
+
+fluent_bit_main_outputs:
+- '{{ fluent_bit_output_systemd }}'
+- '{{ fluent_bit_output_ntfy }}'
diff --git a/hosts b/hosts
index fbdcb9f..f6f611d 100644
--- a/hosts
+++ b/hosts
@@ -130,6 +130,10 @@ k8s-node
 [loki]
 loki1.pyrocufflink.blue
 
+[md-ntfy]
+chromie.pyrocufflink.blue
+nvr2.pyrocufflink.blue
+
 [minio-backups]
 chromie.pyrocufflink.blue