fluent-bit: Make ntfy pipeline steps optional

Most hosts will not need to send any messages to ntfy.  Let's define the
ntfy pipeline stages only for the machines that need them.  There are
currently two use cases for ntfy:

* MD RAID status messages (from Chromie and nvr2)
* WAN Link status messages (from gw1)

Breaking up the pipeline into smaller pieces allows both of these use
cases to define their appropriate filters while still sharing the common
steps.  The other machines that have no use for these steps now omit
them entirely.

group_vars/all.yml

@@ -142,23 +142,7 @@ dnf_automatic_schedule: >-
      | string
   }} *-*-* 04:00:00 America/Chicago
 
-fluent_bit_default_filters:
-- name: record_modifier
-  match: '*'
-  record:
-  - hostname ${HOSTNAME}
-# Avoid log amplification from logging the result of sending logs!
-- name: grep
-  match: host.fluent-bit.service
-  exclude: message \[output:http:.+\] .+, HTTP status=200$
-- name: rewrite_tag
-  alias: ntfy
-  match: host.*
-  rule: transport kernel ntfy true
-- name: grep
-  match: ntfy
-  alias: ntfy.filter
-  regex: message ^md
+fluent_bit_ntfy_common_filters:
 - name: lua
   alias: ntfy.populate
   match: ntfy
@@ -180,6 +164,18 @@ fluent_bit_default_filters:
   - tags
   - topic
 
+fluent_bit_common_filters:
+- name: record_modifier
+  match: '*'
+  record:
+  - hostname ${HOSTNAME}
+# Avoid log amplification from logging the result of sending logs!
+- name: grep
+  match: host.fluent-bit.service
+  exclude: message \[output:http:.+\] .+, HTTP status=200$
+
+fluent_bit_default_filters: '{{ fluent_bit_common_filters }}'
+
 fluent_bit_filters: '{{ fluent_bit_default_filters }}'
 
 fluent_bit_output_template_victorialogs:
@@ -193,12 +189,15 @@ fluent_bit_output_template_victorialogs:
   json_date_format: iso8601
   log_response_payload: false
 
-fluent_bit_output_systemd:
+_fluent_bit_output_systemd:
   name: http
   alias: victorialogs
   match: host.*
   uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date
 
+fluent_bit_output_systemd: >-
+  {{ _fluent_bit_output_systemd | combine(fluent_bit_output_template_victorialogs) }}
+
 fluent_bit_output_ntfy:
   name: http
   alias: ntfy
@@ -215,7 +214,6 @@ fluent_bit_output_ntfy:
   log_response_payload: false
 
 fluent_bit_main_outputs:
-- '{{ fluent_bit_output_systemd | combine(fluent_bit_output_template_victorialogs) }}'
-- '{{ fluent_bit_output_ntfy }}'
+- '{{ fluent_bit_output_systemd }}'
 
 fluent_bit_outputs: '{{ fluent_bit_main_outputs }}'

group_vars/md-ntfy.yml

@@ -0,0 +1,20 @@
+fluent_bit_ntfy_filters:
+- name: rewrite_tag
+  alias: ntfy
+  match: host.*
+  rule: transport kernel ntfy true
+- name: grep
+  match: ntfy
+  alias: ntfy.filter
+  regex: message ^md
+
+fluent_bit_default_filters: >-
+  {{
+    fluent_bit_common_filters +
+    fluent_bit_ntfy_filters +
+    fluent_bit_ntfy_common_filters
+  }}
+
+fluent_bit_main_outputs:
+- '{{ fluent_bit_output_systemd }}'
+- '{{ fluent_bit_output_ntfy }}'

host_vars/gw1.pyrocufflink.blue/main.yml

@@ -60,3 +60,24 @@ chrony_allow:
 - 172.24.100.0/24
 - 192.168.1.0/24
 - fd68:c2d2:500e:3e00::/56
+
+fluent_bit_ntfy_filters:
+- name: rewrite_tag
+  alias: ntfy
+  match: host.*
+  rule: transport kernel ntfy true
+- name: grep
+  match: ntfy
+  alias: ntfy.filter
+  regex: message wan.+Link
+
+fluent_bit_default_filters: >-
+  {{
+    fluent_bit_common_filters +
+    fluent_bit_ntfy_filters +
+    fluent_bit_ntfy_common_filters
+  }}
+
+fluent_bit_main_outputs:
+- '{{ fluent_bit_output_systemd }}'
+- '{{ fluent_bit_output_ntfy }}'

hosts

@@ -130,6 +130,10 @@ k8s-node
 [loki]
 loki1.pyrocufflink.blue
 
+[md-ntfy]
+chromie.pyrocufflink.blue
+nvr2.pyrocufflink.blue
+
 [minio-backups]
 chromie.pyrocufflink.blue