From d0bffdeb150e7accd4b62b12fa9577a2893c5846 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 15 Sep 2025 07:51:39 -0500 Subject: [PATCH 01/10] r/fluent-bit: Support configuring parsers When ingesting logs from sources other than systemd, such as unstructured log files written by uncooperative services, it may be necessary to define custom parsers. --- roles/fluent-bit/defaults/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/fluent-bit/defaults/main.yml b/roles/fluent-bit/defaults/main.yml index 2093a8e..b50d6f4 100644 --- a/roles/fluent-bit/defaults/main.yml +++ b/roles/fluent-bit/defaults/main.yml @@ -1,5 +1,7 @@ fluent_bit_config: service: '{{ fluent_bit_config_service }}' + parsers: '{{ fluent_bit_parsers }}' + multiline_parsers: '{{ fluent_bit_multiline_parsers }}' pipeline: '{{ fluent_bit_pipeline }}' fluent_bit_config_service: @@ -24,6 +26,10 @@ fluent_bit_input_systemd: lowercase: true strip_underscores: true +fluent_bit_parsers: [] + +fluent_bit_multiline_parsers: [] + fluent_bit_filters: [] fluent_bit_outputs: From 0331a55b3eb03b3dfa1243bf79032c55a619f51b Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 15 Sep 2025 07:53:13 -0500 Subject: [PATCH 02/10] r/fluent-bit: Set HOSTNAME environment variable Fluent-bit does not have any native capability for setting a field with the hostname of the machine, but it can set a field with the value of an environment variable. Thus, we can set the `HOSTNAME` environment variable and then use that to set the field in the pipeline. --- roles/fluent-bit/files/fluent-bit.service | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/fluent-bit/files/fluent-bit.service b/roles/fluent-bit/files/fluent-bit.service index c2b1b6f..6bbc134 100644 --- a/roles/fluent-bit/files/fluent-bit.service +++ b/roles/fluent-bit/files/fluent-bit.service @@ -10,6 +10,7 @@ StartLimitBurst=5 Type=exec ExecStart=/usr/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml -Y ExecReload=/bin/kill -HUP $MAINPID +Environment=HOSTNAME=%H StateDirectory=fluent-bit Restart=always RestartSec=1 From 75061c4d78c29bf9de1d7d8607fc290995476713 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 15 Sep 2025 07:55:43 -0500 Subject: [PATCH 03/10] all: Split up Fluent Bit vars Instead of defining the common values for Fluent bit inputs, filters, and outputs directly in the variables used by the _fluent-bit_ role, we need to split these into reusable pieces. This way, hosts and groups that need to use a slightly different pipeline configuration can access the default values without having to redefine them. --- group_vars/all.yml | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/group_vars/all.yml b/group_vars/all.yml index e34d219..2a6aa74 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -142,7 +142,7 @@ dnf_automatic_schedule: >- | string }} *-*-* 04:00:00 America/Chicago -fluent_bit_filters: +fluent_bit_default_filters: # Avoid log amplification from logging the result of sending logs! - name: grep match: host.fluent-bit.service @@ -175,21 +175,28 @@ fluent_bit_filters: - message - tags - topic -fluent_bit_outputs: -- name: http - alias: victorialogs - match: host.* + +fluent_bit_filters: '{{ fluent_bit_default_filters }}' + +fluent_bit_output_template_victorialogs: host: logs.pyrocufflink.blue port: 443 tls: true tls.verify: true tls.verify_hostname: true tls.ca_file: /etc/pki/ca-trust/source/anchors/dch-root-ca-r2.crt - uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date format: json_lines json_date_format: iso8601 log_response_payload: false -- name: http + +fluent_bit_output_systemd: + name: http + alias: victorialogs + match: host.* + uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date + +fluent_bit_output_ntfy: + name: http alias: ntfy workers: 1 match: ntfy @@ -202,3 +209,9 @@ fluent_bit_outputs: format: json_lines json_date_key: false log_response_payload: false + +fluent_bit_main_outputs: +- '{{ fluent_bit_output_systemd | combine(fluent_bit_output_template_victorialogs) }}' +- '{{ fluent_bit_output_ntfy }}' + +fluent_bit_outputs: '{{ fluent_bit_main_outputs }}' From 414cb828e18eaee767f1398cc70f5982dc4c9200 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 15 Sep 2025 07:58:29 -0500 Subject: [PATCH 04/10] unifi: Configure Fluent Bit for Unifi server The Unifi Network server writes a bunch of log files that we need to forward to Victoria Logs. This commit introduces components to the Fluent Bit pipeline to read these files with the `tail` input plugin, parse them using regular expressions to extract the correct time stamp from the messages, and send them to Victoria Logs. --- group_vars/unifi/fluent-bit.yml | 93 +++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 group_vars/unifi/fluent-bit.yml diff --git a/group_vars/unifi/fluent-bit.yml b/group_vars/unifi/fluent-bit.yml new file mode 100644 index 0000000..39c4008 --- /dev/null +++ b/group_vars/unifi/fluent-bit.yml @@ -0,0 +1,93 @@ +fluent_bit_multiline_parsers: +- name: multiline_unifi + type: regex + flush_timeout: 500 + rules: + - state: start_state + regex: ^\[\d{4}-\d{2}-\d{2}T.* + next_state: cont + - state: cont + regex: ^[^\[].* + next_state: cont +- name: multiline_mongod + type: regex + flush_timeout: 500 + rules: + - state: start_state + regex: ^\d{4}-\d{2}-\d{2}T.* + next_state: cont + - state: cont + regex: ^(?!\d{4}-\d{2}-\d{2}T).* + next_state: cont + +unifi_server_logs: +- /var/log/unifi/migration.log +- /var/log/unifi/server.log +- /var/log/unifi/startup.log +- /var/log/unifi/state.log +- /var/log/unifi/tasks.log + +fluent_bit_input_unifi: + name: tail + alias: unifi.server + tag: unifi.server + path: '{{ unifi_server_logs | join(",") }}' + path_key: filename + multiline.parser: multiline_unifi + db: /var/lib/fluent-bit/unifi.db + read_from_head: true + +fluent_bit_input_unifi_mongod: + name: tail + alias: unifi.mongod + tag: unifi.mongod + path: /var/log/unifi/mongod.log + path_key: filename + multiline.parser: multiline_mongod + db: /var/lib/fluent-bit/unifi-mongod.db + read_from_head: true + +fluent_bit_unifi_inputs: +- '{{ fluent_bit_input_unifi }}' +- '{{ fluent_bit_input_unifi_mongod }}' + +fluent_bit_inputs: '{{ fluent_bit_default_inputs + fluent_bit_unifi_inputs }}' + +fluent_bit_parsers: +- name: unifi-server + format: regex + regex: /^\[(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2},\d{3}([\+-]\d{4}|Z))\] (?.*)/m + time_key: timestamp + time_format: '%Y-%m-%dT%H:%M:%S,%L%z' +- name: mongod + format: regex + regex: /^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}[\+-]\d{4}) (?.*)/m + time_key: timestamp + time_format: '%Y-%m-%dT%H:%M:%S.%L%z' + +fluent_bit_unifi_filters: +- name: parser + alias: unifi-server + match: unifi.server + key_name: log + parser: unifi-server + reserve_data: true +- name: parser + alias: unifi-mongod + match: unifi.mongod + key_name: log + parser: mongod + reserve_data: true + +fluent_bit_filters: '{{ fluent_bit_default_filters + fluent_bit_unifi_filters }}' + +fluent_bit_output_unifi: + name: http + alias: unifi + match: unifi.* + uri: /insert/jsonline?_stream_fields=hostname,filename&_msg_field=log&_time_field=date + +fluent_bit_unifi_outputs: +- '{{ fluent_bit_output_unifi | combine(fluent_bit_output_template_victorialogs) }}' + +fluent_bit_outputs: '{{ fluent_bit_main_outputs + fluent_bit_unifi_outputs }}' From 3d4bf3dd6cd289f2426956d34dcec3c8d7f2785e Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 15 Sep 2025 08:00:16 -0500 Subject: [PATCH 05/10] fluent-bit: Add hostname field to all records Messages from sources other than the systemd journal do not have a `hostname` field by default. This could make filtering logs difficult if there are multiple servers that host the same application. Thus, we need to inject the host name statically into every record, to ensure they can be correctly traced to their source machine. --- group_vars/all.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/group_vars/all.yml b/group_vars/all.yml index 2a6aa74..cc4e676 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -143,6 +143,10 @@ dnf_automatic_schedule: >- }} *-*-* 04:00:00 America/Chicago fluent_bit_default_filters: +- name: record_modifier + match: '*' + record: + - hostname ${HOSTNAME} # Avoid log amplification from logging the result of sending logs! - name: grep match: host.fluent-bit.service From faf48229183b578e49465d424caadbe0eca4634f Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 15 Sep 2025 08:01:42 -0500 Subject: [PATCH 06/10] fluent-bit: Ignore all HTTP output status messages If the Fluent Bit pipeline includes multiple HTTP outputs, we need to supporess the `HTTP status=200` messages from _all_ of them. --- group_vars/all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/all.yml b/group_vars/all.yml index cc4e676..b7a8487 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -150,7 +150,7 @@ fluent_bit_default_filters: # Avoid log amplification from logging the result of sending logs! - name: grep match: host.fluent-bit.service - exclude: message \[output:http:victorialogs\] .+, HTTP status=200$ + exclude: message \[output:http:.+\] .+, HTTP status=200$ - name: rewrite_tag alias: ntfy match: host.* From 2cba5eb2e4434807da9372762247a97b40d27e6b Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 15 Sep 2025 10:46:45 -0500 Subject: [PATCH 07/10] fluent-bit: Make ntfy pipeline steps optional Most hosts will not need to send any messages to ntfy. Let's define the ntfy pipeline stages only for the machines that need them. There are currently two use cases for ntfy: * MD RAID status messages (from Chromie and nvr2) * WAN Link status messages (from gw1) Breaking up the pipeline into smaller pieces allows both of these use cases to define their appropriate filters while still sharing the common steps. The other machines that have no use for these steps now omit them entirely. --- group_vars/all.yml | 38 +++++++++++------------- group_vars/md-ntfy.yml | 20 +++++++++++++ host_vars/gw1.pyrocufflink.blue/main.yml | 21 +++++++++++++ hosts | 4 +++ 4 files changed, 63 insertions(+), 20 deletions(-) create mode 100644 group_vars/md-ntfy.yml diff --git a/group_vars/all.yml b/group_vars/all.yml index b7a8487..c76dfed 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -142,23 +142,7 @@ dnf_automatic_schedule: >- | string }} *-*-* 04:00:00 America/Chicago -fluent_bit_default_filters: -- name: record_modifier - match: '*' - record: - - hostname ${HOSTNAME} -# Avoid log amplification from logging the result of sending logs! -- name: grep - match: host.fluent-bit.service - exclude: message \[output:http:.+\] .+, HTTP status=200$ -- name: rewrite_tag - alias: ntfy - match: host.* - rule: transport kernel ntfy true -- name: grep - match: ntfy - alias: ntfy.filter - regex: message ^md +fluent_bit_ntfy_common_filters: - name: lua alias: ntfy.populate match: ntfy @@ -180,6 +164,18 @@ fluent_bit_default_filters: - tags - topic +fluent_bit_common_filters: +- name: record_modifier + match: '*' + record: + - hostname ${HOSTNAME} +# Avoid log amplification from logging the result of sending logs! +- name: grep + match: host.fluent-bit.service + exclude: message \[output:http:.+\] .+, HTTP status=200$ + +fluent_bit_default_filters: '{{ fluent_bit_common_filters }}' + fluent_bit_filters: '{{ fluent_bit_default_filters }}' fluent_bit_output_template_victorialogs: @@ -193,12 +189,15 @@ fluent_bit_output_template_victorialogs: json_date_format: iso8601 log_response_payload: false -fluent_bit_output_systemd: +_fluent_bit_output_systemd: name: http alias: victorialogs match: host.* uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date +fluent_bit_output_systemd: >- + {{ _fluent_bit_output_systemd | combine(fluent_bit_output_template_victorialogs) }} + fluent_bit_output_ntfy: name: http alias: ntfy @@ -215,7 +214,6 @@ fluent_bit_output_ntfy: log_response_payload: false fluent_bit_main_outputs: -- '{{ fluent_bit_output_systemd | combine(fluent_bit_output_template_victorialogs) }}' -- '{{ fluent_bit_output_ntfy }}' +- '{{ fluent_bit_output_systemd }}' fluent_bit_outputs: '{{ fluent_bit_main_outputs }}' diff --git a/group_vars/md-ntfy.yml b/group_vars/md-ntfy.yml new file mode 100644 index 0000000..f5a0332 --- /dev/null +++ b/group_vars/md-ntfy.yml @@ -0,0 +1,20 @@ +fluent_bit_ntfy_filters: +- name: rewrite_tag + alias: ntfy + match: host.* + rule: transport kernel ntfy true +- name: grep + match: ntfy + alias: ntfy.filter + regex: message ^md + +fluent_bit_default_filters: >- + {{ + fluent_bit_common_filters + + fluent_bit_ntfy_filters + + fluent_bit_ntfy_common_filters + }} + +fluent_bit_main_outputs: +- '{{ fluent_bit_output_systemd }}' +- '{{ fluent_bit_output_ntfy }}' diff --git a/host_vars/gw1.pyrocufflink.blue/main.yml b/host_vars/gw1.pyrocufflink.blue/main.yml index cdcfdf8..b1d1e85 100644 --- a/host_vars/gw1.pyrocufflink.blue/main.yml +++ b/host_vars/gw1.pyrocufflink.blue/main.yml @@ -60,3 +60,24 @@ chrony_allow: - 172.24.100.0/24 - 192.168.1.0/24 - fd68:c2d2:500e:3e00::/56 + +fluent_bit_ntfy_filters: +- name: rewrite_tag + alias: ntfy + match: host.* + rule: transport kernel ntfy true +- name: grep + match: ntfy + alias: ntfy.filter + regex: message wan.+Link + +fluent_bit_default_filters: >- + {{ + fluent_bit_common_filters + + fluent_bit_ntfy_filters + + fluent_bit_ntfy_common_filters + }} + +fluent_bit_main_outputs: +- '{{ fluent_bit_output_systemd }}' +- '{{ fluent_bit_output_ntfy }}' diff --git a/hosts b/hosts index fbdcb9f..f6f611d 100644 --- a/hosts +++ b/hosts @@ -130,6 +130,10 @@ k8s-node [loki] loki1.pyrocufflink.blue +[md-ntfy] +chromie.pyrocufflink.blue +nvr2.pyrocufflink.blue + [minio-backups] chromie.pyrocufflink.blue From c2d26f1f59673def2ebb935be0841adb4f91172f Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 15 Sep 2025 10:49:32 -0500 Subject: [PATCH 08/10] r/fluent-bit: Drop network.target requirement The _network.target_ unit should be used for ordering only. Listing it as a `Requires=` dependency can cause _fluent-bit.service_ to fail to start at all if the network takes slightly too long to initialize at boot. --- roles/fluent-bit/files/fluent-bit.service | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/fluent-bit/files/fluent-bit.service b/roles/fluent-bit/files/fluent-bit.service index 6bbc134..fa9a4ec 100644 --- a/roles/fluent-bit/files/fluent-bit.service +++ b/roles/fluent-bit/files/fluent-bit.service @@ -1,7 +1,6 @@ [Unit] Description=Fluent Bit Documentation=https://docs.fluentbit.io/manual/ -Requires=network.target After=network.target StartLimitIntervalSec=5 StartLimitBurst=5 From 4601b4d092e8058cc17aba6a1e27981046a5c025 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 15 Sep 2025 11:13:01 -0500 Subject: [PATCH 09/10] victoria-logs: Update to v1.33.1 --- group_vars/victoria-logs.yml | 14 +++++++------- roles/victoria-logs/defaults/main.yml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/group_vars/victoria-logs.yml b/group_vars/victoria-logs.yml index 1bf0b20..544d0b0 100644 --- a/group_vars/victoria-logs.yml +++ b/group_vars/victoria-logs.yml @@ -4,13 +4,13 @@ data_volumes: mountpoint: /var/lib/victoria-logs victoria_logs_extra_args: -- '-syslog.listenAddr.tcp :601' -- '-syslog.listenAddr.udp :514' -- '-syslog.extraFields.udp ''{}''' -- '-syslog.streamFields.udp ''["hostname","app_name","proc_id"]''' -- '-syslog.listenAddr.udp :6666' -- '-syslog.extraFields.udp ''{"stream":"netconsole"}''' -- '-syslog.streamFields.udp ''["stream"]''' +- '-syslog.listenAddr.tcp=:601' +- '-syslog.listenAddr.udp=:514' +- '-syslog.extraFields.udp=''{}''' +- '-syslog.streamFields.udp=''["hostname","app_name","proc_id"]''' +- '-syslog.listenAddr.udp=:6666' +- '-syslog.extraFields.udp=''{"stream":"netconsole"}''' +- '-syslog.streamFields.udp=''["stream"]''' victoria_logs_publish_ports: - '514:514/udp' diff --git a/roles/victoria-logs/defaults/main.yml b/roles/victoria-logs/defaults/main.yml index 7a26837..09c11ac 100644 --- a/roles/victoria-logs/defaults/main.yml +++ b/roles/victoria-logs/defaults/main.yml @@ -1,7 +1,7 @@ # vim: set ft=yaml.jinja : -victoria_logs_version: 1.23.3 -victoria_logs_container_image_tag: v{{ victoria_logs_version }}-victorialogs -victoria_logs_container_image_repo: docker.io/victoriametrics/victoria-logs +victoria_logs_version: 1.33.1 +victoria_logs_container_image_tag: v{{ victoria_logs_version }} +victoria_logs_container_image_repo: quay.io/victoriametrics/victoria-logs victoria_logs_container_image: >- {{ victoria_logs_container_image_repo }}:{{ victoria_logs_container_image_tag }} From f1b61a8d0aa0111aa0ce4bdcb9f46375c80b5402 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Mon, 24 Nov 2025 07:47:35 -0600 Subject: [PATCH 10/10] v-l: Enable useRemoteIP for syslog Victoria Logs can now record the source address for syslog messages in a `remoteIP` field. This has to be enabled specifically, although I can't think of a reason why someone would _not_ want to record that information. --- group_vars/victoria-logs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/group_vars/victoria-logs.yml b/group_vars/victoria-logs.yml index 544d0b0..370b708 100644 --- a/group_vars/victoria-logs.yml +++ b/group_vars/victoria-logs.yml @@ -5,7 +5,9 @@ data_volumes: victoria_logs_extra_args: - '-syslog.listenAddr.tcp=:601' +- '-syslog.useRemoteIP.tcp=true' - '-syslog.listenAddr.udp=:514' +- '-syslog.useRemoteIP.udp=true' - '-syslog.extraFields.udp=''{}''' - '-syslog.streamFields.udp=''["hostname","app_name","proc_id"]''' - '-syslog.listenAddr.udp=:6666'