Merge branch 'fluent-bit'

group_vars/all.yml

@@ -142,19 +142,7 @@ dnf_automatic_schedule: >-
      | string
   }} *-*-* 04:00:00 America/Chicago
 
-fluent_bit_filters:
+fluent_bit_ntfy_common_filters:
-# Avoid log amplification from logging the result of sending logs!
-- name: grep
-  match: host.fluent-bit.service
-  exclude: message \[output:http:victorialogs\] .+, HTTP status=200$
-- name: rewrite_tag
-  alias: ntfy
-  match: host.*
-  rule: transport kernel ntfy true
-- name: grep
-  match: ntfy
-  alias: ntfy.filter
-  regex: message ^md
 - name: lua
   alias: ntfy.populate
   match: ntfy
@@ -175,21 +163,43 @@ fluent_bit_filters:
   - message
   - tags
   - topic
-fluent_bit_outputs:
+
-- name: http
+fluent_bit_common_filters:
-  alias: victorialogs
+- name: record_modifier
-  match: host.*
+  match: '*'
+  record:
+  - hostname ${HOSTNAME}
+# Avoid log amplification from logging the result of sending logs!
+- name: grep
+  match: host.fluent-bit.service
+  exclude: message \[output:http:.+\] .+, HTTP status=200$
+
+fluent_bit_default_filters: '{{ fluent_bit_common_filters }}'
+
+fluent_bit_filters: '{{ fluent_bit_default_filters }}'
+
+fluent_bit_output_template_victorialogs:
   host: logs.pyrocufflink.blue
   port: 443
   tls: true
   tls.verify: true
   tls.verify_hostname: true
   tls.ca_file: /etc/pki/ca-trust/source/anchors/dch-root-ca-r2.crt
-  uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date
   format: json_lines
   json_date_format: iso8601
   log_response_payload: false
-- name: http
+
+_fluent_bit_output_systemd:
+  name: http
+  alias: victorialogs
+  match: host.*
+  uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date
+
+fluent_bit_output_systemd: >-
+  {{ _fluent_bit_output_systemd | combine(fluent_bit_output_template_victorialogs) }}
+
+fluent_bit_output_ntfy:
+  name: http
   alias: ntfy
   workers: 1
   match: ntfy
@@ -202,3 +212,8 @@ fluent_bit_outputs:
   format: json_lines
   json_date_key: false
   log_response_payload: false
+
+fluent_bit_main_outputs:
+- '{{ fluent_bit_output_systemd }}'
+
+fluent_bit_outputs: '{{ fluent_bit_main_outputs }}'

group_vars/md-ntfy.yml

@@ -0,0 +1,20 @@
+fluent_bit_ntfy_filters:
+- name: rewrite_tag
+  alias: ntfy
+  match: host.*
+  rule: transport kernel ntfy true
+- name: grep
+  match: ntfy
+  alias: ntfy.filter
+  regex: message ^md
+
+fluent_bit_default_filters: >-
+  {{
+    fluent_bit_common_filters +
+    fluent_bit_ntfy_filters +
+    fluent_bit_ntfy_common_filters
+  }}
+
+fluent_bit_main_outputs:
+- '{{ fluent_bit_output_systemd }}'
+- '{{ fluent_bit_output_ntfy }}'

group_vars/unifi/fluent-bit.yml

@@ -0,0 +1,93 @@
+fluent_bit_multiline_parsers:
+- name: multiline_unifi
+  type: regex
+  flush_timeout: 500
+  rules:
+  - state: start_state
+    regex: ^\[\d{4}-\d{2}-\d{2}T.*
+    next_state: cont
+  - state: cont
+    regex: ^[^\[].*
+    next_state: cont
+- name: multiline_mongod
+  type: regex
+  flush_timeout: 500
+  rules:
+  - state: start_state
+    regex: ^\d{4}-\d{2}-\d{2}T.*
+    next_state: cont
+  - state: cont
+    regex: ^(?!\d{4}-\d{2}-\d{2}T).*
+    next_state: cont
+
+unifi_server_logs:
+- /var/log/unifi/migration.log
+- /var/log/unifi/server.log
+- /var/log/unifi/startup.log
+- /var/log/unifi/state.log
+- /var/log/unifi/tasks.log
+
+fluent_bit_input_unifi:
+  name: tail
+  alias: unifi.server
+  tag: unifi.server
+  path: '{{ unifi_server_logs | join(",") }}'
+  path_key: filename
+  multiline.parser: multiline_unifi
+  db: /var/lib/fluent-bit/unifi.db
+  read_from_head: true
+
+fluent_bit_input_unifi_mongod:
+  name: tail
+  alias: unifi.mongod
+  tag: unifi.mongod
+  path: /var/log/unifi/mongod.log
+  path_key: filename
+  multiline.parser: multiline_mongod
+  db: /var/lib/fluent-bit/unifi-mongod.db
+  read_from_head: true
+
+fluent_bit_unifi_inputs:
+- '{{ fluent_bit_input_unifi }}'
+- '{{ fluent_bit_input_unifi_mongod }}'
+
+fluent_bit_inputs: '{{ fluent_bit_default_inputs + fluent_bit_unifi_inputs }}'
+
+fluent_bit_parsers:
+- name: unifi-server
+  format: regex
+  regex: /^\[(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2},\d{3}([\+-]\d{4}|Z))\] (?<log>.*)/m
+  time_key: timestamp
+  time_format: '%Y-%m-%dT%H:%M:%S,%L%z'
+- name: mongod
+  format: regex
+  regex: /^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}[\+-]\d{4}) (?<log>.*)/m
+  time_key: timestamp
+  time_format: '%Y-%m-%dT%H:%M:%S.%L%z'
+
+fluent_bit_unifi_filters:
+- name: parser
+  alias: unifi-server
+  match: unifi.server
+  key_name: log
+  parser: unifi-server
+  reserve_data: true
+- name: parser
+  alias: unifi-mongod
+  match: unifi.mongod
+  key_name: log
+  parser: mongod
+  reserve_data: true
+
+fluent_bit_filters: '{{ fluent_bit_default_filters + fluent_bit_unifi_filters }}'
+
+fluent_bit_output_unifi:
+  name: http
+  alias: unifi
+  match: unifi.*
+  uri: /insert/jsonline?_stream_fields=hostname,filename&_msg_field=log&_time_field=date
+
+fluent_bit_unifi_outputs:
+- '{{ fluent_bit_output_unifi | combine(fluent_bit_output_template_victorialogs) }}'
+
+fluent_bit_outputs: '{{ fluent_bit_main_outputs + fluent_bit_unifi_outputs }}'

group_vars/victoria-logs.yml

@@ -4,13 +4,15 @@ data_volumes:
   mountpoint: /var/lib/victoria-logs
 
 victoria_logs_extra_args:
-- '-syslog.listenAddr.tcp :601'
+- '-syslog.listenAddr.tcp=:601'
-- '-syslog.listenAddr.udp :514'
+- '-syslog.useRemoteIP.tcp=true'
-- '-syslog.extraFields.udp ''{}'''
+- '-syslog.listenAddr.udp=:514'
-- '-syslog.streamFields.udp ''["hostname","app_name","proc_id"]'''
+- '-syslog.useRemoteIP.udp=true'
-- '-syslog.listenAddr.udp :6666'
+- '-syslog.extraFields.udp=''{}'''
-- '-syslog.extraFields.udp ''{"stream":"netconsole"}'''
+- '-syslog.streamFields.udp=''["hostname","app_name","proc_id"]'''
-- '-syslog.streamFields.udp ''["stream"]'''
+- '-syslog.listenAddr.udp=:6666'
+- '-syslog.extraFields.udp=''{"stream":"netconsole"}'''
+- '-syslog.streamFields.udp=''["stream"]'''
 
 victoria_logs_publish_ports:
 - '514:514/udp'

host_vars/gw1.pyrocufflink.blue/main.yml

@@ -60,3 +60,24 @@ chrony_allow:
 - 172.24.100.0/24
 - 192.168.1.0/24
 - fd68:c2d2:500e:3e00::/56
+
+fluent_bit_ntfy_filters:
+- name: rewrite_tag
+  alias: ntfy
+  match: host.*
+  rule: transport kernel ntfy true
+- name: grep
+  match: ntfy
+  alias: ntfy.filter
+  regex: message wan.+Link
+
+fluent_bit_default_filters: >-
+  {{
+    fluent_bit_common_filters +
+    fluent_bit_ntfy_filters +
+    fluent_bit_ntfy_common_filters
+  }}
+
+fluent_bit_main_outputs:
+- '{{ fluent_bit_output_systemd }}'
+- '{{ fluent_bit_output_ntfy }}'

hosts

@@ -131,6 +131,10 @@ k8s-node
 [loki]
 loki1.pyrocufflink.blue
 
+[md-ntfy]
+chromie.pyrocufflink.blue
+nvr2.pyrocufflink.blue
+
 [minio-backups]
 chromie.pyrocufflink.blue
 

roles/fluent-bit/defaults/main.yml

@@ -1,5 +1,7 @@
 fluent_bit_config:
   service: '{{ fluent_bit_config_service }}'
+  parsers: '{{ fluent_bit_parsers }}'
+  multiline_parsers: '{{ fluent_bit_multiline_parsers }}'
   pipeline: '{{ fluent_bit_pipeline }}'
 
 fluent_bit_config_service:
@@ -24,6 +26,10 @@ fluent_bit_input_systemd:
   lowercase: true
   strip_underscores: true
 
+fluent_bit_parsers: []
+
+fluent_bit_multiline_parsers: []
+
 fluent_bit_filters: []
 
 fluent_bit_outputs:

roles/fluent-bit/files/fluent-bit.service

@@ -1,7 +1,6 @@
 [Unit]
 Description=Fluent Bit
 Documentation=https://docs.fluentbit.io/manual/
-Requires=network.target
 After=network.target
 StartLimitIntervalSec=5
 StartLimitBurst=5
@@ -10,6 +9,7 @@ StartLimitBurst=5
 Type=exec
 ExecStart=/usr/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml -Y
 ExecReload=/bin/kill -HUP $MAINPID
+Environment=HOSTNAME=%H
 StateDirectory=fluent-bit
 Restart=always
 RestartSec=1

roles/victoria-logs/defaults/main.yml

@@ -1,7 +1,7 @@
 # vim: set ft=yaml.jinja :
-victoria_logs_version: 1.23.3
+victoria_logs_version: 1.33.1
-victoria_logs_container_image_tag: v{{ victoria_logs_version }}-victorialogs
+victoria_logs_container_image_tag: v{{ victoria_logs_version }}
-victoria_logs_container_image_repo: docker.io/victoriametrics/victoria-logs
+victoria_logs_container_image_repo: quay.io/victoriametrics/victoria-logs
 victoria_logs_container_image: >-
   {{ victoria_logs_container_image_repo }}:{{ victoria_logs_container_image_tag }}