Dustin C. Hatch
414cb828e1
The Unifi Network server writes a bunch of log files that we need to forward to Victoria Logs. This commit introduces components to the Fluent Bit pipeline to read these files with the `tail` input plugin, parse them using regular expressions to extract the correct time stamp from the messages, and send them to Victoria Logs.
94 lines
2.3 KiB
YAML
94 lines
2.3 KiB
YAML
fluent_bit_multiline_parsers:
|
|
- name: multiline_unifi
|
|
type: regex
|
|
flush_timeout: 500
|
|
rules:
|
|
- state: start_state
|
|
regex: ^\[\d{4}-\d{2}-\d{2}T.*
|
|
next_state: cont
|
|
- state: cont
|
|
regex: ^[^\[].*
|
|
next_state: cont
|
|
- name: multiline_mongod
|
|
type: regex
|
|
flush_timeout: 500
|
|
rules:
|
|
- state: start_state
|
|
regex: ^\d{4}-\d{2}-\d{2}T.*
|
|
next_state: cont
|
|
- state: cont
|
|
regex: ^(?!\d{4}-\d{2}-\d{2}T).*
|
|
next_state: cont
|
|
|
|
unifi_server_logs:
|
|
- /var/log/unifi/migration.log
|
|
- /var/log/unifi/server.log
|
|
- /var/log/unifi/startup.log
|
|
- /var/log/unifi/state.log
|
|
- /var/log/unifi/tasks.log
|
|
|
|
fluent_bit_input_unifi:
|
|
name: tail
|
|
alias: unifi.server
|
|
tag: unifi.server
|
|
path: '{{ unifi_server_logs | join(",") }}'
|
|
path_key: filename
|
|
multiline.parser: multiline_unifi
|
|
db: /var/lib/fluent-bit/unifi.db
|
|
read_from_head: true
|
|
|
|
fluent_bit_input_unifi_mongod:
|
|
name: tail
|
|
alias: unifi.mongod
|
|
tag: unifi.mongod
|
|
path: /var/log/unifi/mongod.log
|
|
path_key: filename
|
|
multiline.parser: multiline_mongod
|
|
db: /var/lib/fluent-bit/unifi-mongod.db
|
|
read_from_head: true
|
|
|
|
fluent_bit_unifi_inputs:
|
|
- '{{ fluent_bit_input_unifi }}'
|
|
- '{{ fluent_bit_input_unifi_mongod }}'
|
|
|
|
fluent_bit_inputs: '{{ fluent_bit_default_inputs + fluent_bit_unifi_inputs }}'
|
|
|
|
fluent_bit_parsers:
|
|
- name: unifi-server
|
|
format: regex
|
|
regex: /^\[(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2},\d{3}([\+-]\d{4}|Z))\] (?<log>.*)/m
|
|
time_key: timestamp
|
|
time_format: '%Y-%m-%dT%H:%M:%S,%L%z'
|
|
- name: mongod
|
|
format: regex
|
|
regex: /^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}[\+-]\d{4}) (?<log>.*)/m
|
|
time_key: timestamp
|
|
time_format: '%Y-%m-%dT%H:%M:%S.%L%z'
|
|
|
|
fluent_bit_unifi_filters:
|
|
- name: parser
|
|
alias: unifi-server
|
|
match: unifi.server
|
|
key_name: log
|
|
parser: unifi-server
|
|
reserve_data: true
|
|
- name: parser
|
|
alias: unifi-mongod
|
|
match: unifi.mongod
|
|
key_name: log
|
|
parser: mongod
|
|
reserve_data: true
|
|
|
|
fluent_bit_filters: '{{ fluent_bit_default_filters + fluent_bit_unifi_filters }}'
|
|
|
|
fluent_bit_output_unifi:
|
|
name: http
|
|
alias: unifi
|
|
match: unifi.*
|
|
uri: /insert/jsonline?_stream_fields=hostname,filename&_msg_field=log&_time_field=date
|
|
|
|
fluent_bit_unifi_outputs:
|
|
- '{{ fluent_bit_output_unifi | combine(fluent_bit_output_template_victorialogs) }}'
|
|
|
|
fluent_bit_outputs: '{{ fluent_bit_main_outputs + fluent_bit_unifi_outputs }}'
|