unifi: Configure Fluent Bit for Unifi server
The Unifi Network server writes a bunch of log files that we need to forward to Victoria Logs. This commit introduces components to the Fluent Bit pipeline to read these files with the `tail` input plugin, parse them using regular expressions to extract the correct time stamp from the messages, and send them to Victoria Logs.
This commit is contained in:
93
group_vars/unifi/fluent-bit.yml
Normal file
93
group_vars/unifi/fluent-bit.yml
Normal file
@@ -0,0 +1,93 @@
|
||||
fluent_bit_multiline_parsers:
|
||||
- name: multiline_unifi
|
||||
type: regex
|
||||
flush_timeout: 500
|
||||
rules:
|
||||
- state: start_state
|
||||
regex: ^\[\d{4}-\d{2}-\d{2}T.*
|
||||
next_state: cont
|
||||
- state: cont
|
||||
regex: ^[^\[].*
|
||||
next_state: cont
|
||||
- name: multiline_mongod
|
||||
type: regex
|
||||
flush_timeout: 500
|
||||
rules:
|
||||
- state: start_state
|
||||
regex: ^\d{4}-\d{2}-\d{2}T.*
|
||||
next_state: cont
|
||||
- state: cont
|
||||
regex: ^(?!\d{4}-\d{2}-\d{2}T).*
|
||||
next_state: cont
|
||||
|
||||
unifi_server_logs:
|
||||
- /var/log/unifi/migration.log
|
||||
- /var/log/unifi/server.log
|
||||
- /var/log/unifi/startup.log
|
||||
- /var/log/unifi/state.log
|
||||
- /var/log/unifi/tasks.log
|
||||
|
||||
fluent_bit_input_unifi:
|
||||
name: tail
|
||||
alias: unifi.server
|
||||
tag: unifi.server
|
||||
path: '{{ unifi_server_logs | join(",") }}'
|
||||
path_key: filename
|
||||
multiline.parser: multiline_unifi
|
||||
db: /var/lib/fluent-bit/unifi.db
|
||||
read_from_head: true
|
||||
|
||||
fluent_bit_input_unifi_mongod:
|
||||
name: tail
|
||||
alias: unifi.mongod
|
||||
tag: unifi.mongod
|
||||
path: /var/log/unifi/mongod.log
|
||||
path_key: filename
|
||||
multiline.parser: multiline_mongod
|
||||
db: /var/lib/fluent-bit/unifi-mongod.db
|
||||
read_from_head: true
|
||||
|
||||
fluent_bit_unifi_inputs:
|
||||
- '{{ fluent_bit_input_unifi }}'
|
||||
- '{{ fluent_bit_input_unifi_mongod }}'
|
||||
|
||||
fluent_bit_inputs: '{{ fluent_bit_default_inputs + fluent_bit_unifi_inputs }}'
|
||||
|
||||
fluent_bit_parsers:
|
||||
- name: unifi-server
|
||||
format: regex
|
||||
regex: /^\[(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2},\d{3}([\+-]\d{4}|Z))\] (?<log>.*)/m
|
||||
time_key: timestamp
|
||||
time_format: '%Y-%m-%dT%H:%M:%S,%L%z'
|
||||
- name: mongod
|
||||
format: regex
|
||||
regex: /^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}[\+-]\d{4}) (?<log>.*)/m
|
||||
time_key: timestamp
|
||||
time_format: '%Y-%m-%dT%H:%M:%S.%L%z'
|
||||
|
||||
fluent_bit_unifi_filters:
|
||||
- name: parser
|
||||
alias: unifi-server
|
||||
match: unifi.server
|
||||
key_name: log
|
||||
parser: unifi-server
|
||||
reserve_data: true
|
||||
- name: parser
|
||||
alias: unifi-mongod
|
||||
match: unifi.mongod
|
||||
key_name: log
|
||||
parser: mongod
|
||||
reserve_data: true
|
||||
|
||||
fluent_bit_filters: '{{ fluent_bit_default_filters + fluent_bit_unifi_filters }}'
|
||||
|
||||
fluent_bit_output_unifi:
|
||||
name: http
|
||||
alias: unifi
|
||||
match: unifi.*
|
||||
uri: /insert/jsonline?_stream_fields=hostname,filename&_msg_field=log&_time_field=date
|
||||
|
||||
fluent_bit_unifi_outputs:
|
||||
- '{{ fluent_bit_output_unifi | combine(fluent_bit_output_template_victorialogs) }}'
|
||||
|
||||
fluent_bit_outputs: '{{ fluent_bit_main_outputs + fluent_bit_unifi_outputs }}'
|
||||
Reference in New Issue
Block a user