diff --git a/group_vars/all.yml b/group_vars/all.yml index e34d219..c76dfed 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -142,19 +142,7 @@ dnf_automatic_schedule: >- | string }} *-*-* 04:00:00 America/Chicago -fluent_bit_filters: -# Avoid log amplification from logging the result of sending logs! -- name: grep - match: host.fluent-bit.service - exclude: message \[output:http:victorialogs\] .+, HTTP status=200$ -- name: rewrite_tag - alias: ntfy - match: host.* - rule: transport kernel ntfy true -- name: grep - match: ntfy - alias: ntfy.filter - regex: message ^md +fluent_bit_ntfy_common_filters: - name: lua alias: ntfy.populate match: ntfy @@ -175,21 +163,43 @@ fluent_bit_filters: - message - tags - topic -fluent_bit_outputs: -- name: http - alias: victorialogs - match: host.* + +fluent_bit_common_filters: +- name: record_modifier + match: '*' + record: + - hostname ${HOSTNAME} +# Avoid log amplification from logging the result of sending logs! +- name: grep + match: host.fluent-bit.service + exclude: message \[output:http:.+\] .+, HTTP status=200$ + +fluent_bit_default_filters: '{{ fluent_bit_common_filters }}' + +fluent_bit_filters: '{{ fluent_bit_default_filters }}' + +fluent_bit_output_template_victorialogs: host: logs.pyrocufflink.blue port: 443 tls: true tls.verify: true tls.verify_hostname: true tls.ca_file: /etc/pki/ca-trust/source/anchors/dch-root-ca-r2.crt - uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date format: json_lines json_date_format: iso8601 log_response_payload: false -- name: http + +_fluent_bit_output_systemd: + name: http + alias: victorialogs + match: host.* + uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date + +fluent_bit_output_systemd: >- + {{ _fluent_bit_output_systemd | combine(fluent_bit_output_template_victorialogs) }} + +fluent_bit_output_ntfy: + name: http alias: ntfy workers: 1 match: ntfy @@ -202,3 +212,8 @@ fluent_bit_outputs: format: json_lines json_date_key: false log_response_payload: false + +fluent_bit_main_outputs: +- '{{ fluent_bit_output_systemd }}' + +fluent_bit_outputs: '{{ fluent_bit_main_outputs }}' diff --git a/group_vars/md-ntfy.yml b/group_vars/md-ntfy.yml new file mode 100644 index 0000000..f5a0332 --- /dev/null +++ b/group_vars/md-ntfy.yml @@ -0,0 +1,20 @@ +fluent_bit_ntfy_filters: +- name: rewrite_tag + alias: ntfy + match: host.* + rule: transport kernel ntfy true +- name: grep + match: ntfy + alias: ntfy.filter + regex: message ^md + +fluent_bit_default_filters: >- + {{ + fluent_bit_common_filters + + fluent_bit_ntfy_filters + + fluent_bit_ntfy_common_filters + }} + +fluent_bit_main_outputs: +- '{{ fluent_bit_output_systemd }}' +- '{{ fluent_bit_output_ntfy }}' diff --git a/group_vars/unifi/fluent-bit.yml b/group_vars/unifi/fluent-bit.yml new file mode 100644 index 0000000..39c4008 --- /dev/null +++ b/group_vars/unifi/fluent-bit.yml @@ -0,0 +1,93 @@ +fluent_bit_multiline_parsers: +- name: multiline_unifi + type: regex + flush_timeout: 500 + rules: + - state: start_state + regex: ^\[\d{4}-\d{2}-\d{2}T.* + next_state: cont + - state: cont + regex: ^[^\[].* + next_state: cont +- name: multiline_mongod + type: regex + flush_timeout: 500 + rules: + - state: start_state + regex: ^\d{4}-\d{2}-\d{2}T.* + next_state: cont + - state: cont + regex: ^(?!\d{4}-\d{2}-\d{2}T).* + next_state: cont + +unifi_server_logs: +- /var/log/unifi/migration.log +- /var/log/unifi/server.log +- /var/log/unifi/startup.log +- /var/log/unifi/state.log +- /var/log/unifi/tasks.log + +fluent_bit_input_unifi: + name: tail + alias: unifi.server + tag: unifi.server + path: '{{ unifi_server_logs | join(",") }}' + path_key: filename + multiline.parser: multiline_unifi + db: /var/lib/fluent-bit/unifi.db + read_from_head: true + +fluent_bit_input_unifi_mongod: + name: tail + alias: unifi.mongod + tag: unifi.mongod + path: /var/log/unifi/mongod.log + path_key: filename + multiline.parser: multiline_mongod + db: /var/lib/fluent-bit/unifi-mongod.db + read_from_head: true + +fluent_bit_unifi_inputs: +- '{{ fluent_bit_input_unifi }}' +- '{{ fluent_bit_input_unifi_mongod }}' + +fluent_bit_inputs: '{{ fluent_bit_default_inputs + fluent_bit_unifi_inputs }}' + +fluent_bit_parsers: +- name: unifi-server + format: regex + regex: /^\[(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2},\d{3}([\+-]\d{4}|Z))\] (?.*)/m + time_key: timestamp + time_format: '%Y-%m-%dT%H:%M:%S,%L%z' +- name: mongod + format: regex + regex: /^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}[\+-]\d{4}) (?.*)/m + time_key: timestamp + time_format: '%Y-%m-%dT%H:%M:%S.%L%z' + +fluent_bit_unifi_filters: +- name: parser + alias: unifi-server + match: unifi.server + key_name: log + parser: unifi-server + reserve_data: true +- name: parser + alias: unifi-mongod + match: unifi.mongod + key_name: log + parser: mongod + reserve_data: true + +fluent_bit_filters: '{{ fluent_bit_default_filters + fluent_bit_unifi_filters }}' + +fluent_bit_output_unifi: + name: http + alias: unifi + match: unifi.* + uri: /insert/jsonline?_stream_fields=hostname,filename&_msg_field=log&_time_field=date + +fluent_bit_unifi_outputs: +- '{{ fluent_bit_output_unifi | combine(fluent_bit_output_template_victorialogs) }}' + +fluent_bit_outputs: '{{ fluent_bit_main_outputs + fluent_bit_unifi_outputs }}' diff --git a/group_vars/victoria-logs.yml b/group_vars/victoria-logs.yml index 1bf0b20..370b708 100644 --- a/group_vars/victoria-logs.yml +++ b/group_vars/victoria-logs.yml @@ -4,13 +4,15 @@ data_volumes: mountpoint: /var/lib/victoria-logs victoria_logs_extra_args: -- '-syslog.listenAddr.tcp :601' -- '-syslog.listenAddr.udp :514' -- '-syslog.extraFields.udp ''{}''' -- '-syslog.streamFields.udp ''["hostname","app_name","proc_id"]''' -- '-syslog.listenAddr.udp :6666' -- '-syslog.extraFields.udp ''{"stream":"netconsole"}''' -- '-syslog.streamFields.udp ''["stream"]''' +- '-syslog.listenAddr.tcp=:601' +- '-syslog.useRemoteIP.tcp=true' +- '-syslog.listenAddr.udp=:514' +- '-syslog.useRemoteIP.udp=true' +- '-syslog.extraFields.udp=''{}''' +- '-syslog.streamFields.udp=''["hostname","app_name","proc_id"]''' +- '-syslog.listenAddr.udp=:6666' +- '-syslog.extraFields.udp=''{"stream":"netconsole"}''' +- '-syslog.streamFields.udp=''["stream"]''' victoria_logs_publish_ports: - '514:514/udp' diff --git a/host_vars/gw1.pyrocufflink.blue/main.yml b/host_vars/gw1.pyrocufflink.blue/main.yml index cdcfdf8..b1d1e85 100644 --- a/host_vars/gw1.pyrocufflink.blue/main.yml +++ b/host_vars/gw1.pyrocufflink.blue/main.yml @@ -60,3 +60,24 @@ chrony_allow: - 172.24.100.0/24 - 192.168.1.0/24 - fd68:c2d2:500e:3e00::/56 + +fluent_bit_ntfy_filters: +- name: rewrite_tag + alias: ntfy + match: host.* + rule: transport kernel ntfy true +- name: grep + match: ntfy + alias: ntfy.filter + regex: message wan.+Link + +fluent_bit_default_filters: >- + {{ + fluent_bit_common_filters + + fluent_bit_ntfy_filters + + fluent_bit_ntfy_common_filters + }} + +fluent_bit_main_outputs: +- '{{ fluent_bit_output_systemd }}' +- '{{ fluent_bit_output_ntfy }}' diff --git a/hosts b/hosts index 178a8cb..83bf22d 100644 --- a/hosts +++ b/hosts @@ -131,6 +131,10 @@ k8s-node [loki] loki1.pyrocufflink.blue +[md-ntfy] +chromie.pyrocufflink.blue +nvr2.pyrocufflink.blue + [minio-backups] chromie.pyrocufflink.blue diff --git a/roles/fluent-bit/defaults/main.yml b/roles/fluent-bit/defaults/main.yml index 2093a8e..b50d6f4 100644 --- a/roles/fluent-bit/defaults/main.yml +++ b/roles/fluent-bit/defaults/main.yml @@ -1,5 +1,7 @@ fluent_bit_config: service: '{{ fluent_bit_config_service }}' + parsers: '{{ fluent_bit_parsers }}' + multiline_parsers: '{{ fluent_bit_multiline_parsers }}' pipeline: '{{ fluent_bit_pipeline }}' fluent_bit_config_service: @@ -24,6 +26,10 @@ fluent_bit_input_systemd: lowercase: true strip_underscores: true +fluent_bit_parsers: [] + +fluent_bit_multiline_parsers: [] + fluent_bit_filters: [] fluent_bit_outputs: diff --git a/roles/fluent-bit/files/fluent-bit.service b/roles/fluent-bit/files/fluent-bit.service index c2b1b6f..fa9a4ec 100644 --- a/roles/fluent-bit/files/fluent-bit.service +++ b/roles/fluent-bit/files/fluent-bit.service @@ -1,7 +1,6 @@ [Unit] Description=Fluent Bit Documentation=https://docs.fluentbit.io/manual/ -Requires=network.target After=network.target StartLimitIntervalSec=5 StartLimitBurst=5 @@ -10,6 +9,7 @@ StartLimitBurst=5 Type=exec ExecStart=/usr/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml -Y ExecReload=/bin/kill -HUP $MAINPID +Environment=HOSTNAME=%H StateDirectory=fluent-bit Restart=always RestartSec=1 diff --git a/roles/victoria-logs/defaults/main.yml b/roles/victoria-logs/defaults/main.yml index 7a26837..09c11ac 100644 --- a/roles/victoria-logs/defaults/main.yml +++ b/roles/victoria-logs/defaults/main.yml @@ -1,7 +1,7 @@ # vim: set ft=yaml.jinja : -victoria_logs_version: 1.23.3 -victoria_logs_container_image_tag: v{{ victoria_logs_version }}-victorialogs -victoria_logs_container_image_repo: docker.io/victoriametrics/victoria-logs +victoria_logs_version: 1.33.1 +victoria_logs_container_image_tag: v{{ victoria_logs_version }} +victoria_logs_container_image_repo: quay.io/victoriametrics/victoria-logs victoria_logs_container_image: >- {{ victoria_logs_container_image_repo }}:{{ victoria_logs_container_image_tag }}