fluent-bit: Deploy log collector for Victoria Logs

[fluent-bit][0] is a generic, highly-configurable log collector.  It was
apparently initially developed for fluentd, but is has so many output
capabilities that it works wil many different log aggregation systems,
including Victoria Logs.

Although Victoria Logs supports the Loki input format, and therefore
_Promtail_ would work, I want to try to avoid depending on third-party
repositories.  _fluent-bit_ is packaged by Fedora, so there shouldn't be
any dependency issues, etc.

[0]: https://fluentbit.io

fluent-bit.yml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+  - role: fluent-bit
+    tags: fluent-bit

group_vars/all.yml

@@ -141,3 +141,23 @@ dnf_automatic_schedule: >-
      | random(seed=inventory_hostname)
      | string
   }} *-*-* 04:00:00 America/Chicago
+
+fluent_bit_filters:
+# Avoid log amplification from logging the result of sending logs!
+- name: grep
+  match: host.fluent-bit.service
+  exclude: message \[output:http:victorialogs\] .+, HTTP status=200$
+fluent_bit_outputs:
+- name: http
+  alias: victorialogs
+  match: host.*
+  host: logs.pyrocufflink.blue
+  port: 443
+  tls: true
+  tls.verify: true
+  tls.verify_hostname: true
+  tls.ca_file: /etc/pki/ca-trust/source/anchors/dch-root-ca-r2.crt
+  uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date
+  format: json_lines
+  json_date_format: iso8601
+  log_response_payload: false

host-setup.yml

@@ -2,6 +2,7 @@
 
 - import_playbook: users.yml
 - import_playbook: collectd.yml
-- import_playbook: promtail.yml
+- import_playbook: dch-root-ca.yml
+- import_playbook: fluent-bit.yml
 - import_playbook: auto-updates.yml
 - import_playbook: datavol.yml

roles/fluent-bit/defaults/main.yml

@@ -0,0 +1,34 @@
+fluent_bit_config:
+  service: '{{ fluent_bit_config_service }}'
+  pipeline: '{{ fluent_bit_pipeline }}'
+
+fluent_bit_config_service:
+  log_level: '{{ fluent_bit_log_level }}'
+
+fluent_bit_log_level: info
+
+fluent_bit_pipeline:
+  inputs: '{{ fluent_bit_inputs }}'
+  filters: '{{ fluent_bit_filters }}'
+  outputs: '{{ fluent_bit_outputs }}'
+
+fluent_bit_inputs: '{{ fluent_bit_default_inputs }}'
+
+fluent_bit_default_inputs:
+- '{{ fluent_bit_input_systemd }}'
+
+fluent_bit_input_systemd:
+  name: systemd
+  tag: host.*
+  db: /var/lib/fluent-bit/journal
+  lowercase: true
+  strip_underscores: true
+
+fluent_bit_filters: []
+
+fluent_bit_outputs:
+- '{{ fluent_bit_null_output }}'
+
+fluent_bit_null_output:
+  name: null
+  match: '*'

roles/fluent-bit/files/fluent-bit.service

@@ -0,0 +1,36 @@
+[Unit]
+Description=Fluent Bit
+Documentation=https://docs.fluentbit.io/manual/
+Requires=network.target
+After=network.target
+StartLimitIntervalSec=5
+StartLimitBurst=5
+
+[Service]
+Type=exec
+ExecStart=/usr/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml -Y
+ExecReload=/bin/kill -HUP $MAINPID
+StateDirectory=fluent-bit
+Restart=always
+RestartSec=1
+BindPaths=%S/fluent-bit
+CapabilityBoundingSet=CAP_DAC_READ_SEARCH
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+PrivateDevices=yes
+PrivateTmp=yes
+ProtectControlGroups=yes
+ProtectHome=yes
+ProtectHostname=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+ProtectSystem=strict
+ReadOnlyPaths=/var/log
+ReadWritePaths=%S/fluent-bit
+RestrictNamespaces=yes
+RestrictRealtime=yes
+SystemCallArchitectures=native
+TemporaryFileSystem=%S:ro
+
+[Install]
+WantedBy=multi-user.target

roles/fluent-bit/handlers/main.yml

@@ -0,0 +1,9 @@
+- name: restart fluent-bit
+  service:
+    name: fluent-bit
+    state: restarted
+
+- name: reload fluent-bit
+  service:
+    name: fluent-bit
+    state: reloaded

roles/fluent-bit/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+- role: systemd-base

roles/fluent-bit/tasks/main.yml

@@ -0,0 +1,53 @@
+- name: ensure fluent-bit is installed
+  package:
+    name: fluent-bit
+    state: present
+  tags:
+  - install
+
+- name: ensure fluent-bit is configured
+  copy:
+    dest: /etc/fluent-bit/fluent-bit.yml
+    content: '{{ fluent_bit_config | to_nice_yaml(indent=2) }}'
+    owner: root
+    group: root
+    mode: u=rw,go=
+  notify:
+  - restart fluent-bit
+  tags:
+  - config
+
+# The default unit configuration for fluent-bit.service sucks.  It runs
+# as root without any kind of restrictions or sandboxing, forces the
+# "classic" configuration format (which is deprecated in favor of
+# YAML), and does not support hot reload.  It's very simple, so we can
+# replace it completely without too much worry about upstream changes.
+- name: ensure custom fluent-bit systemd service unit file is installed
+  copy:
+    src: fluent-bit.service
+    dest: /etc/systemd/system/fluent-bit.service
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - reload systemd
+  - restart fluent-bit
+  tags:
+  - systemd
+
+- name: ensure fluent-bit starts at boot
+  service:
+    name: fluent-bit
+    enabled: true
+  tags:
+  - service
+
+- name: flush handlers
+  meta: flush_handlers
+
+- name: ensure fluent-bit is running
+  service:
+    name: fluent-bit
+    state: started
+  tags:
+  - service