Merge remote-tracking branch 'refs/remotes/origin/master'

fluent-bit.yml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+  - role: fluent-bit
+    tags: fluent-bit

group_vars/all.yml

@@ -141,3 +141,64 @@ dnf_automatic_schedule: >-
      | random(seed=inventory_hostname)
      | string
   }} *-*-* 04:00:00 America/Chicago
+
+fluent_bit_filters:
+# Avoid log amplification from logging the result of sending logs!
+- name: grep
+  match: host.fluent-bit.service
+  exclude: message \[output:http:victorialogs\] .+, HTTP status=200$
+- name: rewrite_tag
+  alias: ntfy
+  match: host.*
+  rule: transport kernel ntfy true
+- name: grep
+  match: ntfy
+  alias: ntfy.filter
+  regex: message ^md
+- name: lua
+  alias: ntfy.populate
+  match: ntfy
+  code: |
+    function ntfy_transform(tag, timestamp, record)
+        record["topic"] = "alerts"
+        record["tags"] = {
+          record["hostname"],
+          record["syslog_identifier"],
+        }
+        return 1, timestamp, record
+    end
+  call: ntfy_transform
+- name: record_modifier
+  alias: ntfy.clean
+  match: ntfy
+  allowlist_key:
+  - message
+  - tags
+  - topic
+fluent_bit_outputs:
+- name: http
+  alias: victorialogs
+  match: host.*
+  host: logs.pyrocufflink.blue
+  port: 443
+  tls: true
+  tls.verify: true
+  tls.verify_hostname: true
+  tls.ca_file: /etc/pki/ca-trust/source/anchors/dch-root-ca-r2.crt
+  uri: /insert/jsonline?_stream_fields=hostname,systemd_unit&_msg_field=message&_time_field=date
+  format: json_lines
+  json_date_format: iso8601
+  log_response_payload: false
+- name: http
+  alias: ntfy
+  workers: 1
+  match: ntfy
+  host: ntfy.pyrocufflink.blue
+  port: 443
+  tls: true
+  tls.verify: true
+  tls.verify_hostname: true
+  uri: /
+  format: json_lines
+  json_date_key: false
+  log_response_payload: false

host-setup.yml

@@ -2,6 +2,7 @@
 
 - import_playbook: users.yml
 - import_playbook: collectd.yml
-- import_playbook: promtail.yml
+- import_playbook: dch-root-ca.yml
+- import_playbook: fluent-bit.yml
 - import_playbook: auto-updates.yml
 - import_playbook: datavol.yml

raid-array.yml

@@ -29,14 +29,35 @@
     tags:
     - mdadm
     - mdadm-create
+  - name: ensure md-auto-readd script is installed
+    copy:
+      content: |+
+        #!/bin/sh
+
+        main() {
+            md_dev=$(readlink -e /dev/disk/by-id/md-name-"$2")
+            if [ -z "${md_dev}" ]; then
+                printf 'Could not find block device for MD raid %s\n' "$2" >&2
+                exit 1
+            fi
+            exec mdadm --re-add "${md_dev}" "$1"
+        }
+
+        main "$@" 2>&1 | logger
+      dest: /usr/local/libexec/md-auto-readd
+      owner: root
+      group: root
+      mode: u=rwx,go=rx
+    tags:
+    - script
   - name: ensure raid auto re-add udev rule exists
     copy:
       content: >+
-        ENV{ID_FS_LABEL}=="{{ md_name }}",
+        ACTION=="add",
         ENV{ID_FS_TYPE}=="linux_raid_member",
         ENV{ID_FS_USAGE}=="raid",
-        RUN+="/usr/sbin/mdadm --re-add /dev/disk/by-id/md-name-{{ md_name }} $devnode"
-      dest: /etc/udev/rules.d/80-{{ md_name }}.rules
+        RUN+="/usr/local/libexec/md-auto-readd $devnode $env{ID_FS_LABEL}"
+      dest: /etc/udev/rules.d/80-md-auto-readd.rules
       mode: u=rw,go=r
       owner: root
       group: root

roles/fluent-bit/defaults/main.yml

@@ -0,0 +1,34 @@
+fluent_bit_config:
+  service: '{{ fluent_bit_config_service }}'
+  pipeline: '{{ fluent_bit_pipeline }}'
+
+fluent_bit_config_service:
+  log_level: '{{ fluent_bit_log_level }}'
+
+fluent_bit_log_level: info
+
+fluent_bit_pipeline:
+  inputs: '{{ fluent_bit_inputs }}'
+  filters: '{{ fluent_bit_filters }}'
+  outputs: '{{ fluent_bit_outputs }}'
+
+fluent_bit_inputs: '{{ fluent_bit_default_inputs }}'
+
+fluent_bit_default_inputs:
+- '{{ fluent_bit_input_systemd }}'
+
+fluent_bit_input_systemd:
+  name: systemd
+  tag: host.*
+  db: /var/lib/fluent-bit/journal
+  lowercase: true
+  strip_underscores: true
+
+fluent_bit_filters: []
+
+fluent_bit_outputs:
+- '{{ fluent_bit_null_output }}'
+
+fluent_bit_null_output:
+  name: null
+  match: '*'

roles/fluent-bit/files/fluent-bit.service

@@ -0,0 +1,36 @@
+[Unit]
+Description=Fluent Bit
+Documentation=https://docs.fluentbit.io/manual/
+Requires=network.target
+After=network.target
+StartLimitIntervalSec=5
+StartLimitBurst=5
+
+[Service]
+Type=exec
+ExecStart=/usr/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yml -Y
+ExecReload=/bin/kill -HUP $MAINPID
+StateDirectory=fluent-bit
+Restart=always
+RestartSec=1
+BindPaths=%S/fluent-bit
+CapabilityBoundingSet=CAP_DAC_READ_SEARCH
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+PrivateDevices=yes
+PrivateTmp=yes
+ProtectControlGroups=yes
+ProtectHome=yes
+ProtectHostname=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+ProtectSystem=strict
+ReadOnlyPaths=/var/log
+ReadWritePaths=%S/fluent-bit
+RestrictNamespaces=yes
+RestrictRealtime=yes
+SystemCallArchitectures=native
+TemporaryFileSystem=%S:ro
+
+[Install]
+WantedBy=multi-user.target

roles/fluent-bit/handlers/main.yml

@@ -0,0 +1,9 @@
+- name: restart fluent-bit
+  service:
+    name: fluent-bit
+    state: restarted
+
+- name: reload fluent-bit
+  service:
+    name: fluent-bit
+    state: reloaded

roles/fluent-bit/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+- role: systemd-base

roles/fluent-bit/tasks/main.yml

@@ -0,0 +1,53 @@
+- name: ensure fluent-bit is installed
+  package:
+    name: fluent-bit
+    state: present
+  tags:
+  - install
+
+- name: ensure fluent-bit is configured
+  copy:
+    dest: /etc/fluent-bit/fluent-bit.yml
+    content: '{{ fluent_bit_config | to_nice_yaml(indent=2) }}'
+    owner: root
+    group: root
+    mode: u=rw,go=
+  notify:
+  - restart fluent-bit
+  tags:
+  - config
+
+# The default unit configuration for fluent-bit.service sucks.  It runs
+# as root without any kind of restrictions or sandboxing, forces the
+# "classic" configuration format (which is deprecated in favor of
+# YAML), and does not support hot reload.  It's very simple, so we can
+# replace it completely without too much worry about upstream changes.
+- name: ensure custom fluent-bit systemd service unit file is installed
+  copy:
+    src: fluent-bit.service
+    dest: /etc/systemd/system/fluent-bit.service
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - reload systemd
+  - restart fluent-bit
+  tags:
+  - systemd
+
+- name: ensure fluent-bit starts at boot
+  service:
+    name: fluent-bit
+    enabled: true
+  tags:
+  - service
+
+- name: flush handlers
+  meta: flush_handlers
+
+- name: ensure fluent-bit is running
+  service:
+    name: fluent-bit
+    state: started
+  tags:
+  - service