fluent-bit: create configmap for kubernetes nodes

The last step in replacing Loki with Victoria Logs is to ingest logs
from Kubernetes pods.  Like Promtail, Fluent Bit is capable of
augmenting log records with Kubernetes metadata, so we can search for
logs by pod name, namespace, etc.  This of course requires access to the
Kubernetes API, and the easiest way to provide that is to run Fluent Bit
as a Kubernetes pod, granting its service account the appropriate
permissions.

Since Fluent Bit also collects logs from the systemd journal, I want to
make sure the configuration for that function stays the same on
Kubernetes nodes as on all other servers.  One way to do that would be
to run two different instances of Fluent Bit: one managed by Ansible
that collects journal messages, and another managed by Kubernetes that
collects pod logs.  This seems like unnecessary overhead, so I have
chosen a hybrid approach.  Ansible manages the configuration for the
process running in Kubernetes.

fluent-bit-configmap.yml

@@ -0,0 +1,6 @@
+- hosts: kubelet
+  run_once: true
+  gather_facts: false
+  become: false
+  roles:
+  - role: fluent-bit-configmap

group_vars/kubelet.yml

@@ -23,3 +23,48 @@ container_registry_certs:
   docker-hub.proxy.pyrocufflink.blue:
     ca: >-
       {{ lookup('file', 'dch-root-ca-r2.crt') }}
+
+fluent_bit_parsers:
+- name: kube-tag
+  format: regex
+  regex: >-
+    (?<namespace_name>[^_]+)\.(?<pod_name>[^_]+)\.(?<pod_id>[^_]+)\.(?<container_name>[^_]+)$
+
+fluent_bit_input_kube:
+  name: tail
+  tag: kube.<namespace>.<pod>.<pod_id>.<container>
+  tag_regex: >-
+    ^/var/log/pods/(?<namespace>[^_]+)_(?<pod>[^_]+)_(?<pod_id>[^_]+)/(?<container>[^/]+)/[0-9]+\.log$
+  path: /var/log/pods/*/*/*.log
+  path_key: filename
+  multiline.parser: docker, cri
+  db: /var/lib/fluent-bit/kube.db
+  read_from_head: true
+
+fluent_bit_inputs: '{{ fluent_bit_default_inputs + [fluent_bit_input_kube] }}'
+
+fluent_bit_kube_filters:
+- name: kubernetes
+  match: kube.*
+  merge_log: true
+  kube_tag_prefix: kube.
+  regex_parser: kube-tag
+# Avoid log amplification from logging the result of sending logs!
+- name: grep
+  match: kube.*
+  exclude: log \[output:http:.+\] .+, HTTP status=200$
+
+fluent_bit_filters: '{{ fluent_bit_default_filters + fluent_bit_kube_filters }}'
+
+_fluent_bit_output_kube:
+  name: http
+  alias: kube
+  match: kube.*
+  uri: /insert/jsonline?_stream_fields=kubernetes.namespace_name,kubernetes.pod_name,kubernetes.container_name&_msg_field=log&_time_field=time
+
+fluent_bit_output_kube: >-
+  {{ _fluent_bit_output_kube | combine(fluent_bit_output_template_victorialogs) }}
+
+fluent_bit_outputs:
+- '{{ fluent_bit_output_systemd }}'
+- '{{ fluent_bit_output_kube }}'

roles/fluent-bit-configmap/handlers/main.yml

@@ -0,0 +1,15 @@
+- name: restart fluent-bit daemonset
+  delegate_to: localhost
+  become: false
+  kubernetes.core.k8s:
+    api_version: apps/v1
+    kind: DaemonSet
+    namespace: fluent-bit
+    name: fluent-bit
+    definition:
+      spec:
+        template:
+          metadata:
+            annotations:
+              kubectl.kubernetes.io/restartedAt: >-
+                {{ '%Y-%m-%dT%H:%M:%S%z' | strftime }}

roles/fluent-bit-configmap/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+- role: fluent-bit-config

roles/fluent-bit-configmap/tasks/main.yml

@@ -0,0 +1,21 @@
+- name: ensure fluent-bit configmap is populated
+  delegate_to: localhost
+  become: false
+  kubernetes.core.k8s:
+    namespace: fluent-bit
+    definition:
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: fluent-bit
+        labels:
+          app.kubernetes.io/name: fluent-bit
+          app.kubernetes.io/component: fluent-bit
+          app.kubernetes.io/part-of: fluent-bit
+      data:
+        fluent-bit.yml: |+
+          {{ fluent_bit_config | to_nice_yaml(indent=2) }}
+  notify:
+  - restart fluent-bit daemonset
+  tags:
+  - configmap

roles/fluent-bit/meta/main.yml

@@ -1,2 +1,3 @@
 dependencies:
+- role: fluent-bit-config
 - role: systemd-base