From 5dba0aec8f7a62d3782661656e2bdc4301b1b95c Mon Sep 17 00:00:00 2001
From: "Dustin C. Hatch" <dustin@hatch.name>
Date: Fri, 28 Nov 2025 21:25:19 -0600
Subject: [PATCH] fluent-bit: create configmap for kubernetes nodes

The last step in replacing Loki with Victoria Logs is to ingest logs
from Kubernetes pods.  Like Promtail, Fluent Bit is capable of
augmenting log records with Kubernetes metadata, so we can search for
logs by pod name, namespace, etc.  This of course requires access to the
Kubernetes API, and the easiest way to provide that is to run Fluent Bit
as a Kubernetes pod, granting its service account the appropriate
permissions.

Since Fluent Bit also collects logs from the systemd journal, I want to
make sure the configuration for that function stays the same on
Kubernetes nodes as on all other servers.  One way to do that would be
to run two different instances of Fluent Bit: one managed by Ansible
that collects journal messages, and another managed by Kubernetes that
collects pod logs.  This seems like unnecessary overhead, so I have
chosen a hybrid approach.  Ansible manages the configuration for the
process running in Kubernetes.
---
 fluent-bit-configmap.yml                      |  6 +++
 group_vars/kubelet.yml                        | 45 +++++++++++++++++++
 .../defaults/main.yml                         |  0
 roles/fluent-bit-configmap/handlers/main.yml  | 15 +++++++
 roles/fluent-bit-configmap/meta/main.yml      |  2 +
 roles/fluent-bit-configmap/tasks/main.yml     | 21 +++++++++
 roles/fluent-bit/meta/main.yml                |  1 +
 7 files changed, 90 insertions(+)
 create mode 100644 fluent-bit-configmap.yml
 rename roles/{fluent-bit => fluent-bit-config}/defaults/main.yml (100%)
 create mode 100644 roles/fluent-bit-configmap/handlers/main.yml
 create mode 100644 roles/fluent-bit-configmap/meta/main.yml
 create mode 100644 roles/fluent-bit-configmap/tasks/main.yml

diff --git a/fluent-bit-configmap.yml b/fluent-bit-configmap.yml
new file mode 100644
index 0000000..3795190
--- /dev/null
+++ b/fluent-bit-configmap.yml
@@ -0,0 +1,6 @@
+- hosts: kubelet
+  run_once: true
+  gather_facts: false
+  become: false
+  roles:
+  - role: fluent-bit-configmap
diff --git a/group_vars/kubelet.yml b/group_vars/kubelet.yml
index e972f39..8086101 100644
--- a/group_vars/kubelet.yml
+++ b/group_vars/kubelet.yml
@@ -23,3 +23,48 @@ container_registry_certs:
   docker-hub.proxy.pyrocufflink.blue:
     ca: >-
       {{ lookup('file', 'dch-root-ca-r2.crt') }}
+
+fluent_bit_parsers:
+- name: kube-tag
+  format: regex
+  regex: >-
+    (?<namespace_name>[^_]+)\.(?<pod_name>[^_]+)\.(?<pod_id>[^_]+)\.(?<container_name>[^_]+)$
+
+fluent_bit_input_kube:
+  name: tail
+  tag: kube.<namespace>.<pod>.<pod_id>.<container>
+  tag_regex: >-
+    ^/var/log/pods/(?<namespace>[^_]+)_(?<pod>[^_]+)_(?<pod_id>[^_]+)/(?<container>[^/]+)/[0-9]+\.log$
+  path: /var/log/pods/*/*/*.log
+  path_key: filename
+  multiline.parser: docker, cri
+  db: /var/lib/fluent-bit/kube.db
+  read_from_head: true
+
+fluent_bit_inputs: '{{ fluent_bit_default_inputs + [fluent_bit_input_kube] }}'
+
+fluent_bit_kube_filters:
+- name: kubernetes
+  match: kube.*
+  merge_log: true
+  kube_tag_prefix: kube.
+  regex_parser: kube-tag
+# Avoid log amplification from logging the result of sending logs!
+- name: grep
+  match: kube.*
+  exclude: log \[output:http:.+\] .+, HTTP status=200$
+
+fluent_bit_filters: '{{ fluent_bit_default_filters + fluent_bit_kube_filters }}'
+
+_fluent_bit_output_kube:
+  name: http
+  alias: kube
+  match: kube.*
+  uri: /insert/jsonline?_stream_fields=kubernetes.namespace_name,kubernetes.pod_name,kubernetes.container_name&_msg_field=log&_time_field=time
+
+fluent_bit_output_kube: >-
+  {{ _fluent_bit_output_kube | combine(fluent_bit_output_template_victorialogs) }}
+
+fluent_bit_outputs:
+- '{{ fluent_bit_output_systemd }}'
+- '{{ fluent_bit_output_kube }}'
diff --git a/roles/fluent-bit/defaults/main.yml b/roles/fluent-bit-config/defaults/main.yml
similarity index 100%
rename from roles/fluent-bit/defaults/main.yml
rename to roles/fluent-bit-config/defaults/main.yml
diff --git a/roles/fluent-bit-configmap/handlers/main.yml b/roles/fluent-bit-configmap/handlers/main.yml
new file mode 100644
index 0000000..376927a
--- /dev/null
+++ b/roles/fluent-bit-configmap/handlers/main.yml
@@ -0,0 +1,15 @@
+- name: restart fluent-bit daemonset
+  delegate_to: localhost
+  become: false
+  kubernetes.core.k8s:
+    api_version: apps/v1
+    kind: DaemonSet
+    namespace: fluent-bit
+    name: fluent-bit
+    definition:
+      spec:
+        template:
+          metadata:
+            annotations:
+              kubectl.kubernetes.io/restartedAt: >-
+                {{ '%Y-%m-%dT%H:%M:%S%z' | strftime }}
diff --git a/roles/fluent-bit-configmap/meta/main.yml b/roles/fluent-bit-configmap/meta/main.yml
new file mode 100644
index 0000000..2ae5597
--- /dev/null
+++ b/roles/fluent-bit-configmap/meta/main.yml
@@ -0,0 +1,2 @@
+dependencies:
+- role: fluent-bit-config
diff --git a/roles/fluent-bit-configmap/tasks/main.yml b/roles/fluent-bit-configmap/tasks/main.yml
new file mode 100644
index 0000000..ee44a47
--- /dev/null
+++ b/roles/fluent-bit-configmap/tasks/main.yml
@@ -0,0 +1,21 @@
+- name: ensure fluent-bit configmap is populated
+  delegate_to: localhost
+  become: false
+  kubernetes.core.k8s:
+    namespace: fluent-bit
+    definition:
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: fluent-bit
+        labels:
+          app.kubernetes.io/name: fluent-bit
+          app.kubernetes.io/component: fluent-bit
+          app.kubernetes.io/part-of: fluent-bit
+      data:
+        fluent-bit.yml: |+
+          {{ fluent_bit_config | to_nice_yaml(indent=2) }}
+  notify:
+  - restart fluent-bit daemonset
+  tags:
+  - configmap
diff --git a/roles/fluent-bit/meta/main.yml b/roles/fluent-bit/meta/main.yml
index bbb2a56..7acd67e 100644
--- a/roles/fluent-bit/meta/main.yml
+++ b/roles/fluent-bit/meta/main.yml
@@ -1,2 +1,3 @@
 dependencies:
+- role: fluent-bit-config
 - role: systemd-base