diff --git a/fluent-bit-configmap.yml b/fluent-bit-configmap.yml new file mode 100644 index 0000000..3795190 --- /dev/null +++ b/fluent-bit-configmap.yml @@ -0,0 +1,6 @@ +- hosts: kubelet + run_once: true + gather_facts: false + become: false + roles: + - role: fluent-bit-configmap diff --git a/group_vars/kubelet.yml b/group_vars/kubelet.yml index e972f39..8086101 100644 --- a/group_vars/kubelet.yml +++ b/group_vars/kubelet.yml @@ -23,3 +23,48 @@ container_registry_certs: docker-hub.proxy.pyrocufflink.blue: ca: >- {{ lookup('file', 'dch-root-ca-r2.crt') }} + +fluent_bit_parsers: +- name: kube-tag + format: regex + regex: >- + (?[^_]+)\.(?[^_]+)\.(?[^_]+)\.(?[^_]+)$ + +fluent_bit_input_kube: + name: tail + tag: kube.... + tag_regex: >- + ^/var/log/pods/(?[^_]+)_(?[^_]+)_(?[^_]+)/(?[^/]+)/[0-9]+\.log$ + path: /var/log/pods/*/*/*.log + path_key: filename + multiline.parser: docker, cri + db: /var/lib/fluent-bit/kube.db + read_from_head: true + +fluent_bit_inputs: '{{ fluent_bit_default_inputs + [fluent_bit_input_kube] }}' + +fluent_bit_kube_filters: +- name: kubernetes + match: kube.* + merge_log: true + kube_tag_prefix: kube. + regex_parser: kube-tag +# Avoid log amplification from logging the result of sending logs! +- name: grep + match: kube.* + exclude: log \[output:http:.+\] .+, HTTP status=200$ + +fluent_bit_filters: '{{ fluent_bit_default_filters + fluent_bit_kube_filters }}' + +_fluent_bit_output_kube: + name: http + alias: kube + match: kube.* + uri: /insert/jsonline?_stream_fields=kubernetes.namespace_name,kubernetes.pod_name,kubernetes.container_name&_msg_field=log&_time_field=time + +fluent_bit_output_kube: >- + {{ _fluent_bit_output_kube | combine(fluent_bit_output_template_victorialogs) }} + +fluent_bit_outputs: +- '{{ fluent_bit_output_systemd }}' +- '{{ fluent_bit_output_kube }}' diff --git a/roles/fluent-bit/defaults/main.yml b/roles/fluent-bit-config/defaults/main.yml similarity index 100% rename from roles/fluent-bit/defaults/main.yml rename to roles/fluent-bit-config/defaults/main.yml diff --git a/roles/fluent-bit-configmap/handlers/main.yml b/roles/fluent-bit-configmap/handlers/main.yml new file mode 100644 index 0000000..376927a --- /dev/null +++ b/roles/fluent-bit-configmap/handlers/main.yml @@ -0,0 +1,15 @@ +- name: restart fluent-bit daemonset + delegate_to: localhost + become: false + kubernetes.core.k8s: + api_version: apps/v1 + kind: DaemonSet + namespace: fluent-bit + name: fluent-bit + definition: + spec: + template: + metadata: + annotations: + kubectl.kubernetes.io/restartedAt: >- + {{ '%Y-%m-%dT%H:%M:%S%z' | strftime }} diff --git a/roles/fluent-bit-configmap/meta/main.yml b/roles/fluent-bit-configmap/meta/main.yml new file mode 100644 index 0000000..2ae5597 --- /dev/null +++ b/roles/fluent-bit-configmap/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: +- role: fluent-bit-config diff --git a/roles/fluent-bit-configmap/tasks/main.yml b/roles/fluent-bit-configmap/tasks/main.yml new file mode 100644 index 0000000..ee44a47 --- /dev/null +++ b/roles/fluent-bit-configmap/tasks/main.yml @@ -0,0 +1,21 @@ +- name: ensure fluent-bit configmap is populated + delegate_to: localhost + become: false + kubernetes.core.k8s: + namespace: fluent-bit + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: fluent-bit + labels: + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/component: fluent-bit + app.kubernetes.io/part-of: fluent-bit + data: + fluent-bit.yml: |+ + {{ fluent_bit_config | to_nice_yaml(indent=2) }} + notify: + - restart fluent-bit daemonset + tags: + - configmap diff --git a/roles/fluent-bit/meta/main.yml b/roles/fluent-bit/meta/main.yml index bbb2a56..7acd67e 100644 --- a/roles/fluent-bit/meta/main.yml +++ b/roles/fluent-bit/meta/main.yml @@ -1,2 +1,3 @@ dependencies: +- role: fluent-bit-config - role: systemd-base