promtail: Deploy as DaemonSet

Running Promtail in a pod controlled by a DaemonSet allows it to access
the Kubernetes API via a ServiceAccount token.  Since it needs the API
in order to discover the Pods running on the current node in order to
find their log files, this makes the authentication process a lot
simpler.

promtail/config.yml

@@ -0,0 +1,111 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+  enable_runtime_reload: true
+
+clients:
+- url: https://loki.pyrocufflink.blue/loki/api/v1/push
+  tls_config:
+    ca_file: /run/dch-ca/dch-root-ca.crt
+
+positions:
+  filename: /var/lib/promtail/positions
+
+scrape_configs:
+- job_name: journal
+  journal:
+    json: false
+    labels:
+      job: systemd-journal
+  relabel_configs:
+  - source_labels:
+    - __journal__hostname
+    target_label: hostname
+  - source_labels:
+    - __journal__systemd_unit
+    target_label: unit
+  - source_labels:
+    - __journal_syslog_identifier
+    target_label: syslog_identifier
+  - source_labels:
+    - __journal_priority
+    target_label: priority
+  - source_labels:
+    - __journal_message_id
+    target_label: message_id
+  - source_labels:
+    - __journal__comm
+    target_label: command
+  - source_labels:
+    - __journal__transport
+    target_label: transport
+
+- job_name: pods
+  kubernetes_sd_configs:
+  - role: pod
+  pipeline_stages:
+  - cri: {}
+  relabel_configs:
+  # Magic label: tell Promtail to filter out pods that are not running locally
+  - source_labels: [__meta_kubernetes_pod_node_name]
+    target_label: __host__
+  - target_label: job
+    replacement: kubernetes-pods
+  # Build the log file path:
+  #   /var/log/pods/{namespace}_{pod_name}_{pod_uid}/{container_name}/*.log
+  - source_labels:
+    - __meta_kubernetes_namespace
+    - __meta_kubernetes_pod_name
+    - __meta_kubernetes_pod_uid
+    separator: _
+    target_label: __path__
+    replacement: /var/log/pods/$1
+  - source_labels:
+    - __path__
+    - __meta_kubernetes_pod_container_name
+    separator: /
+    target_label: __path__
+    replacement: '$1/*.log'
+  - source_labels: [__meta_kubernetes_pod_node_name]
+    target_label: node_name
+  - source_labels: [__meta_kubernetes_namespace]
+    target_label: namespace
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: pod
+  - source_labels: [__meta_kubernetes_pod_container_name]
+    target_label: container
+  - source_labels: [__meta_kubernetes_pod_controller_name]
+    regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})?
+    action: replace
+    target_label: __tmp_controller_name
+  # Set `app` to the first non-empty label from
+  # - app.kubernetes.io/name
+  # - app
+  # If none present, use the pod controller (e.g. Deployment) name.
+  # Fall back to pod name if none found.
+  - source_labels:
+    - __meta_kubernetes_pod_label_app_kubernetes_io_name
+    - __meta_kubernetes_pod_label_app
+    - __tmp_controller_name
+    - __meta_kubernetes_pod_name
+    regex: ^;*([^;]+)(;.*)?$
+    action: replace
+    target_label: app
+  # Set `instance` to the first non-empty label from
+  # - app.kubernetes.io/instance
+  # - instance
+  - source_labels:
+    - __meta_kubernetes_pod_label_app_kubernetes_io_instance
+    - __meta_kubernetes_pod_label_instance
+    regex: ^;*([^;]+)(;.*)?$
+    action: replace
+    target_label: instance
+  # Set `component` to the first non-empty label from
+  # - app.kubernetes.io/component
+  # - component
+  - source_labels:
+    - __meta_kubernetes_pod_label_app_kubernetes_io_component
+    - __meta_kubernetes_pod_label_component
+    regex: ^;*([^;]+)(;.*)?$
+    action: replace
+    target_label: component

promtail/kustomization.yaml

@@ -0,0 +1,41 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: promtail
+
+labels:
+- pairs:
+    app.kubernetes.io/instance: promtail
+    app.kubernetes.io/part-of: promtail
+  includeSelectors: false
+
+resources:
+- namespace.yaml
+- promtail.yaml
+- ../dch-root-ca
+
+configMapGenerator:
+- name: promtail
+  files:
+  - config.yml
+
+patches:
+- patch: |
+    apiVersion: apps/v1
+    kind: DaemonSet
+    metadata:
+      name: promtail
+    spec:
+      template:
+        spec:
+          containers:
+          - name: promtail
+            volumeMounts:
+            - mountPath: /run/dch-ca
+              name: dch-ca
+              readOnly: true
+          volumes:
+          - name: dch-ca
+            configMap:
+              name: dch-root-ca
+              optional: true

promtail/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: promtail
+  labels:
+    app.kubernetes.io/name: promtail

promtail/promtail.yaml

@@ -0,0 +1,137 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: promtail
+  labels:
+    app.kubernetes.io/name: promtail
+    app.kubernetes.io/component: promtail
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: promtail
+  labels:
+    app.kubernetes.io/name: promtail
+    app.kubernetes.io/component: promtail
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: promtail
+  labels:
+    app.kubernetes.io/name: promtail
+    app.kubernetes.io/component: promtail
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: promtail
+subjects:
+- kind: ServiceAccount
+  name: promtail
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: promtail
+  labels:
+    app.kubernetes.io/name: promtail
+    app.kubernetes.io/component: promtail
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: promtail
+      app.kubernetes.io/component: promtail
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: promtail
+        app.kubernetes.io/component: promtail
+    spec:
+      containers:
+      - name: promtail
+        image: docker.io/grafana/promtail:2.9.4
+        args:
+        - -config.file=/etc/promtail/config.yml
+        env:
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 9080
+          name: http
+        readinessProbe: &probe
+          httpGet:
+            port: http
+            path: /ready
+          periodSeconds: 60
+        startupProbe:
+          <<: *probe
+          periodSeconds: 1
+          successThreshold: 1
+          failureThreshold: 30
+          timeoutSeconds: 1
+        securityContext:
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /etc/machine-id
+          name: machine-id
+          readOnly: true
+        - mountPath: /etc/promtail
+          name: config
+          readOnly: true
+        - mountPath: /run/log
+          name: run-log
+          readOnly: true
+        - mountPath: /tmp
+          name: tmp
+          subPath: tmp
+        - mountPath: /var/lib/promtail
+          name: promtail
+        - mountPath: /var/log
+          name: var-log
+          readOnly: true
+      securityContext:
+        seLinuxOptions:
+          # confined containers do not have access to /var/log
+          type: spc_t
+      serviceAccountName: promtail
+      tolerations:
+      - effect: NoExecute
+        operator: Exists
+      - effect: NoSchedule
+        operator: Exists
+      volumes:
+      - name: config
+        configMap:
+          name: promtail
+      - name: machine-id
+        hostPath:
+          path: /etc/machine-id
+          type: File
+      - name: promtail
+        hostPath:
+          path: /var/lib/promtail
+          type: DirectoryOrCreate
+      - name: run-log
+        hostPath:
+          path: /run/log
+          type: Directory
+      - name: tmp
+        emptyDir: {}
+      - name: var-log
+        hostPath:
+          path: /var/log
+          type: Directory