grafana: Configure Loki datastore

Usually, Grafana datastores are configured using its web GUI.  When
setting up a datastore that requires TLS client authentication, the
client certificate and private key have to be pasted into the form.
For certificates that renew frequently, this method would require a
frequent manual effort.  Fortunately, Grafana supports defining
datastores via its "provisioning" mechanism, reading the configuration
from YAML files on the filesystem.

grafana/datasources/loki.yml

@@ -0,0 +1,14 @@
+apiVersion: 1
+
+datasources:
+- name: Loki
+  type: loki
+  access: proxy
+  url: https://loki.pyrocufflink.blue
+  jsonData:
+    tlsAuth: true
+    tlsAuthWithCACert: true
+  secureJsonData:
+    tlsCACert: $__file{/run/dch-ca/dch-root-ca.crt}
+    tlsClientCert: $__file{/run/secrets/du5t1n.me/loki/tls.crt}
+    tlsClientKey: $__file{/run/secrets/du5t1n.me/loki/tls.key}

grafana/grafana.yaml

@@ -73,6 +73,9 @@ spec:
         - mountPath: /etc/grafana
           name: config
           readOnly: true
+        - mountPath: /etc/grafana/provisioning/datasources
+          name: datasources
+          readOnly: true
         - mountPath: /run/secrets/grafana
           name: secrets
           readOnly: true
@@ -86,6 +89,10 @@ spec:
       - name: config
         configMap:
           name: grafana
+      - name: datasources
+        configMap:
+          name: datasources
+          optional: true
       - name: grafana
         persistentVolumeClaim:
           claimName: grafana

grafana/kustomization.yaml

@@ -16,9 +16,41 @@ resources:
 - grafana.yaml
 - ingress.yaml
 - secrets.yaml
+- loki-cert.yaml
+- ../dch-root-ca
 
 configMapGenerator:
 - name: grafana
   files:
   - grafana.ini
   - ldap.toml
+
+- name: datasources
+  files:
+  - datasources/loki.yml
+
+patches:
+- patch: |-
+    apiVersion: apps/v1
+    kind: StatefulSet
+    metadata:
+      name: grafana
+    spec:
+      template:
+        spec:
+          containers:
+          - name: grafana
+            volumeMounts:
+            - mountPath: /run/dch-ca
+              name: dch-ca
+              readOnly: true
+            - mountPath: /run/secrets/du5t1n.me/loki
+              name: loki-client-cert
+              readOnly: true
+          volumes:
+          - name: dch-ca
+            configMap:
+              name: dch-root-ca
+          - name: loki-client-cert
+            secret:
+              secretName: loki-client-cert

grafana/loki-cert.yaml

@@ -0,0 +1,12 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: loki-client-cert
+spec:
+  commonName: grafana
+  privateKey:
+    algorithm: Ed25519
+  secretName: loki-client-cert
+  issuerRef:
+    name: loki-ca
+    kind: ClusterIssuer