loki: Deploy Caddy in front of Loki

Grafana Loki explicitly eschews built-in authentication.  In fact, its
[documentation][0] states:

> Operators are expected to run an authenticating reverse proxy in front
> of your services.

While I don't really want to require authentication for agents sending
logs, I definitely want to restrict querying and viewing logs to trusted
users.

There are _many_ reverse proxy servers available, and normally I would
choose _nginx_.  In this case, though, I decided to try Caddy, mostly
because of its built-in ACME support.  I wasn't really happy with how
the `fetchcert` system turned out, particularly using the Kubernetes API
token for authentication.  Since the token will eventually expire, it
will require manual intervention to renew, thus mostly defeating the
purpose of having an auto-renewing certificate.  So instead of using
_cert-manager_ to issue the certificate and store it in Kubernetes, and
then having `fetchcert` download it via the Kubernetes API, I set up
_step-ca_ to handle issuing the certificate directly to the server. When
Caddy starts up, it contacts _step-ca_ via ACME and handles the
challenge verification automatically.  Further, it will automatically
renew the certificate as necessary, again using ACME.

I didn't spend a lot of time optimizing the Caddy configuration, so
there's some duplication there (i.e. the multiple `reverse_proxy`
statements), but the configuration works as desired.  Clients may
provide a certificate, which will be verified against the trusted issuer
CA.  If the certificate is valid, the client may access any Loki
resource.  Clients that do not provide a certificate can only access the
ingestion path, as well as the "ready" and "metrics" resources.

[0]: https://grafana.com/docs/loki/latest/operations/authentication/

app/loki/templates.cue

@@ -23,4 +23,38 @@ templates: [...instructions.#RenderInstruction] & [
 			]
 		}
 	},
+	{
+		template: "loki/caddy-client-ca.crt"
+		dest:     "/etc/caddy/client-ca.crt"
+		hooks: {
+			changed: [{run: "systemctl try-reload-or-restart caddy"}]
+		}
+	},
+	{
+		template: "loki/caddy-acme-ca.crt"
+		dest:     "/etc/caddy/acme-ca.crt"
+		hooks: {
+			changed: [{run: "systemctl try-reload-or-restart caddy"}]
+		}
+	},
+	{
+		template: "loki/Caddyfile"
+		dest:     "/etc/caddy/Caddyfile"
+		hooks: {
+			changed: [{run: "systemctl try-reload-or-restart caddy"}]
+		}
+	},
+	{
+		template: "loki/caddy.container"
+		dest:     "/etc/containers/systemd/caddy.container"
+		hooks: {
+			changed: [
+				{
+					run:       "systemctl daemon-reload"
+					immediate: true
+				},
+				{run: "systemctl restart caddy"},
+			]
+		}
+	},
 ]

env/prod/loki.cue

@@ -0,0 +1,34 @@
+package prod
+
+loki: caddy: {
+	acme_ca: """
+		-----BEGIN CERTIFICATE-----
+		MIICTzCCAgGgAwIBAgIUDNTFsSYYl8xsEcg9kTatxvOSkmUwBQYDK2VwMEAxCzAJ
+		BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxFzAVBgNVBAMMDkRD
+		SCBSb290IENBIFIzMB4XDTI0MDIxNzIwMjk0M1oXDTI1MDIxNzIwMjk0M1owOzEL
+		MAkGA1UEBhMCVVMxGDAWBgNVBAoMD0R1c3RpbiBDLiBIYXRjaDESMBAGA1UEAwwJ
+		RENIIENBIFIzMCowBQYDK2VwAyEA50stJ8iW6/f+uECPxAJwpSfQDRQg4/AgKJY2
+		lpd3uNijggEQMIIBDDAdBgNVHQ4EFgQUtiqtFaZZ/c4IfWXV5SjJIOPbmoowHwYD
+		VR0jBBgwFoAUtmjEAcG9apstYyBr8MACUb2J2jkwEgYDVR0TAQH/BAgwBgEB/wIB
+		ADALBgNVHQ8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMEwG
+		CCsGAQUFBwEBBEAwPjA8BggrBgEFBQcwAoYwaHR0cHM6Ly9kdXN0aW4uaGF0Y2gu
+		bmFtZS9kY2gtY2EvZGNoLXJvb3QtY2EuY3J0MDwGA1UdHwQ1MDMwMaAvoC2GK2h0
+		dHBzOi8vZHVzdGluLmhhdGNoLm5hbWUvZGNoLWNhL2RjaC1jYS5jcmwwBQYDK2Vw
+		A0EAACaKAJAKejpFXQV+mgPdDXaylvakc4rCEs1pFhPXbbMMGflNOeiiy+c+aMwt
+		yfObaZ8/YiXxCSjL6/KzRSSjAQ==
+		-----END CERTIFICATE-----
+		"""
+	client_ca: """
+		-----BEGIN CERTIFICATE-----
+		MIIBlDCCAUagAwIBAgIUGNZ/ASP8F2ytev3YplTk4jA5a2EwBQYDK2VwMEgxCzAJ
+		BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxDTALBgNVBAsMBExv
+		a2kxEDAOBgNVBAMMB0xva2kgQ0EwHhcNMjQwMjIwMTUwMTQxWhcNMzQwMjIwMTUw
+		MTQxWjBIMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPRHVzdGluIEMuIEhhdGNoMQ0w
+		CwYDVQQLDARMb2tpMRAwDgYDVQQDDAdMb2tpIENBMCowBQYDK2VwAyEAnmMawEIo
+		WfzFaLgpSiaPD+DHg28NHknMFcs7XpyTM9CjQjBAMB0GA1UdDgQWBBTFth3c4S/f
+		y0BphQy9SucnKN2pLzASBgNVHRMBAf8ECDAGAQH/AgEAMAsGA1UdDwQEAwIBBjAF
+		BgMrZXADQQCn0JWERsXdJA4kMM45ZXhVgAciwLNQ8ikoucsJcbWBp7bSMjcMVi51
+		I+slotQvQES/vfqp/zZFNl7KKyeeQ0sD
+		-----END CERTIFICATE-----
+		"""
+}

host/loki0.pyrocufflink.blue.cue

@@ -7,32 +7,4 @@ sudo: prod.sudo
 
 promtail: prod.#promtail
 
-fetchcert: prod.fetchcert.loki & {
-	token: """
-		-----BEGIN AGE ENCRYPTED FILE-----
-		YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtNTZzeW9XeWoycVdQa092
-		N0VYL2grR0lLY1c4QXl2VHI3NmsxM253UlNVCmZLbFZWakJGVG9WakkyYmpJL1VR
-		YmVQQXRCTlhrQk9UYUE5UkRFZUlwNlkKLS0tIGg4R25ZaVhUU1BFVjdac2NqMVpQ
-		QmZRTndBalZndVF0VFpxdHBRemhNS1EKrNZG179fh2aS/3FOaM1xCHRG4uOt5jyx
-		1m5h3Q9y2u7EbcbZHLIZR3wkQfsfscK1PS0+H0NiYAgh9u2L2kdhcLcesb3fhmSy
-		svHzW2q1ZkJ8DSwH3xCRBuKmH4Q172NcVUPzI39CgsI5SkqZdKjWnK9JJAs43Ihr
-		cM90hUN+5t50byUSzwTCmNY4xVW3N/pWMfrethCYk9E8cXts/L3A3EpgpIi3qrKn
-		gj2VfrvpHAWVcggX1rZVFlQwBg4LnPWMNztl5VRYIvwfJghykEjMlzkysLm3Q2is
-		/w+kthpBzYAvI4c1Tfx3/uMRVcWnmUgz15viKlqohVaAl9PHQ2y/te9w9D5ZtcYs
-		D33hfA7Aux9t18WJ/ru09rEJl649Al7ZxQd73upf9QrWGzkX4luHO85n8CBmcsuh
-		+ZcM1HMLiuxGCW6xyq66Eg6t/1pfPWGZtLCsFh4SRgJ6Uuq14FyU32Pkulq+yEMg
-		Sq2ZRUXU+e3M6/HcUhb+QQUTQF1wPHyEukUlecLGDd3i+xpjOrL5Eg7LjKVAv8Yj
-		8U1yiYjgRHfdkvT27RJC/rxuf674vU8H8na3jGXrPARMqq4L4B0XkUzclJZMzSPC
-		cSTaEIgb5OpfWmMb4uC0p76vHYhr4XX3iIVpivfxaDLAgyx06D4/oXALcgjcCHWY
-		/7m5t8MbIGqluqcJLYRhSQ+G/aWiyZG3zlgRfpOIyVzQHwQwGf2CLh6ygv9n5cWP
-		Gr0ZfcyVps734gVsDNqZ3vTy4nxjTueUiUpNqRaznzxT/z7Mq9/i0s1aoWBef0PV
-		MZL0jxyMeQUfRf0DdP/iPqkTU5hxw8/yqwuu2i3TJImVQ8ga8O3InyvN577mPihE
-		EqFjRl1jZr+Uip0+SPz+CSLIgBJ8rpAo/HTpue6Oe88rYtC0437YQtcWpB3rnARD
-		uggtP70SfvS7FWFCbYy7nxZrUcDMloD5gcIYNobkWQZhGdGvXDGVxB/FT8Rg6tAU
-		EOpaSSc3wOmHpnB6qCyCJ45mb6HwRCGoZmxaG/5uWreys0R8AJsMIq8vFVAS3sDo
-		EONNYMWtlAZg8XOZcSgSnKpUF5VWlt+3HLkpwQkTBq3SvjvMd6shybPVGVNxMwbU
-		a2gey9Kv4lq8Suvvrn31DeYErGwUYy0qMwTL1a4Q8I08kMg6lqqaPotIC63RSlUu
-		SEoarQ==
-		-----END AGE ENCRYPTED FILE-----
-		"""
-}
+loki: prod.loki

instructions/loki0.pyrocufflink.blue.cue

@@ -2,7 +2,6 @@ import (
 	"list"
 
 	"du5t1n.me/cfg/app/collectd"
-	"du5t1n.me/cfg/app/fetchcert"
 	"du5t1n.me/cfg/app/promtail"
 	"du5t1n.me/cfg/app/loki"
 	"du5t1n.me/cfg/env/prod"
@@ -11,7 +10,6 @@ import (
 render: list.Concat([
 	prod.templates,
 	collectd.templates,
-	fetchcert.templates,
 	loki.templates,
 	promtail.templates,
 ])

templates/loki/Caddyfile

@@ -0,0 +1,30 @@
+loki.pyrocufflink.blue {
+	tls {
+		client_auth {
+			mode verify_if_given
+			trusted_ca_cert_file /etc/caddy/client-ca.crt
+		}
+	}
+	@anonymous {
+		expression {tls_client_subject} == null
+	}
+	handle @anonymous {
+		route /loki/api/v1/push {
+			reverse_proxy 127.0.0.1:3100
+		}
+		route /metrics {
+			reverse_proxy 127.0.0.1:3100
+		}
+		route /ready {
+			reverse_proxy 127.0.0.1:3100
+		}
+		respond 403
+	}
+	handle {
+		reverse_proxy 127.0.0.1:3100
+	}
+	tls loki@pyrocufflink.blue {
+		ca https://ca.pyrocufflink.blue:32599/acme/acme/directory
+		ca_root /etc/caddy/acme-ca.crt
+	}
+}

templates/loki/caddy-acme-ca.crt

@@ -0,0 +1 @@
+{{ loki.caddy.acme_ca }}

templates/loki/caddy-client-ca.crt

@@ -0,0 +1 @@
+{{ loki.caddy.client_ca }}

templates/loki/caddy.container

@@ -0,0 +1,22 @@
+[Unit]
+Description=Caddy web server
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+Image=docker.io/library/caddy:2
+Volume=/etc/caddy:/etc/caddy:ro
+Volume=/var/lib/caddy/config:/config/caddy:rw,z
+Volume=/var/lib/caddy/data:/data/caddy:rw,z
+ReadOnly=yes
+ReadOnlyTmpfs=yes
+Network=host
+AddCapability=CAP_NET_BIND_SERVICE
+DropCapability=all
+
+[Service]
+StateDirectory=%N/data %N/config
+ExecReload=/usr/bin/podman exec systemd-%N caddy reload -c /etc/caddy/Caddyfile
+
+[Install]
+WantedBy=multi-user.target

templates/loki/config.yml

@@ -2,9 +2,7 @@ auth_enabled: false
 
 server:
   http_listen_port: 3100
-  http_tls_config:
-    cert_file: /etc/loki/server.cer
-    key_file: /etc/loki/server.key
+  http_listen_address: 127.0.0.1
   grpc_listen_port: 9096
 
 common:

templates/loki/loki.container

@@ -13,7 +13,7 @@ Image=docker.io/grafana/loki:2.9.4
 Exec=-config.file=/etc/loki/config.yml
 Volume=%S/%P:/var/lib/loki:rw,Z,U
 Volume=/etc/loki:/etc/loki:ro
-PublishPort=3100:3100
+Network=host
 
 [Install]
 WantedBy=multi-user.target