From 878ff7acb5b28af64aaeb0ef8841c85ddc0a3cd8 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Tue, 20 Feb 2024 07:25:30 -0600 Subject: [PATCH] loki: Deploy Caddy in front of Loki Grafana Loki explicitly eschews built-in authentication. In fact, its [documentation][0] states: > Operators are expected to run an authenticating reverse proxy in front > of your services. While I don't really want to require authentication for agents sending logs, I definitely want to restrict querying and viewing logs to trusted users. There are _many_ reverse proxy servers available, and normally I would choose _nginx_. In this case, though, I decided to try Caddy, mostly because of its built-in ACME support. I wasn't really happy with how the `fetchcert` system turned out, particularly using the Kubernetes API token for authentication. Since the token will eventually expire, it will require manual intervention to renew, thus mostly defeating the purpose of having an auto-renewing certificate. So instead of using _cert-manager_ to issue the certificate and store it in Kubernetes, and then having `fetchcert` download it via the Kubernetes API, I set up _step-ca_ to handle issuing the certificate directly to the server. When Caddy starts up, it contacts _step-ca_ via ACME and handles the challenge verification automatically. Further, it will automatically renew the certificate as necessary, again using ACME. I didn't spend a lot of time optimizing the Caddy configuration, so there's some duplication there (i.e. the multiple `reverse_proxy` statements), but the configuration works as desired. Clients may provide a certificate, which will be verified against the trusted issuer CA. If the certificate is valid, the client may access any Loki resource. Clients that do not provide a certificate can only access the ingestion path, as well as the "ready" and "metrics" resources. [0]: https://grafana.com/docs/loki/latest/operations/authentication/ --- app/loki/templates.cue | 34 ++++++++++++++++++++++++ env/prod/loki.cue | 34 ++++++++++++++++++++++++ host/loki0.pyrocufflink.blue.cue | 30 +-------------------- instructions/loki0.pyrocufflink.blue.cue | 2 -- templates/loki/Caddyfile | 30 +++++++++++++++++++++ templates/loki/caddy-acme-ca.crt | 1 + templates/loki/caddy-client-ca.crt | 1 + templates/loki/caddy.container | 22 +++++++++++++++ templates/loki/config.yml | 4 +-- templates/loki/loki.container | 2 +- 10 files changed, 125 insertions(+), 35 deletions(-) create mode 100644 env/prod/loki.cue create mode 100644 templates/loki/Caddyfile create mode 100644 templates/loki/caddy-acme-ca.crt create mode 100644 templates/loki/caddy-client-ca.crt create mode 100644 templates/loki/caddy.container diff --git a/app/loki/templates.cue b/app/loki/templates.cue index a202eaf..401d5f9 100644 --- a/app/loki/templates.cue +++ b/app/loki/templates.cue @@ -23,4 +23,38 @@ templates: [...instructions.#RenderInstruction] & [ ] } }, + { + template: "loki/caddy-client-ca.crt" + dest: "/etc/caddy/client-ca.crt" + hooks: { + changed: [{run: "systemctl try-reload-or-restart caddy"}] + } + }, + { + template: "loki/caddy-acme-ca.crt" + dest: "/etc/caddy/acme-ca.crt" + hooks: { + changed: [{run: "systemctl try-reload-or-restart caddy"}] + } + }, + { + template: "loki/Caddyfile" + dest: "/etc/caddy/Caddyfile" + hooks: { + changed: [{run: "systemctl try-reload-or-restart caddy"}] + } + }, + { + template: "loki/caddy.container" + dest: "/etc/containers/systemd/caddy.container" + hooks: { + changed: [ + { + run: "systemctl daemon-reload" + immediate: true + }, + {run: "systemctl restart caddy"}, + ] + } + }, ] diff --git a/env/prod/loki.cue b/env/prod/loki.cue new file mode 100644 index 0000000..e700645 --- /dev/null +++ b/env/prod/loki.cue @@ -0,0 +1,34 @@ +package prod + +loki: caddy: { + acme_ca: """ + -----BEGIN CERTIFICATE----- + MIICTzCCAgGgAwIBAgIUDNTFsSYYl8xsEcg9kTatxvOSkmUwBQYDK2VwMEAxCzAJ + BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxFzAVBgNVBAMMDkRD + SCBSb290IENBIFIzMB4XDTI0MDIxNzIwMjk0M1oXDTI1MDIxNzIwMjk0M1owOzEL + MAkGA1UEBhMCVVMxGDAWBgNVBAoMD0R1c3RpbiBDLiBIYXRjaDESMBAGA1UEAwwJ + RENIIENBIFIzMCowBQYDK2VwAyEA50stJ8iW6/f+uECPxAJwpSfQDRQg4/AgKJY2 + lpd3uNijggEQMIIBDDAdBgNVHQ4EFgQUtiqtFaZZ/c4IfWXV5SjJIOPbmoowHwYD + VR0jBBgwFoAUtmjEAcG9apstYyBr8MACUb2J2jkwEgYDVR0TAQH/BAgwBgEB/wIB + ADALBgNVHQ8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMEwG + CCsGAQUFBwEBBEAwPjA8BggrBgEFBQcwAoYwaHR0cHM6Ly9kdXN0aW4uaGF0Y2gu + bmFtZS9kY2gtY2EvZGNoLXJvb3QtY2EuY3J0MDwGA1UdHwQ1MDMwMaAvoC2GK2h0 + dHBzOi8vZHVzdGluLmhhdGNoLm5hbWUvZGNoLWNhL2RjaC1jYS5jcmwwBQYDK2Vw + A0EAACaKAJAKejpFXQV+mgPdDXaylvakc4rCEs1pFhPXbbMMGflNOeiiy+c+aMwt + yfObaZ8/YiXxCSjL6/KzRSSjAQ== + -----END CERTIFICATE----- + """ + client_ca: """ + -----BEGIN CERTIFICATE----- + MIIBlDCCAUagAwIBAgIUGNZ/ASP8F2ytev3YplTk4jA5a2EwBQYDK2VwMEgxCzAJ + BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxDTALBgNVBAsMBExv + a2kxEDAOBgNVBAMMB0xva2kgQ0EwHhcNMjQwMjIwMTUwMTQxWhcNMzQwMjIwMTUw + MTQxWjBIMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPRHVzdGluIEMuIEhhdGNoMQ0w + CwYDVQQLDARMb2tpMRAwDgYDVQQDDAdMb2tpIENBMCowBQYDK2VwAyEAnmMawEIo + WfzFaLgpSiaPD+DHg28NHknMFcs7XpyTM9CjQjBAMB0GA1UdDgQWBBTFth3c4S/f + y0BphQy9SucnKN2pLzASBgNVHRMBAf8ECDAGAQH/AgEAMAsGA1UdDwQEAwIBBjAF + BgMrZXADQQCn0JWERsXdJA4kMM45ZXhVgAciwLNQ8ikoucsJcbWBp7bSMjcMVi51 + I+slotQvQES/vfqp/zZFNl7KKyeeQ0sD + -----END CERTIFICATE----- + """ +} diff --git a/host/loki0.pyrocufflink.blue.cue b/host/loki0.pyrocufflink.blue.cue index ebc0c6e..40c2430 100644 --- a/host/loki0.pyrocufflink.blue.cue +++ b/host/loki0.pyrocufflink.blue.cue @@ -7,32 +7,4 @@ sudo: prod.sudo promtail: prod.#promtail -fetchcert: prod.fetchcert.loki & { - token: """ - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtNTZzeW9XeWoycVdQa092 - N0VYL2grR0lLY1c4QXl2VHI3NmsxM253UlNVCmZLbFZWakJGVG9WakkyYmpJL1VR - YmVQQXRCTlhrQk9UYUE5UkRFZUlwNlkKLS0tIGg4R25ZaVhUU1BFVjdac2NqMVpQ - QmZRTndBalZndVF0VFpxdHBRemhNS1EKrNZG179fh2aS/3FOaM1xCHRG4uOt5jyx - 1m5h3Q9y2u7EbcbZHLIZR3wkQfsfscK1PS0+H0NiYAgh9u2L2kdhcLcesb3fhmSy - svHzW2q1ZkJ8DSwH3xCRBuKmH4Q172NcVUPzI39CgsI5SkqZdKjWnK9JJAs43Ihr - cM90hUN+5t50byUSzwTCmNY4xVW3N/pWMfrethCYk9E8cXts/L3A3EpgpIi3qrKn - gj2VfrvpHAWVcggX1rZVFlQwBg4LnPWMNztl5VRYIvwfJghykEjMlzkysLm3Q2is - /w+kthpBzYAvI4c1Tfx3/uMRVcWnmUgz15viKlqohVaAl9PHQ2y/te9w9D5ZtcYs - D33hfA7Aux9t18WJ/ru09rEJl649Al7ZxQd73upf9QrWGzkX4luHO85n8CBmcsuh - +ZcM1HMLiuxGCW6xyq66Eg6t/1pfPWGZtLCsFh4SRgJ6Uuq14FyU32Pkulq+yEMg - Sq2ZRUXU+e3M6/HcUhb+QQUTQF1wPHyEukUlecLGDd3i+xpjOrL5Eg7LjKVAv8Yj - 8U1yiYjgRHfdkvT27RJC/rxuf674vU8H8na3jGXrPARMqq4L4B0XkUzclJZMzSPC - cSTaEIgb5OpfWmMb4uC0p76vHYhr4XX3iIVpivfxaDLAgyx06D4/oXALcgjcCHWY - /7m5t8MbIGqluqcJLYRhSQ+G/aWiyZG3zlgRfpOIyVzQHwQwGf2CLh6ygv9n5cWP - Gr0ZfcyVps734gVsDNqZ3vTy4nxjTueUiUpNqRaznzxT/z7Mq9/i0s1aoWBef0PV - MZL0jxyMeQUfRf0DdP/iPqkTU5hxw8/yqwuu2i3TJImVQ8ga8O3InyvN577mPihE - EqFjRl1jZr+Uip0+SPz+CSLIgBJ8rpAo/HTpue6Oe88rYtC0437YQtcWpB3rnARD - uggtP70SfvS7FWFCbYy7nxZrUcDMloD5gcIYNobkWQZhGdGvXDGVxB/FT8Rg6tAU - EOpaSSc3wOmHpnB6qCyCJ45mb6HwRCGoZmxaG/5uWreys0R8AJsMIq8vFVAS3sDo - EONNYMWtlAZg8XOZcSgSnKpUF5VWlt+3HLkpwQkTBq3SvjvMd6shybPVGVNxMwbU - a2gey9Kv4lq8Suvvrn31DeYErGwUYy0qMwTL1a4Q8I08kMg6lqqaPotIC63RSlUu - SEoarQ== - -----END AGE ENCRYPTED FILE----- - """ -} +loki: prod.loki diff --git a/instructions/loki0.pyrocufflink.blue.cue b/instructions/loki0.pyrocufflink.blue.cue index 7c7e958..8ab3f40 100644 --- a/instructions/loki0.pyrocufflink.blue.cue +++ b/instructions/loki0.pyrocufflink.blue.cue @@ -2,7 +2,6 @@ import ( "list" "du5t1n.me/cfg/app/collectd" - "du5t1n.me/cfg/app/fetchcert" "du5t1n.me/cfg/app/promtail" "du5t1n.me/cfg/app/loki" "du5t1n.me/cfg/env/prod" @@ -11,7 +10,6 @@ import ( render: list.Concat([ prod.templates, collectd.templates, - fetchcert.templates, loki.templates, promtail.templates, ]) diff --git a/templates/loki/Caddyfile b/templates/loki/Caddyfile new file mode 100644 index 0000000..66c8836 --- /dev/null +++ b/templates/loki/Caddyfile @@ -0,0 +1,30 @@ +loki.pyrocufflink.blue { + tls { + client_auth { + mode verify_if_given + trusted_ca_cert_file /etc/caddy/client-ca.crt + } + } + @anonymous { + expression {tls_client_subject} == null + } + handle @anonymous { + route /loki/api/v1/push { + reverse_proxy 127.0.0.1:3100 + } + route /metrics { + reverse_proxy 127.0.0.1:3100 + } + route /ready { + reverse_proxy 127.0.0.1:3100 + } + respond 403 + } + handle { + reverse_proxy 127.0.0.1:3100 + } + tls loki@pyrocufflink.blue { + ca https://ca.pyrocufflink.blue:32599/acme/acme/directory + ca_root /etc/caddy/acme-ca.crt + } +} diff --git a/templates/loki/caddy-acme-ca.crt b/templates/loki/caddy-acme-ca.crt new file mode 100644 index 0000000..e5c3239 --- /dev/null +++ b/templates/loki/caddy-acme-ca.crt @@ -0,0 +1 @@ +{{ loki.caddy.acme_ca }} diff --git a/templates/loki/caddy-client-ca.crt b/templates/loki/caddy-client-ca.crt new file mode 100644 index 0000000..2ad0b49 --- /dev/null +++ b/templates/loki/caddy-client-ca.crt @@ -0,0 +1 @@ +{{ loki.caddy.client_ca }} diff --git a/templates/loki/caddy.container b/templates/loki/caddy.container new file mode 100644 index 0000000..d695785 --- /dev/null +++ b/templates/loki/caddy.container @@ -0,0 +1,22 @@ +[Unit] +Description=Caddy web server +After=network-online.target +Wants=network-online.target + +[Container] +Image=docker.io/library/caddy:2 +Volume=/etc/caddy:/etc/caddy:ro +Volume=/var/lib/caddy/config:/config/caddy:rw,z +Volume=/var/lib/caddy/data:/data/caddy:rw,z +ReadOnly=yes +ReadOnlyTmpfs=yes +Network=host +AddCapability=CAP_NET_BIND_SERVICE +DropCapability=all + +[Service] +StateDirectory=%N/data %N/config +ExecReload=/usr/bin/podman exec systemd-%N caddy reload -c /etc/caddy/Caddyfile + +[Install] +WantedBy=multi-user.target diff --git a/templates/loki/config.yml b/templates/loki/config.yml index 0ddb22d..6359871 100644 --- a/templates/loki/config.yml +++ b/templates/loki/config.yml @@ -2,9 +2,7 @@ auth_enabled: false server: http_listen_port: 3100 - http_tls_config: - cert_file: /etc/loki/server.cer - key_file: /etc/loki/server.key + http_listen_address: 127.0.0.1 grpc_listen_port: 9096 common: diff --git a/templates/loki/loki.container b/templates/loki/loki.container index 98956bc..da8985a 100644 --- a/templates/loki/loki.container +++ b/templates/loki/loki.container @@ -13,7 +13,7 @@ Image=docker.io/grafana/loki:2.9.4 Exec=-config.file=/etc/loki/config.yml Volume=%S/%P:/var/lib/loki:rw,Z,U Volume=/etc/loki:/etc/loki:ro -PublishPort=3100:3100 +Network=host [Install] WantedBy=multi-user.target