Merge branch 'promtail'

app/fetchcert/fetchcert.cue

@@ -0,0 +1,12 @@
+package fetchcert
+
+#Fetchcert: {
+	ca_cert:        string
+	kubernetes_url: string
+	token:          string
+	namespace:      string
+	secret:         string
+	cert:           string
+	key:            string
+	postupdate?:    string
+}

app/fetchcert/templates.cue

@@ -0,0 +1,39 @@
+package fetchcert
+
+import "du5t1n.me/cfg/base/schema/instructions"
+
+templates: [...instructions.#RenderInstruction] & [
+	{
+		template: "fetchcert/ca.crt"
+		dest:     "/etc/fetchcert/ca.crt"
+	},
+	{
+		template: "fetchcert/token"
+		dest:     "/etc/fetchcert/token"
+	},
+	{
+		template: "fetchcert/postupdate.sh"
+		dest:     "/etc/fetchcert/postupdate.sh"
+		mode:     "u=rwx,go=rx"
+	},
+	{
+		template: "fetchcert/fetchcert.container"
+		dest:     "/etc/containers/systemd/fetchcert.container"
+		hooks: {
+			changed: [
+				{run: "systemctl daemon-reload", immediate: true},
+			]
+		}
+	},
+	{
+		template: "fetchcert/fetchcert.timer"
+		dest:     "/etc/systemd/system/fetchcert.timer"
+		hooks: {
+			changed: [
+				{run: "systemctl daemon-reload", immediate: true},
+				{run: "systemctl enable fetchcert.timer"},
+				{run: "systemctl start fetchcert.timer"},
+			]
+		}
+	},
+]

app/loki/templates.cue

@@ -23,4 +23,38 @@ templates: [...instructions.#RenderInstruction] & [
 			]
 		}
 	},
+	{
+		template: "loki/caddy-client-ca.crt"
+		dest:     "/etc/caddy/client-ca.crt"
+		hooks: {
+			changed: [{run: "systemctl try-reload-or-restart caddy"}]
+		}
+	},
+	{
+		template: "loki/caddy-acme-ca.crt"
+		dest:     "/etc/caddy/acme-ca.crt"
+		hooks: {
+			changed: [{run: "systemctl try-reload-or-restart caddy"}]
+		}
+	},
+	{
+		template: "loki/Caddyfile"
+		dest:     "/etc/caddy/Caddyfile"
+		hooks: {
+			changed: [{run: "systemctl try-reload-or-restart caddy"}]
+		}
+	},
+	{
+		template: "loki/caddy.container"
+		dest:     "/etc/containers/systemd/caddy.container"
+		hooks: {
+			changed: [
+				{
+					run:       "systemctl daemon-reload"
+					immediate: true
+				},
+				{run: "systemctl restart caddy"},
+			]
+		}
+	},
 ]

app/promtail/schema/metrics.cue

@@ -0,0 +1,38 @@
+package schema
+
+#BaseMetric: {
+	type:               string
+	description?:       string
+	prefix?:            string
+	source?:            string
+	max_idle_duration?: string
+}
+
+#CounterMetric: {
+	#BaseMetric
+	type: "Counter"
+	config: {
+		match_all?:         bool
+		count_entry_bytes?: bool
+		value?:             string
+		action:             "inc" | "add"
+	}
+}
+
+#GaugeMetric: {
+	#BaseMetric
+	type: "Gauge"
+	config: {
+		value?: string
+		action: "set" | "inc" | "dec" | "add" | "sub"
+	}
+}
+
+#HistogramMetric: {
+	#BaseMetric
+	type: "Histogram"
+	config: {
+		value?: string
+		buckets: [int]
+	}
+}

app/promtail/schema/pipeline.cue

@@ -0,0 +1,106 @@
+package schema
+
+#PipelineStage: {
+	cri?: {
+		max_partial_lines?:              int
+		max_partial_line_size?:          int
+		max_partial_line_size_truncate?: bool
+	}
+
+	decolorize?: null
+
+	drop?: {
+		source?: [...string] | string
+		separator?:           string
+		expression?:          string
+		value?:               string
+		older_than?:          string
+		longer_than?:         string | int
+		drop_counter_reason?: string
+	}
+
+	json?: {
+		expressions?: [string]: string
+		source?:         string
+		drop_malformed?: bool
+	}
+
+	labelallow?: [...string]
+
+	labeldrop?: [...string]
+
+	labels?: [string]: string
+
+	limit?: {
+		rate?:                int
+		burst?:               int
+		by_label_name?:       string
+		max_distinct_labels?: int
+		drop?:                bool
+	}
+
+	logfmt?: {
+		mapping?: [string]: string
+		source?: string
+	}
+
+	match?: {
+		selector:             string
+		pipeline_name?:       string
+		action?:              "keep" | "drop"
+		drop_counter_reason?: string
+		stages?: [#PipelineStage]
+	}
+
+	metrics?: [string]: #CounterMetric | #GaugeMetric | #HistogramMetric
+
+	multiline?: {
+		firstline:      string
+		max_wait_time?: string
+		max_lines?:     int
+	}
+
+	output?: {
+		source: string
+	}
+
+	pack?: {
+		labels: [string]
+		ingest_timestamp?: bool
+	}
+
+	regex?: {
+		expression: string
+		source?:    string
+	}
+
+	replace?: {
+		expression: string
+		source?:    string
+		replace?:   string
+	}
+
+	sampling?: {
+		rate?: float
+	}
+
+	static_labels?: [string]: string
+
+	template?: {
+		source:   string
+		template: string
+	}
+
+	tenant?: {
+		label?:  string
+		source?: string
+		value?:  string
+	}
+
+	timestamp?: {
+		source: string
+		format: string
+		fallback_formats?: [...string]
+		location?: string
+	}
+}

app/promtail/schema/schema.cue

@@ -0,0 +1,62 @@
+package schema
+
+#Client: {
+	url: string
+	tls_config?: {
+		ca_file?:     string
+		cert_file?:   string
+		key_file?:    string
+		server_name?: string
+	}
+}
+
+#CommonScrapeConfig: {
+	labels?: [string]: string
+}
+
+#JournalScrapeConfig: {
+	#CommonScrapeConfig
+	json: bool | *false
+}
+
+#KubernetesScrapeConfig: {
+	#CommonScrapeConfig
+	api_server?: string
+	role:        "node" | "service" | "pod" | "endpoints" | "ingress"
+}
+
+#RelabelConfig: {
+	source_labels?: [...string]
+	separator?:    string
+	target_label?: string
+	regex?:        string
+	modulus?:      int
+	replacement?:  string
+	action?:       "replace" | "keep" | "drop" | "hashmod" | "labelmap" |
+		"labeldrop" | "labelkeep"
+}
+
+#ScrapeConfig: {
+	job_name: string
+	journal?: #JournalScrapeConfig
+	static_configs?: [...#CommonScrapeConfig]
+	kubernetes_sd_configs?: [...#KubernetesScrapeConfig]
+	pipeline_stages?: [...#PipelineStage]
+	relabel_configs?: [...#RelabelConfig]
+}
+
+#PromtailConfig: {
+	server: {
+		http_listen_port:      int | *9080
+		grpc_listen_port:      int | *0
+		enable_runtime_reload: bool | *true
+	}
+
+	clients: [...#Client]
+
+	positions: {
+		filename: string | *"/var/lib/promtail/positions"
+	}
+
+	scrape_configs: [...#ScrapeConfig]
+}

app/promtail/templates.cue

@@ -0,0 +1,33 @@
+package promtail
+
+import "du5t1n.me/cfg/base/schema/instructions"
+
+templates: [...instructions.#RenderInstruction] & [
+	{
+		template: "promtail/ca.crt"
+		dest:     "/etc/promtail/ca.crt"
+		hooks: {
+			changed: [{run: "systemctl try-restart promtail"}]
+		}
+	},
+	{
+		template: "promtail/config.yml"
+		dest:     "/etc/promtail/config.yml"
+		hooks: {
+			changed: [{run: "systemctl try-restart promtail"}]
+		}
+	},
+	{
+		template: "promtail/promtail.container"
+		dest:     "/etc/containers/systemd/promtail.container"
+		hooks: {
+			changed: [
+				{
+					run:       "systemctl daemon-reload"
+					immediate: true
+				},
+				{run: "systemctl restart promtail"},
+			]
+		}
+	},
+]

env/prod/fetchcert.cue

@@ -0,0 +1,40 @@
+package prod
+
+import f "du5t1n.me/cfg/app/fetchcert"
+
+fetchcert: base: f.#Fetchcert & {
+	ca_cert: """
+		-----BEGIN CERTIFICATE-----
+		MIIC/jCCAeagAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
+		cm5ldGVzMB4XDTIyMDgwMTAyNTUzM1oXDTMyMDcyOTAyNTUzM1owFTETMBEGA1UE
+		AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMs6
+		2PUOzIClsAgPv1Mn9CTwzSFMntAn7OppwK5BQ4E5Vd1yMjz3p0uA1ZINv1ORorG0
+		mLl95C7y+CWUGPx+stHKQr/40sLGyypbX+AfjoPzHiDbIcbZEff8X5RwKqzmT9V7
+		Yt29KewADod0z+fqNYa62MJDaUunfwaV8kKFU/WJM8IKv2eJxAtWzvK3iHAFhx0j
+		Xo4TlyINL9V9UMKLf12w6CA3G41uZIBCN3G7aJEm++eGoMdrPZUXlbCpbSztO85/
+		hbulVs+0hCIxWiI+mRmB5OoWlRYL4jA45oK/RtpEqSwZ95zlGNAChmH7rb0pTtNf
+		N0/C2wKAEL4POLx9kscCAwEAAaNZMFcwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
+		/wQFMAMBAf8wHQYDVR0OBBYEFHYActCjEWdtsA+Ju25gxJh/vaLQMBUGA1UdEQQO
+		MAyCCmt1YmVybmV0ZXMwDQYJKoZIhvcNAQELBQADggEBAAfkYHecXUwyqvMSXmqr
+		ETqEzDCBini14s89VDhaDHOXBID9TKMVyeePdEYcPAJz3wo8fbx/+TL37K6hEuo+
+		7bUaamaumznsjg9L0Hth19GvuRKMXJlEpndRmE5K9hnaDLr94MLg9n1qGcEOt6tw
+		O6X5qqHf9AuuL39vt1+kSw6PeZZFZNMDZ8BdiTssw4btjQ2bsWu0wSiOSz/F8iRf
+		2vN5An5dheroDsFs4dZ9gnJ69TmqV1YqQxfRWqCxzfNJbgVm6AoBPwhL1JRuAU4N
+		3nCNoM9n2tLFDojT4un1778UVU91PtcBVdM9Nq+RC2jhXIyLBqsEK0ofOqFYqj3F
+		0EQ=
+		-----END CERTIFICATE-----
+		"""
+	kubernetes_url: "https://kubernetes.pyrocufflink.blue:6443"
+	namespace:      "dch-ca"
+}
+
+fetchcert: loki: fetchcert.base & {
+	secret: "loki"
+	cert:   "loki.cer"
+	key:    "loki.key"
+	postupdate: """
+		install -v -o root -g 10001 -m ugo=r /etc/fetchcert/certs/loki.cer /etc/loki/server.cer
+		install -v -o root -g 10001 -m ug=r,o= /etc/fetchcert/certs/loki.key /etc/loki/server.key
+		systemctl reload loki
+		"""
+}

env/prod/loki.cue

@@ -0,0 +1,34 @@
+package prod
+
+loki: caddy: {
+	acme_ca: """
+		-----BEGIN CERTIFICATE-----
+		MIICTzCCAgGgAwIBAgIUDNTFsSYYl8xsEcg9kTatxvOSkmUwBQYDK2VwMEAxCzAJ
+		BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxFzAVBgNVBAMMDkRD
+		SCBSb290IENBIFIzMB4XDTI0MDIxNzIwMjk0M1oXDTI1MDIxNzIwMjk0M1owOzEL
+		MAkGA1UEBhMCVVMxGDAWBgNVBAoMD0R1c3RpbiBDLiBIYXRjaDESMBAGA1UEAwwJ
+		RENIIENBIFIzMCowBQYDK2VwAyEA50stJ8iW6/f+uECPxAJwpSfQDRQg4/AgKJY2
+		lpd3uNijggEQMIIBDDAdBgNVHQ4EFgQUtiqtFaZZ/c4IfWXV5SjJIOPbmoowHwYD
+		VR0jBBgwFoAUtmjEAcG9apstYyBr8MACUb2J2jkwEgYDVR0TAQH/BAgwBgEB/wIB
+		ADALBgNVHQ8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMEwG
+		CCsGAQUFBwEBBEAwPjA8BggrBgEFBQcwAoYwaHR0cHM6Ly9kdXN0aW4uaGF0Y2gu
+		bmFtZS9kY2gtY2EvZGNoLXJvb3QtY2EuY3J0MDwGA1UdHwQ1MDMwMaAvoC2GK2h0
+		dHBzOi8vZHVzdGluLmhhdGNoLm5hbWUvZGNoLWNhL2RjaC1jYS5jcmwwBQYDK2Vw
+		A0EAACaKAJAKejpFXQV+mgPdDXaylvakc4rCEs1pFhPXbbMMGflNOeiiy+c+aMwt
+		yfObaZ8/YiXxCSjL6/KzRSSjAQ==
+		-----END CERTIFICATE-----
+		"""
+	client_ca: """
+		-----BEGIN CERTIFICATE-----
+		MIIBlDCCAUagAwIBAgIUGNZ/ASP8F2ytev3YplTk4jA5a2EwBQYDK2VwMEgxCzAJ
+		BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxDTALBgNVBAsMBExv
+		a2kxEDAOBgNVBAMMB0xva2kgQ0EwHhcNMjQwMjIwMTUwMTQxWhcNMzQwMjIwMTUw
+		MTQxWjBIMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPRHVzdGluIEMuIEhhdGNoMQ0w
+		CwYDVQQLDARMb2tpMRAwDgYDVQQDDAdMb2tpIENBMCowBQYDK2VwAyEAnmMawEIo
+		WfzFaLgpSiaPD+DHg28NHknMFcs7XpyTM9CjQjBAMB0GA1UdDgQWBBTFth3c4S/f
+		y0BphQy9SucnKN2pLzASBgNVHRMBAf8ECDAGAQH/AgEAMAsGA1UdDwQEAwIBBjAF
+		BgMrZXADQQCn0JWERsXdJA4kMM45ZXhVgAciwLNQ8ikoucsJcbWBp7bSMjcMVi51
+		I+slotQvQES/vfqp/zZFNl7KKyeeQ0sD
+		-----END CERTIFICATE-----
+		"""
+}

env/prod/promtail.cue

@@ -0,0 +1,86 @@
+package prod
+
+import "encoding/yaml"
+
+import "du5t1n.me/cfg/app/promtail/schema"
+
+promtail: {
+	scrape: {
+		journal: schema.#ScrapeConfig & {
+			job_name: "journal"
+			journal: {
+				labels: {
+					job: "systemd-journal"
+				}
+			}
+			relabel_configs: [
+				{
+					source_labels: ["__journal__hostname"]
+					target_label: "hostname"
+				},
+				{
+					source_labels: ["__journal__systemd_unit"]
+					target_label: "unit"
+				},
+				{
+					source_labels: ["__journal_syslog_identifier"]
+					target_label: "syslog_identifier"
+				},
+				{
+					source_labels: ["__journal_priority"]
+					target_label: "priority"
+				},
+				{
+					source_labels: ["__journal_message_id"]
+					target_label: "message_id"
+				},
+				{
+					source_labels: ["__journal__comm"]
+					target_label: "command"
+				},
+				{
+					source_labels: ["__journal__transport"]
+					target_label: "transport"
+				},
+			]
+		}
+	}
+
+	ca: """
+		-----BEGIN CERTIFICATE-----
+		MIIBgTCCATOgAwIBAgIUTf/ZBSJEi8IQb8Ndoxp4/tHB/lcwBQYDK2VwMEAxCzAJ
+		BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxFzAVBgNVBAMMDkRD
+		SCBSb290IENBIFIzMB4XDTI0MDIxNzIwMjkzNloXDTM0MDIxNzIwMjkzNlowQDEL
+		MAkGA1UEBhMCVVMxGDAWBgNVBAoMD0R1c3RpbiBDLiBIYXRjaDEXMBUGA1UEAwwO
+		RENIIFJvb3QgQ0EgUjMwKjAFBgMrZXADIQDORylVcWcxwGDJvsJIc2NctfNfDaIU
+		T6mLebahKdshaKM/MD0wHQYDVR0OBBYEFLZoxAHBvWqbLWMga/DAAlG9ido5MA8G
+		A1UdEwEB/wQFMAMBAf8wCwYDVR0PBAQDAgEGMAUGAytlcANBANLV79joVd9s9bmL
+		0a91HqvOotOnN/416Ek4UTl95jIqy/TvTfRjXX56wSALXqP1iYQM5i3zk3gVEhh4
+		DaY+6wQ=
+		-----END CERTIFICATE-----
+		"""
+
+	config: schema.#PromtailConfig & {
+		clients: [
+			{
+				url: "https://loki.pyrocufflink.blue/loki/api/v1/push"
+				tls_config: {
+					ca_file: "/etc/promtail/ca.crt"
+				}
+			},
+		]
+	}
+}
+
+let Marshal = yaml.Marshal
+
+#promtail: {
+	#scrape: [...schema.#ScrapeConfig] | *[promtail.scrape.journal]
+	ca: string | *promtail.ca
+
+	config: promtail.config & {
+		scrape_configs: #scrape
+	}
+
+	yaml: Marshal(config)
+}

host/loki0.pyrocufflink.blue.cue

@@ -4,3 +4,7 @@ import (
 
 ssh:  prod.ssh
 sudo: prod.sudo
+
+promtail: prod.#promtail
+
+loki: prod.loki

host/loki0.pyrocufflink.blue.post.sh

@@ -1 +1,2 @@
 . scripts/no-coreos-default-sudo.sh
+. scripts/loki-cert.sh

host/nut0.pyrocufflink.blue.cue

@@ -4,15 +4,16 @@ import (
 	"du5t1n.me/cfg/env/prod"
 )
 
+ssh:  prod.ssh
+sudo: prod.sudo
+
+promtail: prod.#promtail
+
+collectd: prod.collectd
+
 nut: prod.nut
 
 nut: monitor: prod.#nut_monitor & {
 	#server:   "localhost"
 	#username: "upsmon"
 }
-
-collectd: prod.collectd
-
-ssh: prod.ssh
-
-sudo: prod.sudo

host/nvr1.pyrocufflink.blue.cue

@@ -5,12 +5,13 @@ import (
 	"du5t1n.me/cfg/app/nut/schema"
 )
 
+ssh:  prod.ssh
+sudo: prod.sudo
+
+promtail: prod.#promtail
+
 nut: monitor: schema.#NutMonitor
 
 nut: monitor: prod.#nut_monitor & {
 	#username: "nvr1"
 }
-
-ssh: prod.ssh
-
-sudo: prod.sudo

host/serial1.pyrocufflink.blue.cue

@@ -6,3 +6,5 @@ import (
 
 ssh:  prod.ssh
 sudo: prod.sudo
+
+promtail: prod.#promtail

instructions/loki0.pyrocufflink.blue.cue

@@ -2,12 +2,14 @@ import (
 	"list"
 
 	"du5t1n.me/cfg/app/collectd"
+	"du5t1n.me/cfg/app/promtail"
 	"du5t1n.me/cfg/app/loki"
 	"du5t1n.me/cfg/env/prod"
 )
 
 render: list.Concat([
+	prod.templates,
 	collectd.templates,
 	loki.templates,
-	prod.templates,
+	promtail.templates,
 ])

instructions/nut0.pyrocufflink.blue.cue

@@ -5,13 +5,15 @@ import (
 
 	"du5t1n.me/cfg/app/collectd"
 	"du5t1n.me/cfg/app/nut"
+	"du5t1n.me/cfg/app/promtail"
 	"du5t1n.me/cfg/env/prod"
 )
 
 render: list.Concat([
+	prod.templates,
 	collectd.templates,
+	promtail.templates,
 	nut.templates,
 	nut.monitor.templates,
 	nut.collectd.templates,
-	prod.templates,
 ])

instructions/nvr1.pyrocufflink.blue.cue

@@ -4,11 +4,13 @@ import (
 	"list"
 
 	"du5t1n.me/cfg/app/nut"
+	"du5t1n.me/cfg/app/promtail"
 	"du5t1n.me/cfg/env/prod"
 )
 
 render: list.Concat([
+	prod.templates,
+	promtail.templates,
 	nut.sysusers.templates,
 	nut.monitor.templates,
-	prod.templates,
 ])

instructions/serial1.pyrocufflink.blue.cue

@@ -4,10 +4,12 @@ import (
 	"list"
 
 	"du5t1n.me/cfg/app/collectd"
+	"du5t1n.me/cfg/app/promtail"
 	"du5t1n.me/cfg/env/prod"
 )
 
 render: list.Concat([
-	collectd.templates,
 	prod.templates,
+	collectd.templates,
+	promtail.templates,
 ])

scripts/loki-cert.sh

@@ -0,0 +1,3 @@
+if [ ! -f /host/etc/loki/server.cer ] || [ ! -f /host/etc/loki/server.key ]; then
+    systemctl start fetchcert
+fi

templates/fetchcert/ca.crt

@@ -0,0 +1 @@
+{{ fetchcert.ca_cert }}

templates/fetchcert/fetchcert.container

@@ -0,0 +1,22 @@
+[Unit]
+Description=Fetch HTTPS certificate from Kubernetes Secret API
+Wants=network-online.target
+After=network-online.target
+
+[Container]
+Image=git.pyrocufflink.net/containerimages/fetchcert
+Exec={{ fetchcert.namespace }} {{ fetchcert.secret }} /etc/fetchcert/certs/{{ fetchcert.key }} /etc/fetchcert/certs/{{ fetchcert.cert }}
+ReadOnly=yes
+ReadOnlyTmpfs=yes
+Volume=/etc/fetchcert:/etc/fetchcert:ro
+Volume=/etc/fetchcert/certs:/etc/fetchcert/certs:rw,z
+Environment=KUBERNETES_URL={{ fetchcert.kubernetes_url }}
+AddCapability=CAP_CHOWN
+DropCapability=all
+NoNewPrivileges=yes
+
+[Service]
+Type=oneshot
+SuccessExitStatus=20
+ExecStartPre=/bin/mkdir -p /etc/fetchcert/certs
+ExecStopPost=-/etc/fetchcert/postupdate.sh

templates/fetchcert/fetchcert.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Periodically fetch certificate from Kubernetes
+
+[Timer]
+OnCalendar=*-*-* 0:0:0
+RandomizedDelaySec=8h
+
+[Install]
+WantedBy=timers.target

templates/fetchcert/postupdate.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if [ ${EXIT_STATUS:-1} -ne 20 ]; then
+    exit 0
+fi
+{% if fetchcert.postupdate %}
+{{ fetchcert.postupdate }}
+{% endif -%}

templates/fetchcert/token

@@ -0,0 +1 @@
+{{ fetchcert.token | decrypt }}

templates/loki/Caddyfile

@@ -0,0 +1,33 @@
+loki.pyrocufflink.blue {
+	tls {
+		client_auth {
+			mode verify_if_given
+			trusted_ca_cert_file /etc/caddy/client-ca.crt
+		}
+	}
+	@anonymous {
+		expression {tls_client_subject} == null
+	}
+	@grafana {
+		header X-Grafana-User *
+	}
+	handle @anonymous {
+		route /loki/api/v1/push {
+			reverse_proxy 127.0.0.1:3100
+		}
+		route /metrics {
+			reverse_proxy 127.0.0.1:3100
+		}
+		route /ready {
+			reverse_proxy 127.0.0.1:3100
+		}
+		respond 403
+	}
+	handle @grafana {
+		reverse_proxy 127.0.0.1:3100
+	}
+	tls loki@pyrocufflink.blue {
+		ca https://ca.pyrocufflink.blue:32599/acme/acme/directory
+		ca_root /etc/caddy/acme-ca.crt
+	}
+}

templates/loki/caddy-acme-ca.crt

@@ -0,0 +1 @@
+{{ loki.caddy.acme_ca }}

templates/loki/caddy-client-ca.crt

@@ -0,0 +1 @@
+{{ loki.caddy.client_ca }}

templates/loki/caddy.container

@@ -0,0 +1,22 @@
+[Unit]
+Description=Caddy web server
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+Image=docker.io/library/caddy:2
+Volume=/etc/caddy:/etc/caddy:ro
+Volume=/var/lib/caddy/config:/config/caddy:rw,z
+Volume=/var/lib/caddy/data:/data/caddy:rw,z
+ReadOnly=yes
+ReadOnlyTmpfs=yes
+Network=host
+AddCapability=CAP_NET_BIND_SERVICE
+DropCapability=all
+
+[Service]
+StateDirectory=%N/data %N/config
+ExecReload=/usr/bin/podman exec systemd-%N caddy reload -c /etc/caddy/Caddyfile
+
+[Install]
+WantedBy=multi-user.target

templates/loki/config.yml

@@ -2,6 +2,7 @@ auth_enabled: false
 
 server:
   http_listen_port: 3100
+  http_listen_address: 127.0.0.1
   grpc_listen_port: 9096
 
 common:

templates/loki/loki.container

@@ -6,13 +6,14 @@ Wants=network-online.target
 
 [Service]
 StateDirectory=%P
+ExecReload=/usr/bin/podman kill --cidfile=%t/%N.cid --signal HUP
 
 [Container]
 Image=docker.io/grafana/loki:2.9.4
 Exec=-config.file=/etc/loki/config.yml
 Volume=%S/%P:/var/lib/loki:rw,Z,U
 Volume=/etc/loki:/etc/loki:ro
-PublishPort=3100:3100
+Network=host
 
 [Install]
 WantedBy=multi-user.target

templates/promtail/ca.crt

@@ -0,0 +1 @@
+{{ promtail.ca }}

templates/promtail/config.yml

@@ -0,0 +1 @@
+{{ promtail.yaml -}}

templates/promtail/promtail.container

@@ -0,0 +1,24 @@
+# vim: set ft=systemd :
+[Unit]
+Description=Grafana Loki Promtail Agent
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+StateDirectory=%P
+TimeoutStartSec=7m
+
+[Container]
+Image=docker.io/grafana/promtail:2.9.4
+Exec=-config.file=/etc/promtail/config.yml
+Volume=%S/%P:/var/lib/promtail:rw,Z,U
+Volume=/etc/machine-id:/etc/machine-id:ro
+Volume=/etc/promtail:/etc/promtail:ro
+Volume=/run/log:/run/log:ro
+Volume=/var/log:/var/log:ro
+PublishPort=9080:9080
+# container_t is not allowed to read var_log_t / syslogd_var_run_t
+SecurityLabelDisable=true
+
+[Install]
+WantedBy=multi-user.target