From 51aaccc8615c9bbe330657e972753f384834dc68 Mon Sep 17 00:00:00 2001
From: "Dustin C. Hatch" <dustin@hatch.name>
Date: Wed, 17 Jan 2024 17:11:49 -0600
Subject: [PATCH] collectd: Deploy collectd in a container

I keep going back-and-forth on whether or not collectd should run in a
container on Fedora CoreOS machines.  On the one hand, running it
directly on the host allows it to monitor filesystem usage by mount
point, which is consistent with how non-FCOS machines are monitored.
On the other hand, installing packages on FCOS with `rpm-ostree` is a
nightmare.  It's _incredibly_ slow.  There's also occasionally issues
installing packages if the base layer has not been updated in a while
and the new packages require an existing package to be updated.

For the NUT server specifically, I have changed my mind again: the
*collectd-nut* package depends on *nut-client*, which in turn depends on
Python.  I definitely want to avoid installing Python on the host, but I
do not want to lose the ability to monitor the UPSs via collectd.  Using
a container, I can strip out the unnecessary bits of *nut-client* and
avoid installing Python at all.  I think that's worth having to monitor
filesystem usage by device instead of by mount point.
---
 app/collectd/schema/schema.cue              |  5 +++
 app/collectd/templates.cue                  | 40 +++++++++++++++++++++
 templates/collectd/collectd-df.conf         |  8 +++++
 templates/collectd/collectd-plugins.conf    |  9 +++++
 templates/collectd/collectd-prometheus.conf |  5 +++
 templates/collectd/collectd.container       | 21 +++++++++++
 6 files changed, 88 insertions(+)
 create mode 100644 app/collectd/schema/schema.cue
 create mode 100644 app/collectd/templates.cue
 create mode 100644 templates/collectd/collectd-df.conf
 create mode 100644 templates/collectd/collectd-plugins.conf
 create mode 100644 templates/collectd/collectd-prometheus.conf
 create mode 100644 templates/collectd/collectd.container

diff --git a/app/collectd/schema/schema.cue b/app/collectd/schema/schema.cue
new file mode 100644
index 0000000..28ebfcc
--- /dev/null
+++ b/app/collectd/schema/schema.cue
@@ -0,0 +1,5 @@
+package schema
+
+#Collectd: {
+	...
+}
diff --git a/app/collectd/templates.cue b/app/collectd/templates.cue
new file mode 100644
index 0000000..33b0ecd
--- /dev/null
+++ b/app/collectd/templates.cue
@@ -0,0 +1,40 @@
+package collectd
+
+import "du5t1n.me/cfg/base/schema/instructions"
+
+templates: [...instructions.#RenderInstruction] & [
+	{
+		template: "collectd/collectd-df.conf"
+		dest:     "/etc/collectd.d/df.conf"
+		hooks: {
+			changed: [{run: "systemctl try-restart collectd"}]
+		}
+	},
+	{
+		template: "collectd/collectd-plugins.conf"
+		dest:     "/etc/collectd.d/plugins.conf"
+		hooks: {
+			changed: [{run: "systemctl try-restart collectd"}]
+		}
+	},
+	{
+		template: "collectd/collectd-prometheus.conf"
+		dest:     "/etc/collectd.d/prometheus.conf"
+		hooks: {
+			changed: [{run: "systemctl try-restart collectd"}]
+		}
+	},
+	{
+		template: "collectd/collectd.container"
+		dest:     "/etc/containers/systemd/collectd.container"
+		hooks: {
+			changed: [
+				{
+					run:       "systemctl daemon-reload"
+					immediate: true
+				},
+				{run: "systemctl restart collectd"},
+			]
+		}
+	},
+]
diff --git a/templates/collectd/collectd-df.conf b/templates/collectd/collectd-df.conf
new file mode 100644
index 0000000..f628bba
--- /dev/null
+++ b/templates/collectd/collectd-df.conf
@@ -0,0 +1,8 @@
+LoadPlugin df
+
+<Plugin df>
+    ReportByDevice true
+
+    FSType overlay
+    IgnoreSelected true
+</Plugin>
diff --git a/templates/collectd/collectd-plugins.conf b/templates/collectd/collectd-plugins.conf
new file mode 100644
index 0000000..c961a39
--- /dev/null
+++ b/templates/collectd/collectd-plugins.conf
@@ -0,0 +1,9 @@
+LoadPlugin chrony
+LoadPlugin cpufreq
+LoadPlugin disk
+LoadPlugin entropy
+LoadPlugin processes
+LoadPlugin swap
+LoadPlugin tcpconns
+LoadPlugin thermal
+LoadPlugin uptime
diff --git a/templates/collectd/collectd-prometheus.conf b/templates/collectd/collectd-prometheus.conf
new file mode 100644
index 0000000..8283102
--- /dev/null
+++ b/templates/collectd/collectd-prometheus.conf
@@ -0,0 +1,5 @@
+LoadPlugin write_prometheus
+
+<Plugin write_prometheus>
+    Port 9103
+</Plugin>
diff --git a/templates/collectd/collectd.container b/templates/collectd/collectd.container
new file mode 100644
index 0000000..e257c45
--- /dev/null
+++ b/templates/collectd/collectd.container
@@ -0,0 +1,21 @@
+# vim: set ft=systemd :
+[Unit]
+Description=Collectd statistics daemon
+After=network.target
+Wants=network.target
+
+[Container]
+Image=git.pyrocufflink.net/containerimages/collectd:latest
+Volume=/etc/collectd.d:/etc/collectd.d:ro
+Volume=/run:/run:rw
+Tmpfs=/tmp
+Network=host
+SecurityLabelDisable=true
+PodmanArgs=--privileged
+PodmanArgs=--ipc=host
+PodmanArgs=--uts=host
+PodmanArgs=--pid=host
+PodmanArgs=--cgroupns=host
+
+[Install]
+WantedBy=multi-user.target