Dustin C. Hatch
8f088fb6ae
Since *mtrcs0.pyrocufflink.blue* (the Metrics Pi) seems to be dying, I decided to move monitoring and alerting into Kubernetes. I was originally planning to have a single, dedicated virtual machine for Victoria Metrics and Grafana, similar to how the Metrics Pi was set up, but running Fedora CoreOS instead of a custom Buildroot-based OS. While I was working on the Ignition configuration for the VM, it occurred to me that monitoring would be interrupted frequently, since FCOS updates weekly and all updates require a reboot. I would rather not have that many gaps in the data. Ultimately I decided that deploying a cluster with Kubernetes would probably be more robust and reliable, as updates can be performed without any downtime at all. I chose not to use the Victoria Metrics Operator, but rather handle the resource definitions myself. Victoria Metrics components are not particularly difficult to deploy, so the overhead of running the operator and using its custom resources would not be worth the minor convenience it provides.
87 lines
2.0 KiB
YAML
87 lines
2.0 KiB
YAML
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: alertmanager
|
|
labels:
|
|
app.kubernetes.io/name: alertmanager
|
|
app.kubernetes.io/component: alertmanager
|
|
spec:
|
|
ports:
|
|
- port: 9093
|
|
name: alertmanager
|
|
selector:
|
|
app.kubernetes.io/name: alertmanager
|
|
app.kubernetes.io/component: alertmanager
|
|
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: alertmanager
|
|
labels:
|
|
app.kubernetes.io/name: alertmanager
|
|
app.kubernetes.io/component: alertmanager
|
|
spec:
|
|
serviceName: alertmanager
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: alertmanager
|
|
app.kubernetes.io/component: alertmanager
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: alertmanager
|
|
app.kubernetes.io/component: alertmanager
|
|
spec:
|
|
containers:
|
|
- name: alertmanager
|
|
image: docker.io/prom/alertmanager:v0.26.0
|
|
ports:
|
|
- containerPort: 9093
|
|
name: http
|
|
readinessProbe: &probe
|
|
httpGet:
|
|
port: http
|
|
path: /-/ready
|
|
periodSeconds: 60
|
|
startupProbe:
|
|
<<: *probe
|
|
periodSeconds: 1
|
|
successThreshold: 1
|
|
failureThreshold: 30
|
|
timeoutSeconds: 1
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
readOnlyRootFilesystem: true
|
|
volumeMounts:
|
|
- mountPath: /etc/alertmanager
|
|
name: config
|
|
readOnly: true
|
|
- mountPath: /alertmanager
|
|
name: alertmanager
|
|
subPath: data
|
|
securityContext:
|
|
fsGroup: 2093
|
|
runAsGroup: 2093
|
|
runAsNonRoot: true
|
|
runAsUser: 2093
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: alertmanager
|
|
volumeClaimTemplates:
|
|
- apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: alertmanager
|
|
labels:
|
|
app.kubernetes.io/name: alertmanager
|
|
app.kubernetes.io/component: alertmanager
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
resources:
|
|
requests:
|
|
storage: 4G
|