From ee3e078b20c365fb44ae5f95fdab4e38871e85fb Mon Sep 17 00:00:00 2001
From: "Dustin C. Hatch" <dustin@hatch.name>
Date: Thu, 17 Oct 2024 06:58:48 -0500
Subject: [PATCH] v-m/alerts: Add alerts for Restic backups

---
 victoria-metrics/alerts.yml | 40 +++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/victoria-metrics/alerts.yml b/victoria-metrics/alerts.yml
index 0bcf521..5ea036c 100644
--- a/victoria-metrics/alerts.yml
+++ b/victoria-metrics/alerts.yml
@@ -107,3 +107,43 @@ groups:
     expr: >-
       count(longhorn_volume_robustness==3) > 0
     for: 5m
+
+- name: Restic
+  rules:
+  - alert: Repository Check Failed
+    expr: >-
+      min(restic_check_success) by (job) < 1
+    annotations:
+      summary: Errors found in restic repository data
+      description: >-
+        The Restic repository has one or more problems that may result in data
+        loss.  Check the restic-exporter log for more information and correct
+        the issue as soon as possible.
+  - alert: Last Backup Age
+    expr: >-
+      time() - restic_backup_timestamp > 604800
+    annotations:
+      summary: A Restic client has not backed up recently
+      description: >-
+        Clients are scheduled to back up every day, but at least one has not
+        been backed up in at least 7 days.  Check the Restic configuration on
+        that system to ensure backups are running properly.
+  - alert: No File Changes
+    expr: >-
+      max_over_time(
+        abs(
+          delta(
+            sum(restic_backup_size_total{
+              client_hostname!="pxe0.pyrocufflink.blue",
+              client_hostname!="web0.pyrocufflink.blue",
+            })
+              by (client_hostname, client_username)
+          )
+        )[7d]
+      ) == 0
+    annotations:
+      summary: The size of the Restic backup has not changed
+      description: >-
+        The size of the Restic backup for a particular client has not changed
+        in at least 7 days.  This may indicate that the backup configuration
+        is incorrect.