Dustin C. Hatch
d74e26d527
I don't like having alerts sent by e-mail. Since I don't get e-mail notifications on my watch, I often do not see alerts for quite some time. They are also much harder to read in an e-mail client (Fastmail web an K-9 Mail both display them poorly). I would much rather have them delivered via _ntfy_, just like all the rest of the ephemeral notifications I receive. Fortunately, it is easy enough to integrate Alertmanager and _ntfy_ using the webhook notifier in Alertmanager. Since _ntfy_ does not natively support the Alertmanager webhook API, though, a bridge is necessary to translate from one data format to the other. There are a few options for this bridge, but I chose [alexbakker/alertmanager-ntfy][0] because it looked the most complete while also having the simplest configuration format. Sadly, it does not expose any Prometheus metrics itself, and since it's deployed in the _victoria-metrics_ namespace, it needs to be explicitly excluded from the VMAgent scrape configuration. [0]: https://github.com/alexbakker/alertmanager-ntfy
205 lines
4.7 KiB
YAML
205 lines
4.7 KiB
YAML
apiVersion: kustomize.config.k8s.io/v1beta1
|
|
kind: Kustomization
|
|
|
|
namespace: victoria-metrics
|
|
|
|
labels:
|
|
- pairs:
|
|
app.kubernetes.io/instance: victoria-metrics
|
|
includeSelectors: true
|
|
- pairs:
|
|
app.kubernetes.io/part-of: victoria-metrics
|
|
includeSelectors: false
|
|
|
|
resources:
|
|
- namespace.yaml
|
|
- secrets.yaml
|
|
- vmstorage.yaml
|
|
- vmselect.yaml
|
|
- vminsert.yaml
|
|
- vmagent.yaml
|
|
- vmalert.yaml
|
|
- alertmanager.yaml
|
|
- alertmanager-ntfy.yaml
|
|
- blackbox-exporter.yaml
|
|
- ingress.yaml
|
|
- ../dch-root-ca
|
|
|
|
configMapGenerator:
|
|
- name: vmagent
|
|
files:
|
|
- scrape.yml
|
|
options:
|
|
disableNameSuffixHash: true
|
|
|
|
- name: vmalert-rules
|
|
files:
|
|
- alerts.yml
|
|
options:
|
|
disableNameSuffixHash: true
|
|
|
|
- name: alertmanager
|
|
files:
|
|
- alertmanager.yml=alertmanager.config.yml
|
|
options:
|
|
disableNameSuffixHash: true
|
|
|
|
- name: alertmanager-ntfy
|
|
files:
|
|
- config.yml=alertmanager-ntfy.config.yml
|
|
|
|
- name: blackbox
|
|
files:
|
|
- blackbox.yml
|
|
options:
|
|
disableNameSuffixHash: true
|
|
|
|
replicas:
|
|
# When changing the number of vmstorage replicas, be sure to update
|
|
# the storageNode value for vmselect and vminsert. Also, the
|
|
# replicationFactor setting may need adjusted.
|
|
- name: vmstorage
|
|
count: 3
|
|
- name: vmselect
|
|
count: 2
|
|
- name: vminsert
|
|
count: 2
|
|
- name: vmagent
|
|
count: 2
|
|
- name: vmalert
|
|
count: 2
|
|
# When changing the number of alertmanager replicas, be sure to update
|
|
# the notifier URL value for vmalert and the peer addresses provided to
|
|
# Alertmanager itself.
|
|
- name: alertmanager
|
|
count: 2
|
|
|
|
patches:
|
|
- patch: |
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: vmstorage
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: vmstorage
|
|
env:
|
|
- name: vmstorage_dedup_minScrapeInterval
|
|
value: 1m
|
|
- name: vmstorage_retentionPeriod
|
|
value: 5y
|
|
|
|
- patch: |
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vmselect
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: vmselect
|
|
env:
|
|
- name: vmselect_storageNode
|
|
value: vmstorage-0.vmstorage,vmstorage-1.vmstorage,vmstorage-2.vmstorage
|
|
- name: vmselect_replicationFactor
|
|
value: '2'
|
|
|
|
- patch: |
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vminsert
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: vminsert
|
|
env:
|
|
- name: vminsert_storageNode
|
|
value: vmstorage-0.vmstorage,vmstorage-1.vmstorage,vmstorage-2.vmstorage
|
|
- name: vminsert_dedup_minScrapeInterval
|
|
value: 1m
|
|
- name: vminsert_replicationFactor
|
|
value: '2'
|
|
|
|
- patch: |
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: vmagent
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: vmagent
|
|
env:
|
|
- name: SCRAPE_GRAYLOG_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vmagent
|
|
key: graylog.token
|
|
optional: true
|
|
volumeMounts:
|
|
- mountPath: /run/dch-ca
|
|
name: dch-ca
|
|
readOnly: true
|
|
- mountPath: /run/secrets/vmagent
|
|
name: secrets
|
|
readOnly: true
|
|
- mountPath: /scrape/collectd
|
|
name: scrape-collectd
|
|
readOnly: true
|
|
volumes:
|
|
- name: scrape-collectd
|
|
configMap:
|
|
name: scrape-collectd
|
|
optional: true
|
|
- name: secrets
|
|
secret:
|
|
secretName: vmagent
|
|
- name: dch-ca
|
|
configMap:
|
|
name: dch-root-ca
|
|
optional: true
|
|
|
|
- patch: |
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vmalert
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: vmalert
|
|
env:
|
|
- name: vmalert_http_pathPrefix
|
|
value: /vmalert
|
|
- name: vmalert_notifier_url
|
|
value: http://alertmanager-0.alertmanager:9093,http://alertmanager-1.alertmanager:9093
|
|
startupProbe:
|
|
httpGet:
|
|
path: /vmalert/health
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /vmalert/health
|
|
|
|
- patch: |
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: alertmanager
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: alertmanager
|
|
args:
|
|
- --config.file=/etc/alertmanager/alertmanager.yml
|
|
- --storage.path=/alertmanager
|
|
- --cluster.peer=alertmanager-0.alertmanager:9094
|
|
- --cluster.peer=alertmanager-1.alertmanager:9094
|