dustin/configpolicy
1
1
Fork 0
Code Issues Releases Activity

hosts: Add mtrcs0.p.r

Browse Source 
*mtrcs0.pyrocufflink.red* is a Raspberry Pi CM4 on a Waveshare
CM4-IO-BASE-B carrier board with a NVMe SSD.  It runs a custom OS built
using Buildroot, and is not a member of the *pyrocufflink.blue* AD
domain.

*mtrcs0.p.r* hosts Victoria Metrics/`vmagent`, `vmalert`, AlertManager,
and Grafana.  I've created a unique group and playbook for it,
*metricspi*, to manage all these applications together.
This commit is contained in:
Dustin
2022-02-26 11:01:20 -06:00
parent 7c654031f0
commit 4ddbc9f256
14 changed files with 447 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
group_vars/metricspi/alertmanager.yml Normal file
View File

@@ -0,0 +1,24 @@
alertmanager_config:
global:
smtp_from: prometheus@pyrocufflink.blue
smtp_smarthost: mail.pyrocufflink.blue:25
smtp_require_tls: false
route:
receiver: default-email
group_by:
- ...
routes:
- receiver: default-email
group_wait: 1m
group_by:
- alertname
repeat_interval: 120h
match:
job: homeassistant
receivers:
- name: default-email
email_configs:
- to: gyrfalcon@ebonfire.com
send_resolved: true

48
group_vars/metricspi/alerts.yml Normal file
View File

@@ -0,0 +1,48 @@
vmalert_rules:
groups:
- name: default alert
rules:
- alert: DiskUsage
expr: >-
sum(collectd_df_df_complex{type!="free"}) by (instance, df) / sum(collectd_df_df_complex{df!="var-log"}) by (instance, df) > .75
or sum(collectd_df_df_complex{type!="free"}) by (instance, df) / sum(collectd_df_df_complex{df="var-log"}) by (instance, df) > .95
for: 2h
- alert: TheWebsiteIsDown
expr: >-
probe_success{job="websites"} == 0
for: 10m
- alert: Missing Metrics
expr: >-
up{instance!~"vmhost.*"} == 0
for: 10m
- alert: NUT is offline
expr: >-
absent(collectd_nut_percent)
- name: Bitwarden
rules:
- alert: vaultwarden is not running
expr: >-
collectd_processes_ps_count_processes{processes="vaultwarden"} < 1
for: 5m
- name: Active Directory
rules:
- alert: samba is not running
expr: >-
collectd_processes_ps_count_processes{processes=~"samba|smbd|winbindd|krb5kdc"} < 1
for: 5m
- name: Graylog
rules:
- alert: unprocessed messages
expr: >-
org_graylog2_journal_entries_uncommitted > 100
for: 1h
- name: mdraid
rules:
- alert: mdraid missing disk
expr: collectd_md_md_disks{type="missing"} != 0
- alert: mdraid failed disk
expr: collectd_md_md_disks{type="failed"} != 0

45
group_vars/metricspi/blackbox.yml Normal file
View File

@@ -0,0 +1,45 @@
blackbox_modules:
icmp:
prober: icmp
timeout: 5s
http:
prober: http
timeout: 5s
http:
method: GET
headers:
Accept-Language: en-US
Accept-Charset: utf-8
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
tcp:
prober: tcp
timeout: 5s
dns_recursive:
prober: dns
timeout: 5s
dns:
query_name: news.ycombinator.com
query_type: A
dns_pyrocufflink:
prober: dns
timeout: 2s
dns:
query_name: pyrocufflink.blue
query_type: SOA
validate_answer_rrs:
fail_if_not_matches_regexp:
- 'pyrocufflink\.blue\.\t\d+\tIN\tSOA\tdc\d+\.pyrocufflink\.blue.*'
smtp:
prober: tcp
timeout: 5s
tcp:
query_response:
- expect: "^220 ([^ ]+) ESMTP (.+)$"
- send: 'EHLO prober\r'
- expect: '^250[ -]SMTPUTF8'
- send: 'QUIT\r'

18
group_vars/metricspi/grafana.yml Normal file
View File

@@ -0,0 +1,18 @@
grafana_allow_anonymous: true
grafana_allow_sign_up: false
grafana_allow_org_create: false
grafana_ldap: true
grafana_ldap_allow_sign_up: true
grafana_ldap_host: dc2.pyrocufflink.blue
grafana_ldap_use_ssl: true
grafana_ldap_root_ca_cert: /etc/ssl/certs/dch-root-ca.crt
grafana_ldap_bind_dn: CN=svc.grafana,CN=Users,DC=pyrocufflink,DC=blue
grafana_ldap_bind_password: '{{ vault_grafana_ldap_bind_password }}'
grafana_ldap_search_filter: '(sAMAccountName=%s)'
grafana_ldap_base_dn: DC=pyrocufflink,DC=blue
grafana_ldap_group_mapping:
- group_dn: CN=Grafana Admins,CN=Users,DC=pyrocufflinke,DC=blue
role: Admin
grafana_admin: true
- group_dn: '*'
role: Viewer

2
group_vars/metricspi/main.yml Normal file
View File

@@ -0,0 +1,2 @@
vm_config: null
victoria_metrics_server_name: metrics.pyrocufflink.blue

231
group_vars/metricspi/scrape.yml Normal file
View File

@@ -0,0 +1,231 @@
vmagent_scrape_configs:
- job_name: vmagent
static_configs:
- targets:
- '[::1]:8429'
- job_name: speedtest
scrape_interval: 10m
scrape_timeout: 1m
metrics_path: /probe
static_configs:
- targets:
- 172.30.0.1:9516
- job_name: blackbox
metrics_path: /probe
params:
module:
- icmp
static_configs:
- targets:
- 1.1.1.1
- 8.8.8.8
- 9.9.9.9
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: '[::1]:9115'
- job_name: brandon
scrape_interval: 5s
metrics_path: /probe
params:
module:
- icmp
static_configs:
- targets:
- 173.172.96.1
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: '[::1]:9115'
- job_name: websites
scrape_interval: 5m
metrics_path: /probe
params:
module:
- http
static_configs:
- targets:
- http://dustin.hatch.name/
- https://darkchestofwonders.us/
- http://nratonpass.com/
- http://pyrocufflink.net/
- http://ebonfire.com/
- http://chmod777.sh/
- https://hatch.chat/_matrix/client/versions
- https://nextcloud.pyrocufflink.net/
- https://bitwarden.pyrocufflink.blue/
- https://git.pyrocufflink.blue/
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: '[::1]:9115'
- job_name: sensors
scrape_interval: 10s
file_sd_configs:
- files:
- sensors.yml
- job_name: graylog
scrape_interval: 1m
scheme: https
metrics_path: /api/plugins/org.graylog.plugins.metrics.prometheus/metrics
basic_auth:
username: >-
{{ vault_graylog_scrape_token }}
password: token
static_configs:
- targets:
- graylog.pyrocufflink.blue:443
- job_name: collectd
scrape_interval: 10s
honor_labels: true
static_configs:
- targets:
- gw1.pyrocufflink.blue
- vmhost0.pyrocufflink.blue
- vmhost1.pyrocufflink.blue
file_sd_configs:
- files:
- /etc/prometheus/scrape-collectd.yml
relabel_configs:
- source_labels: [__address__]
target_label: __address__
replacement: '$1:9103'
- job_name: homeassistant
scrape_interval: 1m
scheme: https
metrics_path: /api/prometheus
bearer_token: >-
{{ vault_homeassistant_scrape_token }}
static_configs:
- targets:
- homeassistant.pyrocufflink.blue
- job_name: sambadc
scrape_interval: 1m
metrics_path: /probe
params:
module:
- tcp
dns_sd_configs:
- names:
- _ldap._tcp.pyrocufflink.blue
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: '[::1]:9115'
- job_name: gitea
scrape_interval: 1m
scheme: https
static_configs:
- targets:
- git.pyrocufflink.blue
- job_name: synapse
scrape_interval: 1m
metrics_path: /_synapse/metrics
static_configs:
- targets:
- matrix0.pyrocufflink.blue
relabel_configs:
- source_labels: [__address__]
target_label: instance
- source_labels: [__address__]
target_label: __address__
replacement: '$1:9000'
- job_name: dns_recursive
scrape_interval: 1m
metrics_path: /probe
params:
module:
- dns_recursive
static_configs:
- targets:
- 172.30.0.4
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: '[::1]:9115'
- job_name: dns_pyrocufflink
scrape_interval: 1m
metrics_path: /probe
params:
module:
- dns_pyrocufflink
static_configs:
- targets:
- 172.30.0.10
- 172.30.0.9
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: '[::1]:9115'
- job_name: smtp
scrape_interval: 1m
metrics_path: /probe
params:
module:
- smtp
dns_sd_configs:
- names:
- mail.pyrocufflink.blue
type: A
port: 25
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: '[::1]:9115'
- job_name: unifi
scrape_interval: 1m
static_configs:
- targets:
- unifi.pyrocufflink.blue:9130
- job_name: jenkins
scrape_interval: 1m
metrics_path: /prometheus/
scheme: https
static_configs:
- targets:
- jenkins.pyrocufflink.blue
- job_name: burp
scrape_interval: 270s
scrape_timeout: 30s
static_configs:
- targets:
- burp.pyrocufflink.blue:9645

24
group_vars/metricspi/secrets Normal file
View File

@@ -0,0 +1,24 @@
$ANSIBLE_VAULT;1.1;AES256
33663232353935666561613765313939376662396331326332373133343437343663323761633066
3361353962646561633066353632373139666562323863330a346339323831376664636366336332
63333662303862633938643734366662623434633562383033663637336330306161613336346135
3566386161363438660a393436353461366433643738313661656434303462376436626162623061
30626361623737636337383336626331633332393233656263303731313239663838313635356432
34333733636139633766316162333266613533636234303830326632303765616330663034636430
63373866373662383438636563643833323765653666663837663162356464326332396138656233
33326465313437343232316132613538626537333332613531613332343566626337613835313639
61353464633239353863313561646235316466393762306238306538376531353831643930336231
34643366633664633937656232316166393835333461363564303938623633653432316561323032
65666330316534306564653731366632613934343966653034393661363035356639656662613339
36373131346561343731666331623833633932373765356263363434666566616131663833656364
32663136303838376537346636633363666630386339633564633662343035653737616439626163
63363234613237656639333035396539626337323932626632666663393864613063306138323465
34343761613637656631323938323465376535613461313231323737636235396535363234646437
34633230333236326331333738323161376230373766393336643636376339396530316632363638
65373530613565663236666561326539343033633230616561633538313632343036346261323134
65663166353632656635653365366664313139396562336133656336396334363063653332393136
32663838363237626562366662383035333762366432323734616633346334646365323733303763
38663461333431613630303330343764633963646432363537616466626133346136623535656539
39353536633364653563366466363338643730663866626531653164663232323663653366363266
62363535363835336262646236346637633033353731666335373663663537356362656362626265
643234623230343334656464633134326136

4
group_vars/metricspi/vmalert.yml Normal file
View File

@@ -0,0 +1,4 @@
vmalert_datasource_url: http://[::1]:8428
vmalert_notifier_url: http://[::1]:9093
vmalert_remote_read_url: http://[::1]:8428
vmalert_remote_write_url: http://[::1]:8428