Merge branch 'loki'

deploy/loki1.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+# vim: set ts=4 :
+
+if ! virsh list --all --name | grep -qF loki1; then
+	./newvm.sh loki1 \
+		--fedora 40 \
+		--memory 4096,currentMemory=2048 \
+		--vcpus 2 \
+		--network network=prod,mac=52:54:00:51:3c:e9 \
+		--no-console \
+		-- \
+		--disk pool=default,size=128,cache=none \
+		|| exit
+	sleep 15
+fi
+
+ANSIBLE_HOST_KEY_CHECKING=false \
+ansible-playbook \
+	-l loki1.pyrocufflink.blue \
+	wait-for-host.yml \
+	bootstrap.yml \
+	datavol.yml \
+	pyrocufflink.yml \
+	loki.yml \
+	collectd.yml \
+	promtail.yml \
+	-u root \
+	-e ansible_host=loki1.local \
+	-e @join.creds \
+	|| exit

group_vars/loki.yml

@@ -0,0 +1,21 @@
+data_volumes:
+- dev: /dev/vdb
+  fstype: btrfs
+  mountpoint: /var/lib/loki
+
+loki_caddy_acme:
+  email: loki@pyrocufflink.blue
+  url: https://ca.pyrocufflink.blue/acme/acme/directory
+
+loki_caddy_client_ca: |+
+  -----BEGIN CERTIFICATE-----
+  MIIBlDCCAUagAwIBAgIUGNZ/ASP8F2ytev3YplTk4jA5a2EwBQYDK2VwMEgxCzAJ
+  BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxDTALBgNVBAsMBExv
+  a2kxEDAOBgNVBAMMB0xva2kgQ0EwHhcNMjQwMjIwMTUwMTQxWhcNMzQwMjIwMTUw
+  MTQxWjBIMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPRHVzdGluIEMuIEhhdGNoMQ0w
+  CwYDVQQLDARMb2tpMRAwDgYDVQQDDAdMb2tpIENBMCowBQYDK2VwAyEAnmMawEIo
+  WfzFaLgpSiaPD+DHg28NHknMFcs7XpyTM9CjQjBAMB0GA1UdDgQWBBTFth3c4S/f
+  y0BphQy9SucnKN2pLzASBgNVHRMBAf8ECDAGAQH/AgEAMAsGA1UdDwQEAwIBBjAF
+  BgMrZXADQQCn0JWERsXdJA4kMM45ZXhVgAciwLNQ8ikoucsJcbWBp7bSMjcMVi51
+  I+slotQvQES/vfqp/zZFNl7KKyeeQ0sD
+  -----END CERTIFICATE-----

group_vars/vm-hosts.yml

@@ -245,7 +245,7 @@ vm_autostart:
 - dc-grumbly
 - dc-headphone
 - delay 30s
-- loki0
+- loki1
 - delay 10s
 - db0
 - k8s-ctrl0

hosts

@@ -89,6 +89,9 @@ k8s-ctrl0.pyrocufflink.blue
 k8s-controller
 k8s-node
 
+[loki]
+loki1.pyrocufflink.blue
+
 [minio-backups]
 chromie.pyrocufflink.blue
 
@@ -143,6 +146,7 @@ file0.pyrocufflink.blue
 git0.pyrocufflink.blue
 haproxy0.pyrocufflink.blue
 k8s-ctrl0.pyrocufflink.blue
+loki1.pyrocufflink.blue
 nvr2.pyrocufflink.blue
 pxe0.pyrocufflink.blue
 smtp1.pyrocufflink.blue

loki.yml

@@ -0,0 +1,8 @@
+- hosts: loki
+  roles:
+  - role: loki
+    tags:
+    - loki
+  - role: loki-caddy
+    tags:
+    - loki-caddy

migration/loki0-to-loki1.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+# vim: set sw=4 ts=4 sts=4 et :
+
+set -e
+
+cleanup() {
+    eval $(ssh-agent -k)
+}
+
+trap cleanup INT TERM QUIT EXIT
+
+eval $(ssh-agent)
+sshca-cli user login
+ssh-add -K
+
+: || {
+ssh root@loki0.pyrocufflink.blue cat '>' .ssh/known_hosts <<EOF
+@cert-authority *.pyrocufflink.blue ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII24CZGosLMTny0a2eDB6KOG47FhlwVkTEFQNAYzKV0t
+EOF
+
+ssh root@loki0.pyrocufflink.blue btrfs subvol snapshot -r /var /var/.snapshot
+
+ssh root@loki0.pyrocufflink.blue btrfs send /var/.snapshot \| ssh root@loki1.pyrocufflink.blue btrfs receive /var/lib/loki
+
+ssh root@loki0.pyrocufflink.blue systemctl stop loki
+ssh root@loki0.pyrocufflink.blue btrfs subvol snapshot -r /var /var/.snapshot2
+ssh root@loki0.pyrocufflink.blue btrfs send -p /var/.snapshot /var/.snapshot2 \| ssh root@loki1.pyrocufflink.blue btrfs receive /var/lib/loki
+}
+
+ssh root@loki1.pyrocufflink.blue systemctl stop loki
+ssh root@loki1.pyrocufflink.blue cd /var/lib/loki '&&' rm -rf rules chunks tsdb-shipper-active tsdb-shipper-cache wal compactor
+ssh root@loki1.pyrocufflink.blue cp -a --reflink=always /var/lib/loki/.snapshot2/lib/loki/. /var/lib/loki
+
+ssh root@loki1.pyrocufflink.blue systemctl start loki
+
+nsupdate -g <<EOF
+del loki.pyrocufflink.blue
+add loki.pyrocufflink.blue 300 A 172.30.0.14
+send
+EOF

roles/loki-caddy/defaults/main.yml

@@ -0,0 +1 @@
+loki_caddy_server_name: loki.{{ ansible_domain }}

roles/loki-caddy/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+- role: caddy
+  tags: caddy

roles/loki-caddy/tasks/main.yml

@@ -0,0 +1,24 @@
+- name: ensure caddy is configured to proxy for loki
+  template:
+    src: Caddyfile.j2
+    dest: /etc/caddy/Caddyfile.d/loki.caddyfile
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - reload caddy
+  tags:
+  - config
+
+- name: ensure client ca is configured
+  copy:
+    dest: /etc/caddy/loki-client-ca.crt
+    content: >-
+      {{ loki_caddy_client_ca|d('') }}
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - reload caddy
+  tags:
+  - cert

roles/loki-caddy/templates/Caddyfile.j2

@@ -0,0 +1,33 @@
+{# vim: set sw=4 ts=4 sts=4 et : #}
+{{ loki_caddy_server_name }} {
+    tls {
+        client_auth {
+            mode verify_if_given
+            trusted_ca_cert_file /etc/caddy/loki-client-ca.crt
+        }
+    }
+    @anonymous {
+        expression {tls_client_subject} == null
+    }
+    @grafana {
+        header X-Grafana-User *
+    }
+    handle @anonymous {
+        route /loki/api/v1/push {
+            reverse_proxy 127.0.0.1:3100
+        }
+        route /metrics {
+            reverse_proxy 127.0.0.1:3100
+        }
+        route /ready {
+            reverse_proxy 127.0.0.1:3100
+        }
+        respond 403
+    }
+    handle @grafana {
+        reverse_proxy 127.0.0.1:3100
+    }
+    tls {{ loki_caddy_acme.email }} {
+        ca {{ loki_caddy_acme.url }}
+    }
+}

roles/loki/defaults/main.yml

@@ -0,0 +1,39 @@
+loki_config:
+  auth_enabled: false
+
+  server:
+    http_listen_port: 3100
+    http_listen_address: 127.0.0.1
+    grpc_listen_port: 9096
+
+  common:
+    instance_addr: 127.0.0.1
+    path_prefix: /var/lib/loki
+    storage:
+      filesystem:
+        chunks_directory: /var/lib/loki/chunks
+        rules_directory: /var/lib/loki/rules
+    replication_factor: 1
+    ring:
+      kvstore:
+        store: inmemory
+
+  query_range:
+    results_cache:
+      cache:
+        embedded_cache:
+          enabled: true
+          max_size_mb: 100
+
+  schema_config:
+    configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v12
+      index:
+        prefix: index_
+        period: 24h
+
+  query_scheduler:
+    max_outstanding_requests_per_tenant: 1024

roles/loki/files/loki.container

@@ -0,0 +1,25 @@
+# vim: set ft=systemd :
+[Unit]
+Description=Grafana Loki
+After=network-online.target
+Wants=network-online.target
+StartLimitIntervalSec=1m
+StartLimitBurst=60
+
+[Service]
+ExecStartPre=/bin/install -o 10001 -g 10001 -d %S/%P
+ExecStartPre=/bin/chcon -t container_file_t %S/%P
+ExecReload=/usr/bin/podman kill --cidfile=%t/%N.cid --signal HUP
+TimeoutStartSec=5m
+Restart=always
+RstartSec=1s
+
+[Container]
+Image=docker.io/grafana/loki:2.9.4
+Exec=-config.file=/etc/loki/config.yml
+Volume=%S/%P:/var/lib/loki:rw
+Volume=/etc/loki:/etc/loki:ro
+Network=host
+
+[Install]
+WantedBy=multi-user.target

roles/loki/meta/main.yml

@@ -0,0 +1,2 @@
+dependencies:
+- systemd-base

roles/loki/tasks/main.yml

@@ -0,0 +1,67 @@
+- name: ensure required packages are installed
+  package:
+    name:
+    - podman
+    state: present
+  tags:
+  - install
+
+- name: ensure loki container unit is configured
+  copy:
+    src: loki.container
+    dest: /etc/containers/systemd/loki.container
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - reload systemd
+  tags:
+  - container
+
+- name: ensure loki configuration directory exists
+  file:
+    path: /etc/loki
+    owner: root
+    group: root
+    state: directory
+  tags:
+  - config
+- name: ensure loki is configured
+  copy:
+    dest: /etc/loki/config.yml
+    content: >-
+      {{ loki_config|to_nice_yaml(indent=2) }}
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  tags:
+  - config
+
+- name: flush handlers
+  meta: flush_handlers
+
+- name: ensure loki starts at boot
+  service:
+    name: loki
+    enabled: true
+  tags:
+  - service
+- name: ensure loki is running
+  service:
+    name: loki
+    state: started
+  tags:
+  - service
+
+- name: ensure firewall is configured for loki
+  firewalld:
+    port: '{{ item }}'
+    state: enabled
+    immediate: true
+    permanent: true
+  loop:
+  - 3100/tcp
+  - 9096/tcp
+  when: host_uses_firewalld|d(true)|bool
+  tags:
+  - firewalld