diff --git a/deploy/loki1.sh b/deploy/loki1.sh new file mode 100644 index 0000000..1fffbfd --- /dev/null +++ b/deploy/loki1.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# vim: set ts=4 : + +if ! virsh list --all --name | grep -qF loki1; then + ./newvm.sh loki1 \ + --fedora 40 \ + --memory 4096,currentMemory=2048 \ + --vcpus 2 \ + --network network=prod,mac=52:54:00:51:3c:e9 \ + --no-console \ + -- \ + --disk pool=default,size=128,cache=none \ + || exit + sleep 15 +fi + +ANSIBLE_HOST_KEY_CHECKING=false \ +ansible-playbook \ + -l loki1.pyrocufflink.blue \ + wait-for-host.yml \ + bootstrap.yml \ + datavol.yml \ + pyrocufflink.yml \ + loki.yml \ + collectd.yml \ + promtail.yml \ + -u root \ + -e ansible_host=loki1.local \ + -e @join.creds \ + || exit diff --git a/group_vars/loki.yml b/group_vars/loki.yml new file mode 100644 index 0000000..c4ef8b3 --- /dev/null +++ b/group_vars/loki.yml @@ -0,0 +1,21 @@ +data_volumes: +- dev: /dev/vdb + fstype: btrfs + mountpoint: /var/lib/loki + +loki_caddy_acme: + email: loki@pyrocufflink.blue + url: https://ca.pyrocufflink.blue/acme/acme/directory + +loki_caddy_client_ca: |+ + -----BEGIN CERTIFICATE----- + MIIBlDCCAUagAwIBAgIUGNZ/ASP8F2ytev3YplTk4jA5a2EwBQYDK2VwMEgxCzAJ + BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxDTALBgNVBAsMBExv + a2kxEDAOBgNVBAMMB0xva2kgQ0EwHhcNMjQwMjIwMTUwMTQxWhcNMzQwMjIwMTUw + MTQxWjBIMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPRHVzdGluIEMuIEhhdGNoMQ0w + CwYDVQQLDARMb2tpMRAwDgYDVQQDDAdMb2tpIENBMCowBQYDK2VwAyEAnmMawEIo + WfzFaLgpSiaPD+DHg28NHknMFcs7XpyTM9CjQjBAMB0GA1UdDgQWBBTFth3c4S/f + y0BphQy9SucnKN2pLzASBgNVHRMBAf8ECDAGAQH/AgEAMAsGA1UdDwQEAwIBBjAF + BgMrZXADQQCn0JWERsXdJA4kMM45ZXhVgAciwLNQ8ikoucsJcbWBp7bSMjcMVi51 + I+slotQvQES/vfqp/zZFNl7KKyeeQ0sD + -----END CERTIFICATE----- diff --git a/group_vars/vm-hosts.yml b/group_vars/vm-hosts.yml index d33f3d5..f36d9d9 100644 --- a/group_vars/vm-hosts.yml +++ b/group_vars/vm-hosts.yml @@ -245,7 +245,7 @@ vm_autostart: - dc-grumbly - dc-headphone - delay 30s -- loki0 +- loki1 - delay 10s - db0 - k8s-ctrl0 diff --git a/hosts b/hosts index fdcabe8..9c792ab 100644 --- a/hosts +++ b/hosts @@ -89,6 +89,9 @@ k8s-ctrl0.pyrocufflink.blue k8s-controller k8s-node +[loki] +loki1.pyrocufflink.blue + [minio-backups] chromie.pyrocufflink.blue @@ -143,6 +146,7 @@ file0.pyrocufflink.blue git0.pyrocufflink.blue haproxy0.pyrocufflink.blue k8s-ctrl0.pyrocufflink.blue +loki1.pyrocufflink.blue nvr2.pyrocufflink.blue pxe0.pyrocufflink.blue smtp1.pyrocufflink.blue diff --git a/loki.yml b/loki.yml new file mode 100644 index 0000000..3d1e1ba --- /dev/null +++ b/loki.yml @@ -0,0 +1,8 @@ +- hosts: loki + roles: + - role: loki + tags: + - loki + - role: loki-caddy + tags: + - loki-caddy diff --git a/migration/loki0-to-loki1.sh b/migration/loki0-to-loki1.sh new file mode 100644 index 0000000..110f7b4 --- /dev/null +++ b/migration/loki0-to-loki1.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# vim: set sw=4 ts=4 sts=4 et : + +set -e + +cleanup() { + eval $(ssh-agent -k) +} + +trap cleanup INT TERM QUIT EXIT + +eval $(ssh-agent) +sshca-cli user login +ssh-add -K + +: || { +ssh root@loki0.pyrocufflink.blue cat '>' .ssh/known_hosts <- + {{ loki_caddy_client_ca|d('') }} + owner: root + group: root + mode: u=rw,go=r + notify: + - reload caddy + tags: + - cert diff --git a/roles/loki-caddy/templates/Caddyfile.j2 b/roles/loki-caddy/templates/Caddyfile.j2 new file mode 100644 index 0000000..8c675b9 --- /dev/null +++ b/roles/loki-caddy/templates/Caddyfile.j2 @@ -0,0 +1,33 @@ +{# vim: set sw=4 ts=4 sts=4 et : #} +{{ loki_caddy_server_name }} { + tls { + client_auth { + mode verify_if_given + trusted_ca_cert_file /etc/caddy/loki-client-ca.crt + } + } + @anonymous { + expression {tls_client_subject} == null + } + @grafana { + header X-Grafana-User * + } + handle @anonymous { + route /loki/api/v1/push { + reverse_proxy 127.0.0.1:3100 + } + route /metrics { + reverse_proxy 127.0.0.1:3100 + } + route /ready { + reverse_proxy 127.0.0.1:3100 + } + respond 403 + } + handle @grafana { + reverse_proxy 127.0.0.1:3100 + } + tls {{ loki_caddy_acme.email }} { + ca {{ loki_caddy_acme.url }} + } +} diff --git a/roles/loki/defaults/main.yml b/roles/loki/defaults/main.yml new file mode 100644 index 0000000..91ae040 --- /dev/null +++ b/roles/loki/defaults/main.yml @@ -0,0 +1,39 @@ +loki_config: + auth_enabled: false + + server: + http_listen_port: 3100 + http_listen_address: 127.0.0.1 + grpc_listen_port: 9096 + + common: + instance_addr: 127.0.0.1 + path_prefix: /var/lib/loki + storage: + filesystem: + chunks_directory: /var/lib/loki/chunks + rules_directory: /var/lib/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + + query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + + schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v12 + index: + prefix: index_ + period: 24h + + query_scheduler: + max_outstanding_requests_per_tenant: 1024 diff --git a/roles/loki/files/loki.container b/roles/loki/files/loki.container new file mode 100644 index 0000000..20e3a70 --- /dev/null +++ b/roles/loki/files/loki.container @@ -0,0 +1,25 @@ +# vim: set ft=systemd : +[Unit] +Description=Grafana Loki +After=network-online.target +Wants=network-online.target +StartLimitIntervalSec=1m +StartLimitBurst=60 + +[Service] +ExecStartPre=/bin/install -o 10001 -g 10001 -d %S/%P +ExecStartPre=/bin/chcon -t container_file_t %S/%P +ExecReload=/usr/bin/podman kill --cidfile=%t/%N.cid --signal HUP +TimeoutStartSec=5m +Restart=always +RstartSec=1s + +[Container] +Image=docker.io/grafana/loki:2.9.4 +Exec=-config.file=/etc/loki/config.yml +Volume=%S/%P:/var/lib/loki:rw +Volume=/etc/loki:/etc/loki:ro +Network=host + +[Install] +WantedBy=multi-user.target diff --git a/roles/loki/meta/main.yml b/roles/loki/meta/main.yml new file mode 100644 index 0000000..e2827fe --- /dev/null +++ b/roles/loki/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: +- systemd-base diff --git a/roles/loki/tasks/main.yml b/roles/loki/tasks/main.yml new file mode 100644 index 0000000..551e5ca --- /dev/null +++ b/roles/loki/tasks/main.yml @@ -0,0 +1,67 @@ +- name: ensure required packages are installed + package: + name: + - podman + state: present + tags: + - install + +- name: ensure loki container unit is configured + copy: + src: loki.container + dest: /etc/containers/systemd/loki.container + owner: root + group: root + mode: u=rw,go=r + notify: + - reload systemd + tags: + - container + +- name: ensure loki configuration directory exists + file: + path: /etc/loki + owner: root + group: root + state: directory + tags: + - config +- name: ensure loki is configured + copy: + dest: /etc/loki/config.yml + content: >- + {{ loki_config|to_nice_yaml(indent=2) }} + owner: root + group: root + mode: u=rw,go=r + tags: + - config + +- name: flush handlers + meta: flush_handlers + +- name: ensure loki starts at boot + service: + name: loki + enabled: true + tags: + - service +- name: ensure loki is running + service: + name: loki + state: started + tags: + - service + +- name: ensure firewall is configured for loki + firewalld: + port: '{{ item }}' + state: enabled + immediate: true + permanent: true + loop: + - 3100/tcp + - 9096/tcp + when: host_uses_firewalld|d(true)|bool + tags: + - firewalld