r/victoria-logs: Deploy VictoriaLogs

I've become rather frusted witih Grafana Loki lately.  It has several
bugs that affect my usage, including issues with counting and
aggregation, completely broken retention and cleanup, spamming itself
with bogus error log messages, and more.  Now that VitoriaLogs has
first-class support in Grafana and support for alerts, it seems like a
good time to try it out.  It's under very active development, with bugs
getting fixed extremely quickly, and new features added constantly.
Indeed, as I was experimenting with it, I thought, "it would be nice if
the web UI could decode ANSI escapes for terminal colors," and just a
few days later, that feature was added!  Native support for syslog is
also a huge benefit, as it will allow me to collect logs directly from
network devices, without first collecting them into a file on the Unifi
controller.

This new role deploys VictoriaLogs in a manner very similar to how I
have Loki set up, as a systemd-managed Podman container.   As it has no
built-in authentication or authorization, we rely on Caddy to handle
that.  As with Loki, mTLS is used to prevent anonymous access to
querying the logs, however, authentication via Authelia is also an
option for human+browser usage.  I'm re-using the same certificate
authority as with Loki to simplify Grafana configuration.  Eventually, I
would like to have a more robust PKI, probably using OpenBao, at which
point I will (hopefully) have decided which log database I will be
using, and can use a proper CA for it.

group_vars/victoria-logs-test.yml

@@ -0,0 +1 @@
+victoria_logs_caddy_server_name: logs-test.pyrocufflink.blue

group_vars/victoria-logs.yml

@@ -0,0 +1,35 @@
+victoria_logs_extra_args:
+- '-syslog.listenAddr.tcp :601'
+- '-syslog.listenAddr.udp :514'
+
+victoria_logs_publish_ports:
+- '514:514/udp'
+- '601:601'
+#- '6514:6514'
+
+victoria_logs_firewall_ports:
+- 514/udp
+- 601/tcp
+#- 6514/tcp
+
+victoria_logs_caddy_server_name: logs.pyrocufflink.blue
+victoria_logs_tls_client_auth: true
+victoria_logs_tls_ca_cert: |+
+  -----BEGIN CERTIFICATE-----
+  MIIBlDCCAUagAwIBAgIUGNZ/ASP8F2ytev3YplTk4jA5a2EwBQYDK2VwMEgxCzAJ
+  BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxDTALBgNVBAsMBExv
+  a2kxEDAOBgNVBAMMB0xva2kgQ0EwHhcNMjQwMjIwMTUwMTQxWhcNMzQwMjIwMTUw
+  MTQxWjBIMQswCQYDVQQGEwJVUzEYMBYGA1UECgwPRHVzdGluIEMuIEhhdGNoMQ0w
+  CwYDVQQLDARMb2tpMRAwDgYDVQQDDAdMb2tpIENBMCowBQYDK2VwAyEAnmMawEIo
+  WfzFaLgpSiaPD+DHg28NHknMFcs7XpyTM9CjQjBAMB0GA1UdDgQWBBTFth3c4S/f
+  y0BphQy9SucnKN2pLzASBgNVHRMBAf8ECDAGAQH/AgEAMAsGA1UdDwQEAwIBBjAF
+  BgMrZXADQQCn0JWERsXdJA4kMM45ZXhVgAciwLNQ8ikoucsJcbWBp7bSMjcMVi51
+  I+slotQvQES/vfqp/zZFNl7KKyeeQ0sD
+  -----END CERTIFICATE-----
+victoria_logs_caddy_forward_auth:
+  url: https://auth.pyrocufflink.blue
+  path: /api/verify
+  location: '?rd={scheme}://{host}{uri}'
+victoria_logs_caddy_acme:
+  email: victoria_logs@pyrocufflink.blue
+  url: https://ca.pyrocufflink.blue/acme/acme/directory

hosts

@@ -238,6 +238,13 @@ vps
 [unifi:children]
 unifi-test
 
+[victoria-logs]
+
+[victoria-logs:children]
+victoria-logs-test
+
+[victoria-logs-test]
+
 [vm-hosts]
 vmhost0.pyrocufflink.blue
 vmhost1.pyrocufflink.blue

roles/victoria-logs/defaults/main.yml

@@ -0,0 +1,14 @@
+# vim: set ft=yaml.jinja :
+victoria_logs_version: 1.19.0
+victoria_logs_container_image_tag: v{{ victoria_logs_version }}-victorialogs
+victoria_logs_container_image_repo: docker.io/victoriametrics/victoria-logs
+victoria_logs_container_image: >-
+  {{ victoria_logs_container_image_repo }}:{{ victoria_logs_container_image_tag }}
+
+victoria_logs_publish_ports: []
+
+victoria_logs_extra_args: []
+victoria_logs_args: >-
+  {{ victoria_logs_default_args + victoria_logs_extra_args }}
+
+victoria_logs_tls_client_auth: false

roles/victoria-logs/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: restart victoria-logs
+  service:
+    name: victoria-logs
+    state: restarted

roles/victoria-logs/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+- systemd-base
+- caddy

roles/victoria-logs/tasks/main.yml

@@ -0,0 +1,106 @@
+- name: ensure victoria logs group exists
+  group:
+    name: victoria-logs
+    system: true
+  tags:
+  - user
+  - group
+- name: ensure victoria logs user exists
+  user:
+    name: victoria-logs
+    system: true
+    home: /var/lib/victoria-logs
+    group: victoria-logs
+    createhome: false
+  register: victoria_logs_user
+  tags:
+  - user
+
+- name: ensure podman is installed
+  package:
+    name: podman
+    state: present
+  tags:
+  - install
+
+- name: ensure victoria logs container image is present
+  podman_image:
+    name: '{{ victoria_logs_container_image_repo }}'
+    tag: '{{ victoria_logs_container_image_tag }}'
+    state: present
+  tags:
+  - container
+  - container-image
+
+- name: ensure victoria logs systemd container unit is configured
+  template:
+    src: victoria-logs.container.j2
+    dest: /etc/containers/systemd/victoria-logs.container
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - reload systemd
+  - restart victoria-logs
+  tags:
+  - systemd
+  - container
+
+- name: flush handlers
+  meta: flush_handlers
+
+- name: ensure victoria logs is running
+  service:
+    name: victoria-logs
+    state: started
+  tags:
+  - service
+
+- name: ensure firewall is configured for victoria logs
+  firewalld:
+    port: '{{ item }}'
+    immediate: true
+    permanent: true
+    state: enabled
+  loop: '{{ victoria_logs_firewall_ports }}'
+  when: host_uses_firewalld|d(true)
+  tags:
+  - firewalld
+
+- name: ensure caddy tls client auth ca certificate is installed
+  copy:
+    dest: /etc/caddy/victoria-logs-ca.crt
+    content: |+
+      {{ victoria_logs_tls_ca_cert }}
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  when: victoria_logs_tls_ca_cert|d(none)
+  notify:
+  - reload caddy
+  tags:
+  - caddy
+  - cert
+- name: ensure caddy is configured to proxy for victoria logs
+  template:
+    src: Caddyfile.j2
+    dest: /etc/caddy/Caddyfile.d/victoria-logs.caddyfile
+    owner: root
+    group: root
+    mode: u=rw,go=r
+  notify:
+  - reload caddy
+  tags:
+  - caddy
+  - config
+
+- name: flush_handlers
+  meta: flush_handlers
+
+- name: ensure caddy is running
+  service:
+    name: caddy
+    state: started
+  tags:
+  - caddy
+  - service

roles/victoria-logs/templates/Caddyfile.j2

@@ -0,0 +1,47 @@
+{# vim: set sw=4 ts=4 sts=4 et : #}
+{{ victoria_logs_caddy_server_name }} {
+{% if victoria_logs_tls_client_auth %}
+    tls {
+        client_auth {
+            mode verify_if_given
+            trust_pool file /etc/caddy/victoria-logs-ca.crt
+        }
+    }
+    @nocert {
+        expression {tls_client_subject} == null
+    }
+{% if victoria_logs_caddy_forward_auth|d %}
+    handle @nocert {
+        forward_auth /select/* {{ victoria_logs_caddy_forward_auth.url }} {
+            uri {{ victoria_logs_caddy_forward_auth.path }}
+            header_up Host {upstream_hostport}
+
+            @unauthorized status 401
+            handle_response @unauthorized {
+                respond "" 303
+                header Location {{ victoria_logs_caddy_forward_auth.url }}{{ victoria_logs_caddy_forward_auth.location }}
+             }
+        }
+    }
+{% endif %}
+{% elif victoria_logs_caddy_forward_auth|d %}
+    forward_auth /select/* {{ victoria_logs_caddy_forward_auth.url }} {
+        uri {{ victoria_logs_caddy_forward_auth.path }}
+        header_up Host {upstream_hostport}
+
+        @unauthorized status 401
+        handle_response @unauthorized {
+            respond "" 303
+            header Location {{ victoria_logs_caddy_forward_auth.url }}{{ victoria_logs_caddy_forward_auth.location }}
+         }
+    }
+
+{% endif %}
+    reverse_proxy 127.0.0.1:9428
+{% if victoria_logs_caddy_acme|d %}
+
+    tls {{ victoria_logs_caddy_acme.email }} {
+        ca {{ victoria_logs_caddy_acme.url }}
+    }
+{% endif %}
+}

roles/victoria-logs/templates/victoria-logs.container.j2

@@ -0,0 +1,47 @@
+[Unit]
+Description=Victoria Logs
+Wants=network.target
+After=network.target
+StartLimitIntervalSec=1s
+StartLimitBurst=1
+
+[Container]
+Image={{ victoria_logs_container_image }}
+Pull=never
+Exec={{ victoria_logs_args | join(' ') }}
+User={{ victoria_logs_user.uid }}
+Group={{ victoria_logs_user.group }}
+Volume=%S/%p:/data:rw,z,U
+NoNewPrivileges=yes
+ReadOnly=yes
+ReadOnlyTmpfs=yes
+AddCapability=CAP_NET_BIND_SERVICE
+PublishPort=9428:9428
+{% for portspec in victoria_logs_publish_ports %}
+PublishPort={{ portspec }}
+{% endfor %}
+HealthCmd=/usr/bin/wget -q -O /dev/null 127.0.0.1:9428/health
+HealthInterval=1m
+HealthOnFailure=stop
+
+[Service]
+StateDirectory=%p
+Restart=always
+RestartSec=1
+MemoryDenyWriteExecute=yes
+PrivateTmp=yes
+ProtectClock=yes
+ProtectHome=yes
+ProtectKernelLogs=yes
+ProtectKernelModules=yes
+ProtectProc=invisible
+ProtectSystem=strict
+ReadWritePaths=%S/%p
+ReadWritePaths=%S/containers/storage
+ReadWritePaths=%t
+ReadWritePaths=/etc/containers/networks
+RestrictRealtime=yes
+UMask=0077
+
+[Install]
+WantedBy=multi-user.target

roles/victoria-logs/vars/main.yml

@@ -0,0 +1,3 @@
+victoria_logs_default_args:
+- '-httpListenAddr 0.0.0.0:9428'
+- -storageDataPath /data

victoria-logs.yml

@@ -0,0 +1,5 @@
+- hosts: victoria-logs
+  roles:
+  - role: victoria-logs
+    tags:
+    - victoria-logs