diff --git a/app/promtail/schema/metrics.cue b/app/promtail/schema/metrics.cue new file mode 100644 index 0000000..6e3ca84 --- /dev/null +++ b/app/promtail/schema/metrics.cue @@ -0,0 +1,38 @@ +package schema + +#BaseMetric: { + type: string + description?: string + prefix?: string + source?: string + max_idle_duration?: string +} + +#CounterMetric: { + #BaseMetric + type: "Counter" + config: { + match_all?: bool + count_entry_bytes?: bool + value?: string + action: "inc" | "add" + } +} + +#GaugeMetric: { + #BaseMetric + type: "Gauge" + config: { + value?: string + action: "set" | "inc" | "dec" | "add" | "sub" + } +} + +#HistogramMetric: { + #BaseMetric + type: "Histogram" + config: { + value?: string + buckets: [int] + } +} diff --git a/app/promtail/schema/pipeline.cue b/app/promtail/schema/pipeline.cue new file mode 100644 index 0000000..d707907 --- /dev/null +++ b/app/promtail/schema/pipeline.cue @@ -0,0 +1,106 @@ +package schema + +#PipelineStage: { + cri?: { + max_partial_lines?: int + max_partial_line_size?: int + max_partial_line_size_truncate?: bool + } + + decolorize?: null + + drop?: { + source?: [...string] | string + separator?: string + expression?: string + value?: string + older_than?: string + longer_than?: string | int + drop_counter_reason?: string + } + + json?: { + expressions?: [string]: string + source?: string + drop_malformed?: bool + } + + labelallow?: [...string] + + labeldrop?: [...string] + + labels?: [string]: string + + limit?: { + rate?: int + burst?: int + by_label_name?: string + max_distinct_labels?: int + drop?: bool + } + + logfmt?: { + mapping?: [string]: string + source?: string + } + + match?: { + selector: string + pipeline_name?: string + action?: "keep" | "drop" + drop_counter_reason?: string + stages?: [#PipelineStage] + } + + metrics?: [string]: #CounterMetric | #GaugeMetric | #HistogramMetric + + multiline?: { + firstline: string + max_wait_time?: string + max_lines?: int + } + + output?: { + source: string + } + + pack?: { + labels: [string] + ingest_timestamp?: bool + } + + regex?: { + expression: string + source?: string + } + + replace?: { + expression: string + source?: string + replace?: string + } + + sampling?: { + rate?: float + } + + static_labels?: [string]: string + + template?: { + source: string + template: string + } + + tenant?: { + label?: string + source?: string + value?: string + } + + timestamp?: { + source: string + format: string + fallback_formats?: [...string] + location?: string + } +} diff --git a/app/promtail/schema/schema.cue b/app/promtail/schema/schema.cue new file mode 100644 index 0000000..41f72fa --- /dev/null +++ b/app/promtail/schema/schema.cue @@ -0,0 +1,62 @@ +package schema + +#Client: { + url: string + tls_config?: { + ca_file?: string + cert_file?: string + key_file?: string + server_name?: string + } +} + +#CommonScrapeConfig: { + labels?: [string]: string +} + +#JournalScrapeConfig: { + #CommonScrapeConfig + json: bool | *false +} + +#KubernetesScrapeConfig: { + #CommonScrapeConfig + api_server?: string + role: "node" | "service" | "pod" | "endpoints" | "ingress" +} + +#RelabelConfig: { + source_labels?: [...string] + separator?: string + target_label?: string + regex?: string + modulus?: int + replacement?: string + action?: "replace" | "keep" | "drop" | "hashmod" | "labelmap" | + "labeldrop" | "labelkeep" +} + +#ScrapeConfig: { + job_name: string + journal?: #JournalScrapeConfig + static_configs?: [...#CommonScrapeConfig] + kubernetes_sd_configs?: [...#KubernetesScrapeConfig] + pipeline_stages?: [...#PipelineStage] + relabel_configs?: [...#RelabelConfig] +} + +#PromtailConfig: { + server: { + http_listen_port: int | *9080 + grpc_listen_port: int | *0 + enable_runtime_reload: bool | *true + } + + clients: [...#Client] + + positions: { + filename: string | *"/var/lib/promtail/positions" + } + + scrape_configs: [...#ScrapeConfig] +} diff --git a/app/promtail/templates.cue b/app/promtail/templates.cue new file mode 100644 index 0000000..4930503 --- /dev/null +++ b/app/promtail/templates.cue @@ -0,0 +1,33 @@ +package promtail + +import "du5t1n.me/cfg/base/schema/instructions" + +templates: [...instructions.#RenderInstruction] & [ + { + template: "promtail/ca.crt" + dest: "/etc/promtail/ca.crt" + hooks: { + changed: [{run: "systemctl try-restart promtail"}] + } + }, + { + template: "promtail/config.yml" + dest: "/etc/promtail/config.yml" + hooks: { + changed: [{run: "systemctl try-restart promtail"}] + } + }, + { + template: "promtail/promtail.container" + dest: "/etc/containers/systemd/promtail.container" + hooks: { + changed: [ + { + run: "systemctl daemon-reload" + immediate: true + }, + {run: "systemctl restart promtail"}, + ] + } + }, +] diff --git a/env/prod/promtail.cue b/env/prod/promtail.cue new file mode 100644 index 0000000..13c2f3e --- /dev/null +++ b/env/prod/promtail.cue @@ -0,0 +1,86 @@ +package prod + +import "encoding/yaml" + +import "du5t1n.me/cfg/app/promtail/schema" + +promtail: { + scrape: { + journal: schema.#ScrapeConfig & { + job_name: "journal" + journal: { + labels: { + job: "systemd-journal" + } + } + relabel_configs: [ + { + source_labels: ["__journal__hostname"] + target_label: "hostname" + }, + { + source_labels: ["__journal__systemd_unit"] + target_label: "unit" + }, + { + source_labels: ["__journal_syslog_identifier"] + target_label: "syslog_identifier" + }, + { + source_labels: ["__journal_priority"] + target_label: "priority" + }, + { + source_labels: ["__journal_message_id"] + target_label: "message_id" + }, + { + source_labels: ["__journal__comm"] + target_label: "command" + }, + { + source_labels: ["__journal__transport"] + target_label: "transport" + }, + ] + } + } + + ca: """ + -----BEGIN CERTIFICATE----- + MIIBgTCCATOgAwIBAgIUTf/ZBSJEi8IQb8Ndoxp4/tHB/lcwBQYDK2VwMEAxCzAJ + BgNVBAYTAlVTMRgwFgYDVQQKDA9EdXN0aW4gQy4gSGF0Y2gxFzAVBgNVBAMMDkRD + SCBSb290IENBIFIzMB4XDTI0MDIxNzIwMjkzNloXDTM0MDIxNzIwMjkzNlowQDEL + MAkGA1UEBhMCVVMxGDAWBgNVBAoMD0R1c3RpbiBDLiBIYXRjaDEXMBUGA1UEAwwO + RENIIFJvb3QgQ0EgUjMwKjAFBgMrZXADIQDORylVcWcxwGDJvsJIc2NctfNfDaIU + T6mLebahKdshaKM/MD0wHQYDVR0OBBYEFLZoxAHBvWqbLWMga/DAAlG9ido5MA8G + A1UdEwEB/wQFMAMBAf8wCwYDVR0PBAQDAgEGMAUGAytlcANBANLV79joVd9s9bmL + 0a91HqvOotOnN/416Ek4UTl95jIqy/TvTfRjXX56wSALXqP1iYQM5i3zk3gVEhh4 + DaY+6wQ= + -----END CERTIFICATE----- + """ + + config: schema.#PromtailConfig & { + clients: [ + { + url: "https://loki.pyrocufflink.blue:3100/loki/api/v1/push" + tls_config: { + ca_file: "/etc/promtail/ca.crt" + } + }, + ] + } +} + +let Marshal = yaml.Marshal + +#promtail: { + #scrape: [...schema.#ScrapeConfig] | *[promtail.scrape.journal] + ca: string | *promtail.ca + + config: promtail.config & { + scrape_configs: #scrape + } + + yaml: Marshal(config) +} diff --git a/host/loki0.pyrocufflink.blue.cue b/host/loki0.pyrocufflink.blue.cue index 81409b1..ebc0c6e 100644 --- a/host/loki0.pyrocufflink.blue.cue +++ b/host/loki0.pyrocufflink.blue.cue @@ -5,6 +5,8 @@ import ( ssh: prod.ssh sudo: prod.sudo +promtail: prod.#promtail + fetchcert: prod.fetchcert.loki & { token: """ -----BEGIN AGE ENCRYPTED FILE----- diff --git a/instructions/loki0.pyrocufflink.blue.cue b/instructions/loki0.pyrocufflink.blue.cue index 0c5cbd0..7c7e958 100644 --- a/instructions/loki0.pyrocufflink.blue.cue +++ b/instructions/loki0.pyrocufflink.blue.cue @@ -3,6 +3,7 @@ import ( "du5t1n.me/cfg/app/collectd" "du5t1n.me/cfg/app/fetchcert" + "du5t1n.me/cfg/app/promtail" "du5t1n.me/cfg/app/loki" "du5t1n.me/cfg/env/prod" ) @@ -12,4 +13,5 @@ render: list.Concat([ collectd.templates, fetchcert.templates, loki.templates, + promtail.templates, ]) diff --git a/templates/promtail/ca.crt b/templates/promtail/ca.crt new file mode 100644 index 0000000..33645b2 --- /dev/null +++ b/templates/promtail/ca.crt @@ -0,0 +1 @@ +{{ promtail.ca }} diff --git a/templates/promtail/config.yml b/templates/promtail/config.yml new file mode 100644 index 0000000..3b2e7bb --- /dev/null +++ b/templates/promtail/config.yml @@ -0,0 +1 @@ +{{ promtail.yaml -}} diff --git a/templates/promtail/promtail.container b/templates/promtail/promtail.container new file mode 100644 index 0000000..a80334f --- /dev/null +++ b/templates/promtail/promtail.container @@ -0,0 +1,23 @@ +# vim: set ft=systemd : +[Unit] +Description=Grafana Loki Promtail Agent +After=network-online.target +Wants=network-online.target + +[Service] +StateDirectory=%P + +[Container] +Image=docker.io/grafana/promtail:2.9.4 +Exec=-config.file=/etc/promtail/config.yml +Volume=%S/%P:/var/lib/promtail:rw,Z,U +Volume=/etc/machine-id:/etc/machine-id:ro +Volume=/etc/promtail:/etc/promtail:ro +Volume=/run/log:/run/log:ro +Volume=/var/log:/var/log:ro +PublishPort=9080:9080 +# container_t is not allowed to read var_log_t / syslogd_var_run_t +SecurityLabelDisable=true + +[Install] +WantedBy=multi-user.target