From 40da9db0e95c3f19c6dbbb9eb770e716bc3449e6 Mon Sep 17 00:00:00 2001 From: Seth Etter Date: Sun, 30 Aug 2026 21:40:55 -0500 Subject: [PATCH] observability: provision host + logs dashboards as code Panel queries validated against live Mimir/Loki; dashboard JSON validated in a throwaway local grafana:12.4.9 container. --- observability/README.md | 7 +- .../dashboards/boards/host-overview.json | 254 ++++++++++++++++++ .../files/dashboards/boards/logs.json | 108 ++++++++ .../files/dashboards/dashboards.yaml | 12 + .../ansible/roles/svc-grafana/tasks/main.yaml | 12 + 5 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 observability/ansible/roles/svc-grafana/files/dashboards/boards/host-overview.json create mode 100644 observability/ansible/roles/svc-grafana/files/dashboards/boards/logs.json create mode 100644 observability/ansible/roles/svc-grafana/files/dashboards/dashboards.yaml diff --git a/observability/README.md b/observability/README.md index 2d75ff6..3024139 100644 --- a/observability/README.md +++ b/observability/README.md @@ -30,6 +30,9 @@ on every monitored host. Alertmanager. If critical alerting ever needs to survive Grafana being down, the upgrade path is Prometheus-style rule files evaluated by Mimir's/Loki's ruler + a standalone Alertmanager. +- **Dashboards are provisioned as code** — JSON committed under + `roles/svc-grafana/files/dashboards/boards/`, loaded read-only from the + `Infrastructure` folder. - **Why WireGuard over VPC + firewall**: VPC rules are IP-based, unencrypted, region-locked, and DO-only. The mesh adds encryption in transit and per-node key identity, and works for any future host @@ -96,6 +99,8 @@ alloy_mimir_url: "http://10.77.0.1:9009/api/v1/push" ## Open questions / TODO - Alert destination is email for now; consider Slack/Discord webhook. -- No dashboard provisioning yet — build in UI, export, commit. +- Dashboards are provisioned as code (see below); new ones: add JSON under + `roles/svc-grafana/files/dashboards/boards/`, validate queries against + Mimir/Loki, then `just deploy`. - Consider pinning image tags to exact versions once validated. - Sizing: `s-1vcpu-2gb` to start; bump `size` if Mimir gets OOM-happy. diff --git a/observability/ansible/roles/svc-grafana/files/dashboards/boards/host-overview.json b/observability/ansible/roles/svc-grafana/files/dashboards/boards/host-overview.json new file mode 100644 index 0000000..903d65a --- /dev/null +++ b/observability/ansible/roles/svc-grafana/files/dashboards/boards/host-overview.json @@ -0,0 +1,254 @@ +{ + "uid": "host-overview", + "title": "Host Overview", + "description": "Host metrics from Alloy's embedded node_exporter, one series per host.", + "schemaVersion": 41, + "version": 1, + "editable": true, + "refresh": "1m", + "time": { "from": "now-6h", "to": "now" }, + "templating": { + "list": [ + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { "type": "prometheus", "uid": "mimir" }, + "query": { "refId": "var", "query": "label_values(up, host)" }, + "includeAll": true, + "allValue": ".*", + "multi": true, + "sort": 1, + "current": { "selected": true, "text": ["All"], "value": ["$__all"] } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Uptime", + "datasource": { "type": "prometheus", "uid": "mimir" }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 0 }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 86400 }, + { "color": "green", "value": 604800 } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "value", + "graphMode": "none" + }, + "targets": [ + { + "refId": "A", + "expr": "time() - node_boot_time_seconds{host=~\"$host\"}", + "legendFormat": "{{host}}" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "CPU usage", + "datasource": { "type": "prometheus", "uid": "mimir" }, + "gridPos": { "h": 8, "w": 9, "x": 6, "y": 0 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { + "refId": "A", + "expr": "100 - avg by (host) (rate(node_cpu_seconds_total{mode=\"idle\", host=~\"$host\"}[$__rate_interval])) * 100", + "legendFormat": "{{host}}" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Load (1m)", + "datasource": { "type": "prometheus", "uid": "mimir" }, + "gridPos": { "h": 8, "w": 9, "x": 15, "y": 0 }, + "fieldConfig": { + "defaults": { + "min": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { + "refId": "A", + "expr": "node_load1{host=~\"$host\"}", + "legendFormat": "{{host}}" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Memory used", + "description": "Total minus available.", + "datasource": { "type": "prometheus", "uid": "mimir" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 20, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { + "refId": "A", + "expr": "node_memory_MemTotal_bytes{host=~\"$host\"} - node_memory_MemAvailable_bytes{host=~\"$host\"}", + "legendFormat": "{{host}}" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Root filesystem usage", + "datasource": { "type": "prometheus", "uid": "mimir" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "thresholdsStyle": { "mode": "line" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "transparent", "value": null }, + { "color": "red", "value": 85 } + ] + } + }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { + "refId": "A", + "expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\", host=~\"$host\"} / node_filesystem_size_bytes{mountpoint=\"/\", host=~\"$host\"})", + "legendFormat": "{{host}}" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Disk I/O", + "datasource": { "type": "prometheus", "uid": "mimir" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "min": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "read" }, + "properties": [] + } + ] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { + "refId": "A", + "expr": "rate(node_disk_read_bytes_total{host=~\"$host\"}[$__rate_interval])", + "legendFormat": "{{host}} {{device}} read" + }, + { + "refId": "B", + "expr": "rate(node_disk_written_bytes_total{host=~\"$host\"}[$__rate_interval])", + "legendFormat": "{{host}} {{device}} write" + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Network traffic", + "datasource": { "type": "prometheus", "uid": "mimir" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "min": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { + "refId": "A", + "expr": "rate(node_network_receive_bytes_total{device!~\"lo\", host=~\"$host\"}[$__rate_interval])", + "legendFormat": "{{host}} rx" + }, + { + "refId": "B", + "expr": "rate(node_network_transmit_bytes_total{device!~\"lo\", host=~\"$host\"}[$__rate_interval])", + "legendFormat": "{{host}} tx" + } + ] + } + ] +} diff --git a/observability/ansible/roles/svc-grafana/files/dashboards/boards/logs.json b/observability/ansible/roles/svc-grafana/files/dashboards/boards/logs.json new file mode 100644 index 0000000..be7fff2 --- /dev/null +++ b/observability/ansible/roles/svc-grafana/files/dashboards/boards/logs.json @@ -0,0 +1,108 @@ +{ + "uid": "logs", + "title": "Logs", + "description": "Container (docker) and systemd journal logs from Loki.", + "schemaVersion": 41, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { "from": "now-1h", "to": "now" }, + "templating": { + "list": [ + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { "type": "loki", "uid": "loki" }, + "query": { "refId": "var", "query": "label_values(host)" }, + "includeAll": true, + "allValue": ".+", + "multi": true, + "sort": 1, + "current": { "selected": true, "text": ["All"], "value": ["$__all"] } + }, + { + "name": "container", + "label": "Container", + "type": "query", + "datasource": { "type": "loki", "uid": "loki" }, + "query": { "refId": "var", "query": "label_values(container)" }, + "includeAll": true, + "allValue": ".+", + "multi": true, + "sort": 1, + "current": { "selected": true, "text": ["All"], "value": ["$__all"] } + }, + { + "name": "unit", + "label": "Journal unit", + "type": "query", + "datasource": { "type": "loki", "uid": "loki" }, + "query": { "refId": "var", "query": "label_values(unit)" }, + "includeAll": true, + "allValue": ".+", + "multi": true, + "sort": 1, + "current": { "selected": true, "text": ["All"], "value": ["$__all"] } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Log volume by host", + "datasource": { "type": "loki", "uid": "loki" }, + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "custom": { + "drawStyle": "bars", + "lineWidth": 0, + "fillOpacity": 70, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "targets": [ + { + "refId": "A", + "expr": "sum by (host) (count_over_time({host=~\"$host\"}[1m]))", + "legendFormat": "{{host}}" + } + ] + }, + { + "id": 2, + "type": "logs", + "title": "Container logs", + "datasource": { "type": "loki", "uid": "loki" }, + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 7 }, + "options": { "sortOrder": "Descending", "showTime": true, "wrapLogMessage": true }, + "targets": [ + { + "refId": "A", + "expr": "{host=~\"$host\", container=~\"$container\"}" + } + ] + }, + { + "id": 3, + "type": "logs", + "title": "Journal (systemd units)", + "datasource": { "type": "loki", "uid": "loki" }, + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 19 }, + "options": { "sortOrder": "Descending", "showTime": true, "wrapLogMessage": true }, + "targets": [ + { + "refId": "A", + "expr": "{host=~\"$host\", unit=~\"$unit\"}" + } + ] + } + ] +} diff --git a/observability/ansible/roles/svc-grafana/files/dashboards/dashboards.yaml b/observability/ansible/roles/svc-grafana/files/dashboards/dashboards.yaml new file mode 100644 index 0000000..63f1a28 --- /dev/null +++ b/observability/ansible/roles/svc-grafana/files/dashboards/dashboards.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +# Dashboards are committed as JSON under boards/ and copied here by the +# svc-grafana role. The provider scans the boards/ subdir so this config +# file itself is not picked up as a dashboard. + +providers: + - name: infra + folder: Infrastructure + type: file + options: + path: /etc/grafana/provisioning/dashboards/boards diff --git a/observability/ansible/roles/svc-grafana/tasks/main.yaml b/observability/ansible/roles/svc-grafana/tasks/main.yaml index 6f32ec1..86e15c2 100644 --- a/observability/ansible/roles/svc-grafana/tasks/main.yaml +++ b/observability/ansible/roles/svc-grafana/tasks/main.yaml @@ -24,6 +24,7 @@ loop: - "{{ svc_grafana_deploy_dir }}" - "{{ svc_grafana_deploy_dir }}/provisioning/datasources" + - "{{ svc_grafana_deploy_dir }}/provisioning/dashboards" - "{{ svc_grafana_deploy_dir }}/provisioning/alerting" - name: render grafana.env @@ -46,6 +47,17 @@ mode: "0644" notify: restart svc-grafana +- name: copy dashboard provisioning (config + JSON, from role files) + become: true + ansible.builtin.copy: + src: dashboards/ + dest: "{{ svc_grafana_deploy_dir }}/provisioning/dashboards/" + owner: root + group: root + mode: "0644" + directory_mode: "0755" + notify: restart svc-grafana + - name: render alert rule provisioning become: true ansible.builtin.template: -- 2.51.2