From 5540cd85cc4ef8376f233ac7ea9a57e3bf304c81 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Thu, 1 Oct 2026 21:56:59 -0500 Subject: [PATCH] deploy: route stream alerts to Discord through Alertmanager The alert rules had no notification route, so the 2026-10-01 trickle was only visible to someone looking at a dashboard. The box now runs an alertmanager service that posts critical and warning alerts to the Discord webhook zlay already pages into, and drops info alerts. Adds the in-repo sources for the box's alertmanager.yml and prometheus.yml, and documents the webhook file, the rule-reload procedure (restart, not up -d), and a one-command end-to-end test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TTKhZqkDD8TLAvjhzi9HLf --- deploy/README.md | 33 +++++++++++++++++++++++++----- deploy/prometheus/alertmanager.yml | 20 ++++++++++++++++++ deploy/prometheus/prometheus.yml | 12 +++++++++++ deploy/prometheus/rules.yml | 4 ++-- 4 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 deploy/prometheus/alertmanager.yml create mode 100644 deploy/prometheus/prometheus.yml diff --git a/deploy/README.md b/deploy/README.md index 8056b6f..0190069 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -118,12 +118,35 @@ semantic row. the box's Prometheus reads its own copy at `/opt/stream-experiment/rules.yml` (mounted to `/etc/prometheus/rules.yml`, listed under `rule_files` in `prometheus.yml`), so a rule change here is not live until that copy is -replaced and Prometheus restarted (`docker compose up -d prometheus`; the -container has no `--web.enable-lifecycle`, so there is no reload endpoint). -Wired up 2026-08-29; before that the box had no rules loaded at all. Validate with `promtool check rules deploy/prometheus/rules.yml` +replaced and Prometheus restarted (`docker compose restart prometheus`; the +container has no `--web.enable-lifecycle`, so there is no reload endpoint, and +`up -d` does not restart a container whose compose definition is unchanged). +Write the box copy in place (`cat new > rules.yml`): the file is a single-file +bind mount. Wired up 2026-08-29; before that the box had no rules loaded at +all. Validate with `promtool check rules deploy/prometheus/rules.yml` (e.g. via `docker run --rm -v "$PWD/deploy/prometheus:/rules:ro" --entrypoint -promtool prom/prometheus check rules /rules/rules.yml`). The rules have no -notification route; they surface only in Prometheus/Grafana alert state. +promtool prom/prometheus check rules /rules/rules.yml`). + +### paging + +Since 2026-10-02 an `alertmanager` service in the box's compose stack +(`prom/alertmanager:v0.33.0`, pinned by digest) posts `critical` and `warning` +alerts to Discord; `info` alerts are dropped. `prometheus/alertmanager.yml` +and `prometheus/prometheus.yml` are the in-repo sources for the box copies at +`/opt/stream-experiment/`. The compose file itself lives only on the box. + +The webhook URL is the one zlay's Alertmanager uses (the +`zlay-discord-webhook` secret in the zlay cluster's `monitoring` namespace). +It is on the box as `/opt/stream-experiment/discord-webhook`, owner 65534, +mode 400, and is read through `webhook_url_file`. Rotating the webhook means +replacing both copies. To prove the route end to end: + +```sh +docker compose exec -T alertmanager amtool --alertmanager.url=http://127.0.0.1:9093 \ + alert add StreamPagingTest severity=warning job=stream +docker compose exec -T alertmanager wget -qO- http://127.0.0.1:9093/metrics \ + | grep 'alertmanager_notifications.*discord' +``` ## dashboard gate diff --git a/deploy/prometheus/alertmanager.yml b/deploy/prometheus/alertmanager.yml new file mode 100644 index 0000000..35f8704 --- /dev/null +++ b/deploy/prometheus/alertmanager.yml @@ -0,0 +1,20 @@ +# Alertmanager for the stream box: critical and warning alerts go to Discord, +# the same channel zlay pages into. The webhook URL is a secret and lives only +# on the box at /opt/stream-experiment/discord-webhook (owner 65534, mode 400). +route: + receiver: discord + group_by: ["alertname"] + group_wait: 1m + group_interval: 5m + repeat_interval: 4h + routes: + # StreamProcessRestarted fires once per restart by design. + - receiver: "null" + matchers: + - severity = "info" +receivers: + - name: "null" + - name: discord + discord_configs: + - webhook_url_file: /etc/alertmanager/discord-webhook + send_resolved: true diff --git a/deploy/prometheus/prometheus.yml b/deploy/prometheus/prometheus.yml new file mode 100644 index 0000000..9c44d5f --- /dev/null +++ b/deploy/prometheus/prometheus.yml @@ -0,0 +1,12 @@ +global: + scrape_interval: 15s +scrape_configs: + - job_name: stream + static_configs: + - targets: ["stream:6060"] +rule_files: + - /etc/prometheus/rules.yml +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] diff --git a/deploy/prometheus/rules.yml b/deploy/prometheus/rules.yml index d4d1946..00205fd 100644 --- a/deploy/prometheus/rules.yml +++ b/deploy/prometheus/rules.yml @@ -2,8 +2,8 @@ # docker compose stack at /opt/stream-experiment/ on the box; copy this file # into its rule_files path. This is the in-repo source of truth for the rules. # -# There is no notification route (no alertmanager / ntfy). These rules only -# show up in Prometheus' alert state and Grafana until the operator wires one. +# Alertmanager on the box routes severity critical and warning to Discord and +# drops info (alertmanager.yml, deploy/README.md "paging"). groups: - name: stream rules: -- 2.51.2