diff --git a/deploy/README.md b/deploy/README.md index 8056b6f..0190069 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -118,12 +118,35 @@ semantic row. the box's Prometheus reads its own copy at `/opt/stream-experiment/rules.yml` (mounted to `/etc/prometheus/rules.yml`, listed under `rule_files` in `prometheus.yml`), so a rule change here is not live until that copy is -replaced and Prometheus restarted (`docker compose up -d prometheus`; the -container has no `--web.enable-lifecycle`, so there is no reload endpoint). -Wired up 2026-08-29; before that the box had no rules loaded at all. Validate with `promtool check rules deploy/prometheus/rules.yml` +replaced and Prometheus restarted (`docker compose restart prometheus`; the +container has no `--web.enable-lifecycle`, so there is no reload endpoint, and +`up -d` does not restart a container whose compose definition is unchanged). +Write the box copy in place (`cat new > rules.yml`): the file is a single-file +bind mount. Wired up 2026-08-29; before that the box had no rules loaded at +all. Validate with `promtool check rules deploy/prometheus/rules.yml` (e.g. via `docker run --rm -v "$PWD/deploy/prometheus:/rules:ro" --entrypoint -promtool prom/prometheus check rules /rules/rules.yml`). The rules have no -notification route; they surface only in Prometheus/Grafana alert state. +promtool prom/prometheus check rules /rules/rules.yml`). + +### paging + +Since 2026-10-02 an `alertmanager` service in the box's compose stack +(`prom/alertmanager:v0.33.0`, pinned by digest) posts `critical` and `warning` +alerts to Discord; `info` alerts are dropped. `prometheus/alertmanager.yml` +and `prometheus/prometheus.yml` are the in-repo sources for the box copies at +`/opt/stream-experiment/`. The compose file itself lives only on the box. + +The webhook URL is the one zlay's Alertmanager uses (the +`zlay-discord-webhook` secret in the zlay cluster's `monitoring` namespace). +It is on the box as `/opt/stream-experiment/discord-webhook`, owner 65534, +mode 400, and is read through `webhook_url_file`. Rotating the webhook means +replacing both copies. To prove the route end to end: + +```sh +docker compose exec -T alertmanager amtool --alertmanager.url=http://127.0.0.1:9093 \ + alert add StreamPagingTest severity=warning job=stream +docker compose exec -T alertmanager wget -qO- http://127.0.0.1:9093/metrics \ + | grep 'alertmanager_notifications.*discord' +``` ## dashboard gate diff --git a/deploy/prometheus/alertmanager.yml b/deploy/prometheus/alertmanager.yml new file mode 100644 index 0000000..35f8704 --- /dev/null +++ b/deploy/prometheus/alertmanager.yml @@ -0,0 +1,20 @@ +# Alertmanager for the stream box: critical and warning alerts go to Discord, +# the same channel zlay pages into. The webhook URL is a secret and lives only +# on the box at /opt/stream-experiment/discord-webhook (owner 65534, mode 400). +route: + receiver: discord + group_by: ["alertname"] + group_wait: 1m + group_interval: 5m + repeat_interval: 4h + routes: + # StreamProcessRestarted fires once per restart by design. + - receiver: "null" + matchers: + - severity = "info" +receivers: + - name: "null" + - name: discord + discord_configs: + - webhook_url_file: /etc/alertmanager/discord-webhook + send_resolved: true diff --git a/deploy/prometheus/prometheus.yml b/deploy/prometheus/prometheus.yml new file mode 100644 index 0000000..9c44d5f --- /dev/null +++ b/deploy/prometheus/prometheus.yml @@ -0,0 +1,12 @@ +global: + scrape_interval: 15s +scrape_configs: + - job_name: stream + static_configs: + - targets: ["stream:6060"] +rule_files: + - /etc/prometheus/rules.yml +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] diff --git a/deploy/prometheus/rules.yml b/deploy/prometheus/rules.yml index d4d1946..00205fd 100644 --- a/deploy/prometheus/rules.yml +++ b/deploy/prometheus/rules.yml @@ -2,8 +2,8 @@ # docker compose stack at /opt/stream-experiment/ on the box; copy this file # into its rule_files path. This is the in-repo source of truth for the rules. # -# There is no notification route (no alertmanager / ntfy). These rules only -# show up in Prometheus' alert state and Grafana until the operator wires one. +# Alertmanager on the box routes severity critical and warning to Discord and +# drops info (alertmanager.yml, deploy/README.md "paging"). groups: - name: stream rules: