From a72b42afe74d3ef1f614e460f201cb26ed0b3d85 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Sun, 19 Jul 2026 14:36:44 -0500 Subject: [PATCH] adapt dashboard to honest process metrics --- README.md | 10 +- deploy/README.md | 11 +- deploy/grafana/stream.json | 7298 +++++++++++++++++++++++++++++ docs/grafana-dashboard.md | 48 +- docs/semantic-parity.md | 2 +- justfile | 28 + src/internal/metrics.zig | 82 +- src/internal/process_metrics.zig | 189 + src/internal/server.zig | 5 +- tests/process_metrics_contract.py | 79 + tests/test_dashboard_contract.py | 24 +- tools/adapt_dashboard.py | 118 + tools/check_dashboard.py | 15 +- 13 files changed, 7878 insertions(+), 31 deletions(-) create mode 100644 deploy/grafana/stream.json create mode 100644 src/internal/process_metrics.zig create mode 100644 tests/process_metrics_contract.py create mode 100644 tools/adapt_dashboard.py diff --git a/README.md b/README.md index 7b45cc5..be87c24 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,9 @@ upstream bootstrap/compaction/subscribe condensations (every deliberate divergence is written down there), lessons carried over from zlay, and durable product/copy context for public surfaces. `docs/semantic-parity.md` is the top-level deployment gate. `docs/grafana-dashboard.md` records the -checksum-pinned upstream dashboard reuse and its fail-closed metric contract. -The dashboard bytes and checker are fully offline: run `just dashboard-test`, -then `just dashboard-contract http://127.0.0.1:6008/metrics` against a local -candidate. +checksum-pinned upstream source, deterministic Stream adaptation, and its +fail-closed metric contract. The dashboard and process receipts are fully +offline: run `just dashboard-test`, `just process-metrics-contract`, then +`just dashboard-contract http://127.0.0.1:6008/metrics` against a local +candidate. The final command intentionally remains red until every upstream +lifecycle family has a real producer. diff --git a/deploy/README.md b/deploy/README.md index 93aa0f6..8e00d7f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -53,7 +53,10 @@ Absolute throughput still needs the dedicated-node soak before public cutover. ## dashboard gate `grafana/jetstream-upstream.json` is the byte-exact dashboard from the pinned -Jetstream V2 reference. Before provisioning it, run -`just dashboard-contract `. The command is -network-independent when given a saved scrape and refuses promotion while any -queried family is absent. +Jetstream V2 reference. `grafana/stream.json` is the deployable artifact, +generated deterministically from that source with only the Go runtime row +adapted to honest process metrics. Run `just dashboard-test`, then +`just process-metrics-contract`, then +`just dashboard-contract `. These checks are +network-independent when given a saved scrape or the loopback harness and +refuse promotion while any queried family is absent. diff --git a/deploy/grafana/stream.json b/deploy/grafana/stream.json new file mode 100644 index 0000000..d93df34 --- /dev/null +++ b/deploy/grafana/stream.json @@ -0,0 +1,7298 @@ +{ + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.4.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "name": "Annotations & Alerts", + "type": "dashboard", + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)" + }, + { + "name": "Phase transitions", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "hide": false, + "iconColor": "#4a3aa7", + "expr": "changes(max(jetstream_orchestrator_phase{job=~\"$job\",instance=~\"$instance\"})[$__interval:]) > 0", + "titleFormat": "phase transition", + "textFormat": "orchestrator phase changed" + } + ] + }, + "description": "Stream operational dashboard adapted from the checksum-pinned Jetstream V2 dashboard. Protocol and lifecycle rows are unchanged; only the Go runtime row uses Stream process metrics.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "title": "jetstream repo", + "url": "https://github.com/bluesky-social/jetstream", + "type": "link", + "icon": "external link", + "targetBlank": true, + "tags": [], + "asDropdown": false, + "includeVars": false, + "keepTime": false + } + ], + "panels": [ + { + "id": 9, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 1, + "type": "stat", + "title": "Phase", + "description": "Ingestion lifecycle phase. 0 is reserved for a process that never reached the dispatch loop.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "unset", + "color": "#898781", + "index": 0 + }, + "1": { + "text": "bootstrap", + "color": "#2a78d6", + "index": 1 + }, + "2": { + "text": "merging", + "color": "#eda100", + "index": 2 + }, + "3": { + "text": "steady state", + "color": "#008300", + "index": 3 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "jetstream_orchestrator_phase{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "Build", + "description": "Version label from jetstream_build_info.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "name", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "jetstream_build_info{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "{{version}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Uptime", + "description": "Uptime of the most recently started selected instance — shortest uptime surfaces recent restarts.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "time() - max(process_start_time_seconds{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Firehose events/s", + "description": "Upstream subscribeRepos events decoded per second.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "cps", + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_events_received_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Appended events/s", + "description": "Events durably appended to the active segment per second. During backfill this is dominated by repo downloads.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "cps", + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_ingest_events_appended_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Upstream freshness", + "description": "Seconds since steady-state live ingest last saw an upstream event. No data until steady state.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#eda100", + "value": 60 + }, + { + "color": "#e34948", + "value": 300 + } + ] + }, + "unit": "s", + "noValue": "pre-steady-state" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max((time() - jetstream_livestream_last_seen_upstream_event_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"}) and (jetstream_livestream_last_seen_upstream_event_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"} > 0))", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 7, + "type": "stat", + "title": "Data dir free", + "description": "Free bytes on the filesystem holding the data directory.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#e34948", + "value": null + }, + { + "color": "#eda100", + "value": 50000000000.0 + }, + { + "color": "green", + "value": 250000000000.0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "min(jetstream_data_dir_free_bytes{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 8, + "type": "stat", + "title": "Subscribers", + "description": "Connected /subscribe websocket clients.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(jetstream_subscribe_subscribers{job=~\"$job\",instance=~\"$instance\"}) or vector(0)", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 15, + "type": "row", + "title": "Live firehose", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "panels": [] + }, + { + "id": 10, + "type": "timeseries", + "title": "Firehose intake", + "description": "Upstream subscribeRepos decode results. Unknown events are archival holes a jetstream upgrade would fix.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "events received/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "decode errors/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown events/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#eb6834" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_events_received_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "events received/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_decode_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "decode errors/s", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_unknown_events_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "unknown events/s", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Relay sequence gaps", + "description": "Forward gaps in the relay's seq stream: upstream data loss, not local decode trouble. Should be flat at zero.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "gaps/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "missed seqs/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#eb6834" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_sequence_gaps_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "gaps/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_sequence_gap_missed_seqs_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "missed seqs/s", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Connection health", + "description": "Websocket reconnects and op=-1 error frames from the relay. Error-frame series appear only after the first occurrence.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "reconnects/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#eda100" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_reconnects_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "reconnects/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (code) (rate(jetstream_livestream_stream_error_frames_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "error frame {{code}}/s", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Replay & verify drops", + "description": "Deliberate steady-state drops (relay seq replays, stale resyncs) vs verify-queue overflow, which is permanent archival loss.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "verify queue drops/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_replayed_account_events_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "replayed #account/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_replayed_identity_events_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "replayed #identity/s", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_stale_resyncs_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "stale resyncs/s", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_livestream_verify_queue_dropped_events_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "verify queue drops/s", + "refId": "D", + "range": true, + "instant": false + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Verifier failures", + "description": "Verifier rejections by error class. A sustained 'other' rate means a new class needs categorizing.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (kind) (rate(jetstream_verifier_failures_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{kind}}/s", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 22, + "type": "row", + "title": "Ingest writer & segments", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "panels": [] + }, + { + "id": 16, + "type": "timeseries", + "title": "Append throughput", + "description": "Writer.Append successes and failures. Append errors indicate writer trouble and should be flat at zero.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 23 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "events appended/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "append errors/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_ingest_events_appended_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "events appended/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_ingest_append_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "append errors/s", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 17, + "type": "timeseries", + "title": "Dropped at validation gate", + "description": "Upstream records dropped at the ingest validation gate. Spec-invalid input vs unrepresentable vs upstream omissions.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 23 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (source, reason) (rate(jetstream_ingest_dropped_events_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{source}}/{{reason}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 18, + "type": "timeseries", + "title": "Blocks & rotations", + "description": "zstd block fsync rate and segment rotations (rotations shown as count per interval — they are rare).", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 23 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_ingest_blocks_flushed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "blocks flushed/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(jetstream_ingest_segments_rotated_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", + "legendFormat": "segments rotated", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 19, + "type": "timeseries", + "title": "Active segment size", + "description": "Compressed bytes written since the active segment's header. Saw-tooth resets on rotation.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 31 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "active segment bytes" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_ingest_active_segment_bytes{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "active segment bytes", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 20, + "type": "timeseries", + "title": "Segment seal latency", + "description": "End-to-end Writer.Seal duration (flush + footer + header pwrite + fsyncs), successful seals only.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 31 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_segment_seal_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_segment_seal_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 21, + "type": "timeseries", + "title": "Readable log", + "description": "In-memory readable log serving hot /subscribe reads. Pinned overrun means retention budget exceeded by undurable bytes.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 31 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "pinned overrun bytes" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_ingest_readable_log_bytes{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "resident bytes", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_ingest_readable_log_pinned_bytes{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "pinned bytes", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_ingest_readable_log_pinned_overrun_bytes{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "pinned overrun bytes", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 31, + "type": "row", + "title": "Backfill & retry", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "panels": [] + }, + { + "id": 23, + "type": "stat", + "title": "Backfill progress", + "description": "Repos completed this Run vs DIDs discovered via listRepos. Approximate: discovery keeps running during backfill.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 40 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#2a78d6", + "value": null + } + ] + }, + "unit": "percentunit", + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(jetstream_backfill_progress_completed{job=~\"$job\",instance=~\"$instance\"}) / clamp_min(sum(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"}), 1)", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 24, + "type": "stat", + "title": "Repos completed", + "description": "Repos the engine reported complete in the current Run.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 40 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(jetstream_backfill_progress_completed{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 25, + "type": "stat", + "title": "Repos failed (run)", + "description": "DIDs that exhausted their retry budget within the Run.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 40 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "#eda100", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(jetstream_backfill_failed_total{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 26, + "type": "stat", + "title": "Completion queue depth", + "description": "Repo completions waiting for a writer durable metadata batch.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 40 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "#eda100", + "value": 1000 + }, + { + "color": "#e34948", + "value": 10000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_backfill_completion_queue_depth{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 27, + "type": "timeseries", + "title": "Repo throughput", + "description": "listRepos discovery vs getRepo completion vs terminal failure.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 44 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "completed/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#008300" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "discovered/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "discovered/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_backfill_completed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "completed/s", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_backfill_failed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "failed/s", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 28, + "type": "timeseries", + "title": "HandleRepo latency", + "description": "Per-repo download+append duration, successful repos only.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 44 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_backfill_handle_repo_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(jetstream_backfill_handle_repo_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_backfill_handle_repo_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 29, + "type": "timeseries", + "title": "Completion durability wait", + "description": "Time a repo completion waited between being queued and becoming durable. A rising tail is a durability backlog.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 52 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_backfill_completion_queue_wait_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_backfill_completion_queue_wait_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 30, + "type": "timeseries", + "title": "Steady-state failed-repo retry", + "description": "Retry loop that drains repos which failed initial backfill.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 52 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "succeeded/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#008300" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "skipped (host parked)/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#eda100" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_backfill_failed_repo_retry_attempts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "attempts/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_backfill_failed_repo_retry_succeeded_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "succeeded/s", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_backfill_failed_repo_retry_failed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "failed/s", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_backfill_failed_repo_retry_skipped_host_parked_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "skipped (host parked)/s", + "refId": "D", + "range": true, + "instant": false + } + ] + }, + { + "id": 41, + "type": "row", + "title": "Serving: /subscribe & archive API", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "panels": [] + }, + { + "id": 32, + "type": "timeseries", + "title": "Subscribers by compression", + "description": "Connected /subscribe clients by negotiated wire compression.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 61 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (compression) (jetstream_subscribe_subscribers{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "{{compression}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 33, + "type": "timeseries", + "title": "Events sent to subscribers", + "description": "JSON frames written to clients, plus events removed by per-subscriber filters before encoding.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 61 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (compression) (rate(jetstream_subscribe_events_sent_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{compression}}/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_subscribe_events_filtered_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "filtered/s", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 34, + "type": "timeseries", + "title": "Egress bytes", + "description": "Bytes written to /subscribe clients (post-compression for zstd) and archive getBlock downloads. sent/encoded ratio approximates the compression ratio.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 61 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (compression) (rate(jetstream_subscribe_bytes_sent_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "subscribe {{compression}}/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_subscribe_bytes_encoded_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "pre-compression encoded/s", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_getblock_served_bytes_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "getBlock served/s", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 35, + "type": "timeseries", + "title": "Subscriber drops & errors", + "description": "Per-subscriber trouble. Encode errors and adversarial drops should be flat at zero.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 69 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "encode errors/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "adversarial drops/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (reason) (rate(jetstream_subscribe_events_skipped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "skipped {{reason}}/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_subscribe_events_oversize_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "oversize/s", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_subscribe_encode_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "encode errors/s", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_subscribe_adversarial_drops_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "adversarial drops/s", + "refId": "D", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (reason) (rate(jetstream_subscribe_options_update_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "options error {{reason}}/s", + "refId": "E", + "range": true, + "instant": false + } + ] + }, + { + "id": 36, + "type": "timeseries", + "title": "Cursor requests by mode", + "description": "New /subscribe connections by cursor resolution outcome.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 69 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (mode) (rate(jetstream_subscribe_cursor_requests_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{mode}}/s", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 37, + "type": "timeseries", + "title": "Hot vs cold reads", + "description": "ReadFrom calls served from the in-memory readable log vs falling through to the cold disk reader.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 69 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "hot (readable log)/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cold (disk)/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#eb6834" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_subscribe_hot_reads_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "hot (readable log)/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_subscribe_cold_reads_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "cold (disk)/s", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 38, + "type": "timeseries", + "title": "HTTP requests", + "description": "Public-listener HTTP request rate by handler and status.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 77 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (handler, code) (rate(jetstream_http_request_duration_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{handler}} {{code}}/s", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 39, + "type": "timeseries", + "title": "API latency (p99)", + "description": "99th percentile latency of the public API surfaces.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 77 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le,handler) (rate(jetstream_http_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "http {{handler}}", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_getblock_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "getBlock", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_subscribe_cursor_resolve_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "cursor resolve", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 40, + "type": "timeseries", + "title": "getBlock requests", + "description": "Archive getBlock requests by outcome. Series appear after the first request.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 77 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ok/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#008300" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (result) (rate(jetstream_getblock_requests_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{result}}/s", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 51, + "type": "row", + "title": "Storage: compaction, tombstones & manifest", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 85 + }, + "panels": [] + }, + { + "id": 42, + "type": "timeseries", + "title": "Compaction passes", + "description": "Delete/update compaction passes by result, plus passes triggered early by the tombstone cap.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 86 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ok/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#008300" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "early (tombstone cap)/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#eda100" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (result) (rate(jetstream_compaction_passes_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{result}}/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_compaction_passes_early_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "early (tombstone cap)/s", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 43, + "type": "timeseries", + "title": "Compaction pass duration", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 86 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_compaction_pass_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_compaction_pass_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 44, + "type": "timeseries", + "title": "Rows dropped & tombstones collected", + "description": "Compaction work: rows physically dropped from rewritten segments and tombstones collected by kind.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 86 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (reason) (rate(jetstream_compaction_rows_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "dropped {{reason}}/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (kind) (rate(jetstream_compaction_tombstones_collected_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "collected {{kind}}/s", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 45, + "type": "timeseries", + "title": "Tombstone set", + "description": "Entries in the live in-memory tombstone set. Drops to zero after each compaction pass.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 94 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_compaction_tombstone_set_entries{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "entries", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 46, + "type": "timeseries", + "title": "Tombstone set memory", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 94 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_compaction_tombstone_set_bytes{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "bytes", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 47, + "type": "timeseries", + "title": "Compaction watermark lag", + "description": "witnessed_at lag between the sealed segment tip and the compaction watermark (header granularity).", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 94 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_compaction_watermark_lag_seconds{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "lag", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 48, + "type": "timeseries", + "title": "Compaction rewrite I/O", + "description": "Segment-file bytes rewritten by compaction.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 94 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_compaction_bytes_rewritten_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "bytes/s", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 49, + "type": "timeseries", + "title": "Sealed segments", + "description": "Sealed segments tracked by the manifest, and compaction rewrite activity per interval.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 102 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_manifest_segments_loaded{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "segments loaded", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(jetstream_compaction_segments_rewritten_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", + "legendFormat": "rewritten (per interval)", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(jetstream_compaction_segments_skipped_clean_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", + "legendFormat": "skipped clean (per interval)", + "refId": "C", + "range": true, + "instant": false + } + ] + }, + { + "id": 50, + "type": "timeseries", + "title": "Block index cache", + "description": "Manifest BlockIndex lookups served from resident metadata. No data while there are no cold lookups.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 102 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_manifest_block_index_cache_hits_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(jetstream_manifest_block_index_cache_hits_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) + sum(rate(jetstream_manifest_block_index_cache_misses_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])), 1e-9)", + "legendFormat": "hit ratio", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 55, + "type": "row", + "title": "Store & integrity", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 110 + }, + "panels": [] + }, + { + "id": 52, + "type": "timeseries", + "title": "Store ops", + "description": "Pebble metadata-store operations by op and outcome. Any error series is a crash-loud-worthy signal.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 111 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (op, status) (rate(jetstream_store_op_duration_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{op}} {{status}}/s", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 53, + "type": "timeseries", + "title": "Store op latency (p99)", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 111 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le,op) (rate(jetstream_store_op_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "{{op}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 54, + "type": "timeseries", + "title": "Integrity signals (expect zero)", + "description": "Data-integrity error counters from every subsystem. All of these should be flat at zero; investigate any movement.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 111 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ingest append errors/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_ingest_append_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "ingest append errors/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_backfill_on_fail_store_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "backfill OnFail store errors/s", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_backfill_completion_stage_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "completion stage errors/s", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_import_rows_corrupt_offset_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "import corrupt offsets/s", + "refId": "D", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (op) (rate(jetstream_store_op_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",status=\"error\"}[$__rate_interval]))", + "legendFormat": "store {{op}} errors/s", + "refId": "E", + "range": true, + "instant": false + } + ] + }, + { + "id": 61, + "type": "row", + "title": "Cutover & merge", + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 119 + }, + "panels": [ + { + "id": 56, + "type": "timeseries", + "title": "Phase timeline", + "description": "1=bootstrap, 2=merging, 3=steady state. Transitions are also marked as dashboard annotations.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 120 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "phase" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#4a3aa7" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_orchestrator_phase{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "phase", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 57, + "type": "timeseries", + "title": "Merge rev filter", + "description": "live_segments/ events surviving the rev filter into the steady-state segments vs already covered by backfill.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 120 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "kept/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#008300" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dropped (rev covered)/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#898781" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_orchestrator_merge_events_kept_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "kept/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_orchestrator_merge_events_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "dropped (rev covered)/s", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 58, + "type": "timeseries", + "title": "Merge progress", + "description": "Run-once merge counters, shown as raw totals.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 120 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(jetstream_orchestrator_merge_segments_consumed_total{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "segments consumed", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(jetstream_orchestrator_merge_did_lookups_total{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "DID lookups", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(jetstream_orchestrator_merge_repo_revs_updated_total{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "repo revs updated", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(jetstream_orchestrator_merge_dids_discovered_post_bootstrap_total{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "DIDs discovered post-bootstrap", + "refId": "D", + "range": true, + "instant": false + } + ] + }, + { + "id": 59, + "type": "bargauge", + "title": "Cutover state durations", + "description": "Total wall-clock seconds spent in each cutover state across this process lifetime.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 128 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "#4a3aa7" + }, + "mappings": [], + "unit": "s", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#4a3aa7", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "horizontal", + "displayMode": "basic", + "valueMode": "color", + "showUnfilled": true, + "sizing": "auto", + "namePlacement": "auto", + "minVizWidth": 8, + "minVizHeight": 16, + "maxVizHeight": 300, + "legend": { + "showLegend": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (state) (jetstream_orchestrator_state_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "{{state}}", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 60, + "type": "timeseries", + "title": "Phase transitions", + "description": "Phase transitions per interval; normally each fires exactly once per data-dir lifetime.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 128 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (from, to) (increase(jetstream_orchestrator_phase_transitions_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", + "legendFormat": "{{from}} → {{to}}", + "refId": "A", + "range": true, + "instant": false + } + ] + } + ] + }, + { + "id": 68, + "type": "row", + "title": "Timestamp import", + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 136 + }, + "panels": [ + { + "id": 62, + "type": "stat", + "title": "Import phase", + "description": "Current timestamp-import phase.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 137 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "idle", + "color": "text", + "index": 0 + }, + "1": { + "text": "parse + bucket", + "color": "#2a78d6", + "index": 1 + }, + "2": { + "text": "apply", + "color": "#eb6834", + "index": 2 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(jetstream_import_phase{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 63, + "type": "timeseries", + "title": "Import rows", + "description": "Timestamp-import pipeline throughput across phases.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 137 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_import_rows_parsed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "parsed/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (scope) (rate(jetstream_import_rows_matched_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "matched {{scope}}/s", + "refId": "B", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_import_rows_mutated_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "mutated/s", + "refId": "C", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_import_dids_matched_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "DIDs matched/s", + "refId": "D", + "range": true, + "instant": false + } + ] + }, + { + "id": 64, + "type": "timeseries", + "title": "Import rows rejected", + "description": "CSV rows rejected during parse, by reason.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 137 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (reason) (rate(jetstream_import_rows_rejected_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{reason}}/s", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 65, + "type": "timeseries", + "title": "Import rewrite activity", + "description": "Phase C segment rewrites per interval.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 145 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(jetstream_import_segments_examined_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", + "legendFormat": "segments examined", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(jetstream_import_segments_patched_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", + "legendFormat": "segments patched", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 66, + "type": "timeseries", + "title": "Import rewrite I/O", + "description": "On-disk segment bytes rewritten by import.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 10, + "y": 145 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(jetstream_import_bytes_rewritten_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "bytes/s", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 67, + "type": "timeseries", + "title": "Import jobs", + "description": "Terminal import jobs per interval by result.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 145 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ok" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#008300" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (result) (increase(jetstream_import_jobs_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", + "legendFormat": "{{result}}", + "refId": "A", + "range": true, + "instant": false + } + ] + } + ] + }, + { + "id": 75, + "type": "row", + "title": "Process runtime", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 153 + }, + "panels": [] + }, + { + "id": 69, + "type": "timeseries", + "title": "CPU", + "description": "Process CPU usage in cores vs the scheduler limit.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 154 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "cores used" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "GOMAXPROCS" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#898781" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "cores used", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 70, + "type": "timeseries", + "title": "Memory", + "description": "Resident set vs Go heap. GOMEMLIMIT is hidden when unset.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 154 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "RSS" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "heap in use" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#1baf7a" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "GC goal" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#898781" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "RSS", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(process_virtual_memory_bytes{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "virtual", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 71, + "type": "timeseries", + "title": "Threads", + "description": "A monotonic goroutine climb is a leak.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 154 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "goroutines" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OS threads" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#898781" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(stream_process_threads{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "OS threads", + "refId": "A", + "range": true, + "instant": false + } + ] + }, + { + "id": 72, + "type": "timeseries", + "title": "File descriptors", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 162 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "open" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#898781" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(process_open_fds{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "open", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(process_max_fds{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "max", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 73, + "type": "timeseries", + "title": "Page faults", + "description": "Stop-the-world pause durations from the Go GC summary.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 162 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(process_minor_page_faults_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "minor/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(process_major_page_faults_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "major/s", + "refId": "B", + "range": true, + "instant": false + } + ] + }, + { + "id": 74, + "type": "timeseries", + "title": "Context switches", + "description": "Process-level network throughput (procfs; Linux only).", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 162 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#1baf7a" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(process_voluntary_context_switches_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "voluntary/s", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(process_involuntary_context_switches_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "involuntary/s", + "refId": "B", + "range": true, + "instant": false + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "stream", + "jetstream", + "atproto" + ], + "templating": { + "list": [ + { + "name": "datasource", + "label": "Data source", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "includeAll": false, + "multi": false, + "options": [], + "refresh": 1, + "regex": "", + "skipUrlSync": false + }, + { + "name": "job", + "label": "Job", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(jetstream_build_info, job)", + "refId": "job" + }, + "definition": "label_values(jetstream_build_info, job)", + "current": {}, + "hide": 0, + "includeAll": true, + "allValue": ".*", + "multi": true, + "options": [], + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1 + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(jetstream_build_info{job=~\"$job\"}, instance)", + "refId": "instance" + }, + "definition": "label_values(jetstream_build_info{job=~\"$job\"}, instance)", + "current": {}, + "hide": 0, + "includeAll": true, + "allValue": ".*", + "multi": true, + "options": [], + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1 + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Stream / Jetstream V2", + "uid": "stream-jetstream-v2", + "version": 1, + "weekStart": "" +} diff --git a/docs/grafana-dashboard.md b/docs/grafana-dashboard.md index b6d213e..e03f54a 100644 --- a/docs/grafana-dashboard.md +++ b/docs/grafana-dashboard.md @@ -1,4 +1,4 @@ -# Canonical Jetstream V2 Grafana dashboard +# Jetstream V2 and Stream Grafana dashboards Stream reuses the upstream dashboard rather than maintaining a lookalike. The reviewed artifact is: @@ -9,13 +9,20 @@ The reviewed artifact is: The exact reviewed bytes are vendored at `deploy/grafana/jetstream-upstream.json`; deployment does not depend on the -network or a moving branch. `just dashboard-test` verifies the checksum and -the contract parser. `just dashboard-contract ` -extracts every metric family referenced by PromQL and compares it with the -candidate's exposition, including histogram child series. It exits non-zero -and prints every missing family. This is deliberate: declaring placeholder -zero-valued metrics would make a dashboard appear healthy while measuring -nothing. +network or a moving branch. Stream provisions +`deploy/grafana/stream.json` (SHA-256 +`e27537b5a951bc1b5a6133aaef068b7b4b1eb2abff2e2f283722697cf6fe7017`). It is +deterministically generated by `tools/adapt_dashboard.py` from the pinned +source. The generator refuses a changed source checksum, and the tests prove +every non-runtime panel remains structurally identical to upstream. + +`just dashboard-test` verifies both checksums, regeneration, and the contract +parser. `just dashboard-contract ` extracts every +metric family referenced by the Stream dashboard's PromQL and compares it with +the candidate's exposition, including histogram child series. It exits +non-zero and prints every missing family. This is deliberate: declaring +placeholder zero-valued metrics would make a dashboard appear healthy while +measuring nothing. ## Reuse boundary @@ -25,15 +32,32 @@ timestamp import. Their queries are part of the parity contract, so Stream emits the same `jetstream_*` family only after it owns the same measurement boundary. -The Go runtime row is the sole expected adaptation. Stream must not synthesize -`go_*` series. Before provisioning, those panels will be replaced with real -process/Zig/host signals: RSS, virtual memory, thread count, CPU, allocator -arenas where available, file descriptors, and process start/build identity. +The Go runtime row is the sole adaptation. Stream does not synthesize `go_*` +series. It retains upstream's same-semantics standard process measurements for +start time, cumulative CPU, current RSS, and file descriptors; adds current +virtual memory and OS thread count; and replaces Go GC and fictional +per-process network panels with real cumulative page-fault and context-switch +counters. Values come from `getrusage` plus Linux procfs or Darwin +`proc_pidinfo`. Unavailable measurements are omitted, never reported as zero. +`just process-metrics-contract` launches the production binary with only +loopback dependencies and validates types, values, invariants, monotonicity, +and the absence of `go_*` series across independent HTTP scrapes. ## Implemented canonical producers The following dashboard families already have non-placeholder producers: +- `process_start_time_seconds` +- `process_cpu_seconds_total` +- `process_resident_memory_bytes` +- `process_virtual_memory_bytes` +- `process_open_fds` +- `process_max_fds` +- `process_minor_page_faults_total` +- `process_major_page_faults_total` +- `process_voluntary_context_switches_total` +- `process_involuntary_context_switches_total` +- `stream_process_threads` - `jetstream_build_info` - `jetstream_livestream_events_received_total` - `jetstream_livestream_decode_errors_total` diff --git a/docs/semantic-parity.md b/docs/semantic-parity.md index ea23c2d..40c5960 100644 --- a/docs/semantic-parity.md +++ b/docs/semantic-parity.md @@ -20,7 +20,7 @@ this document or `bootstrap-semantic-parity.md` is open. | Sync 1.1 resync ordering | Upstream serializes per DID, drops stale async repairs, and emits a sync tombstone plus authoritative replacements | **closed at the implementation boundary.** The live coordinator has 32 real fetch workers, a 64-job queue, a five-minute fetch budget plus the 1,000 B/s-for-30-seconds slow-transfer guard, a bounded 16,384-entry 5/minute-per-DID limiter, 2,048 pending commits with drop-oldest overflow, fetch outside the DID lane, authenticated apply with older/equal-contradictory head rejection, ordered pending replay, and trigger-ticket-gated outbox delivery. Commit divergence schedules async repair; `#sync` divergence attempts inline repair and queues one retry only for transient failure, preserving the original relay cursor only on inline success. The writer emits `sync` then bounded 1,024-row `create_resync` batches and stages the fetched/pending chain checkpoints under the archive lock before any matching fsync. Loopback-only tests exercise a real HTTP getRepo mmap, real DID resolution/signature, buffering while the response is held open, slow-transfer cancellation, durable RocksDB promotion/cursor coupling, v2 tail publication, and sealed JSS row order. | | Timestamp import | Canonical dashboard and API include timestamp import lifecycle/rewrite metrics and behavior | **Closed at the implementation boundary.** The format preserves distinct `witnessed_at` and sentinel-zero `indexed_at` columns. Subscriber v1/v2 encoding applies upstream's `indexed_at != 0 ? indexed_at : witnessed_at` display rule while ranges and timestamp cursors remain anchored to immutable witness envelopes. The strict seekable RFC 4180 parser matches upstream's header, row rejection, byte-offset, CID, RFC3339, 64 KiB, and bounded-sampling contracts in a forward-only allocation-free pass. A separate Bloom-filtered RocksDB rule store bulk-loads sorted external SST chunks with CSV-order last-write-wins, reconstructs its resident collection gate on open, and implements specific-CID precedence with all-version fallback. Its central archive-lock hook stamps live, repair, bootstrap, and merge materializations before sequence assignment, and feeds the mutated rows to immediate delivery so hot and cold encodings agree. The sealed-JSS patch primitive preserves block topology/envelopes, clean frames, and the opaque bloom/collection footer while atomically changing only `indexed_at`; it is exercised against an upstream-produced segment. A concrete sealed-segment bloom catalog, bounded DID/FD bucketer, fsynced packed offsets, positioned revalidation, and collision-safe per-segment patch plan implement Phase B/C with specific-CID precedence and idempotent disk-to-disk coverage. A dedicated archive rewrite mutex serializes the real delete compactor with timestamp patching while leaving live append available. The unified metadata DB synchronously persists the current-job pointer, complete job record, and per-segment done set. The real runner activates rules before force-seal, durably crosses the bucketed handoff, checkpoints only after patch fsync/rename/dir-sync, retains partial cancellation progress, and resumes after full archive/rule/metadata reopen. The background manager canonicalizes symlinks, confines regular files, enforces one durable nonterminal job, and adopts it on startup. Bearer-gated import/status XRPC matches the pinned lexicons, fixed-width digest comparison, error names/statuses, steady-state admission, and current/by-id status behavior; a real loopback HTTP run exercised it against the local simulator. Every canonical import metric is produced at its real parse, route, patch, or durable-terminal boundary; cancellation remains a resumable pause and is not counted as terminal failure. | | Status/diagnostics | Repo attempts/error class/final PDS and per-host aggregates survive restart and are operator-queryable | **closed at the implementation boundary.** Durable v5 repo rows distinguish initial Backfill.Rev from latest Rev/UpdatedAt, retain resolved Handle/PDS and reserved record/byte fields, and atomically maintain `handle/`. Merge source cursor commits refresh latest revisions without mutating backfill watermarks. Same-batch `host/` totals, active/status counts, cumulative error classes, and five bounded recent samples survive restart; host moves and active flips are atomic with the source repo row. `/status?tab=hosts` reads those rows without a whole-network scan and matches upstream ordering and error presentation. Account queries preserve resolver-first handle semantics, the legacy `did`/`handle` aliases, and missing-identity hydration, then reconstruct records last-writer-wins from checksum-verified, DID-bloom-pruned sealed JSS plus rotation-safe active/pending and bootstrap-live snapshots. The resulting canonical MST is compared with the PDS commit root through real Sync 1.1 `getLatestCommit`/`getBlocks`; the page renders the same match fields and error state as upstream. `HEAD` performs lookup without verification; successful status responses are `no-store` and carry a generation timestamp. Expensive verification uses upstream's exact 4/source-IP/minute fixed-window limiter, 4,096-entry bound, stale pruning, oldest eviction, and explicit disable option. The offline receipt uses a real HTTP Stream route, real RocksDB row, real JSS archive, signed commit, and loopback PLC/PDS; four separate connections verify successfully, a fifth changing only its ephemeral port is blocked without another upstream request, HEAD remains side-effect-free, and disabling the limiter performs a new authoritative verification. | -| Prometheus/Grafana | Every canonical dashboard query maps to a real, same-semantics producer; runtime-specific panels use honest Zig/process metrics | exact checksum-pinned upstream dashboard is vendored with an offline fail-closed scrape contract; canonical build/live-firehose/archive-append/drop/verifier/backfill/timestamp-import producers are present. The complete canonical subscriber family set is exported from real scan, delivery, rejection, cursor, and disconnect boundaries; `none`, `deflate`, and `zstd` labels now arise only from the actual negotiated delivery path. Live frame inspection distinguishes gaps, bounded-label server errors, forward-compatible unknowns, and malformed CBOR at the raw transport boundary; a Zat reconnect callback measures attempts rather than connection failures. **Remaining storage/API/orchestrator families and Go-row adaptation open.** | +| Prometheus/Grafana | Every canonical dashboard query maps to a real, same-semantics producer; runtime-specific panels use honest Zig/process metrics | exact checksum-pinned upstream dashboard is vendored with an offline fail-closed scrape contract; canonical build/live-firehose/archive-append/drop/verifier/backfill/timestamp-import producers are present. The complete canonical subscriber family set is exported from real scan, delivery, rejection, cursor, and disconnect boundaries; `none`, `deflate`, and `zstd` labels now arise only from the actual negotiated delivery path. Live frame inspection distinguishes gaps, bounded-label server errors, forward-compatible unknowns, and malformed CBOR at the raw transport boundary; a Zat reconnect callback measures attempts rather than connection failures. The only dashboard divergence is now a deterministic, checksum-pinned process-runtime row: same-semantics start/CPU/RSS/FD measurements, real VM/thread/page-fault/context-switch signals, no `go_*` aliases, and no host-network counters mislabeled as process I/O. A production-binary loopback receipt proves values and cumulative monotonicity on independent scrapes. **Remaining storage/API/orchestrator/backfill/compaction families open.** | | Oracle | Event-log equivalence, final-state convergence, crash/power-loss, verifier repair, hostile input, and anti-vacuity receipts | **open — current crash matrix is necessary but substantially smaller than upstream's oracle** | ## Experiment admission diff --git a/justfile b/justfile index 609d089..0a8af8c 100644 --- a/justfile +++ b/justfile @@ -11,12 +11,40 @@ test: # Offline: validate the vendored upstream Grafana dashboard and its parser. dashboard-test: + python3 tools/adapt_dashboard.py --check python3 -m unittest tests/test_dashboard_contract.py # METRICS may be a saved scrape, localhost URL, or '-' for stdin. dashboard-contract METRICS: python3 tools/check_dashboard.py --metrics {{METRICS}} +# Launch the production binary offline and prove its process metrics are real, +# complete, and monotonic across independent HTTP scrapes. +process-metrics-contract: + #!/usr/bin/env bash + set -euo pipefail + DATA=$(mktemp -d) + LOG=$(mktemp) + PORT=${STREAM_PROCESS_METRICS_PORT:-6020} + cleanup() { + if [[ -n "${pid:-}" ]]; then kill "$pid" 2>/dev/null || true; wait "$pid" 2>/dev/null || true; fi + rm -rf "$DATA" "$LOG" + } + trap cleanup EXIT + python3 -c 'import socket,sys; s=socket.socket(); s.bind(("127.0.0.1",int(sys.argv[1]))); s.close()' "$PORT" + zig build -Doptimize=ReleaseSafe + ./zig-out/bin/stream --port="$PORT" --data-dir="$DATA" \ + --upstream=ws://127.0.0.1:17999 --plc=http://127.0.0.1:17999 \ + --relay-http=http://127.0.0.1:17999 --compaction-interval=0 \ + --retry-interval=0 --no-verify >"$LOG" 2>&1 & + pid=$! + for _ in {1..100}; do + if curl -fsS "http://127.0.0.1:$PORT/healthz" >/dev/null 2>&1; then break; fi + if ! kill -0 "$pid" 2>/dev/null; then cat "$LOG" >&2; exit 1; fi + sleep 0.1 + done + STREAM_BASE_URL="http://127.0.0.1:$PORT" python3 tests/process_metrics_contract.py + # Seed a real sealed archive, serve it, then exercise all four archive XRPCs # plus whole-segment, block-plan, bounded, and live-cutover paths through the # exact pinned upstream Go client. Go and uv are forced offline; Zig's pinned diff --git a/src/internal/metrics.zig b/src/internal/metrics.zig index b049670..90b07a4 100644 --- a/src/internal/metrics.zig +++ b/src/internal/metrics.zig @@ -11,6 +11,7 @@ const std = @import("std"); const build_options = @import("build_options"); const convert = @import("convert.zig"); +const process_metrics = @import("process_metrics.zig"); const timestamp_parse = @import("timestamp/parse.zig"); const Io = std.Io; @@ -134,6 +135,7 @@ pub fn format( subscribers_active: u32, tail: TailGauges, now_s: i64, + process: process_metrics.Snapshot, ) []const u8 { var w: Io.Writer = .fixed(buf); @@ -174,6 +176,62 @@ pub fn format( tail.tip, }) catch {}; + w.print( + \\# TYPE process_start_time_seconds gauge + \\process_start_time_seconds {d} + \\ + , .{stats.start_time_s}) catch {}; + if (process.cpu_us) |value| w.print( + \\# TYPE process_cpu_seconds_total counter + \\process_cpu_seconds_total {d}.{d:0>6} + \\ + , .{ value / std.time.us_per_s, value % std.time.us_per_s }) catch {}; + if (process.resident_memory_bytes) |value| w.print( + \\# TYPE process_resident_memory_bytes gauge + \\process_resident_memory_bytes {d} + \\ + , .{value}) catch {}; + if (process.virtual_memory_bytes) |value| w.print( + \\# TYPE process_virtual_memory_bytes gauge + \\process_virtual_memory_bytes {d} + \\ + , .{value}) catch {}; + if (process.threads) |value| w.print( + \\# TYPE stream_process_threads gauge + \\stream_process_threads {d} + \\ + , .{value}) catch {}; + if (process.open_fds) |value| w.print( + \\# TYPE process_open_fds gauge + \\process_open_fds {d} + \\ + , .{value}) catch {}; + if (process.max_fds) |value| w.print( + \\# TYPE process_max_fds gauge + \\process_max_fds {d} + \\ + , .{value}) catch {}; + if (process.minor_page_faults) |value| w.print( + \\# TYPE process_minor_page_faults_total counter + \\process_minor_page_faults_total {d} + \\ + , .{value}) catch {}; + if (process.major_page_faults) |value| w.print( + \\# TYPE process_major_page_faults_total counter + \\process_major_page_faults_total {d} + \\ + , .{value}) catch {}; + if (process.voluntary_context_switches) |value| w.print( + \\# TYPE process_voluntary_context_switches_total counter + \\process_voluntary_context_switches_total {d} + \\ + , .{value}) catch {}; + if (process.involuntary_context_switches) |value| w.print( + \\# TYPE process_involuntary_context_switches_total counter + \\process_involuntary_context_switches_total {d} + \\ + , .{value}) catch {}; + w.print( \\# TYPE stream_resync_total counter \\stream_resync_total{{result="completed"}} {d} @@ -555,12 +613,34 @@ test "format renders counters" { _ = stats.subscribe_options_updates_total.fetchAdd(16, .monotonic); var buf: [16 * 1024]u8 = undefined; - const out = format(&buf, &stats, 2, .{ .entries = 5, .bytes = 100, .base = 1, .tip = 6 }, 60); + const out = format(&buf, &stats, 2, .{ .entries = 5, .bytes = 100, .base = 1, .tip = 6 }, 60, .{ + .cpu_us = 1_250_000, + .resident_memory_bytes = 4096, + .virtual_memory_bytes = 8192, + .threads = 7, + .open_fds = 11, + .max_fds = 1024, + .minor_page_faults = 31, + .major_page_faults = 2, + .voluntary_context_switches = 41, + .involuntary_context_switches = 3, + }); try std.testing.expect(std.mem.indexOf(u8, out, "stream_events_total 7") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_verify_total{result=\"valid\"} 3") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_dropped_events_total{reason=\"invalid_signature\"} 1") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_build_info{git_sha=") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_subscribers_active 2") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "process_start_time_seconds 0") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "process_cpu_seconds_total 1.250000") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "process_resident_memory_bytes 4096") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "process_virtual_memory_bytes 8192") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "stream_process_threads 7") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "process_open_fds 11") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "process_max_fds 1024") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "process_minor_page_faults_total 31") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "process_major_page_faults_total 2") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "process_voluntary_context_switches_total 41") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "process_involuntary_context_switches_total 3") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_backfill_repos_total{result=\"complete\"} 11") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_backfill_workers_active 4") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_backfill_queue_depth 23") != null); diff --git a/src/internal/process_metrics.zig b/src/internal/process_metrics.zig new file mode 100644 index 0000000..4fe5e2f --- /dev/null +++ b/src/internal/process_metrics.zig @@ -0,0 +1,189 @@ +//! Honest process measurements for the Grafana runtime row. +//! +//! The upstream dashboard gets these from the Go Prometheus collector. Stream +//! measures the same process boundaries directly instead of publishing +//! synthetic `go_*` series. Linux memory/thread values come from procfs; +//! Darwin uses the kernel's proc_pidinfo task snapshot. CPU is cumulative +//! getrusage time on both platforms, and FD counts enumerate this process's +//! descriptor directory. + +const std = @import("std"); + +const Io = std.Io; + +pub const Snapshot = struct { + cpu_us: ?u64 = null, + resident_memory_bytes: ?u64 = null, + virtual_memory_bytes: ?u64 = null, + threads: ?u64 = null, + open_fds: ?u64 = null, + max_fds: ?u64 = null, + minor_page_faults: ?u64 = null, + major_page_faults: ?u64 = null, + voluntary_context_switches: ?u64 = null, + involuntary_context_switches: ?u64 = null, +}; + +pub fn collect(io: Io) Snapshot { + var result: Snapshot = .{}; + + const usage = std.posix.getrusage(std.posix.rusage.SELF); + result.cpu_us = timevalMicros(usage.utime) +| timevalMicros(usage.stime); + result.minor_page_faults = nonNegative(usage.minflt); + result.major_page_faults = nonNegative(usage.majflt); + result.voluntary_context_switches = nonNegative(usage.nvcsw); + result.involuntary_context_switches = nonNegative(usage.nivcsw); + + const limits = std.posix.getrlimit(.NOFILE) catch null; + if (limits) |value| { + // The soft limit is the process's effective maximum. The hard limit + // is merely the ceiling to which a privileged process may raise it. + if (value.cur != std.math.maxInt(std.posix.rlim_t)) + result.max_fds = @intCast(value.cur); + } + result.open_fds = countOpenFds(io); + + switch (@import("builtin").os.tag) { + .linux => { + var buf: [16 * 1024]u8 = undefined; + const status = Io.Dir.cwd().readFile(io, "/proc/self/status", &buf) catch return result; + if (status.len == buf.len) return result; + const parsed = parseLinuxStatus(status); + result.resident_memory_bytes = parsed.resident_memory_bytes; + result.virtual_memory_bytes = parsed.virtual_memory_bytes; + result.threads = parsed.threads; + }, + .macos => { + if (darwinTaskInfo()) |task| { + result.resident_memory_bytes = task.resident_size; + result.virtual_memory_bytes = task.virtual_size; + if (task.thread_count >= 0) result.threads = @intCast(task.thread_count); + } + }, + else => {}, + } + return result; +} + +fn timevalMicros(value: anytype) u64 { + if (value.sec < 0 or value.usec < 0) return 0; + return @as(u64, @intCast(value.sec)) *| std.time.us_per_s +| @as(u64, @intCast(value.usec)); +} + +fn nonNegative(value: anytype) ?u64 { + if (value < 0) return null; + return @intCast(value); +} + +fn countOpenFds(io: Io) ?u64 { + const path = switch (@import("builtin").os.tag) { + .linux => "/proc/self/fd", + .macos => "/dev/fd", + else => return null, + }; + var dir = Io.Dir.cwd().openDir(io, path, .{ .iterate = true }) catch return null; + defer dir.close(io); + var iterator = dir.iterate(); + var count: u64 = 0; + while (iterator.next(io) catch return null) |_| count += 1; + // Opening the descriptor directory itself adds one descriptor to the + // listing. Remove that observation artifact from the process count. + return count -| 1; +} + +const LinuxStatus = struct { + resident_memory_bytes: ?u64 = null, + virtual_memory_bytes: ?u64 = null, + threads: ?u64 = null, +}; + +fn parseLinuxStatus(bytes: []const u8) LinuxStatus { + var result: LinuxStatus = .{}; + var lines = std.mem.splitScalar(u8, bytes, '\n'); + while (lines.next()) |line| { + if (parseKbField(line, "VmRSS:")) |value| result.resident_memory_bytes = value; + if (parseKbField(line, "VmSize:")) |value| result.virtual_memory_bytes = value; + if (parseIntegerField(line, "Threads:")) |value| result.threads = value; + } + return result; +} + +fn parseKbField(line: []const u8, prefix: []const u8) ?u64 { + if (!std.mem.startsWith(u8, line, prefix)) return null; + const rest = std.mem.trim(u8, line[prefix.len..], " \t"); + if (!std.mem.endsWith(u8, rest, "kB")) return null; + const digits = std.mem.trim(u8, rest[0 .. rest.len - 2], " \t"); + const kib = std.fmt.parseInt(u64, digits, 10) catch return null; + return std.math.mul(u64, kib, 1024) catch null; +} + +fn parseIntegerField(line: []const u8, prefix: []const u8) ?u64 { + if (!std.mem.startsWith(u8, line, prefix)) return null; + return std.fmt.parseInt(u64, std.mem.trim(u8, line[prefix.len..], " \t"), 10) catch null; +} + +const DarwinTaskInfo = extern struct { + virtual_size: u64, + resident_size: u64, + total_user: u64, + total_system: u64, + threads_user: u64, + threads_system: u64, + policy: i32, + faults: i32, + pageins: i32, + cow_faults: i32, + messages_sent: i32, + messages_received: i32, + syscalls_mach: i32, + syscalls_unix: i32, + context_switches: i32, + thread_count: i32, + running_thread_count: i32, + priority: i32, +}; + +const Darwin = struct { + extern "c" fn proc_pidinfo(pid: std.posix.pid_t, flavor: c_int, arg: u64, buffer: *anyopaque, size: c_int) c_int; +}; + +fn darwinTaskInfo() ?DarwinTaskInfo { + if (@import("builtin").os.tag != .macos) return null; + var info: DarwinTaskInfo = undefined; + const written = Darwin.proc_pidinfo(std.posix.system.getpid(), 4, 0, &info, @sizeOf(DarwinTaskInfo)); + if (written != @sizeOf(DarwinTaskInfo)) return null; + return info; +} + +test "linux status parsing preserves procfs units" { + const parsed = parseLinuxStatus("Name:\tstream\n" ++ + "VmSize:\t 987654 kB\n" ++ + "VmRSS:\t 12345 kB\n" ++ + "Threads:\t17\n"); + try std.testing.expectEqual(@as(?u64, 987654 * 1024), parsed.virtual_memory_bytes); + try std.testing.expectEqual(@as(?u64, 12345 * 1024), parsed.resident_memory_bytes); + try std.testing.expectEqual(@as(?u64, 17), parsed.threads); +} + +test "malformed procfs values remain unavailable" { + const parsed = parseLinuxStatus("VmSize:\tnope kB\n" ++ + "VmRSS:\t18446744073709551615 kB\n" ++ + "Threads:\t-1\n"); + try std.testing.expectEqual(@as(?u64, null), parsed.virtual_memory_bytes); + try std.testing.expectEqual(@as(?u64, null), parsed.resident_memory_bytes); + try std.testing.expectEqual(@as(?u64, null), parsed.threads); +} + +test "live process snapshot has real runtime measurements" { + const snapshot = collect(std.testing.io); + try std.testing.expect(snapshot.cpu_us != null); + try std.testing.expect(snapshot.resident_memory_bytes != null); + try std.testing.expect(snapshot.virtual_memory_bytes != null); + try std.testing.expect(snapshot.threads != null and snapshot.threads.? > 0); + try std.testing.expect(snapshot.open_fds != null and snapshot.open_fds.? > 0); + try std.testing.expect(snapshot.max_fds != null and snapshot.max_fds.? >= snapshot.open_fds.?); + try std.testing.expect(snapshot.minor_page_faults != null); + try std.testing.expect(snapshot.major_page_faults != null); + try std.testing.expect(snapshot.voluntary_context_switches != null); + try std.testing.expect(snapshot.involuntary_context_switches != null); +} diff --git a/src/internal/server.zig b/src/internal/server.zig index 04f7a18..10f151d 100644 --- a/src/internal/server.zig +++ b/src/internal/server.zig @@ -16,6 +16,7 @@ const cold = @import("cold.zig"); const filter_mod = @import("filter.zig"); const homepage = @import("homepage.zig"); const metrics = @import("metrics.zig"); +const process_metrics = @import("process_metrics.zig"); const repo_action_limiter = @import("repo_action_limiter.zig"); const repo_export = @import("repo_export.zig"); const repo_store = @import("bootstrap/repo_store.zig"); @@ -764,13 +765,13 @@ pub const Handler = struct { if (std.mem.eql(u8, path, "/metrics")) { const g = hub.tail.gauges(); const now_s: i64 = @divTrunc(Io.Timestamp.now(hub.io, .real).toMicroseconds(), std.time.us_per_s); - var buf: [16 * 1024]u8 = undefined; + var buf: [24 * 1024]u8 = undefined; const out = metrics.format(&buf, hub.stats, hub.active.load(.monotonic), .{ .entries = g.entries, .bytes = g.bytes, .base = g.base, .tip = g.tip, - }, now_s); + }, now_s, process_metrics.collect(hub.io)); var extra_buf: [256]u8 = undefined; var extra: []const u8 = ""; if (hub.pipeline_gauges) |pg| { diff --git a/tests/process_metrics_contract.py b/tests/process_metrics_contract.py new file mode 100644 index 0000000..0ecb5a6 --- /dev/null +++ b/tests/process_metrics_contract.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Empirical contract for Stream's real process metrics endpoint.""" + +from __future__ import annotations + +import math +import os +import re +import time +import urllib.request + + +BASE = os.environ.get("STREAM_BASE_URL", "http://127.0.0.1:6008") +TYPE_RE = re.compile(r"^# TYPE (\S+) (\S+)$") +SAMPLE_RE = re.compile(r"^(\S+?)(?:\{[^}]*\})?\s+([^\s]+)$") + +EXPECTED_TYPES = { + "process_start_time_seconds": "gauge", + "process_cpu_seconds_total": "counter", + "process_resident_memory_bytes": "gauge", + "process_virtual_memory_bytes": "gauge", + "stream_process_threads": "gauge", + "process_open_fds": "gauge", + "process_max_fds": "gauge", + "process_minor_page_faults_total": "counter", + "process_major_page_faults_total": "counter", + "process_voluntary_context_switches_total": "counter", + "process_involuntary_context_switches_total": "counter", +} +COUNTERS = {name for name, kind in EXPECTED_TYPES.items() if kind == "counter"} + + +def scrape() -> tuple[str, dict[str, str], dict[str, float]]: + with urllib.request.urlopen(BASE + "/metrics", timeout=5) as response: + assert response.status == 200 + raw = response.read().decode() + types: dict[str, str] = {} + samples: dict[str, float] = {} + for line in raw.splitlines(): + if match := TYPE_RE.match(line): + types[match.group(1)] = match.group(2) + elif not line.startswith("#") and (match := SAMPLE_RE.match(line)): + samples[match.group(1)] = float(match.group(2)) + return raw, types, samples + + +def main() -> None: + first_raw, first_types, first = scrape() + assert not re.search(r"(?m)^go_", first_raw), "Stream emitted synthetic Go metrics" + for name, kind in EXPECTED_TYPES.items(): + assert first_types.get(name) == kind, (name, first_types.get(name), kind) + assert name in first and math.isfinite(first[name]) and first[name] >= 0, (name, first.get(name)) + + assert 0 < first["process_start_time_seconds"] <= time.time() + assert 0 < first["process_resident_memory_bytes"] <= first["process_virtual_memory_bytes"] + assert first["stream_process_threads"] >= 1 + assert 0 < first["process_open_fds"] <= first["process_max_fds"] + + # Exercise the handler between snapshots, then prove cumulative process + # counters never moved backwards and identity stayed fixed. + for _ in range(8): + urllib.request.urlopen(BASE + "/healthz", timeout=5).read() + _, second_types, second = scrape() + assert second_types == first_types + assert second["process_start_time_seconds"] == first["process_start_time_seconds"] + for name in COUNTERS: + assert second[name] >= first[name], (name, first[name], second[name]) + + print( + "process metrics PASS: " + f"rss={int(second['process_resident_memory_bytes'])} " + f"vm={int(second['process_virtual_memory_bytes'])} " + f"threads={int(second['stream_process_threads'])} " + f"fds={int(second['process_open_fds'])}/{int(second['process_max_fds'])}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_dashboard_contract.py b/tests/test_dashboard_contract.py index f8a2d81..46af914 100644 --- a/tests/test_dashboard_contract.py +++ b/tests/test_dashboard_contract.py @@ -1,4 +1,5 @@ import importlib.util +import json import pathlib import unittest @@ -11,13 +12,32 @@ SPEC.loader.exec_module(check_dashboard) class DashboardContractTest(unittest.TestCase): - def test_pinned_dashboard_checksum_and_import_families(self): + def test_pinned_dashboard_checksums_and_import_families(self): + for path, expected in check_dashboard.EXPECTED_SHA256.items(): + self.assertEqual(expected, check_dashboard.hashlib.sha256(path.read_bytes()).hexdigest()) raw = check_dashboard.DEFAULT_DASHBOARD.read_bytes() - self.assertEqual(check_dashboard.EXPECTED_SHA256, check_dashboard.hashlib.sha256(raw).hexdigest()) required = check_dashboard.dashboard_families(raw) self.assertIn("jetstream_import_jobs_total", required) self.assertIn("jetstream_import_rows_rejected_total", required) + def test_stream_runtime_row_has_no_go_or_fake_network_families(self): + required = check_dashboard.dashboard_families(check_dashboard.DEFAULT_DASHBOARD.read_bytes()) + self.assertFalse(any(name.startswith("go_") for name in required)) + self.assertNotIn("process_network_receive_bytes_total", required) + self.assertNotIn("process_network_transmit_bytes_total", required) + self.assertIn("process_virtual_memory_bytes", required) + self.assertIn("stream_process_threads", required) + self.assertIn("process_major_page_faults_total", required) + self.assertIn("process_involuntary_context_switches_total", required) + + def test_non_runtime_panels_are_upstream_exact(self): + upstream = json.loads(check_dashboard.UPSTREAM_DASHBOARD.read_bytes()) + stream = json.loads(check_dashboard.DEFAULT_DASHBOARD.read_bytes()) + runtime_ids = {69, 70, 71, 72, 73, 74, 75} + upstream_panels = {panel["id"]: panel for panel in upstream["panels"] if panel["id"] not in runtime_ids} + stream_panels = {panel["id"]: panel for panel in stream["panels"] if panel["id"] not in runtime_ids} + self.assertEqual(upstream_panels, stream_panels) + def test_histogram_type_expands_queryable_children(self): present = check_dashboard.scrape_families( "# TYPE request_seconds histogram\nrequest_seconds_bucket{le=\"1\"} 2\n" diff --git a/tools/adapt_dashboard.py b/tools/adapt_dashboard.py new file mode 100644 index 0000000..c12beff --- /dev/null +++ b/tools/adapt_dashboard.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Deterministically adapt only Jetstream's Go runtime row for Stream.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import pathlib +import sys + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SOURCE = ROOT / "deploy/grafana/jetstream-upstream.json" +OUTPUT = ROOT / "deploy/grafana/stream.json" +SOURCE_SHA256 = "ef6033c182d8c3f9d88af7d91e61126724efbe72e6b940984c9de12e9530cd91" + + +def target(template: dict, expr: str, legend: str, ref_id: str) -> dict: + value = copy.deepcopy(template) + value.update(expr=expr, legendFormat=legend, refId=ref_id) + return value + + +def adapt(raw: bytes) -> bytes: + digest = hashlib.sha256(raw).hexdigest() + if digest != SOURCE_SHA256: + raise ValueError(f"upstream dashboard checksum mismatch: expected {SOURCE_SHA256}, got {digest}") + + dashboard = json.loads(raw) + panels = {panel.get("id"): panel for panel in dashboard["panels"]} + expected = { + 3: "Uptime", + 69: "CPU", + 70: "Memory", + 71: "Goroutines & threads", + 72: "File descriptors", + 73: "GC pauses", + 74: "Process network I/O", + 75: "Process & Go runtime", + } + for panel_id, title in expected.items(): + if panels.get(panel_id, {}).get("title") != title: + raise ValueError(f"upstream runtime panel {panel_id} is no longer {title!r}") + + selector = '{job=~"$job",instance=~"$instance"}' + rate = "[$__rate_interval]" + panels[75]["title"] = "Process runtime" + + # Standard process collector families retain their upstream meaning. + panels[69]["targets"] = [copy.deepcopy(panels[69]["targets"][0])] + + memory_template = panels[70]["targets"][0] + panels[70]["targets"] = [ + target(memory_template, f"max(process_resident_memory_bytes{selector})", "RSS", "A"), + target(memory_template, f"max(process_virtual_memory_bytes{selector})", "virtual", "B"), + ] + + thread_template = panels[71]["targets"][0] + panels[71]["title"] = "Threads" + panels[71]["targets"] = [ + target(thread_template, f"max(stream_process_threads{selector})", "OS threads", "A") + ] + + fault_template = panels[73]["targets"][0] + panels[73]["title"] = "Page faults" + panels[73]["targets"] = [ + target(fault_template, f"sum(rate(process_minor_page_faults_total{selector}{rate}))", "minor/s", "A"), + target(fault_template, f"sum(rate(process_major_page_faults_total{selector}{rate}))", "major/s", "B"), + ] + panels[73]["fieldConfig"]["defaults"]["unit"] = "ops" + + switch_template = panels[74]["targets"][0] + panels[74]["title"] = "Context switches" + panels[74]["targets"] = [ + target( + switch_template, + f"sum(rate(process_voluntary_context_switches_total{selector}{rate}))", + "voluntary/s", + "A", + ), + target( + switch_template, + f"sum(rate(process_involuntary_context_switches_total{selector}{rate}))", + "involuntary/s", + "B", + ), + ] + panels[74]["fieldConfig"]["defaults"]["unit"] = "ops" + + dashboard["title"] = "Stream / Jetstream V2" + dashboard["uid"] = "stream-jetstream-v2" + dashboard["tags"] = ["stream", "jetstream", "atproto"] + dashboard["description"] = ( + "Stream operational dashboard adapted from the checksum-pinned Jetstream V2 dashboard. " + "Protocol and lifecycle rows are unchanged; only the Go runtime row uses Stream process metrics." + ) + return (json.dumps(dashboard, indent=2, ensure_ascii=False) + "\n").encode() + + +def main() -> int: + generated = adapt(SOURCE.read_bytes()) + if len(sys.argv) == 2 and sys.argv[1] == "--check": + if not OUTPUT.exists() or OUTPUT.read_bytes() != generated: + print(f"{OUTPUT} is stale; run tools/adapt_dashboard.py", file=sys.stderr) + return 1 + print(f"stream dashboard sha256={hashlib.sha256(generated).hexdigest()}") + return 0 + if len(sys.argv) != 1: + print("usage: tools/adapt_dashboard.py [--check]", file=sys.stderr) + return 2 + OUTPUT.write_bytes(generated) + print(f"wrote {OUTPUT} sha256={hashlib.sha256(generated).hexdigest()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/check_dashboard.py b/tools/check_dashboard.py index 301bd98..d045c1d 100755 --- a/tools/check_dashboard.py +++ b/tools/check_dashboard.py @@ -13,9 +13,13 @@ import urllib.request ROOT = pathlib.Path(__file__).resolve().parents[1] -DEFAULT_DASHBOARD = ROOT / "deploy/grafana/jetstream-upstream.json" -EXPECTED_SHA256 = "ef6033c182d8c3f9d88af7d91e61126724efbe72e6b940984c9de12e9530cd91" -METRIC_RE = re.compile(r"\b(?:jetstream|go|process)_[a-zA-Z_:][a-zA-Z0-9_:]*") +UPSTREAM_DASHBOARD = ROOT / "deploy/grafana/jetstream-upstream.json" +DEFAULT_DASHBOARD = ROOT / "deploy/grafana/stream.json" +EXPECTED_SHA256 = { + UPSTREAM_DASHBOARD.resolve(): "ef6033c182d8c3f9d88af7d91e61126724efbe72e6b940984c9de12e9530cd91", + DEFAULT_DASHBOARD.resolve(): "e27537b5a951bc1b5a6133aaef068b7b4b1eb2abff2e2f283722697cf6fe7017", +} +METRIC_RE = re.compile(r"\b(?:jetstream|go|process|stream)_[a-zA-Z_:][a-zA-Z0-9_:]*") TYPE_RE = re.compile(r"^# TYPE ([a-zA-Z_:][a-zA-Z0-9_:]*) (\w+)$") SAMPLE_RE = re.compile(r"^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{|\s)") @@ -81,8 +85,9 @@ def main() -> int: raw = args.dashboard.read_bytes() digest = hashlib.sha256(raw).hexdigest() - if args.dashboard.resolve() == DEFAULT_DASHBOARD.resolve() and digest != EXPECTED_SHA256: - print(f"dashboard checksum mismatch: expected {EXPECTED_SHA256}, got {digest}", file=sys.stderr) + expected_digest = EXPECTED_SHA256.get(args.dashboard.resolve()) + if expected_digest is not None and digest != expected_digest: + print(f"dashboard checksum mismatch: expected {expected_digest}, got {digest}", file=sys.stderr) return 2 required = dashboard_families(raw) -- 2.51.2