{ "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.4.0" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } ], "annotations": { "list": [ { "builtIn": 1, "name": "Annotations & Alerts", "type": "dashboard", "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)" }, { "name": "Phase transitions", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "enable": true, "hide": false, "iconColor": "#4a3aa7", "expr": "changes(max(jetstream_orchestrator_phase{job=~\"$job\",instance=~\"$instance\"})[$__interval:]) > 0", "titleFormat": "phase transition", "textFormat": "orchestrator phase changed" } ] }, "description": "Stream operational dashboard adapted from the checksum-pinned Jetstream V2 dashboard. Protocol and lifecycle rows are unchanged; the runtime row uses Stream process metrics, and final Stream-only rows expose backfill work over time and scheduler health.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [ { "title": "jetstream repo", "url": "https://github.com/bluesky-social/jetstream", "type": "link", "icon": "external link", "targetBlank": true, "tags": [], "asDropdown": false, "includeVars": false, "keepTime": false } ], "panels": [ { "id": 9, "type": "row", "title": "Overview", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "panels": [] }, { "id": 1, "type": "stat", "title": "Phase", "description": "Ingestion lifecycle phase. 0 is reserved for a process that never reached the dispatch loop.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 0, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "0": { "text": "unset", "color": "#898781", "index": 0 }, "1": { "text": "bootstrap", "color": "#2a78d6", "index": 1 }, "2": { "text": "merging", "color": "#eda100", "index": 2 }, "3": { "text": "steady state", "color": "#008300", "index": 3 } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "jetstream_orchestrator_phase{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 2, "type": "stat", "title": "Build", "description": "Version label from jetstream_build_info.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 3, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "name", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "jetstream_build_info{job=~\"$job\",instance=~\"$instance\"}", "legendFormat": "{{version}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 3, "type": "stat", "title": "Uptime", "description": "Uptime of the most recently started selected instance — shortest uptime surfaces recent restarts.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 6, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "s" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "time() - max(process_start_time_seconds{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 4, "type": "stat", "title": "Firehose events/s", "description": "Upstream subscribeRepos events decoded per second.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 9, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "cps", "decimals": 1 }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_events_received_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 5, "type": "stat", "title": "Appended events/s", "description": "Events durably appended to the active segment per second. During backfill this is dominated by repo downloads.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 12, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "cps", "decimals": 1 }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_ingest_events_appended_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 6, "type": "stat", "title": "Upstream freshness", "description": "Seconds since steady-state live ingest last saw an upstream event. No data until steady state.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 15, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#eda100", "value": 60 }, { "color": "#e34948", "value": 300 } ] }, "unit": "s", "noValue": "pre-steady-state" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max((time() - jetstream_livestream_last_seen_upstream_event_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"}) and (jetstream_livestream_last_seen_upstream_event_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"} > 0))", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 7, "type": "stat", "title": "Data dir free", "description": "Free bytes on the filesystem holding the data directory.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 18, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#e34948", "value": null }, { "color": "#eda100", "value": 50000000000.0 }, { "color": "green", "value": 250000000000.0 } ] }, "unit": "bytes" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "min(jetstream_data_dir_free_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 8, "type": "stat", "title": "Subscribers", "description": "Connected /subscribe websocket clients.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 3, "x": 21, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_subscribe_subscribers{job=~\"$job\",instance=~\"$instance\"}) or vector(0)", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 15, "type": "row", "title": "Live firehose", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "panels": [] }, { "id": 10, "type": "timeseries", "title": "Firehose intake", "description": "Upstream subscribeRepos decode results. Unknown events are archival holes a jetstream upgrade would fix.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "events received/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "decode errors/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "unknown events/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eb6834" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_events_received_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "events received/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_decode_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "decode errors/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_unknown_events_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "unknown events/s", "refId": "C", "range": true, "instant": false } ] }, { "id": 11, "type": "timeseries", "title": "Relay sequence gaps", "description": "Forward gaps in the relay's seq stream: upstream data loss, not local decode trouble. Should be flat at zero.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "gaps/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "missed seqs/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eb6834" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_sequence_gaps_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "gaps/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_sequence_gap_missed_seqs_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "missed seqs/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 12, "type": "timeseries", "title": "Connection health", "description": "Websocket reconnects and op=-1 error frames from the relay. Error-frame series appear only after the first occurrence.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "reconnects/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eda100" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_reconnects_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "reconnects/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (code) (rate(jetstream_livestream_stream_error_frames_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "error frame {{code}}/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 13, "type": "timeseries", "title": "Replay & verify drops", "description": "Deliberate steady-state drops (relay seq replays, stale resyncs) vs verify-queue overflow, which is permanent archival loss.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "verify queue drops/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_replayed_account_events_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "replayed #account/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_replayed_identity_events_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "replayed #identity/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_stale_resyncs_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "stale resyncs/s", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_livestream_verify_queue_dropped_events_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "verify queue drops/s", "refId": "D", "range": true, "instant": false } ] }, { "id": 14, "type": "timeseries", "title": "Verifier failures", "description": "Verifier rejections by error class. A sustained 'other' rate means a new class needs categorizing.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind) (rate(jetstream_verifier_failures_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{kind}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 22, "type": "row", "title": "Ingest writer & segments", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "panels": [] }, { "id": 16, "type": "timeseries", "title": "Append throughput", "description": "Writer.Append successes and failures. Append errors indicate writer trouble and should be flat at zero.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 23 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "events appended/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "append errors/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_ingest_events_appended_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "events appended/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_ingest_append_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "append errors/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 17, "type": "timeseries", "title": "Dropped at validation gate", "description": "Upstream records dropped at the ingest validation gate. Spec-invalid input vs unrepresentable vs upstream omissions.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 23 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (source, reason) (rate(jetstream_ingest_dropped_events_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{source}}/{{reason}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 18, "type": "timeseries", "title": "Blocks & rotations", "description": "zstd block fsync rate and segment rotations (rotations shown as count per interval — they are rare).", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 23 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_ingest_blocks_flushed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "blocks flushed/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(jetstream_ingest_segments_rotated_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "segments rotated", "refId": "B", "range": true, "instant": false } ] }, { "id": 19, "type": "timeseries", "title": "Active segment size", "description": "Compressed bytes written since the active segment's header. Saw-tooth resets on rotation.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 31 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "active segment bytes" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_ingest_active_segment_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "active segment bytes", "refId": "A", "range": true, "instant": false } ] }, { "id": 20, "type": "timeseries", "title": "Segment seal latency", "description": "End-to-end Writer.Seal duration (flush + footer + header pwrite + fsyncs), successful seals only.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 31 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_segment_seal_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_segment_seal_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "B", "range": true, "instant": false } ] }, { "id": 21, "type": "timeseries", "title": "Readable log", "description": "In-memory readable log serving hot /subscribe reads. Pinned overrun means retention budget exceeded by undurable bytes.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 31 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "pinned overrun bytes" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_ingest_readable_log_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "resident bytes", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_ingest_readable_log_pinned_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "pinned bytes", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_ingest_readable_log_pinned_overrun_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "pinned overrun bytes", "refId": "C", "range": true, "instant": false } ] }, { "id": 31, "type": "row", "title": "Backfill & retry", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, "panels": [] }, { "id": 23, "type": "stat", "title": "Backfill progress", "description": "Repos completed this Run vs DIDs discovered via listRepos. Approximate: discovery keeps running during backfill.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 40 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#2a78d6", "value": null } ] }, "unit": "percentunit", "decimals": 1 }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_backfill_progress_completed{job=~\"$job\",instance=~\"$instance\"}) / clamp_min(sum(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"}), 1)", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 24, "type": "stat", "title": "Repos completed", "description": "Repos the engine reported complete in the current Run.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 40 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_backfill_progress_completed{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 25, "type": "stat", "title": "Repos failed (run)", "description": "DIDs that exhausted their retry budget within the Run.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 40 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "#eda100", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_backfill_failed_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 26, "type": "stat", "title": "Completion queue depth", "description": "Repo completions waiting for a writer durable metadata batch.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 40 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "#eda100", "value": 1000 }, { "color": "#e34948", "value": 10000 } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_backfill_completion_queue_depth{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 27, "type": "timeseries", "title": "Repo throughput", "description": "listRepos discovery vs getRepo completion vs terminal failure.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "completed/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "failed/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "discovered/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "discovered/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_completed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "completed/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_failed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "failed/s", "refId": "C", "range": true, "instant": false } ] }, { "id": 28, "type": "timeseries", "title": "HandleRepo latency", "description": "Per-repo download+append duration, successful repos only.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_backfill_handle_repo_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(jetstream_backfill_handle_repo_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_backfill_handle_repo_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "C", "range": true, "instant": false } ] }, { "id": 29, "type": "timeseries", "title": "Completion durability wait", "description": "Time a repo completion waited between being queued and becoming durable. A rising tail is a durability backlog.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_backfill_completion_queue_wait_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_backfill_completion_queue_wait_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "B", "range": true, "instant": false } ] }, { "id": 30, "type": "timeseries", "title": "Steady-state failed-repo retry", "description": "Retry loop that drains repos which failed initial backfill.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 52 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "succeeded/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "failed/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "skipped (host parked)/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eda100" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_failed_repo_retry_attempts_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "attempts/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_failed_repo_retry_succeeded_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "succeeded/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_failed_repo_retry_failed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "failed/s", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_failed_repo_retry_skipped_host_parked_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "skipped (host parked)/s", "refId": "D", "range": true, "instant": false } ] }, { "id": 41, "type": "row", "title": "Serving: /subscribe & archive API", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 60 }, "panels": [] }, { "id": 32, "type": "timeseries", "title": "Subscribers by compression", "description": "Connected /subscribe clients by negotiated wire compression.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 61 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (compression) (jetstream_subscribe_subscribers{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "{{compression}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 33, "type": "timeseries", "title": "Events sent to subscribers", "description": "JSON frames written to clients, plus events removed by per-subscriber filters before encoding.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 61 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (compression) (rate(jetstream_subscribe_events_sent_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{compression}}/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_events_filtered_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "filtered/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 34, "type": "timeseries", "title": "Egress bytes", "description": "Bytes written to /subscribe clients (post-compression for zstd) and archive getBlock downloads. sent/encoded ratio approximates the compression ratio.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 61 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (compression) (rate(jetstream_subscribe_bytes_sent_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "subscribe {{compression}}/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_bytes_encoded_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "pre-compression encoded/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_getblock_served_bytes_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "getBlock served/s", "refId": "C", "range": true, "instant": false } ] }, { "id": 35, "type": "timeseries", "title": "Subscriber drops & errors", "description": "Per-subscriber trouble. Encode errors and adversarial drops should be flat at zero.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 69 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "encode errors/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "adversarial drops/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(jetstream_subscribe_events_skipped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "skipped {{reason}}/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_events_oversize_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "oversize/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_encode_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "encode errors/s", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_adversarial_drops_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "adversarial drops/s", "refId": "D", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(jetstream_subscribe_options_update_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "options error {{reason}}/s", "refId": "E", "range": true, "instant": false } ] }, { "id": 36, "type": "timeseries", "title": "Cursor requests by mode", "description": "New /subscribe connections by cursor resolution outcome.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 69 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (mode) (rate(jetstream_subscribe_cursor_requests_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{mode}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 37, "type": "timeseries", "title": "Hot vs cold reads", "description": "ReadFrom calls served from the in-memory readable log vs falling through to the cold disk reader.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 69 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "hot (readable log)/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "cold (disk)/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eb6834" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_hot_reads_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "hot (readable log)/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_subscribe_cold_reads_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "cold (disk)/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 38, "type": "timeseries", "title": "HTTP requests", "description": "Public-listener HTTP request rate by handler and status.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 77 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (handler, code) (rate(jetstream_http_request_duration_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{handler}} {{code}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 39, "type": "timeseries", "title": "API latency (p99)", "description": "99th percentile latency of the public API surfaces.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 77 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le,handler) (rate(jetstream_http_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "http {{handler}}", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_getblock_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "getBlock", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_subscribe_cursor_resolve_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "cursor resolve", "refId": "C", "range": true, "instant": false } ] }, { "id": 40, "type": "timeseries", "title": "getBlock requests", "description": "Archive getBlock requests by outcome. Series appear after the first request.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 77 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "ok/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "error/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (result) (rate(jetstream_getblock_requests_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{result}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 51, "type": "row", "title": "Storage: compaction, tombstones & manifest", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 85 }, "panels": [] }, { "id": 42, "type": "timeseries", "title": "Compaction passes", "description": "Delete/update compaction passes by result, plus passes triggered early by the tombstone cap.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 86 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "ok/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "error/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "early (tombstone cap)/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eda100" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (result) (rate(jetstream_compaction_passes_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{result}}/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_compaction_passes_early_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "early (tombstone cap)/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 43, "type": "timeseries", "title": "Compaction pass duration", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 86 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(jetstream_compaction_pass_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(jetstream_compaction_pass_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "B", "range": true, "instant": false } ] }, { "id": 44, "type": "timeseries", "title": "Rows dropped & tombstones collected", "description": "Compaction work: rows physically dropped from rewritten segments and tombstones collected by kind.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 86 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(jetstream_compaction_rows_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "dropped {{reason}}/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (kind) (rate(jetstream_compaction_tombstones_collected_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "collected {{kind}}/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 45, "type": "timeseries", "title": "Tombstone set", "description": "Entries in the live in-memory tombstone set. Drops to zero after each compaction pass.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 94 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_compaction_tombstone_set_entries{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "entries", "refId": "A", "range": true, "instant": false } ] }, { "id": 46, "type": "timeseries", "title": "Tombstone set memory", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 94 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_compaction_tombstone_set_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "bytes", "refId": "A", "range": true, "instant": false } ] }, { "id": 47, "type": "timeseries", "title": "Compaction watermark lag", "description": "witnessed_at lag between the sealed segment tip and the compaction watermark (header granularity).", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 94 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_compaction_watermark_lag_seconds{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "lag", "refId": "A", "range": true, "instant": false } ] }, { "id": 48, "type": "timeseries", "title": "Compaction rewrite I/O", "description": "Segment-file bytes rewritten by compaction.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 94 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_compaction_bytes_rewritten_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "bytes/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 49, "type": "timeseries", "title": "Sealed segments", "description": "Sealed segments tracked by the manifest, and compaction rewrite activity per interval.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 102 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_manifest_segments_loaded{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "segments loaded", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(jetstream_compaction_segments_rewritten_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "rewritten (per interval)", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(jetstream_compaction_segments_skipped_clean_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "skipped clean (per interval)", "refId": "C", "range": true, "instant": false } ] }, { "id": 50, "type": "timeseries", "title": "Block index cache", "description": "Manifest BlockIndex lookups served from resident metadata. No data while there are no cold lookups.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 102 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_manifest_block_index_cache_hits_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(jetstream_manifest_block_index_cache_hits_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])) + sum(rate(jetstream_manifest_block_index_cache_misses_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])), 1e-9)", "legendFormat": "hit ratio", "refId": "A", "range": true, "instant": false } ] }, { "id": 55, "type": "row", "title": "Store & integrity", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 110 }, "panels": [] }, { "id": 52, "type": "timeseries", "title": "Store ops", "description": "Pebble metadata-store operations by op and outcome. Any error series is a crash-loud-worthy signal.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 111 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (op, status) (rate(jetstream_store_op_duration_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{op}} {{status}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 53, "type": "timeseries", "title": "Store op latency (p99)", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 111 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le,op) (rate(jetstream_store_op_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "{{op}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 54, "type": "timeseries", "title": "Integrity signals (expect zero)", "description": "Data-integrity error counters from every subsystem. All of these should be flat at zero; investigate any movement.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 111 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "ingest append errors/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_ingest_append_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "ingest append errors/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_on_fail_store_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "backfill OnFail store errors/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_backfill_completion_stage_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "completion stage errors/s", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_import_rows_corrupt_offset_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "import corrupt offsets/s", "refId": "D", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (op) (rate(jetstream_store_op_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",status=\"error\"}[$__rate_interval]))", "legendFormat": "store {{op}} errors/s", "refId": "E", "range": true, "instant": false } ] }, { "id": 61, "type": "row", "title": "Cutover & merge", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 119 }, "panels": [ { "id": 56, "type": "timeseries", "title": "Phase timeline", "description": "1=bootstrap, 2=merging, 3=steady state. Transitions are also marked as dashboard annotations.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 120 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "phase" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#4a3aa7" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_orchestrator_phase{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "phase", "refId": "A", "range": true, "instant": false } ] }, { "id": 57, "type": "timeseries", "title": "Merge rev filter", "description": "live_segments/ events surviving the rev filter into the steady-state segments vs already covered by backfill.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 120 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "kept/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "dropped (rev covered)/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#898781" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_orchestrator_merge_events_kept_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "kept/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_orchestrator_merge_events_dropped_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "dropped (rev covered)/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 58, "type": "timeseries", "title": "Merge progress", "description": "Run-once merge counters, shown as raw totals.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 120 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_orchestrator_merge_segments_consumed_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "segments consumed", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_orchestrator_merge_did_lookups_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "DID lookups", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_orchestrator_merge_repo_revs_updated_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "repo revs updated", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_orchestrator_merge_dids_discovered_post_bootstrap_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "DIDs discovered post-bootstrap", "refId": "D", "range": true, "instant": false } ] }, { "id": 59, "type": "bargauge", "title": "Cutover state durations", "description": "Total wall-clock seconds spent in each cutover state across this process lifetime.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 128 }, "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "#4a3aa7" }, "mappings": [], "unit": "s", "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#4a3aa7", "value": null } ] } }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "horizontal", "displayMode": "basic", "valueMode": "color", "showUnfilled": true, "sizing": "auto", "namePlacement": "auto", "minVizWidth": 8, "minVizHeight": 16, "maxVizHeight": 300, "legend": { "showLegend": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (state) (jetstream_orchestrator_state_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "{{state}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 60, "type": "timeseries", "title": "Phase transitions", "description": "Phase transitions per interval; normally each fires exactly once per data-dir lifetime.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 16, "x": 8, "y": 128 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (from, to) (increase(jetstream_orchestrator_phase_transitions_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "{{from}} → {{to}}", "refId": "A", "range": true, "instant": false } ] } ] }, { "id": 68, "type": "row", "title": "Timestamp import", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 136 }, "panels": [ { "id": 62, "type": "stat", "title": "Import phase", "description": "Current timestamp-import phase.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 4, "x": 0, "y": 137 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "0": { "text": "idle", "color": "text", "index": 0 }, "1": { "text": "parse + bucket", "color": "#2a78d6", "index": 1 }, "2": { "text": "apply", "color": "#eb6834", "index": 2 } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short" }, "overrides": [] }, "options": { "reduceOptions": { "values": false, "calcs": [ "lastNotNull" ], "fields": "" }, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "wideLayout": true, "percentChangeColorMode": "standard", "showPercentChange": false }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(jetstream_import_phase{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "range": true, "instant": false } ] }, { "id": 63, "type": "timeseries", "title": "Import rows", "description": "Timestamp-import pipeline throughput across phases.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 10, "x": 4, "y": 137 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_import_rows_parsed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "parsed/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (scope) (rate(jetstream_import_rows_matched_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "matched {{scope}}/s", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_import_rows_mutated_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "mutated/s", "refId": "C", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_import_dids_matched_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "DIDs matched/s", "refId": "D", "range": true, "instant": false } ] }, { "id": 64, "type": "timeseries", "title": "Import rows rejected", "description": "CSV rows rejected during parse, by reason.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 10, "x": 14, "y": 137 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(jetstream_import_rows_rejected_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{reason}}/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 65, "type": "timeseries", "title": "Import rewrite activity", "description": "Phase C segment rewrites per interval.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 10, "x": 0, "y": 145 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(jetstream_import_segments_examined_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "segments examined", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(jetstream_import_segments_patched_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "segments patched", "refId": "B", "range": true, "instant": false } ] }, { "id": 66, "type": "timeseries", "title": "Import rewrite I/O", "description": "On-disk segment bytes rewritten by import.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 10, "x": 10, "y": 145 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(jetstream_import_bytes_rewritten_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "bytes/s", "refId": "A", "range": true, "instant": false } ] }, { "id": 67, "type": "timeseries", "title": "Import jobs", "description": "Terminal import jobs per interval by result.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 4, "x": 20, "y": 145 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "ok" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#008300" } } ] }, { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (result) (increase(jetstream_import_jobs_total{job=~\"$job\",instance=~\"$instance\"}[$__interval]))", "legendFormat": "{{result}}", "refId": "A", "range": true, "instant": false } ] } ] }, { "id": 75, "type": "row", "title": "Process runtime", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 153 }, "panels": [] }, { "id": 69, "type": "timeseries", "title": "CPU", "description": "Process CPU usage in cores vs the scheduler limit.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 154 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "cores used" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "GOMAXPROCS" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#898781" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "cores used", "refId": "A", "range": true, "instant": false } ] }, { "id": 70, "type": "timeseries", "title": "Memory", "description": "Resident set vs Go heap. GOMEMLIMIT is hidden when unset.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 154 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "RSS" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "heap in use" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#1baf7a" } } ] }, { "matcher": { "id": "byName", "options": "GC goal" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#898781" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "RSS", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(process_virtual_memory_bytes{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "virtual", "refId": "B", "range": true, "instant": false } ] }, { "id": 71, "type": "timeseries", "title": "Threads", "description": "A monotonic goroutine climb is a leak.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 154 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "goroutines" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "OS threads" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#898781" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(stream_process_threads{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "OS threads", "refId": "A", "range": true, "instant": false } ] }, { "id": 72, "type": "timeseries", "title": "File descriptors", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 162 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "open" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "max" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#898781" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(process_open_fds{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "open", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(process_max_fds{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "max", "refId": "B", "range": true, "instant": false } ] }, { "id": 73, "type": "timeseries", "title": "Page faults", "description": "Stop-the-world pause durations from the Go GC summary.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 162 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(process_minor_page_faults_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "minor/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(process_major_page_faults_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "major/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 74, "type": "timeseries", "title": "Context switches", "description": "Process-level network throughput (procfs; Linux only).", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 162 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "receive/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "transmit/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#1baf7a" } } ] } ] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(process_voluntary_context_switches_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "voluntary/s", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(process_involuntary_context_switches_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "involuntary/s", "refId": "B", "range": true, "instant": false } ] }, { "id": 80, "type": "row", "title": "Backfill progress over time", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 170 }, "panels": [], "description": "The final network denominator is unknown until listRepos discovery finishes." }, { "id": 81, "type": "timeseries", "title": "Backfill repositories over time", "description": "Absolute repository counts across the current backfill run. listRepos is fetched in pages of 1,000; Stream follows upstream Jetstream V2 by accumulating about 100,000 entries before shuffling and dispatching a batch. Durably committed advances only after that batch checkpoint. These lines are not a percentage of the full network.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 171 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "events received/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#2a78d6" } } ] }, { "matcher": { "id": "byName", "options": "decode errors/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#e34948" } } ] }, { "matcher": { "id": "byName", "options": "unknown events/s" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "#eb6834" } } ] } ] }, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "discovered", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(jetstream_backfill_completion_queued_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "processed", "refId": "B", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(stream_backfill_repos_durable{status=\"complete\",job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "durably committed", "refId": "C", "range": true, "instant": false } ] }, { "id": 76, "type": "row", "title": "Stream pipeline diagnostics", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 175 }, "panels": [], "description": "Live during bootstrap and steady state; gaps and terminal failures are never inferred from archive throughput." }, { "id": 77, "type": "timeseries", "title": "Pipeline tickets", "description": "Cumulative scheduler boundaries. Emitted is the historical name for every settled ticket, including explicit drops and verifier rejections.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 176 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(stream_pipeline_ticket{job=~\"$job\",instance=~\"$instance\"}) by (stage)", "legendFormat": "{{stage}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 78, "type": "timeseries", "title": "Pipeline stage gaps", "description": "Current unsettled work split between the unclaimed verifier queue and claimed work awaiting settlement. Their sum is submitted-to-emitted.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 176 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(stream_pipeline_stage_gap{job=~\"$job\",instance=~\"$instance\"}) by (boundary)", "legendFormat": "{{boundary}}", "refId": "A", "range": true, "instant": false } ] }, { "id": 79, "type": "timeseries", "title": "Pipeline terminal failures", "description": "Any non-zero value is deployment-significant. The gauge marks the current pipeline fatal; the counters retain its first failing stage.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 176 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 6, "gradientMode": "none", "showPoints": "never", "pointSize": 4, "spanNulls": true, "insertNulls": false, "barAlignment": 0, "barWidthFactor": 0.6, "axisPlacement": "auto", "axisLabel": "", "axisColorMode": "text", "axisBorderShow": false, "axisCenteredZero": false, "scaleDistribution": { "type": "linear" }, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "stacking": { "mode": "none", "group": "A" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "short", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(stream_pipeline_fatal{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "fatal", "refId": "A", "range": true, "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(stream_pipeline_failures_total{job=~\"$job\",instance=~\"$instance\"}) by (stage)", "legendFormat": "{{stage}} failures", "refId": "B", "range": true, "instant": false } ] } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "stream", "jetstream", "atproto" ], "templating": { "list": [ { "name": "datasource", "label": "Data source", "type": "datasource", "query": "prometheus", "current": {}, "hide": 0, "includeAll": false, "multi": false, "options": [], "refresh": 1, "regex": "", "skipUrlSync": false }, { "name": "job", "label": "Job", "type": "query", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "query": { "query": "label_values(jetstream_build_info, job)", "refId": "job" }, "definition": "label_values(jetstream_build_info, job)", "current": {}, "hide": 0, "includeAll": true, "allValue": ".*", "multi": true, "options": [], "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1 }, { "name": "instance", "label": "Instance", "type": "query", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "query": { "query": "label_values(jetstream_build_info{job=~\"$job\"}, instance)", "refId": "instance" }, "definition": "label_values(jetstream_build_info{job=~\"$job\"}, instance)", "current": {}, "hide": 0, "includeAll": true, "allValue": ".*", "multi": true, "options": [], "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1 } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Stream / Jetstream V2", "uid": "stream-jetstream-v2", "version": 1, "weekStart": "" }