diff --git a/deploy/grafana/stream.json b/deploy/grafana/stream.json index 45e76cd..70777db 100644 --- a/deploy/grafana/stream.json +++ b/deploy/grafana/stream.json @@ -42,7 +42,7 @@ } ] }, - "description": "Stream operational dashboard adapted from the checksum-pinned Jetstream V2 dashboard. Protocol and lifecycle rows are unchanged; the runtime row uses Stream process metrics, and final Stream-only rows expose current-page backfill work and scheduler health.", + "description": "Stream operational dashboard adapted from the checksum-pinned Jetstream V2 dashboard. Protocol and lifecycle rows are unchanged; the runtime row uses Stream process metrics, and final Stream-only rows expose current-batch backfill work and scheduler health.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -7214,7 +7214,7 @@ { "id": 80, "type": "row", - "title": "Backfill work in progress", + "title": "Backfill batch progress", "collapsed": false, "gridPos": { "h": 1, @@ -7227,15 +7227,15 @@ }, { "id": 81, - "type": "bargauge", - "title": "Current discovery page", - "description": "Processed repositories move continuously; durably committed repositories advance after the page checkpoint. This is progress through the current listRepos page, not a percentage of the full network.", + "type": "timeseries", + "title": "Current dispatch batch over time", + "description": "Absolute repository counts in the current process. listRepos is fetched in pages of 1,000; Stream follows upstream Jetstream V2 by accumulating about 100,000 entries before shuffling and dispatching a batch. Durably committed advances only after that batch checkpoint. These lines are not a percentage of the full network.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 4, + "h": 8, "w": 24, "x": 0, "y": 171 @@ -7243,44 +7243,114 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 6, + "gradientMode": "none", + "showPoints": "never", + "pointSize": 4, + "spanNulls": true, + "insertNulls": false, + "barAlignment": 0, + "barWidthFactor": 0.6, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "axisBorderShow": false, + "axisCenteredZero": false, + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "#2a78d6", + "color": "green", "value": null } ] }, - "unit": "percentunit", - "decimals": 1, - "min": 0, - "max": 1 + "unit": "short", + "min": 0 }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "events received/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#2a78d6" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "decode errors/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#e34948" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown events/s" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#eb6834" + } + } + ] + } + ] }, "options": { - "reduceOptions": { - "values": false, + "legend": { "calcs": [ "lastNotNull" ], - "fields": "" + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, - "orientation": "horizontal", - "displayMode": "basic", - "valueMode": "color", - "showUnfilled": true, - "sizing": "auto", - "namePlacement": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "maxVizHeight": 300, - "legend": { - "showLegend": false + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ @@ -7289,22 +7359,33 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(jetstream_backfill_completion_queued_total{job=~\"$job\",instance=~\"$instance\"}) / clamp_min(sum(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"}), 1)", + "expr": "sum(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"})", + "legendFormat": "discovered", + "refId": "A", + "range": true, + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(jetstream_backfill_completion_queued_total{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "processed", "refId": "B", - "range": false, - "instant": true + "range": true, + "instant": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(jetstream_backfill_progress_completed{job=~\"$job\",instance=~\"$instance\"}) / clamp_min(sum(jetstream_backfill_discovered_total{job=~\"$job\",instance=~\"$instance\"}), 1)", + "expr": "sum(jetstream_backfill_progress_completed{job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "durably committed", - "refId": "A", - "range": false, - "instant": true + "refId": "C", + "range": true, + "instant": false } ] }, diff --git a/docs/grafana-dashboard.md b/docs/grafana-dashboard.md index 7138a90..01b74a5 100644 --- a/docs/grafana-dashboard.md +++ b/docs/grafana-dashboard.md @@ -11,7 +11,7 @@ The exact reviewed bytes are vendored at `deploy/grafana/jetstream-upstream.json`; deployment does not depend on the network or a moving branch. Stream provisions `deploy/grafana/stream.json` (SHA-256 -`7f73d578d941584e70f6f90745a9417596e973538cdb4c80e0473cead92e7f66`). It is +`630cc34ad8cc22944871c00759b905c8b4eef25e7d4a792b4822dd983d3f46e8`). It is deterministically generated by `tools/adapt_dashboard.py` from the pinned source. The generator refuses a changed source checksum, and the tests prove every upstream non-runtime panel remains structurally identical. Stream adds @@ -193,7 +193,14 @@ The following dashboard families already have non-placeholder producers: - `jetstream_compaction_watermark_seq` - `jetstream_compaction_watermark_lag_seconds` -The backfill families are measured at Stream's actual page checkpoint: worker +The Stream-only backfill chart plots absolute repository counts over time: +`discovered`, `processed`, and `durably committed`. It deliberately does not +present a whole-network percentage while `listRepos` is still discovering the +denominator. Upstream requests are pages of 1,000 entries; the production +default accumulates roughly 100,000 entries into a shuffled dispatch/checkpoint +batch before advancing durable progress. + +The backfill families are measured at Stream's actual dispatch-batch checkpoint: worker results enter the completion queue after download/handling, archive bytes are flushed first, and only the succeeding RocksDB transition batch removes them from queue depth and reports completion. This is the semantic counterpart of diff --git a/tests/test_dashboard_contract.py b/tests/test_dashboard_contract.py index b947c6b..fd1a05a 100644 --- a/tests/test_dashboard_contract.py +++ b/tests/test_dashboard_contract.py @@ -58,29 +58,36 @@ class DashboardContractTest(unittest.TestCase): "stream_pipeline_failures_total", }.issubset(required)) - def test_stream_progress_bar_reuses_upstream_ratio_and_names_its_caveat(self): - upstream = json.loads(check_dashboard.UPSTREAM_DASHBOARD.read_bytes()) + def test_stream_progress_chart_names_upstream_batch_semantics(self): stream = json.loads(check_dashboard.DEFAULT_DASHBOARD.read_bytes()) - upstream_panels = {panel["id"]: panel for panel in upstream["panels"]} stream_panels = {panel["id"]: panel for panel in stream["panels"]} progress = stream_panels[81] - self.assertEqual("bargauge", progress["type"]) - self.assertEqual("Current discovery page", progress["title"]) + self.assertEqual("timeseries", progress["type"]) + self.assertEqual("Current dispatch batch over time", progress["title"]) + self.assertIn("pages of 1,000", progress["description"]) + self.assertIn("100,000", progress["description"]) self.assertIn("not a percentage of the full network", progress["description"]) self.assertEqual( - upstream_panels[23]["targets"][0]["expr"], - progress["targets"][1]["expr"], + [ + "jetstream_backfill_discovered_total", + "jetstream_backfill_completion_queued_total", + "jetstream_backfill_progress_completed", + ], + [ + check_dashboard.METRIC_RE.findall(target["expr"])[0] + for target in progress["targets"] + ], ) - self.assertIn( - "jetstream_backfill_completion_queued_total", - progress["targets"][0]["expr"], + self.assertEqual( + ["discovered", "processed", "durably committed"], + [target["legendFormat"] for target in progress["targets"]], ) - self.assertEqual("processed", progress["targets"][0]["legendFormat"]) - self.assertEqual("durably committed", progress["targets"][1]["legendFormat"]) - self.assertTrue(all(target["instant"] for target in progress["targets"])) + self.assertTrue(all(target["range"] for target in progress["targets"])) + self.assertTrue(all(not target["instant"] for target in progress["targets"])) self.assertEqual(0, progress["fieldConfig"]["defaults"]["min"]) - self.assertEqual(1, progress["fieldConfig"]["defaults"]["max"]) + self.assertNotIn("max", progress["fieldConfig"]["defaults"]) + self.assertEqual("short", progress["fieldConfig"]["defaults"]["unit"]) def test_histogram_type_expands_queryable_children(self): present = check_dashboard.scrape_families( diff --git a/tools/adapt_dashboard.py b/tools/adapt_dashboard.py index ad15ba0..8fc4175 100644 --- a/tools/adapt_dashboard.py +++ b/tools/adapt_dashboard.py @@ -40,51 +40,59 @@ def progress_panel(template: dict) -> dict: panel = copy.deepcopy(template) panel.update( id=81, - type="bargauge", - title="Current discovery page", + type="timeseries", + title="Current dispatch batch over time", description=( - "Processed repositories move continuously; durably committed repositories advance " - "after the page checkpoint. This is progress through the current listRepos page, " - "not a percentage of the full network." + "Absolute repository counts in the current process. listRepos is fetched in pages " + "of 1,000; Stream follows upstream Jetstream V2 by accumulating about 100,000 " + "entries before shuffling and dispatching a batch. Durably committed advances only " + "after that batch checkpoint. These lines are not a percentage of the full network." ), - gridPos={"h": 4, "w": 24, "x": 0, "y": 171}, + gridPos={"h": 8, "w": 24, "x": 0, "y": 171}, ) + panel["fieldConfig"]["defaults"]["unit"] = "short" panel["fieldConfig"]["defaults"]["min"] = 0 - panel["fieldConfig"]["defaults"]["max"] = 1 panel["options"] = { - "reduceOptions": { - "values": False, + "legend": { "calcs": ["lastNotNull"], - "fields": "", + "displayMode": "table", + "placement": "bottom", + "showLegend": True, }, - "orientation": "horizontal", - "displayMode": "basic", - "valueMode": "color", - "showUnfilled": True, - "sizing": "auto", - "namePlacement": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "maxVizHeight": 300, - "legend": {"showLegend": False}, + "tooltip": {"mode": "multi", "sort": "desc"}, } - durable = panel["targets"][0] - durable.update( - legendFormat="durably committed", - range=False, - instant=True, + discovered = panel["targets"][0] + discovered.update( + expr=( + 'sum(jetstream_backfill_discovered_total' + '{job=~"$job",instance=~"$instance"})' + ), + legendFormat="discovered", + refId="A", + range=True, + instant=False, ) - processed = copy.deepcopy(durable) + processed = copy.deepcopy(discovered) processed.update( expr=( - 'sum(jetstream_backfill_completion_queued_total{job=~"$job",instance=~"$instance"}) ' - '/ clamp_min(sum(jetstream_backfill_discovered_total' - '{job=~"$job",instance=~"$instance"}), 1)' + 'sum(jetstream_backfill_completion_queued_total' + '{job=~"$job",instance=~"$instance"})' ), legendFormat="processed", refId="B", ) - panel["targets"] = [processed, durable] + durable = copy.deepcopy(discovered) + durable.update( + expr=( + 'sum(jetstream_backfill_progress_completed' + '{job=~"$job",instance=~"$instance"})' + ), + legendFormat="durably committed", + refId="C", + range=True, + instant=False, + ) + panel["targets"] = [discovered, processed, durable] return panel @@ -154,17 +162,18 @@ def adapt(raw: bytes) -> bytes: ] panels[74]["fieldConfig"]["defaults"]["unit"] = "ops" - # Keep upstream's approximate progress ratio intact, but repeat it as a - # phone-legible bar whose copy makes the expanding discovery denominator - # explicit. This is presentation only: the PromQL remains upstream's. + # Show absolute repository counts over time. The upstream completion ratio + # remains in its original panel, but it is approximate while listRepos is + # still expanding the denominator. progress_row = copy.deepcopy(panels[75]) progress_row.update( id=80, - title="Backfill work in progress", + title="Backfill batch progress", description="The final network denominator is unknown until listRepos discovery finishes.", gridPos={"h": 1, "w": 24, "x": 0, "y": 170}, ) - progress = progress_panel(panels[23]) + progress_template = next(panel for panel in dashboard["panels"] if panel.get("type") == "timeseries") + progress = progress_panel(progress_template) # Upstream has no scheduler between its websocket reader and archive # writer. Stream does, so retain every upstream protocol/lifecycle panel @@ -247,7 +256,7 @@ def adapt(raw: bytes) -> bytes: dashboard["description"] = ( "Stream operational dashboard adapted from the checksum-pinned Jetstream V2 dashboard. " "Protocol and lifecycle rows are unchanged; the runtime row uses Stream process metrics, " - "and final Stream-only rows expose current-page backfill work and scheduler health." + "and final Stream-only rows expose current-batch backfill work and scheduler health." ) return (json.dumps(dashboard, indent=2, ensure_ascii=False) + "\n").encode() diff --git a/tools/check_dashboard.py b/tools/check_dashboard.py index 9107f65..a2feb6d 100755 --- a/tools/check_dashboard.py +++ b/tools/check_dashboard.py @@ -17,7 +17,7 @@ UPSTREAM_DASHBOARD = ROOT / "deploy/grafana/jetstream-upstream.json" DEFAULT_DASHBOARD = ROOT / "deploy/grafana/stream.json" EXPECTED_SHA256 = { UPSTREAM_DASHBOARD.resolve(): "ef6033c182d8c3f9d88af7d91e61126724efbe72e6b940984c9de12e9530cd91", - DEFAULT_DASHBOARD.resolve(): "7f73d578d941584e70f6f90745a9417596e973538cdb4c80e0473cead92e7f66", + DEFAULT_DASHBOARD.resolve(): "630cc34ad8cc22944871c00759b905c8b4eef25e7d4a792b4822dd983d3f46e8", } METRIC_RE = re.compile(r"\b(?:jetstream|go|process|stream)_[a-zA-Z_:][a-zA-Z0-9_:]*") TYPE_RE = re.compile(r"^# TYPE ([a-zA-Z_:][a-zA-Z0-9_:]*) (\w+)$")