Something went wrong. Try again.
Tangled infrastructure definitions in Nix
Something went wrong. Try again.
25 kB · 975 lines
JSON
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976{ "uid": "gitmirror", "title": "gitmirror", "description": "gitmirror XRPC traffic, latency, blocking-pool saturation, and per-repository detail.", "tags": [ "gitmirror", "git" ], "schemaVersion": 38, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "templating": { "list": [ { "name": "slow_ms", "label": "slow threshold (ms)", "description": "Requests slower than this appear in the per-repo slow requests panel.", "type": "textbox", "query": "1000", "current": { "selected": true, "text": "1000", "value": "1000" }, "options": [ { "selected": true, "text": "1000", "value": "1000" } ], "skipUrlSync": false }, { "name": "search", "label": "log search", "description": "Case-insensitive text filter for the per-repo log panels. Paste a repo DID to follow one repository.", "type": "textbox", "query": "", "current": { "selected": true, "text": "", "value": "" }, "options": [ { "selected": true, "text": "", "value": "" } ], "skipUrlSync": false } ] }, "panels": [ { "type": "row", "title": "health", "collapsed": false, "panels": [], "id": 2, "gridPos": { "x": 0, "y": 0, "w": 24, "h": 1 } }, { "type": "stat", "title": "metrics target reachable", "targets": [ { "refId": "A", "expr": "up{job=\"gitmirror\"}" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] }, "unit": "percentunit", "decimals": 0 }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "textMode": "value", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "description": "Whether Prometheus can currently reach gitmirror's metrics endpoint. Zero means the process is down, the port moved, or the tailscale route broke \u2014 every other panel is stale while this is zero.", "id": 3, "gridPos": { "x": 0, "y": 1, "w": 4, "h": 4 } }, { "type": "stat", "title": "requests / s", "targets": [ { "refId": "A", "expr": "sum(rate(gitmirror_xrpc_requests_total{job=\"gitmirror\"}[5m]))" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "reqps", "decimals": 2 }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "textMode": "value", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "description": "XRPC requests per second across every method, averaged over the last five minutes.", "id": 4, "gridPos": { "x": 4, "y": 1, "w": 4, "h": 4 } }, { "type": "stat", "title": "server error rate", "targets": [ { "refId": "A", "expr": "(sum(rate(gitmirror_xrpc_requests_total{job=\"gitmirror\",status=~\"5..\"}[5m])) or vector(0)) / clamp_min(sum(rate(gitmirror_xrpc_requests_total{job=\"gitmirror\"}[5m])), 0.0001)" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.01 }, { "color": "red", "value": 0.05 } ] }, "unit": "percentunit", "decimals": 1 }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "textMode": "value", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "description": "Share of requests gitmirror itself failed, over the last five minutes. Requests rejected because the caller asked for something missing or was not authorised are 4xx and are deliberately not counted here.", "id": 5, "gridPos": { "x": 8, "y": 1, "w": 4, "h": 4 } }, { "type": "stat", "title": "requests in flight", "targets": [ { "refId": "A", "expr": "sum(gitmirror_xrpc_in_flight{job=\"gitmirror\"})" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "textMode": "value", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "description": "Requests being handled at this instant. A number that stays high while the request rate is flat means requests are taking longer to finish than they are arriving.", "id": 6, "gridPos": { "x": 12, "y": 1, "w": 4, "h": 4 } }, { "type": "stat", "title": "blocking threads busy", "targets": [ { "refId": "A", "expr": "sum(gitmirror_blocking_running{job=\"gitmirror\"})" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "textMode": "value", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "description": "Threads in the blocking pool currently occupied by git work or by a push to a knot. The pool holds 512 threads; this is how many of them are in use.", "id": 7, "gridPos": { "x": 16, "y": 1, "w": 4, "h": 4 } }, { "type": "stat", "title": "tasks waiting for a thread", "targets": [ { "refId": "A", "expr": "sum(gitmirror_blocking_queued{job=\"gitmirror\"})" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 16 } ] }, "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "textMode": "value", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "description": "Git tasks that have been handed to the blocking pool but have not started yet, because every thread is busy. Anything above zero for a sustained period means the pool is the bottleneck, not git.", "id": 8, "gridPos": { "x": 20, "y": 1, "w": 4, "h": 4 } }, { "type": "row", "title": "xrpc traffic", "collapsed": false, "panels": [], "id": 9, "gridPos": { "x": 0, "y": 5, "w": 24, "h": 1 } }, { "type": "timeseries", "title": "requests / s by method", "targets": [ { "refId": "A", "expr": "sum by (path) (rate(gitmirror_xrpc_requests_total{job=\"gitmirror\"}[5m]))", "legendFormat": "{{path}}" } ], "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 1, "showPoints": "never" }, "min": 0, "unit": "reqps", "decimals": 2 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "id": 10, "gridPos": { "x": 0, "y": 6, "w": 8, "h": 7 }, "description": "Request rate for each XRPC method. Requests that matched no route are grouped as \"unknown\"." }, { "type": "timeseries", "title": "requests / s by status", "targets": [ { "refId": "A", "expr": "sum by (status) (rate(gitmirror_xrpc_requests_total{job=\"gitmirror\"}[5m]))", "legendFormat": "{{status}}" } ], "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 1, "showPoints": "never", "stacking": { "mode": "normal", "group": "A" } }, "min": 0, "unit": "reqps", "decimals": 2 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "id": 11, "gridPos": { "x": 8, "y": 6, "w": 8, "h": 7 }, "description": "Request rate split by HTTP status. 2xx worked, 4xx means the caller asked for something gitmirror could not serve, 5xx means gitmirror failed." }, { "type": "timeseries", "title": "failures / s by cause", "targets": [ { "refId": "A", "expr": "sum by (error) (rate(gitmirror_xrpc_requests_total{job=\"gitmirror\",error!=\"none\"}[5m]))", "legendFormat": "{{error}}" } ], "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 1, "showPoints": "never" }, "min": 0, "unit": "reqps", "decimals": 2 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "id": 12, "gridPos": { "x": 16, "y": 6, "w": 8, "h": 7 }, "description": "Failures named by cause rather than by status code, so a missing repository is told apart from a failed revision comparison. Requests rejected before reaching a handler \u2014 a malformed query, a missing token \u2014 have no cause name and are absent here; watch 4xx on the panel to the left for those." }, { "type": "row", "title": "latency", "collapsed": false, "panels": [], "id": 13, "gridPos": { "x": 0, "y": 13, "w": 24, "h": 1 } }, { "type": "timeseries", "title": "median request duration by method", "targets": [ { "refId": "A", "expr": "histogram_quantile(0.5, sum by (le, path) (rate(gitmirror_xrpc_duration_seconds_bucket{job=\"gitmirror\"}[5m])))", "legendFormat": "{{path}}" } ], "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 1, "showPoints": "never" }, "min": 0, "unit": "s", "decimals": 2 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "id": 14, "gridPos": { "x": 0, "y": 14, "w": 8, "h": 7 }, "description": "Typical time to serve each method, measured over a rolling recent window. Half of requests finished faster than this." }, { "type": "timeseries", "title": "p99 request duration by method", "targets": [ { "refId": "A", "expr": "histogram_quantile(0.99, sum by (le, path) (rate(gitmirror_xrpc_duration_seconds_bucket{job=\"gitmirror\"}[5m])))", "legendFormat": "{{path}}" } ], "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 1, "showPoints": "never" }, "min": 0, "unit": "s", "decimals": 2 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "id": 15, "gridPos": { "x": 8, "y": 14, "w": 8, "h": 7 }, "description": "The slow tail: one request in a hundred took longer than this. Compare against the median beside it \u2014 a large gap means a few requests are far slower than the rest, usually large repositories or deep histories." }, { "type": "table", "title": "slowest methods in range", "targets": [ { "refId": "A", "expr": "topk(5, 1000 * (sum by (path) (increase(gitmirror_xrpc_duration_seconds_sum{job=\"gitmirror\"}[$__range])) / clamp_min(sum by (path) (increase(gitmirror_xrpc_duration_seconds_count{job=\"gitmirror\"}[$__range])), 1))) > 0", "legendFormat": "{{path}}", "instant": true, "range": false, "format": "table" } ], "fieldConfig": { "defaults": { "unit": "ms", "decimals": 1 }, "overrides": [ { "matcher": { "id": "byName", "options": "path" }, "properties": [ { "id": "custom.width", "value": 360 }, { "id": "custom.wrapText", "value": true } ] }, { "matcher": { "id": "byName", "options": "average response" }, "properties": [ { "id": "custom.width", "value": 110 } ] } ] }, "options": { "cellHeight": "auto", "showHeader": true, "showTypeIcons": false, "sortBy": [ { "displayName": "average response", "desc": true } ], "maxRowHeight": 64 }, "id": 16, "gridPos": { "x": 16, "y": 14, "w": 8, "h": 7 }, "description": "The five methods with the highest average response time during the selected period. Use this to find which method to look at; use the per-repo log panels below to find which repository caused it.", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "quantile": true, "method": true }, "indexByName": { "path": 0, "Value": 1 }, "renameByName": { "Value": "average response" } } } ] }, { "type": "row", "title": "concurrency and the blocking pool", "collapsed": false, "panels": [], "id": 17, "gridPos": { "x": 0, "y": 21, "w": 24, "h": 1 } }, { "type": "timeseries", "title": "requests in flight by method", "targets": [ { "refId": "A", "expr": "gitmirror_xrpc_in_flight{job=\"gitmirror\"}", "legendFormat": "{{path}}" } ], "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 1, "showPoints": "never", "stacking": { "mode": "normal", "group": "A" } }, "min": 0, "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "id": 18, "gridPos": { "x": 0, "y": 22, "w": 8, "h": 7 }, "description": "How many requests of each method are being handled right now. This is what one method monopolising the service looks like." }, { "type": "timeseries", "title": "blocking pool occupancy", "targets": [ { "refId": "A", "expr": "gitmirror_blocking_running{job=\"gitmirror\"}", "legendFormat": "{{kind}} running" }, { "refId": "B", "expr": "gitmirror_blocking_queued{job=\"gitmirror\"}", "legendFormat": "{{kind}} queued" } ], "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 1, "showPoints": "never" }, "min": 0, "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "id": 19, "gridPos": { "x": 8, "y": 22, "w": 8, "h": 7 }, "description": "Threads in the blocking pool, split by what is holding them. \"git\" is local work on a repository; \"transport\" is a thread parked on a knot while a merge or branch deletion is pushed, which occupies the pool just as much. \"queued\" rising above zero means tasks are waiting for a free thread." }, { "type": "timeseries", "title": "wait for a free thread", "targets": [ { "refId": "A", "expr": "histogram_quantile(0.5, sum by (le, kind) (rate(gitmirror_blocking_queue_wait_seconds_bucket{job=\"gitmirror\"}[5m])))", "legendFormat": "{{kind}} median" }, { "refId": "B", "expr": "histogram_quantile(0.99, sum by (le, kind) (rate(gitmirror_blocking_queue_wait_seconds_bucket{job=\"gitmirror\"}[5m])))", "legendFormat": "{{kind}} p99" } ], "fieldConfig": { "defaults": { "custom": { "fillOpacity": 8, "lineWidth": 1, "showPoints": "never" }, "min": 0, "unit": "s", "decimals": 2 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "lastNotNull", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "id": 20, "gridPos": { "x": 16, "y": 22, "w": 8, "h": 7 }, "description": "Time a task sat waiting before a thread picked it up. Normally microseconds. When this climbs at the same time as request duration, the service is not slow at git \u2014 it is out of threads." }, { "type": "row", "title": "per-repo detail", "collapsed": false, "panels": [], "id": 21, "gridPos": { "x": 0, "y": 29, "w": 24, "h": 1 } }, { "id": 22, "type": "logs", "title": "slowest requests by repo", "description": "Individual requests that took longer than the slow threshold above, newest first. Repositories are identified by DID, which is why this is a log panel and not a graph \u2014 there are too many repositories to chart one line each. Open a row for its full structured fields.", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "x": 0, "y": 30, "w": 24, "h": 8 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "editorMode": "code", "expr": "{service_name=\"gitmirror\"} |= \"xrpc served\" | duration_ms > ${slow_ms} | line_format \"{{.duration_ms}}ms {{.path}} repo={{.repo}} {{.base_repo}} status={{.status}}\" |~ \"(?i)${search:regex}\"", "queryType": "range", "refId": "A" } ], "options": { "showLabels": false, "wrapLogMessage": true, "enableLogDetails": true, "showCommonLabels": false, "showTime": true, "sortOrder": "Descending", "showControls": true } }, { "id": 23, "type": "logs", "title": "failed requests by repo", "description": "Every request that failed with a named cause, and the repository it was for. A single repository filling this panel usually means that repository is broken on disk, not that gitmirror is.", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "x": 0, "y": 38, "w": 24, "h": 8 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "editorMode": "code", "expr": "{service_name=\"gitmirror\"} |= \"xrpc served\" | error != \"none\" | line_format \"{{.error}} {{.path}} repo={{.repo}} {{.base_repo}} status={{.status}} {{.duration_ms}}ms\" |~ \"(?i)${search:regex}\"", "queryType": "range", "refId": "A" } ], "options": { "showLabels": false, "wrapLogMessage": true, "enableLogDetails": true, "showCommonLabels": false, "showTime": true, "sortOrder": "Descending", "showControls": true } } ], "version": 1}