From d903015d4e5fc314659adfb70b7f3b637b896f60 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Sat, 3 Oct 2026 19:57:21 +0300
Subject: [PATCH 01/25] web/organizations: show tranquil's reason for
 InvalidDelegation instead of calling it the cap

Signed-off-by: dawn <dawn@tangled.org>
---
 web/src/lib/api/delegation.test.ts | 14 +++++++++++++-
 web/src/lib/api/delegation.ts      |  3 ---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/web/src/lib/api/delegation.test.ts b/web/src/lib/api/delegation.test.ts
index 29614e0d5..9e461a9c9 100644
--- a/web/src/lib/api/delegation.test.ts
+++ b/web/src/lib/api/delegation.test.ts
@@ -54,7 +54,6 @@ describe("delegate provisioning", () => {
 	});
 	it.each([
 		["VerifiedEmailRequired", /Verify an email/],
-		["InvalidDelegation", /reached the limit/],
 		["UpstreamUnavailable", /temporarily unavailable/]
 	])("explains %s", async (error, message) => {
 		const { agent } = makeAgent();
@@ -81,6 +80,19 @@ describe("delegate provisioning", () => {
 			})
 		).rejects.toThrow(/Handle already taken/);
 	});
+	it("shows Tranquil's reason for InvalidDelegation, which is not only the delegate cap", async () => {
+		const { agent } = makeAgent();
+		const message = "Invite-only registration requires an existing account on this PDS";
+		await expect(
+			provisionDelegate(agent, "org.example", {
+				serviceUrl: "https://gate.example",
+				serviceDid: "did:web:tranquil.example",
+				fetch: vi.fn(async () =>
+					Response.json({ error: "InvalidDelegation", message }, { status: 400 })
+				)
+			})
+		).rejects.toThrow(message);
+	});
 	it("does not request a token when the sidecar is not configured", async () => {
 		const { agent, handle } = makeAgent();
 		await expect(provisionDelegate(agent, "org.example", { serviceUrl: "" })).rejects.toThrow(
diff --git a/web/src/lib/api/delegation.ts b/web/src/lib/api/delegation.ts
index a1893504e..510b54319 100644
--- a/web/src/lib/api/delegation.ts
+++ b/web/src/lib/api/delegation.ts
@@ -64,9 +64,6 @@ export const provisionDelegate = async (
 		const messages: Record<string, string> = {
 			VerifiedEmailRequired:
 				"Verify an email address in Settings → Emails before creating an organization.",
-			// Tranquil enforces the per-controller delegate cap and reports it as
-			// InvalidDelegation; the gate relays Tranquil's error code and message.
-			InvalidDelegation: "You have reached the limit on organizations you can create.",
 			UpstreamUnavailable:
 				"Organization creation is temporarily unavailable. Please try again later."
 		};
-- 
2.51.2


From 3ba1b583cac5558ea7abcb46e6ccc001c3be0940 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Sat, 3 Oct 2026 20:37:39 +0300
Subject: [PATCH 02/25] *: fmt

Signed-off-by: dawn <dawn@tangled.org>
---
 idresolver/resolver.go                 |  2 +-
 knot2/crates/knot-xrpc/src/firehose.rs |  6 ++++--
 migrator/migrator.go                   |  2 +-
 nix/modules/migrator.nix               | 29 +++++++++++++-------------
 spindle/engine/mask_test.go            |  2 +-
 spindle/models/pipeline.go             |  5 +++--
 spindle/oidc/issuer_test.go            |  2 +-
 7 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/idresolver/resolver.go b/idresolver/resolver.go
index 75c7015bb..57d8f6007 100644
--- a/idresolver/resolver.go
+++ b/idresolver/resolver.go
@@ -33,7 +33,7 @@ func BaseDirectory(plcUrl string) *identity.BaseDirectory {
 			Transport: &http.Transport{
 				// did documents are fetched for caller-supplied dids, so the transport refuses
 				// non-public addresses; IDRESOLVER_DEV=1 lifts the guard for local plc instances
-				DialContext:     netutil.SSRFDialer(os.Getenv("IDRESOLVER_DEV") != "1").DialContext,
+				DialContext: netutil.SSRFDialer(os.Getenv("IDRESOLVER_DEV") != "1").DialContext,
 				// would want this around 100ms for services doing lots of handle resolution. Impacts PLC connections as well, but not too bad.
 				IdleConnTimeout: time.Millisecond * 1000,
 				MaxIdleConns:    100,
diff --git a/knot2/crates/knot-xrpc/src/firehose.rs b/knot2/crates/knot-xrpc/src/firehose.rs
index a70bde48c..8c35158cf 100644
--- a/knot2/crates/knot-xrpc/src/firehose.rs
+++ b/knot2/crates/knot-xrpc/src/firehose.rs
@@ -1565,7 +1565,10 @@ mod tests {
         };
         assert_eq!(sig.len(), 64, "secp256k1 signatures are 64 compact bytes");
         assert!(
-            matches!(payload.remove("seq"), Some(ipld_core::ipld::Ipld::Integer(_))),
+            matches!(
+                payload.remove("seq"),
+                Some(ipld_core::ipld::Ipld::Integer(_))
+            ),
             "a push event still carries seq on the wire, just outside the signature"
         );
         assert!(
@@ -1630,4 +1633,3 @@ mod tests {
         );
     }
 }
-
diff --git a/migrator/migrator.go b/migrator/migrator.go
index 4d228e5e0..6716b40d9 100644
--- a/migrator/migrator.go
+++ b/migrator/migrator.go
@@ -14,8 +14,8 @@ import (
 	"tangled.org/core/migrator/git"
 	"tangled.org/core/migrator/gitserver"
 	migratoroauth "tangled.org/core/migrator/oauth"
-	"tangled.org/core/netutil"
 	"tangled.org/core/migrator/worker"
+	"tangled.org/core/netutil"
 	"tangled.org/core/xrpc/serviceauth"
 )
 
diff --git a/nix/modules/migrator.nix b/nix/modules/migrator.nix
index 2597fbaaa..ecdb580dc 100644
--- a/nix/modules/migrator.nix
+++ b/nix/modules/migrator.nix
@@ -113,20 +113,21 @@ in
           StateDirectory = "migrator";
           Path = [pkgs.git pkgs.git-lfs];
           EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
-          Environment = [
-            "MIGRATOR_LISTEN_ADDR=${cfg.listenAddr}"
-            "MIGRATOR_HOSTNAME=${cfg.hostname}"
-            "MIGRATOR_DB_PATH=${cfg.dbPath}"
-            "MIGRATOR_WORK_DIR=${cfg.workDir}"
-            "MIGRATOR_CONCURRENCY=${toString cfg.concurrency}"
-            "MIGRATOR_JOB_TIMEOUT=${cfg.jobTimeout}"
-            "MIGRATOR_MAX_DISK_BYTES=${toString cfg.maxDiskBytes}"
-            "MIGRATOR_MAX_PACK_BYTES=${toString cfg.maxPackBytes}"
-            "MIGRATOR_PLC_URL=${cfg.plcUrl}"
-          ]
-          ++ lib.optionals (cfg.appUrl != null) [
-            "MIGRATOR_APP_URL=${cfg.appUrl}"
-          ];
+          Environment =
+            [
+              "MIGRATOR_LISTEN_ADDR=${cfg.listenAddr}"
+              "MIGRATOR_HOSTNAME=${cfg.hostname}"
+              "MIGRATOR_DB_PATH=${cfg.dbPath}"
+              "MIGRATOR_WORK_DIR=${cfg.workDir}"
+              "MIGRATOR_CONCURRENCY=${toString cfg.concurrency}"
+              "MIGRATOR_JOB_TIMEOUT=${cfg.jobTimeout}"
+              "MIGRATOR_MAX_DISK_BYTES=${toString cfg.maxDiskBytes}"
+              "MIGRATOR_MAX_PACK_BYTES=${toString cfg.maxPackBytes}"
+              "MIGRATOR_PLC_URL=${cfg.plcUrl}"
+            ]
+            ++ lib.optionals (cfg.appUrl != null) [
+              "MIGRATOR_APP_URL=${cfg.appUrl}"
+            ];
           LoadCredential = [
             "master-key:${cfg.masterKeyFile}"
             "private-key:${cfg.privateKeyFile}"
diff --git a/spindle/engine/mask_test.go b/spindle/engine/mask_test.go
index d6b34d7cb..666c3dbdb 100644
--- a/spindle/engine/mask_test.go
+++ b/spindle/engine/mask_test.go
@@ -325,7 +325,7 @@ func TestWorkflowProceedsWhenTheIssuerWithholdsTheToken(t *testing.T) {
 	eng := newEchoEngine()
 	// what an identity the issuer refuses to sign looks like by the time a
 	// workflow runs: a minter that yields no token and no error
-	runEchoPipelineRecords(t, eng, &models.Pipeline{		RepoDid:       "did:plc:boltless",
+	runEchoPipelineRecords(t, eng, &models.Pipeline{RepoDid: "did:plc:boltless",
 		TrustedSource: true,
 		MintIDToken:   func(time.Duration, string) (string, error) { return "", nil },
 	}, audienceRecord("build", "vault"))
diff --git a/spindle/models/pipeline.go b/spindle/models/pipeline.go
index c0bc224f9..70ec6f885 100644
--- a/spindle/models/pipeline.go
+++ b/spindle/models/pipeline.go
@@ -16,7 +16,8 @@ type Pipeline struct {
 	// used to resolve cache hash files against the checkout the workflow builds
 	TriggerMetadata *tangled.Pipeline_TriggerMetadata
 	// mint after slot acquisition, unrecordable runs return ("", nil), errors abort the workflow
-	MintIDToken func(workflowTimeBudget time.Duration, audience string) (string, error)}
+	MintIDToken func(workflowTimeBudget time.Duration, audience string) (string, error)
+}
 
 func TrustedPipelineSource(metadata *tangled.Pipeline_TriggerMetadata, target string) bool {
 	targetDid, err := syntax.ParseDID(target)
@@ -63,5 +64,5 @@ type Workflow struct {
 	CacheBindings  []CacheBinding
 	CacheNamespace string
 	Engine         string
-	Audience string
+	Audience       string
 }
diff --git a/spindle/oidc/issuer_test.go b/spindle/oidc/issuer_test.go
index d7a69cde6..83c3a8e84 100644
--- a/spindle/oidc/issuer_test.go
+++ b/spindle/oidc/issuer_test.go
@@ -21,7 +21,7 @@ import (
 )
 
 const (
-	testIssuer  = "https://tokens.spindle.tangled.sh"
+	testIssuer   = "https://tokens.spindle.tangled.sh"
 	testAudience = "//iam.googleapis.com/projects/1/locations/global/workloadIdentityPools/tangled/providers/spindle"
 )
 
-- 
2.51.2


From 33a8d5a635aa172162bc60c93f31d2c3d950a63e Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Sat, 3 Oct 2026 21:14:18 +0300
Subject: [PATCH 03/25] spindle: opt-in cors on xrpc via
 SPINDLE_SERVER_CORS_ORIGINS

Signed-off-by: dawn <dawn@tangled.org>
---
 docker-compose.yml        |  1 +
 docs/DOCS.md              |  1 +
 go.mod                    |  3 +-
 go.sum                    |  2 +
 nix/gomod2nix.toml        |  3 ++
 nix/modules/spindle.nix   |  8 ++++
 spindle/config/config.go  |  1 +
 spindle/xrpc/cors_test.go | 86 +++++++++++++++++++++++++++++++++++++++
 spindle/xrpc/xrpc.go      | 12 ++++++
 9 files changed, 116 insertions(+), 1 deletion(-)
 create mode 100644 spindle/xrpc/cors_test.go

diff --git a/docker-compose.yml b/docker-compose.yml
index b6106fb7b..3ea02cad6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -285,6 +285,7 @@ services:
       SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://hydrant.tngl.boltless.dev/subscribe
       SPINDLE_SERVER_DEV: "true"
       SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot2.tngl.boltless.dev,mirror.tngl.boltless.dev
+      SPINDLE_SERVER_CORS_ORIGINS: "*"
       SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db
       SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev
       SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images
diff --git a/docs/DOCS.md b/docs/DOCS.md
index 9d7e88c59..bc8e7753e 100644
--- a/docs/DOCS.md
+++ b/docs/DOCS.md
@@ -1946,6 +1946,7 @@ Spindle is configured using environment variables. The following environment var
 - `SPINDLE_SERVER_OWNER`: The DID of the owner (required).
 - `SPINDLE_SERVER_LOG_DIR`: The directory to store workflow logs (default: `"/var/log/spindle"`).
 - `SPINDLE_SERVER_DOCKER_SOCKET`: Path to Docker socket to expose to invoked Spindle containers (default: `""`).
+- `SPINDLE_SERVER_CORS_ORIGINS`: Comma-separated origins allowed to call the XRPC API from a browser, or `*` for any. Needed for web frontends that talk to Spindle directly (default: `""`, CORS disabled).
 - `SPINDLE_NIXERY_PIPELINES_NIXERY`: The Nixery URL (default: `"nixery.tangled.sh"`).
 - `SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT`: The default workflow timeout (default: `"5m"`).
 
diff --git a/go.mod b/go.mod
index 9a53c732b..92c14aa4b 100644
--- a/go.mod
+++ b/go.mod
@@ -41,6 +41,7 @@ require (
 	github.com/felixge/httpsnoop v1.0.4
 	github.com/gliderlabs/ssh v0.3.8
 	github.com/go-chi/chi/v5 v5.2.0
+	github.com/go-chi/cors v1.2.2
 	github.com/go-enry/go-enry/v2 v2.9.6
 	github.com/go-git/go-git/v5 v5.19.0
 	github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.5.0
@@ -80,6 +81,7 @@ require (
 	github.com/yuin/goldmark-emoji v1.0.6
 	github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc
 	gitlab.com/staticnoise/goldmark-callout v0.0.0-20240609120641-6366b799e4ab
+	gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b
 	go.abhg.dev/goldmark/mermaid v0.6.0
 	go.opentelemetry.io/contrib/bridges/otelslog v0.18.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0
@@ -305,7 +307,6 @@ require (
 	github.com/wasilibs/go-re2 v1.10.0 // indirect
 	github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
-	gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect
 	gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
 	go.etcd.io/bbolt v1.4.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
diff --git a/go.sum b/go.sum
index b0b095f4f..9d1318cba 100644
--- a/go.sum
+++ b/go.sum
@@ -303,6 +303,8 @@ github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
 github.com/go-chi/chi/v5 v5.2.0 h1:Aj1EtB0qR2Rdo2dG4O94RIU35w2lvQSj6BRA4+qwFL0=
 github.com/go-chi/chi/v5 v5.2.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
+github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
 github.com/go-enry/go-enry/v2 v2.9.6 h1:np63eOtMV56zfYDHnFVgpEVOk8fr2kmylcMnAZUDbSs=
 github.com/go-enry/go-enry/v2 v2.9.6/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8=
 github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo=
diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml
index f8a96134e..8f57aeb45 100644
--- a/nix/gomod2nix.toml
+++ b/nix/gomod2nix.toml
@@ -364,6 +364,9 @@ schema = 3
   [mod."github.com/go-chi/chi/v5"]
     version = "v5.2.0"
     hash = "sha256-rCZ2W5BdWwjtv7SSpHOgpYEHf9ketzdPX+r2500JL8A="
+  [mod."github.com/go-chi/cors"]
+    version = "v1.2.2"
+    hash = "sha256-uLbgPWK2WvrfDD+2LMbh5himuoidBPwDdnDE6agfuBo="
   [mod."github.com/go-enry/go-enry/v2"]
     version = "v2.9.6"
     hash = "sha256-iQWIFp+ddOsKdLn7ObdMmshEMJFcGw9hEzwQH2RNyF4="
diff --git a/nix/modules/spindle.nix b/nix/modules/spindle.nix
index 434e21e94..ddc11397a 100644
--- a/nix/modules/spindle.nix
+++ b/nix/modules/spindle.nix
@@ -83,6 +83,13 @@ in
             description = "Maximum number of jobs queue up";
           };
 
+          corsOrigins = mkOption {
+            type = types.listOf types.str;
+            default = [];
+            example = ["https://tangled.org"];
+            description = "Origins browsers may call the xrpc api from, \"*\" allows any. Empty disables CORS.";
+          };
+
           secrets = {
             provider = mkOption {
               type = types.str;
@@ -724,6 +731,7 @@ in
         "SPINDLE_SERVER_OWNER=${instance.server.owner}"
         "SPINDLE_SERVER_MAX_JOB_COUNT=${toString instance.server.maxJobCount}"
         "SPINDLE_SERVER_QUEUE_SIZE=${toString instance.server.queueSize}"
+        "SPINDLE_SERVER_CORS_ORIGINS=${concatStringsSep "," instance.server.corsOrigins}"
         "SPINDLE_SERVER_SECRETS_PROVIDER=${instance.server.secrets.provider}"
         "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${instance.server.secrets.openbao.proxyAddr}"
         "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${instance.server.secrets.openbao.mount}"
diff --git a/spindle/config/config.go b/spindle/config/config.go
index 7c9d0e84f..61d263020 100644
--- a/spindle/config/config.go
+++ b/spindle/config/config.go
@@ -29,6 +29,7 @@ type Server struct {
 	QueueSize         int      `env:"QUEUE_SIZE, default=100"`
 	MaxJobCount       int      `env:"MAX_JOB_COUNT, default=2"` // max number of pipelines that run at a time
 	DockerSocket      string   `env:"DOCKER_SOCKET"`            // path to a docker socket to expose to workflow containers
+	CorsOrigins       []string `env:"CORS_ORIGINS"`             // origins browsers may call xrpc from, "*" for any. empty leaves cors off
 }
 
 type Tap struct {
diff --git a/spindle/xrpc/cors_test.go b/spindle/xrpc/cors_test.go
new file mode 100644
index 000000000..75a888751
--- /dev/null
+++ b/spindle/xrpc/cors_test.go
@@ -0,0 +1,86 @@
+package xrpc
+
+import (
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"tangled.org/core/api/tangled"
+	"tangled.org/core/spindle/config"
+	"tangled.org/core/xrpc/serviceauth"
+)
+
+func corsRouter(origins ...string) http.Handler {
+	cfg := &config.Config{}
+	cfg.Server.CorsOrigins = origins
+	x := &Xrpc{
+		Config:      cfg,
+		ServiceAuth: serviceauth.NewServiceAuth(slog.Default(), nil, "did:web:spindle.test"),
+	}
+	return x.Router()
+}
+
+func preflight(router http.Handler, origin, nsid, method string) *httptest.ResponseRecorder {
+	req := httptest.NewRequest(http.MethodOptions, "/"+nsid, nil)
+	req.Header.Set("Origin", origin)
+	req.Header.Set("Access-Control-Request-Method", method)
+	req.Header.Set("Access-Control-Request-Headers", "authorization")
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+	return rec
+}
+
+func TestCorsOffByDefault(t *testing.T) {
+	rec := preflight(corsRouter(), "https://tangled.org", tangled.RepoListSecretsNSID, http.MethodGet)
+	if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
+		t.Errorf("allow-origin %q with no origins configured", got)
+	}
+}
+
+func TestCorsAnyOrigin(t *testing.T) {
+	router := corsRouter("*")
+
+	for _, tc := range []struct{ nsid, method string }{
+		{tangled.RepoListSecretsNSID, http.MethodGet},
+		{tangled.RepoAddSecretNSID, http.MethodPost},
+	} {
+		rec := preflight(router, "https://tangled.org", tc.nsid, tc.method)
+		if rec.Code >= 300 {
+			t.Errorf("%s preflight: status %d", tc.nsid, rec.Code)
+		}
+		if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
+			t.Errorf("%s preflight: allow-origin %q", tc.nsid, got)
+		}
+		if got := rec.Header().Get("Access-Control-Allow-Headers"); got == "" {
+			t.Errorf("%s preflight: no allow-headers", tc.nsid)
+		}
+	}
+}
+
+func TestCorsListedOrigins(t *testing.T) {
+	router := corsRouter("https://tangled.org")
+
+	rec := preflight(router, "https://tangled.org", tangled.RepoListSecretsNSID, http.MethodGet)
+	if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://tangled.org" {
+		t.Errorf("listed origin: allow-origin %q", got)
+	}
+
+	rec = preflight(router, "https://evil.example", tangled.RepoListSecretsNSID, http.MethodGet)
+	if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
+		t.Errorf("unlisted origin: allow-origin %q", got)
+	}
+}
+
+func TestCorsHeadersOnRejectedRequests(t *testing.T) {
+	req := httptest.NewRequest(http.MethodGet, "/"+tangled.RepoListSecretsNSID, nil)
+	req.Header.Set("Origin", "https://tangled.org")
+	req.Header.Set("Authorization", "Bearer not.a.jwt")
+	rec := httptest.NewRecorder()
+	corsRouter("*").ServeHTTP(rec, req)
+
+	// without the header the browser hides the xrpc error body and the web app only sees a network error
+	if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
+		t.Errorf("allow-origin %q on a %d", got, rec.Code)
+	}
+}
diff --git a/spindle/xrpc/xrpc.go b/spindle/xrpc/xrpc.go
index b66f6454d..959cbf0bb 100644
--- a/spindle/xrpc/xrpc.go
+++ b/spindle/xrpc/xrpc.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/bluesky-social/indigo/atproto/syntax"
 	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/cors"
 
 	"tangled.org/core/api/org_tangled"
 	"tangled.org/core/api/tangled"
@@ -79,6 +80,17 @@ type Xrpc struct {
 func (x *Xrpc) Router() http.Handler {
 	r := chi.NewRouter()
 
+	// browsers preflight anything carrying a service auth header, and without an
+	// answer they drop the real request before it leaves
+	if x.Config != nil && len(x.Config.Server.CorsOrigins) > 0 {
+		r.Use(cors.Handler(cors.Options{
+			AllowedOrigins: x.Config.Server.CorsOrigins,
+			AllowedMethods: []string{http.MethodGet, http.MethodPost},
+			AllowedHeaders: []string{"Authorization", "Content-Type"},
+			MaxAge:         86400,
+		}))
+	}
+
 	r.Get("/"+org_tangled.CiGetWorkflowDefinitionNSID, x.handleOrgTangledCiGetWorkflowDefinition)
 	r.Get("/"+org_tangled.CiListWorkflowDefinitionsNSID, x.handleOrgTangledCiListWorkflowDefinitions)
 	r.Get("/"+org_tangled.CiSubscribePipelineLogsNSID, x.handleOrgTangledCiSubscribePipelineLogs)
-- 
2.51.2


From 3f4ae6cf6ee3d35a71ce2eed28fb93aed4625ca8 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Sat, 3 Oct 2026 21:17:35 +0300
Subject: [PATCH 04/25] nix: regenerate gomod2nix.toml

Signed-off-by: dawn <dawn@tangled.org>
---
 nix/gomod2nix.toml | 125 ++++++++++++++++++++++-----------------------
 1 file changed, 61 insertions(+), 64 deletions(-)

diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml
index 8f57aeb45..bde133d56 100644
--- a/nix/gomod2nix.toml
+++ b/nix/gomod2nix.toml
@@ -8,8 +8,8 @@ schema = 3
     version = "v1.2.0"
     hash = "sha256-4+42DSj7LAdJyPPxif3b5QRH5MCjedudgylTHBibZlQ="
   [mod."cel.dev/expr"]
-    version = "v0.25.1"
-    hash = "sha256-TEdMxFUPK7IZuCXMufwCkbN+ZZIXSQclljIybFZcByo="
+    version = "v0.25.2"
+    hash = "sha256-J8C2tbW9PcXA71UhcEdyXPR8vmqkEiapZk/RnZz3qyo="
   [mod."dario.cat/mergo"]
     version = "v1.0.1"
     hash = "sha256-wcG6+x0k6KzOSlaPA+1RFxa06/RIAePJTAjjuhLbImw="
@@ -184,8 +184,8 @@ schema = 3
     hash = "sha256-GWm5i1ukuBukV0GMF1rffpbOSSXZdfg6/0pABMiGzLQ="
     replaced = "tangled.sh/oppi.li/go-gitdiff"
   [mod."github.com/bluesky-social/indigo"]
-    version = "v0.0.0-20260821154308-2d6bc194426d"
-    hash = "sha256-M7J3JZlqHYU5VgAoAvJCh8LjnHc7CLfkwjfciJYCUec="
+    version = "v0.0.0-20260918100317-e2c9652127ac"
+    hash = "sha256-wqGt02TC+Q/h7x0HKeDHhOi7hJIB/14Gcqx+2CZpzSE="
     replaced = "tangled.org/ptr.pet/indigo"
   [mod."github.com/bluesky-social/jetstream"]
     version = "v0.0.0-20260226214936-e0274250f654"
@@ -405,8 +405,8 @@ schema = 3
     version = "v0.2.3"
     hash = "sha256-hYHMUdwxVkMOjSKjR7UWO0D0juHdI4wL8JEy5plu/Jc="
   [mod."github.com/goccy/go-json"]
-    version = "v0.10.5"
-    hash = "sha256-/EtlGihP0/7oInzMC5E0InZ4b5Ad3s4xOpqotloi3xw="
+    version = "v0.10.6"
+    hash = "sha256-/REW1EDZ3Ta45hrDNR9YTIlhkxQQI8OeK5TxBNJ47Ho="
   [mod."github.com/godbus/dbus/v5"]
     version = "v5.1.0"
     hash = "sha256-xOCMJpQK3KTmHTPn/CdqI4j0eENCtMmJDgAIoYqYOEY="
@@ -416,9 +416,6 @@ schema = 3
   [mod."github.com/gohugoio/hugo-goldmark-extensions/passthrough"]
     version = "v0.5.0"
     hash = "sha256-8kQnLGnNYUkm3q/cFgf+9P++91OUbMJJNNgUQ1Bl3fg="
-  [mod."github.com/golang-jwt/jwt"]
-    version = "v3.2.2+incompatible"
-    hash = "sha256-LOkpuXhWrFayvVf1GOaOmZI5YKEsgqVSb22aF8LnCEM="
   [mod."github.com/golang-jwt/jwt/v5"]
     version = "v5.3.0"
     hash = "sha256-VdN9Eo74ncMFJEVUSiJ1VRPMbC09FdVGno8wAReseXU="
@@ -522,8 +519,8 @@ schema = 3
     version = "v0.5.2"
     hash = "sha256-DIik12zEKdemFWatKrtlq+89jEnPtjQ1uDpSHoGs5t0="
   [mod."github.com/ipfs/go-cid"]
-    version = "v0.6.0"
-    hash = "sha256-OVKY86pHd0CYXutjP/oJahp3Us7L2+nRUtsSfDurvYs="
+    version = "v0.6.2"
+    hash = "sha256-WiQSjjdK7FXafr6Fj9JkXPeET+z9fyI4Ni1sVjl/k20="
   [mod."github.com/ipfs/go-datastore"]
     version = "v0.9.0"
     hash = "sha256-cxYAC5k4wKOrYU4OkH2gJ76SZ92ixSuWDag6C+P6Pfg="
@@ -570,8 +567,8 @@ schema = 3
     version = "v1.7.0"
     hash = "sha256-jJmMUBdHHSavEvpvZCQnqfiMrqrahD5YNaaSIUdGSgo="
   [mod."github.com/ipld/go-ipld-prime"]
-    version = "v0.21.0"
-    hash = "sha256-aOLwqmDtt3jA4Fb3U+FOBX6T0yOMujuVInJN2a+7iSA="
+    version = "v0.24.0"
+    hash = "sha256-jodwYelhDg+TJxUYu7TSYUJK2NqpFf5WZKMYZdsxbAQ="
   [mod."github.com/jackc/pgpassfile"]
     version = "v1.0.0"
     hash = "sha256-H0nFbC34/3pZUFnuiQk9W7yvAMh6qJDrqvHp+akBPLM="
@@ -579,8 +576,8 @@ schema = 3
     version = "v0.0.0-20240606120523-5a60cdf6a761"
     hash = "sha256-ETpGsLAA2wcm5xJBayr/mZrCE1YsWbnkbSSX3ptrFn0="
   [mod."github.com/jackc/pgx/v5"]
-    version = "v5.8.0"
-    hash = "sha256-Mq5/A/Obcceu6kKxUv30DPC2ZaVvD8Iq/YtmLm1BVec="
+    version = "v5.10.0"
+    hash = "sha256-rDJ6VNA0Pzei2KwZjETA1uOMgb5yuXJRgxUauwID630="
   [mod."github.com/jackc/puddle/v2"]
     version = "v2.2.2"
     hash = "sha256-IUxdu4JYfsCh/qlz2SiUWu7EVPHhyooiVA4oaS2Z6yk="
@@ -597,17 +594,17 @@ schema = 3
     version = "v1.2.0"
     hash = "sha256-Ta7ZOmyX8gG5tzWbY2oES70EJPfI90U7CIJS9EAce0s="
   [mod."github.com/klauspost/compress"]
-    version = "v1.18.0"
-    hash = "sha256-jc5pMU/HCBFOShMcngVwNMhz9wolxjOb579868LtOuk="
+    version = "v1.18.6"
+    hash = "sha256-gULc+A2Clg2HnwR9UNXSIHVGdFY/aGqhf0zlvMxqaSY="
   [mod."github.com/klauspost/cpuid/v2"]
-    version = "v2.3.0"
-    hash = "sha256-50JhbQyT67BK38HIdJihPtjV7orYp96HknI2VP7A9Yc="
+    version = "v2.4.0"
+    hash = "sha256-nvTeRVxDekVNLd25aJ2NslBnG7nT25ewoSTk3iW5Xok="
   [mod."github.com/labstack/echo/v4"]
-    version = "v4.11.3"
-    hash = "sha256-rdqH4HQB/vZyEsoymrEsQ8izjf0m7jhrIxbF6r5ZmBo="
+    version = "v4.15.4"
+    hash = "sha256-Hk2cvSmYRCe2MiUX0G54qLwZkarvgO+5jRoIsV0xl9A="
   [mod."github.com/labstack/gommon"]
-    version = "v0.4.1"
-    hash = "sha256-qfjV9jmtR8I7gC7/Hm02XDbuVLX8UkRNi3wPer8Jkm4="
+    version = "v0.5.0"
+    hash = "sha256-QL7ul2y9VIShKMv/3uAINTWOVikBGJgiBKul7tdRcco="
   [mod."github.com/landlock-lsm/go-landlock"]
     version = "v0.8.1"
     hash = "sha256-7H6/LBmv/d4vDIfZcJ0PeNiNU2PInkw29aQU1yHWxsU="
@@ -615,11 +612,11 @@ schema = 3
     version = "v1.3.0"
     hash = "sha256-6BKrJsfmxie+YFAWzTYVPQfrwjQEXRo+J8LY+50C1BU="
   [mod."github.com/mattn/go-colorable"]
-    version = "v0.1.14"
-    hash = "sha256-JC60PjKj7MvhZmUHTZ9p372FV72I9Mxvli3fivTbxuA="
+    version = "v0.1.15"
+    hash = "sha256-hH+yMJHjboEDQbv399OEjfc50RgtO/edw0r2sJfxEJg="
   [mod."github.com/mattn/go-isatty"]
-    version = "v0.0.20"
-    hash = "sha256-qhw9hWtU5wnyFyuMbKx+7RB8ckQaFQ8D+8GKPkN3HHQ="
+    version = "v0.0.24"
+    hash = "sha256-pmq8KKIb2+qMF3E1cR2B7NAj5joF2tZduyXjIfIlkTk="
   [mod."github.com/mattn/go-localereader"]
     version = "v0.0.1"
     hash = "sha256-JlWckeGaWG+bXK8l8WEdZqmSiTwCA8b1qbmBKa/Fj3E="
@@ -669,8 +666,8 @@ schema = 3
     version = "v1.0.0"
     hash = "sha256-5zYgLeGr3K+uhGKlN3xv0PO67V+2Zw+cezjzNCmAWOE="
   [mod."github.com/mr-tron/base58"]
-    version = "v1.2.0"
-    hash = "sha256-8FzMu3kHUbBX10pUdtGf59Ag7BNupx8ZHeUaodR1/Vk="
+    version = "v1.3.0"
+    hash = "sha256-LN6yaKmfTdAG3T3zFIrZlMtIRPxIot4uWsk4YG6hYZI="
   [mod."github.com/mschoch/smat"]
     version = "v0.2.0"
     hash = "sha256-DZvUJXjIcta3U+zxzgU3wpoGn/V4lpBY7Xme8aQUi+E="
@@ -690,8 +687,8 @@ schema = 3
     version = "v0.2.0"
     hash = "sha256-GKNnAGA0Lb39BDGYBm1ieKdXmho8Pu7ouyfVPXvV0PE="
   [mod."github.com/multiformats/go-multibase"]
-    version = "v0.2.0"
-    hash = "sha256-w+hp6u5bWyd34qe0CX+bq487ADqq6SgRR/JuqRB578s="
+    version = "v0.3.0"
+    hash = "sha256-VQgSUWY1p4UMURMSJVmgrGNlTJI5iLMSJ9T/1tJYzbI="
   [mod."github.com/multiformats/go-multihash"]
     version = "v0.2.3"
     hash = "sha256-zqIIE5jMFzm+qhUrouSF+WdXGeHUEYIQvVnKWWU6mRs="
@@ -729,8 +726,8 @@ schema = 3
     version = "v1.0.1-0.20181226105442-5d4384ee4fb2"
     hash = "sha256-XA4Oj1gdmdV/F/+8kMI+DBxKPthZ768hbKsO3d9Gx90="
   [mod."github.com/polydawn/refmt"]
-    version = "v0.89.1-0.20221221234430-40501e09de1f"
-    hash = "sha256-wBdFROClTHNPYU4IjeKbBXaG7F6j5hZe15gMxiqKvi4="
+    version = "v0.90.0"
+    hash = "sha256-4o0kL8yfyDPm/Txkl67/uGElFrlq45meLOmncCoFmdc="
   [mod."github.com/posthog/posthog-go"]
     version = "v1.5.5"
     hash = "sha256-ouhfDUCXsfpcgaCLfJE9oYprAQHuV61OJzb/aEhT0j8="
@@ -877,8 +874,8 @@ schema = 3
     version = "v0.65.0"
     hash = "sha256-fP/2TfGR6TcCH/ALHCAR2qHJlAJrCi8EU0OY1dRcf8U="
   [mod."go.opentelemetry.io/otel"]
-    version = "v1.43.0"
-    hash = "sha256-oRemJUZhA7AzfUoBbRVA32u/XhMpipxLywHoJ1qsHBs="
+    version = "v1.44.0"
+    hash = "sha256-XTGV9RjOIKyWrrgXJ32IA59lovxXqSeRrpcpC1FAFZA="
   [mod."go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"]
     version = "v0.19.0"
     hash = "sha256-fPT9B5yAIEXYVGBi5gcc7emErpKGUZzY46TV68wCWJ0="
@@ -892,17 +889,17 @@ schema = 3
     version = "v0.19.0"
     hash = "sha256-bxaeA+aHA2VRrl4hfzomoadJyq34e6pzUvW9BLTtC/o="
   [mod."go.opentelemetry.io/otel/metric"]
-    version = "v1.43.0"
-    hash = "sha256-iUfx5AvN2oiqlh2v8/oFa+2jm8RX4kbb6X1EOKRyPPw="
+    version = "v1.44.0"
+    hash = "sha256-e8dVyRoavby/YG7aH2T05IhWHlLvwnMHyPhelAS1mCQ="
   [mod."go.opentelemetry.io/otel/sdk"]
-    version = "v1.43.0"
-    hash = "sha256-Z1uTuALNhRXStiDl0UvYh9+XE2hd9OYe/bxCSuR78uE="
+    version = "v1.44.0"
+    hash = "sha256-rNqJXr5P2f5Qgllp/3E5Rm5lqt1kBhxHXppT5dXw5Cw="
   [mod."go.opentelemetry.io/otel/sdk/log"]
     version = "v0.19.0"
     hash = "sha256-mV2lp63Qi6THZYNmWEBo6puRFdslx3meMC+Sv7vs3iU="
   [mod."go.opentelemetry.io/otel/trace"]
-    version = "v1.43.0"
-    hash = "sha256-LLx1PjBGzDwZ3//Gp14R1DCMlnMCzFxnGYqVUz5jTmk="
+    version = "v1.44.0"
+    hash = "sha256-69HorWRTLLzAHRgLe4nPp13qJthb3qp7RU7c0Y9qV/c="
   [mod."go.opentelemetry.io/proto/otlp"]
     version = "v1.10.0"
     hash = "sha256-IEnbR38ucFLTcuC2FA+gRvZNq2loUqXgDskSqP3+LUM="
@@ -919,8 +916,8 @@ schema = 3
     version = "v2.4.3"
     hash = "sha256-WqfrOUQFvfuORgl1yyVOcsEXU/vwWQHkcVWx3vCxvaw="
   [mod."golang.org/x/crypto"]
-    version = "v0.51.0"
-    hash = "sha256-/R74sc1mcOaOuBeXRQzrXrHAgA5VhNWc6SfQJaxb17U="
+    version = "v0.55.0"
+    hash = "sha256-99uV/ESeqhaDEwqbXxDpePuwzO0JoEHGhqZXZVHq/H4="
   [mod."golang.org/x/exp"]
     version = "v0.0.0-20260410095643-746e56fc9e2f"
     hash = "sha256-JaDJGLIRoJjjvsg3dgfFuo7XApEJO2V4kUDmd58qTLI="
@@ -928,41 +925,41 @@ schema = 3
     version = "v0.31.0"
     hash = "sha256-ZFTlu9+4QToPPLA8C5UcG2eq/lQylq81RoG/WtYo9rg="
   [mod."golang.org/x/mod"]
-    version = "v0.35.0"
-    hash = "sha256-ICEQxokHywOFInDPqoP+go9l1tZSz3roknF5SXPtNV4="
+    version = "v0.38.0"
+    hash = "sha256-BpKfvsmb7ecEDvTBKx9ZHauxoWRkOAZCqg4BrHM2/S0="
   [mod."golang.org/x/net"]
-    version = "v0.55.0"
-    hash = "sha256-Phi2mSmBGOJcvqPPAit3uqF3UP8SKRI9dHj6yTM3s5s="
+    version = "v0.58.0"
+    hash = "sha256-fDOkQJXURrbq2dBGa91bCUsXknCZlo3ru2JIofSOGRo="
   [mod."golang.org/x/sync"]
-    version = "v0.20.0"
-    hash = "sha256-ybcjhCfK6lroUM0yswUvWooW8MOQZBXyiSqoxG6Uy0Y="
+    version = "v0.22.0"
+    hash = "sha256-VZjl0fAM0p/nI81zh+pdBjzxFupx0UcKYXBBpL2ZS7k="
   [mod."golang.org/x/sys"]
-    version = "v0.45.0"
-    hash = "sha256-hkBoNazrDA67ER6sWhb+EKxx9nJ24+nz3zGy+zT5Hvw="
+    version = "v0.47.0"
+    hash = "sha256-TpbRyWWqHjddP6QzUgAbaLd2EE0S+GYNRUIDJd18r98="
   [mod."golang.org/x/text"]
-    version = "v0.37.0"
-    hash = "sha256-8XDOnlPIybcDRy89fkjG5VqtIt5Ku+LmaqYhgKl7i1E="
+    version = "v0.41.0"
+    hash = "sha256-22nHcolG87qSPahT2Ey8S5iGlCLAglE9ObYXO6XZ3ZY="
   [mod."golang.org/x/time"]
-    version = "v0.14.0"
-    hash = "sha256-fVjpq0ieUHVEOTSElDVleMWvfdcqojZchqdUXiC7NnY="
+    version = "v0.15.0"
+    hash = "sha256-5D24A65wn7k93Jj3+918UKjB9ccmGHPBEqjD2XDB92E="
   [mod."golang.org/x/tools"]
-    version = "v0.44.0"
-    hash = "sha256-xuj5FLtSJsAojLLTLXtPdLAIFNTKoVFbDMuqRXmj2W4="
+    version = "v0.48.0"
+    hash = "sha256-9cRNUaup6fexA5S1zc+Ic3aaoxAgKWntyMJ2xaOdiP8="
   [mod."golang.org/x/xerrors"]
     version = "v0.0.0-20240903120638-7835f813f4da"
     hash = "sha256-bE7CcrnAvryNvM26ieJGXqbAtuLwHaGcmtVMsVnksqo="
   [mod."google.golang.org/genproto/googleapis/api"]
-    version = "v0.0.0-20260401024825-9d38bb4040a9"
-    hash = "sha256-BXg1F9SwkNalLgqbjfpmr+KcFrZcZXuubJh990f+1mI="
+    version = "v0.0.0-20260526163538-3dc84a4a5aaa"
+    hash = "sha256-XwBCYWKK47hTvTNUl8HenRcbzU5xIYhRfPfNypUU1Ps="
   [mod."google.golang.org/genproto/googleapis/rpc"]
-    version = "v0.0.0-20260401024825-9d38bb4040a9"
+    version = "v0.0.0-20260817212433-ac3dfec99bb1"
     hash = "sha256-ldJTTb7hhj1mdmzTn9IEkQVwCoj3KRlENZtUSEKHABU="
   [mod."google.golang.org/grpc"]
-    version = "v1.80.0"
-    hash = "sha256-+p50KGJvGWdpB/4f0h477dCAfoOL5m2PzG8BGOekVgY="
+    version = "v1.83.1"
+    hash = "sha256-FpEi28U8I4/pzzfS+Gv4PySLcxkjVTy1pqtRlEDZJ54="
   [mod."google.golang.org/protobuf"]
-    version = "v1.36.11"
-    hash = "sha256-7W+6jntfI/awWL3JP6yQedxqP5S9o3XvPgJ2XxxsIeE="
+    version = "v1.36.12"
+    hash = "sha256-MKd0AdkWEe2A79mGPzOY5I76nMqTsGJKQNPJOKyOwkQ="
   [mod."gopkg.in/fsnotify.v1"]
     version = "v1.4.7"
     hash = "sha256-j/Ts92oXa3k1MFU7Yd8/AqafRTsFn7V2pDKCyDJLah8="
-- 
2.51.2


From 59f339ed37b4b48506dbfa6013678e9299e62981 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Sun, 4 Oct 2026 18:32:02 +0300
Subject: [PATCH 05/25] web: always highlight with oniguruma on the server

Signed-off-by: dawn <dawn@tangled.org>
---
 web/patches/@pierre__diffs@1.4.1.patch | 21 ++++++++++++
 web/pnpm-lock.yaml                     |  6 ++--
 web/src/lib/syntax.test.ts             | 47 ++++++++++++++++++++++++++
 3 files changed, 71 insertions(+), 3 deletions(-)
 create mode 100644 web/src/lib/syntax.test.ts

diff --git a/web/patches/@pierre__diffs@1.4.1.patch b/web/patches/@pierre__diffs@1.4.1.patch
index 2107040db..f23fb6a4c 100644
--- a/web/patches/@pierre__diffs@1.4.1.patch
+++ b/web/patches/@pierre__diffs@1.4.1.patch
@@ -84,3 +84,24 @@ index 804fb157d278ec26787fc0aede422ad205096780..8da1f56f3a672aee29046e0feab4c5b7
  function tokenizeLine(grammar, colorMap, lineText, stateStack, timeLimit, collectBracketIgnoredRanges = true, resolveTokens = true) {
  	const result = grammar.tokenizeLine2(lineText, stateStack, timeLimit);
  	if (result.stoppedEarly) console.warn(`[diffs] Time limit reached when tokenizing line: ${lineText.substring(0, 100)}`);
+diff --git a/dist/highlighter/shared_highlighter.js b/dist/highlighter/shared_highlighter.js
+index e231d3c2396b82a0aa7dee8d3c3352dbcae4adb6..f84d7ee85829eea7cf1df28da33d58c05c9be106 100644
+--- a/dist/highlighter/shared_highlighter.js
++++ b/dist/highlighter/shared_highlighter.js
+@@ -11,11 +11,15 @@ import { createHighlighter, createJavaScriptRegexEngine, createOnigurumaEngine }
+ import { pierreThemes } from "@pierre/theming/themes";
+ //#region src/highlighter/shared_highlighter.ts
+ let highlighter;
++// tangled patch: the js regex engine can backtrack forever on some grammars (go
++// struct literals among them), which blocks the server's event loop for good.
++// on node the shared highlighter is always oniguruma, whatever the caller asks for
++const isNode = globalThis.process?.versions?.node != null;
+ async function getSharedHighlighter({ themes, langs, preferredHighlighter = "shiki-js" }) {
+ 	highlighter ??= createHighlighter({
+ 		themes: [],
+ 		langs: ["text"],
+-		engine: preferredHighlighter === "shiki-wasm" ? createOnigurumaEngine(import("shiki/wasm")) : createJavaScriptRegexEngine()
++		engine: isNode || preferredHighlighter === "shiki-wasm" ? createOnigurumaEngine(import("shiki/wasm")) : createJavaScriptRegexEngine()
+ 	});
+ 	const instance = isHighlighterLoading(highlighter) ? await highlighter : highlighter;
+ 	highlighter = instance;
diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml
index 2a7f448b7..b2eccdf61 100644
--- a/web/pnpm-lock.yaml
+++ b/web/pnpm-lock.yaml
@@ -8,7 +8,7 @@ overrides:
   aria-query: 5.3.2
 
 patchedDependencies:
-  '@pierre/diffs@1.4.1': b08596641b783a2c462c36e037f02db2505428aaa2f439c216330d5a09553f3f
+  '@pierre/diffs@1.4.1': 9878b5f3ecdacb7c23c56f1f24e8c7d95670f0563b14739c06746b8aa5c7a419
   boneyard-js@1.9.0: e9bda6b0720bed34eefee605bd041784ac2849ae4d27120d9815663d342f694f
 
 importers:
@@ -50,7 +50,7 @@ importers:
         version: 1.0.1(markdown-it@14.3.0)
       '@pierre/diffs':
         specifier: ^1.4.1
-        version: 1.4.1(patch_hash=b08596641b783a2c462c36e037f02db2505428aaa2f439c216330d5a09553f3f)(@shikijs/themes@4.3.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
+        version: 1.4.1(patch_hash=9878b5f3ecdacb7c23c56f1f24e8c7d95670f0563b14739c06746b8aa5c7a419)(@shikijs/themes@4.3.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
       '@tanstack/svelte-virtual':
         specifier: ^3.13.35
         version: 3.13.35(svelte@5.56.9(@typescript-eslint/types@8.64.0))
@@ -4447,7 +4447,7 @@ snapshots:
   '@oxc-resolver/binding-win32-x64-msvc@11.24.2':
     optional: true
 
-  '@pierre/diffs@1.4.1(patch_hash=b08596641b783a2c462c36e037f02db2505428aaa2f439c216330d5a09553f3f)(@shikijs/themes@4.3.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
+  '@pierre/diffs@1.4.1(patch_hash=9878b5f3ecdacb7c23c56f1f24e8c7d95670f0563b14739c06746b8aa5c7a419)(@shikijs/themes@4.3.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
     dependencies:
       '@pierre/theme': 2.0.0
       '@pierre/theming': 1.0.1(@pierre/theme@2.0.0)(@shikijs/themes@4.3.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(shiki@4.3.1)
diff --git a/web/src/lib/syntax.test.ts b/web/src/lib/syntax.test.ts
new file mode 100644
index 000000000..a097ed8c6
--- /dev/null
+++ b/web/src/lib/syntax.test.ts
@@ -0,0 +1,47 @@
+import { Worker } from "node:worker_threads";
+import { describe, expect, it } from "vitest";
+
+// shiki's js regex engine backtracks without end on this go struct literal. on the
+// server each blob page with it pinned an instance's event loop for good
+const BACKTRACKING_GO = 'pp := struct {\n\t}{t(*p), strings.TrimSuffix(p.URL.String(), "/")}\n';
+
+// a runaway regex can't be interrupted on its own thread, so this tokenizes in a
+// worker and terminates it at the deadline
+const WORKER = `
+const { parentPort, workerData } = require("node:worker_threads");
+import(workerData.pierre).then(async ({ getSharedHighlighter, registerCustomCSSVariableTheme }) => {
+	registerCustomCSSVariableTheme("test", { background: "#ffffff", foreground: "#000000" });
+	const highlighter = await getSharedHighlighter({
+		themes: ["test"],
+		langs: ["go"],
+		preferredHighlighter: "shiki-js"
+	});
+	highlighter.codeToTokens(workerData.code, { lang: "go", theme: "test" });
+	parentPort.postMessage("done");
+});
+`;
+
+const tokenizesWithin = (code: string, ms: number): Promise<boolean> =>
+	new Promise((resolve, reject) => {
+		const worker = new Worker(WORKER, {
+			eval: true,
+			workerData: { pierre: import.meta.resolve("@pierre/diffs"), code }
+		});
+		const finish = (done: boolean) => {
+			clearTimeout(timer);
+			void worker.terminate();
+			resolve(done);
+		};
+		const timer = setTimeout(() => finish(false), ms);
+		worker.once("message", () => finish(true));
+		worker.once("error", (err) => {
+			clearTimeout(timer);
+			reject(err);
+		});
+	});
+
+describe("shared highlighter on the server", () => {
+	it("uses oniguruma even when a caller asks for the js engine", async () => {
+		expect(await tokenizesWithin(BACKTRACKING_GO, 10_000)).toBe(true);
+	}, 15_000);
+});
-- 
2.51.2


From 1c1bf4ab301b9ded7a5b20e7a97449a3fbc84564 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Sun, 4 Oct 2026 21:21:47 +0300
Subject: [PATCH 06/25] tapc: batch AddRepos so tap stays under sqlite's
 variable cap

Signed-off-by: dawn <dawn@tangled.org>
---
 tapc/tap.go      | 37 +++++++++++++--------------
 tapc/tap_test.go | 65 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+), 20 deletions(-)
 create mode 100644 tapc/tap_test.go

diff --git a/tapc/tap.go b/tapc/tap.go
index c44ca19b5..c0e155cbf 100644
--- a/tapc/tap.go
+++ b/tapc/tap.go
@@ -40,35 +40,32 @@ func NewClient(url, adminPassword string) Client {
 	}
 }
 
-func (c *Client) AddRepos(ctx context.Context, dids []syntax.DID) error {
-	body, err := json.Marshal(map[string][]syntax.DID{"dids": dids})
-	if err != nil {
-		return err
-	}
-	req, err := http.NewRequestWithContext(ctx, "POST", c.Url+"/repos/add", bytes.NewReader(body))
-	if err != nil {
-		return err
-	}
-	req.SetBasicAuth("admin", c.AdminPassword)
-	req.Header.Set("Content-Type", "application/json")
+// tap inserts one /repos/add body as a single sqlite statement at about 9
+// variables per did, so a spindle's whole member list blows past sqlite's
+// 32766 cap and tap 500s
+const addReposBatchSize = 1000
 
-	resp, err := c.HTTPClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("tap: /repos/add failed with status %d", resp.StatusCode)
+func (c *Client) AddRepos(ctx context.Context, dids []syntax.DID) error {
+	for len(dids) > 0 {
+		n := min(len(dids), addReposBatchSize)
+		if err := c.postDids(ctx, "/repos/add", dids[:n]); err != nil {
+			return err
+		}
+		dids = dids[n:]
 	}
 	return nil
 }
 
 func (c *Client) RemoveRepos(ctx context.Context, dids []syntax.DID) error {
+	return c.postDids(ctx, "/repos/remove", dids)
+}
+
+func (c *Client) postDids(ctx context.Context, path string, dids []syntax.DID) error {
 	body, err := json.Marshal(map[string][]syntax.DID{"dids": dids})
 	if err != nil {
 		return err
 	}
-	req, err := http.NewRequestWithContext(ctx, "POST", c.Url+"/repos/remove", bytes.NewReader(body))
+	req, err := http.NewRequestWithContext(ctx, "POST", c.Url+path, bytes.NewReader(body))
 	if err != nil {
 		return err
 	}
@@ -81,7 +78,7 @@ func (c *Client) RemoveRepos(ctx context.Context, dids []syntax.DID) error {
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("tap: /repos/remove failed with status %d", resp.StatusCode)
+		return fmt.Errorf("tap: %s failed with status %d", path, resp.StatusCode)
 	}
 	return nil
 }
diff --git a/tapc/tap_test.go b/tapc/tap_test.go
new file mode 100644
index 000000000..7f236d233
--- /dev/null
+++ b/tapc/tap_test.go
@@ -0,0 +1,65 @@
+package tapc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"testing"
+
+	"github.com/bluesky-social/indigo/atproto/syntax"
+)
+
+func TestAddReposBatchesLargeLists(t *testing.T) {
+	var batches [][]syntax.DID
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body struct {
+			DIDs []syntax.DID `json:"dids"`
+		}
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Errorf("decode body: %v", err)
+		}
+		// same failure real tap gives once sqlite runs out of variables
+		if len(body.DIDs) > addReposBatchSize {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		batches = append(batches, body.DIDs)
+	}))
+	defer srv.Close()
+
+	dids := make([]syntax.DID, 2*addReposBatchSize+500)
+	for i := range dids {
+		dids[i] = syntax.DID(fmt.Sprintf("did:plc:%024d", i))
+	}
+
+	c := NewClient(srv.URL, "pw")
+	if err := c.AddRepos(context.Background(), dids); err != nil {
+		t.Fatalf("AddRepos: %v", err)
+	}
+
+	if len(batches) != 3 {
+		t.Fatalf("got %d batches, want 3", len(batches))
+	}
+	var sent []syntax.DID
+	for _, b := range batches {
+		sent = append(sent, b...)
+	}
+	if !slices.Equal(sent, dids) {
+		t.Fatalf("sent %d dids, want all %d in order", len(sent), len(dids))
+	}
+}
+
+func TestAddReposEmptySendsNothing(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		t.Errorf("unexpected request to %s", r.URL.Path)
+	}))
+	defer srv.Close()
+
+	c := NewClient(srv.URL, "pw")
+	if err := c.AddRepos(context.Background(), nil); err != nil {
+		t.Fatalf("AddRepos: %v", err)
+	}
+}
-- 
2.51.2


From 64d68066217ef4f5f9e86a2054c4e0e81e3fdb79 Mon Sep 17 00:00:00 2001
From: eti <eti@eti.tf>
Date: Mon, 5 Oct 2026 14:53:24 +0200
Subject: [PATCH 07/25] web: prevent rubber bounce

Signed-off-by: eti <eti@eti.tf>
---
 web/src/app.css | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/web/src/app.css b/web/src/app.css
index f1cf4bf06..a54741fe0 100644
--- a/web/src/app.css
+++ b/web/src/app.css
@@ -822,3 +822,14 @@ html {
 p > a {
 	line-break: anywhere;
 }
+
+/* disable rubber bounce on desktop but keep it enabled for mobile devices */
+html {
+	overscroll-behavior-y: none;
+}
+
+@media (any-pointer: coarse) {
+	html {
+		overscroll-behavior-y: auto;
+	}
+}
-- 
2.51.2


From 98a332cab053b8ff5b99fe73babaa23ea051ca73 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Tue, 6 Oct 2026 20:23:39 +0300
Subject: [PATCH 08/25] web: load the migrate page's knots from preferences

Signed-off-by: dawn <dawn@tangled.org>
---
 web/src/lib/api/repoCreationOptions.ts      |  24 ----
 web/src/lib/api/repoCreationTargets.test.ts | 109 -----------------
 web/src/lib/api/repoCreationTargets.ts      | 128 --------------------
 web/src/routes/repo/migrate/+page.svelte    |  21 +++-
 web/src/routes/repo/migrate/+page.ts        |   5 -
 web/src/routes/repo/new/+page.ts            |   4 +-
 web/src/routes/welcome/+page.ts             |   4 -
 7 files changed, 20 insertions(+), 275 deletions(-)
 delete mode 100644 web/src/lib/api/repoCreationOptions.ts
 delete mode 100644 web/src/lib/api/repoCreationTargets.test.ts
 delete mode 100644 web/src/lib/api/repoCreationTargets.ts
 delete mode 100644 web/src/routes/repo/migrate/+page.ts

diff --git a/web/src/lib/api/repoCreationOptions.ts b/web/src/lib/api/repoCreationOptions.ts
deleted file mode 100644
index f99557fa3..000000000
--- a/web/src/lib/api/repoCreationOptions.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-import type { Did } from "@atcute/lexicons/syntax";
-import { createBobbinClient } from "$lib/api/client";
-import { stream } from "$lib/api/load";
-import { availableKnots, availableSpindles } from "$lib/api/repoCreationTargets";
-
-interface RepoCreationParent {
-	auth?: { did: string } | null;
-	publicConfig: { bobbinUrl: string };
-}
-
-export const repoCreationOptions = (
-	parent: RepoCreationParent,
-	fetch: typeof globalThis.fetch
-): { knots: Promise<string[]>; spindles: Promise<string[]> } => {
-	if (!parent.auth?.did) {
-		return { knots: Promise.resolve([]), spindles: Promise.resolve([]) };
-	}
-	const ctx = createBobbinClient({ serviceUrl: parent.publicConfig.bobbinUrl, fetch });
-	const did = parent.auth.did as Did;
-	return {
-		knots: stream(availableKnots(ctx, did).catch(() => [])),
-		spindles: stream(availableSpindles(ctx, did).catch(() => []))
-	};
-};
diff --git a/web/src/lib/api/repoCreationTargets.test.ts b/web/src/lib/api/repoCreationTargets.test.ts
deleted file mode 100644
index 8cff13a3d..000000000
--- a/web/src/lib/api/repoCreationTargets.test.ts
+++ /dev/null
@@ -1,109 +0,0 @@
-import { describe, expect, it, vi } from "vitest";
-import type { Did } from "@atcute/lexicons/syntax";
-import { createBobbinClient, type BobbinContext } from "$lib/api/client";
-import { availableKnots, availableSpindles } from "$lib/api/repoCreationTargets";
-
-const did = "did:plc:alice" as Did;
-
-describe("availableSpindles", () => {
-	it("merges owned spindles and memberships across pages", async () => {
-		const responses = [
-			{
-				items: [
-					{
-						uri: "at://did:plc:alice/sh.tangled.spindle/owned.example",
-						value: {}
-					}
-				],
-				cursor: "next"
-			},
-			{
-				items: [
-					{
-						uri: "at://did:plc:owner/sh.tangled.spindle.member/one",
-						value: { instance: "member.example" }
-					},
-					{
-						uri: "at://did:plc:owner/sh.tangled.spindle.member/two",
-						value: { instance: "owned.example" }
-					},
-					{ uri: "at://did:plc:owner/sh.tangled.spindle.member/bad", value: {} }
-				]
-			},
-			{
-				items: [
-					{
-						uri: "at://did:plc:alice/sh.tangled.spindle/another.example",
-						value: {}
-					}
-				]
-			}
-		];
-		const call = vi.fn(async (_schema: unknown, _options: unknown) => ({
-			ok: true,
-			data: responses.shift()
-		}));
-		const ctx = { xrpc: { call } } as unknown as BobbinContext;
-
-		await expect(availableSpindles(ctx, did)).resolves.toEqual([
-			"another.example",
-			"member.example",
-			"owned.example"
-		]);
-		expect(call).toHaveBeenCalledTimes(3);
-		expect(call.mock.calls[0][1]).toMatchObject({ params: { subject: did, limit: 1_000 } });
-		expect(call.mock.calls[2][1]).toMatchObject({ params: { cursor: "next" } });
-	});
-});
-
-describe("availableKnots", () => {
-	it("merges owned knots and memberships", async () => {
-		const pages: Record<string, unknown> = {
-			"/xrpc/sh.tangled.knot.listKnots": {
-				items: [{ uri: "at://did:plc:alice/sh.tangled.knot/owned.example", value: {} }]
-			},
-			"/xrpc/sh.tangled.knot.listMembers": {
-				items: [
-					{
-						uri: "at://did:plc:owner/sh.tangled.knot.member/one",
-						value: { domain: "member.example" }
-					},
-					{
-						uri: "at://did:web:member.example/sh.tangled.bobbin.knotMember/did:plc:alice",
-						value: {
-							subject: "did:plc:alice",
-							addedBy: "did:plc:owner",
-							createdAt: "2026-06-01T00:00:00.000Z"
-						}
-					},
-					{
-						uri: "at://did:plc:owner/sh.tangled.knot.member/two",
-						value: { domain: "owned.example" }
-					},
-					{ uri: "at://did:plc:owner/sh.tangled.knot.member/bad", value: {} }
-				]
-			}
-		};
-		const fetch = vi.fn<typeof globalThis.fetch>(async (input) => {
-			const asked = new URL(String(input));
-			const page = pages[asked.pathname];
-			return new Response(JSON.stringify(page ?? { error: "MethodNotImplemented" }), {
-				status: page ? 200 : 501,
-				headers: { "content-type": "application/json" }
-			});
-		});
-		const ctx = createBobbinClient({ serviceUrl: "https://bobbin.oyster.cafe", fetch });
-
-		await expect(availableKnots(ctx, did)).resolves.toEqual([
-			"member.example",
-			"owned.example"
-		]);
-		expect(fetch.mock.calls.map(([input]) => new URL(String(input)).pathname)).toEqual([
-			"/xrpc/sh.tangled.knot.listKnots",
-			"/xrpc/sh.tangled.knot.listMembers"
-		]);
-		expect(String(fetch.mock.calls[1][0])).toBe(
-			"https://bobbin.oyster.cafe/xrpc/sh.tangled.knot.listMembers?subject=did%3Aplc%3Aalice&limit=1000"
-		);
-	});
-});
diff --git a/web/src/lib/api/repoCreationTargets.ts b/web/src/lib/api/repoCreationTargets.ts
deleted file mode 100644
index 0e5207d02..000000000
--- a/web/src/lib/api/repoCreationTargets.ts
+++ /dev/null
@@ -1,128 +0,0 @@
-import { ok } from "@atcute/client";
-import type { Did } from "@atcute/lexicons/syntax";
-import type { BobbinContext } from "$lib/api/client";
-import { jsonGet } from "$lib/api/_request";
-import { mainSchema as listKnotsSchema } from "$lib/api/lexicons/types/sh/tangled/knot/listKnots";
-import { mainSchema as listSpindleMembersSchema } from "$lib/api/lexicons/types/sh/tangled/spindle/listMembers";
-import { mainSchema as listSpindlesSchema } from "$lib/api/lexicons/types/sh/tangled/spindle/listSpindles";
-import { parseResourceUri } from "@atcute/lexicons/syntax";
-import { hostForServiceDid } from "$lib/auth/agent";
-
-interface Page<T> {
-	items: T[];
-	cursor?: string;
-}
-
-interface RecordItem {
-	uri: string;
-	value: unknown;
-}
-
-const allPages = async <T>(load: (cursor?: string) => Promise<Page<T>>): Promise<T[]> => {
-	const items: T[] = [];
-	let cursor: string | undefined;
-	do {
-		const page = await load(cursor);
-		items.push(...page.items);
-		cursor = page.cursor || undefined;
-	} while (cursor);
-	return items;
-};
-
-const knotDomain = (value: unknown): string | null => {
-	if (typeof value !== "object" || value === null || !("domain" in value)) return null;
-	return typeof value.domain === "string" && value.domain ? value.domain : null;
-};
-
-const repoFromUri = (uri: unknown): string | null => {
-	if (typeof uri !== "string") return null;
-	try {
-		return parseResourceUri(uri).repo;
-	} catch {
-		return null;
-	}
-};
-
-const recordName = (uri: unknown): string | null => {
-	if (typeof uri !== "string") return null;
-	try {
-		return parseResourceUri(uri).rkey ?? null;
-	} catch {
-		return null;
-	}
-};
-
-const knotMembershipHost = (item: unknown): string | null => {
-	if (typeof item !== "object" || item === null) return null;
-	const record = item as { uri?: unknown; value?: unknown };
-	const repo = repoFromUri(record.uri);
-	const host = repo ? hostForServiceDid(repo) : null;
-	if (host) return host;
-	if (record.value !== undefined) {
-		const domain = knotDomain(record.value);
-		if (domain) return domain;
-	}
-	return null;
-};
-
-const spindleInstance = (value: unknown): string | null => {
-	if (typeof value !== "object" || value === null || !("instance" in value)) return null;
-	return typeof value.instance === "string" && value.instance ? value.instance : null;
-};
-
-const targetNames = (
-	owned: RecordItem[],
-	memberships: RecordItem[],
-	memberName: (item: RecordItem) => string | null
-): string[] => {
-	const names = new Set<string>();
-	for (const item of owned) {
-		const name = recordName(item.uri);
-		if (name) names.add(name);
-	}
-	for (const item of memberships) {
-		const name = memberName(item);
-		if (name) names.add(name);
-	}
-	return [...names].sort((a, b) => a.localeCompare(b));
-};
-
-export const availableKnots = async (ctx: BobbinContext, did: string): Promise<string[]> => {
-	const [owned, memberships] = await Promise.all([
-		allPages((cursor) =>
-			ok(
-				ctx.xrpc.call(listKnotsSchema, {
-					params: { subject: did as Did, limit: 1_000, cursor }
-				})
-			)
-		),
-		allPages((cursor) =>
-			jsonGet<Page<RecordItem>>(ctx, "sh.tangled.knot.listMembers", {
-				subject: did,
-				limit: 1_000,
-				cursor
-			})
-		)
-	]);
-	return targetNames(owned, memberships, knotMembershipHost);
-};
-
-export const availableSpindles = async (ctx: BobbinContext, did: string): Promise<string[]> => {
-	const [owned, memberships] = await Promise.all([
-		allPages((cursor) =>
-			ok(
-				ctx.xrpc.call(listSpindlesSchema, {
-					params: { subject: did as Did, limit: 1_000, cursor }
-				})
-			)
-		),
-		allPages((cursor) =>
-			ok(
-				ctx.xrpc.call(listSpindleMembersSchema, {
-					params: { subject: did as Did, limit: 1_000, cursor }
-				})
-			)
-		)
-	]);
-	return targetNames(owned, memberships, (item) => spindleInstance(item.value));
-};
diff --git a/web/src/routes/repo/migrate/+page.svelte b/web/src/routes/repo/migrate/+page.svelte
index 209f13843..9c661c308 100644
--- a/web/src/routes/repo/migrate/+page.svelte
+++ b/web/src/routes/repo/migrate/+page.svelte
@@ -11,7 +11,9 @@
 	} from "$lib/components/import/UrlRepoImporter.svelte";
 	import Card from "$lib/components/ui/Card.svelte";
 	import { getAuth } from "$lib/auth.svelte";
+	import { createBobbinClient } from "$lib/api/client";
 	import { migrationRunning } from "$lib/api/migrator";
+	import { getPreferences } from "$lib/api/preferences";
 	import type { PageData } from "./$types";
 
 	let { data }: { data: PageData } = $props();
@@ -21,6 +23,21 @@
 	const auth = getAuth();
 	const user = $derived(auth.currentUser);
 
+	// see also, new repo form
+	const knots = $derived.by(async () => {
+		const did = user?.did;
+		const saved = did
+			? await auth
+					.agentFor(did, { relogin: false })
+					.then((agent) =>
+						getPreferences(createBobbinClient({ serviceUrl: auth.bobbinUrl, agent }))
+					)
+					.then((preferences) => preferences.knots)
+					.catch(() => [])
+			: [];
+		return [...new Set([data.publicConfig.primaryKnotHostname, ...saved].filter(Boolean))];
+	});
+
 	const urlRunning = $derived(
 		urlMigration.status === "submitting" ||
 			(urlMigration.status === "success" && migrationRunning(urlMigration.jobStatus))
@@ -60,7 +77,7 @@
 	<Card padding="none" class="p-6">
 		{#if source === "github"}
 			<GitHubImporter
-						knots={data.knots}
+				{knots}
 				migratorUrl={data.publicConfig.migratorUrl}
 				method={methodStep}
 			/>
@@ -69,7 +86,7 @@
 				{@render methodStep()}
 				<Step number={2} title="Details">
 					<UrlRepoImporter
-						knots={data.knots}
+						{knots}
 						migratorUrl={data.publicConfig.migratorUrl}
 						bind:migration={urlMigration}
 					/>
diff --git a/web/src/routes/repo/migrate/+page.ts b/web/src/routes/repo/migrate/+page.ts
deleted file mode 100644
index c045bc7be..000000000
--- a/web/src/routes/repo/migrate/+page.ts
+++ /dev/null
@@ -1,5 +0,0 @@
-import { repoCreationOptions } from "$lib/api/repoCreationOptions";
-import type { PageLoad } from "./$types";
-
-export const load: PageLoad = async (event) =>
-	repoCreationOptions(await event.parent(), event.fetch);
diff --git a/web/src/routes/repo/new/+page.ts b/web/src/routes/repo/new/+page.ts
index 9fb93ef81..1866a6edb 100644
--- a/web/src/routes/repo/new/+page.ts
+++ b/web/src/routes/repo/new/+page.ts
@@ -1,8 +1,6 @@
 import { redirect } from "@sveltejs/kit";
-import { repoCreationOptions } from "$lib/api/repoCreationOptions";
 import type { PageLoad } from "./$types";
 
-export const load: PageLoad = async (event) => {
+export const load: PageLoad = (event) => {
 	if (event.url.searchParams.has("import")) redirect(308, "/repo/migrate");
-	return repoCreationOptions(await event.parent(), event.fetch);
 };
diff --git a/web/src/routes/welcome/+page.ts b/web/src/routes/welcome/+page.ts
index 028187ed9..bb8b4b23b 100644
--- a/web/src/routes/welcome/+page.ts
+++ b/web/src/routes/welcome/+page.ts
@@ -5,7 +5,6 @@ import { viewerRkeyOf } from "$lib/api/enrich";
 import { getTrendingWithViewerState } from "$lib/api/profile";
 import { httpStatusFor, toHttpError } from "$lib/api/load";
 import { getProfile } from "$lib/api/records";
-import { repoCreationOptions } from "$lib/api/repoCreationOptions";
 import { listPubKeys, sshFingerprint } from "$lib/api/settings";
 import { toPerson, toRepoCard } from "$lib/components/timeline/mappers";
 import type { PageLoad } from "./$types";
@@ -24,8 +23,6 @@ export const load: PageLoad = async (event) => {
 		throw cause;
 	});
 
-	const knots = repoCreationOptions(parent, event.fetch).knots;
-
 	try {
 		const [keys, enrichedTrending] = await Promise.all([
 			listPubKeys(ctx, viewer),
@@ -43,7 +40,6 @@ export const load: PageLoad = async (event) => {
 
 		return {
 			profile,
-			knots,
 			keys: welcomeKeys,
 			people: trending.people
 				.filter((profile) => profile.did !== viewer)
-- 
2.51.2


From 14c66d590ec9367d74ce0a9e5051750ad7c610f9 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Tue, 6 Oct 2026 20:54:03 +0300
Subject: [PATCH 09/25] 
 lexicons/{ci,org/tangled/ci},spindle/{engines,engine,models,xrpc}: put step
 outcomes in its log end line

Signed-off-by: dawn <dawn@tangled.org>
---
 api/org_tangled/cbor_gen.go                   | 59 ++++++++++++++++++-
 api/org_tangled/cisubscribePipelineLogs.go    |  2 +
 api/tangled/cbor_gen.go                       | 59 ++++++++++++++++++-
 api/tangled/cisubscribePipelineLogs.go        |  2 +
 lexicons/ci/subscribePipelineLogs.json        |  5 ++
 .../org/tangled/ci/subscribePipelineLogs.json |  8 +++
 spindle/engine/engine.go                      |  2 +-
 spindle/engine/engine_test.go                 | 54 +++++++++++++++++
 spindle/engines/microvm/engine.go             |  4 +-
 spindle/engines/nixery/engine.go              |  4 +-
 spindle/models/logger.go                      | 16 +++++
 spindle/models/models.go                      | 21 ++++++-
 spindle/xrpc/ci_pipeline_subscribe_logs.go    |  1 +
 13 files changed, 229 insertions(+), 8 deletions(-)

diff --git a/api/org_tangled/cbor_gen.go b/api/org_tangled/cbor_gen.go
index 3579d802c..97f8fabe0 100644
--- a/api/org_tangled/cbor_gen.go
+++ b/api/org_tangled/cbor_gen.go
@@ -25,7 +25,7 @@ func (t *CiSubscribePipelineLogs_Control) MarshalCBOR(w io.Writer) error {
 	}
 
 	cw := cbg.NewCborWriter(w)
-	fieldCount := 7
+	fieldCount := 8
 
 	if t.Command == nil {
 		fieldCount--
@@ -35,6 +35,10 @@ func (t *CiSubscribePipelineLogs_Control) MarshalCBOR(w io.Writer) error {
 		fieldCount--
 	}
 
+	if t.Outcome == nil {
+		fieldCount--
+	}
+
 	if t.Status == nil {
 		fieldCount--
 	}
@@ -207,6 +211,38 @@ func (t *CiSubscribePipelineLogs_Control) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
+	// t.Outcome (string) (string)
+	if t.Outcome != nil {
+
+		if len("outcome") > 1000000 {
+			return xerrors.Errorf("Value in field \"outcome\" was too long")
+		}
+
+		if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("outcome"))); err != nil {
+			return err
+		}
+		if _, err := cw.WriteString(string("outcome")); err != nil {
+			return err
+		}
+
+		if t.Outcome == nil {
+			if _, err := cw.Write(cbg.CborNull); err != nil {
+				return err
+			}
+		} else {
+			if len(*t.Outcome) > 1000000 {
+				return xerrors.Errorf("Value in field t.Outcome was too long")
+			}
+
+			if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Outcome))); err != nil {
+				return err
+			}
+			if _, err := cw.WriteString(string(*t.Outcome)); err != nil {
+				return err
+			}
+		}
+	}
+
 	// t.Workflow (string) (string)
 	if len("workflow") > 1000000 {
 		return xerrors.Errorf("Value in field \"workflow\" was too long")
@@ -384,6 +420,27 @@ func (t *CiSubscribePipelineLogs_Control) UnmarshalCBOR(r io.Reader) (err error)
 
 				t.Content = string(sval)
 			}
+			// t.Outcome (string) (string)
+		case "outcome":
+
+			{
+				b, err := cr.ReadByte()
+				if err != nil {
+					return err
+				}
+				if b != cbg.CborNull[0] {
+					if err := cr.UnreadByte(); err != nil {
+						return err
+					}
+
+					sval, err := cbg.ReadStringWithMax(cr, 1000000)
+					if err != nil {
+						return err
+					}
+
+					t.Outcome = (*string)(&sval)
+				}
+			}
 			// t.Workflow (string) (string)
 		case "workflow":
 
diff --git a/api/org_tangled/cisubscribePipelineLogs.go b/api/org_tangled/cisubscribePipelineLogs.go
index 9366ab347..fe8058ced 100644
--- a/api/org_tangled/cisubscribePipelineLogs.go
+++ b/api/org_tangled/cisubscribePipelineLogs.go
@@ -15,6 +15,8 @@ type CiSubscribePipelineLogs_Control struct {
 	Content string  `json:"content" cborgen:"content"`
 	// kind: Step kind
 	Kind *string `json:"kind,omitempty" cborgen:"kind,omitempty"`
+	// outcome: How the step ended, only set on end frames for steps that can fail
+	Outcome *string `json:"outcome,omitempty" cborgen:"outcome,omitempty"`
 	// status: Step status
 	Status *string `json:"status,omitempty" cborgen:"status,omitempty"`
 	// step: Step ID
diff --git a/api/tangled/cbor_gen.go b/api/tangled/cbor_gen.go
index f33f80ae2..c4d14a8f7 100644
--- a/api/tangled/cbor_gen.go
+++ b/api/tangled/cbor_gen.go
@@ -1675,7 +1675,7 @@ func (t *CiSubscribePipelineLogs_Control) MarshalCBOR(w io.Writer) error {
 	}
 
 	cw := cbg.NewCborWriter(w)
-	fieldCount := 7
+	fieldCount := 8
 
 	if t.Command == nil {
 		fieldCount--
@@ -1685,6 +1685,10 @@ func (t *CiSubscribePipelineLogs_Control) MarshalCBOR(w io.Writer) error {
 		fieldCount--
 	}
 
+	if t.Outcome == nil {
+		fieldCount--
+	}
+
 	if t.Status == nil {
 		fieldCount--
 	}
@@ -1857,6 +1861,38 @@ func (t *CiSubscribePipelineLogs_Control) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
+	// t.Outcome (string) (string)
+	if t.Outcome != nil {
+
+		if len("outcome") > 1000000 {
+			return xerrors.Errorf("Value in field \"outcome\" was too long")
+		}
+
+		if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("outcome"))); err != nil {
+			return err
+		}
+		if _, err := cw.WriteString(string("outcome")); err != nil {
+			return err
+		}
+
+		if t.Outcome == nil {
+			if _, err := cw.Write(cbg.CborNull); err != nil {
+				return err
+			}
+		} else {
+			if len(*t.Outcome) > 1000000 {
+				return xerrors.Errorf("Value in field t.Outcome was too long")
+			}
+
+			if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Outcome))); err != nil {
+				return err
+			}
+			if _, err := cw.WriteString(string(*t.Outcome)); err != nil {
+				return err
+			}
+		}
+	}
+
 	// t.Workflow (string) (string)
 	if len("workflow") > 1000000 {
 		return xerrors.Errorf("Value in field \"workflow\" was too long")
@@ -2034,6 +2070,27 @@ func (t *CiSubscribePipelineLogs_Control) UnmarshalCBOR(r io.Reader) (err error)
 
 				t.Content = string(sval)
 			}
+			// t.Outcome (string) (string)
+		case "outcome":
+
+			{
+				b, err := cr.ReadByte()
+				if err != nil {
+					return err
+				}
+				if b != cbg.CborNull[0] {
+					if err := cr.UnreadByte(); err != nil {
+						return err
+					}
+
+					sval, err := cbg.ReadStringWithMax(cr, 1000000)
+					if err != nil {
+						return err
+					}
+
+					t.Outcome = (*string)(&sval)
+				}
+			}
 			// t.Workflow (string) (string)
 		case "workflow":
 
diff --git a/api/tangled/cisubscribePipelineLogs.go b/api/tangled/cisubscribePipelineLogs.go
index 2955d03d6..8a1372399 100644
--- a/api/tangled/cisubscribePipelineLogs.go
+++ b/api/tangled/cisubscribePipelineLogs.go
@@ -15,6 +15,8 @@ type CiSubscribePipelineLogs_Control struct {
 	Content string  `json:"content" cborgen:"content"`
 	// kind: Step kind
 	Kind *string `json:"kind,omitempty" cborgen:"kind,omitempty"`
+	// outcome: How the step ended, only set on end frames for steps that can fail
+	Outcome *string `json:"outcome,omitempty" cborgen:"outcome,omitempty"`
 	// status: Step status
 	Status *string `json:"status,omitempty" cborgen:"status,omitempty"`
 	// step: Step ID
diff --git a/lexicons/ci/subscribePipelineLogs.json b/lexicons/ci/subscribePipelineLogs.json
index 4989b323e..2a9f33ad9 100644
--- a/lexicons/ci/subscribePipelineLogs.json
+++ b/lexicons/ci/subscribePipelineLogs.json
@@ -58,6 +58,11 @@
           "type": "string",
           "enum": ["system", "user"],
           "description": "Step kind"
+        },
+        "outcome": {
+          "type": "string",
+          "enum": ["success", "failed"],
+          "description": "How the step ended, only set on end frames for steps that can fail"
         }
       }
     },
diff --git a/lexicons/org/tangled/ci/subscribePipelineLogs.json b/lexicons/org/tangled/ci/subscribePipelineLogs.json
index fa80f43d9..4dd9d3a42 100644
--- a/lexicons/org/tangled/ci/subscribePipelineLogs.json
+++ b/lexicons/org/tangled/ci/subscribePipelineLogs.json
@@ -92,6 +92,14 @@
             "user"
           ],
           "description": "Step kind"
+        },
+        "outcome": {
+          "type": "string",
+          "enum": [
+            "success",
+            "failed"
+          ],
+          "description": "How the step ended, only set on end frames for steps that can fail"
         }
       }
     },
diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go
index 039772c7a..9811ad536 100644
--- a/spindle/engine/engine.go
+++ b/spindle/engine/engine.go
@@ -671,7 +671,7 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q
 
 					if wfLogger != nil {
 						wfLogger.
-							ControlWriter(stepIdx, step, models.StepStatusEnd).
+							EndWriter(stepIdx, step, models.OutcomeOf(err)).
 							Write([]byte{0})
 					}
 
diff --git a/spindle/engine/engine_test.go b/spindle/engine/engine_test.go
index d60c61c0d..4e71ca1ee 100644
--- a/spindle/engine/engine_test.go
+++ b/spindle/engine/engine_test.go
@@ -2,6 +2,7 @@ package engine
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"log/slog"
 	"os"
@@ -418,6 +419,59 @@ func TestStartWorkflows_FlushesLogBeforeTerminalStatus(t *testing.T) {
 	}
 }
 
+func TestStartWorkflows_EndLinesCarryStepOutcome(t *testing.T) {
+	t.Parallel()
+
+	testDB := newTestDB(t)
+	logger := slog.New(slog.NewTextHandler(os.Stderr, nil))
+	logDir := t.TempDir()
+
+	eng := &mockEngine{
+		runStepFunc: func(ctx context.Context, wid models.WorkflowId, idx int, wfLogger models.WorkflowLogger) error {
+			if idx == 1 {
+				hint := mockStep{name: "Debug shell"}
+				wfLogger.ControlWriter(2, hint, models.StepStatusStart).Write([]byte{0})
+				wfLogger.ControlWriter(2, hint, models.StepStatusEnd).Write([]byte{0})
+				return errors.New("exited with code 1")
+			}
+			return nil
+		},
+	}
+
+	pipelineId := models.PipelineId("outcome-rkey")
+	wid := models.WorkflowId{PipelineId: pipelineId, Name: "outcome"}
+	pipeline := &models.Pipeline{
+		Workflows: map[models.Engine][]models.Workflow{
+			eng: {{Name: wid.Name, Steps: []models.Step{mockStep{name: "build"}, mockStep{name: "test"}}}},
+		},
+	}
+	cfg := &config.Config{Server: config.Server{LogDir: logDir}}
+	StartWorkflows(logger, emptyVault{}, cfg, nil, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId)
+
+	raw, err := os.ReadFile(models.LogFilePath(logDir, wid))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := map[string]models.StepOutcome{}
+	for _, encoded := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
+		var line models.LogLine
+		if err := json.Unmarshal([]byte(encoded), &line); err != nil {
+			t.Fatalf("decode log line %q: %v", encoded, err)
+		}
+		if line.Kind == models.LogKindControl && line.StepStatus == models.StepStatusEnd {
+			got[line.Content] = line.StepOutcome
+		}
+	}
+	want := map[string]models.StepOutcome{
+		"build":       models.StepOutcomeSuccess,
+		"test":        models.StepOutcomeFailed,
+		"Debug shell": "",
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("end line outcomes = %v, want %v", got, want)
+	}
+}
+
 func TestStartWorkflowsCapsTheWorkflowTimeBudget(t *testing.T) {
 	deadline := make(chan time.Time, 1)
 	eng := &mockEngine{
diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go
index 4271ffbda..0c7dbd1d1 100644
--- a/spindle/engines/microvm/engine.go
+++ b/spindle/engines/microvm/engine.go
@@ -322,7 +322,9 @@ func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *m
 	setupStep := Step{name: "microVM setup", kind: models.StepKindSystem}
 
 	wfLogger.ControlWriter(-1, setupStep, models.StepStatusStart).Write([]byte{0})
-	defer wfLogger.ControlWriter(-1, setupStep, models.StepStatusEnd).Write([]byte{0})
+	defer func() {
+		wfLogger.EndWriter(-1, setupStep, models.OutcomeOf(err)).Write([]byte{0})
+	}()
 
 	category := "Failed to setup VM"
 	defer func() {
diff --git a/spindle/engines/nixery/engine.go b/spindle/engines/nixery/engine.go
index 248706984..72c5b51c1 100644
--- a/spindle/engines/nixery/engine.go
+++ b/spindle/engines/nixery/engine.go
@@ -262,7 +262,9 @@ func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *m
 	setupStepIdx := -1
 
 	wfLogger.ControlWriter(setupStepIdx, setupStep, models.StepStatusStart).Write([]byte{0})
-	defer wfLogger.ControlWriter(setupStepIdx, setupStep, models.StepStatusEnd).Write([]byte{0})
+	defer func() {
+		wfLogger.EndWriter(setupStepIdx, setupStep, models.OutcomeOf(err)).Write([]byte{0})
+	}()
 
 	defer func() {
 		if err != nil {
diff --git a/spindle/models/logger.go b/spindle/models/logger.go
index 036c56e1d..28f65baea 100644
--- a/spindle/models/logger.go
+++ b/spindle/models/logger.go
@@ -14,6 +14,7 @@ type WorkflowLogger interface {
 	Close() error
 	DataWriter(idx int, stream string) io.Writer
 	ControlWriter(idx int, step Step, stepStatus StepStatus) io.Writer
+	EndWriter(idx int, step Step, outcome StepOutcome) io.Writer
 }
 
 type NullLogger struct{}
@@ -23,6 +24,9 @@ func (l NullLogger) DataWriter(idx int, stream string) io.Writer { return io.Dis
 func (l NullLogger) ControlWriter(idx int, step Step, stepStatus StepStatus) io.Writer {
 	return io.Discard
 }
+func (l NullLogger) EndWriter(idx int, step Step, outcome StepOutcome) io.Writer {
+	return io.Discard
+}
 
 type FileWorkflowLogger struct {
 	file         *os.File
@@ -91,6 +95,16 @@ func (l *FileWorkflowLogger) ControlWriter(idx int, step Step, stepStatus StepSt
 	}
 }
 
+func (l *FileWorkflowLogger) EndWriter(idx int, step Step, outcome StepOutcome) io.Writer {
+	return &controlWriter{
+		logger:     l,
+		idx:        idx,
+		step:       step,
+		stepStatus: StepStatusEnd,
+		outcome:    outcome,
+	}
+}
+
 type dataWriter struct {
 	logger *FileWorkflowLogger
 	idx    int
@@ -154,10 +168,12 @@ type controlWriter struct {
 	idx        int
 	step       Step
 	stepStatus StepStatus
+	outcome    StepOutcome
 }
 
 func (w *controlWriter) Write(_ []byte) (int, error) {
 	entry := NewControlLogLine(w.idx, w.step, w.stepStatus)
+	entry.StepOutcome = w.outcome
 	if err := w.logger.encoder.Encode(entry); err != nil {
 		return 0, err
 	}
diff --git a/spindle/models/models.go b/spindle/models/models.go
index d5b27925f..ba6d5a33b 100644
--- a/spindle/models/models.go
+++ b/spindle/models/models.go
@@ -82,6 +82,20 @@ var (
 	StepStatusEnd   StepStatus = "end"
 )
 
+type StepOutcome string
+
+var (
+	StepOutcomeSuccess StepOutcome = "success"
+	StepOutcomeFailed  StepOutcome = "failed"
+)
+
+func OutcomeOf(err error) StepOutcome {
+	if err != nil {
+		return StepOutcomeFailed
+	}
+	return StepOutcomeSuccess
+}
+
 type LogLine struct {
 	Kind    LogKind   `json:"kind"`
 	Content string    `json:"content"`
@@ -92,9 +106,10 @@ type LogLine struct {
 	Stream string `json:"stream,omitempty"`
 
 	// fields if kind is "control"
-	StepStatus  StepStatus `json:"step_status,omitempty"`
-	StepKind    StepKind   `json:"step_kind,omitempty"`
-	StepCommand string     `json:"step_command,omitempty"`
+	StepStatus  StepStatus  `json:"step_status,omitempty"`
+	StepKind    StepKind    `json:"step_kind,omitempty"`
+	StepCommand string      `json:"step_command,omitempty"`
+	StepOutcome StepOutcome `json:"step_outcome,omitempty"`
 }
 
 func NewDataLogLine(idx int, content, stream string) LogLine {
diff --git a/spindle/xrpc/ci_pipeline_subscribe_logs.go b/spindle/xrpc/ci_pipeline_subscribe_logs.go
index 0ad7765f6..b86471560 100644
--- a/spindle/xrpc/ci_pipeline_subscribe_logs.go
+++ b/spindle/xrpc/ci_pipeline_subscribe_logs.go
@@ -215,6 +215,7 @@ func (x *Xrpc) handleSubscribeLogs(w http.ResponseWriter, r *http.Request, pipel
 							Command:  strptrOrNil(logLine.StepCommand),
 							Status:   strptrOrNil(string(logLine.StepStatus)),
 							Kind:     strptrOrNil(stepKindStr),
+							Outcome:  strptrOrNil(string(logLine.StepOutcome)),
 						}}
 					} else {
 						streamType := logLine.Stream
-- 
2.51.2


From 58ae5eb48e823ad5c7fd11ca64aef1295f57601b Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Tue, 6 Oct 2026 20:57:03 +0300
Subject: [PATCH 10/25] web/pipelines: color pipeline steps from the outcome
 the spindle reports

Signed-off-by: dawn <dawn@tangled.org>
---
 .../sh/tangled/ci/subscribePipelineLogs.ts    |  6 +++
 .../components/repo/pipelines/logs.test.ts    | 50 ++++++++++++++++++-
 web/src/lib/components/repo/pipelines/logs.ts | 17 ++++++-
 3 files changed, 70 insertions(+), 3 deletions(-)

diff --git a/web/src/lib/api/lexicons/types/sh/tangled/ci/subscribePipelineLogs.ts b/web/src/lib/api/lexicons/types/sh/tangled/ci/subscribePipelineLogs.ts
index 4b5723337..b7bba9d7f 100644
--- a/web/src/lib/api/lexicons/types/sh/tangled/ci/subscribePipelineLogs.ts
+++ b/web/src/lib/api/lexicons/types/sh/tangled/ci/subscribePipelineLogs.ts
@@ -17,6 +17,12 @@ const _controlSchema = /*#__PURE__*/ v.object({
   kind: /*#__PURE__*/ v.optional(
     /*#__PURE__*/ v.literalEnum(["system", "user"]),
   ),
+  /**
+   * How the step ended, only set on end frames for steps that can fail
+   */
+  outcome: /*#__PURE__*/ v.optional(
+    /*#__PURE__*/ v.literalEnum(["failed", "success"]),
+  ),
   /**
    * Step status
    */
diff --git a/web/src/lib/components/repo/pipelines/logs.test.ts b/web/src/lib/components/repo/pipelines/logs.test.ts
index 6e0551918..626dda1dd 100644
--- a/web/src/lib/components/repo/pipelines/logs.test.ts
+++ b/web/src/lib/components/repo/pipelines/logs.test.ts
@@ -21,9 +21,9 @@ const start = (step: number, name: string, over: Record<string, unknown> = {}):
 	}
 });
 
-const end = (step: number, time: string): LogFrame => ({
+const end = (step: number, time: string, outcome?: "success" | "failed"): LogFrame => ({
 	kind: "control",
-	control: { time, workflow: "build", step, content: "", status: "end" }
+	control: { time, workflow: "build", step, content: "", status: "end", outcome }
 });
 
 const line = (step: number, content: string): LogFrame => ({
@@ -194,6 +194,42 @@ describe("stepStatuses", () => {
 	it("returns an empty list when there are no steps", () => {
 		expect(stepStatuses([], "running")).toEqual([]);
 	});
+
+	const debugged = () =>
+		drain([
+			start(3, "build"),
+			end(3, "2026-08-06T12:00:10Z", "success"),
+			start(4, "deploy"),
+			start(6, "Debug shell", { kind: "system" }),
+			end(6, "2026-08-06T12:00:20Z"),
+			end(4, "2026-08-06T12:00:20Z", "failed")
+		]);
+
+	it("marks the step the spindle says failed, not the runner step after it", () => {
+		expect(stepStatuses(debugged(), "failed")).toEqual(["done", "error", "done"]);
+	});
+
+	it("shows a reported failure while the workflow still says running", () => {
+		expect(stepStatuses(debugged(), "running")).toEqual(["done", "error", "done"]);
+	});
+
+	it("keeps the open last step active when outcomes are reported", () => {
+		const steps = drain([
+			start(1, "One"),
+			end(1, "2026-08-06T12:00:10Z", "success"),
+			start(2, "Two")
+		]);
+		expect(stepStatuses(steps, "running")).toEqual(["done", "active"]);
+	});
+
+	it("blames an open last step when a reporting run ends badly without saying why", () => {
+		const steps = drain([
+			start(1, "One"),
+			end(1, "2026-08-06T12:00:10Z", "success"),
+			start(2, "Two")
+		]);
+		expect(stepStatuses(steps, "timeout")).toEqual(["done", "error"]);
+	});
 });
 
 describe("narrate", () => {
@@ -244,6 +280,16 @@ describe("narrate", () => {
 		expect(narrate(steps, "timeout")).toBe("Timed out at build the site");
 	});
 
+	it("says failed, not running, when a step failed before the workflow status caught up", () => {
+		const steps = drain([
+			start(1, "One"),
+			end(1, "2026-08-06T12:00:10Z", "success"),
+			start(2, "deploy"),
+			end(2, "2026-08-06T12:00:20Z", "failed")
+		]);
+		expect(narrate(steps, "running")).toBe("Failed at deploy");
+	});
+
 	it("announces completion once every step is done and the workflow succeeded", () => {
 		const steps = drain([
 			start(1, "One"),
diff --git a/web/src/lib/components/repo/pipelines/logs.ts b/web/src/lib/components/repo/pipelines/logs.ts
index 4a57e0cde..98b7a9f7f 100644
--- a/web/src/lib/components/repo/pipelines/logs.ts
+++ b/web/src/lib/components/repo/pipelines/logs.ts
@@ -13,6 +13,7 @@ export interface LogStep {
 	system: boolean;
 	startedAt: number;
 	finishedAt?: number;
+	outcome?: "success" | "failed";
 	// ansi resolved to html
 	lines: string[];
 }
@@ -63,6 +64,7 @@ export const createLogAccumulator = (): LogAccumulator => {
 				}
 				if (c.status === "end") {
 					step.finishedAt = stamp(c.time);
+					step.outcome = c.outcome;
 					return true;
 				}
 				return false;
@@ -129,6 +131,15 @@ export const stepStatuses = (steps: LogStep[], workflow: PipelineStatus): StepSt
 	if (steps.length === 0) return [];
 	const last = steps.length - 1;
 
+	if (steps.some((step) => step.outcome)) {
+		return steps.map((step, i) => {
+			if (step.outcome === "failed") return "error";
+			if (step.finishedAt || i !== last) return "done";
+			if (isLive(workflow)) return "active";
+			return workflow === "success" ? "done" : "error";
+		});
+	}
+
 	if (isLive(workflow)) {
 		return steps.map((step, i) => (step.finishedAt || i !== last ? "done" : "active"));
 	}
@@ -149,7 +160,11 @@ export const narrate = (steps: LogStep[], workflow: PipelineStatus): string => {
 	}
 
 	const errorIndex = statuses.indexOf("error");
-	if (errorIndex !== -1) return `${STATUS_LABELS[workflow]} at ${steps[errorIndex].name}`;
+	if (errorIndex !== -1) {
+		// a step can report its failure before the workflow status catches up
+		const label = isLive(workflow) ? STATUS_LABELS.failed : STATUS_LABELS[workflow];
+		return `${label} at ${steps[errorIndex].name}`;
+	}
 
 	if (workflow === "success" && statuses.every((status) => status === "done")) {
 		const step = steps.length === 1 ? "step" : "steps";
-- 
2.51.2


From 9e61610025349fa3ebc593ef07c4477e953ebf15 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Tue, 6 Oct 2026 20:58:43 +0300
Subject: [PATCH 11/25] spindle/mill: publish a workflow's result before its VM
 is torn down

Signed-off-by: dawn <dawn@tangled.org>
---
 spindle/db/events.go                          |   2 +-
 spindle/db/mill_state.go                      |  18 ++
 spindle/mill/executor/executor.go             |   5 +
 spindle/mill/executor/observe.go              |  51 +++-
 spindle/mill/executor/outbox.go               |  48 +++-
 spindle/mill/executor/reserved_test.go        |  92 +++++++
 spindle/mill/mill.go                          |  68 +++--
 spindle/mill/mill_test.go                     |  63 +++++
 spindle/mill/proto/gen/mill.pb.go             | 253 ++++++++++++------
 spindle/mill/proto/spindle/mill/v1/mill.proto |  15 ++
 spindle/wipe.go                               |  12 +-
 spindle/wipe_test.go                          |  33 +++
 12 files changed, 543 insertions(+), 117 deletions(-)

diff --git a/spindle/db/events.go b/spindle/db/events.go
index d6bb93467..dc03dd212 100644
--- a/spindle/db/events.go
+++ b/spindle/db/events.go
@@ -87,7 +87,7 @@ func (d *DB) CompleteMillLease(
 	n *notifier.Notifier,
 ) error {
 	return d.ApplyEventBatch(n, func(tx *EventBatchTx) error {
-		if err := tx.InsertStatusEvent(workflowId, status, workflowError, exitCode); err != nil {
+		if err := tx.InsertStatusChange(workflowId, status, workflowError, exitCode); err != nil {
 			return err
 		}
 		return tx.DeleteLease(leaseID)
diff --git a/spindle/db/mill_state.go b/spindle/db/mill_state.go
index 6b79fda62..fe599af0f 100644
--- a/spindle/db/mill_state.go
+++ b/spindle/db/mill_state.go
@@ -3,6 +3,7 @@ package db
 import (
 	"context"
 	"database/sql"
+	"errors"
 	"fmt"
 	"time"
 
@@ -280,6 +281,23 @@ func (tx *EventBatchTx) InsertStatusEvent(wid models.WorkflowId, status string,
 	return insertStatusTx(tx.tx, wid, models.StatusKind(status), workflowError, exitCode)
 }
 
+func (tx *EventBatchTx) InsertStatusChange(wid models.WorkflowId, status string, workflowError *string, exitCode *int64) error {
+	var latest string
+	err := tx.tx.QueryRow(
+		`select status from workflow_statuses
+		 where pipeline_id = ? and workflow = ?
+		 order by id desc limit 1`,
+		string(wid.PipelineId), wid.Name,
+	).Scan(&latest)
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return err
+	}
+	if latest == status {
+		return nil
+	}
+	return tx.InsertStatusEvent(wid, status, workflowError, exitCode)
+}
+
 func (tx *EventBatchTx) DeleteLease(leaseID string) error {
 	_, err := tx.tx.Exec(`delete from mill_leases where lease_id = ?`, leaseID)
 	return err
diff --git a/spindle/mill/executor/executor.go b/spindle/mill/executor/executor.go
index 5d75265c3..c9f63fe0d 100644
--- a/spindle/mill/executor/executor.go
+++ b/spindle/mill/executor/executor.go
@@ -87,6 +87,7 @@ type Executor struct {
 
 	lifecycleCtx context.Context
 	jobsWG       sync.WaitGroup
+	finishWG     sync.WaitGroup
 }
 
 type reservation struct {
@@ -109,6 +110,8 @@ type reservation struct {
 	failureClass               string
 	failureReason              string
 	millRecordsTerminalMetrics bool
+	millAcceptsOutcome         bool
+	reportedStatus             string
 }
 
 type messageEncoder interface {
@@ -209,6 +212,7 @@ reconnect:
 	stopObserver()
 	<-observerDone
 	e.drainEvents(&cursor)
+	e.finishWG.Wait()
 }
 
 func (e *Executor) runSession(ctx context.Context) error {
@@ -632,6 +636,7 @@ func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) {
 	res.wf.CacheBindings = bindings
 	res.committed = true
 	res.millRecordsTerminalMetrics = cl.GetMillRecordsTerminalMetrics()
+	res.millAcceptsOutcome = cl.GetAcceptsAttemptOutcome()
 	if res.ttlTimer != nil {
 		res.ttlTimer.Stop()
 	}
diff --git a/spindle/mill/executor/observe.go b/spindle/mill/executor/observe.go
index 0cee58173..a4256d327 100644
--- a/spindle/mill/executor/observe.go
+++ b/spindle/mill/executor/observe.go
@@ -57,12 +57,54 @@ func (e *Executor) onStatusRow(st *db.StatusRow) error {
 	}
 
 	if models.StatusKind(st.Status).IsFinish() {
-		return e.finishJob(res, st)
+		reported, err := e.reportOutcome(res, st)
+		if err != nil {
+			e.l.Error("report outcome failed", "lease", res.leaseID, "err", err)
+		}
+		if !reported {
+			return e.finishJob(res, st)
+		}
+		// a debug shell can hold the VM for minutes, and waiting here stalls every other job's status
+		e.finishWG.Add(1)
+		go func() {
+			defer e.finishWG.Done()
+			if err := e.finishJob(res, st); err != nil {
+				e.l.Error("finish job failed", "lease", res.leaseID, "err", err)
+			}
+		}()
+		return nil
 	}
 
 	return e.appendStatus(res.leaseID, st)
 }
 
+// false when the run already ended or the mill predates outcomes
+func (e *Executor) reportOutcome(res *reservation, st *db.StatusRow) (bool, error) {
+	e.mu.Lock()
+	if e.active[res.leaseID] != res || !res.millAcceptsOutcome || res.runDone == nil {
+		e.mu.Unlock()
+		return false, nil
+	}
+	select {
+	case <-res.runDone:
+		e.mu.Unlock()
+		return false, nil
+	default:
+	}
+	status := st.Status
+	if res.cancelled {
+		status = string(models.StatusKindCancelled)
+	}
+	res.reportedStatus = status
+	e.mu.Unlock()
+
+	// every log line has to reach the mill before the status does
+	if res.stopTail != nil {
+		res.stopTail()
+	}
+	return true, e.appendOutcome(res.leaseID, status, st)
+}
+
 func (e *Executor) finishJob(res *reservation, st *db.StatusRow) error {
 	e.mu.Lock()
 	if e.active[res.leaseID] != res {
@@ -70,6 +112,7 @@ func (e *Executor) finishJob(res *reservation, st *db.StatusRow) error {
 		return nil
 	}
 	cancelled := res.cancelled
+	reported := res.reportedStatus
 	runDone := res.runDone
 	e.mu.Unlock()
 
@@ -91,7 +134,11 @@ func (e *Executor) finishJob(res *reservation, st *db.StatusRow) error {
 	if cancelled {
 		terminalStatus = string(models.StatusKindCancelled)
 	}
-	if cancelled {
+	if reported != "" {
+		// the mill already published it, so a late cancel can't change it
+		terminalStatus = reported
+	}
+	if cancelled && terminalStatus == string(models.StatusKindCancelled) {
 		failureClass = string(engine.FailureClassUser)
 		failureReason = string(engine.FailureReasonCancelled)
 	}
diff --git a/spindle/mill/executor/outbox.go b/spindle/mill/executor/outbox.go
index 24e38828e..2c6e06ea4 100644
--- a/spindle/mill/executor/outbox.go
+++ b/spindle/mill/executor/outbox.go
@@ -74,6 +74,8 @@ func (e *Executor) appendAndSend(leaseID string, payload any, control bool) erro
 		entry.Payload = payload
 	case *millv1.Event_AttemptResult:
 		entry.Payload = payload
+	case *millv1.Event_AttemptOutcome:
+		entry.Payload = payload
 	case *millv1.Event_CacheUpdate:
 		entry.Payload = payload
 	default:
@@ -85,7 +87,7 @@ func (e *Executor) appendAndSend(leaseID string, payload any, control bool) erro
 	case *millv1.Event_AttemptResult:
 		durable = true
 		isTerminal = true
-	case *millv1.Event_CacheUpdate:
+	case *millv1.Event_AttemptOutcome, *millv1.Event_CacheUpdate:
 		durable = true
 	}
 	entry.Seqno = ^uint64(0)
@@ -186,24 +188,44 @@ func defaultTerminalAttribution(status string) (string, string) {
 	}
 }
 
-func (e *Executor) appendTerminalWithArtifact(
-	leaseID, status string,
-	st *db.StatusRow,
-	ref, hash, failureClass, failureReason string,
-	millRecordsTerminalMetrics bool,
-) error {
-	var terminalStatus millv1.TerminalStatus
+func terminalStatusProto(status string) (millv1.TerminalStatus, error) {
 	switch status {
 	case string(models.StatusKindSuccess):
-		terminalStatus = millv1.TerminalStatus_TERMINAL_STATUS_SUCCESS
+		return millv1.TerminalStatus_TERMINAL_STATUS_SUCCESS, nil
 	case string(models.StatusKindFailed):
-		terminalStatus = millv1.TerminalStatus_TERMINAL_STATUS_FAILED
+		return millv1.TerminalStatus_TERMINAL_STATUS_FAILED, nil
 	case string(models.StatusKindTimeout):
-		terminalStatus = millv1.TerminalStatus_TERMINAL_STATUS_TIMEOUT
+		return millv1.TerminalStatus_TERMINAL_STATUS_TIMEOUT, nil
 	case string(models.StatusKindCancelled):
-		terminalStatus = millv1.TerminalStatus_TERMINAL_STATUS_CANCELLED
+		return millv1.TerminalStatus_TERMINAL_STATUS_CANCELLED, nil
 	default:
-		return fmt.Errorf("unsupported terminal status %q", status)
+		return 0, fmt.Errorf("unsupported terminal status %q", status)
+	}
+}
+
+func (e *Executor) appendOutcome(leaseID, status string, st *db.StatusRow) error {
+	terminalStatus, err := terminalStatusProto(status)
+	if err != nil {
+		return err
+	}
+	exit, errStr := parseStatusExitAndError(st)
+	payload := &millv1.Event_AttemptOutcome{AttemptOutcome: &millv1.AttemptOutcome{
+		Status:   terminalStatus,
+		Error:    errStr,
+		ExitCode: exit,
+	}}
+	return e.appendAndSend(leaseID, payload, true)
+}
+
+func (e *Executor) appendTerminalWithArtifact(
+	leaseID, status string,
+	st *db.StatusRow,
+	ref, hash, failureClass, failureReason string,
+	millRecordsTerminalMetrics bool,
+) error {
+	terminalStatus, err := terminalStatusProto(status)
+	if err != nil {
+		return err
 	}
 
 	exit, errStr := parseStatusExitAndError(st)
diff --git a/spindle/mill/executor/reserved_test.go b/spindle/mill/executor/reserved_test.go
index f51367485..fdf16fc70 100644
--- a/spindle/mill/executor/reserved_test.go
+++ b/spindle/mill/executor/reserved_test.go
@@ -1032,6 +1032,98 @@ func TestFinishJobWaitsForEngineCleanup(t *testing.T) {
 	}
 }
 
+func outboxEvents(t *testing.T, d *db.DB) []*millv1.Event {
+	t.Helper()
+	rows, err := d.ListOutboxRows()
+	if err != nil {
+		t.Fatal(err)
+	}
+	events := make([]*millv1.Event, 0, len(rows))
+	for _, row := range rows {
+		var entry millv1.Event
+		if err := proto.Unmarshal(row.Payload, &entry); err != nil {
+			t.Fatal(err)
+		}
+		events = append(events, &entry)
+	}
+	return events
+}
+
+func TestFinishedStatusGoesOutBeforeEngineCleanup(t *testing.T) {
+	e := testExecutor(t)
+	res := &reservation{
+		leaseID:            "lease-1",
+		wid:                models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"},
+		runDone:            make(chan struct{}),
+		millAcceptsOutcome: true,
+	}
+	e.active[res.leaseID] = res
+
+	returned := make(chan error, 1)
+	go func() {
+		returned <- e.onStatusRow(&db.StatusRow{
+			Pipeline: string(res.wid.PipelineId),
+			Workflow: res.wid.Name,
+			Status:   string(models.StatusKindFailed),
+		})
+	}()
+	select {
+	case err := <-returned:
+		if err != nil {
+			t.Fatal(err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("the status loop waited on engine cleanup")
+	}
+
+	events := outboxEvents(t, e.db)
+	if len(events) != 1 || events[0].GetAttemptOutcome().GetStatus() != millv1.TerminalStatus_TERMINAL_STATUS_FAILED {
+		t.Fatalf("events before cleanup = %v, want one failed outcome", events)
+	}
+
+	e.mu.Lock()
+	res.cancelled = true
+	e.mu.Unlock()
+	close(res.runDone)
+	e.finishWG.Wait()
+
+	events = outboxEvents(t, e.db)
+	if len(events) != 2 || events[1].GetAttemptResult().GetStatus() != millv1.TerminalStatus_TERMINAL_STATUS_FAILED {
+		t.Fatalf("events after cleanup = %v, want the outcome then a failed result", events)
+	}
+}
+
+func TestOlderMillGetsNoOutcome(t *testing.T) {
+	e := testExecutor(t)
+	res := &reservation{
+		leaseID: "lease-1",
+		wid:     models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"},
+		runDone: make(chan struct{}),
+	}
+	e.active[res.leaseID] = res
+
+	returned := make(chan error, 1)
+	go func() {
+		returned <- e.onStatusRow(&db.StatusRow{
+			Pipeline: string(res.wid.PipelineId),
+			Workflow: res.wid.Name,
+			Status:   string(models.StatusKindFailed),
+		})
+	}()
+	time.Sleep(50 * time.Millisecond)
+	if events := outboxEvents(t, e.db); len(events) != 0 {
+		t.Fatalf("events before cleanup = %v, an older mill rejects outcomes", events)
+	}
+	close(res.runDone)
+	if err := <-returned; err != nil {
+		t.Fatal(err)
+	}
+	events := outboxEvents(t, e.db)
+	if len(events) != 1 || events[0].GetAttemptResult() == nil {
+		t.Fatalf("events = %v, want only the result", events)
+	}
+}
+
 func TestReplayRejectsMalformedOutboxRow(t *testing.T) {
 	d := testDB(t)
 	e := &Executor{
diff --git a/spindle/mill/mill.go b/spindle/mill/mill.go
index 176161cb7..7d0d02436 100644
--- a/spindle/mill/mill.go
+++ b/spindle/mill/mill.go
@@ -1013,6 +1013,7 @@ func (m *Mill) commitAndWait(ctx context.Context, wf *models.Workflow, unlocked
 		Tracestate:                 tracestate,
 		MillRecordsTerminalMetrics: lease.millRecordsTerminalMetrics,
 		CacheBindings:              cacheBindingsToProto(wf.CacheBindings),
+		AcceptsAttemptOutcome:      true,
 	}}
 
 	// commit retries ride reconnects, a reservation outlives one
@@ -1107,6 +1108,33 @@ func (m *Mill) waitCommitRetry(ctx context.Context, lease *RemoteLease) (bool, e
 	}
 }
 
+func terminalStatusString(status millv1.TerminalStatus) (string, error) {
+	switch status {
+	case millv1.TerminalStatus_TERMINAL_STATUS_SUCCESS:
+		return string(models.StatusKindSuccess), nil
+	case millv1.TerminalStatus_TERMINAL_STATUS_FAILED:
+		return string(models.StatusKindFailed), nil
+	case millv1.TerminalStatus_TERMINAL_STATUS_TIMEOUT:
+		return string(models.StatusKindTimeout), nil
+	case millv1.TerminalStatus_TERMINAL_STATUS_CANCELLED:
+		return string(models.StatusKindCancelled), nil
+	default:
+		return "", protoErrf("unsupported terminal status %v", status)
+	}
+}
+
+func statusDetails(errMsg string, exitCode int64) (*string, *int64) {
+	var e *string
+	if errMsg != "" {
+		e = &errMsg
+	}
+	var c *int64
+	if exitCode != 0 {
+		c = &exitCode
+	}
+	return e, c
+}
+
 func terminalError(status millv1.TerminalStatus) error {
 	switch status {
 	case millv1.TerminalStatus_TERMINAL_STATUS_SUCCESS:
@@ -1632,32 +1660,30 @@ func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error {
 					}
 				}
 
-			case entry.GetAttemptResult() != nil:
-				ar := entry.GetAttemptResult()
-				statusStr := "success"
-				switch ar.Status {
-				case millv1.TerminalStatus_TERMINAL_STATUS_SUCCESS:
-					statusStr = "success"
-				case millv1.TerminalStatus_TERMINAL_STATUS_FAILED:
-					statusStr = "failed"
-				case millv1.TerminalStatus_TERMINAL_STATUS_TIMEOUT:
-					statusStr = "timeout"
-				case millv1.TerminalStatus_TERMINAL_STATUS_CANCELLED:
-					statusStr = "cancelled"
-				default:
-					return batchApply{}, protoErrf("unsupported terminal status %v", ar.Status)
+			case entry.GetAttemptOutcome() != nil:
+				out := entry.GetAttemptOutcome()
+				statusStr, err := terminalStatusString(out.GetStatus())
+				if err != nil {
+					return batchApply{}, err
 				}
-				var errMsg *string
-				if e := ar.GetError(); e != "" {
-					errMsg = &e
+				if tx != nil {
+					errMsg, exitCode := statusDetails(out.GetError(), out.GetExitCode())
+					if err := tx.InsertStatusChange(lease.wid, statusStr, errMsg, exitCode); err != nil {
+						return batchApply{}, err
+					}
 				}
-				var exitCode *int64
-				if c := ar.GetExitCode(); c != 0 {
-					exitCode = &c
+
+			case entry.GetAttemptResult() != nil:
+				ar := entry.GetAttemptResult()
+				statusStr, err := terminalStatusString(ar.GetStatus())
+				if err != nil {
+					return batchApply{}, err
 				}
 				finishedInBatch[lease.id] = struct{}{}
 				if tx != nil {
-					if err := tx.InsertStatusEvent(lease.wid, statusStr, errMsg, exitCode); err != nil {
+					// an AttemptOutcome may have published this status already
+					errMsg, exitCode := statusDetails(ar.GetError(), ar.GetExitCode())
+					if err := tx.InsertStatusChange(lease.wid, statusStr, errMsg, exitCode); err != nil {
 						return batchApply{}, err
 					}
 					if err := tx.DeleteLease(lease.id); err != nil {
diff --git a/spindle/mill/mill_test.go b/spindle/mill/mill_test.go
index 2cb6aeb0b..c90d2a568 100644
--- a/spindle/mill/mill_test.go
+++ b/spindle/mill/mill_test.go
@@ -753,6 +753,69 @@ func TestAtomicBatchRollback(t *testing.T) {
 	}
 }
 
+func TestAttemptOutcomePublishesStatusBeforeTheLeaseEnds(t *testing.T) {
+	m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
+	sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger())
+	m.attachSession(sess)
+
+	owned := newLease("lease-1", "node-1", "inc-1", "dummy")
+	owned.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"}
+	m.mu.Lock()
+	m.leases[owned.id] = owned
+	m.mu.Unlock()
+
+	if err := m.onEventBatch(sess, &millv1.EventBatch{
+		Epoch: sess.epoch,
+		Events: []*millv1.Event{{
+			Seqno:   1,
+			LeaseId: owned.id,
+			Payload: &millv1.Event_AttemptOutcome{AttemptOutcome: &millv1.AttemptOutcome{
+				Status:   millv1.TerminalStatus_TERMINAL_STATUS_FAILED,
+				Error:    "User step error: exited with code 1",
+				ExitCode: 1,
+			}},
+		}},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if st, err := bdb.GetStatus(owned.wid); err != nil || st != models.StatusKindFailed {
+		t.Fatalf("status after outcome = %q (err %v), want failed", st, err)
+	}
+	if _, ok := pollTerminal(owned); ok {
+		t.Fatal("an outcome finished the lease, the VM is still held")
+	}
+	if owned.getState() == leaseDone {
+		t.Fatal("an outcome sealed the lease")
+	}
+
+	if err := m.onEventBatch(sess, &millv1.EventBatch{
+		Epoch: sess.epoch,
+		Events: []*millv1.Event{{
+			Seqno:   2,
+			LeaseId: owned.id,
+			Payload: &millv1.Event_AttemptResult{AttemptResult: &millv1.AttemptResult{
+				Status: millv1.TerminalStatus_TERMINAL_STATUS_FAILED,
+				Error:  "User step error: exited with code 1",
+			}},
+		}},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := pollTerminal(owned); !ok {
+		t.Fatal("the result after an outcome was not delivered")
+	}
+	var rows int
+	if err := bdb.QueryRow(
+		`select count(*) from workflow_statuses where pipeline_id = ? and workflow = ? and status = 'failed'`,
+		string(owned.wid.PipelineId), owned.wid.Name,
+	).Scan(&rows); err != nil {
+		t.Fatal(err)
+	}
+	if rows != 1 {
+		t.Fatalf("failed status rows = %d, want 1", rows)
+	}
+}
+
 func TestCacheUpdatesCommitWithStreamCursor(t *testing.T) {
 	m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
 	cache, err := storage.NewDisk(t.TempDir())
diff --git a/spindle/mill/proto/gen/mill.pb.go b/spindle/mill/proto/gen/mill.pb.go
index 2b035a406..4372c2608 100644
--- a/spindle/mill/proto/gen/mill.pb.go
+++ b/spindle/mill/proto/gen/mill.pb.go
@@ -927,8 +927,11 @@ type CommitLease struct {
 	Tracestate                 string                 `protobuf:"bytes,4,opt,name=tracestate,proto3" json:"tracestate,omitempty"`
 	MillRecordsTerminalMetrics bool                   `protobuf:"varint,5,opt,name=mill_records_terminal_metrics,json=millRecordsTerminalMetrics,proto3" json:"mill_records_terminal_metrics,omitempty"`
 	CacheBindings              []*CacheBinding        `protobuf:"bytes,6,rep,name=cache_bindings,json=cacheBindings,proto3" json:"cache_bindings,omitempty"`
-	unknownFields              protoimpl.UnknownFields
-	sizeCache                  protoimpl.SizeCache
+	// older mills drop the session on an unknown event payload, so executors only
+	// send AttemptOutcome when this is set
+	AcceptsAttemptOutcome bool `protobuf:"varint,7,opt,name=accepts_attempt_outcome,json=acceptsAttemptOutcome,proto3" json:"accepts_attempt_outcome,omitempty"`
+	unknownFields         protoimpl.UnknownFields
+	sizeCache             protoimpl.SizeCache
 }
 
 func (x *CommitLease) Reset() {
@@ -1003,6 +1006,13 @@ func (x *CommitLease) GetCacheBindings() []*CacheBinding {
 	return nil
 }
 
+func (x *CommitLease) GetAcceptsAttemptOutcome() bool {
+	if x != nil {
+		return x.AcceptsAttemptOutcome
+	}
+	return false
+}
+
 type Committed struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	LeaseId       string                 `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"`
@@ -1394,6 +1404,68 @@ func (x *AttemptResult) GetMillRecordsTerminalMetrics() bool {
 	return false
 }
 
+// the result is final but the executor still holds the VM, so the lease stays
+// charged until the AttemptResult
+type AttemptOutcome struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Status        TerminalStatus         `protobuf:"varint,1,opt,name=status,proto3,enum=spindle.mill.v1.TerminalStatus" json:"status,omitempty"`
+	Error         string                 `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
+	ExitCode      int64                  `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *AttemptOutcome) Reset() {
+	*x = AttemptOutcome{}
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[16]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *AttemptOutcome) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AttemptOutcome) ProtoMessage() {}
+
+func (x *AttemptOutcome) ProtoReflect() protoreflect.Message {
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[16]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use AttemptOutcome.ProtoReflect.Descriptor instead.
+func (*AttemptOutcome) Descriptor() ([]byte, []int) {
+	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16}
+}
+
+func (x *AttemptOutcome) GetStatus() TerminalStatus {
+	if x != nil {
+		return x.Status
+	}
+	return TerminalStatus_TERMINAL_STATUS_UNSPECIFIED
+}
+
+func (x *AttemptOutcome) GetError() string {
+	if x != nil {
+		return x.Error
+	}
+	return ""
+}
+
+func (x *AttemptOutcome) GetExitCode() int64 {
+	if x != nil {
+		return x.ExitCode
+	}
+	return 0
+}
+
 type CacheUpdate struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Action        CacheUpdateAction      `protobuf:"varint,1,opt,name=action,proto3,enum=spindle.mill.v1.CacheUpdateAction" json:"action,omitempty"`
@@ -1406,7 +1478,7 @@ type CacheUpdate struct {
 
 func (x *CacheUpdate) Reset() {
 	*x = CacheUpdate{}
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[16]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[17]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -1418,7 +1490,7 @@ func (x *CacheUpdate) String() string {
 func (*CacheUpdate) ProtoMessage() {}
 
 func (x *CacheUpdate) ProtoReflect() protoreflect.Message {
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[16]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[17]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -1431,7 +1503,7 @@ func (x *CacheUpdate) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use CacheUpdate.ProtoReflect.Descriptor instead.
 func (*CacheUpdate) Descriptor() ([]byte, []int) {
-	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16}
+	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17}
 }
 
 func (x *CacheUpdate) GetAction() CacheUpdateAction {
@@ -1473,7 +1545,7 @@ type LiveLog struct {
 
 func (x *LiveLog) Reset() {
 	*x = LiveLog{}
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[17]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[18]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -1485,7 +1557,7 @@ func (x *LiveLog) String() string {
 func (*LiveLog) ProtoMessage() {}
 
 func (x *LiveLog) ProtoReflect() protoreflect.Message {
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[17]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[18]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -1498,7 +1570,7 @@ func (x *LiveLog) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use LiveLog.ProtoReflect.Descriptor instead.
 func (*LiveLog) Descriptor() ([]byte, []int) {
-	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17}
+	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18}
 }
 
 func (x *LiveLog) GetLeaseId() string {
@@ -1525,6 +1597,7 @@ type Event struct {
 	//	*Event_StatusEvent
 	//	*Event_AttemptResult
 	//	*Event_CacheUpdate
+	//	*Event_AttemptOutcome
 	Payload       isEvent_Payload `protobuf_oneof:"payload"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
@@ -1532,7 +1605,7 @@ type Event struct {
 
 func (x *Event) Reset() {
 	*x = Event{}
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[18]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[19]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -1544,7 +1617,7 @@ func (x *Event) String() string {
 func (*Event) ProtoMessage() {}
 
 func (x *Event) ProtoReflect() protoreflect.Message {
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[18]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[19]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -1557,7 +1630,7 @@ func (x *Event) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Event.ProtoReflect.Descriptor instead.
 func (*Event) Descriptor() ([]byte, []int) {
-	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18}
+	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{19}
 }
 
 func (x *Event) GetSeqno() uint64 {
@@ -1608,6 +1681,15 @@ func (x *Event) GetCacheUpdate() *CacheUpdate {
 	return nil
 }
 
+func (x *Event) GetAttemptOutcome() *AttemptOutcome {
+	if x != nil {
+		if x, ok := x.Payload.(*Event_AttemptOutcome); ok {
+			return x.AttemptOutcome
+		}
+	}
+	return nil
+}
+
 type isEvent_Payload interface {
 	isEvent_Payload()
 }
@@ -1624,12 +1706,18 @@ type Event_CacheUpdate struct {
 	CacheUpdate *CacheUpdate `protobuf:"bytes,5,opt,name=cache_update,json=cacheUpdate,proto3,oneof"`
 }
 
+type Event_AttemptOutcome struct {
+	AttemptOutcome *AttemptOutcome `protobuf:"bytes,6,opt,name=attempt_outcome,json=attemptOutcome,proto3,oneof"`
+}
+
 func (*Event_StatusEvent) isEvent_Payload() {}
 
 func (*Event_AttemptResult) isEvent_Payload() {}
 
 func (*Event_CacheUpdate) isEvent_Payload() {}
 
+func (*Event_AttemptOutcome) isEvent_Payload() {}
+
 // a flushed bundle of events
 type EventBatch struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
@@ -1641,7 +1729,7 @@ type EventBatch struct {
 
 func (x *EventBatch) Reset() {
 	*x = EventBatch{}
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[19]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[20]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -1653,7 +1741,7 @@ func (x *EventBatch) String() string {
 func (*EventBatch) ProtoMessage() {}
 
 func (x *EventBatch) ProtoReflect() protoreflect.Message {
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[19]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[20]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -1666,7 +1754,7 @@ func (x *EventBatch) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use EventBatch.ProtoReflect.Descriptor instead.
 func (*EventBatch) Descriptor() ([]byte, []int) {
-	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{19}
+	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{20}
 }
 
 func (x *EventBatch) GetEpoch() string {
@@ -1694,7 +1782,7 @@ type Ack struct {
 
 func (x *Ack) Reset() {
 	*x = Ack{}
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[20]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[21]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -1706,7 +1794,7 @@ func (x *Ack) String() string {
 func (*Ack) ProtoMessage() {}
 
 func (x *Ack) ProtoReflect() protoreflect.Message {
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[20]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[21]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -1719,7 +1807,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Ack.ProtoReflect.Descriptor instead.
 func (*Ack) Descriptor() ([]byte, []int) {
-	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{20}
+	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{21}
 }
 
 func (x *Ack) GetEpoch() string {
@@ -1759,7 +1847,7 @@ type Message struct {
 
 func (x *Message) Reset() {
 	*x = Message{}
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[21]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[22]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -1771,7 +1859,7 @@ func (x *Message) String() string {
 func (*Message) ProtoMessage() {}
 
 func (x *Message) ProtoReflect() protoreflect.Message {
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[21]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[22]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -1784,7 +1872,7 @@ func (x *Message) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Message.ProtoReflect.Descriptor instead.
 func (*Message) Descriptor() ([]byte, []int) {
-	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{21}
+	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{22}
 }
 
 func (x *Message) GetHello() *Hello {
@@ -1907,7 +1995,7 @@ type QuotaRequest struct {
 
 func (x *QuotaRequest) Reset() {
 	*x = QuotaRequest{}
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[22]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[23]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -1919,7 +2007,7 @@ func (x *QuotaRequest) String() string {
 func (*QuotaRequest) ProtoMessage() {}
 
 func (x *QuotaRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[22]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[23]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -1932,7 +2020,7 @@ func (x *QuotaRequest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use QuotaRequest.ProtoReflect.Descriptor instead.
 func (*QuotaRequest) Descriptor() ([]byte, []int) {
-	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{22}
+	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{23}
 }
 
 func (x *QuotaRequest) GetRequestId() string {
@@ -1999,7 +2087,7 @@ type QuotaResponse struct {
 
 func (x *QuotaResponse) Reset() {
 	*x = QuotaResponse{}
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[23]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[24]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -2011,7 +2099,7 @@ func (x *QuotaResponse) String() string {
 func (*QuotaResponse) ProtoMessage() {}
 
 func (x *QuotaResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_spindle_mill_v1_mill_proto_msgTypes[23]
+	mi := &file_spindle_mill_v1_mill_proto_msgTypes[24]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -2024,7 +2112,7 @@ func (x *QuotaResponse) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use QuotaResponse.ProtoReflect.Descriptor instead.
 func (*QuotaResponse) Descriptor() ([]byte, []int) {
-	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{23}
+	return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{24}
 }
 
 func (x *QuotaResponse) GetRequestId() string {
@@ -2148,7 +2236,7 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" +
 	"\asave_id\x18\x06 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x06saveId\x12\"\n" +
 	"\bsave_key\x18\a \x01(\tB\a\xbaH\x04r\x02\x10\x01R\asaveKey\x12)\n" +
 	"\x10restore_checksum\x18\b \x01(\tR\x0frestoreChecksum\x12,\n" +
-	"\x12restore_size_bytes\x18\t \x01(\x03R\x10restoreSizeBytes\"\xaf\x02\n" +
+	"\x12restore_size_bytes\x18\t \x01(\x03R\x10restoreSizeBytes\"\xe7\x02\n" +
 	"\vCommitLease\x12\"\n" +
 	"\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x121\n" +
 	"\asecrets\x18\x02 \x03(\v2\x17.spindle.mill.v1.SecretR\asecrets\x12 \n" +
@@ -2157,7 +2245,8 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" +
 	"tracestate\x18\x04 \x01(\tR\n" +
 	"tracestate\x12A\n" +
 	"\x1dmill_records_terminal_metrics\x18\x05 \x01(\bR\x1amillRecordsTerminalMetrics\x12D\n" +
-	"\x0ecache_bindings\x18\x06 \x03(\v2\x1d.spindle.mill.v1.CacheBindingR\rcacheBindings\"/\n" +
+	"\x0ecache_bindings\x18\x06 \x03(\v2\x1d.spindle.mill.v1.CacheBindingR\rcacheBindings\x126\n" +
+	"\x17accepts_attempt_outcome\x18\a \x01(\bR\x15acceptsAttemptOutcome\"/\n" +
 	"\tCommitted\x12\"\n" +
 	"\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"2\n" +
 	"\fReleaseLease\x12\"\n" +
@@ -2183,7 +2272,12 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" +
 	"\flog_artifact\x18\x04 \x01(\v2\x1c.spindle.mill.v1.LogArtifactR\vlogArtifact\x12,\n" +
 	"\rfailure_class\x18\x05 \x01(\tB\a\xbaH\x04r\x02( R\ffailureClass\x12.\n" +
 	"\x0efailure_reason\x18\x06 \x01(\tB\a\xbaH\x04r\x02(@R\rfailureReason\x12A\n" +
-	"\x1dmill_records_terminal_metrics\x18\a \x01(\bR\x1amillRecordsTerminalMetrics\"\xb2\x01\n" +
+	"\x1dmill_records_terminal_metrics\x18\a \x01(\bR\x1amillRecordsTerminalMetrics\"\x93\x01\n" +
+	"\x0eAttemptOutcome\x12C\n" +
+	"\x06status\x18\x01 \x01(\x0e2\x1f.spindle.mill.v1.TerminalStatusB\n" +
+	"\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" +
+	"\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" +
+	"\texit_code\x18\x03 \x01(\x03R\bexitCode\"\xb2\x01\n" +
 	"\vCacheUpdate\x12F\n" +
 	"\x06action\x18\x01 \x01(\x0e2\".spindle.mill.v1.CacheUpdateActionB\n" +
 	"\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06action\x12\x17\n" +
@@ -2193,13 +2287,14 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" +
 	"\bchecksum\x18\x05 \x01(\tR\bchecksum\"Q\n" +
 	"\aLiveLog\x12\"\n" +
 	"\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\"\n" +
-	"\braw_json\x18\x02 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\xab\x02\n" +
+	"\braw_json\x18\x02 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\xf7\x02\n" +
 	"\x05Event\x12\x1d\n" +
 	"\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12\"\n" +
 	"\blease_id\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12A\n" +
 	"\fstatus_event\x18\x03 \x01(\v2\x1c.spindle.mill.v1.StatusEventH\x00R\vstatusEvent\x12G\n" +
 	"\x0eattempt_result\x18\x04 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResult\x12A\n" +
-	"\fcache_update\x18\x05 \x01(\v2\x1c.spindle.mill.v1.CacheUpdateH\x00R\vcacheUpdateB\x10\n" +
+	"\fcache_update\x18\x05 \x01(\v2\x1c.spindle.mill.v1.CacheUpdateH\x00R\vcacheUpdate\x12J\n" +
+	"\x0fattempt_outcome\x18\x06 \x01(\v2\x1f.spindle.mill.v1.AttemptOutcomeH\x00R\x0eattemptOutcomeB\x10\n" +
 	"\apayload\x12\x05\xbaH\x02\b\x01\"e\n" +
 	"\n" +
 	"EventBatch\x12\x1d\n" +
@@ -2309,7 +2404,7 @@ func file_spindle_mill_v1_mill_proto_rawDescGZIP() []byte {
 }
 
 var file_spindle_mill_v1_mill_proto_enumTypes = make([]protoimpl.EnumInfo, 5)
-var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 28)
+var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 29)
 var file_spindle_mill_v1_mill_proto_goTypes = []any{
 	(RejectClass)(0),           // 0: spindle.mill.v1.RejectClass
 	(NonterminalStatus)(0),     // 1: spindle.mill.v1.NonterminalStatus
@@ -2332,57 +2427,60 @@ var file_spindle_mill_v1_mill_proto_goTypes = []any{
 	(*StatusEvent)(nil),        // 18: spindle.mill.v1.StatusEvent
 	(*LogArtifact)(nil),        // 19: spindle.mill.v1.LogArtifact
 	(*AttemptResult)(nil),      // 20: spindle.mill.v1.AttemptResult
-	(*CacheUpdate)(nil),        // 21: spindle.mill.v1.CacheUpdate
-	(*LiveLog)(nil),            // 22: spindle.mill.v1.LiveLog
-	(*Event)(nil),              // 23: spindle.mill.v1.Event
-	(*EventBatch)(nil),         // 24: spindle.mill.v1.EventBatch
-	(*Ack)(nil),                // 25: spindle.mill.v1.Ack
-	(*Message)(nil),            // 26: spindle.mill.v1.Message
-	(*QuotaRequest)(nil),       // 27: spindle.mill.v1.QuotaRequest
-	(*QuotaResponse)(nil),      // 28: spindle.mill.v1.QuotaResponse
-	nil,                        // 29: spindle.mill.v1.EngineAvailability.LoadEntry
-	nil,                        // 30: spindle.mill.v1.NodeSnapshot.EnginesEntry
-	nil,                        // 31: spindle.mill.v1.ReserveResult.QuotaResourcesEntry
-	nil,                        // 32: spindle.mill.v1.QuotaRequest.ResourcesEntry
+	(*AttemptOutcome)(nil),     // 21: spindle.mill.v1.AttemptOutcome
+	(*CacheUpdate)(nil),        // 22: spindle.mill.v1.CacheUpdate
+	(*LiveLog)(nil),            // 23: spindle.mill.v1.LiveLog
+	(*Event)(nil),              // 24: spindle.mill.v1.Event
+	(*EventBatch)(nil),         // 25: spindle.mill.v1.EventBatch
+	(*Ack)(nil),                // 26: spindle.mill.v1.Ack
+	(*Message)(nil),            // 27: spindle.mill.v1.Message
+	(*QuotaRequest)(nil),       // 28: spindle.mill.v1.QuotaRequest
+	(*QuotaResponse)(nil),      // 29: spindle.mill.v1.QuotaResponse
+	nil,                        // 30: spindle.mill.v1.EngineAvailability.LoadEntry
+	nil,                        // 31: spindle.mill.v1.NodeSnapshot.EnginesEntry
+	nil,                        // 32: spindle.mill.v1.ReserveResult.QuotaResourcesEntry
+	nil,                        // 33: spindle.mill.v1.QuotaRequest.ResourcesEntry
 }
 var file_spindle_mill_v1_mill_proto_depIdxs = []int32{
-	29, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry
-	30, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry
+	30, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry
+	31, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry
 	0,  // 2: spindle.mill.v1.ReserveResult.reject_class:type_name -> spindle.mill.v1.RejectClass
-	31, // 3: spindle.mill.v1.ReserveResult.quota_resources:type_name -> spindle.mill.v1.ReserveResult.QuotaResourcesEntry
+	32, // 3: spindle.mill.v1.ReserveResult.quota_resources:type_name -> spindle.mill.v1.ReserveResult.QuotaResourcesEntry
 	11, // 4: spindle.mill.v1.CommitLease.secrets:type_name -> spindle.mill.v1.Secret
 	12, // 5: spindle.mill.v1.CommitLease.cache_bindings:type_name -> spindle.mill.v1.CacheBinding
 	1,  // 6: spindle.mill.v1.StatusEvent.status:type_name -> spindle.mill.v1.NonterminalStatus
 	2,  // 7: spindle.mill.v1.AttemptResult.status:type_name -> spindle.mill.v1.TerminalStatus
 	19, // 8: spindle.mill.v1.AttemptResult.log_artifact:type_name -> spindle.mill.v1.LogArtifact
-	3,  // 9: spindle.mill.v1.CacheUpdate.action:type_name -> spindle.mill.v1.CacheUpdateAction
-	18, // 10: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent
-	20, // 11: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult
-	21, // 12: spindle.mill.v1.Event.cache_update:type_name -> spindle.mill.v1.CacheUpdate
-	23, // 13: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event
-	5,  // 14: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello
-	6,  // 15: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume
-	8,  // 16: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot
-	9,  // 17: spindle.mill.v1.Message.reserve_seat:type_name -> spindle.mill.v1.ReserveSeat
-	10, // 18: spindle.mill.v1.Message.reserve_result:type_name -> spindle.mill.v1.ReserveResult
-	13, // 19: spindle.mill.v1.Message.commit_lease:type_name -> spindle.mill.v1.CommitLease
-	14, // 20: spindle.mill.v1.Message.committed:type_name -> spindle.mill.v1.Committed
-	15, // 21: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease
-	16, // 22: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt
-	17, // 23: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck
-	24, // 24: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch
-	25, // 25: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack
-	22, // 26: spindle.mill.v1.Message.live_log:type_name -> spindle.mill.v1.LiveLog
-	27, // 27: spindle.mill.v1.Message.quota_req:type_name -> spindle.mill.v1.QuotaRequest
-	28, // 28: spindle.mill.v1.Message.quota_resp:type_name -> spindle.mill.v1.QuotaResponse
-	4,  // 29: spindle.mill.v1.QuotaRequest.operation:type_name -> spindle.mill.v1.QuotaOperation
-	32, // 30: spindle.mill.v1.QuotaRequest.resources:type_name -> spindle.mill.v1.QuotaRequest.ResourcesEntry
-	7,  // 31: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability
-	32, // [32:32] is the sub-list for method output_type
-	32, // [32:32] is the sub-list for method input_type
-	32, // [32:32] is the sub-list for extension type_name
-	32, // [32:32] is the sub-list for extension extendee
-	0,  // [0:32] is the sub-list for field type_name
+	2,  // 9: spindle.mill.v1.AttemptOutcome.status:type_name -> spindle.mill.v1.TerminalStatus
+	3,  // 10: spindle.mill.v1.CacheUpdate.action:type_name -> spindle.mill.v1.CacheUpdateAction
+	18, // 11: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent
+	20, // 12: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult
+	22, // 13: spindle.mill.v1.Event.cache_update:type_name -> spindle.mill.v1.CacheUpdate
+	21, // 14: spindle.mill.v1.Event.attempt_outcome:type_name -> spindle.mill.v1.AttemptOutcome
+	24, // 15: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event
+	5,  // 16: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello
+	6,  // 17: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume
+	8,  // 18: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot
+	9,  // 19: spindle.mill.v1.Message.reserve_seat:type_name -> spindle.mill.v1.ReserveSeat
+	10, // 20: spindle.mill.v1.Message.reserve_result:type_name -> spindle.mill.v1.ReserveResult
+	13, // 21: spindle.mill.v1.Message.commit_lease:type_name -> spindle.mill.v1.CommitLease
+	14, // 22: spindle.mill.v1.Message.committed:type_name -> spindle.mill.v1.Committed
+	15, // 23: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease
+	16, // 24: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt
+	17, // 25: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck
+	25, // 26: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch
+	26, // 27: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack
+	23, // 28: spindle.mill.v1.Message.live_log:type_name -> spindle.mill.v1.LiveLog
+	28, // 29: spindle.mill.v1.Message.quota_req:type_name -> spindle.mill.v1.QuotaRequest
+	29, // 30: spindle.mill.v1.Message.quota_resp:type_name -> spindle.mill.v1.QuotaResponse
+	4,  // 31: spindle.mill.v1.QuotaRequest.operation:type_name -> spindle.mill.v1.QuotaOperation
+	33, // 32: spindle.mill.v1.QuotaRequest.resources:type_name -> spindle.mill.v1.QuotaRequest.ResourcesEntry
+	7,  // 33: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability
+	34, // [34:34] is the sub-list for method output_type
+	34, // [34:34] is the sub-list for method input_type
+	34, // [34:34] is the sub-list for extension type_name
+	34, // [34:34] is the sub-list for extension extendee
+	0,  // [0:34] is the sub-list for field type_name
 }
 
 func init() { file_spindle_mill_v1_mill_proto_init() }
@@ -2390,10 +2488,11 @@ func file_spindle_mill_v1_mill_proto_init() {
 	if File_spindle_mill_v1_mill_proto != nil {
 		return
 	}
-	file_spindle_mill_v1_mill_proto_msgTypes[18].OneofWrappers = []any{
+	file_spindle_mill_v1_mill_proto_msgTypes[19].OneofWrappers = []any{
 		(*Event_StatusEvent)(nil),
 		(*Event_AttemptResult)(nil),
 		(*Event_CacheUpdate)(nil),
+		(*Event_AttemptOutcome)(nil),
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
@@ -2401,7 +2500,7 @@ func file_spindle_mill_v1_mill_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc)),
 			NumEnums:      5,
-			NumMessages:   28,
+			NumMessages:   29,
 			NumExtensions: 0,
 			NumServices:   0,
 		},
diff --git a/spindle/mill/proto/spindle/mill/v1/mill.proto b/spindle/mill/proto/spindle/mill/v1/mill.proto
index fafb4ed79..711ae329c 100644
--- a/spindle/mill/proto/spindle/mill/v1/mill.proto
+++ b/spindle/mill/proto/spindle/mill/v1/mill.proto
@@ -103,6 +103,9 @@ message CommitLease {
   string tracestate = 4;
   bool mill_records_terminal_metrics = 5;
   repeated CacheBinding cache_bindings = 6;
+  // older mills drop the session on an unknown event payload, so executors only
+  // send AttemptOutcome when this is set
+  bool accepts_attempt_outcome = 7;
 }
 
 message Committed {
@@ -165,6 +168,17 @@ message AttemptResult {
   bool mill_records_terminal_metrics = 7;
 }
 
+// the result is final but the executor still holds the VM, so the lease stays
+// charged until the AttemptResult
+message AttemptOutcome {
+  TerminalStatus status = 1 [(buf.validate.field).enum = {
+    defined_only: true
+    not_in: 0
+  }];
+  string error = 2 [(buf.validate.field).string.max_bytes = 65536];
+  int64 exit_code = 3;
+}
+
 enum CacheUpdateAction {
   CACHE_UPDATE_ACTION_UNSPECIFIED = 0;
   CACHE_UPDATE_ACTION_USED = 1;
@@ -199,6 +213,7 @@ message Event {
     StatusEvent status_event = 3;
     AttemptResult attempt_result = 4;
     CacheUpdate cache_update = 5;
+    AttemptOutcome attempt_outcome = 6;
   }
 }
 
diff --git a/spindle/wipe.go b/spindle/wipe.go
index 009494527..51e7a9d5e 100644
--- a/spindle/wipe.go
+++ b/spindle/wipe.go
@@ -138,12 +138,13 @@ func (s *Spindle) WipeOwner(ctx context.Context, ownerDid syntax.DID, reason str
 func (s *Spindle) cancelRepoWorkflows(ctx context.Context, repoDid syntax.DID, reason string) ([]models.WorkflowId, error) {
 	var wids []models.WorkflowId
 	seen := map[models.WorkflowId]bool{}
-	add := func(rkey, name string) {
+	add := func(rkey, name string) models.WorkflowId {
 		wid := models.WorkflowId{PipelineId: models.PipelineId(rkey), Name: name}
 		if !seen[wid] {
 			seen[wid] = true
 			wids = append(wids, wid)
 		}
+		return wid
 	}
 
 	pipes, err := s.db.ListPipelineWorkflows(repoDid.String())
@@ -158,19 +159,24 @@ func (s *Spindle) cancelRepoWorkflows(ctx context.Context, repoDid syntax.DID, r
 	if err != nil {
 		return nil, fmt.Errorf("list mill leases: %w", err)
 	}
+	leased := map[models.WorkflowId]bool{}
 	for _, lease := range leases {
 		if lease.RepoDID == repoDid.String() {
-			add(lease.PipelineID, lease.Workflow)
+			leased[add(lease.PipelineID, lease.Workflow)] = true
 		}
 	}
 
 	var errs []error
 	for _, wid := range wids {
 		st, err := s.db.GetStatus(wid)
-		if err != nil || !st.IsFinish() {
+		finished := err == nil && st.IsFinish()
+		if !finished {
 			if err := s.db.StatusCancelled(wid, reason, -1, s.n); err != nil {
 				errs = append(errs, err)
 			}
+		}
+		// a finished workflow keeps its lease while a debug shell holds the VM
+		if !finished || leased[wid] {
 			engine.CancelWorkflow(wid)
 			for _, eng := range s.engs {
 				if err := eng.DestroyWorkflow(ctx, wid); err != nil {
diff --git a/spindle/wipe_test.go b/spindle/wipe_test.go
index 3eb070c3a..01b9db269 100644
--- a/spindle/wipe_test.go
+++ b/spindle/wipe_test.go
@@ -314,6 +314,39 @@ func TestWipeOwnerWipesAllOwnedRepos(t *testing.T) {
 	}
 }
 
+type destroyRecorder struct {
+	models.Engine
+	destroyed []models.WorkflowId
+}
+
+func (e *destroyRecorder) DestroyWorkflow(_ context.Context, wid models.WorkflowId) error {
+	e.destroyed = append(e.destroyed, wid)
+	return nil
+}
+
+func TestWipeDestroysFinishedWorkflowStillHoldingItsVM(t *testing.T) {
+	s := newWipeTestSpindle(t)
+	repoDid := syntax.DID("did:plc:repo1")
+	seedWipeRepo(t, s, repoDid, "did:plc:owner1")
+	eng := &destroyRecorder{}
+	s.engs["microvm"] = eng
+
+	wid := models.WorkflowId{PipelineId: models.PipelineId("p1"), Name: "w1"}
+	if err := s.db.StatusFailed(wid, "User step error: exited with code 1", 1, s.n); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := s.cancelRepoWorkflows(context.Background(), repoDid, "banned"); err != nil {
+		t.Fatal(err)
+	}
+	if len(eng.destroyed) != 1 || eng.destroyed[0] != wid {
+		t.Fatalf("destroyed = %v, want the held workflow", eng.destroyed)
+	}
+	if st, err := s.db.GetStatus(wid); err != nil || st != models.StatusKindFailed {
+		t.Fatalf("status = %q (err %v), want the failure kept", st, err)
+	}
+}
+
 func ptr[T any](v T) *T { return &v }
 
 func TestIngestAccountTakedownWipes(t *testing.T) {
-- 
2.51.2


From d6f7164c851beeafac3ea6d3b5ebdea5ec660aa6 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Tue, 6 Oct 2026 22:02:15 +0300
Subject: [PATCH 12/25] migrator/oauth: fill in the app's repo record instead
 of creating over it

Signed-off-by: dawn <dawn@tangled.org>
---
 migrator/oauth/client.go      | 108 +++++++++++-----------------------
 migrator/oauth/client_test.go |  29 +++++----
 2 files changed, 51 insertions(+), 86 deletions(-)

diff --git a/migrator/oauth/client.go b/migrator/oauth/client.go
index d7c4b6ed5..0818f328e 100644
--- a/migrator/oauth/client.go
+++ b/migrator/oauth/client.go
@@ -414,23 +414,51 @@ func (c *Client) putRepoRecord(ctx context.Context, session *indigooauth.ClientS
 		return err
 	}
 
-	body, err := json.Marshal(map[string]any{
+	// the app writes a record without repoDid before the import starts, keeping fields
+	// the migrator never sees (spindle, source), so fill that in rather than replace it
+	existing, found := c.existingRepoRecord(ctx, session, pds, ownerDid, rkey)
+	if !found {
+		return c.writeRecord(ctx, session, pds, "com.atproto.repo.createRecord", map[string]any{
+			"repo":       ownerDid,
+			"collection": tangled.RepoNSID,
+			"rkey":       rkey,
+			"record":     repoRecord(rkey, name, description, host, repoDid, time.Now().UTC()),
+		})
+	}
+	if existing.repoDid == repoDid {
+		c.logger.Info("repo record was already written by an earlier attempt", "rkey", rkey, "repoDid", repoDid)
+		return nil
+	}
+	if existing.repoDid != "" {
+		return fmt.Errorf("a record for %q already names repository %s", rkey, existing.repoDid)
+	}
+	record := existing.value
+	if record == nil {
+		record = make(map[string]any)
+	}
+	record["repoDid"] = repoDid
+	return c.writeRecord(ctx, session, pds, "com.atproto.repo.putRecord", map[string]any{
 		"repo":       ownerDid,
 		"collection": tangled.RepoNSID,
 		"rkey":       rkey,
-		"record":     repoRecord(rkey, name, description, host, repoDid, time.Now().UTC()),
+		"record":     record,
+		"swapRecord": existing.cid,
 	})
+}
+
+func (c *Client) writeRecord(ctx context.Context, session *indigooauth.ClientSession, pds, nsid string, input map[string]any) error {
+	body, err := json.Marshal(input)
 	if err != nil {
 		return err
 	}
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
-		strings.TrimSuffix(pds, "/")+"/xrpc/"+"com.atproto.repo.createRecord", bytes.NewReader(body))
+		strings.TrimSuffix(pds, "/")+"/xrpc/"+nsid, bytes.NewReader(body))
 	if err != nil {
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
 
-	lxm, err := syntax.ParseNSID("com.atproto.repo.createRecord")
+	lxm, err := syntax.ParseNSID(nsid)
 	if err != nil {
 		return err
 	}
@@ -439,34 +467,10 @@ func (c *Client) putRepoRecord(ctx context.Context, session *indigooauth.ClientS
 		return fmt.Errorf("writing repo record: %w", err)
 	}
 	defer resp.Body.Close()
-	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
-		return nil
-	}
-
-	if resp.StatusCode == http.StatusBadRequest || resp.StatusCode == http.StatusConflict {
-		// The record already exists; retry swaps lost to concurrent writers.
-		for attempt := 0; attempt < swapConflictAttempts; attempt++ {
-			existing, found := c.existingRepoRecord(ctx, session, pds, ownerDid, rkey)
-			if !found {
-				break
-			}
-			if existing.repoDid == repoDid {
-				c.logger.Info("repo record was already written by an earlier attempt", "rkey", rkey, "repoDid", repoDid)
-				return nil
-			}
-			if existing.repoDid != "" {
-				return fmt.Errorf("a record for %q already names repository %s", rkey, existing.repoDid)
-			}
-			retry, err := c.patchRepoRecord(ctx, session, pds, ownerDid, rkey, repoDid, existing)
-			if err != nil {
-				return err
-			}
-			if !retry {
-				return nil
-			}
-		}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return responseError("repo record write", resp)
 	}
-	return responseError("repo record write", resp)
+	return nil
 }
 
 // knot field stores the knot's host, not its DID; a DID breaks knot and mirror reads
@@ -494,48 +498,6 @@ var defaultLabels = []string{
 	"at://" + defaultLabelOwner + "/sh.tangled.label.definition/assignee",
 }
 
-const swapConflictAttempts = 3
-
-func (c *Client) patchRepoRecord(ctx context.Context, session *indigooauth.ClientSession, pds, ownerDid, rkey, repoDid string, existing *existingRepoRecord) (bool, error) {
-	record := existing.value
-	if record == nil {
-		record = make(map[string]any)
-	}
-	record["repoDid"] = repoDid
-	putBody, err := json.Marshal(map[string]any{
-		"repo":       ownerDid,
-		"collection": tangled.RepoNSID,
-		"rkey":       rkey,
-		"record":     record,
-		"swapRecord": existing.cid,
-	})
-	if err != nil {
-		return false, err
-	}
-	putReq, err := http.NewRequestWithContext(ctx, http.MethodPost,
-		strings.TrimSuffix(pds, "/")+"/xrpc/com.atproto.repo.putRecord", bytes.NewReader(putBody))
-	if err != nil {
-		return false, err
-	}
-	putReq.Header.Set("Content-Type", "application/json")
-	putLxm, err := syntax.ParseNSID("com.atproto.repo.putRecord")
-	if err != nil {
-		return false, err
-	}
-	putResp, err := session.DoWithAuth(c.http, putReq, putLxm)
-	if err != nil {
-		return false, fmt.Errorf("updating repo record: %w", err)
-	}
-	defer putResp.Body.Close()
-	if putResp.StatusCode >= 200 && putResp.StatusCode < 300 {
-		return false, nil
-	}
-	if putResp.StatusCode == http.StatusBadRequest || putResp.StatusCode == http.StatusConflict {
-		return true, nil
-	}
-	return false, responseError("repo record update", putResp)
-}
-
 func repoRecord(rkey, name, description, knotHost, repoDid string, now time.Time) *tangled.Repo {
 	record := &tangled.Repo{
 		LexiconTypeID: tangled.RepoNSID,
diff --git a/migrator/oauth/client_test.go b/migrator/oauth/client_test.go
index f5b0b8e98..2855b8107 100644
--- a/migrator/oauth/client_test.go
+++ b/migrator/oauth/client_test.go
@@ -500,30 +500,33 @@ func TestPutRepoRecordDifferentRepoDidIsError(t *testing.T) {
 	}
 }
 
-func TestPutRepoRecordRetriesALostSwap(t *testing.T) {
-	client, getRecord, putCalls := setupPutRecordTestWithSwaps(
-		t, &mockPdsRecord{cid: "bafy-initial", repoDid: nil, name: "repo"}, 1)
+func TestPutRepoRecordWithoutARecordCreatesOne(t *testing.T) {
+	client, getRecord, putCalls := setupPutRecordTest(t, nil)
 	err := client.PutRepoRecord(context.Background(), "did:plc:alice", "repo", "repo", "desc", "did:web:knot.example", "did:plc:new-repo")
 	if err != nil {
-		t.Fatalf("PutRepoRecord error = %v, want the retry to settle the record", err)
+		t.Fatalf("PutRepoRecord error = %v, want the record created", err)
 	}
 	current := getRecord()
 	if current == nil || current.repoDid == nil || *current.repoDid != "did:plc:new-repo" {
-		t.Fatalf("record = %v, want it to name did:plc:new-repo after the retry", current)
+		t.Fatalf("record = %v, want it to name did:plc:new-repo", current)
 	}
-	if len(*putCalls) != 2 {
-		t.Fatalf("putRecord attempts = %d, want 2 (one lost swap, one retry)", len(*putCalls))
+	if len(*putCalls) != 0 {
+		t.Fatalf("putRecord calls = %d, want 0", len(*putCalls))
 	}
 }
 
-func TestPutRepoRecordGivesUpAfterRepeatedLostSwaps(t *testing.T) {
-	client, _, putCalls := setupPutRecordTestWithSwaps(
-		t, &mockPdsRecord{cid: "bafy-initial", repoDid: nil, name: "repo"}, 9)
+func TestPutRepoRecordLostSwapIsError(t *testing.T) {
+	// a record changed under us is left alone, the job's own retry reads it again
+	client, getRecord, putCalls := setupPutRecordTestWithSwaps(
+		t, &mockPdsRecord{cid: "bafy-initial", repoDid: nil, name: "repo"}, 1)
 	err := client.PutRepoRecord(context.Background(), "did:plc:alice", "repo", "repo", "desc", "did:web:knot.example", "did:plc:new-repo")
 	if err == nil {
-		t.Fatal("PutRepoRecord error = nil, want a bounded failure rather than an endless retry")
+		t.Fatal("PutRepoRecord error = nil, want the lost swap reported")
+	}
+	if current := getRecord(); current.repoDid != nil {
+		t.Fatalf("record repoDid = %v, want it untouched", *current.repoDid)
 	}
-	if len(*putCalls) != 3 {
-		t.Fatalf("putRecord attempts = %d, want the bound of 3", len(*putCalls))
+	if len(*putCalls) != 1 {
+		t.Fatalf("putRecord attempts = %d, want 1", len(*putCalls))
 	}
 }
-- 
2.51.2


From c9887e2a30770241fcda90abd75b43cb6fa07a7a Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Tue, 6 Oct 2026 22:17:00 +0300
Subject: [PATCH 13/25] migrator/{api,oauth}: ask for the update scope, and for
 a new grant when a pds refuses one

Signed-off-by: dawn <dawn@tangled.org>
---
 migrator/api/grant_test.go     |  1 +
 migrator/api/server.go         | 35 ++++++++++++++++++++++------------
 migrator/api/server_test.go    | 31 ++++++++++++++++++++++++++++++
 migrator/oauth/client.go       |  9 ++++++---
 migrator/oauth/session_test.go | 10 ++++++++++
 migrator/oauth/store_test.go   |  2 +-
 6 files changed, 72 insertions(+), 16 deletions(-)

diff --git a/migrator/api/grant_test.go b/migrator/api/grant_test.go
index 12a8a77d0..d3963e746 100644
--- a/migrator/api/grant_test.go
+++ b/migrator/api/grant_test.go
@@ -159,6 +159,7 @@ func serverWithGrants(t *testing.T, g *grantStub) (*testServer, string) {
 			"rpc:sh.tangled.repo.create?aud=*",
 			"rpc:sh.tangled.repo.describeRepo?aud=*",
 			"repo:sh.tangled.repo?action=create",
+			"repo:sh.tangled.repo?action=update",
 		},
 		AccessToken: "pds-oauth-old", RefreshToken: "refresh-old", DPoPPrivateKeyMultibase: dpopKey.Multibase(),
 	}); err != nil {
diff --git a/migrator/api/server.go b/migrator/api/server.go
index ff1c9f3a8..7f8b19236 100644
--- a/migrator/api/server.go
+++ b/migrator/api/server.go
@@ -214,20 +214,9 @@ func (s *Server) handleCreateTask(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	if s.grants == nil || !s.grants.HasSession(r.Context(), owner) {
-		s.writeError(w, grantRequired(s.cfg.ServiceURL(), owner), http.StatusPreconditionRequired)
+	if !s.requireGrant(w, r, owner) {
 		return
 	}
-	if prober, ok := s.grants.(GrantProber); ok {
-		if err := prober.UsableGrant(r.Context(), owner); err != nil {
-			if errors.Is(err, migratoroauth.ErrGrantRequired) {
-				s.writeError(w, grantRequired(s.cfg.ServiceURL(), owner), http.StatusPreconditionRequired)
-				return
-			}
-			// transient probe failures do not block creation; retried at job time
-			s.logger.Error("grant probe failed", "ownerDid", owner, "err", err)
-		}
-	}
 
 	taskID := uuid.New().String()
 
@@ -379,6 +368,9 @@ func (s *Server) handleRetryJob(w http.ResponseWriter, r *http.Request) {
 		s.writeError(w, jobNotRetryable(job.Status), http.StatusBadRequest)
 		return
 	}
+	if !s.requireGrant(w, r, owner) {
+		return
+	}
 
 	var newEncryptedToken *string
 	if job.Private {
@@ -544,6 +536,25 @@ func taskConflict() xrpcerr.XrpcError {
 	)
 }
 
+// answers 428 with the grant's start url when the owner has no usable grant
+func (s *Server) requireGrant(w http.ResponseWriter, r *http.Request, owner string) bool {
+	if s.grants == nil || !s.grants.HasSession(r.Context(), owner) {
+		s.writeError(w, grantRequired(s.cfg.ServiceURL(), owner), http.StatusPreconditionRequired)
+		return false
+	}
+	if prober, ok := s.grants.(GrantProber); ok {
+		if err := prober.UsableGrant(r.Context(), owner); err != nil {
+			if errors.Is(err, migratoroauth.ErrGrantRequired) {
+				s.writeError(w, grantRequired(s.cfg.ServiceURL(), owner), http.StatusPreconditionRequired)
+				return false
+			}
+			// transient probe failures don't block the request; the job checks again
+			s.logger.Error("grant probe failed", "ownerDid", owner, "err", err)
+		}
+	}
+	return true
+}
+
 func grantRequired(serviceURL, did string) xrpcerr.XrpcError {
 	start := fmt.Sprintf("%s/oauth/start?did=%s&return_to=%s",
 		strings.TrimSuffix(serviceURL, "/"), did, url.QueryEscape(config.DefaultReturnPath))
diff --git a/migrator/api/server_test.go b/migrator/api/server_test.go
index 599c98ff0..8a8063944 100644
--- a/migrator/api/server_test.go
+++ b/migrator/api/server_test.go
@@ -591,6 +591,37 @@ func TestCreateTaskRequiresTheCallersOwnOAuthGrant(t *testing.T) {
 	}
 }
 
+func TestRetryJobRequiresTheCallersOwnOAuthGrant(t *testing.T) {
+	ts := setupTestServer(t)
+	created := ts.do(t, http.MethodPost, "/xrpc/"+org_tangled.TempMigratorCreateTaskNSID,
+		ts.token(t, ts.alice, org_tangled.TempMigratorCreateTaskNSID),
+		&org_tangled.TempMigratorCreateTask_Input{RequestId: "r", Jobs: []*org_tangled.TempMigratorDefs_NewJob{job("one", false)}})
+	task := decodeTask(t, created)
+	id, err := parseJobID(task.Jobs[0].Id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reason := "the pds refused a scope"
+	if err := ts.db.UpdateJobStatus(context.Background(), id, db.StatusAuthorizationRequired, &reason); err != nil {
+		t.Fatal(err)
+	}
+
+	ts.server.grants = denyGrants{}
+	rec := ts.do(t, http.MethodPost, "/xrpc/"+org_tangled.TempMigratorRetryJobNSID,
+		ts.token(t, ts.alice, org_tangled.TempMigratorRetryJobNSID),
+		&org_tangled.TempMigratorRetryJob_Input{TaskId: task.Id, JobId: task.Jobs[0].Id})
+	if rec.Code != http.StatusPreconditionRequired || !strings.Contains(rec.Body.String(), "GrantRequired") {
+		t.Fatalf("want 428 GrantRequired, got %d %s", rec.Code, rec.Body.String())
+	}
+	_, jobs, err := ts.db.GetBatch(context.Background(), task.Id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if jobs[0].Status != db.StatusAuthorizationRequired {
+		t.Fatalf("job status = %q, a retry without a grant must leave it alone", jobs[0].Status)
+	}
+}
+
 func TestCreateTaskRejectsLegacyRepoDidInput(t *testing.T) {
 	ts := setupTestServer(t)
 	body := map[string]any{
diff --git a/migrator/oauth/client.go b/migrator/oauth/client.go
index 0818f328e..9eba8e09f 100644
--- a/migrator/oauth/client.go
+++ b/migrator/oauth/client.go
@@ -37,6 +37,7 @@ var scopes = []string{
 	"rpc:sh.tangled.repo.create?aud=*",
 	"rpc:sh.tangled.repo.describeRepo?aud=*",
 	"repo:sh.tangled.repo?action=create",
+	"repo:sh.tangled.repo?action=update",
 }
 
 type Client struct {
@@ -206,7 +207,8 @@ func (c *Client) withOwnerSession(ctx context.Context, ownerDid string, fn func(
 	return nil
 }
 
-// auth server already revoked the token family, so upstream revoke is omitted
+// the auth server already revoked the token family, or the grant lacks a scope and a new one
+// replaces it, so upstream revoke is omitted
 func (c *Client) dropSession(ctx context.Context, ownerDid string) {
 	c.sessions.Delete(ownerDid)
 	did, err := syntax.ParseDID(ownerDid)
@@ -259,12 +261,13 @@ func (c *Client) UsableGrant(ctx context.Context, ownerDid string) error {
 	})
 }
 
-// invalid_grant is terminal because the auth server invalidates the entire token family
+// invalid_grant is terminal because the auth server invalidates the entire token family, and a
+// pds refusing a scope means the grant predates a scope the migrator now asks for
 func grantErr(err error) error {
 	if err == nil {
 		return nil
 	}
-	if strings.Contains(err.Error(), "invalid_grant") {
+	if strings.Contains(err.Error(), "invalid_grant") || strings.Contains(err.Error(), "ScopeMissingError") {
 		return fmt.Errorf("%w: %s", ErrGrantRequired, err)
 	}
 	return err
diff --git a/migrator/oauth/session_test.go b/migrator/oauth/session_test.go
index 4d28b2f8a..cf5345b5c 100644
--- a/migrator/oauth/session_test.go
+++ b/migrator/oauth/session_test.go
@@ -342,3 +342,13 @@ func TestARegrantReplacesADeadCachedSession(t *testing.T) {
 		t.Fatalf("token refreshes after re-grant = %d, want %d (the fresh family must not need one)", g.refreshes, afterRegrant)
 	}
 }
+
+func TestAScopeThePdsRefusesAsksForANewGrant(t *testing.T) {
+	refused := errors.New(`repo record write returned HTTP 403: {"error":"ScopeMissingError","message":"Missing required scope \"repo:sh.tangled.repo?action=update\""}`)
+	if err := grantErr(refused); !errors.Is(err, ErrGrantRequired) {
+		t.Fatalf("err = %v, want ErrGrantRequired", err)
+	}
+	if err := grantErr(errors.New("repo record write returned HTTP 500")); errors.Is(err, ErrGrantRequired) {
+		t.Fatalf("a pds outage became a refused grant: %v", err)
+	}
+}
diff --git a/migrator/oauth/store_test.go b/migrator/oauth/store_test.go
index fe576cd87..86c7c6abc 100644
--- a/migrator/oauth/store_test.go
+++ b/migrator/oauth/store_test.go
@@ -123,7 +123,7 @@ func TestClientMetadataDeclaresOnlyTheImportGrant(t *testing.T) {
 	if metadata.ClientID != "https://migrator.example.com/oauth-client-metadata.json" || metadata.TokenEndpointAuthMethod != "none" || !metadata.DPoPBoundAccessTokens {
 		t.Fatalf("metadata = %+v", metadata)
 	}
-	if metadata.Scope != "atproto rpc:sh.tangled.repo.create?aud=* rpc:sh.tangled.repo.describeRepo?aud=* repo:sh.tangled.repo?action=create" {
+	if metadata.Scope != "atproto rpc:sh.tangled.repo.create?aud=* rpc:sh.tangled.repo.describeRepo?aud=* repo:sh.tangled.repo?action=create repo:sh.tangled.repo?action=update" {
 		t.Fatalf("scope = %q", metadata.Scope)
 	}
 	if strings.Join(metadata.GrantTypes, " ") != "authorization_code refresh_token" || strings.Join(metadata.ResponseTypes, " ") != "code" {
-- 
2.51.2


From 09cad0e147b9e91ec30e4617d5def393e0a95d5f Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Tue, 6 Oct 2026 23:12:13 +0300
Subject: [PATCH 14/25] web/migrate: give the page the onboarding card's width

Signed-off-by: dawn <dawn@tangled.org>
---
 web/src/routes/repo/migrate/+page.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/routes/repo/migrate/+page.svelte b/web/src/routes/repo/migrate/+page.svelte
index 9c661c308..c5988a5fb 100644
--- a/web/src/routes/repo/migrate/+page.svelte
+++ b/web/src/routes/repo/migrate/+page.svelte
@@ -69,7 +69,7 @@
 {/snippet}
 
 <Meta noindex title="Migrate repositories" />
-<div class="mx-auto w-full max-w-6xl px-4 py-12">
+<div class="mx-auto w-full max-w-wide px-4 py-12">
 	<div class="flex items-center gap-2 px-6 py-2">
 		<Download class="size-5" aria-hidden="true" />
 		<h1 class="typography-heading-4 font-bold text-foreground-default">Migrate repositories</h1>
-- 
2.51.2


From 0f4e47795e9640381f02c8240d1f29840b194443 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Tue, 6 Oct 2026 23:12:13 +0300
Subject: [PATCH 15/25] web/import: retry a failed github migration from its
 row

Signed-off-by: dawn <dawn@tangled.org>
---
 .../components/import/GitHubImporter.svelte   |  1 +
 .../components/import/MigrationStep.svelte    | 39 ++++++++++++-------
 .../import/gitHubImportState.svelte.ts        | 22 ++++++++++-
 3 files changed, 45 insertions(+), 17 deletions(-)

diff --git a/web/src/lib/components/import/GitHubImporter.svelte b/web/src/lib/components/import/GitHubImporter.svelte
index d37de243b..111671a2f 100644
--- a/web/src/lib/components/import/GitHubImporter.svelte
+++ b/web/src/lib/components/import/GitHubImporter.svelte
@@ -96,6 +96,7 @@
 			failed={importer.anyFailed}
 			userHandle={importer.userHandle}
 			bannerNote={migrationsNote}
+			onRetry={importer.retry}
 		/>
 	</StepFlow>
 </div>
diff --git a/web/src/lib/components/import/MigrationStep.svelte b/web/src/lib/components/import/MigrationStep.svelte
index 948f40727..e2b980c5d 100644
--- a/web/src/lib/components/import/MigrationStep.svelte
+++ b/web/src/lib/components/import/MigrationStep.svelte
@@ -18,6 +18,7 @@
 	import { resolve } from "$app/paths";
 	import ArrowRight from "$icon/arrow-right";
 	import CircleUserRound from "$icon/circle-user-round";
+	import RotateCcw from "$icon/rotate-ccw";
 	import { migrationRunning } from "$lib/api/migrator";
 	import Button from "$lib/components/ui/Button.svelte";
 	import MigrationRow, { stageLabel } from "./MigrationRow.svelte";
@@ -34,6 +35,7 @@
 		userHandle?: string;
 		bannerNote?: string;
 		id?: string;
+		onRetry?: (item: MigrationItem) => void;
 	}
 
 	let {
@@ -45,7 +47,8 @@
 		failed,
 		userHandle,
 		bannerNote,
-		id
+		id,
+		onRetry
 	}: Props = $props();
 
 	let bannerVisible = $state(true);
@@ -110,20 +113,26 @@
 							? resolve(`/${userHandle}/${m.task.name}` as "/")
 							: undefined}
 					/>
-					{#if itemFailed && m.status === "error"}
-						<p
-							class="mt-1 px-1 typography-paragraph-small break-all text-foreground-danger"
-						>
-							{m.error}
-						</p>
-					{:else if itemFailed && m.status === "success"}
-						<p
-							class="mt-1 px-1 typography-paragraph-small break-all text-foreground-danger"
-						>
-							{m.jobStatus === "authorization_required"
-								? "The source rejected its saved credential. Reconnect GitHub and start the import again."
-								: (m.jobError ?? "The migration failed")}
-						</p>
+					{#if itemFailed}
+						<div class="mt-1 flex items-start justify-between gap-2 px-1">
+							<p class="typography-paragraph-small break-all text-foreground-danger">
+								{m.status === "error"
+									? m.error
+									: m.jobStatus === "authorization_required"
+										? "The migrator lost access to the source or to your account. Authorize it again and retry."
+										: (m.jobError ?? "The migration failed")}
+							</p>
+							{#if onRetry}
+								<Button
+									size="small"
+									icon={RotateCcw}
+									disabled={isRunning}
+									onclick={() => onRetry(item)}
+								>
+									Retry
+								</Button>
+							{/if}
+						</div>
 					{/if}
 				</div>
 			{/each}
diff --git a/web/src/lib/components/import/gitHubImportState.svelte.ts b/web/src/lib/components/import/gitHubImportState.svelte.ts
index ed287f2c6..8d2ba1ea6 100644
--- a/web/src/lib/components/import/gitHubImportState.svelte.ts
+++ b/web/src/lib/components/import/gitHubImportState.svelte.ts
@@ -1,7 +1,7 @@
 import { page } from "$app/state";
 import { isDid, type Did } from "@atcute/lexicons/syntax";
 import { getAuth } from "$lib/auth.svelte";
-import { serviceDidForHost } from "$lib/auth/agent";
+import { hostForServiceDid, serviceDidForHost } from "$lib/auth/agent";
 import { validateRepoName } from "$lib/api/repoCreate";
 import type { GitHubAccount, GitHubRepo } from "$lib/github";
 import { flowReturnPath, returnPathFrom, startUrlFrom } from "$lib/api/migrator";
@@ -629,6 +629,23 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 
 	const migrate = () => void runQueue([...migratable]);
 
+	// a failed repo goes through createTask again, which brings a fresh grant and github token,
+	// and picks up the knot repo and placeholder record the failed attempt left behind, so it
+	// keeps that attempt's knot
+	const retry = (item: MigrationItem) => {
+		const repo = findRepoForJob({ id: item.migration.jobId, name: item.name });
+		if (!repo) {
+			errorMessage = `Connect the GitHub account that owns ${item.name} to retry it.`;
+			return;
+		}
+		const failed = loadedTasks
+			.flatMap((task) => task.jobs)
+			.find((job) => job.id === item.migration.jobId);
+		const knot = failed ? hostForServiceDid(failed.knotDid) : null;
+		if (knot) updateDraft(repo.id, { knot });
+		void runQueue([repo]);
+	};
+
 	const statusFingerprint = () =>
 		JSON.stringify(
 			repos.map((repo) => {
@@ -767,6 +784,7 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 		select,
 		setAll: setAllSelected,
 		toggleExpand,
-		migrate
+		migrate,
+		retry
 	};
 };
-- 
2.51.2


From def319d62bea227c571e821323eda3c1d903a1fd Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Tue, 6 Oct 2026 23:20:14 +0300
Subject: [PATCH 16/25] web/import: ask before sending the user off to
 authorize the migrator

Signed-off-by: dawn <dawn@tangled.org>
---
 web/src/lib/api/migrator.test.ts              |  9 ++++
 web/src/lib/api/migrator.ts                   |  5 ++
 .../import/GitHubAuthorizeStep.svelte         |  9 +++-
 .../components/import/GitHubImporter.svelte   |  2 +
 .../components/import/MigrationStep.svelte    | 19 +++++--
 .../components/import/UrlRepoImporter.svelte  | 32 ++++++++----
 .../import/gitHubImportState.svelte.ts        | 50 ++++++++++++++++---
 7 files changed, 102 insertions(+), 24 deletions(-)

diff --git a/web/src/lib/api/migrator.test.ts b/web/src/lib/api/migrator.test.ts
index 479307db3..d6923f6ae 100644
--- a/web/src/lib/api/migrator.test.ts
+++ b/web/src/lib/api/migrator.test.ts
@@ -8,6 +8,7 @@ import {
 	createMigrationTask,
 	describeSource,
 	getMigrationTask,
+	grantRefused,
 	listMigrationTasks,
 	migrationRunning,
 	describeSourcePermission,
@@ -237,6 +238,14 @@ describe("migrator client", () => {
 			expect(migrationRunning(running)).toBe(true);
 		expect(migrationRunning("fetching_from_the_future")).toBe(true);
 	});
+
+	it("asks for a new grant on jobs a pds refused a scope for, old failures included", () => {
+		expect(grantRefused("authorization_required")).toBe(true);
+		const refused = `repo record write returned HTTP 403: {"error":"ScopeMissingError"}`;
+		expect(grantRefused("failed", refused)).toBe(true);
+		expect(grantRefused("failed", "repo record write returned HTTP 500")).toBe(false);
+		expect(grantRefused("failed")).toBe(false);
+	});
 });
 
 describe("grant return path", () => {
diff --git a/web/src/lib/api/migrator.ts b/web/src/lib/api/migrator.ts
index a775b8912..04a889213 100644
--- a/web/src/lib/api/migrator.ts
+++ b/web/src/lib/api/migrator.ts
@@ -202,6 +202,11 @@ export const retryMigrationJob = async (
 export const migrationRunning = (status: string): boolean =>
 	!["completed", "failed", "authorization_required"].includes(status);
 
+// jobs that failed on a scope the pds refused, from before the migrator stopped them as
+// authorization_required, need a new grant too
+export const grantRefused = (status: string | undefined, error?: string): boolean =>
+	status === "authorization_required" || /ScopeMissingError|invalid_grant/.test(error ?? "");
+
 export const migrationsInFlight = (tasks: MigrationTask[]): boolean =>
 	tasks.some((task) => task.jobs.some((job) => migrationRunning(job.status)));
 
diff --git a/web/src/lib/components/import/GitHubAuthorizeStep.svelte b/web/src/lib/components/import/GitHubAuthorizeStep.svelte
index 303af9494..1a6e35e97 100644
--- a/web/src/lib/components/import/GitHubAuthorizeStep.svelte
+++ b/web/src/lib/components/import/GitHubAuthorizeStep.svelte
@@ -7,17 +7,24 @@
 	interface Props {
 		number: number;
 		authorized: boolean;
+		// the migrator refused a grant it had, so it has to be given a new one
+		again?: boolean;
 		disabled?: boolean;
 		onAuthorize: () => void;
 	}
 
-	let { number, authorized, disabled = false, onAuthorize }: Props = $props();
+	let { number, authorized, again = false, disabled = false, onAuthorize }: Props = $props();
 </script>
 
 <Step {number} title="Authorize the migrator" disabled={disabled && !authorized}>
 	<p class="typography-paragraph-small text-foreground-muted">
 		The migrator needs your permission to create the imported repositories in your account.
 	</p>
+	{#if again}
+		<p class="mt-1 typography-paragraph-small text-foreground-danger">
+			It lost access to your account, so authorize it again before migrating.
+		</p>
+	{/if}
 	<div class="mt-4 flex items-center justify-end gap-4">
 		{#if authorized}
 			<span
diff --git a/web/src/lib/components/import/GitHubImporter.svelte b/web/src/lib/components/import/GitHubImporter.svelte
index 111671a2f..81e572cb4 100644
--- a/web/src/lib/components/import/GitHubImporter.svelte
+++ b/web/src/lib/components/import/GitHubImporter.svelte
@@ -66,6 +66,7 @@
 		<GitHubAuthorizeStep
 			number={firstStep + 1}
 			authorized={importer.authorized}
+			again={importer.grantNeeded}
 			disabled={isAuthorizeGated({ account: importer.account, disabled: importer.disabled })}
 			onAuthorize={importer.authorize}
 		/>
@@ -97,6 +98,7 @@
 			userHandle={importer.userHandle}
 			bannerNote={migrationsNote}
 			onRetry={importer.retry}
+			onAuthorize={importer.grantNeeded ? importer.authorize : undefined}
 		/>
 	</StepFlow>
 </div>
diff --git a/web/src/lib/components/import/MigrationStep.svelte b/web/src/lib/components/import/MigrationStep.svelte
index e2b980c5d..2864f5ede 100644
--- a/web/src/lib/components/import/MigrationStep.svelte
+++ b/web/src/lib/components/import/MigrationStep.svelte
@@ -19,7 +19,8 @@
 	import ArrowRight from "$icon/arrow-right";
 	import CircleUserRound from "$icon/circle-user-round";
 	import RotateCcw from "$icon/rotate-ccw";
-	import { migrationRunning } from "$lib/api/migrator";
+	import UserKey from "$icon/user-key";
+	import { grantRefused, migrationRunning } from "$lib/api/migrator";
 	import Button from "$lib/components/ui/Button.svelte";
 	import MigrationRow, { stageLabel } from "./MigrationRow.svelte";
 	import StartedBanner from "./StartedBanner.svelte";
@@ -36,6 +37,8 @@
 		bannerNote?: string;
 		id?: string;
 		onRetry?: (item: MigrationItem) => void;
+		// set while a job needs a new grant, which puts an authorize button in the footer
+		onAuthorize?: () => void;
 	}
 
 	let {
@@ -48,7 +51,8 @@
 		userHandle,
 		bannerNote,
 		id,
-		onRetry
+		onRetry,
+		onAuthorize
 	}: Props = $props();
 
 	let bannerVisible = $state(true);
@@ -118,8 +122,8 @@
 							<p class="typography-paragraph-small break-all text-foreground-danger">
 								{m.status === "error"
 									? m.error
-									: m.jobStatus === "authorization_required"
-										? "The migrator lost access to the source or to your account. Authorize it again and retry."
+									: grantRefused(m.jobStatus, m.jobError)
+										? "The migrator lost access to your account or to the source. Authorize it again, then retry."
 										: (m.jobError ?? "The migration failed")}
 							</p>
 							{#if onRetry}
@@ -138,7 +142,12 @@
 			{/each}
 		</div>
 		{#if !isRunning}
-			<div class="mt-4 flex justify-end">
+			<div class="mt-4 flex justify-end gap-2">
+				{#if onAuthorize}
+					<Button variant="primary" icon={UserKey} onclick={onAuthorize}>
+						Authorize the migrator
+					</Button>
+				{/if}
 				<Button
 					href={userHandle
 						? (resolve(`/${userHandle}?tab=repos` as "/") as `/${string}`)
diff --git a/web/src/lib/components/import/UrlRepoImporter.svelte b/web/src/lib/components/import/UrlRepoImporter.svelte
index e8285e52c..52d7ff133 100644
--- a/web/src/lib/components/import/UrlRepoImporter.svelte
+++ b/web/src/lib/components/import/UrlRepoImporter.svelte
@@ -3,6 +3,7 @@
 	import { onDestroy } from "svelte";
 	import { replaceState } from "$app/navigation";
 	import RefreshCw from "$icon/refresh-cw";
+	import UserKey from "$icon/user-key";
 	import { getAuth } from "$lib/auth.svelte";
 	import { importFromUrl, type StartedMigration } from "$lib/api/githubImport";
 	import {
@@ -61,6 +62,7 @@
 	let urlKnot = $state("");
 	let urlError = $state("");
 	let needsSignIn = $state(false);
+	let grantUrl = $state<string | null>(null);
 	let running = $state(false);
 	let fetched = $state(false);
 	let defaultBranch = $state("");
@@ -151,6 +153,7 @@
 			url = cloneUrl;
 			running = true;
 			urlError = "";
+			grantUrl = null;
 			const user = auth.currentUser;
 			if (!user) throw new Error("Sign in first.");
 			const agent = await auth.agentFor(user.did);
@@ -167,26 +170,29 @@
 		} catch (cause) {
 			fetched = false;
 			defaultBranch = "";
-			if (await stashAndAskForGrant(cause)) return;
+			if (askForGrant(cause)) return;
 			urlError = messageOf(cause);
 		} finally {
 			running = false;
 		}
 	};
 
-	const stashAndAskForGrant = async (cause: unknown): Promise<boolean> => {
+	const askForGrant = (cause: unknown): boolean => {
 		if (!(cause instanceof MigrationGrantRequired)) return false;
-		stashDraft();
-		if (cause.startUrl) {
-			window.location.assign(cause.startUrl);
-		} else {
-			urlError =
-				"The migrator needs your authorization, but its start URL is unknown. Try again.";
-		}
+		grantUrl = cause.startUrl;
+		urlError = cause.startUrl
+			? "The migrator needs your authorization before it can import this repository."
+			: "The migrator needs your authorization, but its start URL is unknown. Try again.";
 		migration = { status: "idle" };
 		return true;
 	};
 
+	const grant = () => {
+		if (!grantUrl) return;
+		stashDraft();
+		window.location.assign(grantUrl);
+	};
+
 	// existing sessions cannot widen scope without re-authenticating
 	const signInAgain = async () => {
 		const did = auth.currentDid;
@@ -248,6 +254,7 @@
 			if (!urlKnot) throw new Error("Select a knot.");
 			running = true;
 			urlError = "";
+			grantUrl = null;
 			migration = { status: "submitting" };
 			const agent = await auth.agentFor(user.did);
 			const task = await importFromUrl(
@@ -267,7 +274,7 @@
 			pollDelay = POLL_MIN_MS;
 			schedulePoll();
 		} catch (cause) {
-			if (await stashAndAskForGrant(cause)) return;
+			if (askForGrant(cause)) return;
 			urlError = messageOf(cause);
 			migration = { status: "error", error: urlError };
 		} finally {
@@ -305,6 +312,11 @@
 	{/if}
 </p>
 <ErrorAlert label={urlError} class="mt-2" />
+{#if grantUrl}
+	<div class="mt-2">
+		<Button size="small" icon={UserKey} onclick={grant}>Authorize the migrator</Button>
+	</div>
+{/if}
 {#if needsSignIn}
 	<div class="mt-2 flex flex-wrap items-center gap-2">
 		<Button size="small" onclick={() => void signInAgain()}>Sign in again</Button>
diff --git a/web/src/lib/components/import/gitHubImportState.svelte.ts b/web/src/lib/components/import/gitHubImportState.svelte.ts
index 8d2ba1ea6..552b569a8 100644
--- a/web/src/lib/components/import/gitHubImportState.svelte.ts
+++ b/web/src/lib/components/import/gitHubImportState.svelte.ts
@@ -14,6 +14,7 @@ import {
 } from "$lib/api/githubImport";
 import {
 	getMigrationTask,
+	grantRefused,
 	listMigrationTasks,
 	migrationRunning,
 	MigrationGrantRequired,
@@ -144,6 +145,8 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 	let fetchedKnots = $state<readonly string[]>([]);
 	let grantConfirmed = $state(false);
 	let loadedTasks = $state<MigrationTask[]>([]);
+	let grantAsked = $state(false);
+	let grantedNow = $state(false);
 	let running = $state(false);
 	let sessionTaskCreated = $state(false);
 
@@ -217,13 +220,25 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 		})
 	);
 
+	// a grant made on this visit outranks the failures of jobs that ran before it
+	const grantNeeded = $derived(
+		grantAsked ||
+			(!grantedNow &&
+				migrationItems.some(
+					(item) =>
+						item.migration.status === "success" &&
+						grantRefused(item.migration.jobStatus, item.migration.jobError)
+				))
+	);
+
 	const authorized = $derived(
-		hasGrantEvidence({
-			granted: grantConfirmed,
-			url: page.url,
-			tasks: loadedTasks,
-			sessionCreated: sessionTaskCreated
-		})
+		!grantNeeded &&
+			hasGrantEvidence({
+				granted: grantConfirmed,
+				url: page.url,
+				tasks: loadedTasks,
+				sessionCreated: sessionTaskCreated
+			})
 	);
 
 	const connectUrl = $derived(
@@ -428,6 +443,8 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 			expanded = new Set();
 			grantConfirmed = false;
 			sessionTaskCreated = false;
+			grantAsked = false;
+			grantedNow = false;
 		}
 		if (did) {
 			void loadAccount(did, version);
@@ -447,6 +464,7 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 	const settleGrantLatch = (oauthError: string | null) => {
 		if (page.url.searchParams.get("migrator") === "granted" && !oauthError) {
 			grantConfirmed = true;
+			grantedNow = true;
 			grantLatch.confirm();
 			return;
 		}
@@ -524,6 +542,18 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 		window.location.assign(startUrl);
 	};
 
+	// a refused grant only asks: the picks go back to being selected, and authorize() takes them
+	// along to the migrator once the user says so
+	const askForGrant = (startUrl: string | null, picked: GitHubRepo[]) => {
+		if (!startUrl) {
+			redirectToGrant(startUrl, picked);
+			return;
+		}
+		for (const repo of picked)
+			updateDraft(repo.id, { selected: true, migration: { status: "idle" } });
+		grantAsked = true;
+	};
+
 	const authorize = () => {
 		if (running) return;
 		const did = auth.currentUser?.did;
@@ -575,7 +605,7 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 		} catch (cause) {
 			if (superseded(version, owner.did)) return "stale";
 			if (cause instanceof MigrationGrantRequired) {
-				redirectToGrant(cause.startUrl, batch);
+				askForGrant(cause.startUrl, picked.slice(start));
 				return "grant";
 			}
 			failMigrations(batch, messageOf(cause));
@@ -594,6 +624,7 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 			validateSelections(picked);
 			running = true;
 			errorMessage = "";
+			grantAsked = false;
 			const agent = await auth.agentFor(owner.did);
 			assertOwner(owner.did, version);
 			markSubmitting(picked);
@@ -616,7 +647,7 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 		} catch (cause) {
 			if (superseded(version, owner.did)) return;
 			if (cause instanceof MigrationGrantRequired) {
-				redirectToGrant(cause.startUrl, picked);
+				askForGrant(cause.startUrl, picked);
 				return;
 			}
 			failMigrations(picked, messageOf(cause));
@@ -749,6 +780,9 @@ export const createGitHubImporter = (options: GitHubImporterOptions) => {
 		get authorized() {
 			return authorized;
 		},
+		get grantNeeded() {
+			return grantNeeded;
+		},
 		authorize,
 		disconnect,
 
-- 
2.51.2


From e83f9c48e98aa291da785a7b8abe9652154f7bcc Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Wed, 7 Oct 2026 00:30:31 +0300
Subject: [PATCH 17/25] spindle/mill: give a freed seat to the owner with the
 fewest running workflows

Signed-off-by: dawn <dawn@tangled.org>
---
 spindle/mill/fair.go      |  86 +++++++++++++++++++++
 spindle/mill/fair_test.go | 157 ++++++++++++++++++++++++++++++++++++++
 spindle/mill/mill.go      |  72 +++++++++++------
 3 files changed, 291 insertions(+), 24 deletions(-)
 create mode 100644 spindle/mill/fair.go
 create mode 100644 spindle/mill/fair_test.go

diff --git a/spindle/mill/fair.go b/spindle/mill/fair.go
new file mode 100644
index 000000000..c23bb9c15
--- /dev/null
+++ b/spindle/mill/fair.go
@@ -0,0 +1,86 @@
+package mill
+
+// a workflow waiting in place for a seat. when a seat frees up, waiters bid
+// one at a time, starting with the owner that has the fewest workflows on
+// the fleet, so one owner's burst can't take every seat
+type placeWaiter struct {
+	owner     string
+	engine    string
+	seq       uint64
+	admitting bool
+	// set when a bid in triedGen found no seat. the waiter then sits out and
+	// stops blocking the ones behind it until something changes
+	tried    bool
+	triedGen uint64
+}
+
+func (w *placeWaiter) triedSince(gen uint64) bool {
+	return w.tried && w.triedGen == gen
+}
+
+func (m *Mill) joinPlacement(owner, engine string) *placeWaiter {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.waiterSeq++
+	w := &placeWaiter{owner: owner, engine: engine, seq: m.waiterSeq}
+	m.waiters[w] = struct{}{}
+	return w
+}
+
+func (m *Mill) leavePlacement(w *placeWaiter) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	delete(m.waiters, w)
+	m.wakeWaitersLocked()
+}
+
+// reports whether w may bid now, plus the generation and channel to wait on
+func (m *Mill) placementTurn(w *placeWaiter) (bool, uint64, <-chan struct{}) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if w.triedSince(m.changeGen) {
+		return false, m.changeGen, m.changeCh
+	}
+	running := m.runningByOwnerLocked()
+	for v := range m.waiters {
+		if v == w || v.engine != w.engine || v.admitting || v.triedSince(m.changeGen) {
+			continue
+		}
+		if running[v.owner] < running[w.owner] || (running[v.owner] == running[w.owner] && v.seq < w.seq) {
+			return false, m.changeGen, m.changeCh
+		}
+	}
+	return true, m.changeGen, m.changeCh
+}
+
+// the bid found no seat, so let the next waiter try
+func (m *Mill) passTurn(w *placeWaiter, gen uint64) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	w.tried = true
+	w.triedGen = gen
+	m.wakeWaitersLocked()
+}
+
+func (m *Mill) setAdmitting(w *placeWaiter, admitting bool) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	w.admitting = admitting
+	if !admitting {
+		m.wakeWaitersLocked()
+	}
+}
+
+// admitting waiters count too because they're about to hold a seat
+func (m *Mill) runningByOwnerLocked() map[string]int {
+	running := make(map[string]int)
+	for _, lease := range m.leases {
+		running[lease.ownerDID]++
+	}
+	for w := range m.waiters {
+		if w.admitting {
+			running[w.owner]++
+		}
+	}
+	return running
+}
diff --git a/spindle/mill/fair_test.go b/spindle/mill/fair_test.go
new file mode 100644
index 000000000..5a79011b1
--- /dev/null
+++ b/spindle/mill/fair_test.go
@@ -0,0 +1,157 @@
+package mill
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"tangled.org/core/spindle/models"
+
+	millproto "tangled.org/core/spindle/mill/proto"
+	millv1 "tangled.org/core/spindle/mill/proto/gen"
+)
+
+func workflowOwnedBy(wf *models.Workflow, owner string) *models.Workflow {
+	state := wf.Data.(*millWorkflowState)
+	md := *state.RawPipeline.TriggerMetadata
+	repo := *md.Repo
+	repo.Did = owner
+	md.Repo = &repo
+	state.RawPipeline.TriggerMetadata = &md
+	return wf
+}
+
+// a node that turns bids away until the test frees seats on it
+type seatedNode struct {
+	mu   sync.Mutex
+	free int
+}
+
+func (n *seatedNode) open(m *Mill, seats int) {
+	n.mu.Lock()
+	n.free += seats
+	n.mu.Unlock()
+	m.notifyChange()
+}
+
+func addSeatedNode(t *testing.T, m *Mill) *seatedNode {
+	t.Helper()
+	node := &seatedNode{}
+	var sess *millSession
+	sess = addCandidateSession(t, m, "node-a", nil, 0, scriptedEncoder(func(msg *millproto.Message) error {
+		rs := msg.GetReserveSeat()
+		if rs == nil {
+			return nil
+		}
+		node.mu.Lock()
+		accepted := node.free > 0
+		if accepted {
+			node.free--
+		}
+		node.mu.Unlock()
+		sess.deliver(rs.GetLeaseId(), &millproto.Message{ReserveResult: &millv1.ReserveResult{
+			LeaseId:  rs.GetLeaseId(),
+			Accepted: accepted,
+		}})
+		return nil
+	}))
+	return node
+}
+
+func runningFor(m *Mill, owner string, n int) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for range n {
+		lease := newLease(m.nextLeaseIDLocked(), "node-a", "", "dummy")
+		lease.ownerDID = owner
+		m.leases[lease.id] = lease
+	}
+}
+
+func waitForTriedWaiters(t *testing.T, m *Mill, want int) {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		m.mu.Lock()
+		tried := 0
+		for w := range m.waiters {
+			if w.tried {
+				tried++
+			}
+		}
+		m.mu.Unlock()
+		if tried == want {
+			return
+		}
+		time.Sleep(time.Millisecond)
+	}
+	t.Fatalf("waiters never all bid, want %d", want)
+}
+
+func placeInBackground(t *testing.T, m *Mill, wf *models.Workflow) <-chan string {
+	t.Helper()
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	placed := make(chan string, 1)
+	go func() {
+		wid := models.WorkflowId{PipelineId: models.PipelineId("p"), Name: wf.Name}
+		if _, err := m.place(ctx, "dummy", wid, wf); err != nil {
+			placed <- "error: " + err.Error()
+			return
+		}
+		placed <- wf.Name
+	}()
+	return placed
+}
+
+func nextPlaced(t *testing.T, placed ...<-chan string) string {
+	t.Helper()
+	cases := make(chan string, len(placed))
+	for _, ch := range placed {
+		go func() {
+			if name, ok := <-ch; ok {
+				cases <- name
+			}
+		}()
+	}
+	select {
+	case name := <-cases:
+		return name
+	case <-time.After(2 * time.Second):
+		t.Fatal("no workflow was placed")
+		return ""
+	}
+}
+
+func TestFreedSeatGoesToTheOwnerWithFewerRunning(t *testing.T) {
+	m, _ := quotaMill(t, &scriptedStore{})
+	node := addSeatedNode(t, m)
+	runningFor(m, "did:plc:busy", 2)
+
+	busy := placeInBackground(t, m, workflowOwnedBy(testWorkflow("busy"), "did:plc:busy"))
+	waitForTriedWaiters(t, m, 1)
+	quiet := placeInBackground(t, m, workflowOwnedBy(testWorkflow("quiet"), "did:plc:quiet"))
+	waitForTriedWaiters(t, m, 2)
+
+	node.open(m, 1)
+	if got := nextPlaced(t, busy, quiet); got != "quiet" {
+		t.Fatalf("first freed seat went to %q, want the owner with nothing running", got)
+	}
+}
+
+func TestWaiterThatCannotFitDoesNotBlockTheRest(t *testing.T) {
+	m, _ := quotaMill(t, &scriptedStore{})
+	node := addSeatedNode(t, m)
+	runningFor(m, "did:plc:busy", 2)
+
+	// ranks first but no node has the label it needs
+	gpu := placeInBackground(t, m, workflowOwnedBy(testWorkflowWithRunsOn("gpu", []string{"gpu"}), "did:plc:quiet"))
+	busy := placeInBackground(t, m, workflowOwnedBy(testWorkflow("busy"), "did:plc:busy"))
+	waitForTriedWaiters(t, m, 2)
+
+	node.open(m, 1)
+	if got := nextPlaced(t, gpu, busy); got != "busy" {
+		t.Fatalf("freed seat went to %q, want the waiter that fits", got)
+	}
+}
diff --git a/spindle/mill/mill.go b/spindle/mill/mill.go
index 7d0d02436..455b9d3d1 100644
--- a/spindle/mill/mill.go
+++ b/spindle/mill/mill.go
@@ -92,6 +92,9 @@ type Mill struct {
 	protoStrikes     map[string]int
 	pending          int
 	changeCh         chan struct{}
+	changeGen        uint64
+	waiters          map[*placeWaiter]struct{}
+	waiterSeq        uint64
 
 	cacheDeleteOnce  sync.Once
 	cacheDeleteMu    sync.Mutex
@@ -135,6 +138,7 @@ func New(l *slog.Logger, cfg Config) *Mill {
 		nodeSeqno:    make(map[string]uint64),
 		changeCh:     make(chan struct{}),
 		quotaLeases:  make(map[string]*RemoteLease),
+		waiters:      make(map[*placeWaiter]struct{}),
 	}
 	m.startCacheDeletionWorker()
 	return m
@@ -179,10 +183,13 @@ func (m *Mill) ensureCacheSentinel(store storage.Storage) {
 
 func (m *Mill) nextLeaseID() string {
 	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.nextLeaseIDLocked()
+}
+
+func (m *Mill) nextLeaseIDLocked() string {
 	m.leaseSeq++
-	seq := m.leaseSeq
-	m.mu.Unlock()
-	return fmt.Sprintf("%s-%d", tid.TID(), seq)
+	return fmt.Sprintf("%s-%d", tid.TID(), m.leaseSeq)
 }
 
 func (m *Mill) dropReservation(id string) {
@@ -198,6 +205,11 @@ func (m *Mill) notifyChange() {
 }
 
 func (m *Mill) notifyChangeLocked() {
+	m.changeGen++
+	m.wakeWaitersLocked()
+}
+
+func (m *Mill) wakeWaitersLocked() {
 	close(m.changeCh)
 	m.changeCh = make(chan struct{})
 }
@@ -459,6 +471,13 @@ func (m *Mill) place(ctx context.Context, engineName string, wid models.Workflow
 		m.mu.Unlock()
 	}()
 
+	owner := wf.OwnerDID
+	if id, ok := m.quotaIdentity(wf); ok {
+		owner = id.OwnerDID
+	}
+	w := m.joinPlacement(owner, engineName)
+	defer m.leavePlacement(w)
+
 	for {
 		if err := ctx.Err(); err != nil {
 			placementResult = "cancelled"
@@ -468,30 +487,35 @@ func (m *Mill) place(ctx context.Context, engineName string, wid models.Workflow
 			return nil, err
 		}
 
-		// capture the channel before bidding
-		// a change during the bid makes the next wait return immediately
-		ch := m.currentChangeCh()
-
-		lease, err := m.bid(ctx, engineName, wid, wf)
-		if err != nil {
-			return nil, err
-		}
-		if lease != nil {
-			wf.CacheNamespace = lease.cacheNamespace
-			slot, retry, err := m.admit(ctx, engineName, wid, wf, lease)
+		// the channel comes from before the bid, so a change during the bid
+		// makes the next wait return immediately
+		turn, gen, ch := m.placementTurn(w)
+		if turn {
+			lease, err := m.bid(ctx, engineName, wid, wf)
 			if err != nil {
-				var failure *engine.WorkflowFailure
-				if errors.As(err, &failure) && failure.Class == engine.FailureClassPolicy {
-					placementResult = "rejected"
-				}
 				return nil, err
 			}
-			if slot != nil {
-				placementResult = "success"
-				return slot, nil
-			}
-			if retry {
-				continue
+			if lease == nil {
+				m.passTurn(w, gen)
+			} else {
+				wf.CacheNamespace = lease.cacheNamespace
+				m.setAdmitting(w, true)
+				slot, retry, err := m.admit(ctx, engineName, wid, wf, lease)
+				m.setAdmitting(w, false)
+				if err != nil {
+					var failure *engine.WorkflowFailure
+					if errors.As(err, &failure) && failure.Class == engine.FailureClassPolicy {
+						placementResult = "rejected"
+					}
+					return nil, err
+				}
+				if slot != nil {
+					placementResult = "success"
+					return slot, nil
+				}
+				if retry {
+					continue
+				}
 			}
 		}
 
-- 
2.51.2


From 5e62dde00c478f093d009b54780e735a2ba0c75c Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Wed, 7 Oct 2026 00:38:18 +0300
Subject: [PATCH 18/25] spindle: drop the unused QUEUE_SIZE setting

Signed-off-by: dawn <dawn@tangled.org>
---
 nix/modules/spindle.nix  | 8 --------
 nix/vm.nix               | 1 -
 spindle/config/config.go | 1 -
 3 files changed, 10 deletions(-)

diff --git a/nix/modules/spindle.nix b/nix/modules/spindle.nix
index ddc11397a..876cf13c9 100644
--- a/nix/modules/spindle.nix
+++ b/nix/modules/spindle.nix
@@ -76,13 +76,6 @@ in
             description = "Maximum number of concurrent jobs to run";
           };
 
-          queueSize = mkOption {
-            type = types.int;
-            default = 100;
-            example = 100;
-            description = "Maximum number of jobs queue up";
-          };
-
           corsOrigins = mkOption {
             type = types.listOf types.str;
             default = [];
@@ -730,7 +723,6 @@ in
         "SPINDLE_SERVER_DEV=${lib.boolToString instance.server.dev}"
         "SPINDLE_SERVER_OWNER=${instance.server.owner}"
         "SPINDLE_SERVER_MAX_JOB_COUNT=${toString instance.server.maxJobCount}"
-        "SPINDLE_SERVER_QUEUE_SIZE=${toString instance.server.queueSize}"
         "SPINDLE_SERVER_CORS_ORIGINS=${concatStringsSep "," instance.server.corsOrigins}"
         "SPINDLE_SERVER_SECRETS_PROVIDER=${instance.server.secrets.provider}"
         "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${instance.server.secrets.openbao.proxyAddr}"
diff --git a/nix/vm.nix b/nix/vm.nix
index b312f18b2..bcfddf015 100644
--- a/nix/vm.nix
+++ b/nix/vm.nix
@@ -155,7 +155,6 @@ in
             jetstreamEndpoint = jetstream;
             listenAddr = "0.0.0.0:6555";
             dev = true;
-            queueSize = 100;
             maxJobCount = 2;
             secrets = {
               provider = "sqlite";
diff --git a/spindle/config/config.go b/spindle/config/config.go
index 61d263020..7a8ca0657 100644
--- a/spindle/config/config.go
+++ b/spindle/config/config.go
@@ -26,7 +26,6 @@ type Server struct {
 	Owner             string   `env:"OWNER"`
 	Secrets           Secrets  `env:",prefix=SECRETS_"`
 	LogDir            string   `env:"LOG_DIR, default=/var/log/spindle"`
-	QueueSize         int      `env:"QUEUE_SIZE, default=100"`
 	MaxJobCount       int      `env:"MAX_JOB_COUNT, default=2"` // max number of pipelines that run at a time
 	DockerSocket      string   `env:"DOCKER_SOCKET"`            // path to a docker socket to expose to workflow containers
 	CorsOrigins       []string `env:"CORS_ORIGINS"`             // origins browsers may call xrpc from, "*" for any. empty leaves cors off
-- 
2.51.2


From fc0bae46b827067e7f893c9fd04eab5ff8f2b91e Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Wed, 7 Oct 2026 00:38:18 +0300
Subject: [PATCH 19/25] spindle: dequeue the next pipeline from the owner with
 the fewest running

Signed-off-by: dawn <dawn@tangled.org>
---
 spindle/db/jobs.go          |  65 ++++++++++++++----
 spindle/db/jobs_test.go     |  48 +++++++++++++-
 spindle/queue/queue.go      | 127 ------------------------------------
 spindle/queue/queue_test.go | 103 -----------------------------
 spindle/server.go           |  81 +++++++++++++++--------
 5 files changed, 150 insertions(+), 274 deletions(-)
 delete mode 100644 spindle/queue/queue.go
 delete mode 100644 spindle/queue/queue_test.go

diff --git a/spindle/db/jobs.go b/spindle/db/jobs.go
index 9d2a91768..d85bff8f9 100644
--- a/spindle/db/jobs.go
+++ b/spindle/db/jobs.go
@@ -14,6 +14,7 @@ import (
 
 type JobRow struct {
 	Id          int64
+	Owner       string // the repo owner, or the repo itself if it isn't known yet
 	RepoDid     string
 	PipelineId  models.PipelineId
 	SourceRepo  *tangled.Pipeline_TriggerRepo
@@ -126,23 +127,61 @@ func (d *DB) enqueueJob(
 	}
 	return nil
 }
-func (d *DB) DequeueJob(ctx context.Context) (*JobRow, error) {
-	var row JobRow
+
+// takes the oldest job of the owner with the fewest jobs running, so one
+// owner's burst of pipelines can't hold every job worker
+func (d *DB) DequeueJob(ctx context.Context, running map[string]int) (*JobRow, error) {
+	tx, err := d.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+
+	rows, err := tx.QueryContext(ctx, `
+		select owner, min(id) from (
+			select id, coalesce(
+				(select r.owner from repos r where r.repo_did = jobs.repo_did limit 1),
+				repo_did
+			) as owner
+			from jobs
+		)
+		group by owner
+	`)
+	if err != nil {
+		return nil, err
+	}
+	var owner string
+	var id int64
+	found := false
+	for rows.Next() {
+		var o string
+		var i int64
+		if err := rows.Scan(&o, &i); err != nil {
+			rows.Close()
+			return nil, err
+		}
+		if !found || running[o] < running[owner] || (running[o] == running[owner] && i < id) {
+			owner, id, found = o, i, true
+		}
+	}
+	if err := errors.Join(rows.Err(), rows.Close()); err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, nil
+	}
+
+	row := JobRow{Owner: owner}
 	var sourceRepoStr *string
 	var tplJson string
-	err := d.QueryRowContext(ctx, `
-		delete from jobs
-		where id = (
-			select id from jobs
-			order by id asc
-			limit 1
-		)
+	err = tx.QueryRowContext(ctx, `
+		delete from jobs where id = ?
 		returning id, repo_did, pipeline_id, source_repo, tpl, traceparent, tracestate, created_at_ns
-	`).Scan(&row.Id, &row.RepoDid, &row.PipelineId, &sourceRepoStr, &tplJson, &row.Traceparent, &row.Tracestate, &row.CreatedAtNs)
+	`, id).Scan(&row.Id, &row.RepoDid, &row.PipelineId, &sourceRepoStr, &tplJson, &row.Traceparent, &row.Tracestate, &row.CreatedAtNs)
 	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil
-		}
+		return nil, err
+	}
+	if err := tx.Commit(); err != nil {
 		return nil, err
 	}
 	if err := json.Unmarshal([]byte(tplJson), &row.Tpl); err != nil {
diff --git a/spindle/db/jobs_test.go b/spindle/db/jobs_test.go
index f908f4ab5..596bd4515 100644
--- a/spindle/db/jobs_test.go
+++ b/spindle/db/jobs_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"path/filepath"
+	"slices"
 	"testing"
 	"time"
 
@@ -35,7 +36,7 @@ func TestJobTraceContextSurvivesEnqueue(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	job, err := database.DequeueJob(ctx)
+	job, err := database.DequeueJob(ctx, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -81,7 +82,7 @@ func TestEnqueueJobWithPendingCommitsJobAndStatusesTogether(t *testing.T) {
 	if status != models.StatusKindPending {
 		t.Fatalf("status = %q, want pending", status)
 	}
-	job, err := database.DequeueJob(ctx)
+	job, err := database.DequeueJob(ctx, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -129,7 +130,7 @@ func TestMakeMigratesLegacyJobTimestampToNanoseconds(t *testing.T) {
 		t.Fatal(err)
 	}
 	t.Cleanup(func() { database.Close() })
-	job, err := database.DequeueJob(ctx)
+	job, err := database.DequeueJob(ctx, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -137,3 +138,44 @@ func TestMakeMigratesLegacyJobTimestampToNanoseconds(t *testing.T) {
 		t.Fatalf("created_at_ns = %v, want converted legacy seconds", job)
 	}
 }
+
+func TestDequeueJobPrefersTheOwnerWithFewerRunning(t *testing.T) {
+	ctx := context.Background()
+	database, err := Make(ctx, filepath.Join(t.TempDir(), "spindle.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { database.Close() })
+
+	if err := database.AddRepo(Repo{Knot: "knot.example.com", Owner: "did:plc:busy", Rkey: "busy", RepoDid: "did:plc:busyrepo"}); err != nil {
+		t.Fatal(err)
+	}
+	enqueue := func(repoDid, pipeline string) {
+		t.Helper()
+		if err := database.EnqueueJob(ctx, repoDid, models.PipelineId(pipeline), nil, tangled.Pipeline{}, "", ""); err != nil {
+			t.Fatal(err)
+		}
+	}
+	enqueue("did:plc:busyrepo", "busy-1")
+	enqueue("did:plc:busyrepo", "busy-2")
+	// no repo row yet, so the repo stands in for its owner
+	enqueue("did:plc:quietrepo", "quiet-1")
+
+	running := map[string]int{"did:plc:busy": 1}
+	var order []string
+	for {
+		job, err := database.DequeueJob(ctx, running)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if job == nil {
+			break
+		}
+		order = append(order, string(job.PipelineId))
+		running[job.Owner]++
+	}
+	want := []string{"quiet-1", "busy-1", "busy-2"}
+	if !slices.Equal(order, want) {
+		t.Fatalf("dequeue order = %v, want %v", order, want)
+	}
+}
diff --git a/spindle/queue/queue.go b/spindle/queue/queue.go
deleted file mode 100644
index b90aa09e3..000000000
--- a/spindle/queue/queue.go
+++ /dev/null
@@ -1,127 +0,0 @@
-package queue
-
-import (
-	"slices"
-	"sync"
-
-	"github.com/bluesky-social/indigo/atproto/syntax"
-)
-
-type Job struct {
-	Run    func() error
-	OnFail func(error)
-}
-
-type ownedJob struct {
-	owner syntax.DID
-	job   Job
-}
-
-// prefers users with fewer running jobs, otherwise it's FIFO
-type Queue struct {
-	mu      sync.Mutex
-	cond    *sync.Cond
-	queue   []ownedJob
-	running map[syntax.DID]int
-	maxSize int
-	workers int
-	stopped bool
-	wg      sync.WaitGroup
-}
-
-func NewQueue(queueSize, numWorkers int) *Queue {
-	q := &Queue{
-		maxSize: queueSize,
-		workers: numWorkers,
-		running: make(map[syntax.DID]int),
-	}
-	q.cond = sync.NewCond(&q.mu)
-	return q
-}
-
-// todo(dawn): add a per-user cap so a single user can't fill the queue
-func (q *Queue) Enqueue(owner syntax.DID, job Job) bool {
-	q.mu.Lock()
-	defer q.mu.Unlock()
-	if q.stopped || len(q.queue) >= q.maxSize {
-		return false
-	}
-	q.queue = append(q.queue, ownedJob{owner: owner, job: job})
-	q.cond.Signal()
-	return true
-}
-
-func (q *Queue) Start() {
-	for range q.workers {
-		q.wg.Add(1)
-		go q.worker()
-	}
-}
-
-func (q *Queue) worker() {
-	defer q.wg.Done()
-	for {
-		picked, ok := q.takeNext()
-		if !ok {
-			return
-		}
-
-		err := picked.job.Run()
-		if err != nil && picked.job.OnFail != nil {
-			picked.job.OnFail(err)
-		}
-
-		q.finish(picked.owner)
-	}
-}
-
-// get or wait for the next job
-func (q *Queue) takeNext() (ownedJob, bool) {
-	q.mu.Lock()
-	defer q.mu.Unlock()
-
-	for len(q.queue) == 0 && !q.stopped {
-		q.cond.Wait() // waiting for jobs
-	}
-	if q.stopped && len(q.queue) == 0 {
-		return ownedJob{}, false // no jobs are left and the queue is stopped
-	}
-
-	idx := q.pickBest()
-	picked := q.queue[idx]
-	q.queue = slices.Delete(q.queue, idx, idx+1)
-	q.running[picked.owner]++
-
-	return picked, true
-}
-
-// index of the queued job whose owner has the fewest currently-running jobs,
-// tiebreaking by arrival order.
-func (q *Queue) pickBest() int {
-	best := 0
-	for idx, job := range q.queue {
-		if q.running[job.owner] < q.running[q.queue[best].owner] {
-			best = idx
-		}
-	}
-	return best
-}
-
-// called when finishing a job
-func (q *Queue) finish(owner syntax.DID) {
-	q.mu.Lock()
-	defer q.mu.Unlock()
-
-	q.running[owner]--
-	if q.running[owner] <= 0 {
-		delete(q.running, owner)
-	}
-}
-
-func (q *Queue) Stop() {
-	q.mu.Lock()
-	q.stopped = true
-	q.cond.Broadcast()
-	q.mu.Unlock()
-	q.wg.Wait()
-}
diff --git a/spindle/queue/queue_test.go b/spindle/queue/queue_test.go
deleted file mode 100644
index cadb0ab6d..000000000
--- a/spindle/queue/queue_test.go
+++ /dev/null
@@ -1,103 +0,0 @@
-package queue
-
-import (
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/bluesky-social/indigo/atproto/syntax"
-)
-
-const (
-	alice = syntax.DID("did:plc:alice")
-	eve   = syntax.DID("did:plc:eve")
-	dawn  = syntax.DID("did:plc:dawn")
-)
-
-func TestQueueDrainsAllJobs(t *testing.T) {
-	t.Parallel()
-
-	q := NewQueue(10, 2)
-	q.Start()
-
-	var mu sync.Mutex
-	var ran []string
-	done := make(chan struct{}, 5)
-
-	for _, name := range []string{"a", "b", "c", "d", "e"} {
-		q.Enqueue(dawn, Job{Run: func() error {
-			mu.Lock()
-			ran = append(ran, name)
-			mu.Unlock()
-			done <- struct{}{}
-			return nil
-		}})
-	}
-
-	for range 5 {
-		select {
-		case <-done:
-		case <-time.After(time.Second):
-			t.Fatal("timed out waiting for jobs to finish")
-		}
-	}
-
-	q.Stop()
-
-	if len(ran) != 5 {
-		t.Fatalf("expected 5 jobs, ran %d", len(ran))
-	}
-}
-
-func TestQueueRejectsWhenFull(t *testing.T) {
-	t.Parallel()
-
-	// no workers, so the queue never drains
-	q := NewQueue(2, 0)
-
-	if !q.Enqueue(dawn, Job{Run: func() error { return nil }}) {
-		t.Fatal("first Enqueue() returned false, want true")
-	}
-	if !q.Enqueue(dawn, Job{Run: func() error { return nil }}) {
-		t.Fatal("second Enqueue() returned false, want true")
-	}
-	if q.Enqueue(dawn, Job{Run: func() error { return nil }}) {
-		t.Fatal("third Enqueue() returned true on full queue, want false")
-	}
-}
-
-func TestQueuePrefersOwnerWithFewestRunning(t *testing.T) {
-	t.Parallel()
-
-	// 2 workers. alice gets the first slot; while she's holding it, eve's
-	// job should win the second slot over alice's own queued waiters.
-	q := NewQueue(20, 2)
-
-	releaseAlice1 := make(chan struct{})
-	gotEve := make(chan struct{}, 1)
-
-	q.Enqueue(alice, Job{Run: func() error {
-		<-releaseAlice1
-		return nil
-	}})
-	// alice queues two more
-	q.Enqueue(alice, Job{Run: func() error { return nil }})
-	q.Enqueue(alice, Job{Run: func() error { return nil }})
-	// eve queues one
-	q.Enqueue(eve, Job{Run: func() error {
-		gotEve <- struct{}{}
-		return nil
-	}})
-
-	q.Start()
-
-	// eve should run while alice's first is still held
-	select {
-	case <-gotEve:
-	case <-time.After(time.Second):
-		t.Fatal("eve's job did not run while alice was blocked")
-	}
-
-	close(releaseAlice1)
-	q.Stop()
-}
diff --git a/spindle/server.go b/spindle/server.go
index 3882893fa..9fba209d9 100644
--- a/spindle/server.go
+++ b/spindle/server.go
@@ -78,33 +78,35 @@ type executorClient interface {
 }
 
 type Spindle struct {
-	jc         *jetstream.JetstreamClient
-	tap        *Tap
-	embedTap   *embeddedTap
-	db         *db.DB
-	e          *rbac.Enforcer
-	l          *slog.Logger
-	n          *notifier.Notifier
-	wh         *webhook.Service
-	metrics    *observability.Metrics
-	engs       map[string]models.Engine
-	jobWake    chan struct{}
-	jobWorkers sync.WaitGroup
-	scheduler  *pipelineScheduler
-	cfg        *config.Config
-	feed       *feed.Feed
-	res        *idresolver.Resolver
-	verify     repoverify.Verifier
-	vault      secrets.Manager
-	cache      storage.Storage
-	motd       []byte
-	motdMu     sync.RWMutex
-	rootCtx    context.Context
-	rootCancel context.CancelFunc
-	store      artifactstore.Store
-	stores     *artifactstore.Stores
-	reader     artifactstore.Reader
-	qm         *quota.Manager
+	jc          *jetstream.JetstreamClient
+	tap         *Tap
+	embedTap    *embeddedTap
+	db          *db.DB
+	e           *rbac.Enforcer
+	l           *slog.Logger
+	n           *notifier.Notifier
+	wh          *webhook.Service
+	metrics     *observability.Metrics
+	engs        map[string]models.Engine
+	jobWake     chan struct{}
+	jobWorkers  sync.WaitGroup
+	jobsMu      sync.Mutex
+	jobsByOwner map[string]int
+	scheduler   *pipelineScheduler
+	cfg         *config.Config
+	feed        *feed.Feed
+	res         *idresolver.Resolver
+	verify      repoverify.Verifier
+	vault       secrets.Manager
+	cache       storage.Storage
+	motd        []byte
+	motdMu      sync.RWMutex
+	rootCtx     context.Context
+	rootCancel  context.CancelFunc
+	store       artifactstore.Store
+	stores      *artifactstore.Stores
+	reader      artifactstore.Reader
+	qm          *quota.Manager
 	// set only when this spindle hosts the mill or joins one as an executor
 	mill              *mill.Mill
 	oidc              *oidc.Issuer
@@ -1244,7 +1246,7 @@ func (s *Spindle) StartJobWorkers(ctx context.Context) {
 		go func() {
 			defer s.jobWorkers.Done()
 			for {
-				job, err := s.db.DequeueJob(ctx)
+				job, err := s.dequeueJob(ctx)
 				if err != nil {
 					s.l.Error("failed to dequeue job", "error", err)
 				}
@@ -1259,11 +1261,34 @@ func (s *Spindle) StartJobWorkers(ctx context.Context) {
 				}
 				s.metrics.RecordJobQueueActivity("dequeue")
 				s.runJob(ctx, job)
+				s.finishJob(job.Owner)
 			}
 		}()
 	}
 }
 
+func (s *Spindle) dequeueJob(ctx context.Context) (*db.JobRow, error) {
+	s.jobsMu.Lock()
+	defer s.jobsMu.Unlock()
+	job, err := s.db.DequeueJob(ctx, s.jobsByOwner)
+	if job != nil {
+		if s.jobsByOwner == nil {
+			s.jobsByOwner = make(map[string]int)
+		}
+		s.jobsByOwner[job.Owner]++
+	}
+	return job, err
+}
+
+func (s *Spindle) finishJob(owner string) {
+	s.jobsMu.Lock()
+	defer s.jobsMu.Unlock()
+	s.jobsByOwner[owner]--
+	if s.jobsByOwner[owner] <= 0 {
+		delete(s.jobsByOwner, owner)
+	}
+}
+
 func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) {
 	pipelineId := job.PipelineId
 	repoDID := job.RepoDid
-- 
2.51.2


From 61d041190593d8eff1e67fc67eb59c1b9e4dff4b Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Wed, 7 Oct 2026 00:38:18 +0300
Subject: [PATCH 20/25] spindle/microvm: hold one failed VM per repo and per
 pusher

Signed-off-by: dawn <dawn@tangled.org>
---
 spindle/engines/microvm/README.md     |  4 ++
 spindle/engines/microvm/debug.go      | 76 ++++++++++++++++++++++-----
 spindle/engines/microvm/debug_test.go | 70 ++++++++++++++++++++++++
 spindle/engines/microvm/engine.go     | 13 +++--
 spindle/engines/microvm/vm.go         |  1 +
 5 files changed, 149 insertions(+), 15 deletions(-)

diff --git a/spindle/engines/microvm/README.md b/spindle/engines/microvm/README.md
index be7439e3d..5a79c5c7f 100644
--- a/spindle/engines/microvm/README.md
+++ b/spindle/engines/microvm/README.md
@@ -249,6 +249,10 @@ so you can poke at the failed VM interactively. Spindle terminates the ssh
 connection itself and bridges a pty into the live guest over the agent's
 vsock; the guest stays keyless and never runs an ssh daemon.
 
+A held VM keeps its seat, so each executor holds at most one per repo and one
+per pusher. A newer failure replaces a held VM nobody has connected to yet. While
+someone is connected, new failures from that repo or pusher aren't held at all.
+
 Access mirrors a git push: the ssh username is the job id, and the offered
 public key is sent to the job's repo knot (`sh.tangled.repo.checkPushAllowed`).
 The session is accepted only if that key is allowed to push to the job's repo.
diff --git a/spindle/engines/microvm/debug.go b/spindle/engines/microvm/debug.go
index 0c536e596..15e980216 100644
--- a/spindle/engines/microvm/debug.go
+++ b/spindle/engines/microvm/debug.go
@@ -32,6 +32,22 @@ type debugTarget struct {
 	maxAliveAt time.Time
 	connected  chan struct{} // closed when the user first ssh's in, ending the grace window
 	released   chan struct{} // closed when the user exits the debug shell, to tear down early
+	evicted    chan struct{} // closed when a newer failure takes over the hold
+	holdRepo   string
+	actor      string
+}
+
+func (t debugTarget) sharesHoldWith(other debugTarget) bool {
+	return (t.holdRepo != "" && t.holdRepo == other.holdRepo) || (t.actor != "" && t.actor == other.actor)
+}
+
+func isClosed(ch chan struct{}) bool {
+	select {
+	case <-ch:
+		return true
+	default:
+		return false
+	}
 }
 
 var debugHandleSafe = regexp.MustCompile(`[^a-zA-Z0-9_.-]`)
@@ -53,6 +69,30 @@ func (e *Engine) registerDebugTarget(wid models.WorkflowId, t debugTarget) {
 	e.debug[newDebugHandle(wid)] = t
 }
 
+// a held VM keeps its whole seat, so each repo and each pusher get one at a
+// time. a newer failure replaces a hold nobody has connected to yet, and
+// false means someone is already debugging one
+func (e *Engine) claimDebugHold(wid models.WorkflowId, t debugTarget) bool {
+	e.debugMu.Lock()
+	defer e.debugMu.Unlock()
+	var replaced []string
+	for handle, held := range e.debug {
+		if !held.sharesHoldWith(t) {
+			continue
+		}
+		if isClosed(held.connected) {
+			return false
+		}
+		replaced = append(replaced, handle)
+	}
+	for _, handle := range replaced {
+		close(e.debug[handle].evicted)
+		delete(e.debug, handle)
+	}
+	e.debug[newDebugHandle(wid)] = t
+	return true
+}
+
 func (e *Engine) unregisterDebugTarget(wid models.WorkflowId) {
 	e.debugMu.Lock()
 	defer e.debugMu.Unlock()
@@ -60,16 +100,18 @@ func (e *Engine) unregisterDebugTarget(wid models.WorkflowId) {
 }
 
 // ends the grace window so we hold the VM until the shell exits instead.
-func (e *Engine) markDebugConnected(handle string) {
+// false means a newer failure replaced this hold while the shell was opening
+func (e *Engine) markDebugConnected(handle string) bool {
 	e.debugMu.Lock()
 	defer e.debugMu.Unlock()
-	if t, ok := e.debug[handle]; ok {
-		select {
-		case <-t.connected: // already signalled
-		default:
-			close(t.connected)
-		}
+	t, ok := e.debug[handle]
+	if !ok {
+		return false
+	}
+	if !isClosed(t.connected) {
+		close(t.connected)
 	}
+	return true
 }
 
 func (e *Engine) releaseDebugTarget(handle string) {
@@ -128,12 +170,16 @@ func (e *Engine) OpenDebugSession(ctx context.Context, jobID, term string, rows,
 		return nil, fmt.Errorf("accept debug shell connection: %w", err)
 	}
 
-	e.markDebugConnected(jobID)
+	if !e.markDebugConnected(jobID) {
+		_ = conn.Close()
+		_ = ln.Close()
+		return nil, fmt.Errorf("a newer failed run replaced the microVM for job %q", jobID)
+	}
 
 	return newDebugSession(conn, ln, e.l), nil
 }
 
-func (e *Engine) writeDebugHint(wid models.WorkflowId, idx int, wfLogger models.WorkflowLogger) {
+func (e *Engine) writeDebugHint(wid models.WorkflowId, idx int, wfLogger models.WorkflowLogger, held bool) {
 	if wfLogger == nil {
 		wfLogger = models.NullLogger{}
 	}
@@ -142,11 +188,14 @@ func (e *Engine) writeDebugHint(wid models.WorkflowId, idx int, wfLogger models.
 	wfLogger.ControlWriter(idx, step, models.StepStatusStart).Write([]byte{0})
 	defer wfLogger.ControlWriter(idx, step, models.StepStatusEnd).Write([]byte{0})
 
+	out := wfLogger.DataWriter(idx, "stdout")
+	if !held {
+		fmt.Fprintln(out, "Workflow failed, but someone is already debugging another failed run from this repo or pusher, so this VM won't be kept.")
+		return
+	}
 	ssh := e.cfg.MicroVMPipelines.DebugSSH
-	grace := ssh.GracePeriod
 	cmd := debugSSHCommand(ssh.ListenAddr, e.cfg.Server.Hostname, ssh.Host, ssh.JumpHost, newDebugHandle(wid))
-	out := wfLogger.DataWriter(idx, "stdout")
-	fmt.Fprintf(out, "Workflow failed, connect within %s to debug until shell exit or workflow timeout:\n", grace)
+	fmt.Fprintf(out, "Workflow failed, connect within %s to debug until shell exit or workflow timeout:\n", ssh.GracePeriod)
 	fmt.Fprintf(out, "    %s\n", cmd)
 }
 
@@ -175,6 +224,9 @@ func (e *Engine) maybeRetainForDebug(ctx context.Context, wid models.WorkflowId)
 	case <-graceTimer.C:
 		e.l.Info("nobody ssh'd in within grace; tearing down microVM", "workflow", wid)
 		return
+	case <-target.evicted:
+		e.l.Info("a newer failed run took the debug hold; tearing down microVM", "workflow", wid)
+		return
 	case <-target.connected:
 		e.l.Info("debug shell connected; holding microVM until exit", "workflow", wid)
 	}
diff --git a/spindle/engines/microvm/debug_test.go b/spindle/engines/microvm/debug_test.go
index c7f9c2010..0fc28f2ed 100644
--- a/spindle/engines/microvm/debug_test.go
+++ b/spindle/engines/microvm/debug_test.go
@@ -68,3 +68,73 @@ func TestStepEnvironmentPreservesFailedStepContext(t *testing.T) {
 		}
 	}
 }
+
+func heldTarget(repo, actor string) debugTarget {
+	return debugTarget{
+		holdRepo:  repo,
+		actor:     actor,
+		connected: make(chan struct{}),
+		released:  make(chan struct{}),
+		evicted:   make(chan struct{}),
+	}
+}
+
+func holdWid(name string) models.WorkflowId {
+	return models.WorkflowId{PipelineId: models.PipelineId("p"), Name: name}
+}
+
+func TestNewerFailureReplacesAnUnconnectedHold(t *testing.T) {
+	e := &Engine{debug: make(map[string]debugTarget)}
+	first := heldTarget("did:plc:repo", "did:plc:alice")
+	if !e.claimDebugHold(holdWid("first"), first) {
+		t.Fatal("first failure was not held")
+	}
+	// another repo, but the same pusher
+	if !e.claimDebugHold(holdWid("second"), heldTarget("did:plc:other", "did:plc:alice")) {
+		t.Fatal("newer failure was not held")
+	}
+	if !isClosed(first.evicted) {
+		t.Fatal("the older hold was not told to tear down")
+	}
+	if _, ok := e.lookupDebugTarget(newDebugHandle(holdWid("first"))); ok {
+		t.Fatal("the older hold still accepts debug shells")
+	}
+}
+
+func TestConnectedHoldIsNeverReplaced(t *testing.T) {
+	e := &Engine{debug: make(map[string]debugTarget)}
+	first := heldTarget("did:plc:repo", "did:plc:alice")
+	e.claimDebugHold(holdWid("first"), first)
+	if !e.markDebugConnected(newDebugHandle(holdWid("first"))) {
+		t.Fatal("connecting to the hold failed")
+	}
+
+	if e.claimDebugHold(holdWid("second"), heldTarget("did:plc:repo", "did:plc:bob")) {
+		t.Fatal("a failure in the same repo was held while someone is debugging")
+	}
+	if isClosed(first.evicted) {
+		t.Fatal("the connected hold was torn down")
+	}
+	if !e.claimDebugHold(holdWid("third"), heldTarget("did:plc:other", "did:plc:bob")) {
+		t.Fatal("an unrelated repo and pusher should still get a hold")
+	}
+}
+
+func TestHoldsWithoutAnActorOnlyShareByRepo(t *testing.T) {
+	e := &Engine{debug: make(map[string]debugTarget)}
+	first := heldTarget("did:plc:repo-a", "")
+	e.claimDebugHold(holdWid("first"), first)
+	e.claimDebugHold(holdWid("second"), heldTarget("did:plc:repo-b", ""))
+	if isClosed(first.evicted) {
+		t.Fatal("two repos with no recorded pusher replaced each other")
+	}
+}
+
+func TestReplacedHoldRefusesAShellThatWasStillOpening(t *testing.T) {
+	e := &Engine{debug: make(map[string]debugTarget)}
+	e.claimDebugHold(holdWid("first"), heldTarget("did:plc:repo", "did:plc:alice"))
+	e.claimDebugHold(holdWid("second"), heldTarget("did:plc:repo", "did:plc:alice"))
+	if e.markDebugConnected(newDebugHandle(holdWid("first"))) {
+		t.Fatal("a shell connected to a replaced hold")
+	}
+}
diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go
index 0c7dbd1d1..0e8df9653 100644
--- a/spindle/engines/microvm/engine.go
+++ b/spindle/engines/microvm/engine.go
@@ -264,13 +264,16 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin
 		return nil, engine.ClassifiedFailure(engine.FailureClassUser, engine.FailureReasonConfigurationFailed, err)
 	}
 
-	var ownerDID, repoDID string
+	var ownerDID, repoDID, actor string
 	if md := tpl.TriggerMetadata; md != nil && md.Repo != nil {
 		ownerDID = md.Repo.Did
 		repoDID = ownerDID
 		if md.Repo.RepoDid != nil && *md.Repo.RepoDid != "" {
 			repoDID = *md.Repo.RepoDid
 		}
+		if md.Actor != nil {
+			actor = *md.Actor
+		}
 	}
 	if ownerDID == "" {
 		return nil, engine.ClassifiedFailure(
@@ -301,6 +304,7 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin
 		OwnerDID:                     ownerDID,
 		QuotaStore:                   e.quotaStore,
 		RepoDID:                      repoDID,
+		Actor:                        actor,
 	}
 	return swf, nil
 }
@@ -553,7 +557,7 @@ func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.W
 	if exitCode != 0 {
 		e.l.Debug("step exited non-zero", "workflow", wid, "step", step.Name(), "exitCode", exitCode)
 		if e.cfg.MicroVMPipelines.DebugSSH.Enabled {
-			e.registerDebugTarget(wid, debugTarget{
+			held := e.claimDebugHold(wid, debugTarget{
 				cid:        state.CID,
 				agent:      state.Agent,
 				knot:       w.Environment["TANGLED_REPO_KNOT"],
@@ -562,8 +566,11 @@ func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.W
 				maxAliveAt: state.StartedAt.Add(e.WorkflowTimeout()),
 				connected:  make(chan struct{}),
 				released:   make(chan struct{}),
+				evicted:    make(chan struct{}),
+				holdRepo:   state.RepoDID,
+				actor:      state.Actor,
 			})
-			e.writeDebugHint(wid, len(w.Steps), wfLogger)
+			e.writeDebugHint(wid, len(w.Steps), wfLogger, held)
 		}
 		return engine.ClassifiedFailure(
 			engine.FailureClassUser,
diff --git a/spindle/engines/microvm/vm.go b/spindle/engines/microvm/vm.go
index c5ec4af17..6988fe273 100644
--- a/spindle/engines/microvm/vm.go
+++ b/spindle/engines/microvm/vm.go
@@ -223,6 +223,7 @@ type workflowState struct {
 	NixOSToplevels               nixosToplevelStore
 	OwnerDID                     string
 	RepoDID                      string
+	Actor                        string // who pushed or triggered the run, can be empty on old records
 	QuotaStore                   quota.ReservationStore
 	StartedAt                    time.Time // when the VM booted, for the max-lifetime cap
 	ResourceUsage                *engine.WorkflowResourceUsage
-- 
2.51.2


From 60b56c6b297d1d66672de6e3463d70839cc351af Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Wed, 7 Oct 2026 01:36:29 +0300
Subject: [PATCH 21/25] spindle/engine: say why a workflow failed and how long
 it waited in its finished log

Signed-off-by: dawn <dawn@tangled.org>
---
 spindle/engine/engine.go      | 34 +++++++++++++++++++++++++++++++++-
 spindle/engine/engine_test.go | 10 ++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go
index 9811ad536..7ac52b82d 100644
--- a/spindle/engine/engine.go
+++ b/spindle/engine/engine.go
@@ -207,6 +207,14 @@ func workflowResult(ctx context.Context, err error) string {
 	return "success"
 }
 
+func truncateRunes(s string, n int) string {
+	r := []rune(s)
+	if len(r) <= n {
+		return s
+	}
+	return string(r[:n]) + "…"
+}
+
 func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, qm *quota.Manager, stores *artifactstore.Stores, db *db.DB, n *notifier.Notifier, cacheStore storage.Storage, cacheController CacheController, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) {
 	l.Info("starting all workflows in parallel", "pipeline", pipelineId)
 
@@ -285,11 +293,19 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q
 				if repoDID == "" && pipeline != nil {
 					repoDID = pipeline.RepoDid.String()
 				}
+				var repoName string
+				if pipeline != nil {
+					if md := pipeline.TriggerMetadata; md != nil && md.Repo != nil && md.Repo.Repo != nil {
+						repoName = *md.Repo.Repo
+					}
+				}
 				wl := l.With(
 					"owner_did", w.OwnerDID,
 					"repo_did", repoDID,
+					"repo", repoName,
 					"pipeline_id", pipelineId.String(),
 					"workflow_id", wid.String(),
+					"workflow", wid.Name,
 				)
 
 				if st, err := db.GetStatus(wid); err == nil && st.IsFinish() {
@@ -342,6 +358,7 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q
 					metrics.RecordWorkflowStart(engName)
 				}
 				startTime := time.Now()
+				var runStart time.Time
 				result := "success"
 				remoteExecutionStarted := false
 
@@ -382,9 +399,24 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q
 						}
 					}
 
+					waited := duration
+					if !runStart.IsZero() {
+						waited = runStart.Sub(startTime)
+					}
+					failureClass, failureReason := FailureAttribution(resStr, err)
+					var errMsg string
+					if err != nil {
+						errMsg = truncateRunes(err.Error(), 300)
+					}
+
 					wl.InfoContext(wfCtx, "workflow finished",
 						"result", resStr,
+						"failure_class", failureClass,
+						"failure_reason", failureReason,
+						"error", errMsg,
 						"duration_seconds", duration.Seconds(),
+						"wait_seconds", waited.Seconds(),
+						"run_seconds", (duration - waited).Seconds(),
 						"requested_workflows", reqWorkflows,
 						"requested_vcpus", reqVCPUs,
 						"requested_memory_mib", reqMemoryMiB,
@@ -415,7 +447,6 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q
 						)
 					}
 
-					failureClass, failureReason := FailureAttribution(resStr, err)
 					if remoteStatus {
 						if !remoteExecutionStarted {
 							metrics.RecordWorkflowTerminal(engName, resStr, failureClass, failureReason)
@@ -530,6 +561,7 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q
 					slotAcquired = true
 					remoteExecutionStarted = true
 				}
+				runStart = time.Now()
 
 				// the run's ID token is a credential for this workflow alone: it is
 				// minted now, with the workflow holding its slot, and handed to the
diff --git a/spindle/engine/engine_test.go b/spindle/engine/engine_test.go
index 4e71ca1ee..a53f73084 100644
--- a/spindle/engine/engine_test.go
+++ b/spindle/engine/engine_test.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"log/slog"
+	"math"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -781,6 +782,15 @@ func TestStartWorkflows_ResourceTracking(t *testing.T) {
 	if getAttr(finishedLog, "actual_memory_peak_bytes") != uint64(4096) {
 		t.Errorf("finished log actual_memory_peak_bytes = %v, want 4096", getAttr(finishedLog, "actual_memory_peak_bytes"))
 	}
+	if getAttr(finishedLog, "error") != "" || getAttr(finishedLog, "failure_class") != string(FailureClassNone) {
+		t.Errorf("finished log error = %q, failure_class = %q, want no error", getAttr(finishedLog, "error"), getAttr(finishedLog, "failure_class"))
+	}
+	duration, _ := getAttr(finishedLog, "duration_seconds").(float64)
+	waited, _ := getAttr(finishedLog, "wait_seconds").(float64)
+	ran, _ := getAttr(finishedLog, "run_seconds").(float64)
+	if waited < 0 || ran < 0 || math.Abs(waited+ran-duration) > 1e-6 {
+		t.Errorf("finished log wait %v + run %v, want both >= 0 adding up to duration %v", waited, ran, duration)
+	}
 
 	spans := recorder.Ended()
 	var workflowRunSpan sdktrace.ReadOnlySpan
-- 
2.51.2


From 26b3801ea3bf30d50c1b7f3c9737b7542d2c2501 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Wed, 7 Oct 2026 01:38:07 +0300
Subject: [PATCH 22/25] spindle/mill: report running and waiting workflows per
 repo

Signed-off-by: dawn <dawn@tangled.org>
---
 spindle/mill/fair.go             | 53 ++++++++++++++++++++++++++++++--
 spindle/mill/fair_test.go        | 36 ++++++++++++++++++++++
 spindle/mill/mill.go             |  7 +++--
 spindle/observability/metrics.go | 40 ++++++++++++++++++++++++
 4 files changed, 131 insertions(+), 5 deletions(-)

diff --git a/spindle/mill/fair.go b/spindle/mill/fair.go
index c23bb9c15..99ac6981a 100644
--- a/spindle/mill/fair.go
+++ b/spindle/mill/fair.go
@@ -1,10 +1,17 @@
 package mill
 
+import (
+	"github.com/bluesky-social/indigo/atproto/syntax"
+
+	"tangled.org/core/spindle/observability"
+)
+
 // a workflow waiting in place for a seat. when a seat frees up, waiters bid
 // one at a time, starting with the owner that has the fewest workflows on
 // the fleet, so one owner's burst can't take every seat
 type placeWaiter struct {
 	owner     string
+	repo      string
 	engine    string
 	seq       uint64
 	admitting bool
@@ -18,11 +25,11 @@ func (w *placeWaiter) triedSince(gen uint64) bool {
 	return w.tried && w.triedGen == gen
 }
 
-func (m *Mill) joinPlacement(owner, engine string) *placeWaiter {
+func (m *Mill) joinPlacement(owner, repo, engine string) *placeWaiter {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	m.waiterSeq++
-	w := &placeWaiter{owner: owner, engine: engine, seq: m.waiterSeq}
+	w := &placeWaiter{owner: owner, repo: repo, engine: engine, seq: m.waiterSeq}
 	m.waiters[w] = struct{}{}
 	return w
 }
@@ -84,3 +91,45 @@ func (m *Mill) runningByOwnerLocked() map[string]int {
 	}
 	return running
 }
+
+// what each repo has leased or waiting right now, for the dashboard
+func (m *Mill) workflowCounts() []observability.MillWorkflows {
+	type key struct{ state, owner, repo string }
+	counts := make(map[key]int)
+	m.mu.Lock()
+	for _, lease := range m.leases {
+		counts[key{"running", lease.ownerDID, lease.repoDID}]++
+	}
+	for w := range m.waiters {
+		state := "waiting_seat"
+		if w.admitting {
+			state = "waiting_quota"
+		}
+		counts[key{state, w.owner, w.repo}]++
+	}
+	m.mu.Unlock()
+
+	names := make(map[string]string)
+	out := make([]observability.MillWorkflows, 0, len(counts))
+	for k, n := range counts {
+		name, ok := names[k.repo]
+		if !ok {
+			name = m.repoName(k.repo)
+			names[k.repo] = name
+		}
+		out = append(out, observability.MillWorkflows{State: k.state, OwnerDID: k.owner, RepoDID: k.repo, Repo: name, Count: n})
+	}
+	return out
+}
+
+// forks and repos this spindle doesn't track fall back to their did
+func (m *Mill) repoName(repoDID string) string {
+	if m.db == nil || repoDID == "" {
+		return repoDID
+	}
+	repo, err := m.db.GetRepoByDid(syntax.DID(repoDID))
+	if err != nil || repo.Name == "" {
+		return repoDID
+	}
+	return repo.Name
+}
diff --git a/spindle/mill/fair_test.go b/spindle/mill/fair_test.go
index 5a79011b1..5f12e2f34 100644
--- a/spindle/mill/fair_test.go
+++ b/spindle/mill/fair_test.go
@@ -2,11 +2,15 @@ package mill
 
 import (
 	"context"
+	"slices"
+	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"tangled.org/core/spindle/db"
 	"tangled.org/core/spindle/models"
+	"tangled.org/core/spindle/observability"
 
 	millproto "tangled.org/core/spindle/mill/proto"
 	millv1 "tangled.org/core/spindle/mill/proto/gen"
@@ -155,3 +159,35 @@ func TestWaiterThatCannotFitDoesNotBlockTheRest(t *testing.T) {
 		t.Fatalf("freed seat went to %q, want the waiter that fits", got)
 	}
 }
+
+func TestWorkflowCountsNameTheRepoAndSplitWaiting(t *testing.T) {
+	m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute})
+	if err := bdb.AddRepo(db.Repo{Knot: "knot", Owner: "did:plc:alice", Rkey: "core", RepoDid: "did:plc:core", Name: "core"}); err != nil {
+		t.Fatal(err)
+	}
+
+	m.mu.Lock()
+	for range 2 {
+		lease := newLease(m.nextLeaseIDLocked(), "node-a", "", "dummy")
+		lease.ownerDID, lease.repoDID = "did:plc:alice", "did:plc:core"
+		m.leases[lease.id] = lease
+	}
+	m.mu.Unlock()
+	m.joinPlacement("did:plc:alice", "did:plc:core", "dummy")
+	m.setAdmitting(m.joinPlacement("did:plc:alice", "did:plc:core", "dummy"), true)
+	m.joinPlacement("did:plc:bob", "did:plc:fork", "dummy")
+
+	got := m.workflowCounts()
+	slices.SortFunc(got, func(a, b observability.MillWorkflows) int {
+		return strings.Compare(a.State+a.RepoDID, b.State+b.RepoDID)
+	})
+	want := []observability.MillWorkflows{
+		{State: "running", OwnerDID: "did:plc:alice", RepoDID: "did:plc:core", Repo: "core", Count: 2},
+		{State: "waiting_quota", OwnerDID: "did:plc:alice", RepoDID: "did:plc:core", Repo: "core", Count: 1},
+		{State: "waiting_seat", OwnerDID: "did:plc:alice", RepoDID: "did:plc:core", Repo: "core", Count: 1},
+		{State: "waiting_seat", OwnerDID: "did:plc:bob", RepoDID: "did:plc:fork", Repo: "did:plc:fork", Count: 1},
+	}
+	if !slices.Equal(got, want) {
+		t.Fatalf("workflow counts = %+v, want %+v", got, want)
+	}
+}
diff --git a/spindle/mill/mill.go b/spindle/mill/mill.go
index 455b9d3d1..0446c058d 100644
--- a/spindle/mill/mill.go
+++ b/spindle/mill/mill.go
@@ -471,11 +471,11 @@ func (m *Mill) place(ctx context.Context, engineName string, wid models.Workflow
 		m.mu.Unlock()
 	}()
 
-	owner := wf.OwnerDID
+	owner, repo := wf.OwnerDID, wf.RepoDID
 	if id, ok := m.quotaIdentity(wf); ok {
-		owner = id.OwnerDID
+		owner, repo = id.OwnerDID, id.RepoDID
 	}
-	w := m.joinPlacement(owner, engineName)
+	w := m.joinPlacement(owner, repo, engineName)
 	defer m.leavePlacement(w)
 
 	for {
@@ -2174,6 +2174,7 @@ func (m *Mill) RegisterMetrics(metrics *observability.Metrics) {
 			return float64(disconnected)
 		},
 	)
+	metrics.RegisterMillWorkflows(m.workflowCounts)
 }
 
 func (m *Mill) authorizeAndLockQuota(s *millSession, reservationID, requestedLeaseID string) (*RemoteLease, error) {
diff --git a/spindle/observability/metrics.go b/spindle/observability/metrics.go
index 5dddbf44c..1ce22480f 100644
--- a/spindle/observability/metrics.go
+++ b/spindle/observability/metrics.go
@@ -942,6 +942,46 @@ func (m *Metrics) RegisterMillGauges(
 	}, disconnectedSessions))
 }
 
+type MillWorkflows struct {
+	State    string
+	OwnerDID string
+	RepoDID  string
+	Repo     string
+	Count    int
+}
+
+// repo labels are only safe because a repo has series while it has work on
+// the mill, so the count stays under seats plus max pending
+type millWorkflowsCollector struct {
+	desc     *prometheus.Desc
+	snapshot func() []MillWorkflows
+}
+
+func (c *millWorkflowsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.desc
+}
+
+func (c *millWorkflowsCollector) Collect(ch chan<- prometheus.Metric) {
+	for _, w := range c.snapshot() {
+		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(w.Count), w.State, w.OwnerDID, w.RepoDID, w.Repo)
+	}
+}
+
+// snapshot has to return each state, owner and repo at most once
+func (m *Metrics) RegisterMillWorkflows(snapshot func() []MillWorkflows) {
+	if m == nil {
+		return
+	}
+	m.reg.MustRegister(&millWorkflowsCollector{
+		desc: prometheus.NewDesc(
+			"spindle_mill_workflows",
+			"Workflows on the mill right now by state (running, waiting_seat, waiting_quota) and repo. Repos with none have no series.",
+			[]string{"state", "owner_did", "repo_did", "repo"}, nil,
+		),
+		snapshot: snapshot,
+	})
+}
+
 func (m *Metrics) RecordMillPlacementAdmission(allowed bool, reason string) {
 	if m == nil {
 		return
-- 
2.51.2


From 3ba41476d7bbca1914566074bfe099f2193807fa Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Wed, 7 Oct 2026 02:16:01 +0300
Subject: [PATCH 23/25] spindle/engine: take the repo name for the finished log
 from the repos table

Signed-off-by: dawn <dawn@tangled.org>
---
 spindle/db/repos.go      | 12 ++++++++++++
 spindle/engine/engine.go |  8 +-------
 spindle/mill/fair.go     | 20 ++------------------
 3 files changed, 15 insertions(+), 25 deletions(-)

diff --git a/spindle/db/repos.go b/spindle/db/repos.go
index 54aaaf67c..1ec886a70 100644
--- a/spindle/db/repos.go
+++ b/spindle/db/repos.go
@@ -134,6 +134,18 @@ func (d *DB) GetRepoByDid(repoDid syntax.DID) (*Repo, error) {
 	))
 }
 
+// forks and repos this spindle doesn't track fall back to their did
+func (d *DB) RepoName(repoDid string) string {
+	if d == nil || repoDid == "" {
+		return repoDid
+	}
+	repo, err := d.GetRepoByDid(syntax.DID(repoDid))
+	if err != nil || repo.Name == "" {
+		return repoDid
+	}
+	return repo.Name
+}
+
 func (d *DB) GetRepoByOwnerRkey(owner syntax.DID, rkey syntax.RecordKey) (*Repo, error) {
 	return scanRepo(d.QueryRow(
 		`select knot, owner, rkey, coalesce(repo_did, ''), coalesce(name, '') from repos where owner = ? and rkey = ?`,
diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go
index 7ac52b82d..d9ff27987 100644
--- a/spindle/engine/engine.go
+++ b/spindle/engine/engine.go
@@ -293,16 +293,10 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q
 				if repoDID == "" && pipeline != nil {
 					repoDID = pipeline.RepoDid.String()
 				}
-				var repoName string
-				if pipeline != nil {
-					if md := pipeline.TriggerMetadata; md != nil && md.Repo != nil && md.Repo.Repo != nil {
-						repoName = *md.Repo.Repo
-					}
-				}
 				wl := l.With(
 					"owner_did", w.OwnerDID,
 					"repo_did", repoDID,
-					"repo", repoName,
+					"repo", db.RepoName(repoDID),
 					"pipeline_id", pipelineId.String(),
 					"workflow_id", wid.String(),
 					"workflow", wid.Name,
diff --git a/spindle/mill/fair.go b/spindle/mill/fair.go
index 99ac6981a..dc608ea7e 100644
--- a/spindle/mill/fair.go
+++ b/spindle/mill/fair.go
@@ -1,10 +1,6 @@
 package mill
 
-import (
-	"github.com/bluesky-social/indigo/atproto/syntax"
-
-	"tangled.org/core/spindle/observability"
-)
+import "tangled.org/core/spindle/observability"
 
 // a workflow waiting in place for a seat. when a seat frees up, waiters bid
 // one at a time, starting with the owner that has the fewest workflows on
@@ -114,22 +110,10 @@ func (m *Mill) workflowCounts() []observability.MillWorkflows {
 	for k, n := range counts {
 		name, ok := names[k.repo]
 		if !ok {
-			name = m.repoName(k.repo)
+			name = m.db.RepoName(k.repo)
 			names[k.repo] = name
 		}
 		out = append(out, observability.MillWorkflows{State: k.state, OwnerDID: k.owner, RepoDID: k.repo, Repo: name, Count: n})
 	}
 	return out
 }
-
-// forks and repos this spindle doesn't track fall back to their did
-func (m *Mill) repoName(repoDID string) string {
-	if m.db == nil || repoDID == "" {
-		return repoDID
-	}
-	repo, err := m.db.GetRepoByDid(syntax.DID(repoDID))
-	if err != nil || repo.Name == "" {
-		return repoDID
-	}
-	return repo.Name
-}
-- 
2.51.2


From a48f86056b0347c1a90db9517be5347309ea13b3 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Wed, 7 Oct 2026 13:56:57 +0300
Subject: [PATCH 24/25] shuttle: replace the guest's default substituters with
 the read proxy

Signed-off-by: dawn <dawn@tangled.org>
---
 shuttle/src/nix_config.rs                       | 17 ++++++++---------
 shuttle/src/session.rs                          |  9 +++------
 spindle/engines/microvm/test-spindle-microvm.sh |  2 +-
 3 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/shuttle/src/nix_config.rs b/shuttle/src/nix_config.rs
index 275a5771f..275667d68 100644
--- a/shuttle/src/nix_config.rs
+++ b/shuttle/src/nix_config.rs
@@ -45,10 +45,12 @@ pub struct RuntimeCacheConfig {
 }
 
 // configures nix daemon with the configuration passed from host
-pub async fn configure(init: &v1::Init, read_proxy_url: &str) -> Result<RuntimeCacheConfig> {
-    let read_urls = vec![read_proxy_url.to_owned()];
+pub async fn configure(
+    init: &v1::Init,
+    read_proxy_url: Option<&str>,
+) -> Result<RuntimeCacheConfig> {
     let cfg = RuntimeCacheConfig {
-        read_urls,
+        read_urls: read_proxy_url.map(str::to_owned).into_iter().collect(),
         trusted_public_keys: clean_strings(&init.cache_trusted_public_keys),
     };
 
@@ -64,13 +66,10 @@ pub async fn configure(init: &v1::Init, read_proxy_url: &str) -> Result<RuntimeC
     write_file_atomic(SPINDLE_CACHE_CONFIG, &cache_json, 0o600)?;
 
     let mut nix_conf = String::new();
+    // not extra-substituters, so nix drops its default cache.nixos.org and
+    // every fetch goes through the proxy
     if !cfg.read_urls.is_empty() {
-        writeln!(
-            &mut nix_conf,
-            "extra-substituters = {}",
-            cfg.read_urls.join(" ")
-        )
-        .unwrap();
+        writeln!(&mut nix_conf, "substituters = {}", cfg.read_urls.join(" ")).unwrap();
     }
     if !cfg.trusted_public_keys.is_empty() {
         writeln!(
diff --git a/shuttle/src/session.rs b/shuttle/src/session.rs
index 41ff6c36b..9c1cfc490 100644
--- a/shuttle/src/session.rs
+++ b/shuttle/src/session.rs
@@ -45,12 +45,9 @@ pub async fn run(host_cid: u32, port: u32) -> Result<()> {
     let _dns_proxy = DnsProxy::start(host_cid, init.dns_proxy_port)
         .await
         .context("start dns proxy")?;
-    let _cache_cfg = nix_config::configure(
-        &init,
-        read_proxy.as_ref().map(ReadCacheProxy::url).unwrap_or(""),
-    )
-    .await
-    .context("configure nix cache")?;
+    let _cache_cfg = nix_config::configure(&init, read_proxy.as_ref().map(ReadCacheProxy::url))
+        .await
+        .context("configure nix cache")?;
     let uploader = CacheUploadManager::start(
         write_proxy.as_ref().map(WriteCacheProxy::url).unwrap_or(""),
         out_tx.clone(),
diff --git a/spindle/engines/microvm/test-spindle-microvm.sh b/spindle/engines/microvm/test-spindle-microvm.sh
index 608c52f29..b5f0b4779 100755
--- a/spindle/engines/microvm/test-spindle-microvm.sh
+++ b/spindle/engines/microvm/test-spindle-microvm.sh
@@ -489,7 +489,7 @@ test_realize() {
     out=$(run_vm --name "realize" --timeout "60s" -- /run/current-system/sw/bin/bash -lc '
 set -euo pipefail
 store_path=$1
-cache_url=$(sed -n "s/^extra-substituters = //p" /run/spindle/nix.conf)
+cache_url=$(sed -n "s/^substituters = //p" /run/spindle/nix.conf)
 cache_url=${cache_url%% *}
 if [ -z "$cache_url" ]; then
     echo "error: cache URL not found in /run/spindle/nix.conf" >&2
-- 
2.51.2


From 5c3149cf5d235ed453de033e9298fedd85952546 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Wed, 7 Oct 2026 15:01:33 +0300
Subject: [PATCH 25/25] sitemap: use handles instead of dids in the urls

Signed-off-by: dawn <dawn@tangled.org>
---
 docs/SEARCH_ENGINES.md              | 111 -----------
 sitemap/migrations/0002_handles.sql |  16 ++
 sitemap/src/db.ts                   | 110 +++++++++-
 sitemap/src/hydrant.ts              |  11 +-
 sitemap/src/identity.ts             |  49 +++++
 sitemap/src/index.ts                | 108 +++++++++-
 sitemap/src/types.ts                |   3 +
 sitemap/test/d1.ts                  |  16 +-
 sitemap/test/node.d.ts              |   1 +
 sitemap/test/sitemap.test.ts        |  57 +++++-
 sitemap/test/sync.test.ts           | 298 +++++++++++++++++++++++++++-
 sitemap/wrangler.dev.jsonc          |   9 +
 sitemap/wrangler.jsonc              |   9 +
 13 files changed, 663 insertions(+), 135 deletions(-)
 delete mode 100644 docs/SEARCH_ENGINES.md
 create mode 100644 sitemap/migrations/0002_handles.sql
 create mode 100644 sitemap/src/identity.ts

diff --git a/docs/SEARCH_ENGINES.md b/docs/SEARCH_ENGINES.md
deleted file mode 100644
index 9cf3ce124..000000000
--- a/docs/SEARCH_ENGINES.md
+++ /dev/null
@@ -1,111 +0,0 @@
-# Search Engine Integrations: Google Search Console & IndexNow
-
-This document outlines Tangled's search engine discovery mechanisms, sitemap architecture, verification methods, and instant re-indexing integrations.
-
----
-
-## 1. Dynamic XML Sitemap (TAN-737 & TAN-745)
-
-Tangled serves an automated, dynamically generated XML sitemap adhering to the standard Sitemap Protocol (0.9).
-
-- **URL**: `https://tangled.org/sitemap.xml`
-- **Implementation**: `web/src/routes/sitemap.xml/+server.ts`
-- **Cache-Control**: `public, max-age=3600, s-maxage=3600` (1-hour CDN/browser caching)
-- **Specification Limit**: Hard cap at 50,000 URLs per sitemap specification.
-
-### URL Structure & Priority
-1. **Core Static Pages**:
-   - `https://tangled.org/` — priority `1.0`, changefreq `daily`
-   - `https://tangled.org/about` — priority `0.8`, changefreq `weekly`
-2. **Public Profiles**:
-   - `https://tangled.org/{handle}` — priority `0.7`, changefreq `daily`
-   - Dynamically gathered from trending actors (`sh.tangled.actor.getTrending`) and public timeline events (`sh.tangled.feed.getTimeline`).
-3. **Public Repositories**:
-   - `https://tangled.org/{handle}/{repo}` — priority `0.8`, changefreq `daily`
-   - Includes `<lastmod>` derived from the repository's creation timestamp.
-   - Paginates repository lists for discovered users via `sh.tangled.repo.listRepos` with cursor-based pagination.
-
-### Resilience & Graceful Degradation
-If upstream backend services (Bobbin / XRPC) are temporarily unreachable, degraded, or timing out, the endpoint catches errors silently and serves the static core URLs cleanly with HTTP 200, ensuring search engine crawlers are never served a 5xx error.
-
----
-
-## 2. Google Search Console Verification (TAN-746)
-
-Google Search Console (GSC) is used to monitor search performance, coverage, indexing status, and submit sitemaps.
-
-### Verification Methods
-Tangled supports two primary verification methods:
-
-1. **HTML Meta Tag Verification**:
-   Add the verification tag provided by Google Search Console into the `<head>` of `web/src/app.html`:
-   ```html
-   <meta name="google-site-verification" content="YOUR_GSC_VERIFICATION_TOKEN" />
-   ```
-2. **DNS TXT Record (Recommended for domain-level property)**:
-   Add a TXT record to the DNS configuration for `tangled.org`:
-   ```text
-   Record Type: TXT
-   Host: @
-   Value: google-site-verification=YOUR_GSC_VERIFICATION_TOKEN
-   ```
-
-### Sitemap Submission
-1. Log into [Google Search Console](https://search.google.com/search-console).
-2. Select the `tangled.org` property.
-3. In the left navigation, click **Indexing** > **Sitemaps**.
-4. Under "Add a new sitemap", enter `sitemap.xml` and click **Submit**.
-5. Google will periodically poll and re-index URLs from `https://tangled.org/sitemap.xml`.
-
----
-
-## 3. IndexNow Instant Indexing Protocol (TAN-747)
-
-IndexNow allows websites to immediately notify participating search engines (including Microsoft Bing, Yandex, Seznam, and Naver) whenever content is created, updated, or deleted. Submitted URLs are crawled and reflected in search results within minutes.
-
-### Verification Key
-- **Key**: `d76fae270607bfc761b35c14f3182c9e`
-- **Key Location**: `https://tangled.org/d76fae270607bfc761b35c14f3182c9e.txt`
-- **Static File**: `web/static/d76fae270607bfc761b35c14f3182c9e.txt`
-
-Search engines verify domain ownership by requesting `https://tangled.org/{key}.txt` and verifying that the body matches the key.
-
-### IndexNow Submission Utility
-Implementation located at `web/src/lib/server/indexnow.ts`.
-
-#### API Usage
-```typescript
-import {
-  submitToIndexNow,
-  notifyProfileCreated,
-  notifyRepoCreated
-} from "$lib/server/indexnow";
-
-// Submit an arbitrary list of changed URLs:
-await submitToIndexNow([
-  "https://tangled.org/alice.tangled.sh",
-  "https://tangled.org/alice.tangled.sh/my-repo"
-]);
-
-// Helper when a new public profile is created:
-await notifyProfileCreated("alice.tangled.sh");
-
-// Helper when a new public repository is created:
-await notifyRepoCreated("alice.tangled.sh", "my-repo");
-```
-
-#### Protocol Payload
-Requests are sent via HTTP POST to `https://api.indexnow.org/indexnow`:
-```json
-{
-  "host": "tangled.org",
-  "key": "d76fae270607bfc761b35c14f3182c9e",
-  "keyLocation": "https://tangled.org/d76fae270607bfc761b35c14f3182c9e.txt",
-  "urlList": [
-    "https://tangled.org/alice.tangled.sh/my-repo"
-  ]
-}
-```
-
-#### Fault-Tolerance & Non-Blocking Design
-All IndexNow helper calls catch network and HTTP status errors internally and log warnings to the console without throwing exceptions. User-facing mutation flows (such as repository creation or profile setup) will never fail or be delayed if the external IndexNow API is slow or unavailable.
diff --git a/sitemap/migrations/0002_handles.sql b/sitemap/migrations/0002_handles.sql
new file mode 100644
index 000000000..e11b0e5b1
--- /dev/null
+++ b/sitemap/migrations/0002_handles.sql
@@ -0,0 +1,16 @@
+-- urls stay keyed by did so a handle change never moves one, and rendering swaps the handle in
+ALTER TABLE urls ADD COLUMN did TEXT;
+
+-- the owner did is the one path segment that starts with did:
+UPDATE urls SET did = substr(url, instr(url, '/did:') + 1) WHERE instr(url, '/did:') > 0;
+UPDATE urls SET did = substr(did, 1, instr(did, '/') - 1) WHERE instr(did, '/') > 0;
+
+CREATE INDEX urls_by_did ON urls (did);
+
+-- hydrant's handle for each repo, null when the did doc names none. stale means an
+-- identity event came in since, so it gets read again
+CREATE TABLE handles (
+  did TEXT PRIMARY KEY,
+  handle TEXT,
+  stale INTEGER NOT NULL DEFAULT 0
+) WITHOUT ROWID;
diff --git a/sitemap/src/db.ts b/sitemap/src/db.ts
index 49d8e4478..12b82efa4 100644
--- a/sitemap/src/db.ts
+++ b/sitemap/src/db.ts
@@ -1,9 +1,10 @@
 import type { ExtractedAction } from './hydrant';
+import { withHandle, type RepoHandle } from './identity';
 import type { UrlEntry } from './types';
 import { buildUrlsetXml, measureXmlBytes } from './xml';
 
 // static urls have no record behind them
-export type Change = ExtractedAction | { type: 'upsert'; uri?: undefined; url: string; entry: UrlEntry };
+export type Change = ExtractedAction | { type: 'upsert'; uri?: undefined; did?: undefined; url: string; entry: UrlEntry };
 
 export interface ShardLimits {
   maxUrls: number;
@@ -56,6 +57,10 @@ function setState(db: D1Database, key: string, value: string): D1PreparedStateme
     .bind(key, value);
 }
 
+export async function putState(db: D1Database, key: string, value: string): Promise<void> {
+  await setState(db, key, value).run();
+}
+
 export async function countUrls(db: D1Database): Promise<number> {
   const row = await db.prepare('SELECT count(*) AS n FROM urls').first<{ n: number }>();
   return row?.n ?? 0;
@@ -98,8 +103,16 @@ export async function applyChanges(
     )).map((row) => [row.url, row.shard]),
   );
 
+  const handleOf = new Map(
+    (await lookup<RepoHandle>(
+      db,
+      'SELECT did, handle FROM handles WHERE did IN (SELECT value FROM json_each(?1))',
+      changes.flatMap((change) => (change.did ? [change.did] : [])),
+    )).map((row) => [row.did, row.handle]),
+  );
+
   const active = await activeShard(db);
-  const upserts = new Map<string, UrlEntry & { shard: number; bytes: number }>();
+  const upserts = new Map<string, UrlEntry & { did: string | null; shard: number; bytes: number }>();
   const records = new Map<string, string | null>();
   const dirty = new Set<number>();
   let mutations = 0;
@@ -130,7 +143,9 @@ export async function applyChanges(
       records.set(change.uri, change.url);
     }
 
-    const bytes = urlBytes(change.entry);
+    // sized as it renders, so the shard caps hold with the handle in
+    const handle = change.did ? handleOf.get(change.did) : null;
+    const bytes = urlBytes({ ...change.entry, loc: withHandle(change.url, change.did, handle) });
     let shard = urlShard.get(change.url);
     if (shard == null) {
       if (active.count >= limits.maxUrls || active.bytes + bytes > limits.maxBytes) {
@@ -143,7 +158,7 @@ export async function applyChanges(
       shard = active.shard;
       urlShard.set(change.url, shard);
     }
-    upserts.set(change.url, { ...change.entry, shard, bytes });
+    upserts.set(change.url, { ...change.entry, did: change.did ?? null, shard, bytes });
     dirty.add(shard);
     mutations++;
   }
@@ -153,12 +168,12 @@ export async function applyChanges(
     ...chunked([...upserts.values()]).map((json) =>
       db
         .prepare(
-          `INSERT INTO urls (url, shard, lastmod, changefreq, priority, bytes)
+          `INSERT INTO urls (url, shard, lastmod, changefreq, priority, bytes, did)
            SELECT value ->> 'loc', value ->> 'shard', value ->> 'lastmod', value ->> 'changefreq',
-                  value ->> 'priority', value ->> 'bytes'
+                  value ->> 'priority', value ->> 'bytes', value ->> 'did'
            FROM json_each(?1) WHERE true
            ON CONFLICT (url) DO UPDATE SET shard = excluded.shard, lastmod = excluded.lastmod, changefreq = excluded.changefreq,
-             priority = excluded.priority, bytes = excluded.bytes`,
+             priority = excluded.priority, bytes = excluded.bytes, did = excluded.did`,
         )
         .bind(json),
     ),
@@ -197,11 +212,22 @@ export async function dirtyShards(db: D1Database): Promise<{ shard: number; dirt
 
 export async function shardEntries(db: D1Database, shard: number): Promise<UrlEntry[]> {
   const res = await db
-    .prepare('SELECT url AS loc, lastmod, changefreq, priority FROM urls WHERE shard = ?1 ORDER BY url')
+    .prepare(
+      `SELECT u.url AS loc, u.did, h.handle, u.lastmod, u.changefreq, u.priority
+       FROM urls u LEFT JOIN handles h ON h.did = u.did
+       WHERE u.shard = ?1 ORDER BY u.url`,
+    )
     .bind(shard)
-    .all<{ loc: string; lastmod: string; changefreq: UrlEntry['changefreq'] | null; priority: number | null }>();
+    .all<{
+      loc: string;
+      did: string | null;
+      handle: string | null;
+      lastmod: string;
+      changefreq: UrlEntry['changefreq'] | null;
+      priority: number | null;
+    }>();
   return res.results.map((row) => ({
-    loc: row.loc,
+    loc: withHandle(row.loc, row.did, row.handle),
     lastmod: row.lastmod,
     changefreq: row.changefreq ?? undefined,
     priority: row.priority ?? undefined,
@@ -221,10 +247,72 @@ export async function allShards(db: D1Database): Promise<{ shard: number; lastmo
   return res.results;
 }
 
+// an identity event can mean a new handle, so those get read again
+export async function markStale(db: D1Database, dids: string[]): Promise<void> {
+  const statements = chunked([...new Set(dids)]).map((json) =>
+    db.prepare('UPDATE handles SET stale = 1 WHERE did IN (SELECT value FROM json_each(?1))').bind(json),
+  );
+  if (statements.length > 0) await db.batch(statements);
+}
+
+// owners with no handle read yet, or a stale one, in did order after `after`
+export async function staleOwners(db: D1Database, after: string, limit: number): Promise<string[]> {
+  const res = await db
+    .prepare(
+      `SELECT DISTINCT u.did FROM urls u LEFT JOIN handles h ON h.did = u.did
+       WHERE u.did > ?1 AND (h.did IS NULL OR h.stale = 1)
+       ORDER BY u.did LIMIT ?2`,
+    )
+    .bind(after, limit)
+    .all<{ did: string }>();
+  return res.results.map((row) => row.did);
+}
+
+// a changed handle resizes its owner's urls and rerenders their shards
+export async function saveHandles(db: D1Database, found: RepoHandle[]): Promise<void> {
+  const previous = new Map(
+    (await lookup<RepoHandle>(
+      db,
+      'SELECT did, handle FROM handles WHERE did IN (SELECT value FROM json_each(?1))',
+      found.map((repo) => repo.did),
+    )).map((row) => [row.did, row.handle]),
+  );
+  const changed = found.filter((repo) => repo.handle !== (previous.get(repo.did) ?? null));
+  // dids and handles are plain ascii, so characters are bytes
+  const growth = changed.map((repo) => [
+    repo.did,
+    (repo.handle ?? repo.did).length - (previous.get(repo.did) ?? repo.did).length,
+  ]);
+  const statements: D1PreparedStatement[] = [
+    ...chunked(found.map((repo) => [repo.did, repo.handle])).map((json) =>
+      db
+        .prepare(
+          `INSERT INTO handles (did, handle, stale) SELECT value ->> 0, value ->> 1, 0 FROM json_each(?1) WHERE true
+           ON CONFLICT (did) DO UPDATE SET handle = excluded.handle, stale = 0`,
+        )
+        .bind(json),
+    ),
+    ...chunked(growth).map((json) =>
+      db
+        .prepare('UPDATE urls SET bytes = bytes + (j.value ->> 1) FROM json_each(?1) AS j WHERE urls.did = j.value ->> 0')
+        .bind(json),
+    ),
+    ...chunked(changed.map((repo) => repo.did)).map((json) =>
+      db
+        .prepare(
+          `UPDATE shards SET dirty = dirty + 1
+           WHERE shard IN (SELECT shard FROM urls WHERE did IN (SELECT value FROM json_each(?1)))`,
+        )
+        .bind(json),
+    ),
+  ];
+  if (statements.length > 0) await db.batch(statements);
+}
+
 export async function markSynced(db: D1Database, now: Date): Promise<void> {
   await setState(db, 'last_sync_at', now.toISOString()).run();
 }
 
 export async function clearDb(db: D1Database): Promise<void> {
-  await db.batch(['urls', 'records', 'shards', 'state'].map((table) => db.prepare(`DELETE FROM ${table}`)));
+  await db.batch(['urls', 'records', 'handles', 'shards', 'state'].map((table) => db.prepare(`DELETE FROM ${table}`)));
 }
diff --git a/sitemap/src/hydrant.ts b/sitemap/src/hydrant.ts
index 2d159d2ad..d26e30db6 100644
--- a/sitemap/src/hydrant.ts
+++ b/sitemap/src/hydrant.ts
@@ -4,8 +4,8 @@ import { sanitizeLastmod } from './xml';
 
 // uri is the record's at-uri, deletes carry no record so it's what finds their url
 export type ExtractedAction =
-  | { type: 'delete'; uri: string; url: string }
-  | { type: 'upsert'; uri: string; url: string; entry: UrlEntry };
+  | { type: 'delete'; uri: string; did: string; url: string }
+  | { type: 'upsert'; uri: string; did: string; url: string; entry: UrlEntry };
 
 interface CollectionSpec {
   buildPath: (owner: string, rkey: string, record?: RecordValue) => string;
@@ -31,8 +31,8 @@ export const COLLECTION_SPECS: Record<string, CollectionSpec> = {
   },
 };
 
-// owners are dids rather than handles, so a url never moves when someone
-// changes handle, and the web redirects it to the handle anyway
+// owners are dids here so a url never moves when someone changes handle, and
+// rendering swaps the handle in
 export function extractUrlAction(
   event: JetstreamEvent,
   baseUrl: string,
@@ -55,13 +55,14 @@ export function extractUrlAction(
   const url = `${normalizedBase}/${path}`;
 
   if (operation === 'delete') {
-    return { type: 'delete', uri, url };
+    return { type: 'delete', uri, did: event.did, url };
   }
 
   const lastmod = sanitizeLastmod(record?.createdAt as string | undefined, now);
   return {
     type: 'upsert',
     uri,
+    did: event.did,
     url,
     entry: {
       loc: url,
diff --git a/sitemap/src/identity.ts b/sitemap/src/identity.ts
new file mode 100644
index 000000000..4b7ca7da8
--- /dev/null
+++ b/sitemap/src/identity.ts
@@ -0,0 +1,49 @@
+export interface RepoHandle {
+  did: string;
+  handle: string | null;
+}
+
+// hydrant's management api, which has no auth so it's only reachable over the tunnel
+export interface Repos {
+  // every repo hydrant tracks, in did order, starting after `after`
+  list(after: string | undefined, limit: number): Promise<RepoHandle[]>;
+  // throws when there's no answer, so a flaky read never drops a good handle
+  get(did: string): Promise<string | null>;
+}
+
+// whatever the did doc claims, nobody checked the domain agrees. that's fine for a
+// sitemap because a wrong one just 404s
+function handleOf(repo: { handle?: string }): string | null {
+  const handle = repo.handle?.toLowerCase();
+  return handle && !handle.endsWith('.invalid') ? handle : null;
+}
+
+export function hydrantRepos(fetcher: Pick<Fetcher, 'fetch'>, base: string): Repos {
+  const read = (path: string, params: Record<string, string> = {}) => {
+    const url = new URL(path, base);
+    for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value);
+    return fetcher.fetch(url, { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(10_000) });
+  };
+  return {
+    async list(after, limit) {
+      const res = await read('/repos', after ? { limit: String(limit), cursor: after } : { limit: String(limit) });
+      if (!res.ok) throw new Error(`hydrant /repos: http ${res.status}`);
+      const repos = await res.json<{ did: string; handle?: string }[]>();
+      return repos.map((repo) => ({ did: repo.did, handle: handleOf(repo) }));
+    },
+    async get(did) {
+      const res = await read(`/repos/${did}`);
+      if (res.status === 404) return null;
+      if (!res.ok) throw new Error(`hydrant /repos/${did}: http ${res.status}`);
+      return handleOf(await res.json<{ handle?: string }>());
+    },
+  };
+}
+
+export function withHandle(url: string, did: string | null | undefined, handle: string | null | undefined): string {
+  if (!did || !handle) return url;
+  return url
+    .split('/')
+    .map((segment) => (segment === did ? handle : segment))
+    .join('/');
+}
diff --git a/sitemap/src/index.ts b/sitemap/src/index.ts
index 195790262..17f35ba7a 100644
--- a/sitemap/src/index.ts
+++ b/sitemap/src/index.ts
@@ -1,6 +1,16 @@
 import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep, type WorkflowStepConfig } from 'cloudflare:workers';
-import { applyChanges, countUrls, getState, markSynced } from './db';
+import {
+  applyChanges,
+  countUrls,
+  getState,
+  markStale,
+  markSynced,
+  putState,
+  saveHandles,
+  staleOwners,
+} from './db';
 import { COLLECTION_SPECS, extractUrlAction } from './hydrant';
+import { hydrantRepos, type RepoHandle, type Repos } from './identity';
 import { readJetstream, subscribeJetstream, type Subscribe } from './jetstream';
 import { getStaticUrls, renderShards, resetSitemap } from './sitemap';
 import type { Env } from './types';
@@ -14,12 +24,15 @@ function hydrantUrl(env: Env): string {
 
 export interface SyncDeps {
   subscribe: Subscribe;
+  repos: (env: Env) => Repos;
   idleMs: number;
   closeMs: number;
 }
 
 const defaultDeps: SyncDeps = {
   subscribe: subscribeJetstream,
+  // the vpc service only takes http and routes by itself, so the host there means nothing
+  repos: (env) => (env.HYDRANT ? hydrantRepos(env.HYDRANT, 'http://hydrant') : hydrantRepos(globalThis, hydrantUrl(env))),
   idleMs: 3000,
   closeMs: 10_000,
 };
@@ -94,12 +107,80 @@ export interface SyncOptions {
 export interface SyncResult {
   syncedEvents: number;
   mutatedUrls: number;
+  walkedRepos: number;
+  lookedUpHandles: number;
   newCursor: string;
   totalUrls: number;
 }
 
 const MAX_PAGES = 10;
 
+const REPO_PAGE = 1000;
+const WALK_PAGES = 10;
+const MAX_WALK_STEPS = 100;
+
+// a page of reads that all time out still fits well inside a step
+const LOOKUP_PAGE = 200;
+const LOOKUP_CONCURRENCY = 8;
+const MAX_LOOKUP_STEPS = 10;
+
+// the first run, and the first after a reset, pages through every handle hydrant has
+// instead of reading owners one by one. a walk that gets cut off carries on next run
+async function walkRepos(
+  db: D1Database,
+  repos: Repos,
+  now: Date,
+): Promise<{ repos: number; done: boolean; failed: boolean }> {
+  if (await getState(db, 'handles_walked')) return { repos: 0, done: true, failed: false };
+  let after = await getState(db, 'handles_after');
+  let count = 0;
+  try {
+    for (let page = 0; page < WALK_PAGES; page++) {
+      const found = await repos.list(after, REPO_PAGE);
+      await saveHandles(db, found);
+      count += found.length;
+      if (found.length < REPO_PAGE) {
+        await putState(db, 'handles_walked', now.toISOString());
+        return { repos: count, done: true, failed: false };
+      }
+      after = found[found.length - 1].did;
+      await putState(db, 'handles_after', after);
+    }
+  } catch (err) {
+    // the sitemap still goes out, just with dids where handles are missing
+    console.warn(`[sitemap] couldn't page through hydrant's repos: ${err}`);
+    return { repos: count, done: false, failed: true };
+  }
+  return { repos: count, done: false, failed: false };
+}
+
+// owners new since the walk or touched by an identity event. a failed read stays
+// stale for the next run
+async function lookUpHandles(
+  db: D1Database,
+  repos: Repos,
+  after: string,
+): Promise<{ looked: number; failed: number; last: string }> {
+  const dids = await staleOwners(db, after, LOOKUP_PAGE);
+  const found: RepoHandle[] = [];
+  let failed = 0;
+  let next = 0;
+  const worker = async () => {
+    while (next < dids.length) {
+      const did = dids[next++];
+      try {
+        found.push({ did, handle: await repos.get(did) });
+      } catch {
+        failed++;
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: LOOKUP_CONCURRENCY }, worker));
+  await saveHandles(db, found);
+  if (failed > 0) console.warn(`[sitemap] ${failed} of ${dids.length} handle reads failed`);
+  return { looked: dids.length, failed, last: dids.at(-1) ?? after };
+}
+
 export async function syncFromHydrant(
   env: Env,
   deps: SyncDeps = defaultDeps,
@@ -150,6 +231,8 @@ export async function syncFromHydrant(
         return { events: 0, mutations: 0, more: false };
       }
       const changes = read.events.flatMap((evt) => extractUrlAction(evt, baseUrl, now) ?? []);
+      // before the cursor moves, so a retried batch can't skip it
+      await markStale(db, read.events.flatMap((evt) => (evt.kind === 'identity' ? [evt.did] : [])));
       const mutations = await applyChanges(db, changes, limits, String(read.cursor), now);
       return { events: read.events.length, mutations, more: read.full && !read.error };
     });
@@ -158,6 +241,27 @@ export async function syncFromHydrant(
     if (!batch.more) break;
   }
 
+  const repos = deps.repos(env);
+  let walkedRepos = 0;
+  let walked = false;
+  for (let step = 1; step <= MAX_WALK_STEPS && !walked; step++) {
+    const res = await steps.do(`walk ${step}`, () => walkRepos(db, repos, now));
+    walkedRepos += res.repos;
+    walked = res.done;
+    if (res.failed) break;
+  }
+
+  // until the walk is through, most owners are still in it
+  let lookedUpHandles = 0;
+  let after = '';
+  for (let step = 1; walked && step <= MAX_LOOKUP_STEPS; step++) {
+    const res = await steps.do(`lookups ${step}`, () => lookUpHandles(db, repos, after));
+    lookedUpHandles += res.looked;
+    // nothing answering means hydrant is down, so render with what we have
+    if (res.looked < LOOKUP_PAGE || res.failed === res.looked) break;
+    after = res.last;
+  }
+
   await steps.do('render', () => renderShards(db, env.SITEMAP_BUCKET, baseUrl, now));
 
   return steps.do('finish', async () => {
@@ -165,6 +269,8 @@ export async function syncFromHydrant(
     return {
       syncedEvents,
       mutatedUrls,
+      walkedRepos,
+      lookedUpHandles,
       newCursor: (await getState(db, 'cursor')) ?? '0',
       totalUrls: await countUrls(db),
     };
diff --git a/sitemap/src/types.ts b/sitemap/src/types.ts
index 20425b987..c97d0de1a 100644
--- a/sitemap/src/types.ts
+++ b/sitemap/src/types.ts
@@ -3,6 +3,9 @@ export interface Env {
   SITEMAP_BUCKET: R2Bucket;
   BASE_URL?: string;
   HYDRANT_URL?: string;
+  // a workers vpc service onto the hydrant node with the tunnel, for /repos, which
+  // isn't public. without it /repos goes to HYDRANT_URL, so only a local hydrant works
+  HYDRANT?: Fetcher;
   SHARD_MAX_URLS?: string;
   SHARD_MAX_BYTES?: string;
   SYNC_BATCH_SIZE?: string;
diff --git a/sitemap/test/d1.ts b/sitemap/test/d1.ts
index 7599cd68d..b8a5f9843 100644
--- a/sitemap/test/d1.ts
+++ b/sitemap/test/d1.ts
@@ -1,13 +1,23 @@
-import { readFileSync } from 'node:fs';
+import { readdirSync, readFileSync } from 'node:fs';
 import { DatabaseSync } from 'node:sqlite';
 
+const MIGRATIONS = new URL('../migrations/', import.meta.url);
+
 // enough of d1 over node's sqlite to run the real migration and queries
 export class TestD1 {
   sqlite = new DatabaseSync(':memory:');
   writes = 0;
 
-  constructor() {
-    this.sqlite.exec(readFileSync(new URL('../migrations/0001_init.sql', import.meta.url), 'utf8'));
+  // `stopAt` leaves the later migrations out, to test one against older data
+  constructor(stopAt?: string) {
+    for (const name of readdirSync(MIGRATIONS).sort()) {
+      if (name === stopAt) break;
+      this.migrate(name);
+    }
+  }
+
+  migrate(name: string): void {
+    this.sqlite.exec(readFileSync(new URL(name, MIGRATIONS), 'utf8'));
   }
 
   prepare(sql: string): TestStatement {
diff --git a/sitemap/test/node.d.ts b/sitemap/test/node.d.ts
index 96d8400d5..7032eb5a1 100644
--- a/sitemap/test/node.d.ts
+++ b/sitemap/test/node.d.ts
@@ -18,4 +18,5 @@ interface ImportMeta {
 
 declare module 'node:fs' {
   export function readFileSync(path: URL, encoding: 'utf8'): string;
+  export function readdirSync(path: URL): string[];
 }
diff --git a/sitemap/test/sitemap.test.ts b/sitemap/test/sitemap.test.ts
index d1ed3fbda..5032728c6 100644
--- a/sitemap/test/sitemap.test.ts
+++ b/sitemap/test/sitemap.test.ts
@@ -1,6 +1,7 @@
 import type { JetstreamEvent } from '@atcute/jetstream';
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it, vi } from 'vitest';
 import { extractUrlAction } from '../src/hydrant';
+import { hydrantRepos, withHandle } from '../src/identity';
 import { formatShardFilename, getStaticUrls } from '../src/sitemap';
 import type { ShardInfo, UrlEntry } from '../src/types';
 import { commit, identity } from './events';
@@ -82,6 +83,7 @@ describe('Hydrant Event Processing', () => {
     const action = extractUrlAction(event, baseUrl, now);
     expect(action?.type).toBe('upsert');
     expect(action?.uri).toBe('at://did:plc:alice123/sh.tangled.repo/3mhgx3cxdfq22');
+    expect(action?.did).toBe('did:plc:alice123');
     expect(action?.url).toBe('https://tangled.org/did:plc:alice123/cool-tool');
     expect(action?.type === 'upsert' && action.entry.lastmod).toBe('2026-07-15');
     expect(action?.type === 'upsert' && action.entry.priority).toBe(0.8);
@@ -135,6 +137,59 @@ describe('Hydrant Event Processing', () => {
   });
 });
 
+describe('handles', () => {
+  function hydrant(status: number, body: unknown) {
+    const fetch = vi.fn(async (_url: URL, _init?: RequestInit) => new Response(JSON.stringify(body), { status }));
+    return { fetch, repos: hydrantRepos({ fetch } as any, 'https://hydrant.test') };
+  }
+
+  it('swaps the owner segment for the handle and nothing else', () => {
+    expect(withHandle('https://tangled.org/did:plc:alice/tool', 'did:plc:alice', 'alice.test')).toBe(
+      'https://tangled.org/alice.test/tool',
+    );
+    expect(withHandle('https://tangled.org/strings/did:plc:alice/3mtid', 'did:plc:alice', 'alice.test')).toBe(
+      'https://tangled.org/strings/alice.test/3mtid',
+    );
+    expect(withHandle('https://tangled.org/did:plc:alice/did:plc:alicex', 'did:plc:alice', 'alice.test')).toBe(
+      'https://tangled.org/alice.test/did:plc:alicex',
+    );
+    expect(withHandle('https://tangled.org/did:plc:alice', 'did:plc:alice', null)).toBe('https://tangled.org/did:plc:alice');
+    expect(withHandle('https://tangled.org/about', undefined, undefined)).toBe('https://tangled.org/about');
+  });
+
+  it('pages through the repos as json', async () => {
+    const { fetch, repos } = hydrant(200, [
+      { did: 'did:plc:alice', handle: 'Alice.Test', status: 'synced' },
+      { did: 'did:plc:bob', status: 'synced' },
+      { did: 'did:plc:carol', handle: 'handle.invalid' },
+    ]);
+    expect(await repos.list(undefined, 1000)).toEqual([
+      { did: 'did:plc:alice', handle: 'alice.test' },
+      { did: 'did:plc:bob', handle: null },
+      { did: 'did:plc:carol', handle: null },
+    ]);
+    expect(String(fetch.mock.calls[0][0])).toBe('https://hydrant.test/repos?limit=1000');
+    expect(fetch.mock.calls[0][1]?.headers).toEqual({ accept: 'application/json' });
+
+    await repos.list('did:plc:carol', 2);
+    expect(String(fetch.mock.calls[1][0])).toBe('https://hydrant.test/repos?limit=2&cursor=did%3Aplc%3Acarol');
+  });
+
+  it('reads one repo, with no handle for one hydrant never saw', async () => {
+    const found = hydrant(200, { did: 'did:plc:alice', handle: 'Alice.Test' });
+    expect(await found.repos.get('did:plc:alice')).toBe('alice.test');
+    expect(String(found.fetch.mock.calls[0][0])).toBe('https://hydrant.test/repos/did:plc:alice');
+
+    expect(await hydrant(404, 'repository not found').repos.get('did:plc:alice')).toBeNull();
+  });
+
+  it('throws when hydrant has no answer', async () => {
+    const { repos } = hydrant(502, 'bad gateway');
+    await expect(repos.get('did:plc:alice')).rejects.toThrow('http 502');
+    await expect(repos.list(undefined, 1000)).rejects.toThrow('http 502');
+  });
+});
+
 describe('Shard Management', () => {
   it('formats shard filenames with zero padding', () => {
     expect(formatShardFilename(1)).toBe('sitemap-0001.xml.gz');
diff --git a/sitemap/test/sync.test.ts b/sitemap/test/sync.test.ts
index 403918cce..32218bc81 100644
--- a/sitemap/test/sync.test.ts
+++ b/sitemap/test/sync.test.ts
@@ -1,10 +1,11 @@
 import { describe, expect, it } from 'vitest';
 import type { JetstreamEvent } from '@atcute/jetstream';
 import { parseConfigLimits, syncFromHydrant, type SyncDeps } from '../src/index';
+import type { Repos } from '../src/identity';
 import type { Env } from '../src/types';
 import { TestD1 } from './d1';
 import { commit, deadJetstream, fakeJetstream, identity } from './events';
-import { gzipDecompress } from '../src/xml';
+import { buildUrlsetXml, gzipDecompress, measureXmlBytes } from '../src/xml';
 
 class MockR2Object {
   constructor(private buffer: ArrayBuffer, public httpMetadata?: any) {}
@@ -49,8 +50,54 @@ class MockR2Bucket {
   }
 }
 
-function deps(subscribe: SyncDeps['subscribe']): SyncDeps {
-  return { subscribe, idleMs: 5, closeMs: 5 };
+// hydrant's repos as the tunnel serves them, where nobody has a handle unless a test
+// gives them one. changing `handles` later is someone changing theirs
+function hydrant(handles: Record<string, string | null> = {}): Repos & { asked: string[]; pages: (string | undefined)[] } {
+  const fake = {
+    asked: [] as string[],
+    pages: [] as (string | undefined)[],
+    async list(after: string | undefined, limit: number) {
+      fake.pages.push(after);
+      return Object.keys(handles)
+        .sort()
+        .filter((did) => after === undefined || did > after)
+        .slice(0, limit)
+        .map((did) => ({ did, handle: handles[did] }));
+    },
+    async get(did: string) {
+      fake.asked.push(did);
+      return handles[did] ?? null;
+    },
+  };
+  return fake;
+}
+
+const down: Repos = {
+  async list() {
+    throw new Error('hydrant is down');
+  },
+  async get() {
+    throw new Error('hydrant is down');
+  },
+};
+
+function deps(subscribe: SyncDeps['subscribe'], repos: Repos = hydrant()): SyncDeps {
+  return { subscribe, repos: () => repos, idleMs: 5, closeMs: 5 };
+}
+
+function users(count: number): Record<string, string> {
+  return Object.fromEntries(
+    Array.from({ length: count }, (_, i) => [`did:plc:user${String(i).padStart(4, '0')}`, `user${i}.test`]),
+  );
+}
+
+async function shardXml(bucket: MockR2Bucket, shard: number): Promise<string> {
+  const obj = await bucket.get(`sitemaps/sitemap-${String(shard).padStart(4, '0')}.xml.gz`);
+  return gzipDecompress(await obj!.arrayBuffer());
+}
+
+function handleRow(db: TestD1, did: string): { handle: string | null; stale: number } | undefined {
+  return db.query<{ handle: string | null; stale: number }>('SELECT handle, stale FROM handles WHERE did = ?', did)[0];
 }
 
 function shardOf(db: TestD1, url: string): number | undefined {
@@ -282,3 +329,248 @@ describe('Full syncFromHydrant workflow with Sharding & Multi-Sync', () => {
     });
   });
 });
+
+const EMPTY_SHARD = measureXmlBytes(buildUrlsetXml([]));
+
+// what `applyChanges` thinks a shard weighs has to match what it renders to
+async function expectSizesTracked(db: TestD1, bucket: MockR2Bucket): Promise<void> {
+  for (const { shard, bytes } of db.query<{ shard: number; bytes: number }>(
+    'SELECT shard, sum(bytes) AS bytes FROM urls GROUP BY shard',
+  )) {
+    // `urlBytes` counts a joining newline for every url, and there's one fewer
+    expect(measureXmlBytes(await shardXml(bucket, shard))).toBe(EMPTY_SHARD + bytes - 1);
+  }
+}
+
+describe('handles', () => {
+  const day = 24 * 60 * 60 * 1000;
+  const startedAt = new Date('2026-10-01T00:00:00Z');
+  const env = (db: TestD1, bucket: MockR2Bucket, vars: Partial<Env> = {}): Env => ({
+    SITEMAP_DB: db.asD1(),
+    SITEMAP_BUCKET: bucket as any,
+    BASE_URL: 'https://tangled.org',
+    ...vars,
+  });
+
+  it('renders owners as their handles and keeps the did when there is none', async () => {
+    const db = new TestD1();
+    const bucket = new MockR2Bucket();
+    const jetstream = fakeJetstream(() => [
+      commit(1, 'did:plc:alice', 'sh.tangled.actor.profile', 'self'),
+      commit(2, 'did:plc:alice', 'sh.tangled.repo', '3mtid1', 'create', { name: 'tool' }),
+      commit(3, 'did:plc:alice', 'sh.tangled.string', '3mtid2'),
+      commit(4, 'did:plc:bob', 'sh.tangled.repo', 'thing'),
+    ]);
+    const repos = hydrant({ 'did:plc:alice': 'alice.test' });
+
+    const res = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos));
+    expect(res.walkedRepos).toBe(1);
+    // the walk had alice, bob is the one it didn't know
+    expect(repos.asked).toEqual(['did:plc:bob']);
+    expect(res.lookedUpHandles).toBe(1);
+
+    const xml = await shardXml(bucket, 1);
+    expect(xml).toContain('<loc>https://tangled.org/alice.test</loc>');
+    expect(xml).toContain('<loc>https://tangled.org/alice.test/tool</loc>');
+    expect(xml).toContain('<loc>https://tangled.org/strings/alice.test/3mtid2</loc>');
+    expect(xml).toContain('<loc>https://tangled.org/did:plc:bob/thing</loc>');
+    expect(xml).not.toContain('did:plc:alice');
+    // stored by did, so a handle change never moves them
+    expect(shardOf(db, 'https://tangled.org/did:plc:alice/tool')).toBe(1);
+    await expectSizesTracked(db, bucket);
+  });
+
+  it('walks every repo once, and again after a reset', async () => {
+    const db = new TestD1();
+    const bucket = new MockR2Bucket();
+    const handles = users(2500);
+    const jetstream = fakeJetstream(() => [
+      commit(1, 'did:plc:user0042', 'sh.tangled.repo', 'tool'),
+      commit(2, 'did:plc:user2499', 'sh.tangled.repo', 'tool'),
+    ]);
+    const repos = hydrant(handles);
+
+    const res = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos));
+    expect(res.walkedRepos).toBe(2500);
+    expect(repos.pages).toEqual([undefined, 'did:plc:user0999', 'did:plc:user1999']);
+    expect(repos.asked).toEqual([]);
+    expect(await shardXml(bucket, 1)).toContain('<loc>https://tangled.org/user2499.test/tool</loc>');
+
+    const again = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos));
+    expect(again.walkedRepos).toBe(0);
+    expect(repos.pages).toHaveLength(3);
+
+    await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos), undefined, { reset: true });
+    expect(repos.pages).toHaveLength(6);
+    expect(repos.asked).toEqual([]);
+  });
+
+  it('carries on a walk that broke off from where it got to', async () => {
+    const db = new TestD1();
+    const bucket = new MockR2Bucket();
+    const jetstream = fakeJetstream(() => [commit(1, 'did:plc:user2000', 'sh.tangled.repo', 'tool')]);
+    const repos = hydrant(users(2500));
+    let calls = 0;
+    const flaky: Repos = {
+      list: (after, limit) => (++calls === 2 ? down.list(after, limit) : repos.list(after, limit)),
+      get: (did) => repos.get(did),
+    };
+
+    const res = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, flaky));
+    expect(res.walkedRepos).toBe(1000);
+    // the rest of the walk will have most owners, so nobody gets read one by one yet
+    expect(res.lookedUpHandles).toBe(0);
+    expect(await shardXml(bucket, 1)).toContain('<loc>https://tangled.org/did:plc:user2000/tool</loc>');
+
+    const next = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, flaky));
+    expect(next.walkedRepos).toBe(1500);
+    expect(repos.pages).toEqual([undefined, 'did:plc:user0999', 'did:plc:user1999']);
+    expect(db.query('SELECT did FROM handles')).toHaveLength(2500);
+    expect(await shardXml(bucket, 1)).toContain('<loc>https://tangled.org/user2000.test/tool</loc>');
+  });
+
+  it('rerenders only the shards of an owner whose handle changed', async () => {
+    const db = new TestD1();
+    const bucket = new MockR2Bucket();
+    let events = [
+      commit(1, 'did:plc:alice', 'sh.tangled.repo', 'alice-1'),
+      commit(2, 'did:plc:bob', 'sh.tangled.repo', 'bob-1'),
+      commit(3, 'did:plc:bob', 'sh.tangled.repo', 'bob-2'),
+      commit(4, 'did:plc:bob', 'sh.tangled.repo', 'bob-3'),
+      commit(5, 'did:plc:alice', 'sh.tangled.repo', 'alice-2'),
+    ];
+    const jetstream = fakeJetstream(() => events);
+    const vars = { SHARD_MAX_URLS: '3' };
+    const handles: Record<string, string> = { 'did:plc:alice': 'alice.test', 'did:plc:bob': 'bob.test' };
+    const repos = hydrant(handles);
+
+    await syncFromHydrant(env(db, bucket, vars), deps(jetstream.subscribe, repos), undefined, { startedAt });
+    expect(shardCounts(db)).toEqual([3, 3, 1]);
+
+    handles['did:plc:alice'] = 'alice.example.com';
+    events = [identity(10, 'did:plc:alice', 'alice.example.com')];
+    const puts = bucket.putCount;
+    await syncFromHydrant(env(db, bucket, vars), deps(jetstream.subscribe, repos), undefined, {
+      startedAt: new Date(startedAt.getTime() + day),
+    });
+
+    expect(repos.asked).toEqual(['did:plc:alice']);
+    expect(bucket.putCount - puts).toBe(3);
+    expect(await shardXml(bucket, 1)).toContain('<loc>https://tangled.org/alice.example.com/alice-1</loc>');
+    expect(await shardXml(bucket, 3)).toContain('<loc>https://tangled.org/alice.example.com/alice-2</loc>');
+    expect(await shardXml(bucket, 2)).not.toContain('alice');
+    await expectSizesTracked(db, bucket);
+  });
+
+  it('keeps the old handle when a read fails and tries again next run', async () => {
+    const db = new TestD1();
+    const bucket = new MockR2Bucket();
+    let events = [commit(1, 'did:plc:alice', 'sh.tangled.repo', 'tool')];
+    const jetstream = fakeJetstream(() => events);
+    const handles: Record<string, string> = { 'did:plc:alice': 'alice.test' };
+    const repos = hydrant(handles);
+    await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos), undefined, { startedAt });
+
+    handles['did:plc:alice'] = 'alice.example.com';
+    events = [identity(10, 'did:plc:alice', 'alice.example.com')];
+    await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, down), undefined, { startedAt });
+    expect(await shardXml(bucket, 1)).toContain('<loc>https://tangled.org/alice.test/tool</loc>');
+    expect(handleRow(db, 'did:plc:alice')).toEqual({ handle: 'alice.test', stale: 1 });
+
+    events = [];
+    await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, repos), undefined, { startedAt });
+    expect(await shardXml(bucket, 1)).toContain('<loc>https://tangled.org/alice.example.com/tool</loc>');
+    expect(handleRow(db, 'did:plc:alice')).toEqual({ handle: 'alice.example.com', stale: 0 });
+  });
+
+  it('reads each owner at most once a run, even one that keeps failing', async () => {
+    const db = new TestD1();
+    const bucket = new MockR2Bucket();
+    const owners = Object.keys(users(450));
+    const jetstream = fakeJetstream(() => owners.map((did, i) => commit(i + 1, did, 'sh.tangled.actor.profile', 'self')));
+    const repos = hydrant();
+    const asked: string[] = [];
+    const broken: Repos = {
+      list: (after, limit) => repos.list(after, limit),
+      async get(did) {
+        asked.push(did);
+        if (did === owners[0]) throw new Error('bad doc');
+        return null;
+      },
+    };
+
+    const res = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, broken), undefined, { startedAt });
+    expect(res.lookedUpHandles).toBe(450);
+    expect(asked.sort()).toEqual(owners);
+    expect(handleRow(db, owners[0])).toBeUndefined();
+  });
+
+  it('stops reading after a page where nothing answered', async () => {
+    const db = new TestD1();
+    const bucket = new MockR2Bucket();
+    const owners = Object.keys(users(250));
+    const jetstream = fakeJetstream(() => owners.map((did, i) => commit(i + 1, did, 'sh.tangled.actor.profile', 'self')));
+    const asked: string[] = [];
+    const flaky: Repos = {
+      list: async () => [],
+      async get(did) {
+        asked.push(did);
+        throw new Error('hydrant is down');
+      },
+    };
+
+    const res = await syncFromHydrant(env(db, bucket), deps(jetstream.subscribe, flaky), undefined, { startedAt });
+    expect(asked).toHaveLength(200);
+    expect(res.lookedUpHandles).toBe(200);
+    expect(db.query('SELECT did FROM handles')).toEqual([]);
+    expect(bucket.store.has('sitemaps/sitemap-0001.xml.gz')).toBe(true);
+  });
+
+  it('sizes new urls with the handle they render with', async () => {
+    const db = new TestD1();
+    const bucket = new MockR2Bucket();
+    const handle = 'a-very-long-handle-that-takes-up-a-lot-of-room.example.com';
+    let events = [commit(1, 'did:plc:alice', 'sh.tangled.actor.profile', 'self')];
+    const jetstream = fakeJetstream(() => events);
+    const repos = hydrant({ 'did:plc:alice': handle });
+    const vars = { SHARD_MAX_BYTES: '700' };
+    await syncFromHydrant(env(db, bucket, vars), deps(jetstream.subscribe, repos), undefined, { startedAt });
+
+    events = [1, 2, 3, 4].map((i) => commit(10 + i, 'did:plc:alice', 'sh.tangled.repo', `repo-${i}`));
+    await syncFromHydrant(env(db, bucket, vars), deps(jetstream.subscribe, repos), undefined, { startedAt });
+
+    expect(shardCounts(db).length).toBeGreaterThan(2);
+    for (const { shard } of db.query<{ shard: number }>('SELECT DISTINCT shard FROM urls')) {
+      const xml = await shardXml(bucket, shard);
+      expect(measureXmlBytes(xml)).toBeLessThanOrEqual(700);
+    }
+    expect(await shardXml(bucket, 1)).toContain(`<loc>https://tangled.org/${handle}</loc>`);
+    await expectSizesTracked(db, bucket);
+  });
+});
+
+describe('0002_handles migration', () => {
+  it('fills in owner dids and leaves the handles to the walk', () => {
+    const db = new TestD1('0002_handles.sql');
+    const urls = [
+      'https://tangled.org/',
+      'https://tangled.org/did:plc:alice',
+      'https://tangled.org/did:plc:alice/tool',
+      'https://tangled.org/strings/did:plc:bob/3mtid',
+      'https://tangled.org/did:web:example.com/thing',
+    ];
+    for (const url of urls) {
+      db.sqlite.prepare("INSERT INTO urls (url, shard, lastmod, bytes) VALUES (?, 1, '2026-10-01', 100)").run(url);
+    }
+    db.migrate('0002_handles.sql');
+
+    expect(db.query('SELECT url, did FROM urls ORDER BY url')).toEqual([
+      { url: 'https://tangled.org/', did: null },
+      { url: 'https://tangled.org/did:plc:alice', did: 'did:plc:alice' },
+      { url: 'https://tangled.org/did:plc:alice/tool', did: 'did:plc:alice' },
+      { url: 'https://tangled.org/did:web:example.com/thing', did: 'did:web:example.com' },
+      { url: 'https://tangled.org/strings/did:plc:bob/3mtid', did: 'did:plc:bob' },
+    ]);
+    expect(db.query('SELECT did FROM handles')).toEqual([]);
+  });
+});
diff --git a/sitemap/wrangler.dev.jsonc b/sitemap/wrangler.dev.jsonc
index fccb84164..31b291314 100644
--- a/sitemap/wrangler.dev.jsonc
+++ b/sitemap/wrangler.dev.jsonc
@@ -30,6 +30,15 @@
       "bucket_name": "tangled-sitemaps-dev",
     },
   ],
+  // hydrant's /repos over the tunnel on hyd-gcp-eu-1 (tunnel = true in infra's
+  // terraform/modules/envs/dev.tf)
+  "vpc_services": [
+    {
+      "binding": "HYDRANT",
+      "service_id": "01a11677-64af-78f2-8512-94b028e01487",
+      "remote": true,
+    },
+  ],
   "vars": {
     "BASE_URL": "https://next.tangled.org",
     "HYDRANT_URL": "https://index.tangled.network",
diff --git a/sitemap/wrangler.jsonc b/sitemap/wrangler.jsonc
index e75a416fe..599bcfa5c 100644
--- a/sitemap/wrangler.jsonc
+++ b/sitemap/wrangler.jsonc
@@ -30,6 +30,15 @@
       "bucket_name": "tangled-sitemaps",
     },
   ],
+  // hydrant's /repos over the tunnel on hyd-gcp-prod-eu-1 (tunnel = true in infra's
+  // terraform/modules/envs/prod.tf)
+  "vpc_services": [
+    {
+      "binding": "HYDRANT",
+      "service_id": "01a11694-d405-7c93-a786-e2483ed74577",
+      "remote": true,
+    },
+  ],
   "vars": {
     "BASE_URL": "https://tangled.org",
     "HYDRANT_URL": "https://index.tangled.network",
-- 
2.51.2