diff --git a/web/src/lib/api/_request.ts b/web/src/lib/api/_request.ts index 666ba670a..adcbdc59b 100644 --- a/web/src/lib/api/_request.ts +++ b/web/src/lib/api/_request.ts @@ -54,6 +54,10 @@ export const jsonGet = async ( signal: init?.signal }); if (!response.ok) throw await toResponseError(response); + if (typeof response.text === "function") { + const text = await response.text(); + return (text.length > 0 ? JSON.parse(text) : undefined) as T; + } return (await response.json()) as T; }; @@ -66,14 +70,18 @@ export const jsonPost = async ( const response = await ctx.fetch(buildUrl(ctx.serviceUrl, nsid), { method: "POST", headers: { - "content-type": "application/json", + ...(body !== undefined ? { "content-type": "application/json" } : {}), accept: "application/json", ...init?.headers }, - body: JSON.stringify(body), + body: body !== undefined ? JSON.stringify(body) : undefined, signal: init?.signal }); if (!response.ok) throw await toResponseError(response); + if (typeof response.text === "function") { + const text = await response.text(); + return (text.length > 0 ? JSON.parse(text) : undefined) as T; + } return (await response.json()) as T; }; diff --git a/web/src/lib/api/githubImport.test.ts b/web/src/lib/api/githubImport.test.ts new file mode 100644 index 000000000..75bf8d4a4 --- /dev/null +++ b/web/src/lib/api/githubImport.test.ts @@ -0,0 +1,804 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import type { Did } from "@atcute/lexicons/syntax"; +import type { OAuthUserAgent } from "@atcute/oauth-browser-client"; +import type { GitHubAccount, GitHubRepo } from "$lib/github"; +import type { ProfileRecord } from "$lib/api/records"; +import type { NewMigrationJob } from "$lib/api/migrator"; +import { + canonicalRepoName, + fetchGitHubAccount, + findRepoCollisions, + importGitHubProfile, + importFromUrl, + importGitHubRepo, + importGitHubRepos, + invalidateGitHubAccount, + validateGitHubRepoUrl +} from "$lib/api/githubImport"; + +const mockCreateMigrationTask = vi.fn(); +vi.mock("$lib/api/migrator", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createMigrationTask: (...args: unknown[]) => mockCreateMigrationTask(...args) + }; +}); + +const mockUploadProfileAvatar = vi.fn(); +vi.mock("$lib/api/profile", () => ({ + uploadProfileAvatar: (...args: unknown[]) => mockUploadProfileAvatar(...args) +})); + +const mockPutRecord = vi.fn(); +const mockCreateRecord = vi.fn(); +const mockReadRecord = vi.fn(); +vi.mock("$lib/api/write", () => ({ + createRecord: (...args: unknown[]) => mockCreateRecord(...args), + putRecord: (...args: unknown[]) => mockPutRecord(...args), + readRecord: (...args: unknown[]) => mockReadRecord(...args) +})); + +const mockCreatePubKey = vi.fn(); +vi.mock("$lib/api/settings", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createPubKey: (...args: unknown[]) => mockCreatePubKey(...args) + }; +}); + +const mockListEmails = vi.fn(); +const mockAddEmail = vi.fn(); +vi.mock("$lib/api/emails", () => ({ + listEmails: (...args: unknown[]) => mockListEmails(...args), + addEmail: (...args: unknown[]) => mockAddEmail(...args) +})); + +const mockRpcCall = vi.fn(); +vi.mock("$lib/auth/agent", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createClient: () => ({ + call: (...args: unknown[]) => mockRpcCall(...args) + }) + }; +}); + +vi.mock("$lib/api/deliberi", () => ({ + createDeliberiClient: (opts: unknown) => opts +})); + +const testAgent = { + sub: "did:plc:alice" as Did +} as unknown as OAuthUserAgent; + +const baseRepo: GitHubRepo = { + id: 12345, + name: "sample-repo", + fullName: "octocat/sample-repo", + description: "A great sample project", + cloneUrl: "https://github.com/octocat/sample-repo.git", + htmlUrl: "https://github.com/octocat/sample-repo", + defaultBranch: "main" +}; + +const GITHUB_ORIGIN = "https://github.com"; + +const validate = ( + url: string, + fullName: string, + kind: "htmlUrl" | "cloneUrl", + origin: string = GITHUB_ORIGIN +) => validateGitHubRepoUrl(url, fullName, kind, origin); + +describe("validateGitHubRepoUrl", () => { + it("accepts valid canonical HTTPS github.com URLs matching fullName", () => { + expect( + validate("https://github.com/octocat/sample-repo", "octocat/sample-repo", "htmlUrl") + ).toBe("https://github.com/octocat/sample-repo"); + + expect( + validate( + "https://github.com/octocat/sample-repo.git", + "octocat/sample-repo", + "cloneUrl" + ) + ).toBe("https://github.com/octocat/sample-repo.git"); + + expect( + validate("https://github.com/octocat/sample-repo", "octocat/sample-repo", "cloneUrl") + ).toBe("https://github.com/octocat/sample-repo"); + }); + + it("rejects URLs whose path does not match fullName", () => { + expect(() => + validate( + "https://github.com/someone-else/sample-repo", + "octocat/sample-repo", + "htmlUrl" + ) + ).toThrow(/pathname must match repository/); + }); + + it("rejects non-HTTPS, credentials, nondefault ports, query, and hash", () => { + expect(() => + validate("http://github.com/octocat/sample-repo", "octocat/sample-repo", "htmlUrl") + ).toThrow(/protocol must be https/); + + expect(() => + validate( + "https://user:pass@github.com/octocat/sample-repo", + "octocat/sample-repo", + "htmlUrl" + ) + ).toThrow(/credentials are not allowed/); + + expect(() => + validate( + "https://github.com:8443/octocat/sample-repo", + "octocat/sample-repo", + "htmlUrl" + ) + ).toThrow(/non-default port is not allowed/); + + expect(() => + validate( + "https://github.com/octocat/sample-repo?ref=main", + "octocat/sample-repo", + "htmlUrl" + ) + ).toThrow(/query parameters are not allowed/); + + expect(() => + validate( + "https://github.com/octocat/sample-repo#readme", + "octocat/sample-repo", + "htmlUrl" + ) + ).toThrow(/hash fragments are not allowed/); + }); + + it("checks the host against the account's configured origin", () => { + const stub = "https://github.tngl.boltless.dev"; + expect( + validate(`${stub}/octocat/sample-repo.git`, "octocat/sample-repo", "cloneUrl", stub) + ).toBe(`${stub}/octocat/sample-repo.git`); + + expect(() => + validate( + "https://github.com/octocat/sample-repo.git", + "octocat/sample-repo", + "cloneUrl", + stub + ) + ).toThrow(/host must be github\.tngl\.boltless\.dev/); + + expect(() => + validate( + `${stub}/octocat/sample-repo.git`, + "octocat/sample-repo", + "cloneUrl", + GITHUB_ORIGIN + ) + ).toThrow(/host must be github\.com/); + }); + + it("refuses an origin that is not https, rather than quietly allowing it", () => { + expect(() => + validate( + "http://github.com/octocat/sample-repo.git", + "octocat/sample-repo", + "cloneUrl", + "http://github.com" + ) + ).toThrow(/protocol must be https/); + + expect(() => + validate( + "https://github.com/octocat/sample-repo.git", + "octocat/sample-repo", + "cloneUrl", + "not a url" + ) + ).toThrow(/is not a URL/); + }); + + it("rejects malformed fullName grammar", () => { + expect(() => + validate("https://github.com/octocat/sample-repo", "single-name", "htmlUrl") + ).toThrow(/Invalid GitHub fullName/); + }); +}); + +describe("canonicalRepoName & single normalization", () => { + it("matches validateRepoName semantics and lowercases", () => { + expect(canonicalRepoName("Demo-Repo.git")).toBe("demo-repo"); + expect(canonicalRepoName("foo.git.git")).toBe("foo.git"); + expect(canonicalRepoName("plain-name")).toBe("plain-name"); + }); +}); + +describe("findRepoCollisions", () => { + it("detects collisions across batch and existing repositories with string or object candidates", () => { + const candidates = ["awesome-app", { name: "Awesome-App.git" }, "other-tool"]; + const existing = ["Other-Tool.git", "prior-repo"]; + const collisions = findRepoCollisions(candidates, existing); + + expect(collisions.get("awesome-app")?.hasCollision).toBe(true); + expect(collisions.get("awesome-app")?.candidates).toEqual([ + "awesome-app", + { name: "Awesome-App.git" } + ]); + expect(collisions.get("awesome-app")?.conflictsWithExisting).toBe(false); + + expect(collisions.get("other-tool")?.hasCollision).toBe(true); + expect(collisions.get("other-tool")?.conflictsWithExisting).toBe(true); + }); +}); + +describe("importGitHubRepo", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockCreateMigrationTask.mockImplementation( + async ( + _agent: unknown, + _host: unknown, + _requestId: unknown, + jobs: NewMigrationJob[] + ) => ({ + id: "task-1", + ownerDid: "did:plc:alice", + createdAt: "2026-01-01T00:00:00Z", + jobs: jobs.map((job, index) => ({ + id: String(index + 1), + name: job.name, + knotDid: job.knotDid, + sourceUrl: job.sourceUrl, + private: false, + status: "queued", + attempts: 0 + })) + }) + ); + }); + + it("hands the clone url to the migrator, with the knot to create the repository on", async () => { + const started = await importGitHubRepo( + testAgent, + "https://migrator.test", + "https://github.com", + { + ownerDid: "did:plc:alice" as Did, + ownerHandle: "alice.tangled.org", + knot: "knot.test", + repo: baseRepo + } + ); + + expect(started).toEqual({ taskId: "task-1", name: "sample-repo", jobId: "1" }); + expect(mockCreateMigrationTask).toHaveBeenCalledTimes(1); + const [, host, requestId, jobs] = mockCreateMigrationTask.mock.calls[0]; + expect(host).toBe("https://migrator.test"); + expect(typeof requestId).toBe("string"); + expect(jobs).toEqual([ + { + name: "sample-repo", + knotDid: "did:web:knot.test", + sourceUrl: "https://github.com/octocat/sample-repo.git" + } + ]); + }); + + it("batches multiple repositories, marking private jobs, preserving knot DIDs, and normalising names", async () => { + const task = await importGitHubRepos( + testAgent, + "https://migrator.test", + "https://github.com", + [ + { + ownerDid: "did:plc:alice" as Did, + ownerHandle: "alice.tangled.org", + knot: "knot.test", + repo: { ...baseRepo, private: true } + }, + { + ownerDid: "did:plc:alice" as Did, + ownerHandle: "alice.tangled.org", + knot: "did:web:custom-knot.test", + name: "second-repo.git.git", + repo: { + ...baseRepo, + id: 99, + name: "second-repo", + fullName: "octocat/second-repo", + cloneUrl: "https://github.com/octocat/second-repo.git" + } + } + ] + ); + + expect(task.id).toBe("task-1"); + expect(mockCreateMigrationTask).toHaveBeenCalledTimes(1); + const [, , , jobs] = mockCreateMigrationTask.mock.calls[0]; + expect(jobs).toHaveLength(2); + expect(jobs[0]).toMatchObject({ + name: "sample-repo", + knotDid: "did:web:knot.test", + private: true + }); + expect(jobs[1]).toMatchObject({ + name: "second-repo.git", + knotDid: "did:web:custom-knot.test" + }); + expect(jobs[1].private).toBeUndefined(); + }); + + it("rejects a clone url that does not belong to the named repository", async () => { + const mismatchedRepo: GitHubRepo = { + ...baseRepo, + cloneUrl: "https://github.com/imposter/sample-repo.git" + }; + + await expect( + importGitHubRepo(testAgent, "https://migrator.test", "https://github.com", { + ownerDid: "did:plc:alice" as Did, + ownerHandle: "alice.tangled.org", + knot: "knot.test", + repo: mismatchedRepo + }) + ).rejects.toThrow(/pathname must match repository/); + + expect(mockCreateMigrationTask).not.toHaveBeenCalled(); + }); +}); + +describe("importGitHubProfile", () => { + const sampleAccount: GitHubAccount = { + publicOrigin: "https://github.com", + login: "octocat", + avatarUrl: "https://avatars.githubusercontent.com/u/583231", + email: "octocat@github.com", + bio: "builds things", + blog: "https://octocat.example.com", + keys: [ + { + id: 1, + title: "Laptop", + key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG123 user@laptop" + }, + { + id: 2, + title: "Desktop", + key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC456 user@desktop" + } + ], + repos: [] + }; + + beforeEach(() => { + vi.clearAllMocks(); + mockReadRecord.mockReset(); + mockReadRecord.mockResolvedValue(null); + mockCreatePubKey.mockReset(); + mockCreatePubKey.mockImplementation(async (agent, name, key) => ({ + uri: `at://${agent.sub}/sh.tangled.publicKey/${name}`, + cid: "bafycreated", + rkey: `rkey-${name}` + })); + }); + + it("preserves existing PDS profile fields and passes swapRecord CID to putRecord", async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { + status: 200, + headers: { "content-type": "image/png" } + }) + ); + + const existingProfile: ProfileRecord = { + $type: "sh.tangled.actor.profile", + bluesky: true, + description: "Existing Tangled Bio", + location: "San Francisco", + links: ["https://example.com"] + }; + + mockReadRecord.mockResolvedValue({ + value: existingProfile, + written: { + uri: "at://did:plc:alice/sh.tangled.actor.profile/self", + cid: "bafk-existing-cid" + } + }); + mockRpcCall.mockResolvedValue({ ok: true, data: { records: [] } }); + + mockUploadProfileAvatar.mockResolvedValue({ + $type: "blob", + ref: { $link: "new-blob-cid" }, + mimeType: "image/png", + size: 3 + }); + + const result = await importGitHubProfile( + testAgent, + { deliberiUrl: "https://deliberi.test", fetch: fetchMock }, + sampleAccount, + { avatar: true, keys: false, email: false } + ); + + expect(result.errors).toEqual([]); + expect(result.imported).toContain("avatar"); + expect(fetchMock).toHaveBeenCalledWith("/_internal/github/avatar"); + expect(mockUploadProfileAvatar).toHaveBeenCalled(); + expect(mockPutRecord).toHaveBeenCalledWith( + testAgent, + "sh.tangled.actor.profile", + "self", + { + $type: "sh.tangled.actor.profile", + bluesky: true, + description: "Existing Tangled Bio", + location: "San Francisco", + links: ["https://example.com"], + avatar: { + $type: "blob", + ref: { $link: "new-blob-cid" }, + mimeType: "image/png", + size: 3 + } + }, + "bafk-existing-cid" + ); + }); + + it("regression: a failed profile read never writes or blanks the profile", async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { + status: 200, + headers: { "content-type": "image/png" } + }) + ); + + mockUploadProfileAvatar.mockResolvedValue({ + $type: "blob", + ref: { $link: "blob-cid" }, + mimeType: "image/png", + size: 3 + }); + + mockReadRecord.mockRejectedValue(new Error("Connection reset by peer")); + + const result = await importGitHubProfile( + testAgent, + { deliberiUrl: "https://deliberi.test", fetch: fetchMock }, + sampleAccount, + { avatar: true, keys: false, email: false } + ); + + expect(mockPutRecord).not.toHaveBeenCalled(); + expect(mockCreateRecord).not.toHaveBeenCalled(); + expect(result.errors).toHaveLength(1); + expect(result.errors[0]).toContain("Connection reset by peer"); + expect(result.imported).not.toContain("avatar"); + }); + + it("creates an absent profile without overwriting a concurrent creation", async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { + status: 200, + headers: { "content-type": "image/png" } + }) + ); + + mockUploadProfileAvatar.mockResolvedValue({ + $type: "blob", + ref: { $link: "fresh-blob-cid" }, + mimeType: "image/png", + size: 3 + }); + + mockRpcCall.mockResolvedValue({ + ok: false, + status: 404, + headers: new Headers(), + data: { error: "RecordNotFound" } + }); + + const result = await importGitHubProfile( + testAgent, + { deliberiUrl: "https://deliberi.test", fetch: fetchMock }, + sampleAccount, + { avatar: true, keys: false, email: false } + ); + + expect(result.errors).toEqual([]); + expect(result.imported).toContain("avatar"); + expect(mockPutRecord).not.toHaveBeenCalled(); + expect(mockCreateRecord).toHaveBeenCalledWith( + testAgent, + "sh.tangled.actor.profile", + { + $type: "sh.tangled.actor.profile", + bluesky: false, + avatar: { + $type: "blob", + ref: { $link: "fresh-blob-cid" }, + mimeType: "image/png", + size: 3 + } + }, + "self" + ); + }); + + it("reads all pages of PDS public keys and dedups raw SSH algorithm+base64, avoiding retries and duplicates", async () => { + const pdsPage1 = { + records: [ + { + value: { + key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG123 old-comment" + } + } + ], + cursor: "cursor-2" + }; + const pdsPage2 = { + records: [], + cursor: undefined + }; + + mockRpcCall.mockImplementation(async (_schema, options) => { + if (options?.params?.collection === "sh.tangled.publicKey") { + if (options?.params?.cursor === "cursor-2") { + return { ok: true, data: pdsPage2 }; + } + return { ok: true, data: pdsPage1 }; + } + return { ok: true, data: { records: [] } }; + }); + + const accountWithDupes: GitHubAccount = { + ...sampleAccount, + keys: [ + ...sampleAccount.keys, + { + id: 3, + title: "Desktop Duplicate", + key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC456 different@comment" + }, + { + id: 4, + title: "Broken Key", + key: "malformed" + } + ] + }; + + const result = await importGitHubProfile( + testAgent, + { deliberiUrl: "https://deliberi.test" }, + accountWithDupes, + { avatar: false, keys: true, email: false } + ); + + expect(mockCreatePubKey).toHaveBeenCalledTimes(1); + expect(mockCreatePubKey).toHaveBeenCalledWith( + testAgent, + "Desktop", + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC456 user@desktop" + ); + expect(result.imported).toContain("key:2"); + expect(result.keys).toEqual([ + { + rkey: "rkey-Desktop", + name: "Desktop", + key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC456 user@desktop" + } + ]); + expect(result.errors.some((e) => e.includes("Invalid SSH key format"))).toBe(true); + }); + + it("handles per-key errors independently without aborting remaining keys", async () => { + mockRpcCall.mockResolvedValue({ ok: true, data: { records: [] } }); + mockCreatePubKey + .mockRejectedValueOnce(new Error("Network timeout writing record")) + .mockResolvedValueOnce({ uri: "at://uri", cid: "cid", rkey: "rkey" }); + + const result = await importGitHubProfile( + testAgent, + { deliberiUrl: "https://deliberi.test" }, + sampleAccount, + { avatar: false, keys: true, email: false } + ); + + expect(mockCreatePubKey).toHaveBeenCalledTimes(2); + expect(result.imported).toEqual(["key:2"]); + expect(result.errors).toHaveLength(1); + expect(result.errors[0]).toContain("Network timeout writing record"); + }); + + it.each([ + ["absent", [{ address: "existing@tangled.org", verified: true, primary: true }], true], + [ + "already registered", + [{ address: "octocat@github.com", verified: true, primary: false }], + false + ] + ])("imports email when %s in deliberi", async (_case, existingEmails, shouldAdd) => { + mockListEmails.mockResolvedValue({ emails: existingEmails }); + + const result = await importGitHubProfile( + testAgent, + { deliberiUrl: "https://deliberi.test" }, + sampleAccount, + { avatar: false, keys: false, email: true } + ); + + expect(mockListEmails).toHaveBeenCalledTimes(1); + if (shouldAdd) { + expect(mockAddEmail).toHaveBeenCalledWith(expect.anything(), "octocat@github.com"); + expect(result.imported.some((msg) => msg.includes("verification required"))).toBe(true); + } else { + expect(mockAddEmail).not.toHaveBeenCalled(); + } + expect(result.errors).toEqual([]); + }); + + it("executes best-effort independent operations when multiple options are selected", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(new Response("Internal error", { status: 500 })); + + mockRpcCall.mockResolvedValue({ ok: true, data: { records: [] } }); + mockCreatePubKey.mockResolvedValue({ uri: "at://uri", cid: "cid", rkey: "rkey" }); + mockListEmails.mockResolvedValue({ emails: [] }); + + const result = await importGitHubProfile( + testAgent, + { deliberiUrl: "https://deliberi.test", fetch: fetchMock }, + sampleAccount, + { avatar: true, keys: true, email: true } + ); + + expect(result.errors).toHaveLength(1); + expect(result.errors[0]).toContain("failed to fetch avatar"); + expect(result.imported).toEqual([ + "key:1", + "key:2", + "email:octocat@github.com (verification required)" + ]); + }); +}); + +describe("importFromUrl", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockCreateMigrationTask.mockImplementation( + async ( + _agent: unknown, + _host: unknown, + _requestId: unknown, + jobs: NewMigrationJob[] + ) => ({ + id: "task-url", + ownerDid: "did:plc:alice", + createdAt: "2026-01-01T00:00:00Z", + jobs: jobs.map((job, index) => ({ + id: String(index + 1), + name: job.name, + knotDid: job.knotDid, + sourceUrl: job.sourceUrl, + private: false, + status: "queued", + attempts: 0 + })) + }) + ); + }); + + it("hands any https git host to the migrator, with the description", async () => { + const started = await importFromUrl(testAgent, "https://migrator.test", { + sourceUrl: "https://git.example.com/team/project.git", + name: "project", + description: "a small tool", + knot: "knot.test" + }); + + expect(started).toEqual({ taskId: "task-url", name: "project", jobId: "1" }); + const [, host, requestId, jobs] = mockCreateMigrationTask.mock.calls[0]; + expect(host).toBe("https://migrator.test"); + expect(typeof requestId).toBe("string"); + expect(jobs).toEqual([ + { + name: "project", + knotDid: "did:web:knot.test", + sourceUrl: "https://git.example.com/team/project.git", + description: "a small tool" + } + ]); + }); + + it.each([ + ["non-https scheme", "http://git.example.com/team/project.git", /https/], + ["custom port", "https://git.example.com:8443/team/project.git", /custom port/] + ])("refuses an invalid source URL with %s", async (_case, sourceUrl, pattern) => { + await expect( + importFromUrl(testAgent, "https://migrator.test", { + sourceUrl, + name: "project", + knot: "knot.test" + }) + ).rejects.toThrow(pattern); + expect(mockCreateMigrationTask).not.toHaveBeenCalled(); + }); + + it("accepts an explicit 443 port", async () => { + await importFromUrl(testAgent, "https://migrator.test", { + sourceUrl: "https://git.example.com:443/team/project.git", + name: "project", + knot: "knot.test" + }); + const [, , , jobs] = mockCreateMigrationTask.mock.calls[0]; + expect(jobs[0].sourceUrl).toBe("https://git.example.com/team/project.git"); + }); +}); + +describe("fetchGitHubAccount", () => { + const accountOf = (login: string): GitHubAccount => ({ + publicOrigin: "https://github.com", + bio: null, + blog: null, + login, + avatarUrl: "", + email: null, + keys: [], + repos: [] + }); + const fetcherOf = () => { + const fetcher = vi.fn(); + return fetcher; + }; + + it("serves repeated loads from the cache and refetches after invalidation", async () => { + const fetcher = fetcherOf(); + fetcher.mockImplementation(async () => Response.json(accountOf("octocat"))); + + expect((await fetchGitHubAccount(fetcher))?.login).toBe("octocat"); + expect((await fetchGitHubAccount(fetcher))?.login).toBe("octocat"); + expect(fetcher).toHaveBeenCalledTimes(1); + + invalidateGitHubAccount(fetcher); + await fetchGitHubAccount(fetcher); + expect(fetcher).toHaveBeenCalledTimes(2); + }); + + it("caches per fetcher, so test doubles never share entries", async () => { + const first = fetcherOf(); + const second = fetcherOf(); + first.mockResolvedValue(Response.json(accountOf("octocat"))); + second.mockResolvedValue(Response.json(accountOf("monalisa"))); + + await fetchGitHubAccount(first); + expect((await fetchGitHubAccount(second))?.login).toBe("monalisa"); + expect(first).toHaveBeenCalledTimes(1); + }); + + it.each([ + ["401 signed-out", 401, async (res: Promise) => expect(await res).toBeNull()], + [ + "500 failure", + 500, + async (res: Promise) => expect(res).rejects.toThrow(/Try connecting again/) + ] + ])("does not cache %s responses", async (_name, status, assertFirst) => { + const fetcher = fetcherOf(); + fetcher.mockResolvedValue(new Response("err", { status })); + + await assertFirst(fetchGitHubAccount(fetcher)); + fetcher.mockResolvedValue(Response.json(accountOf("octocat"))); + expect((await fetchGitHubAccount(fetcher))?.login).toBe("octocat"); + expect(fetcher).toHaveBeenCalledTimes(2); + }); +}); diff --git a/web/src/lib/api/githubImport.ts b/web/src/lib/api/githubImport.ts new file mode 100644 index 000000000..4ed4951f0 --- /dev/null +++ b/web/src/lib/api/githubImport.ts @@ -0,0 +1,440 @@ +import { ok } from "@atcute/client"; +import { mainSchema as listRecordsSchema } from "@atcute/atproto/types/repo/listRecords"; +import { isDid, type Did, type Nsid } from "@atcute/lexicons/syntax"; +import type { OAuthUserAgent } from "@atcute/oauth-browser-client"; +import { createClient, serviceDidForHost } from "$lib/auth/agent"; +import { validateRepoName } from "$lib/api/repoCreate"; +import { createMigrationTask, type MigrationTask } from "$lib/api/migrator"; +import { uploadProfileAvatar } from "$lib/api/profile"; +import { createRecord, putRecord, readRecord } from "$lib/api/write"; +import { createPubKey } from "$lib/api/settings"; +import { addEmail, listEmails } from "$lib/api/emails"; +import { createDeliberiClient } from "$lib/api/deliberi"; +import type { GitHubAccount, GitHubRepo } from "$lib/github"; +import type { ProfileRecord } from "$lib/api/records"; + +const ACCOUNT_CACHE_MS = 30_000; +const accountCache = new WeakMap(); + +export const fetchGitHubAccount = async ( + fetcher: typeof fetch = fetch +): Promise => { + const cached = accountCache.get(fetcher); + if (cached && Date.now() - cached.at < ACCOUNT_CACHE_MS) return cached.account; + const response = await fetcher("/_internal/github", { cache: "no-store" }); + if (response.status === 401) { + accountCache.delete(fetcher); + return null; + } + if (!response.ok) throw new Error("Could not load your GitHub account. Try connecting again."); + const account = (await response.json()) as GitHubAccount; + accountCache.set(fetcher, { account, at: Date.now() }); + return account; +}; + +export const invalidateGitHubAccount = (fetcher: typeof fetch = fetch) => + accountCache.delete(fetcher); + +const PROFILE_COLLECTION = "sh.tangled.actor.profile" as Nsid; +const PUBLIC_KEY_COLLECTION = "sh.tangled.publicKey" as Nsid; + +const GITHUB_REPO_GRAMMAR = /^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$/; + +const messageOf = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + +// localinfra serves a github stub on its own origin; github.com cannot be hardcoded +const hostOf = (origin: string): string => { + let parsed: URL; + try { + parsed = new URL(origin); + } catch { + throw new Error(`Invalid GitHub origin: "${origin}" is not a URL.`); + } + if (parsed.protocol !== "https:") { + throw new Error(`Invalid GitHub origin: protocol must be https.`); + } + return parsed.hostname.toLowerCase(); +}; + +export const validateGitHubRepoUrl = ( + urlString: string, + expectedFullName: string, + kind: "htmlUrl" | "cloneUrl", + origin: string +): string => { + if (!GITHUB_REPO_GRAMMAR.test(expectedFullName)) { + throw new Error(`Invalid GitHub fullName: "${expectedFullName}".`); + } + let parsed: URL; + try { + parsed = new URL(urlString); + } catch { + throw new Error(`Invalid GitHub ${kind}: malformed URL.`); + } + if (parsed.protocol !== "https:") { + throw new Error(`Invalid GitHub ${kind}: protocol must be https.`); + } + const allowedHost = hostOf(origin); + if (parsed.hostname.toLowerCase() !== allowedHost) { + throw new Error(`Invalid GitHub ${kind}: host must be ${allowedHost}.`); + } + if (parsed.username || parsed.password) { + throw new Error(`Invalid GitHub ${kind}: credentials are not allowed.`); + } + if (parsed.port !== "") { + throw new Error(`Invalid GitHub ${kind}: non-default port is not allowed.`); + } + if (parsed.search !== "") { + throw new Error(`Invalid GitHub ${kind}: query parameters are not allowed.`); + } + if (parsed.hash !== "") { + throw new Error(`Invalid GitHub ${kind}: hash fragments are not allowed.`); + } + + const allowedPaths = + kind === "cloneUrl" + ? [`/${expectedFullName}.git`, `/${expectedFullName}`] + : [`/${expectedFullName}`]; + + if (!allowedPaths.includes(parsed.pathname)) { + throw new Error( + `Invalid GitHub ${kind}: pathname must match repository "${expectedFullName}".` + ); + } + + return parsed.toString(); +}; + +export const canonicalRepoName = (rawName: string): string => + validateRepoName(rawName).toLowerCase(); + +export interface RepoCollision { + canonical: string; + candidates: T[]; + conflictsWithExisting: boolean; + hasCollision: boolean; +} + +export const findRepoCollisions = ( + candidates: Iterable, + existing: Iterable = [] +): Map> => { + const existingSet = new Set( + Array.from(existing).flatMap((item) => { + try { + return [canonicalRepoName(item)]; + } catch { + return []; + } + }) + ); + + const groups = new Map(); + for (const candidate of candidates) { + const rawName = typeof candidate === "string" ? candidate : candidate.name; + const canonical = canonicalRepoName(rawName); + const bucket = groups.get(canonical); + if (bucket) { + bucket.push(candidate); + } else { + groups.set(canonical, [candidate]); + } + } + + return new Map( + Array.from(groups, ([canonical, items]) => { + const conflictsWithExisting = existingSet.has(canonical); + return [ + canonical, + { + canonical, + candidates: items, + conflictsWithExisting, + hasCollision: items.length > 1 || conflictsWithExisting + } + ]; + }) + ); +}; + +export interface ImportGitHubRepoInput { + ownerDid: Did; + ownerHandle: string; + knot: string; + spindle?: string; + repo: GitHubRepo; + name?: string; + description?: string; +} + +export interface ImportUrlInput { + sourceUrl: string; + name: string; + description?: string; + knot: string; +} + +const knotDidOf = (knot: string): Did => { + const did = knot.startsWith("did:") ? knot : serviceDidForHost(knot); + if (!isDid(did)) throw new Error(`${knot} is not a did`); + return did; +}; + +type Uri = `${string}:${string}`; + +const validateUrlSource = (raw: string): Uri => { + let parsed: URL; + try { + parsed = new URL(raw.trim()); + } catch { + throw new Error("That is not a URL."); + } + if (parsed.protocol !== "https:") throw new Error("The source URL must be https."); + if (parsed.username || parsed.password) + throw new Error("The source URL must not carry credentials."); + if (parsed.port !== "" && parsed.port !== "443") + throw new Error("The source URL must not use a custom port."); + if (parsed.pathname.replace(/\/+$/, "").split("/").filter(Boolean).length === 0) + throw new Error("The source URL must name a repository."); + return parsed.toString() as Uri; +}; + +export interface StartedMigration { + taskId: string; + name: string; + jobId?: string; +} + +export const importFromUrl = async ( + agent: OAuthUserAgent, + migratorUrl: string, + input: ImportUrlInput, + options?: { fetch?: typeof globalThis.fetch; signal?: AbortSignal } +): Promise => { + const sourceUrl = validateUrlSource(input.sourceUrl); + const name = validateRepoName(input.name); + const description = input.description?.trim() || undefined; + const task = await createMigrationTask( + agent, + migratorUrl, + crypto.randomUUID(), + [ + { + name, + knotDid: knotDidOf(input.knot), + sourceUrl, + ...(description !== undefined ? { description } : {}) + } + ], + options + ); + const first = task.jobs[0]; + return { taskId: task.id, name: first?.name ?? name, jobId: first?.id }; +}; + +export const importGitHubRepos = async ( + agent: OAuthUserAgent, + migratorUrl: string, + origin: string, + inputs: ImportGitHubRepoInput[], + options?: { fetch?: typeof globalThis.fetch; signal?: AbortSignal } +): Promise => { + if (inputs.length === 0) throw new Error("Select at least one repository to import."); + const jobs = inputs.map((input) => { + const cloneUrl = validateGitHubRepoUrl( + input.repo.cloneUrl, + input.repo.fullName, + "cloneUrl", + origin + ); + const name = validateRepoName(input.name ?? input.repo.name); + const description = input.description?.trim() || undefined; + return { + name, + knotDid: knotDidOf(input.knot), + sourceUrl: validateUrlSource(cloneUrl), + ...(input.repo.private === true ? { private: true } : {}), + ...(description !== undefined ? { description } : {}) + }; + }); + return createMigrationTask(agent, migratorUrl, crypto.randomUUID(), jobs, options); +}; + +export const importGitHubRepo = async ( + agent: OAuthUserAgent, + migratorUrl: string, + origin: string, + input: ImportGitHubRepoInput, + options?: { fetch?: typeof globalThis.fetch; signal?: AbortSignal } +): Promise => { + const task = await importGitHubRepos(agent, migratorUrl, origin, [input], options); + const first = task.jobs[0]; + return { + taskId: task.id, + name: first?.name ?? input.name ?? input.repo.name, + jobId: first?.id + }; +}; + +const normalizeSshKey = (rawKey: string): string | null => { + const parts = rawKey.trim().split(/\s+/); + if (parts.length < 2) return null; + const [algorithm, base64Blob] = parts; + if (!/^[a-zA-Z0-9@._-]+$/.test(algorithm)) return null; + if (!/^[A-Za-z0-9+/=]+$/.test(base64Blob)) return null; + return `${algorithm} ${base64Blob}`; +}; + +export interface ImportGitHubProfileEndpoints { + bobbinUrl?: string; + deliberiUrl: string; + fetch?: typeof globalThis.fetch; +} + +export interface ImportGitHubProfileOptions { + avatar: boolean; + keys: boolean; + email: boolean; +} + +export interface ImportedGitHubKey { + rkey: string; + name: string; + key: string; +} + +export interface ImportGitHubProfileResult { + imported: string[]; + errors: string[]; + keys: ImportedGitHubKey[]; +} + +export const importGitHubProfile = async ( + agent: OAuthUserAgent, + endpoints: ImportGitHubProfileEndpoints, + account: GitHubAccount, + options: ImportGitHubProfileOptions +): Promise => { + const fetchFn = endpoints.fetch ?? globalThis.fetch; + const imported: string[] = []; + const errors: string[] = []; + + if (options.avatar) { + try { + const res = await fetchFn("/_internal/github/avatar"); + if (!res.ok) { + throw new Error(`failed to fetch avatar (${res.status})`); + } + const rawBlob = await res.blob(); + const contentType = res.headers.get("content-type")?.split(";")[0]?.trim(); + const image = + contentType && !rawBlob.type ? new Blob([rawBlob], { type: contentType }) : rawBlob; + const avatarBlob = await uploadProfileAvatar(agent, image); + + const existing = await readRecord(agent, PROFILE_COLLECTION, "self"); + const existingProfile = existing?.value; + const existingCid = existing?.written?.cid; + + const record: ProfileRecord = { + $type: "sh.tangled.actor.profile", + bluesky: false, + ...(existingProfile ?? {}), + avatar: avatarBlob + }; + + if (existingProfile) { + await putRecord(agent, PROFILE_COLLECTION, "self", record, existingCid); + } else { + await createRecord(agent, PROFILE_COLLECTION, record, "self"); + } + imported.push("avatar"); + } catch (cause) { + errors.push(`Failed to import avatar: ${messageOf(cause)}`); + } + } + + const writtenKeys: ImportedGitHubKey[] = []; + + if (options.keys) { + try { + const rpc = createClient(agent); + const existingTokens = new Set(); + let cursor: string | undefined; + + do { + const page = await ok( + rpc.call(listRecordsSchema, { + params: { + repo: agent.sub, + collection: PUBLIC_KEY_COLLECTION, + limit: 100, + ...(cursor ? { cursor } : {}) + } + }) + ); + for (const rec of page.records) { + const val = rec.value as { key?: unknown }; + if (typeof val?.key === "string") { + const token = normalizeSshKey(val.key); + if (token) existingTokens.add(token); + } + } + cursor = page.cursor; + } while (cursor); + + const seenInBatch = new Set(); + for (const key of account.keys) { + const rawKey = key.key?.trim() ?? ""; + const token = normalizeSshKey(rawKey); + if (!token) { + errors.push(`Invalid SSH key format for key ${key.title || key.id}.`); + continue; + } + if (existingTokens.has(token) || seenInBatch.has(token)) { + continue; + } + seenInBatch.add(token); + + try { + const title = key.title?.trim() || `github-${key.id}`; + const written = await createPubKey(agent, title, rawKey); + writtenKeys.push({ rkey: written.rkey, name: title, key: rawKey }); + existingTokens.add(token); + imported.push(`key:${key.id}`); + } catch (cause) { + errors.push(`Failed to import key ${key.title || key.id}: ${messageOf(cause)}`); + } + } + } catch (cause) { + errors.push(`Failed to load existing keys: ${messageOf(cause)}`); + } + } + + if (options.email) { + if (!account.email || !account.email.trim()) { + errors.push("GitHub account does not have an email address."); + } else { + const targetEmail = account.email.trim(); + try { + const deliberi = createDeliberiClient({ + deliberiUrl: endpoints.deliberiUrl, + agent, + fetch: fetchFn + }); + const { emails } = await listEmails(deliberi); + const normalizedTarget = targetEmail.toLowerCase(); + const exists = emails.some((e) => e.address.toLowerCase() === normalizedTarget); + + if (!exists) { + await addEmail(deliberi, targetEmail); + imported.push(`email:${targetEmail} (verification required)`); + } + } catch (cause) { + errors.push(`Failed to import email ${targetEmail}: ${messageOf(cause)}`); + } + } + } + + return { imported, errors, keys: writtenKeys }; +}; diff --git a/web/src/lib/api/lexicons/types/sh/tangled/repo/create.ts b/web/src/lib/api/lexicons/types/sh/tangled/repo/create.ts index 5bbc3afce..08c4b551d 100644 --- a/web/src/lib/api/lexicons/types/sh/tangled/repo/create.ts +++ b/web/src/lib/api/lexicons/types/sh/tangled/repo/create.ts @@ -24,9 +24,11 @@ const _mainSchema = /*#__PURE__*/ v.procedure("sh.tangled.repo.create", { */ rkey: /*#__PURE__*/ v.recordKeyString(), /** - * A source URL to clone from, populate this when forking or importing a repository. + * A source to clone from, populate this when forking or importing a repository. */ - source: /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), + get source() { + return /*#__PURE__*/ v.optional(sourceSchema); + }, }), }, output: { @@ -40,12 +42,30 @@ const _mainSchema = /*#__PURE__*/ v.procedure("sh.tangled.repo.create", { }), }, }); +const _sourceSchema = /*#__PURE__*/ v.object({ + $type: /*#__PURE__*/ v.optional( + /*#__PURE__*/ v.literal("sh.tangled.repo.create#source"), + ), + /** + * fork tracks the source as upstream; import is a one-time copy. + */ + kind: /*#__PURE__*/ v.string<"fork" | "import" | (string & {})>(), + /** + * Git URL to clone from + */ + url: /*#__PURE__*/ v.genericUriString(), +}); type main$schematype = typeof _mainSchema; +type source$schematype = typeof _sourceSchema; export interface mainSchema extends main$schematype {} +export interface sourceSchema extends source$schematype {} export const mainSchema = _mainSchema as mainSchema; +export const sourceSchema = _sourceSchema as sourceSchema; + +export interface Source extends v.InferInput {} export interface $params {} export interface $input extends v.InferXRPCBodyInput {} diff --git a/web/src/lib/api/lexicons/types/sh/tangled/repo/describeRepo.ts b/web/src/lib/api/lexicons/types/sh/tangled/repo/describeRepo.ts index 2a5be2fb7..c28515ce3 100644 --- a/web/src/lib/api/lexicons/types/sh/tangled/repo/describeRepo.ts +++ b/web/src/lib/api/lexicons/types/sh/tangled/repo/describeRepo.ts @@ -12,10 +12,22 @@ const _mainSchema = /*#__PURE__*/ v.query("sh.tangled.repo.describeRepo", { output: { type: "lex", schema: /*#__PURE__*/ v.object({ + /** + * Status of the clone from source. Omitted if not created from a source. + */ + content: /*#__PURE__*/ v.optional( + /*#__PURE__*/ v.string< + "failed" | "fetching" | "partial" | "present" | (string & {}) + >(), + ), /** * DID of the current owner according to the knot. */ ownerDid: /*#__PURE__*/ v.didString(), + /** + * Reason for partial or failed content. + */ + reason: /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), repoDid: /*#__PURE__*/ v.didString(), /** * Current rkey of the sh.tangled.repo record tracked by this knot diff --git a/web/src/lib/api/migrator.test.ts b/web/src/lib/api/migrator.test.ts new file mode 100644 index 000000000..d1d74b370 --- /dev/null +++ b/web/src/lib/api/migrator.test.ts @@ -0,0 +1,392 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { type ClientResponseError } from "@atcute/client"; +import { missingPermissions } from "$lib/auth/scopes"; +import oauthMetadata from "$lib/oauth-client-metadata"; +import type { OAuthUserAgent } from "@atcute/oauth-browser-client"; +import type { Did } from "@atcute/lexicons/syntax"; +import { + createMigrationTask, + describeSource, + getMigrationTask, + listMigrationTasks, + migrationRunning, + describeSourcePermission, + missingDescribeSourceScope, + migrationsInFlight, + migrationsUnfinished, + retryMigrationJob, + flowReturnPath, + returnPathFrom, + startUrlFrom, + MigrationGrantRequired +} from "./migrator"; +import type { MigrationTask } from "./migrator"; + +const agent = { + sub: "did:plc:alice" as Did +} as unknown as OAuthUserAgent; + +vi.mock("$lib/auth/agent", () => ({ + mintServiceAuth: vi.fn(async (_agent, { aud, lxm }) => `mock-token-for-${aud}-${lxm}`), + serviceDidForHost: (host: string) => + `did:web:${host.replace(/^https?:\/\//, "").replace(/\/+$/, "")}` +})); + +describe("migrator client", () => { + const host = "https://migrator.test"; + + it("createMigrationTask posts to broker with minted auth and parses response", async () => { + const customFetch: typeof fetch = vi.fn( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = + typeof input === "string" + ? input + : input instanceof Request + ? input.url + : input.href; + expect(url).toBe("/_internal/github/migrator/createTask"); + expect(init?.method).toBe("POST"); + expect(new Headers(init?.headers).get("authorization")).toBe( + "Bearer mock-token-for-did:web:migrator.test-org.tangled.temp.migrator.createTask" + ); + return new Response( + JSON.stringify({ + id: "task-1", + ownerDid: "did:plc:alice", + createdAt: new Date().toISOString(), + jobs: [ + { + id: "job-1", + name: "repo", + knotDid: "did:web:knot.test", + sourceUrl: "https://github.com/a/b", + status: "queued", + attempts: 0, + private: false + } + ] + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + ); + + const task = await createMigrationTask( + agent, + host, + "req-1", + [ + { + name: "repo", + knotDid: "did:web:knot.test" as Did, + sourceUrl: "https://github.com/a/b" as `https://${string}`, + private: false + } + ], + { fetch: customFetch } + ); + + expect(task.id).toBe("task-1"); + expect(task.jobs).toHaveLength(1); + }); + + it("createMigrationTask maps 428 to MigrationGrantRequired with startUrl", async () => { + const customFetch: typeof fetch = vi.fn(async () => { + return new Response( + JSON.stringify({ + error: "GrantRequired", + message: "grant needed", + startUrl: "https://migrator.test/oauth/start?did=did%3Aplc%3Aalice" + }), + { status: 428, headers: { "content-type": "application/json" } } + ); + }); + + await expect( + createMigrationTask( + agent, + host, + "req-1", + [ + { + name: "repo", + knotDid: "did:web:knot.test" as Did, + sourceUrl: "https://github.com/a/b" as `https://${string}`, + private: false + } + ], + { fetch: customFetch } + ) + ).rejects.toThrow(MigrationGrantRequired); + }); + + it("getMigrationTask includes caller did in startUrl on 428 rather than dropping it", async () => { + vi.resetModules(); + vi.doMock("$lib/api/_request", () => ({ + serviceUrlFor: (h: string) => (h.startsWith("http") ? h : `https://${h}`), + serviceClient: () => ({}), + jsonGet: vi.fn(async () => { + const { ClientResponseError } = await import("@atcute/client"); + throw new ClientResponseError({ + status: 428, + headers: new Headers(), + data: { error: "GrantRequired", message: "grant expired" } + }); + }) + })); + + const { getMigrationTask: getTask } = await import("./migrator"); + let caught: unknown; + try { + await getTask(agent, host, "task-1"); + } catch (e) { + caught = e; + } + + const grant = caught as MigrationGrantRequired; + expect(grant?.name).toBe("MigrationGrantRequired"); + expect(grant.startUrl).toBe("https://migrator.test/oauth/start?did=did%3Aplc%3Aalice"); + }); + + it("describeSource queries the migrator directly and parses name and default branch", async () => { + vi.resetModules(); + const jsonGet = vi.fn(async (_ctx: unknown, _nsid: string, _params: unknown) => ({ + name: "repo", + defaultBranch: "main" + })); + vi.doMock("$lib/api/_request", () => ({ + serviceUrlFor: (h: string) => (h.startsWith("http") ? h : `https://${h}`), + serviceClient: vi.fn(), + jsonGet + })); + + const { describeSource: describe } = await import("./migrator"); + const out = await describe(agent, host, "https://example.com/a/repo.git"); + + expect(jsonGet).toHaveBeenCalledTimes(1); + const [, nsid, params] = jsonGet.mock.calls[0]; + expect(nsid).toBe("org.tangled.temp.migrator.describeSource"); + expect(params).toEqual({ sourceUrl: "https://example.com/a/repo.git" }); + expect(out).toEqual({ name: "repo", defaultBranch: "main" }); + }); + + it("describeSource maps a grant requirement to MigrationGrantRequired", async () => { + vi.resetModules(); + vi.doMock("$lib/api/_request", () => ({ + serviceUrlFor: (h: string) => (h.startsWith("http") ? h : `https://${h}`), + serviceClient: () => ({}), + jsonGet: vi.fn(async () => { + const { ClientResponseError } = await import("@atcute/client"); + throw new ClientResponseError({ + status: 428, + headers: new Headers(), + data: { error: "GrantRequired", message: "grant needed" } + }); + }) + })); + + const { describeSource: describe } = await import("./migrator"); + let caught: unknown; + try { + await describe(agent, host, "https://example.com/a/repo.git"); + } catch (e) { + caught = e; + } + + // re-imported module has its own class instance; match by name instead of instanceof + const grant = caught as MigrationGrantRequired; + expect(grant?.name).toBe("MigrationGrantRequired"); + expect(grant.startUrl).toBe("https://migrator.test/oauth/start?did=did%3Aplc%3Aalice"); + }); + + it("describeSource keeps the server's error tag so the UI can show the message", async () => { + vi.resetModules(); + vi.doMock("$lib/api/_request", () => ({ + serviceUrlFor: (h: string) => (h.startsWith("http") ? h : `https://${h}`), + serviceClient: () => ({}), + jsonGet: vi.fn(async () => { + const { ClientResponseError } = await import("@atcute/client"); + throw new ClientResponseError({ + status: 400, + headers: new Headers(), + data: { + error: "UnreachableSource", + message: "could not read the repository: ls-remote: exit status 128" + } + }); + }) + })); + + const { describeSource: describe } = await import("./migrator"); + let caught: unknown; + try { + await describe(agent, host, "https://example.com/a/repo.git"); + } catch (e) { + caught = e; + } + + const err = caught as ClientResponseError; + expect(err.error).toBe("UnreachableSource"); + expect(err.message).toContain("could not read the repository"); + }); + + it("treats unknown job statuses as running so polling keeps going", () => { + for (const terminal of ["completed", "failed", "authorization_required"]) + expect(migrationRunning(terminal)).toBe(false); + for (const running of ["queued", "cloning", "importing"]) + expect(migrationRunning(running)).toBe(true); + expect(migrationRunning("fetching_from_the_future")).toBe(true); + }); +}); + +describe("grant return path", () => { + afterEach(() => vi.unstubAllGlobals()); + + const job = { + name: "repo", + knotDid: "did:web:knot.test" as Did, + sourceUrl: "https://github.com/a/b" as `https://${string}`, + private: false + }; + + const task = (statuses: string[]) => + ({ + id: "task-1", + ownerDid: "did:plc:alice", + createdAt: "2026-09-17T00:00:00Z", + jobs: statuses.map((status, index) => ({ + id: `job-${index}`, + ...job, + knotDid: "did:web:knot.test", + status, + attempts: 0 + })) + }) as unknown as MigrationTask; + + it("drops the migrator's own transport params and keeps the rest", () => { + expect(returnPathFrom({ pathname: "/repo/import/github", search: "" })).toBe( + "/repo/import/github" + ); + expect( + returnPathFrom({ + pathname: "/repo/import/github", + search: "?return_to=%2Fwelcome&oauth_error=grant_failed&migrator=granted" + }) + ).toBe("/repo/import/github"); + expect( + returnPathFrom({ + pathname: "/repo/import/github", + search: "?oauth_error=grant_failed&page=2" + }) + ).toBe("/repo/import/github?page=2"); + }); + + it("flowReturnPath keeps the params that reopen the flow and drops the error", () => { + expect(flowReturnPath({ pathname: "/welcome", search: "?step=github" })).toBe( + "/welcome?step=github" + ); + expect( + flowReturnPath({ pathname: "/welcome", search: "?step=github&github_error=denied" }) + ).toBe("/welcome?step=github"); + expect( + flowReturnPath({ pathname: "/repo/import/github", search: "?github_error=denied" }) + ).toBe("/repo/import/github"); + expect(flowReturnPath({ pathname: "/welcome", search: "" })).toBe("/welcome"); + }); + + it("startUrlFrom constructs oauth start URL with did and return_to", () => { + expect(startUrlFrom("https://migrator.test", "did:plc:alice", "/repo/migrate")).toBe( + "https://migrator.test/oauth/start?did=did%3Aplc%3Aalice&return_to=%2Frepo%2Fmigrate" + ); + expect(startUrlFrom("https://migrator.test", "did:plc:alice", "")).toBe( + "https://migrator.test/oauth/start?did=did%3Aplc%3Aalice" + ); + expect(startUrlFrom("https://migrator.test", undefined)).toBeNull(); + }); + + it("startUrl sends the grant back to the page that asked for it", async () => { + vi.stubGlobal("window", { + location: { + pathname: "/repo/import/github", + search: "?oauth_error=grant_failed&page=2" + } + }); + const customFetch: typeof fetch = vi.fn( + async () => + new Response(JSON.stringify({ error: "GrantRequired", message: "grant needed" }), { + status: 428, + headers: { "content-type": "application/json" } + }) + ); + + let caught: unknown; + try { + await createMigrationTask(agent, "https://migrator.test", "req-1", [job], { + fetch: customFetch + }); + } catch (cause) { + caught = cause; + } + + expect(caught).toBeInstanceOf(MigrationGrantRequired); + expect((caught as MigrationGrantRequired).startUrl).toBe( + "https://migrator.test/oauth/start?did=did%3Aplc%3Aalice&return_to=%2Frepo%2Fimport%2Fgithub%3Fpage%3D2" + ); + }); + + it("keeps a migration unfinished while it still owes the user something", () => { + expect(migrationsUnfinished([])).toBe(false); + expect(migrationsUnfinished([task(["completed", "failed"])])).toBe(false); + expect(migrationsUnfinished([task(["authorization_required"])])).toBe(true); + expect(migrationsUnfinished([task(["completed"]), task(["queued"])])).toBe(true); + expect(migrationsUnfinished([task(["queued"]), task(["authorization_required"])])).toBe( + true + ); + expect(migrationsUnfinished([task(["completed"]), task(["authorization_required"])])).toBe( + false + ); + }); + + it("reports a migration in flight only while some job is still running", () => { + expect(migrationsInFlight([])).toBe(false); + expect(migrationsInFlight([task([])])).toBe(false); + expect(migrationsInFlight([task(["completed", "failed"])])).toBe(false); + expect(migrationsInFlight([task(["authorization_required"])])).toBe(false); + expect(migrationsInFlight([task(["completed"]), task(["queued"])])).toBe(true); + expect(migrationsInFlight([task(["importing"])])).toBe(true); + }); +}); + +describe("migrator scopes", () => { + const host = "https://migrator.test"; + + const sessionAgent = (scope: string) => + ({ + sub: "did:plc:alice" as Did, + session: { token: { scope } } + }) as unknown as OAuthUserAgent; + + it("the declared oauth scopes cover describeSource", () => { + const wanted = describeSourcePermission(host); + expect(wanted).not.toBeNull(); + expect(missingPermissions(oauthMetadata.scope, [wanted!])).toEqual([]); + }); + + it("describeSourcePermission names the migrator's method and audience", () => { + expect(describeSourcePermission(host)).toEqual({ + resource: "rpc", + lxm: "org.tangled.temp.migrator.describeSource", + aud: "did:web:migrator.test" + }); + }); + + it("missingDescribeSourceScope reports the grant an older session lacks", () => { + expect(missingDescribeSourceScope(sessionAgent("atproto"), host)).toHaveLength(1); + expect( + missingDescribeSourceScope( + sessionAgent("atproto rpc:org.tangled.temp.migrator.describeSource?aud=*"), + host + ) + ).toEqual([]); + }); +}); diff --git a/web/src/lib/api/migrator.ts b/web/src/lib/api/migrator.ts new file mode 100644 index 000000000..8bf082203 --- /dev/null +++ b/web/src/lib/api/migrator.ts @@ -0,0 +1,220 @@ +import { ClientResponseError } from "@atcute/client"; +import { isDid } from "@atcute/lexicons/syntax"; +import type { OAuthUserAgent } from "@atcute/oauth-browser-client"; +import { mintServiceAuth, serviceDidForHost } from "$lib/auth/agent"; +import { missingPermissions, type Permission } from "$lib/auth/scopes"; +import { + jsonGet, + jsonPost, + serviceClient, + serviceUrlFor, + toResponseError +} from "$lib/api/_request"; +import { mainSchema as createTaskSchema } from "$lib/api/lexicons/types/org/tangled/temp/migrator/createTask"; +import type * as Migrator from "$lib/api/lexicons/types/org/tangled/temp/migrator/defs"; +import { mainSchema as describeSourceSchema } from "$lib/api/lexicons/types/org/tangled/temp/migrator/describeSource"; +import type { $output as DescribeSourceOutput } from "$lib/api/lexicons/types/org/tangled/temp/migrator/describeSource"; +import { mainSchema as getTaskSchema } from "$lib/api/lexicons/types/org/tangled/temp/migrator/getTask"; +import { mainSchema as listTasksSchema } from "$lib/api/lexicons/types/org/tangled/temp/migrator/listTasks"; +import type { $output as ListTasksOutput } from "$lib/api/lexicons/types/org/tangled/temp/migrator/listTasks"; +import { mainSchema as retryJobSchema } from "$lib/api/lexicons/types/org/tangled/temp/migrator/retryJob"; + +const CREATE_TASK = createTaskSchema.nsid; +const GET_TASK = getTaskSchema.nsid; +const LIST_TASKS = listTasksSchema.nsid; +const RETRY_JOB = retryJobSchema.nsid; +const DESCRIBE_SOURCE = describeSourceSchema.nsid; + +export const describeSourcePermission = (host: string): Permission | null => { + const aud = serviceDidForHost(host); + return isDid(aud) ? { resource: "rpc", lxm: DESCRIBE_SOURCE, aud } : null; +}; + +export const missingDescribeSourceScope = ( + agent: OAuthUserAgent, + host: string +): readonly Permission[] => { + const wanted = describeSourcePermission(host); + return wanted === null ? [] : missingPermissions(agent.session.token.scope, [wanted]); +}; + +export type MigrationJob = Migrator.Job; +export type MigrationTask = Migrator.Task; +export type NewMigrationJob = Migrator.NewJob; +export type MigrationJobStatus = + "authorization_required" | "cloning" | "completed" | "failed" | "importing" | "queued"; + +export class MigrationGrantRequired extends Error { + readonly startUrl: string | null; + constructor(startUrl: string | null) { + super("The migrator needs your authorization before it can migrate anything."); + this.name = "MigrationGrantRequired"; + this.startUrl = startUrl; + } +} + +const messageOf = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + +// migrator echoes return_to and appends oauth_error; round trips must not nest them +const transportParams = ["return_to", "oauth_error", "migrator"]; + +const withQuery = (pathname: string, params: URLSearchParams): string => { + const query = params.toString(); + return query === "" ? pathname : `${pathname}?${query}`; +}; + +export const returnPathFrom = (location: { pathname: string; search: string }): string => { + const params = new URLSearchParams(location.search); + for (const key of transportParams) params.delete(key); + return withQuery(location.pathname, params); +}; + +export const flowReturnPath = (location: { pathname: string; search: string }): string => { + const params = new URLSearchParams(location.search); + params.delete("github_error"); + return withQuery(location.pathname, params); +}; + +const currentReturnPath = (): string => + typeof window === "undefined" + ? "" + : returnPathFrom({ pathname: window.location.pathname, search: window.location.search }); + +export const startUrlFrom = ( + host: string, + did: string | undefined, + returnTo = currentReturnPath() +): string | null => + did === undefined + ? null + : `${serviceUrlFor(host)}/oauth/start?did=${encodeURIComponent(did)}` + + (returnTo === "" ? "" : `&return_to=${encodeURIComponent(returnTo)}`); + +const extractGrantError = ( + cause: unknown, + host: string, + did: string | undefined +): MigrationGrantRequired | null => + cause instanceof ClientResponseError && + (cause.status === 428 || cause.error === "GrantRequired") + ? new MigrationGrantRequired(startUrlFrom(host, did)) + : null; + +export interface CreateMigrationTaskOptions { + signal?: AbortSignal; + fetch?: typeof globalThis.fetch; +} + +export const createMigrationTask = async ( + agent: OAuthUserAgent, + host: string, + requestId: string, + jobs: NewMigrationJob[], + options?: CreateMigrationTaskOptions | AbortSignal +): Promise => { + if (jobs.length === 0) throw new Error("a migration needs at least one repository"); + const signal = options instanceof AbortSignal ? options : options?.signal; + const customFetch = + (options && !(options instanceof AbortSignal) ? options.fetch : undefined) ?? + globalThis.fetch; + + const token = await mintServiceAuth(agent, { + aud: serviceDidForHost(host), + lxm: CREATE_TASK, + signal + }); + + const response = await customFetch("/_internal/github/migrator/createTask", { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + accept: "application/json" + }, + body: JSON.stringify({ requestId, jobs }), + signal + }); + + if (!response.ok) { + const err = await toResponseError(response); + throw extractGrantError(err, host, agent.sub) ?? err; + } + + const text = await response.text(); + return (text.length > 0 ? JSON.parse(text) : undefined) as MigrationTask; +}; + +export const getMigrationTask = async ( + agent: OAuthUserAgent, + host: string, + taskId: string, + signal?: AbortSignal +): Promise => { + const ctx = serviceClient(agent, host); + try { + return await jsonGet(ctx, GET_TASK, { taskId }, { signal }); + } catch (cause) { + throw extractGrantError(cause, host, agent.sub) ?? cause; + } +}; + +export const listMigrationTasks = async ( + agent: OAuthUserAgent, + host: string, + params?: { limit?: number }, + signal?: AbortSignal +): Promise => { + const ctx = serviceClient(agent, host); + try { + const out = await jsonGet(ctx, LIST_TASKS, params, { signal }); + return out.tasks; + } catch (cause) { + throw extractGrantError(cause, host, agent.sub) ?? cause; + } +}; + +export const describeSource = async ( + agent: OAuthUserAgent, + host: string, + sourceUrl: string, + signal?: AbortSignal +): Promise => { + const ctx = serviceClient(agent, host); + try { + return await jsonGet(ctx, DESCRIBE_SOURCE, { sourceUrl }, { signal }); + } catch (cause) { + throw extractGrantError(cause, host, agent.sub) ?? cause; + } +}; + +export const retryMigrationJob = async ( + agent: OAuthUserAgent, + host: string, + input: { taskId: string; jobId: string; githubToken?: string }, + signal?: AbortSignal +): Promise => { + const ctx = serviceClient(agent, host); + try { + return await jsonPost(ctx, RETRY_JOB, input, { signal }); + } catch (cause) { + throw extractGrantError(cause, host, agent.sub) ?? cause; + } +}; + +// mirrors migrator isterminal; unrecognized status keeps poll alive +export const migrationRunning = (status: string): boolean => + !["completed", "failed", "authorization_required"].includes(status); + +export const migrationsInFlight = (tasks: MigrationTask[]): boolean => + tasks.some((task) => task.jobs.some((job) => migrationRunning(job.status))); + +// only newest batch can wait on a grant; older batches are history +export const migrationsUnfinished = (tasks: MigrationTask[]): boolean => + migrationsInFlight(tasks) || + (tasks[0]?.jobs.some((job) => job.status === "authorization_required") ?? false); + +export const migrationFailed = (job: MigrationJob): string | null => + job.status === "failed" ? job.error?.trim() || "the migration failed" : null; + +export const migrationError = messageOf; diff --git a/web/src/lib/api/repo.test.ts b/web/src/lib/api/repo.test.ts index b61fcc741..e94a1400a 100644 --- a/web/src/lib/api/repo.test.ts +++ b/web/src/lib/api/repo.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { coAuthorsFrom, + importState, logFor, repoNameOf, resolveForkRepoLabels, @@ -186,6 +187,44 @@ describe("logFor", () => { }); }); +describe("importState", () => { + const repo = { uri: "at://did:plc:o/sh.tangled.repo/abc", repoDid: "did:plc:repo" }; + const describeResponse = (body: Record) => + jsonResponse({ ownerDid: "did:plc:o", rkey: "abc", ...body }); + + it("sends the at-uri for bobbin's proxy and repoDid for the knot", async () => { + const fetchMock = vi.fn().mockResolvedValue(describeResponse({})); + await importState(makeCtx(fetchMock), repo); + const url = new URL(String(fetchMock.mock.calls[0][0])); + expect(url.pathname).toBe("/xrpc/sh.tangled.repo.describeRepo"); + expect(url.searchParams.get("repo")).toBe(repo.uri); + expect(url.searchParams.get("repoDid")).toBe(repo.repoDid); + }); + + it("keeps in-flight states and the failure reason", async () => { + const state = (content: string, reason?: string) => { + const fetchMock = vi + .fn() + .mockResolvedValue(describeResponse({ content, reason: reason ?? "" })); + return importState(makeCtx(fetchMock), repo); + }; + expect(await state("fetching")).toEqual({ content: "fetching" }); + expect(await state("partial")).toEqual({ content: "partial" }); + expect(await state("failed", "clone died")).toEqual({ + content: "failed", + reason: "clone died" + }); + }); + + it("reports nothing for present content or a record with no repoDid", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(describeResponse({ content: "present" })); + expect(await importState(makeCtx(fetchMock), repo)).toBeNull(); + expect(await importState(makeCtx(fetchMock), { uri: repo.uri })).toBeNull(); + }); +}); + describe("toBranchSummary", () => { it("reads the nested reference and the go-git commit fields", () => { const branch: BranchEntry = { diff --git a/web/src/lib/api/repo.ts b/web/src/lib/api/repo.ts index 0f3c6136b..30f69d566 100644 --- a/web/src/lib/api/repo.ts +++ b/web/src/lib/api/repo.ts @@ -6,10 +6,12 @@ import { authorOf, enrich, target } from "$lib/api/enrich"; import { getRepoByName, type RecordList, type RecordView, type RepoRecord } from "$lib/api/records"; import { branches as knotBranches, + describeRepo as knotDescribeRepo, log as knotLog, tag as knotTag, tags as knotTags } from "$lib/api/knot"; +import type * as DescribeRepo from "$lib/api/lexicons/types/sh/tangled/repo/describeRepo"; import { httpStatusFor } from "$lib/api/load"; import { didFromUri, rkeyFromUri } from "$lib/api/uri"; import type * as Tree from "$lib/api/lexicons/types/sh/tangled/repo/tree"; @@ -476,3 +478,36 @@ export const tagsFor = ( export const tagFor = (ctx: BobbinContext, repo: string, tag: string, init?: XrpcRequestInit) => knotTag(ctx, { repo, tag }, init); + +type ImportContent = "fetching" | "partial" | "failed"; + +export interface ImportState { + content: ImportContent; + reason?: string; + source?: string; +} + +// bobbin proxy resolves knot from at-uri, knot keys on repoDid; both required +const describeRepo = ( + ctx: BobbinContext, + repo: { uri: string; repoDid?: string }, + init?: XrpcRequestInit +): Promise => + knotDescribeRepo( + ctx, + { repo: repo.uri, repoDid: repo.repoDid ?? "" } as DescribeRepo.$params, + init + ); + +export const importState = async ( + ctx: BobbinContext, + repo: { uri: string; repoDid?: string }, + init?: XrpcRequestInit +): Promise => { + if (!repo.repoDid) return null; + const out = await describeRepo(ctx, repo, init); + const { content, reason } = out; + if (!content || content === "present") return null; + if (content === "failed") return { content: "failed", reason }; + return { content: content === "fetching" ? "fetching" : "partial" }; +}; diff --git a/web/src/lib/api/repoCreate.test.ts b/web/src/lib/api/repoCreate.test.ts index c7bbfde84..d63e1e57a 100644 --- a/web/src/lib/api/repoCreate.test.ts +++ b/web/src/lib/api/repoCreate.test.ts @@ -9,10 +9,12 @@ const agent = { interface HarnessOptions { createError?: Error; putError?: Error; + readError?: Error; + existing?: { value: Record; cid?: string | null } | null; } // mocks have to be installed before the module is loaded -const load = async ({ createError, putError }: HarnessOptions = {}) => { +const load = async ({ createError, putError, readError, existing = null }: HarnessOptions = {}) => { vi.resetModules(); const events: string[] = []; const records: unknown[] = []; @@ -23,6 +25,7 @@ const load = async ({ createError, putError }: HarnessOptions = {}) => { records.push(record); return { uri: "at://did:plc:alice/sh.tangled.repo/demo", cid: "initial-cid" }; }); + const runCalls: Record[] = []; const putRecord = vi.fn(async (_agent, _collection, _rkey, record, swapRecord) => { events.push("record:put"); if (putError) throw putError; @@ -37,7 +40,23 @@ const load = async ({ createError, putError }: HarnessOptions = {}) => { events.push("record:delete"); }); - vi.doMock("./write", () => ({ createRecord, putRecord, deleteRecord })); + const readRecord = vi.fn(async () => { + if (readError) throw readError; + return existing === null + ? null + : { + value: existing.value, + written: + existing.cid === null + ? null + : { + uri: "at://did:plc:alice/sh.tangled.repo/demo", + cid: existing.cid ?? "existing-cid" + } + }; + }); + + vi.doMock("./write", () => ({ createRecord, putRecord, deleteRecord, readRecord })); vi.doMock("./client", () => ({ createBobbinClient: ({ serviceUrl, @@ -53,7 +72,10 @@ const load = async ({ createError, putError }: HarnessOptions = {}) => { return { ok: true, data: { status: "indexed" } }; } if (caller) knotAgents.push(caller); - if (options.input) knotInputs.push(options.input); + if (options.input) { + knotInputs.push(options.input); + runCalls.push(options.input); + } if (options.input && "repo" in options.input) { events.push("knot:delete"); return { ok: true, data: null }; @@ -70,11 +92,13 @@ const load = async ({ createError, putError }: HarnessOptions = {}) => { api: await import("./repoCreate"), events, records, + runCalls, knotInputs, knotAgents, createRecord, putRecord, - deleteRecord + deleteRecord, + readRecord }; }; @@ -165,20 +189,77 @@ describe("repository creation", () => { expect(events).not.toContain("knot:delete"); }); + it("refuses a name the account already has instead of failing inside the pds", async () => { + const { api, events, readRecord } = await load({ + existing: { + value: { $type: "sh.tangled.repo", knot: "knot.test", repoDid: "did:plc:theirs" } + } + }); + + await expect(api.createRepo(agent, "https://bobbin.test", input)).rejects.toThrow( + 'A repository named "Demo" already exists in your account. Choose another name.' + ); + + expect(readRecord).toHaveBeenCalledWith(agent, "sh.tangled.repo", "demo"); + expect(events).toEqual([]); + }); + + it("resumes a creation that never reached the knot instead of stranding the name", async () => { + const { api, events, runCalls, createRecord, putRecord } = await load({ + existing: { value: { $type: "sh.tangled.repo", knot: "knot.test" } } + }); + + const creation = await api.createRepo(agent, "https://bobbin.test", input); + + expect(creation.repoDid).toBe("did:plc:repo"); + expect(createRecord).not.toHaveBeenCalled(); + expect(events).toEqual([ + "record:put", + "bobbin:await", + "knot:create", + "record:put", + "bobbin:await" + ]); + expect(putRecord.mock.calls.map((call) => call[4])).toEqual([ + "existing-cid", + "committed-cid" + ]); + expect(runCalls.at(-1)).toMatchObject({ rkey: "demo", defaultBranch: "main" }); + }); + + it("refuses an interrupted record left on a different knot", async () => { + const { api, events } = await load({ + existing: { value: { $type: "sh.tangled.repo", knot: "other.knot.test" } } + }); + + await expect(api.createRepo(agent, "https://bobbin.test", input)).rejects.toThrow( + 'A repository named "Demo" already exists in your account. Choose another name.' + ); + expect(events).toEqual([]); + }); + + it("fails immediately when readRecord throws rather than proceeding with blind create", async () => { + const { api } = await load({ readError: new Error("PDS connection dropped") }); + await expect(api.createRepo(agent, "https://bobbin.test", input)).rejects.toThrow( + "PDS connection dropped" + ); + }); + it("passes fork identity and clone source to both halves", async () => { const { api, records, knotInputs } = await load(); await api.createRepo(agent, "https://bobbin.test", { ...input, name: "fork", source: { - record: "did:plc:upstream", - cloneUrl: "https://knot.source/did:plc:upstream" + url: "https://knot.source/did:plc:upstream", + kind: "fork", + record: "did:plc:upstream" } }); expect(records[0]).toMatchObject({ source: "did:plc:upstream" }); expect(knotInputs[0]).toMatchObject({ - source: "https://knot.source/did:plc:upstream" + source: { url: "https://knot.source/did:plc:upstream", kind: "fork" } }); }); }); diff --git a/web/src/lib/api/repoCreate.ts b/web/src/lib/api/repoCreate.ts index 4a14a6e8d..83d39a205 100644 --- a/web/src/lib/api/repoCreate.ts +++ b/web/src/lib/api/repoCreate.ts @@ -9,11 +9,20 @@ import { DEFAULT_LABELS } from "$lib/api/labels"; import { mainSchema as createRepoSchema } from "$lib/api/lexicons/types/sh/tangled/repo/create"; import { mainSchema as deleteRepoSchema } from "$lib/api/lexicons/types/sh/tangled/repo/delete"; import type { RepoRecord } from "$lib/api/records"; -import { createRecord, deleteRecord, putRecord, type WrittenRecord } from "$lib/api/write"; +import { + createRecord, + deleteRecord, + putRecord, + readRecord, + type WrittenRecord +} from "$lib/api/write"; import { pingIndexNow } from "$lib/api/indexnow"; const REPO_COLLECTION = "sh.tangled.repo" as Nsid; +export type RepoCreationSource = + { kind: "fork"; url: string; record: string } | { kind: "import"; url: string }; + export interface RepoCreationInput { ownerDid: Did; ownerHandle: string; @@ -22,10 +31,7 @@ export interface RepoCreationInput { defaultBranch?: string; knot: string; spindle?: string; - source?: { - record: string; - cloneUrl: string; - }; + source?: RepoCreationSource; } export interface CreatedRepo { @@ -52,7 +58,7 @@ export const validateRepoName = (rawName: string): string => { if (rawName.toLowerCase() === "self") throw new Error(`Repository name ${JSON.stringify(rawName)} is reserved.`); - return rawName.endsWith(".git") ? rawName.slice(0, -4) : rawName; + return /\.git$/i.test(rawName) ? rawName.slice(0, -4) : rawName; }; const messageOf = (cause: unknown): string => @@ -130,7 +136,9 @@ const completeCreation = async ( rkey: recordKeyOf(rkey), name: rkey, ...(input.defaultBranch ? { defaultBranch: input.defaultBranch } : {}), - ...(input.source ? { source: input.source.cloneUrl } : {}) + ...(input.source + ? { source: { url: input.source.url as never, kind: input.source.kind } } + : {}) } }) ); @@ -185,11 +193,24 @@ export const createRepo = async ( ...(name !== rkey ? { name } : {}), ...(description ? { description } : {}), ...(spindle ? { spindle } : {}), - ...(input.source ? { source: input.source.record as never } : {}) + ...(input.source?.kind === "fork" ? { source: input.source.record as never } : {}) }; + // records without repoDid are interrupted optimistic creates on this knot, resume to avoid stranding rkey + const existing = await readRecord(agent, REPO_COLLECTION, rkey); + const resuming = + existing !== null && !existing.value.repoDid && existing.value.knot === input.knot + ? existing + : null; + if (existing && !resuming) + throw new Error( + `A repository named "${name}" already exists in your account. Choose another name.` + ); + const bobbin = createBobbinClient({ serviceUrl: bobbinUrl }); - const initial = await createRecord(agent, REPO_COLLECTION, record, rkey); + const initial = resuming + ? await putRecord(agent, REPO_COLLECTION, rkey, record, resuming.written?.cid) + : await createRecord(agent, REPO_COLLECTION, record, rkey); try { await awaitIndexedRecord(bobbin, initial); } catch (cause) { diff --git a/web/src/lib/api/repoIndex.ts b/web/src/lib/api/repoIndex.ts index 011e201ce..ba20841ab 100644 --- a/web/src/lib/api/repoIndex.ts +++ b/web/src/lib/api/repoIndex.ts @@ -22,6 +22,7 @@ import { toTreeCommitSummary, toTreeEntrySummary } from "$lib/api/repo"; +import { importState, type ImportState } from "$lib/api/repo"; import { renderDocument } from "$lib/markup"; import type { CommitSummary, LanguageSlice, RepoInfo } from "$lib/components/repo/types"; // `/tree/{ref}` is this same page at another ref, so they share a load @@ -123,6 +124,7 @@ const readmeOf = (tree: { readme?: { filename: string; contents: string } } | nu export interface RepoIndexOptions { requireRef?: boolean; + importStatus?: boolean; } // for the handle and the row reads like the rest of the site. a plain git @@ -170,7 +172,7 @@ export const loadRepoIndex = ( ref?: string, // a ref from the url has to resolve or a typo looks like a repo with no // files. the default branch renders whatever the knot managed to answer - { requireRef = false }: RepoIndexOptions = {} + { requireRef = false, importStatus: withImportStatus = false }: RepoIndexOptions = {} ) => { const repoP = Promise.resolve(parent.repo); @@ -234,11 +236,24 @@ export const loadRepoIndex = ( renderReadme(resolved.readme, repo, parent.publicConfig, event, resolved.ref) ); + const importStatusP = withImportStatus + ? repoP.then(async (repo): Promise => { + if (!repo.importSource) return null; + const ctx = createBobbinClient({ + serviceUrl: parent.publicConfig.bobbinUrl, + fetch: event.fetch + }); + const state = await orNull(importState(ctx, repo)); + return state ? { ...state, source: repo.importSource } : null; + }) + : Promise.resolve(null); + return { ref: ref ?? "", content: stream(content), languages: stream(languages), - readmeHtml: stream(readmeHtml) + readmeHtml: stream(readmeHtml), + importStatus: stream(importStatusP) }; }; diff --git a/web/src/lib/api/write.test.ts b/web/src/lib/api/write.test.ts index 8098dfb38..9c146dbf4 100644 --- a/web/src/lib/api/write.test.ts +++ b/web/src/lib/api/write.test.ts @@ -34,20 +34,21 @@ describe("record writes", () => { expect(read).toHaveBeenCalledTimes(2); }); - it("forwards the swap guard so a concurrent edit loses", async () => { + it.each([ + [ + "putRecord", + (w: any) => w.putRecord(agent, collection, "3lk", record, "bafyread"), + "bafyread" + ], + [ + "deleteRecord", + (w: any) => w.deleteRecord(agent, collection, "3lk", "bafycreated"), + "bafycreated" + ] + ])("forwards the swap guard on %s", async (_name, op, swapRecord) => { const { write, inputs } = await load(); - - await write.putRecord(agent, collection, "3lk", record, "bafyread"); - - expect(inputs[0]).toMatchObject({ rkey: "3lk", swapRecord: "bafyread" }); - }); - - it("forwards the swap guard on deletes used for rollback", async () => { - const { write, inputs } = await load(); - - await write.deleteRecord(agent, collection, "3lk", "bafycreated"); - - expect(inputs[0]).toMatchObject({ rkey: "3lk", swapRecord: "bafycreated" }); + await op(write); + expect(inputs[0]).toMatchObject({ rkey: "3lk", swapRecord }); }); it("takes a minted rkey instead of letting the pds assign one", async () => { @@ -80,3 +81,60 @@ describe("record writes", () => { expect(read).toHaveBeenCalledTimes(1); }); }); + +describe("readRecord", () => { + it("returns record value and written metadata on 200", async () => { + vi.resetModules(); + vi.doMock("$app/environment", () => ({ browser: true })); + vi.doMock("$lib/auth/agent", () => ({ + createClient: () => ({ + call: async () => ({ + ok: true, + data: { + uri: "at://did:plc:writer/c/1", + cid: "bafy123", + value: { title: "found" } + } + }) + }) + })); + const write = await import("./write"); + const result = await write.readRecord<{ title: string }>(agent, collection, "3lk"); + expect(result).toEqual({ + value: { title: "found" }, + written: { uri: "at://did:plc:writer/c/1", cid: "bafy123" } + }); + }); + + it.each(["RecordNotFound", "CouldNotFindRecord"])("returns null on %s", async (error) => { + vi.resetModules(); + vi.doMock("$app/environment", () => ({ browser: true })); + vi.doMock("$lib/auth/agent", () => ({ + createClient: () => ({ + call: async () => ({ + ok: false, + data: { error, message: "not found" } + }) + }) + })); + const write = await import("./write"); + const result = await write.readRecord(agent, collection, "3lk"); + expect(result).toBeNull(); + }); + + it("rethrows non-404 errors instead of swallowing into null", async () => { + vi.resetModules(); + vi.doMock("$app/environment", () => ({ browser: true })); + vi.doMock("$lib/auth/agent", () => ({ + createClient: () => ({ + call: async () => ({ + ok: false, + status: 500, + data: { error: "InternalServerError", message: "pds down" } + }) + }) + })); + const write = await import("./write"); + await expect(write.readRecord(agent, collection, "3lk")).rejects.toThrow(); + }); +}); diff --git a/web/src/lib/api/write.ts b/web/src/lib/api/write.ts index b5921c083..ad7b1c2e5 100644 --- a/web/src/lib/api/write.ts +++ b/web/src/lib/api/write.ts @@ -1,5 +1,6 @@ import { ok } from "@atcute/client"; import { mainSchema as createRecordSchema } from "@atcute/atproto/types/repo/createRecord"; +import { mainSchema as getRecordSchema } from "@atcute/atproto/types/repo/getRecord"; import { mainSchema as deleteRecordSchema } from "@atcute/atproto/types/repo/deleteRecord"; import { mainSchema as putRecordSchema } from "@atcute/atproto/types/repo/putRecord"; import type { Cid, Nsid, ResourceUri } from "@atcute/lexicons/syntax"; @@ -13,6 +14,31 @@ export interface WrittenRecord { cid: Cid; } +interface ReadRecord { + value: T; + /** getRecord returns cid only when present, null can't be used as swapRecord */ + written: WrittenRecord | null; +} + +export const readRecord = async ( + agent: OAuthUserAgent, + collection: Nsid, + rkey: string +): Promise | null> => { + const response = await createClient(agent).call(getRecordSchema, { + params: { repo: agent.sub, collection, rkey: recordKeyOf(rkey) } + }); + if (response.ok) + return { + value: response.data.value as T, + written: response.data.cid ? { uri: response.data.uri, cid: response.data.cid } : null + }; + const error = (response.data as { error?: string } | undefined)?.error; + if (error === "RecordNotFound" || error === "CouldNotFindRecord") return null; + await ok(response); + return null; +}; + // every record write in the app goes through here, so the read cache is dropped // on any mutation instead of each write site remembering which keys it dirties export const createRecord = async ( diff --git a/web/src/lib/auth/agent.test.ts b/web/src/lib/auth/agent.test.ts new file mode 100644 index 000000000..9f8afb51c --- /dev/null +++ b/web/src/lib/auth/agent.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, test } from "vitest"; +import { hostForServiceDid, serviceDidForHost } from "$lib/auth/agent"; + +describe("serviceDidForHost", () => { + test.each([ + ["migrator.tngl.boltless.dev", "did:web:migrator.tngl.boltless.dev"], + ["https://migrator.tngl.boltless.dev/", "did:web:migrator.tngl.boltless.dev"], + ["http://knot.oyster.cafe:8443", "did:web:knot.oyster.cafe%3A8443"] + ])("maps %s to %s", (input, expected) => { + expect(serviceDidForHost(input)).toBe(expected); + }); + + test("round-trips with hostForServiceDid", () => { + const host = "migrator.tngl.boltless.dev"; + expect(hostForServiceDid(serviceDidForHost(host))).toBe(host); + }); +}); diff --git a/web/src/lib/auth/agent.ts b/web/src/lib/auth/agent.ts index 3e4f95b50..47d96e9bd 100644 --- a/web/src/lib/auth/agent.ts +++ b/web/src/lib/auth/agent.ts @@ -20,8 +20,13 @@ export interface ServiceAuthOptions { signal?: AbortSignal; } +// pds refuses did:web:https%3A//host as an invalid did +const hostOf = (hostOrUrl: string): string => + /^[a-z][a-z0-9+.-]*:\/\//i.test(hostOrUrl) ? new URL(hostOrUrl).host : hostOrUrl.replace(/\/+$/, ""); + // did:web service id, with ports percent-encoded like serviceauth didweb. -export const serviceDidForHost = (host: string): string => `did:web:${host.replace(/:/g, "%3A")}`; +export const serviceDidForHost = (hostOrUrl: string): string => + `did:web:${hostOf(hostOrUrl).replace(/:/g, "%3A")}`; export const hostForServiceDid = (did: string): string | null => { if (!isDid(did) || !did.startsWith("did:web:")) return null; diff --git a/web/src/lib/components/repo/ImportStatusBanner.svelte b/web/src/lib/components/repo/ImportStatusBanner.svelte new file mode 100644 index 000000000..e91023274 --- /dev/null +++ b/web/src/lib/components/repo/ImportStatusBanner.svelte @@ -0,0 +1,70 @@ + + +{#if view.content === "failed"} + +{:else} +
+ +

+ This repository is still importing from {view.source ?? "its source"}. It will fill in + here as the import progresses. +

+
+{/if} diff --git a/web/src/lib/components/repo/RepoIndexView.stories.svelte b/web/src/lib/components/repo/RepoIndexView.stories.svelte index bca44fe71..a42ecdcbc 100644 --- a/web/src/lib/components/repo/RepoIndexView.stories.svelte +++ b/web/src/lib/components/repo/RepoIndexView.stories.svelte @@ -89,7 +89,8 @@ { name: "TypeScript", percentage: 70, share: 70 }, { name: "Svelte", percentage: 30, share: 30 } ]), - readmeHtml: streamed.readmeHtml ?? Promise.resolve("

tangled

") + readmeHtml: streamed.readmeHtml ?? Promise.resolve("

tangled

"), + importStatus: Promise.resolve(null) }); const data = indexData(); diff --git a/web/src/lib/components/repo/SourceSelector.svelte b/web/src/lib/components/repo/SourceSelector.svelte new file mode 100644 index 000000000..e9499dcb2 --- /dev/null +++ b/web/src/lib/components/repo/SourceSelector.svelte @@ -0,0 +1,201 @@ + + + + +
+
+ {#each tabs as tab (tab.mode)} + + {/each} +
+ + {#if source === "patch"} + + {#snippet children({ id })} +