From 9fb2aea4e261f3b8a3e571629a84f97a291ca4be Mon Sep 17 00:00:00 2001 From: dawn Date: Thu, 30 Jul 2026 16:15:12 +0300 Subject: [PATCH] web/api: use knotmirror for repo git reads when set Signed-off-by: dawn --- web/src/lib/api/gitclient.test.ts | 193 ++++++++++++++++++ web/src/lib/api/gitclient.ts | 118 +++++++++++ web/src/lib/api/knotmirror.test.ts | 87 +++++++- web/src/lib/api/knotmirror.ts | 103 +++++++++- web/src/lib/api/pagination.test.ts | 17 +- web/src/lib/api/pagination.ts | 11 +- web/src/lib/api/repo.test.ts | 15 ++ web/src/lib/api/repo.ts | 29 ++- web/src/lib/api/repoIndex.test.ts | 6 +- web/src/lib/api/repoIndex.ts | 98 +++------ .../repo/RepoIndexView.stories.svelte | 10 +- .../lib/components/repo/RepoIndexView.svelte | 6 +- web/src/lib/server/repo.ts | 45 ++++ web/src/routes/[handle]/[repo]/+layout.ts | 11 +- .../[repo]/raw/[ref]/[...path]/+server.ts | 29 +-- 15 files changed, 647 insertions(+), 131 deletions(-) create mode 100644 web/src/lib/api/gitclient.test.ts create mode 100644 web/src/lib/api/gitclient.ts create mode 100644 web/src/lib/server/repo.ts diff --git a/web/src/lib/api/gitclient.test.ts b/web/src/lib/api/gitclient.test.ts new file mode 100644 index 000000000..855bd041b --- /dev/null +++ b/web/src/lib/api/gitclient.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it, vi } from "vitest"; +import { + blobRawUrl, + branches, + gitTarget, + log, + resolveDefaultBranch, + tag, + tags, + tree +} from "./gitclient"; + +const config = { bobbinUrl: "https://bobbin.example", knotMirrorUrl: "https://km.example" }; +const repo = { + uri: "at://did:plc:owner/sh.tangled.repo/core", + repoDid: "did:plc:reporepo" +}; + +const jsonResponse = (body: unknown): Response => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" } + }); + +describe("gitTarget", () => { + it("routes git ops to the knot mirror keyed by repo DID when set", () => { + const target = gitTarget(config, repo, globalThis.fetch); + expect(target.ctx.serviceUrl).toBe("https://km.example"); + expect(target.repo).toBe("did:plc:reporepo"); + expect(target.via).toBe("mirror"); + }); + + it("uses bobbin keyed by at-uri without a mirror", () => { + const target = gitTarget({ ...config, knotMirrorUrl: "" }, repo, globalThis.fetch); + expect(target.ctx.serviceUrl).toBe("https://bobbin.example"); + expect(target.repo).toBe(repo.uri); + expect(target.via).toBe("bobbin"); + }); + + it("falls back to bobbin when the repo DID is unknown", () => { + const target = gitTarget(config, { uri: repo.uri }, globalThis.fetch); + expect(target.ctx.serviceUrl).toBe("https://bobbin.example"); + expect(target.repo).toBe(repo.uri); + expect(target.via).toBe("bobbin"); + }); +}); + +describe("resolveDefaultBranch", () => { + it("reads the is_default flag out of the mirror's branch list", async () => { + const fetchMock = vi.fn().mockResolvedValue( + jsonResponse({ + branches: [ + { reference: { name: "sv-fe", hash: "aaa" } }, + { reference: { name: "master", hash: "bbb" }, is_default: true } + ] + }) + ); + const target = gitTarget(config, repo, fetchMock); + + await expect(resolveDefaultBranch(target)).resolves.toBe("master"); + + const url = new URL(String(fetchMock.mock.calls[0][0])); + expect(url.pathname).toBe("/xrpc/sh.tangled.git.temp.listBranches"); + expect(url.searchParams.get("repo")).toBe(repo.repoDid); + }); + + it("asks the knot endpoint directly on the bobbin path", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(jsonResponse({ name: "main", hash: "ccc", when: "" })); + const target = gitTarget({ ...config, knotMirrorUrl: "" }, repo, fetchMock); + + await expect(resolveDefaultBranch(target)).resolves.toBe("main"); + + const url = new URL(String(fetchMock.mock.calls[0][0])); + expect(url.pathname).toBe("/xrpc/sh.tangled.repo.getDefaultBranch"); + }); + + it("returns null instead of throwing when the backend is down", async () => { + const fetchMock = vi + .fn() + .mockRejectedValue(new TypeError("fetch failed")); + const target = gitTarget(config, repo, fetchMock); + + await expect(resolveDefaultBranch(target)).resolves.toBeNull(); + }); +}); + +describe("endpoint routing", () => { + const mirrorTarget = (fetchMock: typeof globalThis.fetch) => gitTarget(config, repo, fetchMock); + const bobbinTarget = (fetchMock: typeof globalThis.fetch) => + gitTarget({ ...config, knotMirrorUrl: "" }, repo, fetchMock); + const calledUrl = (fetchMock: ReturnType>, call = 0) => + new URL(String(fetchMock.mock.calls[call][0])); + + it("sends log to the mirror's listCommits keyed by DID, cursor intact", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(jsonResponse({ commits: [] })); + + await log(mirrorTarget(fetchMock), { ref: "main", limit: 10, cursor: "abc" }); + + const url = calledUrl(fetchMock); + expect(url.origin).toBe("https://km.example"); + expect(url.pathname).toBe("/xrpc/sh.tangled.git.temp.listCommits"); + expect(url.searchParams.get("repo")).toBe(repo.repoDid); + expect(url.searchParams.get("cursor")).toBe("abc"); + }); + + it("sends log to bobbin's repo.log keyed by at-uri, cursor intact", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(jsonResponse({ commits: [] })); + + await log(bobbinTarget(fetchMock), { ref: "main", limit: 10, cursor: "abc" }); + + const url = calledUrl(fetchMock); + expect(url.origin).toBe("https://bobbin.example"); + expect(url.pathname).toBe("/xrpc/sh.tangled.repo.log"); + expect(url.searchParams.get("repo")).toBe(repo.uri); + expect(url.searchParams.get("cursor")).toBe("abc"); + }); + + it("sends tree to the mirror's getTree with the path param", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(jsonResponse({ files: [] })); + + await tree(mirrorTarget(fetchMock), { ref: "main", path: "src/lib" }); + + const url = calledUrl(fetchMock); + expect(url.pathname).toBe("/xrpc/sh.tangled.git.temp.getTree"); + expect(url.searchParams.get("repo")).toBe(repo.repoDid); + expect(url.searchParams.get("path")).toBe("src/lib"); + }); + + it("sends tree to bobbin's repo.tree", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(jsonResponse({ files: [] })); + + await tree(bobbinTarget(fetchMock), { ref: "main" }); + + const url = calledUrl(fetchMock); + expect(url.pathname).toBe("/xrpc/sh.tangled.repo.tree"); + expect(url.searchParams.get("repo")).toBe(repo.uri); + }); + + it("sends branches and tags to the legacy list endpoints on the mirror", async () => { + const fetchMock = vi + .fn() + .mockImplementation(() => Promise.resolve(jsonResponse({ branches: [], tags: [] }))); + const target = mirrorTarget(fetchMock); + + await branches(target, 5); + await tags(target, 5); + + expect(calledUrl(fetchMock).pathname).toBe("/xrpc/sh.tangled.git.temp.listBranches"); + expect(calledUrl(fetchMock, 1).pathname).toBe("/xrpc/sh.tangled.git.temp.listTags"); + }); + + it("routes single-tag reads to getTag on the mirror and repo.tag on bobbin", async () => { + const fetchMock = vi + .fn() + .mockImplementation(() => Promise.resolve(jsonResponse({ tag: undefined }))); + + await tag(mirrorTarget(fetchMock), "v1.0.0"); + expect(calledUrl(fetchMock).pathname).toBe("/xrpc/sh.tangled.git.temp.getTag"); + + await tag(bobbinTarget(fetchMock), "v1.0.0"); + expect(calledUrl(fetchMock, 1).pathname).toBe("/xrpc/sh.tangled.repo.tag"); + }); + + it("builds the raw url against the mirror without a raw flag", () => { + const url = new URL( + blobRawUrl(mirrorTarget(globalThis.fetch), { ref: "main", path: "README.md" }) + ); + expect(url.origin).toBe("https://km.example"); + expect(url.pathname).toBe("/xrpc/sh.tangled.git.temp.getBlob"); + expect(url.searchParams.get("repo")).toBe(repo.repoDid); + expect(url.searchParams.has("raw")).toBe(false); + }); + + it("builds the raw url against bobbin with raw=true", () => { + const url = new URL( + blobRawUrl(bobbinTarget(globalThis.fetch), { ref: "main", path: "README.md" }) + ); + expect(url.origin).toBe("https://bobbin.example"); + expect(url.pathname).toBe("/xrpc/sh.tangled.repo.blob"); + expect(url.searchParams.get("repo")).toBe(repo.uri); + expect(url.searchParams.get("raw")).toBe("true"); + }); +}); diff --git a/web/src/lib/api/gitclient.ts b/web/src/lib/api/gitclient.ts new file mode 100644 index 000000000..e7ba9c3dd --- /dev/null +++ b/web/src/lib/api/gitclient.ts @@ -0,0 +1,118 @@ +import { createBobbinClient, type BobbinContext } from "./client"; +import { + branches as knotMirrorBranches, + blob as knotMirrorBlob, + blobRawUrl as knotMirrorBlobRawUrl, + createKnotMirrorClient, + languages as knotMirrorLanguages, + log as knotMirrorLog, + getTag as knotMirrorGetTag, + tags as knotMirrorTags, + tree as knotMirrorTree +} from "./knotmirror"; +import { + blob as knotBlob, + blobRawUrl as knotBlobRawUrl, + getDefaultBranch, + languages as knotLanguages, + tree as knotTree +} from "./knot"; +import { branchesFor, logFor, tagFor, tagsFor } from "./repo"; +import type { Did, ResourceUri } from "@atcute/lexicons"; + +export interface GitServiceConfig { + bobbinUrl: string; + knotMirrorUrl: string; +} + +export interface GitTarget { + ctx: BobbinContext; + repo: Did | ResourceUri; + via: "mirror" | "bobbin"; +} + +export const gitTarget = ( + config: GitServiceConfig, + repo: { uri: string; repoDid?: string }, + fetch: typeof globalThis.fetch +): GitTarget => + config.knotMirrorUrl && repo.repoDid + ? { + ctx: createKnotMirrorClient(config.knotMirrorUrl, fetch), + repo: repo.repoDid as Did, + via: "mirror" + } + : { + ctx: createBobbinClient({ serviceUrl: config.bobbinUrl, fetch }), + repo: repo.uri as ResourceUri, + via: "bobbin" + }; + +// the mirror has no getDefaultBranch, the default falls out of the branch +// list's is_default flag instead +export const resolveDefaultBranch = ( + target: GitTarget, + init?: { signal?: AbortSignal } +): Promise => + target.via === "mirror" + ? (async () => { + for (let offset = 0; ; offset += 100) { + const response = await knotMirrorBranches( + target.ctx, + { repo: target.repo, limit: 100, cursor: offset ? `${offset}` : undefined }, + init + ); + const list = response.branches ?? []; + const found = list.find((branch) => branch.is_default)?.reference.name; + if (found) return found; + if (list.length < 100) return null; + } + })().catch(() => null) + : getDefaultBranch(target.ctx, { repo: target.repo }) + .then((branch) => branch.name) + .catch(() => null); + +export const tree = (target: GitTarget, params: { ref: string; path?: string }) => + target.via === "mirror" + ? knotMirrorTree(target.ctx, { repo: target.repo, ...params }) + : knotTree(target.ctx, { repo: target.repo, ...params }); + +export const log = (target: GitTarget, params: { ref: string; limit: number; cursor?: string }) => + target.via === "mirror" + ? knotMirrorLog(target.ctx, { repo: target.repo, ...params }) + : logFor(target.ctx, target.repo, params.ref, params.limit, params.cursor); + +export const branches = (target: GitTarget, limit: number, cursor?: string) => + target.via === "mirror" + ? knotMirrorBranches(target.ctx, { repo: target.repo, limit, cursor }) + : branchesFor(target.ctx, target.repo, limit, cursor); + +export const tags = (target: GitTarget, limit: number, cursor?: string) => + target.via === "mirror" + ? knotMirrorTags(target.ctx, { repo: target.repo, limit, cursor }) + : tagsFor(target.ctx, target.repo, limit, cursor); + +export const tag = (target: GitTarget, name: string) => + target.via === "mirror" + ? knotMirrorGetTag(target.ctx, { repo: target.repo, tag: name }) + : tagFor(target.ctx, target.repo, name); + +export const languages = ( + target: GitTarget, + ref: string +): Promise<{ languages?: { name: string; size: number }[] }> => + target.via === "mirror" + ? knotMirrorLanguages(target.ctx, { repo: target.repo, ref }) + : knotLanguages(target.ctx, { repo: target.repo, ref }); + +// only the mirror needs the inline cap, the knot enforces its own and 413s +// past it +export const blob = (target: GitTarget, params: { ref: string; path: string }, maxBytes: number) => + target.via === "mirror" + ? knotMirrorBlob(target.ctx, { repo: target.repo, ...params }, maxBytes) + : knotBlob(target.ctx, { repo: target.repo as ResourceUri, ...params }); + +export const blobRawUrl = (target: GitTarget, params: { ref: string; path: string }): string => + target.via === "mirror" + ? knotMirrorBlobRawUrl(target.ctx.serviceUrl, { repo: target.repo, ...params }) + : knotBlobRawUrl(target.ctx.serviceUrl, { repo: target.repo, ...params }); diff --git a/web/src/lib/api/knotmirror.test.ts b/web/src/lib/api/knotmirror.test.ts index bf8d94b39..4c2ca5c2a 100644 --- a/web/src/lib/api/knotmirror.test.ts +++ b/web/src/lib/api/knotmirror.test.ts @@ -8,7 +8,7 @@ const jsonResponse = (body: unknown): Response => }); describe("knotmirror.tree", () => { - it("adapts the temporary tree response and fetches the readme blob", async () => { + it("fetches the readme blob", async () => { const fetchMock = vi.fn().mockImplementation(async (input) => { const url = new URL(String(input)); if (url.pathname.endsWith("getTree")) { @@ -51,7 +51,7 @@ describe("knotmirror.tree", () => { }); describe("knotmirror.languages", () => { - it("uses the temporary languages endpoint", async () => { + it("uses the languages endpoint", async () => { const fetchMock = vi .fn() .mockResolvedValue( @@ -65,3 +65,86 @@ describe("knotmirror.languages", () => { expect(url.pathname).toBe("/xrpc/sh.tangled.git.temp.listLanguages"); }); }); + +describe("knotmirror.blob", () => { + const entryBody = { + name: "hello.txt", + mode: "0100644", + size: 5, + lastCommit: { + hash: "abc", + message: "hi", + author: { name: "dawn", email: "dawn@tangled.org", when: "2026-07-01T00:00:00Z" } + } + }; + + const blobFetch = (content: Uint8Array, entry: unknown = entryBody) => + vi.fn().mockImplementation(async (input) => { + const url = new URL(String(input)); + return url.pathname.endsWith("getEntry") + ? jsonResponse(entry) + : new Response(content.slice().buffer, { status: 200 }); + }); + + it("creates the repo blob shape from getEntry plus raw content", async () => { + const fetchMock = blobFetch(new TextEncoder().encode("hello")); + const ctx = knotmirror.createKnotMirrorClient("https://mirror.test", fetchMock); + + const output = await knotmirror.blob( + ctx, + { repo: "did:plc:repo", ref: "main", path: "hello.txt" }, + 1024 + ); + + expect(output).toMatchObject({ + path: "hello.txt", + size: 5, + isBinary: false, + encoding: "utf-8", + content: "hello", + lastCommit: { hash: "abc", when: "2026-07-01T00:00:00Z" } + }); + }); + + it("skips the content fetch when the entry is over the inline cap", async () => { + const fetchMock = blobFetch(new TextEncoder().encode("hello"), { ...entryBody, size: 2048 }); + const ctx = knotmirror.createKnotMirrorClient("https://mirror.test", fetchMock); + + const output = await knotmirror.blob( + ctx, + { repo: "did:plc:repo", ref: "main", path: "hello.txt" }, + 1024 + ); + + expect(output.fileTooLarge).toBe(true); + expect(output.content).toBeUndefined(); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("marks content with null bytes as binary", async () => { + const fetchMock = blobFetch(new Uint8Array([0x50, 0x4b, 0x00, 0x00])); + const ctx = knotmirror.createKnotMirrorClient("https://mirror.test", fetchMock); + + const output = await knotmirror.blob( + ctx, + { repo: "did:plc:repo", ref: "main", path: "hello.txt" }, + 1024 + ); + + expect(output.isBinary).toBe(true); + expect(output.content).toBeUndefined(); + }); + + it("treats a gitlink mode as a submodule", async () => { + const fetchMock = blobFetch(new Uint8Array(), { ...entryBody, mode: "0160000" }); + const ctx = knotmirror.createKnotMirrorClient("https://mirror.test", fetchMock); + + const output = await knotmirror.blob( + ctx, + { repo: "did:plc:repo", ref: "main", path: "hello.txt" }, + 1024 + ); + + expect(output.submodule).toEqual({ name: "hello.txt", url: "" }); + }); +}); diff --git a/web/src/lib/api/knotmirror.ts b/web/src/lib/api/knotmirror.ts index c15fbd346..d59472ccd 100644 --- a/web/src/lib/api/knotmirror.ts +++ b/web/src/lib/api/knotmirror.ts @@ -1,16 +1,22 @@ import type { BobbinContext, XrpcRequestInit } from "./client"; import { createBobbinClient } from "./client"; -import { jsonGet, rawGet } from "./_request"; +import { buildUrl, jsonGet, rawGet } from "./_request"; +import type { RepoTagResponse } from "./repo"; +import type * as RepoBlob from "./lexicons/types/sh/tangled/repo/blob"; import type * as LegacyTree from "./lexicons/types/sh/tangled/git/temp/getTree"; import type * as Tree from "./lexicons/types/sh/tangled/repo/tree"; import { treeEntryKind, type BranchesResponse, type LogResponse, type TagsResponse } from "./repo"; const TREE_NSID = "sh.tangled.git.temp.getTree"; +const ENTRY_NSID = "sh.tangled.git.temp.getEntry"; const BLOB_NSID = "sh.tangled.git.temp.getBlob"; const LOG_NSID = "sh.tangled.git.temp.listCommits"; const BRANCHES_NSID = "sh.tangled.git.temp.listBranches"; const TAGS_NSID = "sh.tangled.git.temp.listTags"; const LANGUAGES_NSID = "sh.tangled.git.temp.listLanguages"; +const GET_TAG_NSID = "sh.tangled.git.temp.getTag"; + +const MAX_README_BYTES = 1 << 20; export interface KnotMirrorLanguagesResponse { languages?: { name: string; size: number }[]; @@ -21,6 +27,88 @@ export interface KnotMirrorLanguagesResponse { export const createKnotMirrorClient = (serviceUrl: string, fetch: typeof globalThis.fetch) => createBobbinClient({ serviceUrl, fetch }); +export interface KnotMirrorEntry { + name: string; + mode: string; + oid?: string; + size?: number; + lastCommit?: { + hash: string; + message?: string; + author?: { name: string; email: string; when: string }; + committer?: { name: string; email: string; when: string }; + }; +} + +export const entry = ( + ctx: BobbinContext, + params: { repo: string; ref: string; path: string }, + init?: XrpcRequestInit +) => jsonGet(ctx, ENTRY_NSID, params, init); + +export const blobRawUrl = ( + serviceUrl: string, + params: { repo: string; ref: string; path: string } +): string => buildUrl(serviceUrl, BLOB_NSID, params).toString(); + +// same heuristic as git: a null byte in the first chunk means binary +const looksBinary = (bytes: Uint8Array): boolean => bytes.subarray(0, 8000).includes(0); + +// temp.getBlob answers raw bytes only, metadata comes from getEntry and +// the binary flag from a content sniff +export const blob = async ( + ctx: BobbinContext, + params: { repo: string; ref: string; path: string }, + maxBytes: number, + init?: XrpcRequestInit +): Promise => { + const meta = await entry(ctx, params, init); + + const lastCommit = meta.lastCommit + ? { + hash: meta.lastCommit.hash, + message: meta.lastCommit.message ?? "", + when: meta.lastCommit.committer?.when ?? meta.lastCommit.author?.when ?? "", + author: meta.lastCommit.author + } + : undefined; + + // a gitlink is a submodule pointer, not a file + if (treeEntryKind(meta.mode) === "submodule") { + return { + path: params.path, + ref: params.ref, + size: meta.size, + submodule: { name: meta.name, url: "" } + }; + } + + if (meta.size !== undefined && meta.size > maxBytes) { + return { path: params.path, ref: params.ref, size: meta.size, fileTooLarge: true, lastCommit }; + } + + const bytes = new Uint8Array(await (await rawGet(ctx, BLOB_NSID, params, init)).arrayBuffer()); + if (looksBinary(bytes)) { + return { + path: params.path, + ref: params.ref, + size: meta.size ?? bytes.length, + isBinary: true, + lastCommit + }; + } + + return { + path: params.path, + ref: params.ref, + size: meta.size ?? bytes.length, + isBinary: false, + encoding: "utf-8", + content: new TextDecoder().decode(bytes), + lastCommit + }; +}; + const normalizeSignature = ( signature: LegacyTree.Signature | undefined ): Tree.Signature | undefined => @@ -64,6 +152,7 @@ export const tree = async ( const result = normalizeTree(response); const readme = response.files.find((entry) => isReadmeFile(entry.name, entry.mode)); if (!readme) return result; + if (readme.size !== undefined && readme.size > MAX_README_BYTES) return result; try { const blob = await rawGet( @@ -80,22 +169,28 @@ export const tree = async ( export const log = ( ctx: BobbinContext, - params: { repo: string; ref: string; limit: number }, + params: { repo: string; ref: string; limit: number; cursor?: string }, init?: XrpcRequestInit ) => jsonGet(ctx, LOG_NSID, params, init); export const branches = ( ctx: BobbinContext, - params: { repo: string; limit: number }, + params: { repo: string; limit: number; cursor?: string }, init?: XrpcRequestInit ) => jsonGet(ctx, BRANCHES_NSID, params, init); export const tags = ( ctx: BobbinContext, - params: { repo: string; limit: number }, + params: { repo: string; limit: number; cursor?: string }, init?: XrpcRequestInit ) => jsonGet(ctx, TAGS_NSID, params, init); +export const getTag = ( + ctx: BobbinContext, + params: { repo: string; tag: string }, + init?: XrpcRequestInit +) => jsonGet(ctx, GET_TAG_NSID, params, init); + export const languages = ( ctx: BobbinContext, params: { repo: string; ref: string }, diff --git a/web/src/lib/api/pagination.test.ts b/web/src/lib/api/pagination.test.ts index 923d6186f..292c5025a 100644 --- a/web/src/lib/api/pagination.test.ts +++ b/web/src/lib/api/pagination.test.ts @@ -94,13 +94,24 @@ describe("paginateBy", () => { }); it("maxPages caps loader invocations", async () => { + let page = 0; + const load = vi.fn(async () => { + page += 1; + return { items: [0], cursor: `p${page}` }; + }); + const out: number[] = []; + for await (const n of paginateBy(load, { maxPages: 3 })) out.push(n); + expect(load).toHaveBeenCalledTimes(3); + expect(out).toHaveLength(3); + }); + + it("stops when the cursor stops advancing", async () => { const load = vi.fn(async () => ({ items: [0], cursor: "always" })); const out: number[] = []; - for await (const n of paginateBy(load, { maxPages: 3 })) out.push(n); - expect(load).toHaveBeenCalledTimes(3); - expect(out).toHaveLength(3); + for await (const n of paginateBy(load)) out.push(n); + expect(load).toHaveBeenCalledTimes(2); }); }); diff --git a/web/src/lib/api/pagination.ts b/web/src/lib/api/pagination.ts index d873d0b20..9204974ee 100644 --- a/web/src/lib/api/pagination.ts +++ b/web/src/lib/api/pagination.ts @@ -30,7 +30,6 @@ export type PageItem = export type PageParams = Omit, "cursor">; export interface PaginateOptions extends XrpcRequestInit { - /** stop after this many network round-trips. */ maxPages?: number; } @@ -70,7 +69,9 @@ export async function* pages( options ); yield data; - cursor = (data as CursorPage).cursor; + const next = (data as CursorPage).cursor; + if (next !== undefined && next === cursor) break; + cursor = next; if (options.maxPages && ++seen >= options.maxPages) break; } while (cursor); } @@ -87,7 +88,6 @@ export async function* items( } export interface CollectOptions extends PaginateOptions { - /** stop once this many items are gathered. */ max?: number; } @@ -105,7 +105,6 @@ export const collect = async ( return out; }; -// cursor walker for schema-less list endpoints. export async function* paginateBy( load: (cursor: string | undefined) => Promise<{ items: readonly T[]; cursor?: string | null }>, options: { maxPages?: number } = {} @@ -115,7 +114,9 @@ export async function* paginateBy( do { const page = await load(cursor); for (const item of page.items) yield item; - cursor = page.cursor ?? undefined; + const next = page.cursor ?? undefined; + if (next !== undefined && next === cursor) break; + cursor = next; if (options.maxPages && ++seen >= options.maxPages) break; } while (cursor); } diff --git a/web/src/lib/api/repo.test.ts b/web/src/lib/api/repo.test.ts index 7786b9104..381645d14 100644 --- a/web/src/lib/api/repo.test.ts +++ b/web/src/lib/api/repo.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { + logFor, repoNameOf, resolveRepoByName, sortTreeEntries, @@ -98,6 +99,20 @@ describe("toCommitSummary", () => { }); }); +describe("logFor", () => { + it("passes the cursor through to the knot", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(jsonResponse({ commits: [], total: 42 })); + await logFor(makeCtx(fetchMock), "at://did:plc:o/sh.tangled.repo/abc", "master", 20, "40"); + const url = new URL(String(fetchMock.mock.calls[0][0])); + expect(url.pathname).toBe("/xrpc/sh.tangled.repo.log"); + expect(url.searchParams.get("ref")).toBe("master"); + expect(url.searchParams.get("limit")).toBe("20"); + expect(url.searchParams.get("cursor")).toBe("40"); + }); +}); + describe("toBranchSummary", () => { it("reads the nested reference and the go-git commit fields", () => { const branch: BranchEntry = { diff --git a/web/src/lib/api/repo.ts b/web/src/lib/api/repo.ts index 83048848b..821340b7d 100644 --- a/web/src/lib/api/repo.ts +++ b/web/src/lib/api/repo.ts @@ -1,6 +1,6 @@ import { ClientResponseError, type BobbinContext, type XrpcRequestInit } from "./client"; import { getRepoByName, type RecordView, type RepoRecord } from "./records"; -import { branches as knotBranches, log as knotLog, tags as knotTags } from "./knot"; +import { branches as knotBranches, log as knotLog, tag as knotTag, tags as knotTags } from "./knot"; import { httpStatusFor } from "./load"; import { rkeyFromUri } from "./uri"; import type * as Tree from "./lexicons/types/sh/tangled/repo/tree"; @@ -62,7 +62,7 @@ export interface TagEntry { tag?: { Tagger?: GitSignature; Message?: string; - /** the commit an annotated tag points at, bytes like every go-git hash */ + // the commit an annotated tag points at, bytes like every go-git hash Target?: number[]; }; } @@ -72,6 +72,10 @@ export interface TagsResponse { total?: number; } +export interface RepoTagResponse { + tag?: TagEntry; +} + // newer repos get tid rkeys and keep their display name in the record export const repoNameOf = (view: RecordView): string => view.value.name ?? rkeyFromUri(view.uri); @@ -158,7 +162,7 @@ export const toBranchSummary = (branch: BranchEntry): BranchSummary => ({ export interface TagSummary { name: string; hash: string; - /** an annotated tag has its own hash, this is the commit it points at */ + // an annotated tag has its own hash, this is the commit it points at commitHash: string; when?: string; message?: string; @@ -231,20 +235,31 @@ export const sortTreeEntries = (entries: TreeEntrySummary[]): TreeEntrySummary[] return a.name.localeCompare(b.name); }); +// the log cursor is a numeric offset encoded as a string export const logFor = ( ctx: BobbinContext, repo: string, ref: string, limit: number, + cursor?: string, init?: XrpcRequestInit -) => knotLog(ctx, { repo, ref, limit }, init); +) => knotLog(ctx, { repo, ref, limit, cursor }, init); export const branchesFor = ( ctx: BobbinContext, repo: string, limit: number, + cursor?: string, + init?: XrpcRequestInit +) => knotBranches(ctx, { repo, limit, cursor }, init); + +export const tagsFor = ( + ctx: BobbinContext, + repo: string, + limit: number, + cursor?: string, init?: XrpcRequestInit -) => knotBranches(ctx, { repo, limit }, init); +) => knotTags(ctx, { repo, limit, cursor }, init); -export const tagsFor = (ctx: BobbinContext, repo: string, limit: number, init?: XrpcRequestInit) => - knotTags(ctx, { repo, limit }, init); +export const tagFor = (ctx: BobbinContext, repo: string, tag: string, init?: XrpcRequestInit) => + knotTag(ctx, { repo, tag }, init); diff --git a/web/src/lib/api/repoIndex.test.ts b/web/src/lib/api/repoIndex.test.ts index 8da78ef6c..33078d352 100644 --- a/web/src/lib/api/repoIndex.test.ts +++ b/web/src/lib/api/repoIndex.test.ts @@ -11,7 +11,7 @@ describe("classifyRepoAvailability", () => { [unsupported(), unsupported(), unsupported()].map((cause) => ({ value: null, error: cause })) ); - expect(result).toEqual({ needsUpgrade: true, knotUnreachable: false }); + expect(result).toBe("needs-upgrade"); }); it("keeps transport failures as unreachable", () => { @@ -19,7 +19,7 @@ describe("classifyRepoAvailability", () => { ["tree", "log", "branches"].map(() => ({ value: null, error: new TypeError("offline") })) ); - expect(result).toEqual({ needsUpgrade: false, knotUnreachable: true }); + expect(result).toBe("unreachable"); }); it("does not mistake a missing ref for an unavailable knot", () => { @@ -29,6 +29,6 @@ describe("classifyRepoAvailability", () => { { value: { branches: [] }, error: null } ]); - expect(result).toEqual({ needsUpgrade: false, knotUnreachable: false }); + expect(result).toBe("ok"); }); }); diff --git a/web/src/lib/api/repoIndex.ts b/web/src/lib/api/repoIndex.ts index 49ff046ed..772690240 100644 --- a/web/src/lib/api/repoIndex.ts +++ b/web/src/lib/api/repoIndex.ts @@ -1,21 +1,17 @@ import { error } from "@sveltejs/kit"; -import { ClientResponseError, createBobbinClient } from "$lib/api/client"; -import { languages as knotLanguages, tree as knotTree } from "$lib/api/knot"; +import { ClientResponseError } from "$lib/api/client"; import { - branches as knotMirrorBranches, - createKnotMirrorClient, - languages as knotMirrorLanguages, - log as knotMirrorLog, - tags as knotMirrorTags, - tree as knotMirrorTree -} from "$lib/api/knotmirror"; + branches as gitBranches, + gitTarget, + languages as gitLanguages, + log as gitLog, + tags as gitTags, + tree as gitTree +} from "$lib/api/gitclient"; import { parallel } from "$lib/api/load"; import { - branchesFor, - logFor, sortTreeEntries, tagsByCommitHash, - tagsFor, toBranchSummary, toCommitSummary, toTagSummary, @@ -24,15 +20,13 @@ import { } from "$lib/api/repo"; import { renderDocument } from "$lib/markup"; import type { LanguageSlice, RepoInfo } from "$lib/components/repo/types"; -import type * as Tree from "./lexicons/types/sh/tangled/repo/tree"; - // `/tree/{ref}` is this same page at another ref, so they share a load const COMMIT_LIMIT = 10; const BRANCH_LIMIT = 5; const TAG_LIMIT = 5; // a knot only ever lists 100 refs, so any total we get is really a minimum -const REF_LIMIT = 100; +export const REF_LIMIT = 100; export interface RepoParent { publicConfig: { bobbinUrl: string; knotMirrorUrl: string; camoEnabled: boolean }; @@ -44,14 +38,6 @@ export interface RepoLoadEvent { url: URL; } -interface RepoDataSource { - tree: (ref: string, path?: string) => Promise; - log: (ref: string, limit: number) => Promise>>; - branches: (limit: number) => Promise>>; - tags: (limit: number) => Promise>>; - languages: (ref: string) => Promise<{ languages?: { name: string; size: number }[] }>; -} - const orNull = (promise: Promise): Promise => promise.catch(() => null); interface Attempt { @@ -68,16 +54,15 @@ const attempt = (promise: Promise): Promise> => const isUnsupported = (cause: unknown): boolean => cause instanceof ClientResponseError && cause.status === 404; +export type RepoAvailability = "ok" | "empty" | "needs-upgrade" | "unreachable"; + export const classifyRepoAvailability = ( attempts: readonly { value: unknown | null; error: unknown | null }[] -) => { - const needsUpgrade = attempts.every( - (result) => result.value === null && isUnsupported(result.error) - ); - return { - needsUpgrade, - knotUnreachable: !needsUpgrade && attempts.every((result) => result.value === null) - }; +): Exclude => { + if (attempts.every((result) => result.value === null && isUnsupported(result.error))) { + return "needs-upgrade"; + } + return attempts.every((result) => result.value === null) ? "unreachable" : "ok"; }; const toLanguageSlices = (languages: { name: string; size: number }[]): LanguageSlice[] => { @@ -107,7 +92,7 @@ const refNames = (branches: { name: string }[], tags: { name: string }[]) => ({ capped: branches.length >= REF_LIMIT || tags.length >= REF_LIMIT }); -const renderReadme = ( +export const renderReadme = ( readme: { filename: string; contents: string } | null, parent: RepoParent, event: RepoLoadEvent, @@ -130,30 +115,6 @@ const readmeOf = (tree: { readme?: { filename: string; contents: string } } | nu return readme?.filename ? readme : null; }; -const repoDataSource = (event: RepoLoadEvent, parent: RepoParent): RepoDataSource => { - if (parent.publicConfig.knotMirrorUrl && parent.repo.repoDid) { - const ctx = createKnotMirrorClient(parent.publicConfig.knotMirrorUrl, event.fetch); - const repo = parent.repo.repoDid; - return { - tree: (ref, path) => knotMirrorTree(ctx, { repo, ref, path }), - log: (ref, limit) => knotMirrorLog(ctx, { repo, ref, limit }), - branches: (limit) => knotMirrorBranches(ctx, { repo, limit }), - tags: (limit) => knotMirrorTags(ctx, { repo, limit }), - languages: (ref) => knotMirrorLanguages(ctx, { repo, ref }) - }; - } - - const ctx = createBobbinClient({ serviceUrl: parent.publicConfig.bobbinUrl, fetch: event.fetch }); - const repo = parent.repo.uri; - return { - tree: (ref, path) => knotTree(ctx, { repo, ref, path }), - log: (ref, limit) => logFor(ctx, repo, ref, limit), - branches: (limit) => branchesFor(ctx, repo, limit), - tags: (limit) => tagsFor(ctx, repo, limit), - languages: (ref) => knotLanguages(ctx, { repo, ref }) - }; -}; - export interface RepoIndexOptions { requireRef?: boolean; } @@ -166,15 +127,15 @@ export const loadRepoIndex = async ( // files. the default branch renders whatever the knot managed to answer { requireRef = false }: RepoIndexOptions = {} ) => { - const source = repoDataSource(event, parent); + const git = gitTarget(parent.publicConfig, parent.repo, event.fetch); // each list falls back on its own, so half a page still renders const results = await parallel({ - tree: attempt(source.tree(ref)), - log: attempt(source.log(ref, COMMIT_LIMIT)), - branches: attempt(source.branches(REF_LIMIT)), - tags: attempt(source.tags(REF_LIMIT)), - languages: attempt(source.languages(ref)) + tree: attempt(gitTree(git, { ref })), + log: attempt(gitLog(git, { ref, limit: COMMIT_LIMIT })), + branches: attempt(gitBranches(git, REF_LIMIT)), + tags: attempt(gitTags(git, REF_LIMIT)), + languages: attempt(gitLanguages(git, ref)) }); const branches = (results.branches.value?.branches ?? []).map(toBranchSummary); @@ -188,8 +149,9 @@ export const loadRepoIndex = async ( const readmeHtml = await renderReadme(readme, parent, event, ref); const contentAttempts = [results.tree, results.log, results.branches]; - const { needsUpgrade, knotUnreachable } = classifyRepoAvailability(contentAttempts); - const isEmpty = !knotUnreachable && files.length === 0 && branches.length === 0; + const knot = classifyRepoAvailability(contentAttempts); + const availability: RepoAvailability = + knot === "ok" && files.length === 0 && branches.length === 0 ? "empty" : knot; // there are refs but not this one, so it is not a real ref. an empty repo has // no refs at all and still gets a page @@ -199,9 +161,7 @@ export const loadRepoIndex = async ( return { ref, - isEmpty, - needsUpgrade, - knotUnreachable, + availability, files, readme, readmeHtml, @@ -224,10 +184,10 @@ export const loadRepoTree = async ( ref: string, path: string ) => { - const source = repoDataSource(event, parent); + const git = gitTarget(parent.publicConfig, parent.repo, event.fetch); // the tree is the whole page here, so a miss is just a 404 - const tree = await orNull(source.tree(ref, path)); + const tree = await orNull(gitTree(git, { ref, path })); const files = sortTreeEntries((tree?.files ?? []).map(toTreeEntrySummary)); // git cannot store an empty directory. so nothing here means the path is a // file, or was never there diff --git a/web/src/lib/components/repo/RepoIndexView.stories.svelte b/web/src/lib/components/repo/RepoIndexView.stories.svelte index e0d8481df..1f6417055 100644 --- a/web/src/lib/components/repo/RepoIndexView.stories.svelte +++ b/web/src/lib/components/repo/RepoIndexView.stories.svelte @@ -18,9 +18,7 @@ const data = { ref: "main", - isEmpty: false, - needsUpgrade: false, - knotUnreachable: false, + availability: "ok", files: [ { name: "src", kind: "directory" as const, size: 0 }, { name: "README.md", kind: "file" as const, size: 512 } @@ -65,7 +63,7 @@ } satisfies Awaited>; const emptyData = { ...data, - isEmpty: true, + availability: "empty", files: [], readme: null, readmeHtml: null, @@ -90,8 +88,8 @@ - - + + diff --git a/web/src/lib/components/repo/RepoIndexView.svelte b/web/src/lib/components/repo/RepoIndexView.svelte index 283a41029..bac0b7646 100644 --- a/web/src/lib/components/repo/RepoIndexView.svelte +++ b/web/src/lib/components/repo/RepoIndexView.svelte @@ -30,7 +30,7 @@ - {#if data.needsUpgrade} + {#if data.availability === "needs-upgrade"}
@@ -47,14 +47,14 @@

- {:else if data.knotUnreachable} + {:else if data.availability === "unreachable"}
- {:else if data.isEmpty} + {:else if data.availability === "empty"} {:else} {#if data.languages.length > 0} diff --git a/web/src/lib/server/repo.ts b/web/src/lib/server/repo.ts new file mode 100644 index 000000000..aa8b98160 --- /dev/null +++ b/web/src/lib/server/repo.ts @@ -0,0 +1,45 @@ +import { error } from "@sveltejs/kit"; +import { createBobbinClient, type BobbinContext } from "$lib/api/client"; +import { gitTarget, type GitTarget } from "$lib/api/gitclient"; +import { resolveMiniDoc, type MiniDoc } from "$lib/api/identity"; +import { toHttpError } from "$lib/api/load"; +import type { RecordView, RepoRecord } from "$lib/api/records"; +import { resolveRepoByName } from "$lib/api/repo"; +import { getConfig } from "$lib/server/config"; + +export interface ResolvedRepo { + ctx: BobbinContext; + git: GitTarget; + doc: MiniDoc; + view: RecordView; +} + +// the handle/repo preamble shared by the raw-content routes, params arrive +// already decoded +export const resolveRepoFromParams = async (event: { + params: { handle?: string; repo?: string }; + fetch: typeof fetch; +}): Promise => { + const identifier = event.params.handle ?? ""; + const name = event.params.repo ?? ""; + +// rejects bare words, unrelated paths must 404 instead of resolving as actors + if (!identifier.startsWith("did:") && !identifier.includes(".")) { + error(404, "Not found"); + } + + const config = getConfig(); + const ctx = createBobbinClient({ serviceUrl: config.bobbinUrl, fetch: event.fetch }); + + const doc = await resolveMiniDoc(ctx, identifier).catch((cause) => + toHttpError(cause, "Could not resolve user") + ); + const view = await resolveRepoByName(ctx, doc.did, name).catch((cause) => + toHttpError(cause, "Could not load repository") + ); + if (!view) error(404, `${doc.handle}/${name} does not exist`); + + const git = gitTarget(config, { uri: view.uri, repoDid: view.value.repoDid }, event.fetch); + + return { ctx, git, doc, view }; +}; diff --git a/web/src/routes/[handle]/[repo]/+layout.ts b/web/src/routes/[handle]/[repo]/+layout.ts index 52cefb924..bcbd1764d 100644 --- a/web/src/routes/[handle]/[repo]/+layout.ts +++ b/web/src/routes/[handle]/[repo]/+layout.ts @@ -1,9 +1,9 @@ import { error, redirect } from "@sveltejs/kit"; import { createBobbinClient } from "$lib/api/client"; import { count } from "$lib/api/count"; +import { gitTarget, resolveDefaultBranch } from "$lib/api/gitclient"; import { getStarRkey } from "$lib/api/graph"; import { IdentityCache, resolveMiniDoc } from "$lib/api/identity"; -import { getDefaultBranch } from "$lib/api/knot"; import { parallel, toHttpError } from "$lib/api/load"; import { getRepo } from "$lib/api/records"; import { repoNameOf, resolveRepoByName } from "$lib/api/repo"; @@ -61,13 +61,12 @@ export const load: LayoutLoad = async (event) => { const record = view.value; const repoDid = record.repoDid; const viewerDid = parent.auth?.did; + const git = gitTarget(parent.publicConfig, { uri: view.uri, repoDid }, event.fetch); const stats = await parallel({ - // only the knot knows the default branch. a knot that is down or still - // syncing shouldn't take out the whole layout - defaultBranch: getDefaultBranch(ctx, { repo: view.uri }) - .then((branch) => branch.name) - .catch(() => null), + // only the git backend knows the default branch, and one that is down + // or still syncing shouldn't take out the whole layout + defaultBranch: resolveDefaultBranch(git), stars: repoDid ? count(ctx, "sh.tangled.feed.countStars", repoDid).catch(() => null) : Promise.resolve(null), diff --git a/web/src/routes/[handle]/[repo]/raw/[ref]/[...path]/+server.ts b/web/src/routes/[handle]/[repo]/raw/[ref]/[...path]/+server.ts index 22e970a03..68206fdb0 100644 --- a/web/src/routes/[handle]/[repo]/raw/[ref]/[...path]/+server.ts +++ b/web/src/routes/[handle]/[repo]/raw/[ref]/[...path]/+server.ts @@ -1,41 +1,24 @@ import { error } from "@sveltejs/kit"; -import { createBobbinClient } from "$lib/api/client"; -import { resolveMiniDoc } from "$lib/api/identity"; -import { blobRawUrl } from "$lib/api/knot"; -import { toHttpError } from "$lib/api/load"; -import { resolveRepoByName } from "$lib/api/repo"; -import { getConfig } from "$lib/server/config"; +import { blobRawUrl } from "$lib/api/gitclient"; +import { resolveRepoFromParams } from "$lib/server/repo"; import type { RequestHandler } from "./$types"; // redirecting instead of proxying keeps repo content off our own origin, same // as camo and avatar export const GET: RequestHandler = async (event) => { - const identifier = decodeURIComponent(event.params.handle); - const name = decodeURIComponent(event.params.repo); const { ref, path } = event.params; - // rejects bare words so unrelated paths 404 instead of resolving as actors - if (!identifier.startsWith("did:") && !identifier.includes(".")) { - error(404, "Not found"); - } if (path === "") error(404, "Not found"); - const { bobbinUrl } = getConfig(); - const ctx = createBobbinClient({ serviceUrl: bobbinUrl, fetch: event.fetch }); - - const doc = await resolveMiniDoc(ctx, identifier).catch((cause) => - toHttpError(cause, "Could not resolve user") - ); - const view = await resolveRepoByName(ctx, doc.did, name).catch((cause) => - toHttpError(cause, "Could not load repository") - ); - if (!view) error(404, `${doc.handle}/${name} does not exist`); + const { git } = await resolveRepoFromParams(event); // a branch points at a new commit after every push, so this cannot cache + const location = blobRawUrl(git, { ref, path }); + return new Response(null, { status: 302, headers: { - location: blobRawUrl(bobbinUrl, { repo: view.uri, ref, path }), + location, "cache-control": "public, no-cache" } }); -- 2.51.2