From c1cf1faf2b272a0f783eb0d3da765367dc2c40dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20R=C3=B8mer=20Hesselbjerg?= Date: Wed, 23 Sep 2026 23:05:35 +0200 Subject: [PATCH] chore: simplify/cleanup/rename in discover --- src/cli.ts | 11 ++++----- src/discover.ts | 63 ++++++++++++++++++++----------------------------- src/scan.ts | 4 ++-- src/tangled.ts | 23 ++++++------------ 4 files changed, 39 insertions(+), 62 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index d9535ad..ad8f0e9 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,4 +1,3 @@ -import { existsSync } from 'node:fs'; import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { parseArgs } from 'node:util'; @@ -14,12 +13,12 @@ npm run report [-- ] `; async function scan(): Promise { - const result = await scanRepos(await discoverRepos()); - const path = join(SCANS_DIR, `${result.date}.json`); - const replaced = existsSync(path); + const repos = await discoverRepos(); + const result = await scanRepos(repos); + const scanFilePath = join(SCANS_DIR, `${result.date}.json`); await mkdir(SCANS_DIR, { recursive: true }); - await writeFile(path, JSON.stringify(result, null, 2) + '\n', 'utf8'); - console.log(`→ ${path}${replaced ? " (replaced today's)" : ''}`); + await writeFile(scanFilePath, JSON.stringify(result, null, 2) + '\n', 'utf8'); + console.log(`→ ${scanFilePath}`); } /** diff --git a/src/discover.ts b/src/discover.ts index 250ff9f..7ad443e 100644 --- a/src/discover.ts +++ b/src/discover.ts @@ -1,17 +1,13 @@ -/** - * Discover repos on Tangled's own knot that still run GitHub workflows. - * - * The knot lists every repo it hosts, the mirror answers the folder probes, - * and the appview names the candidates. Every step is an unauthenticated GET. - * Repos on self-hosted knots are out of scope. - */ - import { DidResolver, getHandle, MemoryCache } from '@atproto/identity'; import { parseCanonicalResourceUri } from '@atcute/lexicons'; import type { Main as RepoRecord } from '@atcute/tangled/types/repo'; import type { $output as GetRepoOutput } from '@atcute/tangled/types/repo/getRepoByRepoDid'; import type { $output as ListReposOutput } from '@atcute/tangled/types/sync/listRepos'; -import { listFiles, listWorkflowFiles, USER_AGENT } from './tangled.ts'; +import { + listRepoFiles, + listGithubWorkflowFiles, + USER_AGENT, +} from './tangled.ts'; /** * A candidate repo, named from its `sh.tangled.repo` record. @@ -25,7 +21,7 @@ export type Repo = { const KNOT = 'https://knot1.tangled.sh'; const APPVIEW = 'https://api.tangled.org'; const PAGE_SIZE = 1000; -const PROGRESS_EVERY = 200; +const PROGRESS_INTERVAL = 200; const resolver = new DidResolver({ didCache: new MemoryCache() }); @@ -53,26 +49,11 @@ async function* listKnotRepos(): AsyncGenerator< } /** - * Whether the repo has GitHub workflow files and no `.tangled` folder. - * Throws when the mirror cannot read the repo. - */ -async function isCandidate(repoDid: string): Promise { - const [workflows, tangled] = await Promise.all([ - listWorkflowFiles(repoDid), - listFiles(repoDid, '.tangled'), - ]); - return workflows.length > 0 && tangled.length === 0; -} - -/** - * Name a repo from the current `sh.tangled.repo` record the appview holds - * for it. Yields `null` when the appview does not know the repo or the - * owner's handle does not resolve. - * - * Older records use the repo name as rkey and carry no `name`; newer ones - * use a TID rkey and carry the name in the record. + * Resolve a repo DID to its owner's handle and repo name via the appview's + * `sh.tangled.repo` record. Returns `null` when the appview does not know + * the repo or the owner's handle does not resolve. */ -async function describeRepo(repoDid: string): Promise { +async function resolveRepo(repoDid: string): Promise { const url = new URL('/xrpc/sh.tangled.repo.getRepoByRepoDid', APPVIEW); url.searchParams.set('repoDid', repoDid); const res = await fetch(url, { headers: { 'User-Agent': USER_AGENT } }); @@ -87,24 +68,25 @@ async function describeRepo(repoDid: string): Promise { if (!handle) { return null; } + // Some records have no `name` field, use rkey instead. return { handle, name: record.name ?? rkey, repoDid }; } /** - * Probe every active repo on the knot that has a default branch, and name - * the candidates. Archived and disabled repos are left out. + * Probe every active repo on the knot that has a default branch. + * Excludes archived and disabled repos. */ export async function discoverRepos(): Promise { const repos: Repo[] = []; - let listed = 0; + let total = 0; let inactive = 0; let empty = 0; let unreadable = 0; let unnamed = 0; for await (const entry of listKnotRepos()) { - listed++; - if (listed % PROGRESS_EVERY === 0) { - console.log(`${listed} listed, ${repos.length} candidates so far`); + total++; + if (total % PROGRESS_INTERVAL === 0) { + console.log(`${total} listed, ${repos.length} candidates found`); } if (entry.status !== 'active') { inactive++; @@ -115,14 +97,19 @@ export async function discoverRepos(): Promise { continue; } try { - if (!(await isCandidate(entry.repo))) { + // Check if repo has GH workflows but no Tangled + const [githubWorkflows, tangled] = await Promise.all([ + listGithubWorkflowFiles(entry.repo), + listRepoFiles(entry.repo, '.tangled'), + ]); + if (githubWorkflows.length === 0 || tangled.length > 0) { continue; } } catch { unreadable++; continue; } - const repo = await describeRepo(entry.repo); + const repo = await resolveRepo(entry.repo); if (!repo) { unnamed++; continue; @@ -130,7 +117,7 @@ export async function discoverRepos(): Promise { repos.push(repo); } console.log( - `${listed} repos on ${KNOT}: ${inactive} inactive, ${empty} empty, ` + + `${total} repos on ${KNOT}: ${inactive} inactive, ${empty} empty, ` + `${unreadable} unreadable, ${unnamed} unnamed, ` + `${repos.length} candidates`, ); diff --git a/src/scan.ts b/src/scan.ts index 8fe7c1f..f138bf8 100644 --- a/src/scan.ts +++ b/src/scan.ts @@ -4,7 +4,7 @@ import { type Conversion, } from './convert.ts'; import type { Repo } from './discover.ts'; -import { getBlob, listWorkflowFiles } from './tangled.ts'; +import { getBlob, listGithubWorkflowFiles } from './tangled.ts'; /** * The conversion outcome of one workflow file. `owner` is the handle of the @@ -30,7 +30,7 @@ export type Scan = { * Convert every GitHub workflow in `repo`. */ async function scanRepo(repo: Repo): Promise { - const files = await listWorkflowFiles(repo.repoDid); + const files = await listGithubWorkflowFiles(repo.repoDid); return Promise.all( files.map(async (file) => { const conversion = convertWorkflow(await getBlob(repo.repoDid, file)); diff --git a/src/tangled.ts b/src/tangled.ts index 8453f04..fb0e435 100644 --- a/src/tangled.ts +++ b/src/tangled.ts @@ -1,10 +1,3 @@ -/** - * Read-only access to git contents of Tangled repos. - * - * Reads go through the mirror rather than the knot named in a repo record; - * `knot1.tangled.sh` 404s on every `sh.tangled.git.temp.*` call. - */ - import type { $output as TreeOutput } from '@atcute/tangled/types/git/temp/getTree'; const MIRROR = 'https://mirror-fsn.tangled.network'; @@ -24,14 +17,10 @@ function xrpc( } /** - * List the entry names in `path` at the repo's default branch. A missing path - * yields an empty list. - * - * Knot1 repos answer 404 for a missing path, but self-hosted knots answer 200 - * with an empty list for any path, so the empty list is the only reliable - * "absent" signal. + * List the entry names in `path` at the repo's default branch. The mirror + * answers 404 for a missing path, which yields an empty list. */ -export async function listFiles( +export async function listRepoFiles( repoDid: string, path: string, ): Promise { @@ -64,8 +53,10 @@ const YAML_EXTENSIONS = ['.yml', '.yaml'] as const; * List the GitHub workflow files at the repo's default branch as * repo-relative paths (`.github/workflows/.yml|.yaml`). */ -export async function listWorkflowFiles(repoDid: string): Promise { - const names = await listFiles(repoDid, WORKFLOWS_DIR); +export async function listGithubWorkflowFiles( + repoDid: string, +): Promise { + const names = await listRepoFiles(repoDid, WORKFLOWS_DIR); return names .filter((name) => YAML_EXTENSIONS.some((ext) => name.endsWith(ext))) .map((name) => `${WORKFLOWS_DIR}/${name}`); -- 2.51.2