From a4300eab82a6379ba36754f8916b47969d292b77 Mon Sep 17 00:00:00 2001 From: Owais Jamil Date: Mon, 23 Mar 2026 09:56:41 -0500 Subject: [PATCH] docs: reorganize and add overview to README --- README.md | 96 +++++- apps/twisted/.env.example | 1 + apps/twisted/README.md | 19 +- apps/twisted/docs/specs/phase-3.md | 66 ---- apps/twisted/src/core/config/project.ts | 12 + .../src/features/activity/ActivityPage.vue | 2 +- .../src/features/explore/ExplorePage.vue | 303 +++++++++++++++++- apps/twisted/src/features/home/HomePage.vue | 4 +- .../src/features/profile/UserProfilePage.vue | 30 +- .../src/services/project-api/client.ts | 31 ++ .../src/services/project-api/queries.ts | 198 ++++++++++++ apps/twisted/src/vite-env.d.ts | 8 + docs/README.md | 13 + .../api}/specs/01-architecture.md | 64 ++-- .../api}/specs/02-tangled-lexicons.md | 0 .../docs => docs/api}/specs/03-data-model.md | 0 .../api}/specs/04-data-pipeline.md | 0 .../api/docs => docs/api}/specs/05-search.md | 172 +++++----- .../docs => docs/api}/specs/06-operations.md | 114 ++++++- .../api}/specs/07-graph-backfill.md | 28 +- docs/api/specs/08-app-integration.md | 89 +++++ .../api/docs => docs/api}/specs/README.md | 5 +- docs/api/tasks/README.md | 40 +++ .../docs => docs/api}/tasks/phase-1-mvp.md | 171 +++++----- .../api}/tasks/phase-2-semantic.md | 12 - .../docs => docs/api}/tasks/phase-3-hybrid.md | 2 - .../api}/tasks/phase-4-quality.md | 2 - .../twisted/docs => docs/app}/specs/README.md | 28 +- .../docs => docs/app}/specs/phase-1.md | 0 .../docs => docs/app}/specs/phase-2.md | 0 docs/app/specs/phase-3.md | 68 ++++ .../docs => docs/app}/specs/phase-4.md | 0 .../docs => docs/app}/specs/phase-5.md | 0 .../docs => docs/app}/specs/phase-6.md | 18 +- .../docs => docs/app}/specs/phase-7.md | 2 +- .../docs => docs/app}/tasks/phase-1.md | 0 .../docs => docs/app}/tasks/phase-2.md | 0 .../docs => docs/app}/tasks/phase-3.md | 0 .../docs => docs/app}/tasks/phase-4.md | 0 .../docs => docs/app}/tasks/phase-5.md | 0 .../docs => docs/app}/tasks/phase-6.md | 35 +- .../docs => docs/app}/tasks/phase-7.md | 0 justfile | 2 +- packages/api/README.md | 2 +- packages/api/docs/tasks/README.md | 38 --- 45 files changed, 1254 insertions(+), 421 deletions(-) create mode 100644 apps/twisted/.env.example delete mode 100644 apps/twisted/docs/specs/phase-3.md create mode 100644 apps/twisted/src/core/config/project.ts create mode 100644 apps/twisted/src/services/project-api/client.ts create mode 100644 apps/twisted/src/services/project-api/queries.ts create mode 100644 docs/README.md rename {packages/api/docs => docs/api}/specs/01-architecture.md (81%) rename {packages/api/docs => docs/api}/specs/02-tangled-lexicons.md (100%) rename {packages/api/docs => docs/api}/specs/03-data-model.md (100%) rename {packages/api/docs => docs/api}/specs/04-data-pipeline.md (100%) rename {packages/api/docs => docs/api}/specs/05-search.md (59%) rename {packages/api/docs => docs/api}/specs/06-operations.md (77%) rename {packages/api/docs => docs/api}/specs/07-graph-backfill.md (80%) create mode 100644 docs/api/specs/08-app-integration.md rename {packages/api/docs => docs/api}/specs/README.md (73%) create mode 100644 docs/api/tasks/README.md rename {packages/api/docs => docs/api}/tasks/phase-1-mvp.md (80%) rename {packages/api/docs => docs/api}/tasks/phase-2-semantic.md (94%) rename {packages/api/docs => docs/api}/tasks/phase-3-hybrid.md (99%) rename {packages/api/docs => docs/api}/tasks/phase-4-quality.md (99%) rename {apps/twisted/docs => docs/app}/specs/README.md (73%) rename {apps/twisted/docs => docs/app}/specs/phase-1.md (100%) rename {apps/twisted/docs => docs/app}/specs/phase-2.md (100%) create mode 100644 docs/app/specs/phase-3.md rename {apps/twisted/docs => docs/app}/specs/phase-4.md (100%) rename {apps/twisted/docs => docs/app}/specs/phase-5.md (100%) rename {apps/twisted/docs => docs/app}/specs/phase-6.md (79%) rename {apps/twisted/docs => docs/app}/specs/phase-7.md (94%) rename {apps/twisted/docs => docs/app}/tasks/phase-1.md (100%) rename {apps/twisted/docs => docs/app}/tasks/phase-2.md (100%) rename {apps/twisted/docs => docs/app}/tasks/phase-3.md (100%) rename {apps/twisted/docs => docs/app}/tasks/phase-4.md (100%) rename {apps/twisted/docs => docs/app}/tasks/phase-5.md (100%) rename {apps/twisted/docs => docs/app}/tasks/phase-6.md (67%) rename {apps/twisted/docs => docs/app}/tasks/phase-7.md (100%) delete mode 100644 packages/api/docs/tasks/README.md diff --git a/README.md b/README.md index a229406..03e291c 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,102 @@ -# Twisted Monorepo +# Twisted -- `apps/twisted`: Ionic/Vue client -- `packages/api`: Go API copied from `~/Projects/TWISTER` +Twisted is a monorepo for a Tangled mobile client and the supporting Tap-backed indexing API. + +## Projects + +- `apps/twisted`: Ionic Vue client for browsing Tangled repos, profiles, issues, PRs, and indexed search results +- `packages/api`: Go service that consumes Tangled records through Tap, fills gaps in the public Tangled API, and serves search +- `docs`: top-level specs and plans, split by project under `docs/app` and `docs/api` + +## Architecture + +The app still uses Tangled's public knot and PDS APIs for canonical repo and profile data. The API project adds two complementary capabilities: + +1. Global search over indexed Tangled content +2. Index-backed summaries for data that is hard to derive from the public API alone, such as followers + +That keeps direct browsing honest while giving the client one place to ask for cross-network discovery and graph augmentation. ## Development -Use the top-level `justfile` for common tasks: +Use the top-level [`justfile`](justfile) for common workflows: ```bash just dev just build just test +just api-run-api ``` -The existing client package still works directly from `apps/twisted`. +To enable indexed search in the client, set `VITE_TWISTER_API_BASE_URL` in `apps/twisted/.env`. + +## Infrastructure Setup + +### Turso + +Use one Turso database per environment, for example: + +- `twister-dev` +- `twister-prod` + +Do not introduce separate app variable names for dev and prod. Always use the same variables: + +- `TURSO_DATABASE_URL` +- `TURSO_AUTH_TOKEN` + +Only the values change per environment. + +Example: + +```bash +# Development +TURSO_DATABASE_URL=libsql://twister-dev-your-org.turso.io +TURSO_AUTH_TOKEN=... + +# Production +TURSO_DATABASE_URL=libsql://twister-prod-your-org.turso.io +TURSO_AUTH_TOKEN=... +``` + +### Railway + +Create or reuse one Railway project containing: + +- existing `tap` +- `api` running `twister api` +- `indexer` running `twister indexer` + +Set these shared variables on the Railway services: + +- `TURSO_DATABASE_URL` +- `TURSO_AUTH_TOKEN` +- `LOG_LEVEL` +- `LOG_FORMAT` + +Set these API-specific variables: + +- `HTTP_BIND_ADDR` +- `SEARCH_DEFAULT_LIMIT` +- `SEARCH_MAX_LIMIT` + +Set these indexer-specific variables: + +- `TAP_URL` +- `TAP_AUTH_PASSWORD` +- `INDEXED_COLLECTIONS` + +If you use separate Railway environments for dev and prod, keep the same variable names in both and only swap the Turso values. + +### First Bootstrap + +For a brand-new environment: + +1. Point `TURSO_DATABASE_URL` and `TURSO_AUTH_TOKEN` at the target database. +2. Deploy `api` and `indexer` on Railway. +3. Verify API readiness and indexer health. +4. Run `twister backfill` with your seed file. +5. Treat the environment as search-ready only after historical backfill completes. + +## Docs + +- Index: [`docs/README.md`](docs/README.md) diff --git a/apps/twisted/.env.example b/apps/twisted/.env.example new file mode 100644 index 0000000..e53c663 --- /dev/null +++ b/apps/twisted/.env.example @@ -0,0 +1 @@ +VITE_TWISTER_API_BASE_URL=http://localhost:8080 diff --git a/apps/twisted/README.md b/apps/twisted/README.md index f2496c6..826be52 100644 --- a/apps/twisted/README.md +++ b/apps/twisted/README.md @@ -1,18 +1,3 @@ -# Twisted +# Twisted App -A mobile client for [Tangled](https://tangled.org). - -## Development - -Run the mobile apps with Capacitor: - -```bash -pnpm cap run ios -pnpm cap run android -``` - -Or to test the web version: - -```bash -pnpm dev -``` +Ionic Vue client for the Twisted monorepo. diff --git a/apps/twisted/docs/specs/phase-3.md b/apps/twisted/docs/specs/phase-3.md deleted file mode 100644 index 39f85f0..0000000 --- a/apps/twisted/docs/specs/phase-3.md +++ /dev/null @@ -1,66 +0,0 @@ -# Phase 3 — Deferred Search and Activity - -## Goal - -Preserve honest product boundaries before search is implemented as a separate project. Public browsing continues through known AT Protocol handles on Home, while Explore and Activity stay visible as clearly labeled in-progress placeholders. - -## Current Product Shape - -### Home - -Home is the temporary public entry point for unauthenticated browsing: - -- Enter a known AT Protocol handle -- Open that user's profile directly -- Resolve the handle to DID + PDS via AT Protocol identity -- List that user's public Tangled repos inline and open one directly - -This keeps public browsing fully real without implying that global discovery already exists. - -### Explore - -Explore remains a tab-level placeholder: - -- No global repo search -- No global user search -- No curated fallback discovery pretending to be search -- Empty state should explicitly say search is in progress - -### Activity - -Activity also remains a tab-level placeholder: - -- No public timeline yet -- No curated public feed fallback -- Empty state should explicitly say activity is in progress - -## Identity and Routing - -The Home handle flow continues to use the existing AT Protocol resolution path: - -1. Resolve `handle -> DID` via `com.atproto.identity.resolveHandle` -2. Fetch the DID document and extract the PDS endpoint -3. Query the user's PDS for `sh.tangled.repo` records via `com.atproto.repo.listRecords` -4. Route to existing profile and repo detail screens - -No backend search index, feed service, or additional dependency is introduced in this phase. - -## UI Expectations - -- Home shows one handle input plus explicit actions for profile jump and repo browsing -- Home shows loading, invalid-handle, no-repos, and resolved-repo-list states -- Explore shows a static in-progress empty state -- Activity shows a static in-progress empty state -- Profile remains unchanged - -## Deferred Work - -The following work is intentionally deferred out of this phase: - -- Search indexing and ranking -- Search result UI and recent searches -- Trending or suggested discovery sections -- Public activity feed ingestion, pagination, and caching -- Jetstream or appview timeline investigation - -These capabilities will be revisited when search and feed work are scheduled independently. diff --git a/apps/twisted/src/core/config/project.ts b/apps/twisted/src/core/config/project.ts new file mode 100644 index 0000000..03f6113 --- /dev/null +++ b/apps/twisted/src/core/config/project.ts @@ -0,0 +1,12 @@ +const rawTwisterApiBaseUrl = import.meta.env.VITE_TWISTER_API_BASE_URL?.trim() ?? ""; + +export const twisterApiBaseUrl = rawTwisterApiBaseUrl.replace(/\/+$/, ""); +export const hasTwisterApi = twisterApiBaseUrl.length > 0; + +export function getTwisterApiUrl(path: string): string { + if (!hasTwisterApi) { + throw new Error("Twister API base URL is not configured."); + } + + return new URL(path.replace(/^\/+/, ""), `${twisterApiBaseUrl}/`).toString(); +} diff --git a/apps/twisted/src/features/activity/ActivityPage.vue b/apps/twisted/src/features/activity/ActivityPage.vue index 8c2fafc..4e564bb 100644 --- a/apps/twisted/src/features/activity/ActivityPage.vue +++ b/apps/twisted/src/features/activity/ActivityPage.vue @@ -16,7 +16,7 @@ + message="The public activity feed is still in progress. This tab stays as a placeholder until the indexed feed work is ready." /> diff --git a/apps/twisted/src/features/explore/ExplorePage.vue b/apps/twisted/src/features/explore/ExplorePage.vue index a97bb3f..ba97493 100644 --- a/apps/twisted/src/features/explore/ExplorePage.vue +++ b/apps/twisted/src/features/explore/ExplorePage.vue @@ -13,16 +13,307 @@ - +
+

Indexed Search

+

Search the Tangled network through the project index.

+

+ Explore uses the Twister index for global search. Open any result to continue browsing through Tangled's + public repo and profile APIs. +

+
+ +
+ + + + + All + Repos + People + + +
+ + Search + + Clear +
+ +

+ Search results and follower counts come from the project index when available. +

+

+ Set VITE_TWISTER_API_BASE_URL to enable global search and index-backed graph summaries. +

+
+ +
+ + + + + + + + + + + +
+ + diff --git a/apps/twisted/src/features/home/HomePage.vue b/apps/twisted/src/features/home/HomePage.vue index b02fd01..91e4265 100644 --- a/apps/twisted/src/features/home/HomePage.vue +++ b/apps/twisted/src/features/home/HomePage.vue @@ -45,7 +45,9 @@ -

Repo browsing is temporary here until search ships in a separate project.

+

+ Home is still the fastest way to jump to a known handle directly. +

diff --git a/apps/twisted/src/features/profile/UserProfilePage.vue b/apps/twisted/src/features/profile/UserProfilePage.vue index 5b4e687..9dc6361 100644 --- a/apps/twisted/src/features/profile/UserProfilePage.vue +++ b/apps/twisted/src/features/profile/UserProfilePage.vue @@ -183,6 +183,7 @@ import { useUserPullRequests, useUserFollowing, } from "@/services/tangled/queries.js"; +import { useIndexedProfileSummary } from "@/services/project-api/queries.js"; import type { IssueSummary } from "@/domain/models/issue.js"; import type { PullRequestSummary } from "@/domain/models/pull-request.js"; import type { RepoSummary } from "@/domain/models/repo.js"; @@ -208,6 +209,7 @@ const stringsQuery = useUserStrings(pds, did, { enabled: hasIdentity }); const issuesQuery = useUserIssues(pds, did, handle, { enabled: hasIdentity }); const pullRequestsQuery = useUserPullRequests(pds, did, handle, { enabled: hasIdentity }); const followingQuery = useUserFollowing(pds, did, { enabled: hasIdentity }); +const indexedProfileSummaryQuery = useIndexedProfileSummary(did, { enabled: hasIdentity }); const profile = computed(() => profileQuery.data.value); const repos = computed(() => reposQuery.data.value ?? []); @@ -215,17 +217,31 @@ const strings = computed(() => stringsQuery.data.value ?? []); const issues = computed(() => issuesQuery.data.value ?? []); const pullRequests = computed(() => pullRequestsQuery.data.value ?? []); const following = computed(() => followingQuery.data.value ?? []); +const indexedProfileSummary = computed(() => indexedProfileSummaryQuery.data.value); const pinnedUris = computed(() => (profile.value as { pinnedRepos?: string[] } | undefined)?.pinnedRepos ?? []); const pinnedRepos = computed(() => repos.value.filter((r) => pinnedUris.value.includes(r.atUri))); const otherRepos = computed(() => repos.value.filter((repo) => !pinnedUris.value.includes(repo.atUri))); -const stats = computed(() => [ - { label: "repos", value: repos.value.length }, - { label: "strings", value: strings.value.length }, - { label: "issues", value: issues.value.length }, - { label: "prs", value: pullRequests.value.length }, - { label: "following", value: following.value.length }, -]); +const stats = computed(() => { + const values = [ + { label: "repos", value: repos.value.length }, + { label: "strings", value: strings.value.length }, + { label: "issues", value: issues.value.length }, + { label: "prs", value: pullRequests.value.length }, + ]; + + if (indexedProfileSummary.value?.followerCount != null) { + values.splice(1, 0, { label: "followers", value: indexedProfileSummary.value.followerCount }); + } + + values.splice( + indexedProfileSummary.value?.followerCount != null ? 2 : 1, + 0, + { label: "following", value: indexedProfileSummary.value?.followingCount ?? following.value.length }, + ); + + return values; +}); const isLoading = computed(() => identity.isPending.value || profileQuery.isPending.value); const isError = computed(() => identity.isError.value || profileQuery.isError.value); diff --git a/apps/twisted/src/services/project-api/client.ts b/apps/twisted/src/services/project-api/client.ts new file mode 100644 index 0000000..093a342 --- /dev/null +++ b/apps/twisted/src/services/project-api/client.ts @@ -0,0 +1,31 @@ +import { getTwisterApiUrl, hasTwisterApi } from "@/core/config/project.js"; + +type ErrorPayload = { error?: string; message?: string }; + +export async function fetchProjectApiJson(path: string, init?: RequestInit): Promise { + if (!hasTwisterApi) { + throw new Error("Twister API base URL is not configured."); + } + + const response = await fetch(getTwisterApiUrl(path), { + ...init, + headers: { Accept: "application/json", ...(init?.headers ?? {}) }, + }); + + if (!response.ok) { + const fallbackMessage = `Project API request failed with status ${response.status}.`; + + try { + const payload = (await response.json()) as ErrorPayload; + throw new Error(payload.message ?? payload.error ?? fallbackMessage); + } catch (error) { + if (error instanceof Error && error.message !== "Unexpected end of JSON input") { + throw error; + } + + throw new Error(fallbackMessage, { cause: error }); + } + } + + return (await response.json()) as T; +} diff --git a/apps/twisted/src/services/project-api/queries.ts b/apps/twisted/src/services/project-api/queries.ts new file mode 100644 index 0000000..3298044 --- /dev/null +++ b/apps/twisted/src/services/project-api/queries.ts @@ -0,0 +1,198 @@ +import { useQuery } from "@tanstack/vue-query"; +import { computed, toValue } from "vue"; +import type { MaybeRef } from "vue"; +import { hasTwisterApi } from "@/core/config/project.js"; +import type { RepoSummary } from "@/domain/models/repo.js"; +import type { UserSummary } from "@/domain/models/user.js"; +import { fetchProjectApiJson } from "./client.js"; + +const MIN = 60_000; + +export type ProjectSearchMode = "keyword" | "semantic" | "hybrid"; +export type ProjectSearchType = "all" | "repo" | "profile"; + +type ProjectSearchResult = { + id: string; + did?: string; + at_uri?: string; + collection: string; + record_type: string; + title: string; + body_snippet?: string; + summary?: string; + repo_name?: string; + author_handle?: string; + score?: number; + matched_by?: string[]; + created_at?: string; + updated_at?: string; + primary_language?: string; + stars?: number; + follower_count?: number; + following_count?: number; +}; + +type ProjectSearchResponse = { + query: string; + mode: ProjectSearchMode; + total: number; + limit: number; + offset: number; + results: ProjectSearchResult[]; +}; + +type IndexedProfileSummaryResponse = { + did: string; + handle?: string; + follower_count?: number; + following_count?: number; + indexed_at?: string; +}; + +export type IndexedProfileSummary = { + did: string; + handle?: string; + followerCount?: number; + followingCount?: number; + indexedAt?: string; +}; + +export type ProjectSearchResults = { + query: string; + mode: ProjectSearchMode; + total: number; + repos: RepoSummary[]; + profiles: UserSummary[]; +}; + +function stripHighlight(html?: string): string | undefined { + if (!html) return undefined; + + const text = html.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim(); + return text || undefined; +} + +function parseAtUriRkey(atUri?: string, fallback = ""): string { + if (!atUri) return fallback; + + const segments = atUri.split("/"); + return segments[segments.length - 1] || fallback; +} + +function toRepoSummary(result: ProjectSearchResult): RepoSummary { + const atUri = result.at_uri ?? result.id; + const repoName = result.repo_name ?? result.title; + + return { + atUri, + rkey: parseAtUriRkey(result.at_uri, result.id), + ownerDid: result.did ?? "", + ownerHandle: result.author_handle ?? "unknown", + name: repoName, + description: result.summary ?? stripHighlight(result.body_snippet), + primaryLanguage: result.primary_language, + stars: result.stars, + updatedAt: result.updated_at ?? result.created_at, + knot: "", + }; +} + +function toUserSummary(result: ProjectSearchResult): UserSummary { + const handle = result.author_handle ?? result.title; + const displayName = result.title !== handle ? result.title : undefined; + + return { + did: result.did ?? "", + handle, + displayName, + bio: result.summary ?? stripHighlight(result.body_snippet), + followerCount: result.follower_count, + followingCount: result.following_count, + }; +} + +function normalizeProfileSummary(summary: IndexedProfileSummaryResponse): IndexedProfileSummary { + return { + did: summary.did, + handle: summary.handle, + followerCount: summary.follower_count, + followingCount: summary.following_count, + indexedAt: summary.indexed_at, + }; +} + +export function useProjectSearch( + query: MaybeRef, + options: { + type?: MaybeRef; + mode?: MaybeRef; + limit?: number; + enabled?: MaybeRef; + } = {}, +) { + const normalizedQuery = computed(() => toValue(query).trim()); + const normalizedType = computed(() => toValue(options.type) ?? "all"); + const normalizedMode = computed(() => toValue(options.mode) ?? "keyword"); + const enabled = computed( + () => + hasTwisterApi && + normalizedQuery.value.length > 0 && + (options.enabled === undefined || !!toValue(options.enabled)), + ); + + return useQuery({ + queryKey: computed(() => ["projectSearch", normalizedQuery.value, normalizedType.value, normalizedMode.value]), + queryFn: async (): Promise => { + const params = new URLSearchParams({ + q: normalizedQuery.value, + mode: normalizedMode.value, + limit: String(options.limit ?? 20), + }); + + if (normalizedType.value !== "all") { + params.set("type", normalizedType.value); + } + + const response = await fetchProjectApiJson(`/search?${params.toString()}`); + const repos = response.results.filter((result) => result.record_type === "repo").map(toRepoSummary); + const profiles = response.results.filter((result) => result.record_type === "profile").map(toUserSummary); + + return { + query: response.query, + mode: response.mode, + total: response.total, + repos, + profiles, + }; + }, + enabled, + staleTime: 2 * MIN, + gcTime: 10 * MIN, + }); +} + +export function useIndexedProfileSummary( + did: MaybeRef, + options: { enabled?: MaybeRef } = {}, +) { + const normalizedDid = computed(() => toValue(did).trim()); + const enabled = computed( + () => + hasTwisterApi && + normalizedDid.value.length > 0 && + (options.enabled === undefined || !!toValue(options.enabled)), + ); + + return useQuery({ + queryKey: computed(() => ["indexedProfileSummary", normalizedDid.value]), + queryFn: async () => { + const response = await fetchProjectApiJson( + `/profiles/${encodeURIComponent(normalizedDid.value)}/summary`, + ); + return normalizeProfileSummary(response); + }, + enabled, + staleTime: 10 * MIN, + gcTime: 30 * MIN, + }); +} diff --git a/apps/twisted/src/vite-env.d.ts b/apps/twisted/src/vite-env.d.ts index 2b71d9c..ed63eda 100644 --- a/apps/twisted/src/vite-env.d.ts +++ b/apps/twisted/src/vite-env.d.ts @@ -2,6 +2,14 @@ /// /// +interface ImportMetaEnv { + readonly VITE_TWISTER_API_BASE_URL?: string; +} + +interface ImportMeta { + readonly env: ImportMetaEnv; +} + declare module "markdown-it" { type MarkdownIt = { render(content: string): string; diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..4e0293b --- /dev/null +++ b/docs/README.md @@ -0,0 +1,13 @@ +# Twisted Documentation + +Documentation is organized by project: + +- [`app/`](app/) for the Ionic/Vue client +- [`api/`](api/) for the Go Tap/index/search service + +## Quick Links + +- App spec index: [`app/specs/README.md`](app/specs/README.md) +- App task index: [`app/tasks/phase-6.md`](app/tasks/phase-6.md) +- API spec index: [`api/specs/README.md`](api/specs/README.md) +- API task index: [`api/tasks/README.md`](api/tasks/README.md) diff --git a/packages/api/docs/specs/01-architecture.md b/docs/api/specs/01-architecture.md similarity index 81% rename from packages/api/docs/specs/01-architecture.md rename to docs/api/specs/01-architecture.md index 5ff3edd..6eec7e8 100644 --- a/packages/api/docs/specs/01-architecture.md +++ b/docs/api/specs/01-architecture.md @@ -7,42 +7,43 @@ updated: 2026-03-22 Build a Go-based search service for Tangled content on AT Protocol that: -* ingests Tangled records through **Tap** (already deployed on Railway) -* denormalizes them into internal search documents -* indexes them in **Turso/libSQL** -* exposes a search API with **keyword**, **semantic**, and **hybrid** retrieval modes +- ingests Tangled records through **Tap** (already deployed on Railway) +- denormalizes them into internal search documents +- indexes them in **Turso/libSQL** +- exposes a search API with **keyword**, **semantic**, and **hybrid** retrieval modes +- exposes index-backed summary APIs for data the public Tangled APIs do not answer efficiently, such as followers ## 2. Functional Goals The system shall: -* index Tangled-specific ATProto collections under the `sh.tangled.*` namespace -* support initial backfill and continuous incremental sync via Tap -* support lexical retrieval using Turso's Tantivy-backed FTS -* support semantic retrieval using vector embeddings -* support hybrid ranking combining lexical and semantic signals -* expose stable HTTP APIs for search and document lookup -* support deployment on **Railway** +- index Tangled-specific ATProto collections under the `sh.tangled.*` namespace +- support initial backfill and continuous incremental sync via Tap +- support lexical retrieval using Turso's Tantivy-backed FTS +- support semantic retrieval using vector embeddings +- support hybrid ranking combining lexical and semantic signals +- expose stable HTTP APIs for search, document lookup, and graph/profile summaries +- support deployment on **Railway** ## 3. Non-Functional Goals The system shall prioritize: -* **correctness of sync** — cursors never advance ahead of committed data -* **operational simplicity** — single binary, subcommand-driven -* **incremental delivery** — keyword search ships before embeddings -* **small deployable services** — process groups, not microservices -* **reindexability** — any document or collection can be re-normalized and re-indexed -* **low coupling** — sync, indexing, and serving are independent concerns +- **correctness of sync** — cursors never advance ahead of committed data +- **operational simplicity** — single binary, subcommand-driven +- **incremental delivery** — keyword search ships before embeddings +- **small deployable services** — process groups, not microservices +- **reindexability** — any document or collection can be re-normalized and re-indexed +- **low coupling** — sync, indexing, and serving are independent concerns ## 4. Out of Scope (v1) -* code-aware symbol search -* sourcegraph-style structural search -* personalized ranking -* access control beyond public/private visibility flags in indexed records -* full analytics pipeline -* custom ANN infrastructure outside Turso/libSQL +- code-aware symbol search +- sourcegraph-style structural search +- personalized ranking +- access control beyond public/private visibility flags in indexed records +- full analytics pipeline +- custom ANN infrastructure outside Turso/libSQL ## 5. Design Principles @@ -50,12 +51,14 @@ The system shall prioritize: 2. **The indexer owns denormalization.** Raw ATProto records are never queried directly by the public API. -3. **Search serves denormalized documents.** Search ranking depends on the document model, not transport. +3. **The public API serves denormalized projections.** Search ranking and graph summaries depend on the indexed document model, not transport. 4. **Keyword search is the baseline.** Semantic and hybrid search are layered on top. 5. **Embeddings are asynchronous.** Ingestion is never blocked on vector generation unless explicitly configured. +6. **Twister complements public Tangled APIs.** Repo detail stays on knots/PDSes; the index adds discovery and cross-network summaries. + ## 6. External Systems - **AT Protocol network** — source of all Tangled content @@ -93,17 +96,18 @@ ATProto Firehose / PDS ├─ keyword search (fts_match / fts_score) ├─ semantic search (vector_top_k) ├─ hybrid search (weighted merge) + ├─ profile and graph summaries └─ document fetch ``` ## 8. Runtime Units -| Unit | Role | Deployment | -| -------------- | ----------------------------------- | ------------------------------- | -| `api` | HTTP search and document API | Railway service (public) | -| `indexer` | Tap consumer, normalizer, DB writer | Railway service (internal) | -| `embed-worker` | Async embedding generation | Optional Railway service | -| `tap` | ATProto sync | Railway (already deployed) | +| Unit | Role | Deployment | +| -------------- | -------------------------------------------- | -------------------------- | +| `api` | HTTP search, graph summary, and document API | Railway service (public) | +| `indexer` | Tap consumer, normalizer, DB writer | Railway service (internal) | +| `embed-worker` | Async embedding generation | Optional Railway service | +| `tap` | ATProto sync | Railway (already deployed) | ## 9. Repository Structure diff --git a/packages/api/docs/specs/02-tangled-lexicons.md b/docs/api/specs/02-tangled-lexicons.md similarity index 100% rename from packages/api/docs/specs/02-tangled-lexicons.md rename to docs/api/specs/02-tangled-lexicons.md diff --git a/packages/api/docs/specs/03-data-model.md b/docs/api/specs/03-data-model.md similarity index 100% rename from packages/api/docs/specs/03-data-model.md rename to docs/api/specs/03-data-model.md diff --git a/packages/api/docs/specs/04-data-pipeline.md b/docs/api/specs/04-data-pipeline.md similarity index 100% rename from packages/api/docs/specs/04-data-pipeline.md rename to docs/api/specs/04-data-pipeline.md diff --git a/packages/api/docs/specs/05-search.md b/docs/api/specs/05-search.md similarity index 59% rename from packages/api/docs/specs/05-search.md rename to docs/api/specs/05-search.md index 7fd5f8f..5d38308 100644 --- a/packages/api/docs/specs/05-search.md +++ b/docs/api/specs/05-search.md @@ -3,15 +3,15 @@ title: "Spec 05 — Search" updated: 2026-03-22 --- -Covers all search modes, the API contract, scoring, and filtering. +Covers all search modes, the public search API contract, scoring, and filtering. ## 1. Search Modes -| Mode | Backing | Available | -|------|---------|-----------| -| `keyword` | Turso Tantivy-backed FTS | MVP | -| `semantic` | Vector similarity (DiskANN index) | Phase 2 | -| `hybrid` | Weighted merge of keyword + semantic | Phase 3 | +| Mode | Backing | Available | +| ---------- | ------------------------------------ | --------- | +| `keyword` | Turso Tantivy-backed FTS | MVP | +| `semantic` | Vector similarity (DiskANN index) | Phase 2 | +| `hybrid` | Weighted merge of keyword + semantic | Phase 3 | ## 2. Keyword Search @@ -35,14 +35,14 @@ LIMIT ? OFFSET ?; Configured in the FTS index definition: -| Field | Weight | Rationale | -|-------|--------|-----------| -| `title` | 3.0 | Highest signal for relevance | -| `repo_name` | 2.5 | Exact repo lookups should rank first | -| `author_handle` | 2.0 | Author search is common | -| `summary` | 1.5 | More focused than body | -| `tags_json` | 1.2 | Topic matching | -| `body` | 1.0 | Baseline | +| Field | Weight | Rationale | +| --------------- | ------ | ------------------------------------ | +| `title` | 3.0 | Highest signal for relevance | +| `repo_name` | 2.5 | Exact repo lookups should rank first | +| `author_handle` | 2.0 | Author search is common | +| `summary` | 1.5 | More focused than body | +| `tags_json` | 1.2 | Topic matching | +| `body` | 1.0 | Baseline | ### Query Features @@ -84,7 +84,7 @@ WHERE d.deleted_at IS NULL; Cosine distance ranges from 0 (identical) to 2 (opposite). Normalize to a 0–1 relevance score: -``` +```text semantic_score = 1.0 - (distance / 2.0) ``` @@ -92,7 +92,7 @@ semantic_score = 1.0 - (distance / 2.0) ### v1: Weighted Score Blending -``` +```text hybrid_score = 0.65 * keyword_score_normalized + 0.35 * semantic_score_normalized ``` @@ -100,7 +100,7 @@ hybrid_score = 0.65 * keyword_score_normalized + 0.35 * semantic_score_normalize Keyword (BM25) scores are unbounded. Normalize using min-max within the result set: -``` +```text keyword_normalized = (score - min_score) / (max_score - min_score) ``` @@ -121,7 +121,7 @@ Semantic scores are already bounded after the distance-to-relevance conversion. If keyword and semantic score scales prove unstable under weighted blending, replace with RRF: -``` +```text rrf_score = Σ 1 / (k + rank_i) ``` @@ -131,15 +131,15 @@ where `k` is a constant (typically 60) and `rank_i` is the document's rank in ea All search modes support these filters, applied as SQL WHERE clauses: -| Filter | Parameter | SQL | -|--------|-----------|-----| -| Collection | `collection` | `d.collection = ?` | -| Author | `author` | `d.author_handle = ?` or `d.did = ?` | -| Repo | `repo` | `d.repo_name = ?` or `d.repo_did = ?` | -| Record type | `type` | `d.record_type = ?` | -| Language | `language` | `d.language = ?` | -| Date range | `from`, `to` | `d.created_at >= ?` and `d.created_at <= ?` | -| State | `state` | Join to `record_state` table | +| Filter | Parameter | SQL | +| ----------- | ------------ | ------------------------------------------- | +| Collection | `collection` | `d.collection = ?` | +| Author | `author` | `d.author_handle = ?` or `d.did = ?` | +| Repo | `repo` | `d.repo_name = ?` or `d.repo_did = ?` | +| Record type | `type` | `d.record_type = ?` | +| Language | `language` | `d.language = ?` | +| Date range | `from`, `to` | `d.created_at >= ?` and `d.created_at <= ?` | +| State | `state` | Join to `record_state` table | ## 6. Embedding Eligibility @@ -204,46 +204,52 @@ Admin endpoints are disabled by default. Enable with `ENABLE_ADMIN_ENDPOINTS=tru ```json { - "query": "rust markdown tui", - "mode": "hybrid", - "total": 142, - "limit": 20, - "offset": 0, - "results": [ - { - "id": "did:plc:abc|sh.tangled.repo|3kb3fge5lm32x", - "collection": "sh.tangled.repo", - "record_type": "repo", - "title": "glow-rs", - "body_snippet": "A TUI markdown viewer inspired by Glow...", - "summary": "Rust TUI markdown viewer", - "repo_name": "glow-rs", - "author_handle": "desertthunder.dev", - "score": 0.842, - "matched_by": ["keyword", "semantic"], - "created_at": "2026-03-20T10:00:00Z", - "updated_at": "2026-03-22T15:03:11Z" - } - ] + "query": "rust markdown tui", + "mode": "hybrid", + "total": 142, + "limit": 20, + "offset": 0, + "results": [ + { + "id": "did:plc:abc|sh.tangled.repo|3kb3fge5lm32x", + "collection": "sh.tangled.repo", + "record_type": "repo", + "title": "glow-rs", + "body_snippet": "A TUI markdown viewer inspired by Glow...", + "summary": "Rust TUI markdown viewer", + "repo_name": "glow-rs", + "author_handle": "desertthunder.dev", + "score": 0.842, + "matched_by": ["keyword", "semantic"], + "created_at": "2026-03-20T10:00:00Z", + "updated_at": "2026-03-22T15:03:11Z" + } + ] } ``` ### Result Fields -| Field | Type | Description | -| --------------- | -------- | --------------------------------------- | -| `id` | string | Document stable ID | -| `collection` | string | ATProto collection NSID | -| `record_type` | string | Normalized type label | -| `title` | string | Document title | -| `body_snippet` | string | Highlighted body excerpt | -| `summary` | string | Short description | -| `repo_name` | string | Repository name (if applicable) | -| `author_handle` | string | Author handle | -| `score` | float | Relevance score (0–1) | -| `matched_by` | string[] | Which search modes produced this result | -| `created_at` | string | ISO 8601 creation timestamp | -| `updated_at` | string | ISO 8601 last update timestamp | +| Field | Type | Description | +| ------------------ | -------- | ------------------------------------------- | +| `id` | string | Document stable ID | +| `collection` | string | ATProto collection NSID | +| `record_type` | string | Normalized type label | +| `title` | string | Document title | +| `body_snippet` | string | Highlighted body excerpt | +| `summary` | string | Short description | +| `repo_name` | string | Repository name (if applicable) | +| `author_handle` | string | Author handle | +| `did` | string | Author DID when available | +| `at_uri` | string | Canonical AT URI when available | +| `primary_language` | string | Primary language for repo results | +| `stars` | number | Indexed star count for repo results | +| `follower_count` | number | Indexed follower count for profile results | +| `following_count` | number | Indexed following count for profile results | +| `score` | float | Relevance score (0–1) | +| `matched_by` | string[] | Which search modes produced this result | +| `created_at` | string | ISO 8601 creation timestamp | +| `updated_at` | string | ISO 8601 last update timestamp | ## 10. Document Response @@ -251,24 +257,24 @@ Admin endpoints are disabled by default. Enable with `ENABLE_ADMIN_ENDPOINTS=tru ```json { - "id": "did:plc:abc|sh.tangled.repo|3kb3fge5lm32x", - "did": "did:plc:abc", - "collection": "sh.tangled.repo", - "rkey": "3kb3fge5lm32x", - "at_uri": "at://did:plc:abc/sh.tangled.repo/3kb3fge5lm32x", - "cid": "bafyreig...", - "record_type": "repo", - "title": "glow-rs", - "body": "A TUI markdown viewer inspired by Glow, written in Rust.", - "summary": "Rust TUI markdown viewer", - "repo_name": "glow-rs", - "author_handle": "desertthunder.dev", - "tags_json": "[\"rust\", \"tui\", \"markdown\"]", - "language": "en", - "created_at": "2026-03-20T10:00:00Z", - "updated_at": "2026-03-22T15:03:11Z", - "indexed_at": "2026-03-22T15:05:00Z", - "has_embedding": true + "id": "did:plc:abc|sh.tangled.repo|3kb3fge5lm32x", + "did": "did:plc:abc", + "collection": "sh.tangled.repo", + "rkey": "3kb3fge5lm32x", + "at_uri": "at://did:plc:abc/sh.tangled.repo/3kb3fge5lm32x", + "cid": "bafyreig...", + "record_type": "repo", + "title": "glow-rs", + "body": "A TUI markdown viewer inspired by Glow, written in Rust.", + "summary": "Rust TUI markdown viewer", + "repo_name": "glow-rs", + "author_handle": "desertthunder.dev", + "tags_json": "[\"rust\", \"tui\", \"markdown\"]", + "language": "en", + "created_at": "2026-03-20T10:00:00Z", + "updated_at": "2026-03-22T15:03:11Z", + "indexed_at": "2026-03-22T15:05:00Z", + "has_embedding": true } ``` @@ -281,10 +287,7 @@ Admin endpoints are disabled by default. Enable with `ENABLE_ADMIN_ENDPOINTS=tru | 503 | DB unreachable (readiness failure) | ```json -{ - "error": "invalid_parameter", - "message": "limit must be between 1 and 100" -} +{ "error": "invalid_parameter", "message": "limit must be between 1 and 100" } ``` ## 12. API Behavior @@ -294,3 +297,4 @@ Admin endpoints are disabled by default. Enable with `ENABLE_ADMIN_ENDPOINTS=tru - `hybrid` merges both result sets and reranks - All modes exclude documents with `deleted_at IS NOT NULL` by default - Pagination uses `limit`/`offset` (cursor-based pagination deferred) +- Mobile clients may use `type=repo` and `type=profile` to render repo/profile search directly diff --git a/packages/api/docs/specs/06-operations.md b/docs/api/specs/06-operations.md similarity index 77% rename from packages/api/docs/specs/06-operations.md rename to docs/api/specs/06-operations.md index e43b56d..ab039f6 100644 --- a/packages/api/docs/specs/06-operations.md +++ b/docs/api/specs/06-operations.md @@ -1,10 +1,24 @@ --- title: "Spec 06 — Operations" -updated: 2026-03-22 +updated: 2026-03-23 --- Covers configuration, observability, security, and deployment. +## 0. Quick Setup + +Tap is already deployed. For a new environment, the minimum operator work is: + +1. Create or choose a Turso database for that environment +2. Generate a Turso auth token for that database +3. Point `TURSO_DATABASE_URL` and `TURSO_AUTH_TOKEN` at that database +4. Create Railway services for `api` and `indexer` +5. Point `TAP_URL` at the existing Tap deployment +6. Run migrations/start the services +7. Run `twister backfill` before treating the environment as search-ready + +No separate `*_DEV` or `*_PROD` variables are required. Each environment keeps using the same variable names and simply points them at the appropriate Turso database. + ## 1. Configuration All configuration is via environment variables. @@ -85,6 +99,94 @@ LOG_LEVEL=info ENABLE_ADMIN_ENDPOINTS=false ``` +### Environment Selection + +Use the same variable names in every environment: + +- local development can point `TURSO_DATABASE_URL` and `TURSO_AUTH_TOKEN` at `twister-dev` +- production can point those same variables at `twister-prod` + +The application should not care which database it is talking to; only the environment wiring changes. + +## 1.5. Turso Setup + +### Recommended Databases + +Use one Turso database per environment, for example: + +- `twister-dev` +- `twister-prod` + +Keep the app config identical across environments and swap only these values: + +- `TURSO_DATABASE_URL` +- `TURSO_AUTH_TOKEN` + +### Basic Flow + +Using the Turso dashboard or CLI: + +1. Create the database for the target environment +2. Capture its libSQL URL +3. Create an auth token for the service +4. Set `TURSO_DATABASE_URL` and `TURSO_AUTH_TOKEN` in that environment + +Example values: + +```bash +# Development environment +TURSO_DATABASE_URL=libsql://twister-dev-your-org.turso.io +TURSO_AUTH_TOKEN=... + +# Production environment +TURSO_DATABASE_URL=libsql://twister-prod-your-org.turso.io +TURSO_AUTH_TOKEN=... +``` + +### Practical Rule + +Do not introduce `TURSO_DATABASE_URL_DEV`, `TURSO_DATABASE_URL_PROD`, or similar split variables. Railway environments, local shells, and CI should all set the same names with environment-specific values. + +## 1.6. Railway Setup + +### Project Layout + +Create or reuse one Railway project containing: + +- existing `tap` service +- `api` service running `twister api` +- `indexer` service running `twister indexer` + +### Basic Steps + +1. Connect the monorepo to Railway +2. Create the `api` and `indexer` services from the same source repo/Docker image +3. Set shared variables on both services: + - `TURSO_DATABASE_URL` + - `TURSO_AUTH_TOKEN` + - `LOG_LEVEL` + - `LOG_FORMAT` +4. Set API-specific variables: + - `HTTP_BIND_ADDR` + - `SEARCH_DEFAULT_LIMIT` + - `SEARCH_MAX_LIMIT` +5. Set indexer-specific variables: + - `TAP_URL` + - `TAP_AUTH_PASSWORD` + - `INDEXED_COLLECTIONS` +6. Configure health checks +7. Deploy +8. Run backfill against the environment before public validation + +### Dev vs Production on Railway + +If you use multiple Railway environments, keep the same service definitions and variable names in each one. Only the values change: + +- dev Railway environment -> `TURSO_DATABASE_URL=...twister-dev...` +- prod Railway environment -> `TURSO_DATABASE_URL=...twister-prod...` + +This keeps deployment logic simple and avoids conditional application config. + ## 2. Observability ### Structured Logging @@ -262,6 +364,16 @@ INDEXED_COLLECTIONS=sh.tangled.repo,sh.tangled.repo.issue,sh.tangled.repo.pull,s Railway supports referencing other services' variables with `${{service.VAR}}` syntax, which is useful for linking the indexer to Tap's domain. +#### First-Time Bootstrap Checklist + +After the first successful deploy of a new environment: + +1. Confirm API readiness on `/readyz` +2. Confirm indexer health and Tap connectivity +3. Run graph backfill with the environment's seed file +4. Wait for Tap historical sync to settle +5. Verify that search returns known historical repos/profiles + #### Health Checks Railway activates deployments based on health check responses. Configure per-service: diff --git a/packages/api/docs/specs/07-graph-backfill.md b/docs/api/specs/07-graph-backfill.md similarity index 80% rename from packages/api/docs/specs/07-graph-backfill.md rename to docs/api/specs/07-graph-backfill.md index bbb3f57..b88b7c3 100644 --- a/packages/api/docs/specs/07-graph-backfill.md +++ b/docs/api/specs/07-graph-backfill.md @@ -21,6 +21,7 @@ bob.tangled.sh ``` Format: + - One entry per line - Lines starting with `#` are comments - Blank lines are ignored @@ -49,6 +50,7 @@ Higher hop counts discover more users but increase time and may pull in loosely ### Crawl Queue Discovered DIDs are added to a queue, deduplicated by DID. Each entry tracks: + - DID - Discovery hop (distance from seed) - Source (which seed/user led to discovery) @@ -92,14 +94,14 @@ twister backfill --seeds seeds.txt --concurrency 5 ### Flags -| Flag | Default | Description | -|------|---------|-------------| -| `--seeds` | required | Path to seed file | -| `--max-hops` | `2` | Max fan-out depth from seed users | -| `--dry-run` | `false` | List discovered users without submitting to Tap | -| `--concurrency` | `5` | Parallel discovery workers | -| `--batch-size` | `10` | DIDs per `/repos/add` call | -| `--batch-delay` | `1s` | Delay between batches | +| Flag | Default | Description | +| --------------- | -------- | ----------------------------------------------- | +| `--seeds` | required | Path to seed file | +| `--max-hops` | `2` | Max fan-out depth from seed users | +| `--dry-run` | `false` | List discovered users without submitting to Tap | +| `--concurrency` | `5` | Parallel discovery workers | +| `--batch-size` | `10` | DIDs per `/repos/add` call | +| `--batch-delay` | `1s` | Delay between batches | ### Output @@ -127,11 +129,11 @@ The entire backfill process is safe to re-run: ## 8. Configuration -| Variable | Default | Description | -|----------|---------|-------------| -| `TAP_URL` | (existing) | Tap base URL for API calls | -| `TAP_AUTH_PASSWORD` | (existing) | Tap admin auth | +| Variable | Default | Description | +| -------------------- | ---------- | ----------------------------- | +| `TAP_URL` | (existing) | Tap base URL for API calls | +| `TAP_AUTH_PASSWORD` | (existing) | Tap admin auth | | `TURSO_DATABASE_URL` | (existing) | For checking existing records | -| `TURSO_AUTH_TOKEN` | (existing) | DB auth | +| `TURSO_AUTH_TOKEN` | (existing) | DB auth | No new environment variables are needed — backfill reuses existing Tap and DB configuration. diff --git a/docs/api/specs/08-app-integration.md b/docs/api/specs/08-app-integration.md new file mode 100644 index 0000000..60049b4 --- /dev/null +++ b/docs/api/specs/08-app-integration.md @@ -0,0 +1,89 @@ +--- +title: "Spec 08 — App Integration" +updated: 2026-03-23 +--- + +## 1. Purpose + +Define the mobile-facing Twister API surface. + +The Twisted app should keep using Tangled's public knot and PDS APIs for canonical repo/profile detail. Twister is responsible for: + +- cross-network discovery via search +- index-backed summaries for data gaps such as followers + +## 2. Client Boundary + +The mobile client uses Twister only for: + +- Explore search +- index-backed profile summaries +- future feed and notification features + +The mobile client does not use Twister for: + +- repo tree/blob/detail reads +- direct profile record reads +- issue/PR detail reads + +Those remain on Tangled's public APIs. + +## 3. Search Contract + +`GET /search` + +Required query parameters: + +- `q` + +Optional query parameters: + +- `mode=keyword|semantic|hybrid` +- `type=repo|profile` +- `limit` +- `offset` + +For mobile clients, repo and profile results should include: + +- `did` +- `at_uri` +- `record_type` +- `title` +- `summary` +- `repo_name` +- `author_handle` +- `updated_at` +- `primary_language` for repos when known +- `stars` for repos when known +- `follower_count` and `following_count` for profiles when known + +## 4. Profile Summary Contract + +`GET /profiles/{did}/summary` + +Response: + +```json +{ + "did": "did:plc:abc123", + "handle": "desertthunder.dev", + "follower_count": 128, + "following_count": 84, + "indexed_at": "2026-03-23T10:15:00Z" +} +``` + +This endpoint exists because follower counts and follower lists are derived from indexed graph state, not from a single direct public Tangled API call. + +## 5. Failure Handling + +If Twister is unavailable: + +- the app should keep direct known-handle browsing working +- Explore should show a clear "index unavailable" state +- profile pages should omit index-backed follower counts rather than fail entirely + +## 6. Ownership + +- Twister owns search ranking, document normalization, and graph summary derivation +- The app owns result presentation, route transitions, and fallback behavior diff --git a/packages/api/docs/specs/README.md b/docs/api/specs/README.md similarity index 73% rename from packages/api/docs/specs/README.md rename to docs/api/specs/README.md index c60ca15..3a4b804 100644 --- a/packages/api/docs/specs/README.md +++ b/docs/api/specs/README.md @@ -5,8 +5,8 @@ updated: 2026-03-22 # Twister Technical Specifications -Twister is a Go-based search service for [Tangled](https://tangled.org) content on AT Protocol. -It ingests records through [Tap](https://github.com/bluesky-social/indigo/tree/main/cmd/tap), denormalizes them into search documents, indexes them in [Turso/libSQL](https://docs.turso.tech), and exposes keyword, semantic, and hybrid search APIs. +Twister is a Go-based index and search service for [Tangled](https://tangled.org) content on AT Protocol. +It ingests records through [Tap](https://github.com/bluesky-social/indigo/tree/main/cmd/tap), denormalizes them into search documents and graph summaries, indexes them in [Turso/libSQL](https://docs.turso.tech), and exposes public APIs for search and index-backed data gaps. ## Specifications @@ -19,3 +19,4 @@ It ingests records through [Tap](https://github.com/bluesky-social/indigo/tree/m | 5 | [Search](05-search.md) | Search modes, API contract, scoring, filtering | | 6 | [Operations](06-operations.md) | Configuration, observability, security, deployment | | 7 | [Graph Backfill](07-graph-backfill.md) | Seed-based user discovery and content backfill | +| 8 | [App Integration](08-app-integration.md) | Mobile-facing contracts for search and graph summaries | diff --git a/docs/api/tasks/README.md b/docs/api/tasks/README.md new file mode 100644 index 0000000..fb301ae --- /dev/null +++ b/docs/api/tasks/README.md @@ -0,0 +1,40 @@ +--- +title: "Twister — Task Index" +updated: 2026-03-22 +--- + +# Twister Tasks + +Assumes Go, Tap (deployed on Railway), Turso/libSQL, and Railway for deployment. + +## Delivery Strategy + +Build in four phases: + +1. **MVP** — ingestion, graph backfill, keyword search, deployment, operational tooling +2. **Semantic Search** — embeddings, vector retrieval +3. **Hybrid Search** — weighted merge of keyword + semantic +4. **Quality Polish** — ranking refinement, advanced filters, analytics + +Ship keyword search before embeddings. That gives a testable, inspectable baseline before introducing model behavior. +Within MVP, run graph backfill before calling the environment search-ready for users. + +## Phases + +| Phase | Title | Document | Status | +| ----- | --------------- | ------------------------------------------ | --------------------------------------------------------------------- | +| 1 | MVP | [phase-1-mvp.md](phase-1-mvp.md) | In progress (M0–M2 complete; backfill scheduled before public launch) | +| 2 | Semantic Search | [phase-2-semantic.md](phase-2-semantic.md) | Not started | +| 3 | Hybrid Search | [phase-3-hybrid.md](phase-3-hybrid.md) | Not started | +| 4 | Quality Polish | [phase-4-quality.md](phase-4-quality.md) | Not started | + +## MVP Complete When + +- Tap ingests tracked `sh.tangled.*` records +- Documents normalize into a stable store +- Keyword search works publicly +- Index-backed profile summaries can fill public API gaps such as followers +- API and indexer are deployed on Railway +- Restart does not lose sync position +- Reindex exists for repair +- Graph backfill populates initial content from seed users diff --git a/packages/api/docs/tasks/phase-1-mvp.md b/docs/api/tasks/phase-1-mvp.md similarity index 80% rename from packages/api/docs/tasks/phase-1-mvp.md rename to docs/api/tasks/phase-1-mvp.md index bcb8c6b..fdfdccf 100644 --- a/packages/api/docs/tasks/phase-1-mvp.md +++ b/docs/api/tasks/phase-1-mvp.md @@ -17,30 +17,22 @@ Get a searchable product online: ingestion, keyword search, deployment, and oper - Reindex exists for repair - Graph backfill populates initial content from seed users ---- - ## M0 — Repository Bootstrap ✅ Executable layout, local tooling, and development conventions (completed 2026-03-22). ---- - ## M1 — Database Schema and Store Layer ✅ refs: [specs/03-data-model.md](../specs/03-data-model.md) Implemented the Turso/libSQL schema and Go store package for document persistence. ---- - ## M2 — Normalization Layer ✅ refs: [specs/02-tangled-lexicons.md](../specs/02-tangled-lexicons.md), [specs/04-data-pipeline.md](../specs/04-data-pipeline.md) Translate `sh.tangled.*` records into internal search documents. ---- - ## M3 — Tap Client and Ingestion Loop refs: [specs/04-data-pipeline.md](../specs/04-data-pipeline.md), [specs/01-architecture.md](../specs/01-architecture.md) @@ -49,10 +41,6 @@ refs: [specs/04-data-pipeline.md](../specs/04-data-pipeline.md), [specs/01-archi Connect the indexer to Tap (on Railway) and process live events into the store. -### Why Now - -Tap is the point of truth for synchronized ATProto ingestion. It is already deployed on Railway. - ### Deliverables - Tap WebSocket client package (`internal/tapclient/`) @@ -126,9 +114,84 @@ Tap is the point of truth for synchronized ATProto ingestion. It is already depl The system continuously ingests and persists `sh.tangled.*` records from Tap. ---- +## M4 — Graph Backfill from Seed Users + +refs: [specs/07-graph-backfill.md](../specs/07-graph-backfill.md) + +### Goal + +Bootstrap the index with historical Tangled content by discovering and backfilling users from a curated seed set. + +### Deliverables -## M4 — Keyword Search API +- `twister backfill` CLI command +- Seed file parser and documented seed-file format +- Graph fan-out discovery (follows and collaborators) +- Tap `/repos/add` integration for discovered users +- Deduplication against already-tracked repos +- Dry-run mode and progress logging +- Basic operator runbook for first bootstrap and repeat runs + +### Tasks + +- [ ] Implement `backfill` subcommand with flags: + - `--seeds ` — required seed file path + - `--max-hops ` — depth limit for fan-out (default: 2) + - `--dry-run` — print the discovery plan without mutating Tap + - `--concurrency ` — parallel discovery workers (default: 5) + - `--batch-size ` — DIDs per `/repos/add` request + - `--batch-delay ` — delay between Tap registration batches +- [ ] Implement seed file parsing: + - One DID or handle per line + - `#` comments allowed + - Blank lines ignored + - Handles resolved to DIDs before graph expansion +- [ ] Decide and document the initial seed file location for operators: + - Repository-managed example file for format/reference + - Deployment-specific runtime file or mounted secret for real runs +- [ ] Implement graph discovery: + 1. Start from hop-0 seed users + 2. Fetch `sh.tangled.graph.follow` records and collect subject DIDs + 3. Fetch repo collaborators by inspecting repos, issues, PRs, and comments + 4. Enqueue newly discovered DIDs with hop metadata + 5. Stop expanding beyond `max-hops` +- [ ] Track discovery metadata for logs: + - source DID + - hop depth + - discovery reason (`seed`, `follow`, `collaborator`) +- [ ] Integrate with Tap admin endpoints: + - `GET /info/:did` to skip already-tracked repos when practical + - `POST /repos/add` to register new DIDs for backfill +- [ ] Make the command safe to re-run: + - in-memory visited DID set during crawl + - tolerate duplicate `/repos/add` + - rely on index upsert idempotency for re-delivered records +- [ ] Add operator-friendly logging: + - seed count + - users discovered per hop + - already-tracked vs newly-submitted DIDs + - batch progress + - final totals +- [ ] Add a short runbook covering: + - first bootstrap against an empty database + - repeat run after expanding the seed list + - dry-run before production mutation + +### Verification + +- [ ] A small seed file of known Tangled users produces a non-empty discovery graph +- [ ] `--max-hops 1` limits discovery to direct neighbors +- [ ] `--dry-run` does not call Tap mutation endpoints +- [ ] Already-tracked DIDs are reported and not re-submitted unnecessarily +- [ ] Re-running the same seeds is effectively idempotent +- [ ] Newly submitted DIDs cause Tap to begin historical backfill +- [ ] Search results become materially richer after bootstrap than they were under live-only ingestion + +### Exit Criteria + +Operators can bootstrap an empty environment to a usable historical baseline before public rollout. + +## M5 — Keyword Search API refs: [specs/05-search.md](../specs/05-search.md) @@ -136,10 +199,6 @@ refs: [specs/05-search.md](../specs/05-search.md) Expose a usable public search API backed by Turso's Tantivy-backed FTS. -### Why Now - -First real product milestone. Searchable Tangled content without waiting for embeddings. - ### Deliverables - HTTP server (chi or net/http) @@ -200,9 +259,7 @@ First real product milestone. Searchable Tangled content without waiting for emb A user can search Tangled content reliably with keyword search. ---- - -## M5 — Railway Deployment +## M6 — Railway Deployment refs: [specs/06-operations.md](../specs/06-operations.md) @@ -210,10 +267,6 @@ refs: [specs/06-operations.md](../specs/06-operations.md) Deploy the API and indexer as Railway services alongside Tap. -### Why Now - -At this point, the product is useful enough to run continuously. - ### Deliverables - Finalized Dockerfile @@ -253,9 +306,7 @@ At this point, the product is useful enough to run continuously. The system runs as a deployed service with health-checked processes on Railway. ---- - -## M6 — Reindex and Repair +## M7 — Reindex and Repair refs: [specs/05-search.md](../specs/05-search.md) @@ -263,10 +314,6 @@ refs: [specs/05-search.md](../specs/05-search.md) Make the system recoverable and operable with repair tools. -### Why Now - -Search systems are never perfect on first ingestion. Repair tools are needed before production. - ### Deliverables - `twister reindex` command with scoping options @@ -304,9 +351,7 @@ Search systems are never perfect on first ingestion. Repair tools are needed bef Operators can repair bad indexes without rebuilding everything manually. ---- - -## M7 — Observability +## M8 — Observability refs: [specs/06-operations.md](../specs/06-operations.md) @@ -349,59 +394,3 @@ Make the system diagnosable in production. ### Exit Criteria The system is maintainable without guesswork. - ---- - -## M-New — Graph Backfill from Seed Users - -refs: [specs/07-graph-backfill.md](../specs/07-graph-backfill.md) - -### Goal - -Bootstrap the search index with existing Tangled content by discovering and backfilling users from a seed set. - -### Why Now - -Before MVP launch, the index needs existing content. Live ingestion only captures new events — backfill populates historical data. - -### Deliverables - -- `twister backfill` CLI command -- Seed file parser -- Graph fan-out discovery (follows/collaborators) -- Tap `/repos/add` integration for discovered users -- Deduplication against already-indexed users -- Progress logging - -### Tasks - -- [ ] Implement `backfill` subcommand with flags: - - `--seeds ` — path to seed file (one DID or handle per line) - - `--max-hops ` — depth limit for fan-out (default: 2) - - `--dry-run` — show discovered users without triggering backfill - - `--concurrency ` — parallel discovery workers (default: 5) -- [ ] Implement seed file parser (supports DIDs and handles, comments with `#`) -- [ ] Implement graph fan-out: - 1. For each seed user, resolve DID if handle provided - 2. Fetch `sh.tangled.graph.follow` records for the user - 3. Fetch collaborators from repos owned by the user - 4. Add discovered DIDs to the crawl queue - 5. Repeat up to `max-hops` depth -- [ ] Integrate with Tap `/repos/add` to register discovered DIDs for tracking -- [ ] Deduplicate: skip DIDs already tracked by Tap (check via `/info/:did`) -- [ ] Log progress: seeds processed, users discovered per hop, DIDs submitted to Tap -- [ ] Handle rate limiting and errors gracefully (retry with backoff) -- [ ] Make idempotent: safe to re-run; Tap handles duplicate `/repos/add` calls - -### Verification - -- [ ] Running with a seed file of 3 known users discovers their followers -- [ ] `--max-hops 1` limits discovery to direct connections only -- [ ] `--dry-run` lists discovered DIDs without calling Tap -- [ ] Already-tracked users are skipped -- [ ] Re-running the same seed file produces no duplicate work -- [ ] Tap begins backfilling records for newly added DIDs - -### Exit Criteria - -The index contains historical content from the seed user graph, not just new events. diff --git a/packages/api/docs/tasks/phase-2-semantic.md b/docs/api/tasks/phase-2-semantic.md similarity index 94% rename from packages/api/docs/tasks/phase-2-semantic.md rename to docs/api/tasks/phase-2-semantic.md index 6e39dcb..11bc72c 100644 --- a/packages/api/docs/tasks/phase-2-semantic.md +++ b/docs/api/tasks/phase-2-semantic.md @@ -7,8 +7,6 @@ updated: 2026-03-22 Add embedding generation and vector-based retrieval on top of the keyword baseline. ---- - ## M8 — Embedding Pipeline refs: [specs/03-data-model.md](../specs/03-data-model.md), [specs/05-search.md](../specs/05-search.md) @@ -17,10 +15,6 @@ refs: [specs/03-data-model.md](../specs/03-data-model.md), [specs/05-search.md]( Add asynchronous embedding generation without blocking ingestion. -### Why Now - -Only after keyword search is stable should semantic complexity be added. - ### Deliverables - `embedding_jobs` table operational (schema from M1) @@ -70,8 +64,6 @@ Only after keyword search is stable should semantic complexity be added. Embeddings are produced asynchronously and stored durably. ---- - ## M9 — Semantic Search refs: [specs/05-search.md](../specs/05-search.md) @@ -80,10 +72,6 @@ refs: [specs/05-search.md](../specs/05-search.md) Expose vector-based semantic retrieval. -### Why Now - -Natural next step once embeddings exist. Turso/libSQL has native vector search with `vector_top_k`. - ### Deliverables - `GET /search/semantic` endpoint diff --git a/packages/api/docs/tasks/phase-3-hybrid.md b/docs/api/tasks/phase-3-hybrid.md similarity index 99% rename from packages/api/docs/tasks/phase-3-hybrid.md rename to docs/api/tasks/phase-3-hybrid.md index e6d47c0..eb21220 100644 --- a/packages/api/docs/tasks/phase-3-hybrid.md +++ b/docs/api/tasks/phase-3-hybrid.md @@ -7,8 +7,6 @@ updated: 2026-03-22 Merge lexical and semantic search into the default high-quality retrieval mode. ---- - ## M10 — Hybrid Search refs: [specs/05-search.md](../specs/05-search.md) diff --git a/packages/api/docs/tasks/phase-4-quality.md b/docs/api/tasks/phase-4-quality.md similarity index 99% rename from packages/api/docs/tasks/phase-4-quality.md rename to docs/api/tasks/phase-4-quality.md index 0f2dc26..1a97ca1 100644 --- a/packages/api/docs/tasks/phase-4-quality.md +++ b/docs/api/tasks/phase-4-quality.md @@ -7,8 +7,6 @@ updated: 2026-03-22 Improve search quality without changing the core architecture. ---- - ## M11 — Ranking and Quality Polish refs: [specs/05-search.md](../specs/05-search.md) diff --git a/apps/twisted/docs/specs/README.md b/docs/app/specs/README.md similarity index 73% rename from apps/twisted/docs/specs/README.md rename to docs/app/specs/README.md index 13ef541..17b9fde 100644 --- a/apps/twisted/docs/specs/README.md +++ b/docs/app/specs/README.md @@ -12,11 +12,12 @@ A mobile-first Tangled client for iOS, Android, and web. Built with Ionic Vue, C ## What Twisted Does -**Reader and social companion** for Tangled. Focused on browsing, known-handle lookup, and lightweight interactions. +**Reader and social companion** for Tangled. Focused on direct browsing, indexed discovery, and lightweight interactions. - Browse repos, files, READMEs, issues, PRs - Jump to profiles and repos from a known AT Protocol handle -- Explore and Activity placeholders until search/feed work lands +- Search indexed repos and profiles through the Twister API +- Use index-backed graph summaries where the public API is incomplete - Sign in via AT Protocol OAuth - Star repos, follow users, react to content - Offline-capable with cached data @@ -40,20 +41,21 @@ Three layers, strict dependency direction (presentation → domain → data): **Presentation** — Ionic pages, Vue components, composables, Pinia stores. **Domain** — Normalized models (`UserSummary`, `RepoDetail`, `ActivityItem`, etc.), action policies, pagination. -**Data** — `@atcute/client` XRPC calls, `@atcute/tangled` type definitions, local cache, optional BFF. +**Data** — `@atcute/client` XRPC calls, `@atcute/tangled` type definitions, local cache, and the Twister API for search/index-backed summaries. Protocol isolation: no Vue component imports `@atcute/*` directly. All API access flows through `src/services/`. ## Tangled API Surface -Two distinct API hosts: +Three distinct data hosts: | Host | Protocol | Data | | ---------------------------------- | ---------------------------------- | ----------------------------------------------------------------- | | Knots (`us-west.tangled.sh`, etc.) | XRPC at `/xrpc/sh.tangled.*` | Git data: trees, blobs, commits, branches, diffs, tags | | User's PDS | XRPC at `/xrpc/com.atproto.repo.*` | AT Protocol records: repos, issues, PRs, stars, follows, profiles | +| Twister API | HTTP JSON | Global search and index-backed graph/profile summaries | -The appview (`tangled.org`) serves HTML — it's the web UI, not a JSON API. The mobile client talks to knots and PDS servers directly. +The appview (`tangled.org`) serves HTML — it's the web UI, not a JSON API. The mobile client talks to knots and PDS servers directly for canonical detail and uses the Twister API for cross-network discovery. Repo param format: `did:plc:xxx/repoName`. @@ -61,18 +63,18 @@ Repo param format: `did:plc:xxx/repoName`. | Phase | Focus | Spec | Tasks | | ----- | ------------------------------------------------------------------------ | ------------------------------------ | ------------------------------------ | -| 1 | Project shell, tabs, mock data, design system | [specs/phase-1.md](specs/phase-1.md) | [tasks/phase-1.md](tasks/phase-1.md) | -| 2 | Public browsing — repos, files, profiles, issues, PRs | [specs/phase-2.md](specs/phase-2.md) | [tasks/phase-2.md](tasks/phase-2.md) | -| 3 | Deferred search/feed placeholders and Home-first public browsing | [specs/phase-3.md](specs/phase-3.md) | [tasks/phase-3.md](tasks/phase-3.md) | -| 4 | OAuth sign-in, star, follow, react, personalized feed | [specs/phase-4.md](specs/phase-4.md) | [tasks/phase-4.md](tasks/phase-4.md) | -| 5 | Offline persistence, performance, bundle optimization | [specs/phase-5.md](specs/phase-5.md) | [tasks/phase-5.md](tasks/phase-5.md) | -| 6 | Write features (issues, comments, profile edit), BFF, push notifications | [specs/phase-6.md](specs/phase-6.md) | [tasks/phase-6.md](tasks/phase-6.md) | -| 7 | Real-time Jetstream feed, custom feeds, forking, labels, interdiff | [specs/phase-7.md](specs/phase-7.md) | [tasks/phase-7.md](tasks/phase-7.md) | +| 1 | Project shell, tabs, mock data, design system | [phase-1.md](phase-1.md) | [../tasks/phase-1.md](../tasks/phase-1.md) | +| 2 | Public browsing — repos, files, profiles, issues, PRs | [phase-2.md](phase-2.md) | [../tasks/phase-2.md](../tasks/phase-2.md) | +| 3 | Index-backed search and handle-first public browsing | [phase-3.md](phase-3.md) | [../tasks/phase-3.md](../tasks/phase-3.md) | +| 4 | OAuth sign-in, star, follow, react, personalized feed | [phase-4.md](phase-4.md) | [../tasks/phase-4.md](../tasks/phase-4.md) | +| 5 | Offline persistence, performance, bundle optimization | [phase-5.md](phase-5.md) | [../tasks/phase-5.md](../tasks/phase-5.md) | +| 6 | Write features, project service integration, push notifications | [phase-6.md](phase-6.md) | [../tasks/phase-6.md](../tasks/phase-6.md) | +| 7 | Real-time Jetstream feed, custom feeds, forking, labels, interdiff | [phase-7.md](phase-7.md) | [../tasks/phase-7.md](../tasks/phase-7.md) | ## Key Design Decisions 1. **`@atcute` end-to-end** for all AT Protocol interaction — no mixing client stacks. 2. **Tangled lexicon handling in one module boundary** (`src/services/tangled/`) — don't scatter `sh.tangled.*` awareness across pages. 3. **Read-first** — the primary product is a fast reader. Social mutations are a controlled second layer. -4. **Thin BFF when needed** (Phase 6+) for search indexing, personalized feeds, push notifications, and unstable procedure wrapping. +4. **Use the project API sparingly and intentionally.** Search and index-backed graph gaps belong there; canonical repo detail stays on Tangled's public APIs. 5. **Mobile-first, not desktop-forge-first** — prioritize readability, direct browsing, and small focused actions before broader discovery surfaces. diff --git a/apps/twisted/docs/specs/phase-1.md b/docs/app/specs/phase-1.md similarity index 100% rename from apps/twisted/docs/specs/phase-1.md rename to docs/app/specs/phase-1.md diff --git a/apps/twisted/docs/specs/phase-2.md b/docs/app/specs/phase-2.md similarity index 100% rename from apps/twisted/docs/specs/phase-2.md rename to docs/app/specs/phase-2.md diff --git a/docs/app/specs/phase-3.md b/docs/app/specs/phase-3.md new file mode 100644 index 0000000..cc5183b --- /dev/null +++ b/docs/app/specs/phase-3.md @@ -0,0 +1,68 @@ +# Phase 3 — Indexed Search and Honest Discovery + +## Goal + +Introduce global discovery through the Twister project index while preserving honest product boundaries. Home continues to support direct known-handle browsing, Explore becomes index-backed search, and Activity remains a clearly labeled in-progress surface. + +## Current Product Shape + +### Home + +Home is the temporary public entry point for unauthenticated browsing: + +- Enter a known AT Protocol handle +- Open that user's profile directly +- Resolve the handle to DID + PDS via AT Protocol identity +- List that user's public Tangled repos inline and open one directly + +This keeps public browsing fully real while still giving the app a lightweight direct-entry path. + +### Explore + +Explore becomes the network-level discovery surface: + +- Global repo search via the Twister index +- Global profile search via the Twister index +- Empty state should clearly distinguish "index unavailable" from "no results" +- Search results route into the existing profile and repo detail screens + +### Activity + +Activity also remains a tab-level placeholder: + +- No public timeline yet +- No curated public feed fallback +- Empty state should explicitly say activity is in progress + +## Identity and Routing + +The app now uses two read paths: + +1. **Direct handle browsing** + Resolve `handle -> DID` via `com.atproto.identity.resolveHandle` + Fetch the DID document and extract the PDS endpoint + Query the user's PDS for `sh.tangled.repo` records via `com.atproto.repo.listRecords` +2. **Indexed discovery** + Query the Twister API for global search results + Open the selected profile or repo in the existing screens + Continue detail fetching from Tangled's public APIs + +The Twister API is additive, not authoritative for repo detail. It fills discovery and graph gaps; knots and PDSes remain the source of truth for detail screens. + +## UI Expectations + +- Home shows one handle input plus explicit actions for profile jump and repo browsing +- Home shows loading, invalid-handle, no-repos, and resolved-repo-list states +- Explore shows a working search form, loading state, index-unavailable state, and no-results state +- Activity shows a static in-progress empty state +- Profile may show index-backed follower/following summaries when available + +## Deferred Work + +The following work is intentionally deferred out of this phase: + +- Trending or suggested discovery sections +- Public activity feed ingestion, pagination, and caching +- Jetstream or appview timeline investigation + +These capabilities will be revisited after the baseline search and graph-summary integration is stable. diff --git a/apps/twisted/docs/specs/phase-4.md b/docs/app/specs/phase-4.md similarity index 100% rename from apps/twisted/docs/specs/phase-4.md rename to docs/app/specs/phase-4.md diff --git a/apps/twisted/docs/specs/phase-5.md b/docs/app/specs/phase-5.md similarity index 100% rename from apps/twisted/docs/specs/phase-5.md rename to docs/app/specs/phase-5.md diff --git a/apps/twisted/docs/specs/phase-6.md b/docs/app/specs/phase-6.md similarity index 79% rename from apps/twisted/docs/specs/phase-6.md rename to docs/app/specs/phase-6.md index 160c82d..3cfc6ca 100644 --- a/apps/twisted/docs/specs/phase-6.md +++ b/docs/app/specs/phase-6.md @@ -1,10 +1,10 @@ -# Phase 6 — Write Features & Backend Adapter +# Phase 6 — Write Features & Project Services ## Goal -Add authenticated write operations (create issues, comment on PRs/issues, edit profile) and introduce a thin backend (BFF) for operations that don't work well from a pure SPA. +Add authenticated write operations (create issues, comment on PRs/issues, edit profile) and extend the Twister project services only where the client should not or cannot do the work directly. -## Why a Backend +## Why Project Services Some operations are awkward or unsafe from a browser client: @@ -12,18 +12,18 @@ Some operations are awkward or unsafe from a browser client: - **Unstable procedures**: Tangled's API may change — a backend adapter isolates the mobile client from churn - **Push notifications**: require server-side registration and delivery - **Personalized feeds**: server-side aggregation is more efficient than client-side filtering -- **Search**: if no public JSON search API exists, the backend can index and serve search results +- **Graph gaps**: follower lists/counts and other cross-network summaries may require index-backed derivation - **Rate limiting**: backend can batch and deduplicate requests -### BFF Scope +### Service Scope -Thin adapter — not a full backend. Proxies and transforms Tangled/AT Protocol calls. +Thin service layer — not a replacement for Tangled's public APIs. Use it for cross-network aggregation, search, notifications, and operations the SPA should not own. | Endpoint | Purpose | | ------------------------------------ | --------------------------------------------------- | | `POST /auth/session` | OAuth token exchange and session management | | `GET /feed/personalized` | Pre-filtered activity feed for the user | -| `GET /search/repos`, `/search/users` | Search proxy/index | +| `GET /search`, `GET /profiles/:did/summary` | Search and index-backed graph/profile summaries | | `POST /notifications/register` | Push notification device registration | | Passthrough for stable XRPC calls | Avoid duplicating what the client already does well | @@ -56,8 +56,8 @@ Thin adapter — not a full backend. Proxies and transforms Tangled/AT Protocol ## Push Notifications -- Register device token with BFF -- BFF subscribes to Jetstream for the user's relevant events +- Register device token with project services +- Project services subscribe to Jetstream or indexed events relevant to the user - Deliver via APNs (iOS) / FCM (Android) - Notification types: PR activity on your repos, issue comments, new followers, stars diff --git a/apps/twisted/docs/specs/phase-7.md b/docs/app/specs/phase-7.md similarity index 94% rename from apps/twisted/docs/specs/phase-7.md rename to docs/app/specs/phase-7.md index 2342e30..df62314 100644 --- a/apps/twisted/docs/specs/phase-7.md +++ b/docs/app/specs/phase-7.md @@ -44,7 +44,7 @@ Allow users to create saved feed configurations: - "Team" — activity from users I follow - Custom filters: by repo, by user, by event type -Feeds are stored locally in IndexedDB. If a BFF exists, they can optionally sync server-side for push notification filtering. +Feeds are stored locally in IndexedDB. If project services exist, they can optionally sync server-side for push notification filtering. ## Advanced Features diff --git a/apps/twisted/docs/tasks/phase-1.md b/docs/app/tasks/phase-1.md similarity index 100% rename from apps/twisted/docs/tasks/phase-1.md rename to docs/app/tasks/phase-1.md diff --git a/apps/twisted/docs/tasks/phase-2.md b/docs/app/tasks/phase-2.md similarity index 100% rename from apps/twisted/docs/tasks/phase-2.md rename to docs/app/tasks/phase-2.md diff --git a/apps/twisted/docs/tasks/phase-3.md b/docs/app/tasks/phase-3.md similarity index 100% rename from apps/twisted/docs/tasks/phase-3.md rename to docs/app/tasks/phase-3.md diff --git a/apps/twisted/docs/tasks/phase-4.md b/docs/app/tasks/phase-4.md similarity index 100% rename from apps/twisted/docs/tasks/phase-4.md rename to docs/app/tasks/phase-4.md diff --git a/apps/twisted/docs/tasks/phase-5.md b/docs/app/tasks/phase-5.md similarity index 100% rename from apps/twisted/docs/tasks/phase-5.md rename to docs/app/tasks/phase-5.md diff --git a/apps/twisted/docs/tasks/phase-6.md b/docs/app/tasks/phase-6.md similarity index 67% rename from apps/twisted/docs/tasks/phase-6.md rename to docs/app/tasks/phase-6.md index 3696dcb..10a34b6 100644 --- a/apps/twisted/docs/tasks/phase-6.md +++ b/docs/app/tasks/phase-6.md @@ -1,31 +1,30 @@ -# Phase 6 Tasks — Write Features & Backend Adapter +# Phase 6 Tasks — Write Features & Project Services -## Backend (BFF) Setup +## Project Services Setup -- [ ] Choose runtime (Node/Deno/Bun) and framework (Hono/Fastify/Express) -- [ ] Scaffold BFF project with TypeScript -- [ ] Implement health check endpoint -- [ ] Deploy to hosting (Fly.io, Railway, etc.) +- [ ] Decide which write and notification operations belong in `packages/api` versus a separate service +- [ ] Implement health and readiness endpoints for all public client-facing services - [ ] Configure CORS for the mobile app's origins +- [ ] Document the mobile-facing service contract in `docs/api` -## Backend — Auth Proxy +## Project Services — Auth Proxy - [ ] Implement OAuth token exchange endpoint (if moving auth server-side) - [ ] Implement session endpoint that returns user info -- [ ] Decide: keep client-side OAuth or migrate to BFF-mediated auth +- [ ] Decide: keep client-side OAuth or migrate to service-mediated auth -## Backend — Search +## Project Services — Search and Graph -- [ ] Implement search indexer: subscribe to Jetstream, index `sh.tangled.repo` and `sh.tangled.actor.profile` records -- [ ] Implement `GET /search/repos?q=` endpoint -- [ ] Implement `GET /search/users?q=` endpoint -- [ ] Wire mobile client's search service to BFF endpoints +- [ ] Implement `GET /search` endpoint for repo/profile discovery +- [ ] Return enough repo/profile metadata for the mobile client to render result cards directly +- [ ] Implement `GET /profiles/:did/summary` for follower/following counts and other graph-derived gaps +- [ ] Wire mobile client's search and profile summary services to these endpoints -## Backend — Personalized Feed +## Project Services — Personalized Feed - [ ] Implement `GET /feed/personalized` — aggregate activity for the user's follows and stars - [ ] Index relevant events from Jetstream -- [ ] Wire mobile client's feed to BFF endpoint when authenticated +- [ ] Wire mobile client's feed to the project service endpoint when authenticated ## Create Issue @@ -69,12 +68,12 @@ - [ ] Prompt user to re-authorize with expanded scopes - [ ] Handle scope upgrade flow gracefully (no data loss) -## Push Notifications (if BFF exists) +## Push Notifications (if services exist) -- [ ] Implement `POST /notifications/register` on BFF — register device token +- [ ] Implement `POST /notifications/register` — register device token - [ ] Configure Capacitor Push Notifications plugin - [ ] Register device token on login -- [ ] BFF: subscribe to Jetstream events relevant to user, deliver via APNs/FCM +- [ ] Services: subscribe to events relevant to user, deliver via APNs/FCM - [ ] Handle notification tap → deep link to relevant content ## Quality diff --git a/apps/twisted/docs/tasks/phase-7.md b/docs/app/tasks/phase-7.md similarity index 100% rename from apps/twisted/docs/tasks/phase-7.md rename to docs/app/tasks/phase-7.md diff --git a/justfile b/justfile index 6d9e0f3..2c7929a 100644 --- a/justfile +++ b/justfile @@ -41,7 +41,7 @@ app-cap-android: api-build: just --justfile packages/api/justfile build -api-run-api: +api-dev: just --justfile packages/api/justfile run-api api-run-indexer: diff --git a/packages/api/README.md b/packages/api/README.md index 9c729b4..b3dd019 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -1,3 +1,3 @@ # Twister -Tap-based search engine for Tangled. +Tap-based indexing and search service for Tangled. diff --git a/packages/api/docs/tasks/README.md b/packages/api/docs/tasks/README.md deleted file mode 100644 index 1a73837..0000000 --- a/packages/api/docs/tasks/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: "Twister — Task Index" -updated: 2026-03-22 ---- - -# Twister Tasks - -Assumes Go, Tap (deployed on Railway), Turso/libSQL, and Railway for deployment. - -## Delivery Strategy - -Build in four phases: - -1. **MVP** — ingestion, keyword search, deployment, operational tooling, graph backfill -2. **Semantic Search** — embeddings, vector retrieval -3. **Hybrid Search** — weighted merge of keyword + semantic -4. **Quality Polish** — ranking refinement, advanced filters, analytics - -Ship keyword search before embeddings. That gives a testable, inspectable baseline before introducing model behavior. - -## Phases - -| Phase | Title | Document | Status | -| ----- | ----- | -------- | ------ | -| 1 | MVP | [phase-1-mvp.md](phase-1-mvp.md) | In progress (M0–M2 complete) | -| 2 | Semantic Search | [phase-2-semantic.md](phase-2-semantic.md) | Not started | -| 3 | Hybrid Search | [phase-3-hybrid.md](phase-3-hybrid.md) | Not started | -| 4 | Quality Polish | [phase-4-quality.md](phase-4-quality.md) | Not started | - -## MVP Complete When - -- Tap ingests tracked `sh.tangled.*` records -- Documents normalize into a stable store -- Keyword search works publicly -- API and indexer are deployed on Railway -- Restart does not lose sync position -- Reindex exists for repair -- Graph backfill populates initial content from seed users -- 2.51.2