From 8b90cf78ef9c64cc437f6eade539c2e66cf44d82 Mon Sep 17 00:00:00 2001 From: Wesley Finck Date: Wed, 14 Jan 2026 20:47:41 -0800 Subject: [PATCH] feat: add SearchLeafletDocsForUrlUseCase and supporting infrastructure Co-authored-by: aider (anthropic/claude-sonnet-4-20250514) --- .../SearchLeafletDocsForUrlUseCase.ts | 80 ++++++ .../domain/services/LeafletSearchService.ts | 257 ++++++++++++++++++ .../SearchLeafletDocsForUrlController.ts | 37 +++ .../http/routes/searchRoutes.ts | 7 + .../http/factories/ControllerFactory.ts | 5 + .../http/factories/ServiceFactory.ts | 1 + .../http/factories/UseCaseFactory.ts | 5 + src/types/src/api/requests.ts | 6 + src/types/src/api/responses.ts | 6 + src/webapp/api-client/ApiClient.ts | 8 + src/webapp/api-client/clients/QueryClient.ts | 15 + 11 files changed, 427 insertions(+) create mode 100644 src/modules/search/application/use-cases/SearchLeafletDocsForUrlUseCase.ts create mode 100644 src/modules/search/domain/services/LeafletSearchService.ts create mode 100644 src/modules/search/infrastructure/http/controllers/SearchLeafletDocsForUrlController.ts diff --git a/src/modules/search/application/use-cases/SearchLeafletDocsForUrlUseCase.ts b/src/modules/search/application/use-cases/SearchLeafletDocsForUrlUseCase.ts new file mode 100644 index 00000000..66739c09 --- /dev/null +++ b/src/modules/search/application/use-cases/SearchLeafletDocsForUrlUseCase.ts @@ -0,0 +1,80 @@ +import { UseCase } from 'src/shared/core/UseCase'; +import { Result, err, ok } from 'src/shared/core/Result'; +import { AppError } from 'src/shared/core/AppError'; +import { LeafletSearchService } from '../../domain/services/LeafletSearchService'; +import { + SearchLeafletDocsForUrlParams, + SearchLeafletDocsForUrlResponse, +} from '@semble/types'; + +export interface SearchLeafletDocsForUrlRequest + extends SearchLeafletDocsForUrlParams { + userDid?: string; // Optional - for future authenticated features +} + +export type SearchLeafletDocsForUrlResult = Result< + SearchLeafletDocsForUrlResponse, + AppError.UnexpectedError +>; + +export class SearchLeafletDocsForUrlUseCase + implements + UseCase< + SearchLeafletDocsForUrlRequest, + Promise + > +{ + constructor(private leafletSearchService: LeafletSearchService) {} + + async execute( + request: SearchLeafletDocsForUrlRequest, + ): Promise { + try { + if (!request.url || typeof request.url !== 'string') { + return err( + new AppError.UnexpectedError( + new Error('URL parameter is required'), + ), + ); + } + + const searchResult = await this.leafletSearchService.searchLeafletDocsForUrl( + request.url, + request.limit, + request.cursor, + ); + + if (searchResult.isErr()) { + return err(searchResult.error); + } + + const documents = searchResult.value; + + // Transform to UrlView format to match other search endpoints + const urls = documents.map((doc) => ({ + url: doc.url, + metadata: { + url: doc.metadata.url, + title: doc.metadata.title, + description: doc.metadata.description, + author: doc.metadata.author, + publishedDate: doc.metadata.publishedDate?.toISOString(), + siteName: doc.metadata.siteName, + imageUrl: doc.metadata.imageUrl, + type: doc.metadata.type?.toString(), + retrievedAt: doc.metadata.retrievedAt?.toISOString(), + }, + urlLibraryCount: 0, // We don't have this data from Leaflet + urlInLibrary: false, // We don't have this data without user context + })); + + return ok({ + urls, + cursor: null, // Constellation API doesn't seem to use cursors in the example + total: documents.length, + }); + } catch (error: any) { + return err(new AppError.UnexpectedError(error)); + } + } +} diff --git a/src/modules/search/domain/services/LeafletSearchService.ts b/src/modules/search/domain/services/LeafletSearchService.ts new file mode 100644 index 00000000..01cf585e --- /dev/null +++ b/src/modules/search/domain/services/LeafletSearchService.ts @@ -0,0 +1,257 @@ +import { Result, ok, err } from '../../../../shared/core/Result'; +import { AppError } from '../../../../shared/core/AppError'; +import { IAgentService } from '../../../atproto/application/IAgentService'; +import { IMetadataService } from '../../../cards/domain/services/IMetadataService'; +import { UrlMetadata } from '../../../cards/domain/value-objects/UrlMetadata'; + +export interface LeafletLinkingRecord { + did: string; + collection: string; + rkey: string; +} + +export interface LeafletLinksResponse { + total: number; + linking_records: LeafletLinkingRecord[]; + cursor: string | null; +} + +export interface LeafletDocumentRecord { + $type: string; + title: string; + author: string; + description?: string; + publication: string; + publishedAt: string; + pages: any[]; +} + +export interface LeafletPublicationRecord { + $type: string; + name: string; + base_path: string; + description?: string; + preferences?: any; +} + +export interface LeafletDocumentResult { + url: string; + metadata: UrlMetadata; +} + +export class LeafletSearchService { + private readonly CONSTELLATION_BASE_URL = 'https://constellation.microcosm.blue'; + private readonly ATPROTO_XRPC_BASE_URL = 'https://bsky.social/xrpc'; + + constructor( + private agentService: IAgentService, + private metadataService: IMetadataService, + ) {} + + async searchLeafletDocsForUrl( + targetUrl: string, + limit?: number, + cursor?: string, + ): Promise> { + try { + // Step 1: Get backlinks from Constellation + const backlinksResult = await this.getBacklinksFromConstellation( + targetUrl, + limit, + cursor, + ); + if (backlinksResult.isErr()) { + return err(backlinksResult.error); + } + + const backlinks = backlinksResult.value; + const results: LeafletDocumentResult[] = []; + + // Step 2: Process each linking record + for (const record of backlinks.linking_records) { + try { + const documentResult = await this.processLeafletDocument(record); + if (documentResult.isOk()) { + results.push(documentResult.value); + } + } catch (error) { + // Continue processing other records if one fails + console.warn( + `Failed to process leaflet document ${record.did}/${record.rkey}:`, + error, + ); + } + } + + return ok(results); + } catch (error: any) { + return err(new AppError.UnexpectedError(error)); + } + } + + private async getBacklinksFromConstellation( + targetUrl: string, + limit?: number, + cursor?: string, + ): Promise> { + try { + const params = new URLSearchParams({ + target: targetUrl, + collection: 'pub.leaflet.document', + path: '.pages[pub.leaflet.pages.linearDocument].blocks[pub.leaflet.pages.linearDocument#block].block.facets[].features[pub.leaflet.richtext.facet#link].uri', + }); + + if (limit) params.set('limit', limit.toString()); + if (cursor) params.set('cursor', cursor); + + const response = await fetch( + `${this.CONSTELLATION_BASE_URL}/links?${params}`, + ); + + if (!response.ok) { + return err( + new AppError.UnexpectedError( + new Error(`Constellation API error: ${response.statusText}`), + ), + ); + } + + const data: LeafletLinksResponse = await response.json(); + return ok(data); + } catch (error: any) { + return err(new AppError.UnexpectedError(error)); + } + } + + private async processLeafletDocument( + record: LeafletLinkingRecord, + ): Promise> { + try { + // Step 1: Get the document record + const documentResult = await this.getLeafletDocumentRecord( + record.did, + record.rkey, + ); + if (documentResult.isErr()) { + return err(documentResult.error); + } + + const document = documentResult.value; + + // Step 2: Get the publication record + const publicationAtUri = document.publication; + const publicationResult = await this.getLeafletPublicationFromAtUri( + publicationAtUri, + ); + if (publicationResult.isErr()) { + return err(publicationResult.error); + } + + const publication = publicationResult.value; + + // Step 3: Construct the URL + const documentUrl = `https://${publication.base_path}/${record.rkey}`; + + // Step 4: Fetch metadata + const metadataResult = await this.metadataService.getUrlMetadata( + documentUrl, + ); + if (metadataResult.isErr()) { + // If metadata fetch fails, create basic metadata from document + const basicMetadata = UrlMetadata.create({ + url: documentUrl, + title: document.title, + description: document.description, + author: document.author, + publishedDate: new Date(document.publishedAt), + siteName: publication.name, + }); + + if (basicMetadata.isErr()) { + return err(new AppError.UnexpectedError(basicMetadata.error)); + } + + return ok({ + url: documentUrl, + metadata: basicMetadata.value, + }); + } + + return ok({ + url: documentUrl, + metadata: metadataResult.value, + }); + } catch (error: any) { + return err(new AppError.UnexpectedError(error)); + } + } + + private async getLeafletDocumentRecord( + did: string, + rkey: string, + ): Promise> { + try { + const params = new URLSearchParams({ + repo: did, + collection: 'pub.leaflet.document', + rkey: rkey, + }); + + const response = await fetch( + `${this.ATPROTO_XRPC_BASE_URL}/com.atproto.repo.getRecord?${params}`, + ); + + if (!response.ok) { + return err( + new AppError.UnexpectedError( + new Error(`ATProto API error: ${response.statusText}`), + ), + ); + } + + const data = await response.json(); + return ok(data.value as LeafletDocumentRecord); + } catch (error: any) { + return err(new AppError.UnexpectedError(error)); + } + } + + private async getLeafletPublicationFromAtUri( + atUri: string, + ): Promise> { + try { + // Parse AT URI: at://did:plc:6z5botgrc5vekq7j26xnvawq/pub.leaflet.publication/3ly4c4cmyn22t + const uriParts = atUri.replace('at://', '').split('/'); + if (uriParts.length !== 3) { + return err( + new AppError.UnexpectedError(new Error(`Invalid AT URI: ${atUri}`)), + ); + } + + const [did, collection, rkey] = uriParts; + + const params = new URLSearchParams({ + repo: did, + collection: collection, + rkey: rkey, + }); + + const response = await fetch( + `${this.ATPROTO_XRPC_BASE_URL}/com.atproto.repo.getRecord?${params}`, + ); + + if (!response.ok) { + return err( + new AppError.UnexpectedError( + new Error(`ATProto API error: ${response.statusText}`), + ), + ); + } + + const data = await response.json(); + return ok(data.value as LeafletPublicationRecord); + } catch (error: any) { + return err(new AppError.UnexpectedError(error)); + } + } +} diff --git a/src/modules/search/infrastructure/http/controllers/SearchLeafletDocsForUrlController.ts b/src/modules/search/infrastructure/http/controllers/SearchLeafletDocsForUrlController.ts new file mode 100644 index 00000000..08e7fb93 --- /dev/null +++ b/src/modules/search/infrastructure/http/controllers/SearchLeafletDocsForUrlController.ts @@ -0,0 +1,37 @@ +import { Controller } from '../../../../../shared/infrastructure/http/Controller'; +import { Request, Response } from 'express'; +import { SearchLeafletDocsForUrlUseCase } from '../../../application/use-cases/SearchLeafletDocsForUrlUseCase'; +import { AuthenticatedRequest } from '../../../../../shared/infrastructure/http/middleware/AuthMiddleware'; + +export class SearchLeafletDocsForUrlController extends Controller { + constructor( + private searchLeafletDocsForUrlUseCase: SearchLeafletDocsForUrlUseCase, + ) { + super(); + } + + async executeImpl(req: AuthenticatedRequest, res: Response): Promise { + try { + const { url, limit, cursor } = req.query; + + if (!url || typeof url !== 'string') { + return this.badRequest(res, 'Query parameter "url" is required'); + } + + const result = await this.searchLeafletDocsForUrlUseCase.execute({ + url, + limit: limit ? parseInt(limit as string, 10) : undefined, + cursor: cursor as string | undefined, + userDid: req.did, // This will be undefined if not authenticated + }); + + if (result.isErr()) { + return this.fail(res, result.error.message); + } + + return this.ok(res, result.value); + } catch (error: any) { + return this.fail(res, error.message || 'Unknown error'); + } + } +} diff --git a/src/modules/search/infrastructure/http/routes/searchRoutes.ts b/src/modules/search/infrastructure/http/routes/searchRoutes.ts index 68c8e7c5..8da4ec55 100644 --- a/src/modules/search/infrastructure/http/routes/searchRoutes.ts +++ b/src/modules/search/infrastructure/http/routes/searchRoutes.ts @@ -3,6 +3,7 @@ import { GetSimilarUrlsForUrlController } from '../controllers/GetSimilarUrlsFor import { SearchBskyPostsForUrlController } from '../controllers/SearchBskyPostsForUrlController'; import { SemanticSearchUrlsController } from '../controllers/SemanticSearchUrlsController'; import { SearchAtProtoAccountsController } from '../controllers/SearchAtProtoAccountsController'; +import { SearchLeafletDocsForUrlController } from '../controllers/SearchLeafletDocsForUrlController'; import { AuthMiddleware } from '../../../../../shared/infrastructure/http/middleware/AuthMiddleware'; export function createSearchRoutes( @@ -11,6 +12,7 @@ export function createSearchRoutes( searchBskyPostsForUrlController: SearchBskyPostsForUrlController, semanticSearchUrlsController: SemanticSearchUrlsController, searchAtProtoAccountsController: SearchAtProtoAccountsController, + searchLeafletDocsForUrlController: SearchLeafletDocsForUrlController, ): Router { const router = Router(); @@ -34,5 +36,10 @@ export function createSearchRoutes( searchAtProtoAccountsController.execute(req, res), ); + // GET /api/search/leaflet-docs - Search Leaflet documents that link to a URL + router.get('/leaflet-docs', authMiddleware.optionalAuth(), (req, res) => + searchLeafletDocsForUrlController.execute(req, res), + ); + return router; } diff --git a/src/shared/infrastructure/http/factories/ControllerFactory.ts b/src/shared/infrastructure/http/factories/ControllerFactory.ts index 17361879..f4191763 100644 --- a/src/shared/infrastructure/http/factories/ControllerFactory.ts +++ b/src/shared/infrastructure/http/factories/ControllerFactory.ts @@ -24,6 +24,7 @@ import { GetSimilarUrlsForUrlController } from '../../../../modules/search/infra import { SemanticSearchUrlsController } from '../../../../modules/search/infrastructure/http/controllers/SemanticSearchUrlsController'; import { SearchBskyPostsForUrlController } from '../../../../modules/search/infrastructure/http/controllers/SearchBskyPostsForUrlController'; import { SearchAtProtoAccountsController } from '../../../../modules/search/infrastructure/http/controllers/SearchAtProtoAccountsController'; +import { SearchLeafletDocsForUrlController } from '../../../../modules/search/infrastructure/http/controllers/SearchLeafletDocsForUrlController'; import { UseCases } from './UseCaseFactory'; import { GetMyProfileController } from 'src/modules/cards/infrastructure/http/controllers/GetMyProfileController'; import { GetUserProfileController } from 'src/modules/cards/infrastructure/http/controllers/GetUserProfileController'; @@ -86,6 +87,7 @@ export interface Controllers { semanticSearchUrlsController: SemanticSearchUrlsController; searchBskyPostsForUrlController: SearchBskyPostsForUrlController; searchAtProtoAccountsController: SearchAtProtoAccountsController; + searchLeafletDocsForUrlController: SearchLeafletDocsForUrlController; // Notification controllers getMyNotificationsController: GetMyNotificationsController; getUnreadNotificationCountController: GetUnreadNotificationCountController; @@ -224,6 +226,9 @@ export class ControllerFactory { searchAtProtoAccountsController: new SearchAtProtoAccountsController( useCases.searchAtProtoAccountsUseCase, ), + searchLeafletDocsForUrlController: new SearchLeafletDocsForUrlController( + useCases.searchLeafletDocsForUrlUseCase, + ), // Notification controllers getMyNotificationsController: new GetMyNotificationsController( useCases.getMyNotificationsUseCase, diff --git a/src/shared/infrastructure/http/factories/ServiceFactory.ts b/src/shared/infrastructure/http/factories/ServiceFactory.ts index c411f7cf..b5929c61 100644 --- a/src/shared/infrastructure/http/factories/ServiceFactory.ts +++ b/src/shared/infrastructure/http/factories/ServiceFactory.ts @@ -417,3 +417,4 @@ export class ServiceFactory { }; } } +import { LeafletSearchService } from '../../../../modules/search/domain/services/LeafletSearchService'; diff --git a/src/shared/infrastructure/http/factories/UseCaseFactory.ts b/src/shared/infrastructure/http/factories/UseCaseFactory.ts index 47745ece..6dfea485 100644 --- a/src/shared/infrastructure/http/factories/UseCaseFactory.ts +++ b/src/shared/infrastructure/http/factories/UseCaseFactory.ts @@ -37,6 +37,7 @@ import { GetSimilarUrlsForUrlUseCase } from '../../../../modules/search/applicat import { SemanticSearchUrlsUseCase } from '../../../../modules/search/application/useCases/queries/SemanticSearchUrlsUseCase'; import { SearchBskyPostsForUrlUseCase } from '../../../../modules/search/application/use-cases/SearchBskyPostsForUrlUseCase'; import { SearchAtProtoAccountsUseCase } from '../../../../modules/search/application/use-cases/SearchAtProtoAccountsUseCase'; +import { SearchLeafletDocsForUrlUseCase } from '../../../../modules/search/application/use-cases/SearchLeafletDocsForUrlUseCase'; import { ProcessCardFirehoseEventUseCase } from '../../../../modules/atproto/application/useCases/ProcessCardFirehoseEventUseCase'; import { ProcessCollectionFirehoseEventUseCase } from '../../../../modules/atproto/application/useCases/ProcessCollectionFirehoseEventUseCase'; import { ProcessCollectionLinkFirehoseEventUseCase } from '../../../../modules/atproto/application/useCases/ProcessCollectionLinkFirehoseEventUseCase'; @@ -103,6 +104,7 @@ export interface UseCases { semanticSearchUrlsUseCase: SemanticSearchUrlsUseCase; searchBskyPostsForUrlUseCase: SearchBskyPostsForUrlUseCase; searchAtProtoAccountsUseCase: SearchAtProtoAccountsUseCase; + searchLeafletDocsForUrlUseCase: SearchLeafletDocsForUrlUseCase; // Notification use cases getMyNotificationsUseCase: GetMyNotificationsUseCase; getUnreadNotificationCountUseCase: GetUnreadNotificationCountUseCase; @@ -290,6 +292,9 @@ export class UseCaseFactory { searchAtProtoAccountsUseCase: new SearchAtProtoAccountsUseCase( services.atProtoAgentService, ), + searchLeafletDocsForUrlUseCase: new SearchLeafletDocsForUrlUseCase( + services.leafletSearchService, + ), // Notification use cases getMyNotificationsUseCase: new GetMyNotificationsUseCase( repositories.notificationRepository, diff --git a/src/types/src/api/requests.ts b/src/types/src/api/requests.ts index 0c717670..d9c086aa 100644 --- a/src/types/src/api/requests.ts +++ b/src/types/src/api/requests.ts @@ -195,6 +195,12 @@ export interface SearchAtProtoAccountsParams { cursor?: string; } +export interface SearchLeafletDocsForUrlParams { + url: string; + limit?: number; + cursor?: string; +} + export interface GetGemActivityFeedParams extends PaginationParams { // Removed beforeActivityId since we're using page-based pagination urlType?: UrlType; // Filter by URL type diff --git a/src/types/src/api/responses.ts b/src/types/src/api/responses.ts index 8ab11f7d..cfc5013b 100644 --- a/src/types/src/api/responses.ts +++ b/src/types/src/api/responses.ts @@ -319,6 +319,12 @@ export interface SearchAtProtoAccountsResponse { actors: ProfileView[]; } +export interface SearchLeafletDocsForUrlResponse { + urls: UrlView[]; + cursor?: string; + total: number; +} + // Notification types export enum NotificationType { USER_ADDED_YOUR_CARD = 'USER_ADDED_YOUR_CARD', diff --git a/src/webapp/api-client/ApiClient.ts b/src/webapp/api-client/ApiClient.ts index cac17b6b..e79c78dc 100644 --- a/src/webapp/api-client/ApiClient.ts +++ b/src/webapp/api-client/ApiClient.ts @@ -71,6 +71,8 @@ import type { SearchBskyPostsForUrlResponse, SearchAtProtoAccountsParams, SearchAtProtoAccountsResponse, + SearchLeafletDocsForUrlParams, + SearchLeafletDocsForUrlResponse, GetMyNotificationsParams, GetMyNotificationsResponse, GetUnreadNotificationCountResponse, @@ -208,6 +210,12 @@ export class ApiClient { return this.queryClient.searchAtProtoAccounts(params); } + async searchLeafletDocs( + params: SearchLeafletDocsForUrlParams, + ): Promise { + return this.queryClient.searchLeafletDocs(params); + } + // Card operations - delegate to CardClient async addUrlToLibrary( request: AddUrlToLibraryRequest, diff --git a/src/webapp/api-client/clients/QueryClient.ts b/src/webapp/api-client/clients/QueryClient.ts index 104e3438..3b7a7fb4 100644 --- a/src/webapp/api-client/clients/QueryClient.ts +++ b/src/webapp/api-client/clients/QueryClient.ts @@ -30,6 +30,8 @@ import { SearchBskyPostsForUrlResponse, SearchAtProtoAccountsParams, SearchAtProtoAccountsResponse, + SearchLeafletDocsForUrlParams, + SearchLeafletDocsForUrlResponse, } from '@semble/types'; export class QueryClient extends BaseClient { @@ -308,4 +310,17 @@ export class QueryClient extends BaseClient { `/api/search/accounts?${searchParams}`, ); } + + async searchLeafletDocs( + params: SearchLeafletDocsForUrlParams, + ): Promise { + const searchParams = new URLSearchParams({ url: params.url }); + if (params.limit) searchParams.set('limit', params.limit.toString()); + if (params.cursor) searchParams.set('cursor', params.cursor); + + return this.request( + 'GET', + `/api/search/leaflet-docs?${searchParams}`, + ); + } } -- 2.51.2