diff --git a/src/modules/cards/application/useCases/queries/GetGraphDataUseCase.ts b/src/modules/cards/application/useCases/queries/GetGraphDataUseCase.ts index 7c52ac38..1b747640 100644 --- a/src/modules/cards/application/useCases/queries/GetGraphDataUseCase.ts +++ b/src/modules/cards/application/useCases/queries/GetGraphDataUseCase.ts @@ -3,7 +3,8 @@ import { UseCase } from '../../../../../shared/core/UseCase'; import { IGraphQueryRepository } from '../../../domain/IGraphQueryRepository'; export interface GetGraphDataQuery { - // No parameters needed for global graph + page?: number; + limit?: number; } export interface GraphNode { @@ -30,6 +31,7 @@ export interface GraphEdge { export interface GetGraphDataResult { nodes: GraphNode[]; edges: GraphEdge[]; + totalNodeCount: number; } export class GetGraphDataUseCase @@ -39,12 +41,16 @@ export class GetGraphDataUseCase async execute(query: GetGraphDataQuery): Promise> { try { - // Fetch all graph data - const graphData = await this.graphQueryRepo.getGraphData(); + // Fetch graph data with pagination + const graphData = await this.graphQueryRepo.getGraphData( + query.page, + query.limit, + ); return ok({ nodes: graphData.nodes, edges: graphData.edges, + totalNodeCount: graphData.totalNodeCount, }); } catch (error) { return err( diff --git a/src/modules/cards/domain/IGraphQueryRepository.ts b/src/modules/cards/domain/IGraphQueryRepository.ts index 862af42b..9f0b732e 100644 --- a/src/modules/cards/domain/IGraphQueryRepository.ts +++ b/src/modules/cards/domain/IGraphQueryRepository.ts @@ -23,12 +23,16 @@ export interface GraphEdgeDTO { export interface GraphDataDTO { nodes: GraphNodeDTO[]; edges: GraphEdgeDTO[]; + totalNodeCount: number; } export interface IGraphQueryRepository { /** - * Get all nodes and edges for the global graph visualization - * Returns the complete graph structure with all relationships + * Get nodes and edges for the global graph visualization with pagination support + * Returns paginated graph data with total count for calculating pagination metadata + * + * @param page - Page number (1-indexed, defaults to 1) + * @param limit - Number of nodes per page (defaults to 300) */ - getGraphData(): Promise; + getGraphData(page?: number, limit?: number): Promise; } diff --git a/src/modules/cards/infrastructure/http/controllers/GetGraphDataController.ts b/src/modules/cards/infrastructure/http/controllers/GetGraphDataController.ts index bc716ffe..9ca68d7b 100644 --- a/src/modules/cards/infrastructure/http/controllers/GetGraphDataController.ts +++ b/src/modules/cards/infrastructure/http/controllers/GetGraphDataController.ts @@ -10,13 +10,37 @@ export class GetGraphDataController extends Controller { async executeImpl(req: AuthenticatedRequest, res: Response): Promise { try { - const result = await this.getGraphDataUseCase.execute({}); + // Parse pagination parameters from query string + const page = req.query.page ? parseInt(req.query.page as string, 10) : 1; + const limit = req.query.limit + ? parseInt(req.query.limit as string, 10) + : 300; + + const result = await this.getGraphDataUseCase.execute({ page, limit }); if (result.isErr()) { return this.fail(res, result.error); } - return this.ok(res, result.value); + // Calculate pagination metadata + const totalCount = result.value.totalNodeCount; + const totalPages = Math.ceil(totalCount / limit); + const hasMore = page < totalPages; + + // Build response with pagination + const response = { + nodes: result.value.nodes, + edges: result.value.edges, + pagination: { + currentPage: page, + totalPages, + totalCount, + hasMore, + limit, + }, + }; + + return this.ok(res, response); } catch (error: any) { return this.handleError(res, error); } diff --git a/src/modules/cards/infrastructure/repositories/DrizzleGraphQueryRepository.ts b/src/modules/cards/infrastructure/repositories/DrizzleGraphQueryRepository.ts index 5dbb46b7..63e3ffc7 100644 --- a/src/modules/cards/infrastructure/repositories/DrizzleGraphQueryRepository.ts +++ b/src/modules/cards/infrastructure/repositories/DrizzleGraphQueryRepository.ts @@ -12,7 +12,7 @@ export class DrizzleGraphQueryRepository implements IGraphQueryRepository { this.graphQueryService = new GraphQueryService(db); } - async getGraphData(): Promise { - return this.graphQueryService.getGraphData(); + async getGraphData(page?: number, limit?: number): Promise { + return this.graphQueryService.getGraphData(page, limit); } } diff --git a/src/modules/cards/infrastructure/repositories/query-services/GraphQueryService.ts b/src/modules/cards/infrastructure/repositories/query-services/GraphQueryService.ts index 1fa35e00..28516097 100644 --- a/src/modules/cards/infrastructure/repositories/query-services/GraphQueryService.ts +++ b/src/modules/cards/infrastructure/repositories/query-services/GraphQueryService.ts @@ -14,7 +14,10 @@ import { users } from '../../../../user/infrastructure/repositories/schema/user. export class GraphQueryService { constructor(private db: PostgresJsDatabase) {} - async getGraphData(): Promise { + async getGraphData( + page: number = 1, + limit: number = 300, + ): Promise { // Fetch all data in parallel const [ userNodes, @@ -40,10 +43,24 @@ export class GraphQueryService { this.getUrlConnectionEdges(), ]); - // Combine all nodes and edges - const nodes = [...userNodes, ...urlNodes, ...collectionNodes, ...noteNodes]; + // Combine all nodes + const allNodes = [ + ...userNodes, + ...urlNodes, + ...collectionNodes, + ...noteNodes, + ]; + const totalNodeCount = allNodes.length; + + // Apply pagination to nodes + const offset = (page - 1) * limit; + const paginatedNodes = allNodes.slice(offset, offset + limit); - const edges = [ + // Create a Set of loaded node IDs for efficient lookup + const loadedNodeIds = new Set(paginatedNodes.map((node) => node.id)); + + // Filter edges to only include those where BOTH source and target are in loaded nodes + const allEdges = [ ...userFollowEdges, ...collectionFollowEdges, ...authorshipEdges, @@ -52,7 +69,16 @@ export class GraphQueryService { ...urlConnectionEdges, ]; - return { nodes, edges }; + const filteredEdges = allEdges.filter( + (edge) => + loadedNodeIds.has(edge.source) && loadedNodeIds.has(edge.target), + ); + + return { + nodes: paginatedNodes, + edges: filteredEdges, + totalNodeCount, + }; } private async getUserNodes(): Promise { diff --git a/src/modules/cards/tests/utils/InMemoryGraphQueryRepository.ts b/src/modules/cards/tests/utils/InMemoryGraphQueryRepository.ts index ca4374f9..b5e4df69 100644 --- a/src/modules/cards/tests/utils/InMemoryGraphQueryRepository.ts +++ b/src/modules/cards/tests/utils/InMemoryGraphQueryRepository.ts @@ -16,9 +16,10 @@ export class InMemoryGraphQueryRepository implements IGraphQueryRepository { return InMemoryGraphQueryRepository.instance; } - async getGraphData(): Promise { + async getGraphData(page?: number, limit?: number): Promise { // For in-memory implementation, return empty graph // In a real test scenario, you would populate this with test data - return { nodes: [], edges: [] }; + // and apply pagination + return { nodes: [], edges: [], totalNodeCount: 0 }; } } diff --git a/src/types/src/api/requests.ts b/src/types/src/api/requests.ts index 1fa8cd24..ec955426 100644 --- a/src/types/src/api/requests.ts +++ b/src/types/src/api/requests.ts @@ -357,6 +357,6 @@ export interface GetConnectionsParams extends PaginatedSortedParams { } // Graph request types -export interface GetGraphDataParams { - // No parameters needed for global graph in v1 +export interface GetGraphDataParams extends PaginationParams { + // Supports pagination for incremental graph loading } diff --git a/src/types/src/api/responses.ts b/src/types/src/api/responses.ts index b7ad6856..d3ec223c 100644 --- a/src/types/src/api/responses.ts +++ b/src/types/src/api/responses.ts @@ -543,4 +543,5 @@ export interface GraphEdge { export interface GetGraphDataResponse { nodes: GraphNode[]; edges: GraphEdge[]; + pagination: Pagination; } diff --git a/src/webapp/api-client/ApiClient.ts b/src/webapp/api-client/ApiClient.ts index bf03393e..88fcdeb1 100644 --- a/src/webapp/api-client/ApiClient.ts +++ b/src/webapp/api-client/ApiClient.ts @@ -117,6 +117,7 @@ import type { SearchUrlsParams, SearchUrlsResponse, // Graph types + GetGraphDataParams, GetGraphDataResponse, } from '@semble/types'; @@ -516,8 +517,10 @@ export class ApiClient { } // Graph operations - async getGraphData(): Promise { - return this.queryClient.getGraphData(); + async getGraphData( + params?: GetGraphDataParams, + ): Promise { + return this.queryClient.getGraphData(params); } } diff --git a/src/webapp/api-client/clients/QueryClient.ts b/src/webapp/api-client/clients/QueryClient.ts index ad7d4667..74c8512c 100644 --- a/src/webapp/api-client/clients/QueryClient.ts +++ b/src/webapp/api-client/clients/QueryClient.ts @@ -558,8 +558,8 @@ export class QueryClient extends BaseClient { ): Promise { // Check if mock data should be used (for performance testing) // Set NEXT_PUBLIC_USE_MOCK_GRAPH_DATA=true in .env.local to enable - const useMockData = process.env.NEXT_PUBLIC_USE_MOCK_GRAPH_DATA === 'true'; - // const useMockData = true; + // const useMockData = process.env.NEXT_PUBLIC_USE_MOCK_GRAPH_DATA === 'true'; + const useMockData = true; if (useMockData) { const { generateMockGraphData, MOCK_GRAPH_PRESETS } = await import( @@ -573,9 +573,43 @@ export class QueryClient extends BaseClient { // Simulate network delay for realistic testing await new Promise((resolve) => setTimeout(resolve, 500)); - return mockData; + // Apply pagination to mock data + const page = params?.page || 1; + const limit = params?.limit || 300; + const offset = (page - 1) * limit; + const totalCount = mockData.nodes.length; + const totalPages = Math.ceil(totalCount / limit); + const hasMore = page < totalPages; + + const paginatedNodes = mockData.nodes.slice(offset, offset + limit); + const loadedNodeIds = new Set(paginatedNodes.map((n) => n.id)); + const filteredEdges = mockData.edges.filter( + (e) => loadedNodeIds.has(e.source) && loadedNodeIds.has(e.target), + ); + + return { + nodes: paginatedNodes, + edges: filteredEdges, + pagination: { + currentPage: page, + totalPages, + totalCount, + hasMore, + limit, + }, + }; } - return this.request('GET', '/api/graph/data'); + // Build query string with pagination parameters + const searchParams = new URLSearchParams(); + if (params?.page) searchParams.set('page', params.page.toString()); + if (params?.limit) searchParams.set('limit', params.limit.toString()); + + const queryString = searchParams.toString(); + const endpoint = queryString + ? `/api/graph/data?${queryString}` + : '/api/graph/data'; + + return this.request('GET', endpoint); } } diff --git a/src/webapp/api-client/clients/mockGraphData.ts b/src/webapp/api-client/clients/mockGraphData.ts index 41bef1c9..02d96a1d 100644 --- a/src/webapp/api-client/clients/mockGraphData.ts +++ b/src/webapp/api-client/clients/mockGraphData.ts @@ -112,7 +112,18 @@ export function generateMockGraphData( `Generated mock graph data: ${nodes.length} nodes, ${edges.length} edges`, ); - return { nodes, edges }; + // Return with pagination metadata (mock assumes single page with all data) + return { + nodes, + edges, + pagination: { + currentPage: 1, + totalPages: 1, + totalCount: nodes.length, + hasMore: false, + limit: nodes.length, + }, + }; } /** diff --git a/src/webapp/app/(dashboard)/explore/(feeds)/(main)/page.tsx b/src/webapp/app/(dashboard)/explore/(feeds)/(main)/page.tsx index 478dbc8b..b81b8a26 100644 --- a/src/webapp/app/(dashboard)/explore/(feeds)/(main)/page.tsx +++ b/src/webapp/app/(dashboard)/explore/(feeds)/(main)/page.tsx @@ -9,7 +9,9 @@ interface Props { export default async function Page(props: Props) { const params = await props.searchParams; const key = new URLSearchParams( - Object.entries(params).filter((entry): entry is [string, string] => typeof entry[1] === 'string'), + Object.entries(params).filter( + (entry): entry is [string, string] => typeof entry[1] === 'string', + ), ).toString(); return ( diff --git a/src/webapp/features/graph/lib/dal.ts b/src/webapp/features/graph/lib/dal.ts index 942916f4..4ae5c651 100644 --- a/src/webapp/features/graph/lib/dal.ts +++ b/src/webapp/features/graph/lib/dal.ts @@ -3,16 +3,32 @@ import { createSembleClient } from '@/services/client.apiClient'; import { verifySessionOnClient } from '@/lib/auth/dal'; /** - * Fetch graph data from the backend - * Returns all nodes and edges for the current user's knowledge graph + * Fetch a specific page of graph data from the backend + * Returns nodes and edges for the specified page with pagination metadata + * + * @param page - Page number (1-indexed, defaults to 1) + * @param limit - Number of nodes per page (defaults to 300) */ -export const getGraphData = cache(async () => { - // Verify authentication - graph data is personalized - const session = await verifySessionOnClient({ redirectOnFail: true }); - if (!session) throw new Error('No session found'); +export const getGraphDataPage = cache( + async (page: number = 1, limit: number = 300) => { + // Verify authentication - graph data is personalized + const session = await verifySessionOnClient({ redirectOnFail: true }); + if (!session) throw new Error('No session found'); - const client = createSembleClient(); - const response = await client.getGraphData(); + const client = createSembleClient(); + const response = await client.getGraphData({ page, limit }); - return response; + return response; + }, +); + +/** + * Fetch all graph data from the backend (deprecated - use getGraphDataPage for better performance) + * Returns all nodes and edges for the current user's knowledge graph + * + * @deprecated Use getGraphDataPage instead for incremental loading + */ +export const getGraphData = cache(async () => { + // For backward compatibility, fetch page 1 with a large limit + return getGraphDataPage(1, 10000); }); diff --git a/src/webapp/features/graph/lib/graphKeys.ts b/src/webapp/features/graph/lib/graphKeys.ts index 4e9e5d5f..e9e1ed4a 100644 --- a/src/webapp/features/graph/lib/graphKeys.ts +++ b/src/webapp/features/graph/lib/graphKeys.ts @@ -5,5 +5,6 @@ export const graphKeys = { all: () => ['graph'] as const, data: () => [...graphKeys.all(), 'data'] as const, + page: (page: number) => [...graphKeys.all(), 'page', page] as const, node: (nodeId: string) => [...graphKeys.all(), 'node', nodeId] as const, } as const; diff --git a/src/webapp/features/graph/lib/queries/useGraphData.tsx b/src/webapp/features/graph/lib/queries/useGraphData.tsx index b68fde85..b65619d3 100644 --- a/src/webapp/features/graph/lib/queries/useGraphData.tsx +++ b/src/webapp/features/graph/lib/queries/useGraphData.tsx @@ -1,83 +1,183 @@ -import { useSuspenseQuery } from '@tanstack/react-query'; -import { useMemo } from 'react'; +import { useQueries } from '@tanstack/react-query'; +import { useState, useEffect, useRef } from 'react'; import { graphKeys } from '../graphKeys'; -import { getGraphData } from '../dal'; +import { getGraphDataPage } from '../dal'; import type { ProcessedGraphData, ExtendedGraphNode, ExtendedGraphEdge, } from '../../types'; import { calculateNodeSize, getNodeColor } from '../utils/nodeStyles'; +import type { GetGraphDataResponse } from '@semble/types'; /** - * Hook to fetch and process graph data - * Automatically calculates connection counts, node sizes, and colors + * Hook to fetch and process graph data with incremental loading + * Automatically loads data in pages and progressively renders + * Calculates connection counts, node sizes, and colors for all loaded data */ export default function useGraphData() { - // Fetch raw data from backend - const query = useSuspenseQuery({ - queryKey: graphKeys.data(), - queryFn: getGraphData, - staleTime: 5 * 60 * 1000, // 5 minutes (graph data doesn't change frequently) - refetchOnWindowFocus: false, // Don't refetch on focus (expensive operation) + const [pagesToLoad, setPagesToLoad] = useState([1]); + const [totalPages, setTotalPages] = useState(null); + + // Store processed data in state for incremental updates + const [processedData, setProcessedData] = useState({ + nodes: [], + links: [], }); - // Process the data to add visual properties - const processedData: ProcessedGraphData | undefined = useMemo(() => { - if (!query.data) return undefined; + // Track which pages we've already processed to avoid reprocessing + const processedPagesRef = useRef>(new Set()); + + // Maintain stable node references (important for smooth graph transitions) + const nodeMapRef = useRef>(new Map()); + + // Fetch pages in parallel (React Query will dedupe and cache) + const queries = useQueries({ + queries: pagesToLoad.map((page) => ({ + queryKey: graphKeys.page(page), + queryFn: () => getGraphDataPage(page), + staleTime: 5 * 60 * 1000, // 5 minutes + refetchOnWindowFocus: false, + })), + }); - // Step 1: Calculate connection counts - const connectionCounts: Record = {}; - query.data.edges.forEach((edge) => { - const sourceId = - typeof edge.source === 'string' ? edge.source : edge.source; - const targetId = - typeof edge.target === 'string' ? edge.target : edge.target; + // Incrementally merge new pages as they complete + useEffect(() => { + const successfulQueries = queries.filter((q) => q.isSuccess && q.data); + if (successfulQueries.length === 0) return; - connectionCounts[sourceId] = (connectionCounts[sourceId] || 0) + 1; - connectionCounts[targetId] = (connectionCounts[targetId] || 0) + 1; + // Find pages we haven't processed yet + const newPages: { page: number; data: GetGraphDataResponse }[] = []; + successfulQueries.forEach((q, idx) => { + const page = pagesToLoad[idx]; + if (!processedPagesRef.current.has(page)) { + newPages.push({ page, data: q.data as GetGraphDataResponse }); + } }); - // Step 2: Process nodes with visual properties - const processedNodes: ExtendedGraphNode[] = query.data.nodes.map((node) => { - const connectionCount = connectionCounts[node.id] || 0; - const nodeSize = calculateNodeSize(connectionCount); - const nodeColor = getNodeColor(node.type); + // Update pagination metadata and queue next page + const latestData = successfulQueries[successfulQueries.length - 1] + .data as GetGraphDataResponse; + + if (latestData?.pagination) { + setTotalPages(latestData.pagination.totalPages); + + // Queue next page if available + if ( + latestData.pagination.hasMore && + !pagesToLoad.includes(latestData.pagination.currentPage + 1) + ) { + setPagesToLoad((prev) => [ + ...prev, + latestData.pagination.currentPage + 1, + ]); + } + } + + // If no new pages, nothing to merge + if (newPages.length === 0) return; + + // Mark pages as processed + newPages.forEach(({ page }) => processedPagesRef.current.add(page)); + + // Incrementally merge new data + setProcessedData((prevData) => { + // Extract new nodes and edges from newly loaded pages + const newNodes = newPages.flatMap(({ data }) => data.nodes); + const newEdges = newPages.flatMap(({ data }) => data.edges); + + // Add new nodes to map (preserving existing node references) + newNodes.forEach((node) => { + if (!nodeMapRef.current.has(node.id)) { + // Create new extended node + const extendedNode: ExtendedGraphNode = { + ...node, + connectionCount: 0, + val: 0, + color: getNodeColor(node.type), + }; + nodeMapRef.current.set(node.id, extendedNode); + } + }); + + // Combine all edges (previous + new) + const allEdges = [...prevData.links, ...newEdges]; + + // Recalculate connection counts for ALL nodes (new edges affect existing nodes) + const connectionCounts: Record = {}; + allEdges.forEach((edge) => { + // Handle both string IDs and object references (ForceGraph2D mutates edges) + const sourceId = + typeof edge.source === 'string' + ? edge.source + : (edge.source as ExtendedGraphNode).id; + const targetId = + typeof edge.target === 'string' + ? edge.target + : (edge.target as ExtendedGraphNode).id; + + connectionCounts[sourceId] = (connectionCounts[sourceId] || 0) + 1; + connectionCounts[targetId] = (connectionCounts[targetId] || 0) + 1; + }); + + // Update connection counts on existing node objects (mutate in place) + nodeMapRef.current.forEach((node) => { + const connectionCount = connectionCounts[node.id] || 0; + node.connectionCount = connectionCount; + node.val = calculateNodeSize(connectionCount); + // color stays the same + }); + + // Get all nodes as array (stable references for existing nodes) + const allNodes = Array.from(nodeMapRef.current.values()); + + // Create a set of valid node IDs for edge validation + const validNodeIds = new Set(allNodes.map((n) => n.id)); + + // Process edges (add value for thickness if needed) + // Filter out edges that reference non-existent nodes + const processedEdges: ExtendedGraphEdge[] = allEdges + .filter((edge) => { + const sourceId = + typeof edge.source === 'string' + ? edge.source + : (edge.source as ExtendedGraphNode).id; + const targetId = + typeof edge.target === 'string' + ? edge.target + : (edge.target as ExtendedGraphNode).id; + return validNodeIds.has(sourceId) && validNodeIds.has(targetId); + }) + .map((edge) => ({ + ...edge, + value: 1, // Default thickness, can be customized based on edge type + })); return { - ...node, - connectionCount, - val: nodeSize, - color: nodeColor, + nodes: allNodes, + links: processedEdges, }; }); + }, [queries, pagesToLoad]); - // Create a set of valid node IDs for edge validation - const validNodeIds = new Set(processedNodes.map((node) => node.id)); + // Aggregate query state + const isLoading = queries.some((q) => q.isLoading); + const isError = queries.some((q) => q.isError); + const error = queries.find((q) => q.error)?.error; - // Step 3: Process edges (add value for thickness if needed) - // Filter out edges that reference non-existent nodes - const processedEdges: ExtendedGraphEdge[] = query.data.edges - .filter((edge) => { - const sourceId = - typeof edge.source === 'string' ? edge.source : edge.source; - const targetId = - typeof edge.target === 'string' ? edge.target : edge.target; - return validNodeIds.has(sourceId) && validNodeIds.has(targetId); - }) - .map((edge) => ({ - ...edge, - value: 1, // Default thickness, can be customized based on edge type - })); - - return { - nodes: processedNodes, - links: processedEdges, - }; - }, [query.data]); + // Calculate overall loading progress + const loadedPages = queries.filter((q) => q.isSuccess).length; + const loadingProgress = + totalPages && totalPages > 0 ? (loadedPages / totalPages) * 100 : 0; return { - ...query, data: processedData, + isLoading, + isError, + error, + loadingProgress, + loadedPages, + totalPages, + isComplete: totalPages !== null && loadedPages >= totalPages, }; }