diff --git a/.agent/logs/20251223_card_url_type_migration.md b/.agent/logs/20251223_card_url_type_migration.md index ec403637..5f863fa6 100644 --- a/.agent/logs/20251223_card_url_type_migration.md +++ b/.agent/logs/20251223_card_url_type_migration.md @@ -1,7 +1,7 @@ # Card URL Type Migration Plan **Date:** 2025-12-23 -**Status:** Planning Phase +**Status:** Planning Phase ## Overview @@ -10,18 +10,21 @@ Adding a dedicated `urlType` field to the cards table to enable efficient queryi ## Approach: Incremental Migration ### Phase 1: Add Optional Field (Immediate) + - Add `url_type` TEXT column to cards table (nullable) - Update CardMapper to populate urlType for new cards - Add indexes for performance - New cards will be immediately queryable by type ### Phase 2: Hybrid Queries (Temporary) + - Update query services to handle mixed data: - Fast indexed lookup for new cards (urlType field) - Fallback JSON query for old cards (contentData.metadata.type) - Use OR conditions to support both data states ### Phase 3: Backfill Migration (Later) + - Run SQL update to populate urlType for existing cards - Remove fallback JSON queries once migration complete - Simplify query logic to use only indexed field @@ -31,7 +34,7 @@ Adding a dedicated `urlType` field to the cards table to enable efficient queryi ✅ **Zero downtime** - No breaking changes ✅ **Immediate value** - New cards queryable by type right away ✅ **Gradual migration** - Can backfill when convenient -✅ **Performance** - Indexed queries for new data +✅ **Performance** - Indexed queries for new data ## Implementation Notes @@ -42,10 +45,10 @@ Adding a dedicated `urlType` field to the cards table to enable efficient queryi ## Migration SQL (for Phase 3) ```sql -UPDATE cards +UPDATE cards SET url_type = content_data #>> '{metadata,type}' -WHERE type = 'URL' - AND url_type IS NULL +WHERE type = 'URL' + AND url_type IS NULL AND content_data #>> '{metadata,type}' IS NOT NULL; ``` diff --git a/src/modules/cards/infrastructure/repositories/mappers/CardMapper.ts b/src/modules/cards/infrastructure/repositories/mappers/CardMapper.ts index 8bcaa598..c8d646da 100644 --- a/src/modules/cards/infrastructure/repositories/mappers/CardMapper.ts +++ b/src/modules/cards/infrastructure/repositories/mappers/CardMapper.ts @@ -346,7 +346,10 @@ export class CardMapper { type: card.type.value, contentData, url: card.url?.value, - urlType: content.type === CardTypeEnum.URL ? content.urlContent?.metadata?.type : undefined, + urlType: + content.type === CardTypeEnum.URL + ? content.urlContent?.metadata?.type + : undefined, parentCardId: card.parentCardId?.getStringValue(), viaCardId: card.viaCardId?.getStringValue(), libraryCount: card.libraryCount, diff --git a/src/modules/cards/infrastructure/repositories/query-services/CollectionCardQueryService.ts b/src/modules/cards/infrastructure/repositories/query-services/CollectionCardQueryService.ts index b4abd311..c1c8b8fd 100644 --- a/src/modules/cards/infrastructure/repositories/query-services/CollectionCardQueryService.ts +++ b/src/modules/cards/infrastructure/repositories/query-services/CollectionCardQueryService.ts @@ -54,7 +54,7 @@ export class CollectionCardQueryService { eq(collectionCards.collectionId, collectionId), eq(cards.type, CardTypeEnum.URL), ]; - + if (options.urlType) { whereConditions.push(eq(cards.urlType, options.urlType)); } diff --git a/src/modules/cards/infrastructure/repositories/query-services/UrlCardQueryService.ts b/src/modules/cards/infrastructure/repositories/query-services/UrlCardQueryService.ts index 90fcdb9a..7ae94bb7 100644 --- a/src/modules/cards/infrastructure/repositories/query-services/UrlCardQueryService.ts +++ b/src/modules/cards/infrastructure/repositories/query-services/UrlCardQueryService.ts @@ -37,7 +37,7 @@ export class UrlCardQueryService { eq(cards.authorId, userId), eq(cards.type, CardTypeEnum.URL), ]; - + if (options.urlType) { whereConditions.push(eq(cards.urlType, options.urlType)); } diff --git a/src/modules/cards/tests/utils/InMemoryCardQueryRepository.ts b/src/modules/cards/tests/utils/InMemoryCardQueryRepository.ts index 8655c60c..37222cd4 100644 --- a/src/modules/cards/tests/utils/InMemoryCardQueryRepository.ts +++ b/src/modules/cards/tests/utils/InMemoryCardQueryRepository.ts @@ -32,12 +32,10 @@ export class InMemoryCardQueryRepository implements ICardQueryRepository { try { // Get all cards and filter by user's library membership const allCards = this.cardRepository.getAllCards(); - let userCards = allCards - .filter( - (card) => - card.isUrlCard && - card.isInLibrary(CuratorId.create(userId).unwrap()), - ); + let userCards = allCards.filter( + (card) => + card.isUrlCard && card.isInLibrary(CuratorId.create(userId).unwrap()), + ); // Filter by urlType if specified if (options.urlType) { @@ -244,8 +242,7 @@ export class InMemoryCardQueryRepository implements ICardQueryRepository { ); let collectionCards = allCards.filter( (card) => - collectionCardIds.has(card.cardId.getStringValue()) && - card.isUrlCard, + collectionCardIds.has(card.cardId.getStringValue()) && card.isUrlCard, ); // Filter by urlType if specified diff --git a/src/modules/feeds/infrastructure/repositories/DrizzleFeedRepository.ts b/src/modules/feeds/infrastructure/repositories/DrizzleFeedRepository.ts index f6292ba8..93246def 100644 --- a/src/modules/feeds/infrastructure/repositories/DrizzleFeedRepository.ts +++ b/src/modules/feeds/infrastructure/repositories/DrizzleFeedRepository.ts @@ -69,7 +69,10 @@ export class DrizzleFeedRepository implements IFeedRepository { .limit(1); if (beforeActivity.length > 0) { - const conditions = [lt(feedActivities.createdAt, beforeActivity[0]!.createdAt), ...whereConditions]; + const conditions = [ + lt(feedActivities.createdAt, beforeActivity[0]!.createdAt), + ...whereConditions, + ]; activitiesResult = await this.db .select() .from(feedActivities) @@ -82,14 +85,16 @@ export class DrizzleFeedRepository implements IFeedRepository { } } else { // Regular pagination without cursor - const query = this.db - .select() - .from(feedActivities); - + const query = this.db.select().from(feedActivities); + if (whereConditions.length > 0) { - query.where(whereConditions.length > 1 ? and(...whereConditions) : whereConditions[0]); + query.where( + whereConditions.length > 1 + ? and(...whereConditions) + : whereConditions[0], + ); } - + activitiesResult = await query .orderBy(desc(feedActivities.createdAt), desc(feedActivities.id)) .limit(limit) @@ -100,11 +105,15 @@ export class DrizzleFeedRepository implements IFeedRepository { const countQuery = this.db .select({ count: count() }) .from(feedActivities); - + if (whereConditions.length > 0) { - countQuery.where(whereConditions.length > 1 ? and(...whereConditions) : whereConditions[0]); + countQuery.where( + whereConditions.length > 1 + ? and(...whereConditions) + : whereConditions[0], + ); } - + const totalCountResult = await countQuery; const totalCount = totalCountResult[0]?.count || 0; @@ -206,7 +215,7 @@ export class DrizzleFeedRepository implements IFeedRepository { const conditions = [ lt(feedActivities.createdAt, beforeActivity[0]!.createdAt), jsonArrayCondition, - ...whereConditions + ...whereConditions, ]; activitiesResult = await this.db .select() @@ -235,7 +244,7 @@ export class DrizzleFeedRepository implements IFeedRepository { if (options.urlType) { conditions.push(eq(feedActivities.urlType, options.urlType)); } - + const totalCountResult = await this.db .select({ count: count() }) .from(feedActivities) diff --git a/src/modules/feeds/infrastructure/repositories/schema/feedActivity.sql.ts b/src/modules/feeds/infrastructure/repositories/schema/feedActivity.sql.ts index 89bd5746..9f157239 100644 --- a/src/modules/feeds/infrastructure/repositories/schema/feedActivity.sql.ts +++ b/src/modules/feeds/infrastructure/repositories/schema/feedActivity.sql.ts @@ -1,23 +1,44 @@ -import { pgTable, text, timestamp, jsonb, uuid, index } from 'drizzle-orm/pg-core'; +import { + pgTable, + text, + timestamp, + jsonb, + uuid, + index, +} from 'drizzle-orm/pg-core'; -export const feedActivities = pgTable('feed_activities', { - id: uuid('id').primaryKey(), - actorId: text('actor_id').notNull(), // The DID of the user who performed the activity - type: text('type').notNull(), // The type of activity (e.g., 'CARD_COLLECTED') - metadata: jsonb('metadata').notNull(), // Activity-specific metadata - urlType: text('url_type'), // Optional URL type from the card - createdAt: timestamp('created_at').notNull().defaultNow(), -}, (table) => ({ - // Index for filtering by activity type - typeIdx: index('feed_activities_type_idx').on(table.type), - // Index for filtering by URL type - urlTypeIdx: index('feed_activities_url_type_idx').on(table.urlType), - // Index for sorting by creation date (most recent first) - createdAtIdx: index('feed_activities_created_at_idx').on(table.createdAt.desc()), - // Composite index for common query patterns (type + createdAt) - typeCreatedAtIdx: index('feed_activities_type_created_at_idx').on(table.type, table.createdAt.desc()), - // Composite index for URL type filtering with date sorting - urlTypeCreatedAtIdx: index('feed_activities_url_type_created_at_idx').on(table.urlType, table.createdAt.desc()), - // Composite index for filtering by both type and URL type with date sorting - typeUrlTypeCreatedAtIdx: index('feed_activities_type_url_type_created_at_idx').on(table.type, table.urlType, table.createdAt.desc()), -})); +export const feedActivities = pgTable( + 'feed_activities', + { + id: uuid('id').primaryKey(), + actorId: text('actor_id').notNull(), // The DID of the user who performed the activity + type: text('type').notNull(), // The type of activity (e.g., 'CARD_COLLECTED') + metadata: jsonb('metadata').notNull(), // Activity-specific metadata + urlType: text('url_type'), // Optional URL type from the card + createdAt: timestamp('created_at').notNull().defaultNow(), + }, + (table) => ({ + // Index for filtering by activity type + typeIdx: index('feed_activities_type_idx').on(table.type), + // Index for filtering by URL type + urlTypeIdx: index('feed_activities_url_type_idx').on(table.urlType), + // Index for sorting by creation date (most recent first) + createdAtIdx: index('feed_activities_created_at_idx').on( + table.createdAt.desc(), + ), + // Composite index for common query patterns (type + createdAt) + typeCreatedAtIdx: index('feed_activities_type_created_at_idx').on( + table.type, + table.createdAt.desc(), + ), + // Composite index for URL type filtering with date sorting + urlTypeCreatedAtIdx: index('feed_activities_url_type_created_at_idx').on( + table.urlType, + table.createdAt.desc(), + ), + // Composite index for filtering by both type and URL type with date sorting + typeUrlTypeCreatedAtIdx: index( + 'feed_activities_type_url_type_created_at_idx', + ).on(table.type, table.urlType, table.createdAt.desc()), + }), +); diff --git a/src/shared/infrastructure/database/migrations/meta/0010_snapshot.json b/src/shared/infrastructure/database/migrations/meta/0010_snapshot.json index b8ae3341..ffe1cd55 100644 --- a/src/shared/infrastructure/database/migrations/meta/0010_snapshot.json +++ b/src/shared/infrastructure/database/migrations/meta/0010_snapshot.json @@ -251,12 +251,8 @@ "name": "cards_parent_card_id_cards_id_fk", "tableFrom": "cards", "tableTo": "cards", - "columnsFrom": [ - "parent_card_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["parent_card_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -264,12 +260,8 @@ "name": "cards_via_card_id_cards_id_fk", "tableFrom": "cards", "tableTo": "cards", - "columnsFrom": [ - "via_card_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["via_card_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -277,12 +269,8 @@ "name": "cards_published_record_id_published_records_id_fk", "tableFrom": "cards", "tableTo": "published_records", - "columnsFrom": [ - "published_record_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["published_record_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -414,12 +402,8 @@ "name": "collection_cards_collection_id_collections_id_fk", "tableFrom": "collection_cards", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -427,12 +411,8 @@ "name": "collection_cards_card_id_cards_id_fk", "tableFrom": "collection_cards", "tableTo": "cards", - "columnsFrom": [ - "card_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["card_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -440,12 +420,8 @@ "name": "collection_cards_via_card_id_cards_id_fk", "tableFrom": "collection_cards", "tableTo": "cards", - "columnsFrom": [ - "via_card_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["via_card_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -453,12 +429,8 @@ "name": "collection_cards_published_record_id_published_records_id_fk", "tableFrom": "collection_cards", "tableTo": "published_records", - "columnsFrom": [ - "published_record_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["published_record_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -498,12 +470,8 @@ "name": "collection_collaborators_collection_id_collections_id_fk", "tableFrom": "collection_collaborators", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -619,12 +587,8 @@ "name": "collections_published_record_id_published_records_id_fk", "tableFrom": "collections", "tableTo": "published_records", - "columnsFrom": [ - "published_record_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["published_record_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -723,12 +687,8 @@ "name": "library_memberships_card_id_cards_id_fk", "tableFrom": "library_memberships", "tableTo": "cards", - "columnsFrom": [ - "card_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["card_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -736,12 +696,8 @@ "name": "library_memberships_published_record_id_published_records_id_fk", "tableFrom": "library_memberships", "tableTo": "published_records", - "columnsFrom": [ - "published_record_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["published_record_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -749,10 +705,7 @@ "compositePrimaryKeys": { "library_memberships_card_id_user_id_pk": { "name": "library_memberships_card_id_user_id_pk", - "columns": [ - "card_id", - "user_id" - ] + "columns": ["card_id", "user_id"] } }, "uniqueConstraints": {}, @@ -1106,12 +1059,8 @@ "name": "auth_refresh_tokens_user_did_users_id_fk", "tableFrom": "auth_refresh_tokens", "tableTo": "users", - "columnsFrom": [ - "user_did" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_did"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -1171,4 +1120,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/src/shared/infrastructure/database/migrations/meta/0011_snapshot.json b/src/shared/infrastructure/database/migrations/meta/0011_snapshot.json index 8faf164c..0e594703 100644 --- a/src/shared/infrastructure/database/migrations/meta/0011_snapshot.json +++ b/src/shared/infrastructure/database/migrations/meta/0011_snapshot.json @@ -251,12 +251,8 @@ "name": "cards_parent_card_id_cards_id_fk", "tableFrom": "cards", "tableTo": "cards", - "columnsFrom": [ - "parent_card_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["parent_card_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -264,12 +260,8 @@ "name": "cards_via_card_id_cards_id_fk", "tableFrom": "cards", "tableTo": "cards", - "columnsFrom": [ - "via_card_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["via_card_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -277,12 +269,8 @@ "name": "cards_published_record_id_published_records_id_fk", "tableFrom": "cards", "tableTo": "published_records", - "columnsFrom": [ - "published_record_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["published_record_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -414,12 +402,8 @@ "name": "collection_cards_collection_id_collections_id_fk", "tableFrom": "collection_cards", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -427,12 +411,8 @@ "name": "collection_cards_card_id_cards_id_fk", "tableFrom": "collection_cards", "tableTo": "cards", - "columnsFrom": [ - "card_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["card_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -440,12 +420,8 @@ "name": "collection_cards_via_card_id_cards_id_fk", "tableFrom": "collection_cards", "tableTo": "cards", - "columnsFrom": [ - "via_card_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["via_card_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" }, @@ -453,12 +429,8 @@ "name": "collection_cards_published_record_id_published_records_id_fk", "tableFrom": "collection_cards", "tableTo": "published_records", - "columnsFrom": [ - "published_record_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["published_record_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -498,12 +470,8 @@ "name": "collection_collaborators_collection_id_collections_id_fk", "tableFrom": "collection_collaborators", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -619,12 +587,8 @@ "name": "collections_published_record_id_published_records_id_fk", "tableFrom": "collections", "tableTo": "published_records", - "columnsFrom": [ - "published_record_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["published_record_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -723,12 +687,8 @@ "name": "library_memberships_card_id_cards_id_fk", "tableFrom": "library_memberships", "tableTo": "cards", - "columnsFrom": [ - "card_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["card_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -736,12 +696,8 @@ "name": "library_memberships_published_record_id_published_records_id_fk", "tableFrom": "library_memberships", "tableTo": "published_records", - "columnsFrom": [ - "published_record_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["published_record_id"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -749,10 +705,7 @@ "compositePrimaryKeys": { "library_memberships_card_id_user_id_pk": { "name": "library_memberships_card_id_user_id_pk", - "columns": [ - "card_id", - "user_id" - ] + "columns": ["card_id", "user_id"] } }, "uniqueConstraints": {}, @@ -1227,12 +1180,8 @@ "name": "auth_refresh_tokens_user_did_users_id_fk", "tableFrom": "auth_refresh_tokens", "tableTo": "users", - "columnsFrom": [ - "user_did" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_did"], + "columnsTo": ["id"], "onDelete": "no action", "onUpdate": "no action" } @@ -1292,4 +1241,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/src/shared/infrastructure/database/migrations/meta/_journal.json b/src/shared/infrastructure/database/migrations/meta/_journal.json index 99100799..dd39edaa 100644 --- a/src/shared/infrastructure/database/migrations/meta/_journal.json +++ b/src/shared/infrastructure/database/migrations/meta/_journal.json @@ -87,4 +87,4 @@ "breakpoints": true } ] -} \ No newline at end of file +}