diff --git a/services/appview/package.json b/services/appview/package.json index a60c49e..607da93 100644 --- a/services/appview/package.json +++ b/services/appview/package.json @@ -30,6 +30,7 @@ "express": "^4.21.2", "hono": "^4.7.4", "iron-session": "^8.0.4", + "lodash": "^4.17.21", "mongoose": "^8.12.1", "multiformats": "^9.9.0", "pino": "^9.6.0", @@ -40,6 +41,7 @@ "devDependencies": { "@atproto/lex-cli": "^0.6.2", "@types/express": "^5.0.0", + "@types/lodash": "^4.17.16", "@types/node": "^22.13.10", "@types/pg": "^8.11.11", "@types/ws": "^8.18.0", diff --git a/services/appview/pnpm-lock.yaml b/services/appview/pnpm-lock.yaml index 3821c0a..5499f54 100644 --- a/services/appview/pnpm-lock.yaml +++ b/services/appview/pnpm-lock.yaml @@ -60,6 +60,9 @@ dependencies: iron-session: specifier: ^8.0.4 version: 8.0.4 + lodash: + specifier: ^4.17.21 + version: 4.17.21 mongoose: specifier: ^8.12.1 version: 8.12.1 @@ -86,6 +89,9 @@ devDependencies: '@types/express': specifier: ^5.0.0 version: 5.0.0 + '@types/lodash': + specifier: ^4.17.16 + version: 4.17.16 '@types/node': specifier: ^22.13.10 version: 22.13.10 @@ -860,6 +866,10 @@ packages: resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} dev: true + /@types/lodash@4.17.16: + resolution: {integrity: sha512-HX7Em5NYQAXKW+1T+FiuG27NGwzJfCX3s1GjOa7ujxZa52kjJLOr4FUxT+giF6Tgxv1e+/czV/iTtBw27WTU9g==} + dev: true + /@types/mime@1.3.5: resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} dev: true @@ -1895,6 +1905,10 @@ packages: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} dev: true + /lodash@4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + dev: false + /lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} dev: false diff --git a/services/appview/src/middleware/takedown-filter.ts b/services/appview/src/middleware/takedown-filter.ts index c71fdb2..2cee4e9 100644 --- a/services/appview/src/middleware/takedown-filter.ts +++ b/services/appview/src/middleware/takedown-filter.ts @@ -1,5 +1,6 @@ import { Context, Next } from 'hono' import { TakedownService } from '../services/takedown.js' +import { get } from 'lodash' /** * Middleware that filters out taken-down content from responses @@ -13,51 +14,105 @@ export const takedownFilterMiddleware = async (c: Context, next: Next) => { await next() return } - + // Call the next middleware/route handler first await next() - + // Skip filtering if not a JSON response const contentType = c.res.headers.get('Content-Type') if (!contentType || !contentType.includes('application/json')) { return } - + try { // Get the takedown service from context const takedownService = c.get('takedownService') as TakedownService - - // Get the response body + const body = await c.res.json() - - // Process different response formats + + const targetDid = body.did || body.user?.did || body.actor?.did || body.profile?.did || body.subject?.did + if (targetDid) { + const isRepoTakenDown = await takedownService.isRepoTakenDown(targetDid) + if (isRepoTakenDown) { + // For specific user/profile views, return minimal placeholder + if (body.did && body.$type && body.$type.includes('profileView')) { + const takenDownProfile = { + $type: body.$type, + did: body.did, + handle: body.handle || 'unavailable', + moderation: { + takenDown: true, + }, + } + c.res = new Response(JSON.stringify(takenDownProfile), { + status: c.res.status, + headers: c.res.headers, + }) + return + } else { + // For other single-user responses, null out or minimize the content + c.res = new Response(JSON.stringify({ + error: 'Content unavailable - repository has been taken down', + code: 404 + }), { + status: 404, + headers: c.res.headers, + }) + return + } + } + } + + // Continue with specific content type filtering if (body.posts && Array.isArray(body.posts)) { - // For post feeds - const filteredPosts = await filterTakenDownItems(body.posts, takedownService, 'uri') + const filteredPosts = await filterTakenDownItems( + body.posts, + takedownService, + 'uri', + ) body.posts = filteredPosts } else if (body.feed && Array.isArray(body.feed)) { - // For general feeds - const filteredFeed = await filterTakenDownItems(body.feed, takedownService, 'post.uri') + const filteredFeed = await filterTakenDownItems( + body.feed, + takedownService, + 'post.uri', + ) body.feed = filteredFeed } else if (body.thread && body.thread.post) { - // For thread views - const isThreadTakenDown = await takedownService.isTakenDown(body.thread.post.uri) - if (isThreadTakenDown) { - // If the main post is taken down, return empty thread + const isThreadTakenDown = await takedownService.isTakenDown( + body.thread.post.uri, + ) + + // Also check if the thread author repo is taken down + let isAuthorTakenDown = false + if (body.thread.post.author?.did) { + isAuthorTakenDown = await takedownService.isRepoTakenDown( + body.thread.post.author.did + ) + } + + if (isThreadTakenDown || isAuthorTakenDown) { body.thread = null } else if (body.thread.replies) { - // Filter replies if they exist - body.thread.replies = await filterReplies(body.thread.replies, takedownService) + body.thread.replies = await filterReplies( + body.thread.replies, + takedownService, + ) } } // If there are user profiles in the response, filter out taken down repositories if (body.profiles && Array.isArray(body.profiles)) { - const filteredProfiles = await filterTakenDownRepos(body.profiles, takedownService) + const filteredProfiles = await filterTakenDownRepos( + body.profiles, + takedownService, + ) body.profiles = filteredProfiles } else if (body.profile) { if (body.profile.did) { - const isRepoTakenDown = await takedownService.isRepoTakenDown(body.profile.did) + const isRepoTakenDown = await takedownService.isRepoTakenDown( + body.profile.did, + ) if (isRepoTakenDown) { body.profile = null } @@ -72,23 +127,25 @@ export const takedownFilterMiddleware = async (c: Context, next: Next) => { did: body.did, handle: body.handle || 'unavailable', moderation: { - takenDown: true - } + takenDown: true, + }, } - + // Create a new response with the placeholder instead of trying to modify body c.res = new Response(JSON.stringify(takenDownProfile), { status: c.res.status, headers: c.res.headers, }) - + // Skip the rest of the processing return } } else if (body.subject) { // For followers/follows response that has a subject profile if (body.subject.did) { - const isRepoTakenDown = await takedownService.isRepoTakenDown(body.subject.did) + const isRepoTakenDown = await takedownService.isRepoTakenDown( + body.subject.did, + ) if (isRepoTakenDown) { // Keep minimal info about the profile but mark it as taken down body.subject = { @@ -96,22 +153,25 @@ export const takedownFilterMiddleware = async (c: Context, next: Next) => { did: body.subject.did, handle: body.subject.handle || 'unavailable', moderation: { - takenDown: true - } + takenDown: true, + }, } } } - + // Also filter any followers/follows list if (body.followers && Array.isArray(body.followers)) { - body.followers = await filterTakenDownRepos(body.followers, takedownService) + body.followers = await filterTakenDownRepos( + body.followers, + takedownService, + ) } - + if (body.follows && Array.isArray(body.follows)) { body.follows = await filterTakenDownRepos(body.follows, takedownService) } } - + // Set the filtered response c.res = new Response(JSON.stringify(body), { status: c.res.status, @@ -125,55 +185,75 @@ export const takedownFilterMiddleware = async (c: Context, next: Next) => { // Helper function to filter out taken down items async function filterTakenDownItems( - items: any[], + items: Record[], takedownService: TakedownService, - uriPath: string -): Promise { - if (!items || !Array.isArray(items)) return items - - const filteredItems: any[] = [] - + uriPath: string, +) { + if (!items || items.length === 0) { + return items + } + + const filteredItems: Record[] = [] + for (const item of items) { - // Get the URI based on the specified path (handles nested objects) - const uri = uriPath.split('.').reduce((obj, key) => obj && obj[key], item) - + let isTakenDown = false + + // Get URI for this specific content + const uri = get(item, uriPath) as string | undefined if (uri) { - const isTakenDown = await takedownService.isTakenDown(uri) - - // Check if author's repo is taken down - let isAuthorTakenDown = false - if (item.author?.did || (item.post?.author?.did)) { - const authorDid = item.author?.did || item.post?.author?.did - isAuthorTakenDown = await takedownService.isRepoTakenDown(authorDid) - } - - // Keep the item only if neither the content nor the author is taken down - if (!isTakenDown && !isAuthorTakenDown) { - // Filter out taken down images if the item has embeds - if (item.embed?.images && Array.isArray(item.embed.images)) { - item.embed.images = await filterTakenDownBlobs(item.embed.images, takedownService) + isTakenDown = await takedownService.isTakenDown(uri) + } + + // Check if author's repo is taken down + let isAuthorTakenDown = false + // Look for author DID in common locations + const authorDid = get(item, 'author.did') || + get(item, 'post.author.did') || + get(item, 'user.did') || + get(item, 'actor.did') + + if (authorDid) { + isAuthorTakenDown = await takedownService.isRepoTakenDown(authorDid) + } + + // Keep the item only if neither the content nor the author is taken down + if (!isTakenDown && !isAuthorTakenDown) { + // Also check for any embedded items like quotes or replies + if (item.embed && item.embed.record && item.embed.record.author?.did) { + const embedAuthorTakenDown = await takedownService.isRepoTakenDown(item.embed.record.author.did) + if (embedAuthorTakenDown) { + // Null out the embed if from a taken-down repo + item.embed = { + $type: item.embed.$type, + takenDown: true + } + } else if (item.embed.record.uri) { + // Check if the specific embedded content is taken down + const embedContentTakenDown = await takedownService.isTakenDown(item.embed.record.uri) + if (embedContentTakenDown) { + item.embed = { + $type: item.embed.$type, + takenDown: true + } + } } - - filteredItems.push(item) } - } else { - // If URI is not found, keep the item filteredItems.push(item) } } - + return filteredItems } // Helper function to filter out taken down repositories async function filterTakenDownRepos( profiles: any[], - takedownService: TakedownService + takedownService: TakedownService, ): Promise { if (!profiles || !Array.isArray(profiles)) return profiles - + const filteredProfiles: any[] = [] - + for (const profile of profiles) { if (profile.did) { const isRepoTakenDown = await takedownService.isRepoTakenDown(profile.did) @@ -187,8 +267,8 @@ async function filterTakenDownRepos( did: profile.did, handle: profile.handle || 'unavailable', moderation: { - takenDown: true - } + takenDown: true, + }, }) } } else { @@ -196,23 +276,26 @@ async function filterTakenDownRepos( filteredProfiles.push(profile) } } - + return filteredProfiles } // Helper function to filter out taken down blobs/images async function filterTakenDownBlobs( images: any[], - takedownService: TakedownService + takedownService: TakedownService, ): Promise { if (!images || !Array.isArray(images)) return images - + const filteredImages: any[] = [] - + for (const image of images) { // Check if the image is taken down based on blob CID if (image.cid && image.did) { - const isBlobTakenDown = await takedownService.isBlobTakenDown(image.did, image.cid) + const isBlobTakenDown = await takedownService.isBlobTakenDown( + image.did, + image.cid, + ) if (!isBlobTakenDown) { filteredImages.push(image) } @@ -221,40 +304,48 @@ async function filterTakenDownBlobs( filteredImages.push(image) } } - + return filteredImages } // Helper function to recursively filter replies in a thread async function filterReplies( replies: any[], - takedownService: TakedownService + takedownService: TakedownService, ): Promise { if (!replies || !Array.isArray(replies)) return replies - + const filteredReplies: any[] = [] - + for (const reply of replies) { if (reply.post && reply.post.uri) { const isTakenDown = await takedownService.isTakenDown(reply.post.uri) - + // Check if author's repo is taken down let isAuthorTakenDown = false if (reply.post.author?.did) { - isAuthorTakenDown = await takedownService.isRepoTakenDown(reply.post.author.did) + isAuthorTakenDown = await takedownService.isRepoTakenDown( + reply.post.author.did, + ) } - + if (!isTakenDown && !isAuthorTakenDown) { // If this reply has nested replies, filter those too if (reply.replies && Array.isArray(reply.replies)) { reply.replies = await filterReplies(reply.replies, takedownService) } - + // Filter out taken down images in the post - if (reply.post.embed?.images && Array.isArray(reply.post.embed.images)) { - reply.post.embed.images = await filterTakenDownBlobs(reply.post.embed.images, takedownService) + if ( + reply.post.embed?.images && + Array.isArray(reply.post.embed.images) + ) { + reply.post.embed.images = await filterTakenDownBlobs( + reply.post.embed.images, + takedownService, + ) } - + filteredReplies.push(reply) } } else { @@ -262,6 +353,6 @@ async function filterReplies( filteredReplies.push(reply) } } - + return filteredReplies -} \ No newline at end of file +}