From 15e6d077ef1ed34b1e752e8cd33b94f93c1eeb4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andri=20=C3=93skarsson?= Date: Sat, 18 Jan 2025 17:30:07 +0100 Subject: [PATCH] unifying the feed handlers a bit --- .../atproto/domain/jetstream-subscription.ts | 4 ++ .../feeds/{queries => handlers}/following.ts | 2 +- .../atproto/feeds/handlers/only-reposts.ts | 54 +++++++++++++++++++ .../atproto/feeds/handlers/trending-all.ts | 43 +++++++++++++++ packages/atproto/feeds/index.ts | 18 +++---- packages/atproto/feeds/only-reposts.ts | 32 ----------- .../atproto/feeds/queries/following-posts.ts | 25 --------- packages/atproto/feeds/queries/post-query.ts | 6 +-- packages/atproto/helpers/has-label.ts | 8 +++ 9 files changed, 122 insertions(+), 70 deletions(-) rename packages/atproto/feeds/{queries => handlers}/following.ts (96%) create mode 100644 packages/atproto/feeds/handlers/only-reposts.ts create mode 100644 packages/atproto/feeds/handlers/trending-all.ts delete mode 100644 packages/atproto/feeds/only-reposts.ts delete mode 100644 packages/atproto/feeds/queries/following-posts.ts create mode 100644 packages/atproto/helpers/has-label.ts diff --git a/packages/atproto/domain/jetstream-subscription.ts b/packages/atproto/domain/jetstream-subscription.ts index 332d46f..69214c5 100644 --- a/packages/atproto/domain/jetstream-subscription.ts +++ b/packages/atproto/domain/jetstream-subscription.ts @@ -7,6 +7,7 @@ import { import { subMinutes } from "date-fns"; import { desc } from "drizzle-orm"; import type { AtContext } from "../context"; +import { hasLabel } from "../helpers/has-label"; import { prettyBytes } from "../helpers/pretty-bytes"; import { getDids } from "./jetstream-did-list"; import { postTable } from "./post/post.table"; @@ -39,6 +40,9 @@ export async function listenForPosts(ctx: AtContext) { const initialMem = process.memoryUsage().rss; async function handlePost(msg: CommitEvent) { + if (hasLabel(msg.commit.record.labels, ["porn"])) { + return; // We're not interested in posts with this label + } await queuePost(ctx, msg); postCounter++; const mem = process.memoryUsage().rss; diff --git a/packages/atproto/feeds/queries/following.ts b/packages/atproto/feeds/handlers/following.ts similarity index 96% rename from packages/atproto/feeds/queries/following.ts rename to packages/atproto/feeds/handlers/following.ts index a754fdf..606a006 100644 --- a/packages/atproto/feeds/queries/following.ts +++ b/packages/atproto/feeds/handlers/following.ts @@ -2,7 +2,7 @@ import { type AppBskyFeedGetFeedSkeleton } from "@atproto/api"; import type { SkeletonReasonRepost } from "@atproto/api/dist/client/types/app/bsky/feed/defs"; import type { FeedHandlerArgs, FeedHandlerOutput } from ".."; import { toCursor } from "../../helpers/cursor"; -import { postQuery } from "./post-query"; +import { postQuery } from "../queries/post-query"; const PER_PAGE = 30; diff --git a/packages/atproto/feeds/handlers/only-reposts.ts b/packages/atproto/feeds/handlers/only-reposts.ts new file mode 100644 index 0000000..63862bd --- /dev/null +++ b/packages/atproto/feeds/handlers/only-reposts.ts @@ -0,0 +1,54 @@ +import { type AppBskyFeedGetFeedSkeleton } from "@atproto/api"; +import type { SkeletonReasonRepost } from "@atproto/api/dist/client/types/app/bsky/feed/defs"; +import type { FeedHandlerArgs, FeedHandlerOutput } from ".."; +import { toCursor } from "../../helpers/cursor"; +import { postQuery } from "../queries/post-query"; + +const PER_PAGE = 30; + +// TODO: This should in fact follow the same interface as the feed handlers +export async function repostsOnlyFeedHandler( + args: FeedHandlerArgs +): Promise { + const posts = await postQuery({ + ...args, + tagFilters: args.tagFilters ?? [ + { + tag: "tech", + minScore: 70, + }, + ], + options: { + onlyFollows: false, + showPosts: false, + showReposts: true, + }, + }); + + let newCursor: string | undefined; + if (posts.length > 0) { + const lastPost = posts[posts.length - 1]; + if (!lastPost.date) { + throw new Error("wat!"); + } + newCursor = toCursor(new Date(lastPost.date)); + } + + return { + feed: posts.map((p) => { + let reason: SkeletonReasonRepost | undefined = undefined; + if (p.repost) { + // Note: Reason type is required. Client will get mad otherwise + reason = { + $type: "app.bsky.feed.defs#skeletonReasonRepost", + repost: p.repost, + }; + } + return { + post: p.id, + reason, + }; + }), + cursor: newCursor, + } satisfies AppBskyFeedGetFeedSkeleton.OutputSchema; +} diff --git a/packages/atproto/feeds/handlers/trending-all.ts b/packages/atproto/feeds/handlers/trending-all.ts new file mode 100644 index 0000000..d3b8e75 --- /dev/null +++ b/packages/atproto/feeds/handlers/trending-all.ts @@ -0,0 +1,43 @@ +import { type AppBskyFeedGetFeedSkeleton } from "@atproto/api"; +import type { FeedHandlerArgs, FeedHandlerOutput } from ".."; +import { toCursor } from "../../helpers/cursor"; +import { postQuery } from "../queries/post-query"; + +const PER_PAGE = 30; + +// TODO: This should in fact follow the same interface as the feed handlers +export async function trendingAllFeedHandler( + args: FeedHandlerArgs +): Promise { + const posts = await postQuery({ + ...args, + tagFilters: args.tagFilters ?? [ + { + tag: "tech", + minScore: 70, + }, + ], + options: { + onlyFollows: false, + }, + }); + + let newCursor: string | undefined; + if (posts.length > 0) { + const lastPost = posts[posts.length - 1]; + if (!lastPost.date) { + throw new Error("wat!"); + } + newCursor = toCursor(new Date(lastPost.date)); + } + + return { + feed: posts.map((p) => { + return { + post: p.id, + // No reason given for trending; looks silly + }; + }), + cursor: newCursor, + } satisfies AppBskyFeedGetFeedSkeleton.OutputSchema; +} diff --git a/packages/atproto/feeds/index.ts b/packages/atproto/feeds/index.ts index a236596..5e90ecc 100644 --- a/packages/atproto/feeds/index.ts +++ b/packages/atproto/feeds/index.ts @@ -5,12 +5,12 @@ import type { import { config } from "../config"; import type { AtContext } from "../context"; import { getModerationPosts } from "../domain/get-moderation-queue"; -import { getTechAllFeed } from "../domain/get-tech-all-feed"; import type { PostFlags } from "../domain/post/post-flags"; import type { repostTable } from "../domain/post/post-reposts.table"; import type { postTable } from "../domain/post/post.table"; -import { repostsOnlyFeed } from "./only-reposts"; -import { followingFeedHandler } from "./queries/following"; +import { followingFeedHandler } from "./handlers/following"; +import { repostsOnlyFeedHandler } from "./handlers/only-reposts"; +import { trendingAllFeedHandler } from "./handlers/trending-all"; export const DEFAULT_FEED_ACTOR = "did:plc:rrrwbar3wv576qpsymwey5p5"; export type FeedHandlerArgs = { @@ -61,7 +61,7 @@ export const feeds: Array = [ did: config.feedGenDid, displayName: "📜 Following", description: - "Posts from your following; filtered for anything that isn't considered tech related.", + 'Your following feed; filtered for anything that isn\'t technical.\n\nMore specifically: Posts, reposts and replies from those you follow if the content has been classified to be "techy" (clinical term) enough by our army of trained robot hamsters.', //avatar: avatarRef, createdAt: new Date("2024-12-19").toISOString(), }, @@ -73,11 +73,11 @@ export const feeds: Array = [ record: { did: config.feedGenDid, displayName: "📜 Trending", - description: "Curated posts on the subject of technology.", + description: "Experimental. Algorithms subject to change.", //avatar: avatarRef, createdAt: new Date("2024-12-19").toISOString(), }, - handler: getTechAllFeed, + handler: trendingAllFeedHandler, private: false, }, { @@ -85,11 +85,11 @@ export const feeds: Array = [ record: { did: config.feedGenDid, displayName: "📜 Reposts", - description: "Experimental", + description: "Experimental. Algorithms subject to change.", createdAt: new Date("2025-01-11").toISOString(), }, - handler: repostsOnlyFeed, - private: true, + handler: repostsOnlyFeedHandler, + private: false, }, { rkey: "tech-mod", diff --git a/packages/atproto/feeds/only-reposts.ts b/packages/atproto/feeds/only-reposts.ts deleted file mode 100644 index 5852cfa..0000000 --- a/packages/atproto/feeds/only-reposts.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { getOrUpdateFollows } from "../domain/get-or-update-follows"; -import type { FeedHandlerArgs, FeedHandlerOutput } from "../feeds"; -import { toCursor } from "../helpers/cursor"; -import { followingRepostsQuery } from "./queries/following-reposts"; - -export async function repostsOnlyFeed( - args: FeedHandlerArgs -): Promise { - const { ctx, actorDid, cursor, limit = 50 } = args; - - await getOrUpdateFollows(ctx, actorDid); - - //const posts = await followingRepostsQuery(args); - const posts = await followingRepostsQuery(args).limit(limit); - - return { - feed: posts.map((p) => { - const reason = p.repost - ? { - repost: p.repost, - } - : undefined; - return { - post: p.id, - reason, - feedContext: "tech-reposts", - } satisfies FeedHandlerOutput[number]; - }), - cursor: - posts.length > 0 ? toCursor(posts[posts.length - 1].date) : undefined, - }; -} diff --git a/packages/atproto/feeds/queries/following-posts.ts b/packages/atproto/feeds/queries/following-posts.ts deleted file mode 100644 index a9f1d49..0000000 --- a/packages/atproto/feeds/queries/following-posts.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { and, eq, gte } from "drizzle-orm"; -import type { FeedHandlerArgs } from ".."; -import { repostTable } from "../../domain/post/post-reposts.table"; -import { postScores } from "../../domain/post/post-scores.view"; -import { postTable } from "../../domain/post/post.table"; -import { followingSubQuery } from "./sq-following"; - -export function followingPostsQuery(args: FeedHandlerArgs) { - const { ctx, actorDid, limit = 30 } = args; - const fls = followingSubQuery(args); - const postsQuery = ctx.db - .select({ - id: postTable.id, - repost: repostTable.repostUri, - date: postTable.lastMentioned, - }) - .from(postTable) - .innerJoin(fls, eq(postTable.authorId, fls.follows)) - .leftJoin(fls, eq(repostTable.repostAuthorId, fls.follows)) - .innerJoin( - postScores, - and(eq(postScores.postId, postTable.id), gte(postScores.avgScore, 70)) - ); - return postsQuery; -} diff --git a/packages/atproto/feeds/queries/post-query.ts b/packages/atproto/feeds/queries/post-query.ts index 5039b3b..76fa9ae 100644 --- a/packages/atproto/feeds/queries/post-query.ts +++ b/packages/atproto/feeds/queries/post-query.ts @@ -35,10 +35,12 @@ export async function postQuery(args: FeedHandlerArgs) { const textSearch = textSearchSubQuery(ctx.db, search); let filters: Array = [ + // Reposts + options?.showPosts === false ? isNotNull(rpls.created) : undefined, + options?.showReposts === false ? isNull(rpls.created) : undefined, options?.onlyFollows ? or( // Posts that we follow - // TODO: Ignore replies if author isn't someone we follow // - Maybe subquery for replies so I can look up the author.. or get PSQL to parse the URL and( isNotNull(fls.follows), @@ -52,8 +54,6 @@ export async function postQuery(args: FeedHandlerArgs) { ), // Reposts by people we follow and(isNotNull(rpls.created)) - // TODO: Flags - // TODO: Options for reposts/posts/mustfollow ) : undefined, ]; diff --git a/packages/atproto/helpers/has-label.ts b/packages/atproto/helpers/has-label.ts new file mode 100644 index 0000000..7661a81 --- /dev/null +++ b/packages/atproto/helpers/has-label.ts @@ -0,0 +1,8 @@ +export function hasLabel(v: unknown, label: Array): boolean | null { + if (v === undefined || v === null) { + return false; + } + if (!Array.isArray(v)) throw new Error("Label variable not array"); + const labels = v as Array; + return labels.some((l) => label.includes(l)); +} -- 2.51.2