From 9bd9678050352eb5cae8a3b41307f8aacd3e3598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andri=20=C3=93skarsson?= Date: Sun, 12 Jan 2025 15:21:05 +0100 Subject: [PATCH] why is memory such a pain in my butt --- apps/classifier/package.json | 2 +- fly.toml | 2 +- packages/atproto/context.ts | 1 + packages/atproto/db/helpers.ts | 17 ++++++++++ packages/atproto/feeds/index.ts | 21 ++++++++++++ packages/atproto/feeds/only-reposts.ts | 32 +++++++++++++++++++ .../atproto/feeds/queries/following-all.ts | 16 ++++++++++ .../atproto/feeds/queries/following-posts.ts | 25 +++++++++++++++ .../feeds/queries/following-reposts.ts | 30 +++++++++++++++++ packages/atproto/feeds/queries/following.ts | 11 +++++++ 10 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 packages/atproto/db/helpers.ts create mode 100644 packages/atproto/feeds/only-reposts.ts create mode 100644 packages/atproto/feeds/queries/following-all.ts create mode 100644 packages/atproto/feeds/queries/following-posts.ts create mode 100644 packages/atproto/feeds/queries/following-reposts.ts create mode 100644 packages/atproto/feeds/queries/following.ts diff --git a/apps/classifier/package.json b/apps/classifier/package.json index c33250f..7fe9fe3 100644 --- a/apps/classifier/package.json +++ b/apps/classifier/package.json @@ -3,7 +3,7 @@ "module": "index.ts", "type": "module", "scripts": { - "dev": "bun --watch --env-file=../../.env index.ts", + "sdev": "bun --watch --env-file=../../.env index.ts", "prod": "bun index.ts" }, "dependencies": { diff --git a/fly.toml b/fly.toml index f44ef00..836744d 100644 --- a/fly.toml +++ b/fly.toml @@ -35,6 +35,6 @@ jetstream = "jetstream" [[vm]] processes=['worker'] - memory = '1gb' + memory = '2gb' cpu_kind = 'shared' cpus = 1 diff --git a/packages/atproto/context.ts b/packages/atproto/context.ts index 4ebf603..c524ff0 100644 --- a/packages/atproto/context.ts +++ b/packages/atproto/context.ts @@ -12,6 +12,7 @@ export async function createAtContext() { }); const db = drizzle({ client: dbClient, + logger: false, }); await db.execute("select 1"); // Make sure we're connected to DB diff --git a/packages/atproto/db/helpers.ts b/packages/atproto/db/helpers.ts new file mode 100644 index 0000000..fe9bccd --- /dev/null +++ b/packages/atproto/db/helpers.ts @@ -0,0 +1,17 @@ +import { sql, type SQLWrapper } from "drizzle-orm"; +import { PostgresJsDatabase } from "drizzle-orm/postgres-js"; + +export const explainAnalyze = async ( + db: PostgresJsDatabase, + query: T +) => { + const debugResult = await db.execute(sql`EXPLAIN ANALYZE ${query.getSQL()}`); + console.debug(debugResult); + return query; +}; + +export const debugQuery = async (query: T) => { + const sql = JSON.stringify(query.getSQL()); + console.debug(`SQL`, sql); + return query; +}; diff --git a/packages/atproto/feeds/index.ts b/packages/atproto/feeds/index.ts index c1f0b4f..49402c1 100644 --- a/packages/atproto/feeds/index.ts +++ b/packages/atproto/feeds/index.ts @@ -6,6 +6,9 @@ import { config } from "../config"; import type { AtContext } from "../context"; import { getTechAllFeed } from "../domain/get-tech-all-feed"; import { getTechFollowingFeed } from "../domain/get-tech-following-feed"; +import type { repostTable } from "../domain/post/post-reposts.table"; +import type { postTable } from "../domain/post/post.table"; +import { repostsOnlyFeed } from "./only-reposts"; export const DEFAULT_FEED_ACTOR = "did:plc:rrrwbar3wv576qpsymwey5p5"; export type FeedHandlerArgs = { @@ -15,6 +18,14 @@ export type FeedHandlerArgs = { limit?: number; }; +export const FeedDefaultLimit = 30; + +export type FeedSQLSelect = { + id: typeof postTable.id; + date: typeof postTable.lastMentioned | typeof postTable.created; + repost: typeof repostTable.repostUri; +}; + export type FeedHandlerOutput = AppBskyFeedGetFeedSkeleton.Response["data"]; export type FeedDefinition = { @@ -47,4 +58,14 @@ export const feeds: Array = [ }, handler: getTechAllFeed, }, + { + rkey: "tech-reposts", + record: { + did: config.feedGenDid, + displayName: "Tech Reposts", + description: "Experimental", + createdAt: new Date("2025-01-11").toISOString(), + }, + handler: repostsOnlyFeed, + }, ]; diff --git a/packages/atproto/feeds/only-reposts.ts b/packages/atproto/feeds/only-reposts.ts new file mode 100644 index 0000000..5852cfa --- /dev/null +++ b/packages/atproto/feeds/only-reposts.ts @@ -0,0 +1,32 @@ +import { getOrUpdateFollows } from "../domain/get-or-update-follows"; +import type { FeedHandlerArgs, FeedHandlerOutput } from "../feeds"; +import { toCursor } from "../helpers/cursor"; +import { followingRepostsQuery } from "./queries/following-reposts"; + +export async function repostsOnlyFeed( + args: FeedHandlerArgs +): Promise { + const { ctx, actorDid, cursor, limit = 50 } = args; + + await getOrUpdateFollows(ctx, actorDid); + + //const posts = await followingRepostsQuery(args); + const posts = await followingRepostsQuery(args).limit(limit); + + return { + feed: posts.map((p) => { + const reason = p.repost + ? { + repost: p.repost, + } + : undefined; + return { + post: p.id, + reason, + feedContext: "tech-reposts", + } satisfies FeedHandlerOutput[number]; + }), + cursor: + posts.length > 0 ? toCursor(posts[posts.length - 1].date) : undefined, + }; +} diff --git a/packages/atproto/feeds/queries/following-all.ts b/packages/atproto/feeds/queries/following-all.ts new file mode 100644 index 0000000..7a148bf --- /dev/null +++ b/packages/atproto/feeds/queries/following-all.ts @@ -0,0 +1,16 @@ +import { desc } from "drizzle-orm"; +import { union } from "drizzle-orm/pg-core"; +import type { FeedHandlerArgs } from ".."; +import { postTable } from "../../domain/post/post.table"; +import { followingPostsQuery } from "./following-posts"; +import { followingRepostsQuery } from "./following-reposts"; + +export function followingAllQuery(args: FeedHandlerArgs) { + const { ctx, limit = 30, cursor } = args; + const combinedQuery = union(followingPostsQuery, followingRepostsQuery) + .limit(limit) + .orderBy(desc(postTable.lastMentioned)); + return combinedQuery; +} + +// TODO: Read up on WITH clause and CTEs diff --git a/packages/atproto/feeds/queries/following-posts.ts b/packages/atproto/feeds/queries/following-posts.ts new file mode 100644 index 0000000..a2cfe3c --- /dev/null +++ b/packages/atproto/feeds/queries/following-posts.ts @@ -0,0 +1,25 @@ +import { and, eq, gte } from "drizzle-orm"; +import type { FeedHandlerArgs } from ".."; +import { repostTable } from "../../domain/post/post-reposts.table"; +import { postScores } from "../../domain/post/post-scores.view"; +import { postTable } from "../../domain/post/post.table"; +import { followingSubQuery } from "./following"; + +export function followingPostsQuery(args: FeedHandlerArgs) { + const { ctx, actorDid, limit = 30 } = args; + const fls = followingSubQuery(args); + const postsQuery = ctx.db + .select({ + id: postTable.id, + repost: repostTable.repostUri, + date: postTable.lastMentioned, + }) + .from(postTable) + .innerJoin(fls, eq(postTable.authorId, fls.follows)) + .leftJoin(fls, eq(repostTable.authorId, fls.follows)) + .innerJoin( + postScores, + and(eq(postScores.postId, postTable.id), gte(postScores.avgScore, 70)) + ); + return postsQuery; +} diff --git a/packages/atproto/feeds/queries/following-reposts.ts b/packages/atproto/feeds/queries/following-reposts.ts new file mode 100644 index 0000000..f3d4b91 --- /dev/null +++ b/packages/atproto/feeds/queries/following-reposts.ts @@ -0,0 +1,30 @@ +import { and, desc, eq, gte } from "drizzle-orm"; +import type { FeedHandlerArgs } from ".."; +import { repostTable } from "../../domain/post/post-reposts.table"; +import { postScores } from "../../domain/post/post-scores.view"; +import { postTable } from "../../domain/post/post.table"; +import { followingSubQuery } from "./following"; + +export function followingRepostsQuery(args: FeedHandlerArgs) { + const { ctx, limit = 30 } = args; + const fls = followingSubQuery(args); + const repostsQuery = ctx.db + .select({ + id: postTable.id, + repost: repostTable.repostUri, + date: repostTable.created, + }) + .from(postTable) + // Only reposts + .innerJoin(repostTable, eq(repostTable.postId, postTable.id)) + // Only reposts by those I follow + .innerJoin(fls, eq(repostTable.authorId, fls.follows)) + // Only within score condition + .innerJoin( + postScores, + and(eq(postScores.postId, postTable.id), gte(postScores.avgScore, 70)) + ) + .groupBy(postTable.id, repostTable.repostUri) + .orderBy(desc(postTable.lastMentioned)); + return repostsQuery; +} diff --git a/packages/atproto/feeds/queries/following.ts b/packages/atproto/feeds/queries/following.ts new file mode 100644 index 0000000..becc6c7 --- /dev/null +++ b/packages/atproto/feeds/queries/following.ts @@ -0,0 +1,11 @@ +import { eq } from "drizzle-orm"; +import type { FeedHandlerArgs } from ".."; +import { followTable } from "../../domain/user/user-follows.table"; + +export function followingSubQuery({ ctx, actorDid }: FeedHandlerArgs) { + return ctx.db + .select({ follows: followTable.follows }) + .from(followTable) + .where(eq(followTable.followedBy, actorDid)) + .as("fls"); +} -- 2.51.2