From 2c2b6a64fb0e6c49161ac6f5359f19f314dc6746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andri=20=C3=93skarsson?= <account+github@andri.dk>
Date: Sat, 11 Jan 2025 13:38:36 +0100
Subject: [PATCH] classifier task update

---
 apps/worker/crontab                    | 8 ++++++--
 packages/atproto/scripts/classifier.ts | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/apps/worker/crontab b/apps/worker/crontab
index 2d6d93b..549ae1f 100644
--- a/apps/worker/crontab
+++ b/apps/worker/crontab
@@ -1,3 +1,7 @@
-*/15 * * * * fetch-missing-post-records
-*/60 * * * * classify-unclassified-posts
 0 5 * * * cleanup
+0,15,30,45 * * * * fetch-missing-post-records
+5,20,35,50 * * * * classify-unclassified-posts
+
+# dev only
+#*/5 * * * * classify-unclassified-posts
+
diff --git a/packages/atproto/scripts/classifier.ts b/packages/atproto/scripts/classifier.ts
index e39399a..a17461c 100644
--- a/packages/atproto/scripts/classifier.ts
+++ b/packages/atproto/scripts/classifier.ts
@@ -1,4 +1,5 @@
-import { and, eq, inArray, isNull } from "drizzle-orm";
+import { subHours } from "date-fns";
+import { and, eq, gte, inArray, isNull } from "drizzle-orm";
 import { mutedWordsClassifier } from "../classifiers/tech/muted-words";
 import { techLinkClassifier } from "../classifiers/tech/tech-links";
 import { techWordsRegExp } from "../classifiers/tech/tech-words";
@@ -8,6 +9,7 @@ import { createAtContext } from "../context";
 import { classifyPost } from "../domain/classify-manually";
 import { postRecords } from "../domain/post/post-record.table";
 import { postTags } from "../domain/post/post-tag.table";
+import { postTable } from "../domain/post/post.table";
 
 export const LISTEN_NOTIFY_POSTQUEUE = "atproto.postqueue";
 const POSTS_PER_CLASSIFIER_RUN = 500;
@@ -31,7 +33,7 @@ export async function classifier(postUri?: Array<string>) {
       console.warn("Classifier has no name, ignoring");
       continue;
     }
-    // TODO: Pagination, so we only handle 100 posts at a time
+    // TODO: Pagination, so we only handle LIMIT posts at a time
     const unclassifiedPosts = await ctx.db
       .select({
         postId: postRecords.postId,
@@ -39,6 +41,7 @@ export async function classifier(postUri?: Array<string>) {
         cid: postRecords.cid,
       })
       .from(postRecords)
+      .innerJoin(postTable, eq(postRecords.postId, postTable.id))
       .leftJoin(
         postTags,
         and(
@@ -48,6 +51,7 @@ export async function classifier(postUri?: Array<string>) {
       )
       .where(
         and(
+          gte(postTable.lastMentioned, subHours(new Date(), 24)),
           isNull(postTags.postId),
           // If specified, otherwise ignore
           postUri ? inArray(postRecords.postId, postUri) : undefined
-- 
2.54.0