From 68bd2fce4b5b608ca68647319910878ee1aa5ef3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andri=20=C3=93skarsson?= <account+github@andri.dk>
Date: Sun, 12 Jan 2025 15:54:55 +0100
Subject: [PATCH] training now done in its own job

---
 apps/worker/crontab                        | 11 ++++-------
 apps/worker/tasks/train-classifier.ts      |  7 +++++++
 packages/atproto/classifiers/tfjs/index.ts |  6 ++----
 3 files changed, 13 insertions(+), 11 deletions(-)
 create mode 100644 apps/worker/tasks/train-classifier.ts

diff --git a/apps/worker/crontab b/apps/worker/crontab
index e397da5..74ad73c 100644
--- a/apps/worker/crontab
+++ b/apps/worker/crontab
@@ -1,7 +1,4 @@
-0 5 * * * cleanup
-0,10,20,30,40,50 * * * * fetch-missing-post-records
-5,15,25,35,45,55 * * * * classify-unclassified-posts
-
-# dev only
-#*/5 * * * * classify-unclassified-posts
-
+0 5 * * * cleanup ?max=1
+0,30 * * * * fetch-missing-post-records ?max=1
+15,45 * * * * classify-unclassified-posts ?max=1
+0 */2 * * * train-classifier ?max=1
\ No newline at end of file
diff --git a/apps/worker/tasks/train-classifier.ts b/apps/worker/tasks/train-classifier.ts
new file mode 100644
index 0000000..2d5571e
--- /dev/null
+++ b/apps/worker/tasks/train-classifier.ts
@@ -0,0 +1,7 @@
+import { createAtContext } from "@andrioid/atproto";
+import { train } from "@andrioid/atproto/classifiers/tfjs/train";
+
+export default async function trainClassifierTask() {
+  const ctx = await createAtContext();
+  await train(ctx);
+}
diff --git a/packages/atproto/classifiers/tfjs/index.ts b/packages/atproto/classifiers/tfjs/index.ts
index e5dd9ba..05eefbc 100644
--- a/packages/atproto/classifiers/tfjs/index.ts
+++ b/packages/atproto/classifiers/tfjs/index.ts
@@ -9,9 +9,7 @@ import { postTexts } from "../../domain/post/post-texts.table";
 import { trainingSetSize } from "../../domain/training-set-size";
 import type { ClassifierFn } from "../types";
 import { classify } from "./classify";
-import { isModelOutdated } from "./is-model-outdated";
 import { loadModelFromDb } from "./loadModelFromDb";
-import { train } from "./train";
 
 type FnType = (ctx: AtContext) => Promise<ClassifierFn | undefined>;
 const MINIMUM_POST_COUNT = 50;
@@ -23,8 +21,8 @@ export const createBayesClassiferFn: FnType = async (ctx) => {
     console.log("[classifier] not enough training data, not running tfjsbayes");
     return;
   }
-  const loader = (await isModelOutdated(ctx)) ? train : loadModelFromDb;
-  const m = await loader(ctx);
+
+  const m = await loadModelFromDb(ctx);
   console.log(`[classifier] tfjsbyes ready with ${m.uniqueWords.length} words`);
   // fetch the model and stuff
   return async function tfjsBayes({
-- 
2.51.2