From 67f1ebc594651592e43be8967f9dff0d36fedc14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andri=20=C3=93skarsson?= Date: Sat, 28 Dec 2024 08:47:04 +0100 Subject: [PATCH] cleaning up old posts so this doesnt burn a hole in my pocket --- fly.toml | 2 +- .../classifiers/tfjs/is-model-outdated.ts | 5 +- packages/atproto/config.ts | 2 +- .../db/drizzle/0015_gorgeous_santa_claus.sql | 4 + .../db/drizzle/0016_violet_namorita.sql | 2 + .../db/drizzle/0017_conscious_arachne.sql | 3 + .../atproto/db/drizzle/0018_fair_iron_lad.sql | 3 + .../db/drizzle/meta/0015_snapshot.json | 496 ++++++++++++++++++ .../db/drizzle/meta/0016_snapshot.json | 496 ++++++++++++++++++ .../db/drizzle/meta/0017_snapshot.json | 496 ++++++++++++++++++ .../db/drizzle/meta/0018_snapshot.json | 496 ++++++++++++++++++ .../atproto/db/drizzle/meta/_journal.json | 28 + packages/atproto/db/schema.ts | 16 +- packages/atproto/domain/cleanup-old-posts.ts | 29 + packages/atproto/feeds/index.ts | 4 +- packages/atproto/scripts/classifier.ts | 9 + packages/atproto/update-feed.ts | 7 + 17 files changed, 2085 insertions(+), 13 deletions(-) create mode 100644 packages/atproto/db/drizzle/0015_gorgeous_santa_claus.sql create mode 100644 packages/atproto/db/drizzle/0016_violet_namorita.sql create mode 100644 packages/atproto/db/drizzle/0017_conscious_arachne.sql create mode 100644 packages/atproto/db/drizzle/0018_fair_iron_lad.sql create mode 100644 packages/atproto/db/drizzle/meta/0015_snapshot.json create mode 100644 packages/atproto/db/drizzle/meta/0016_snapshot.json create mode 100644 packages/atproto/db/drizzle/meta/0017_snapshot.json create mode 100644 packages/atproto/db/drizzle/meta/0018_snapshot.json create mode 100644 packages/atproto/domain/cleanup-old-posts.ts diff --git a/fly.toml b/fly.toml index b7cbbc6..ff6b393 100644 --- a/fly.toml +++ b/fly.toml @@ -18,7 +18,7 @@ jetstream = "jetstream" force_https = true auto_stop_machines = 'stop' auto_start_machines = true - min_machines_running = 0 + min_machines_running = 1 processes = ['app'] [[vm]] diff --git a/packages/atproto/classifiers/tfjs/is-model-outdated.ts b/packages/atproto/classifiers/tfjs/is-model-outdated.ts index 16f5128..da30ca5 100644 --- a/packages/atproto/classifiers/tfjs/is-model-outdated.ts +++ b/packages/atproto/classifiers/tfjs/is-model-outdated.ts @@ -10,8 +10,9 @@ export async function isModelOutdated(ctx: AtContext): Promise { if (!res || res.length === 0) return true; const createdAt = res[0].createdAt; if (!createdAt) return true; - const oneWeekAgo = new Date(Date.now() - 1 * 24 * 60 * 60 * 1000); - if (createdAt < oneWeekAgo) return true; + // currently half a day while im training, then much higher + const maxModelLifeTime = new Date(Date.now() - 0.5 * 24 * 60 * 60 * 1000); + if (createdAt < maxModelLifeTime) return true; return false; } diff --git a/packages/atproto/config.ts b/packages/atproto/config.ts index 2a51868..4a82c53 100644 --- a/packages/atproto/config.ts +++ b/packages/atproto/config.ts @@ -2,7 +2,7 @@ export const config = (() => { const identifier = process.env["DID_HANDLE"]; const password = process.env["DID_PASSWORD"]; const port = process.env["PORT"] ?? 3000; - const fgHostname = process.env["FEEDGEN_HOSTNAME"] ?? "technical.scrolls.org"; + const fgHostname = process.env["FEEDGEN_HOSTNAME"] ?? "techscrolls.fly.dev"; const recordName = "testfeed"; const pgURL = process.env["PG_URL"] ?? "postgres://devuser:devpass@localhost:5432/devdb"; diff --git a/packages/atproto/db/drizzle/0015_gorgeous_santa_claus.sql b/packages/atproto/db/drizzle/0015_gorgeous_santa_claus.sql new file mode 100644 index 0000000..0231e9c --- /dev/null +++ b/packages/atproto/db/drizzle/0015_gorgeous_santa_claus.sql @@ -0,0 +1,4 @@ +ALTER TABLE "post" ALTER COLUMN "created" SET DATA TYPE timestamp using created::timestamp; +ALTER TABLE "post" ALTER COLUMN "created" SET DEFAULT now(); +ALTER TABLE "post" ALTER COLUMN "modified" SET DATA TYPE timestamp using created::timestamp; +ALTER TABLE "post" ALTER COLUMN "modified" SET DEFAULT now(); \ No newline at end of file diff --git a/packages/atproto/db/drizzle/0016_violet_namorita.sql b/packages/atproto/db/drizzle/0016_violet_namorita.sql new file mode 100644 index 0000000..c0bce43 --- /dev/null +++ b/packages/atproto/db/drizzle/0016_violet_namorita.sql @@ -0,0 +1,2 @@ +ALTER TABLE "post" ALTER COLUMN "created" SET DATA TYPE timestamp with time zone using created::timestamp; +ALTER TABLE "post" ALTER COLUMN "modified" SET DATA TYPE timestamp with time zone using created::timestamp; \ No newline at end of file diff --git a/packages/atproto/db/drizzle/0017_conscious_arachne.sql b/packages/atproto/db/drizzle/0017_conscious_arachne.sql new file mode 100644 index 0000000..5888cc0 --- /dev/null +++ b/packages/atproto/db/drizzle/0017_conscious_arachne.sql @@ -0,0 +1,3 @@ +ALTER TABLE "post_record" DROP CONSTRAINT "post_record_post_id_post_id_fk"; +--> statement-breakpoint +ALTER TABLE "post_record" ADD CONSTRAINT "post_record_post_id_post_id_fk" FOREIGN KEY ("post_id") REFERENCES "public"."post"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/atproto/db/drizzle/0018_fair_iron_lad.sql b/packages/atproto/db/drizzle/0018_fair_iron_lad.sql new file mode 100644 index 0000000..506b01f --- /dev/null +++ b/packages/atproto/db/drizzle/0018_fair_iron_lad.sql @@ -0,0 +1,3 @@ +ALTER TABLE "post_tag" DROP CONSTRAINT "post_tag_post_id_post_id_fk"; +--> statement-breakpoint +ALTER TABLE "post_tag" ADD CONSTRAINT "post_tag_post_id_post_id_fk" FOREIGN KEY ("post_id") REFERENCES "public"."post"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/atproto/db/drizzle/meta/0015_snapshot.json b/packages/atproto/db/drizzle/meta/0015_snapshot.json new file mode 100644 index 0000000..92911c4 --- /dev/null +++ b/packages/atproto/db/drizzle/meta/0015_snapshot.json @@ -0,0 +1,496 @@ +{ + "id": "6dc0b285-4921-4639-828c-ed3c8b5aba65", + "prevId": "8504570b-e728-4686-80ad-1dc6e2c3c8bb", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.app": { + "name": "app", + "schema": "", + "columns": { + "k": { + "name": "k", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "v": { + "name": "v", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.follow": { + "name": "follow", + "schema": "", + "columns": { + "followed_by": { + "name": "followed_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "follows": { + "name": "follows", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "followedby_idx": { + "name": "followedby_idx", + "columns": [ + { + "expression": "followed_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "follow_followed_by_user_did_fk": { + "name": "follow_followed_by_user_did_fk", + "tableFrom": "follow", + "tableTo": "user", + "columnsFrom": [ + "followed_by" + ], + "columnsTo": [ + "did" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "follow_follows_followed_by_pk": { + "name": "follow_follows_followed_by_pk", + "columns": [ + "follows", + "followed_by" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_ranking": { + "name": "post_ranking", + "schema": "", + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "algo": { + "name": "algo", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "post_ranking_post_id_post_id_fk": { + "name": "post_ranking_post_id_post_id_fk", + "tableFrom": "post_ranking", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "post_ranking_tag_id_tag_id_fk": { + "name": "post_ranking_tag_id_tag_id_fk", + "tableFrom": "post_ranking", + "tableTo": "tag", + "columnsFrom": [ + "tag_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_record": { + "name": "post_record", + "schema": "", + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cid": { + "name": "cid", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "post_idx": { + "name": "post_idx", + "columns": [ + { + "expression": "post_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "post_record_post_id_post_id_fk": { + "name": "post_record_post_id_post_id_fk", + "tableFrom": "post_record", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post": { + "name": "post", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created": { + "name": "created", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "modified": { + "name": "modified", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_tag": { + "name": "post_tag", + "schema": "", + "columns": { + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "algo": { + "name": "algo", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tagpost_idx": { + "name": "tagpost_idx", + "columns": [ + { + "expression": "post_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tag_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "post_tag_tag_id_tag_id_fk": { + "name": "post_tag_tag_id_tag_id_fk", + "tableFrom": "post_tag", + "tableTo": "tag", + "columnsFrom": [ + "tag_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "post_tag_post_id_post_id_fk": { + "name": "post_tag_post_id_post_id_fk", + "tableFrom": "post_tag", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "post_tag_algo_post_id_tag_id_unique": { + "name": "post_tag_algo_post_id_tag_id_unique", + "nullsNotDistinct": false, + "columns": [ + "algo", + "post_id", + "tag_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tag": { + "name": "tag", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tfjs_model": { + "name": "tfjs_model", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "model_json": { + "name": "model_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "weights": { + "name": "weights", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "unique_words": { + "name": "unique_words", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "word_index": { + "name": "word_index", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "(CURRENT_TIMESTAMP)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "did": { + "name": "did", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "modified": { + "name": "modified", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "(CURRENT_TIMESTAMP)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": { + "public.posts_tag_avg": { + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "definition": "select \"post_id\", \"tag_id\", round(avg(\"score\"), 0) as \"avg_score\", ARRAY_AGG(DISTINCT \"algo\") as \"algos\" from \"post_tag\" group by \"post_tag\".\"post_id\", \"post_tag\".\"tag_id\"", + "name": "posts_tag_avg", + "schema": "public", + "isExisting": false, + "materialized": false + } + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/atproto/db/drizzle/meta/0016_snapshot.json b/packages/atproto/db/drizzle/meta/0016_snapshot.json new file mode 100644 index 0000000..9427ad2 --- /dev/null +++ b/packages/atproto/db/drizzle/meta/0016_snapshot.json @@ -0,0 +1,496 @@ +{ + "id": "fb08f85c-6265-4eef-876a-cecb7bc5c200", + "prevId": "6dc0b285-4921-4639-828c-ed3c8b5aba65", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.app": { + "name": "app", + "schema": "", + "columns": { + "k": { + "name": "k", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "v": { + "name": "v", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.follow": { + "name": "follow", + "schema": "", + "columns": { + "followed_by": { + "name": "followed_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "follows": { + "name": "follows", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "followedby_idx": { + "name": "followedby_idx", + "columns": [ + { + "expression": "followed_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "follow_followed_by_user_did_fk": { + "name": "follow_followed_by_user_did_fk", + "tableFrom": "follow", + "tableTo": "user", + "columnsFrom": [ + "followed_by" + ], + "columnsTo": [ + "did" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "follow_follows_followed_by_pk": { + "name": "follow_follows_followed_by_pk", + "columns": [ + "follows", + "followed_by" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_ranking": { + "name": "post_ranking", + "schema": "", + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "algo": { + "name": "algo", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "post_ranking_post_id_post_id_fk": { + "name": "post_ranking_post_id_post_id_fk", + "tableFrom": "post_ranking", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "post_ranking_tag_id_tag_id_fk": { + "name": "post_ranking_tag_id_tag_id_fk", + "tableFrom": "post_ranking", + "tableTo": "tag", + "columnsFrom": [ + "tag_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_record": { + "name": "post_record", + "schema": "", + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cid": { + "name": "cid", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "post_idx": { + "name": "post_idx", + "columns": [ + { + "expression": "post_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "post_record_post_id_post_id_fk": { + "name": "post_record_post_id_post_id_fk", + "tableFrom": "post_record", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post": { + "name": "post", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created": { + "name": "created", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "modified": { + "name": "modified", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_tag": { + "name": "post_tag", + "schema": "", + "columns": { + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "algo": { + "name": "algo", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tagpost_idx": { + "name": "tagpost_idx", + "columns": [ + { + "expression": "post_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tag_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "post_tag_tag_id_tag_id_fk": { + "name": "post_tag_tag_id_tag_id_fk", + "tableFrom": "post_tag", + "tableTo": "tag", + "columnsFrom": [ + "tag_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "post_tag_post_id_post_id_fk": { + "name": "post_tag_post_id_post_id_fk", + "tableFrom": "post_tag", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "post_tag_algo_post_id_tag_id_unique": { + "name": "post_tag_algo_post_id_tag_id_unique", + "nullsNotDistinct": false, + "columns": [ + "algo", + "post_id", + "tag_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tag": { + "name": "tag", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tfjs_model": { + "name": "tfjs_model", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "model_json": { + "name": "model_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "weights": { + "name": "weights", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "unique_words": { + "name": "unique_words", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "word_index": { + "name": "word_index", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "(CURRENT_TIMESTAMP)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "did": { + "name": "did", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "modified": { + "name": "modified", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "(CURRENT_TIMESTAMP)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": { + "public.posts_tag_avg": { + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "definition": "select \"post_id\", \"tag_id\", round(avg(\"score\"), 0) as \"avg_score\", ARRAY_AGG(DISTINCT \"algo\") as \"algos\" from \"post_tag\" group by \"post_tag\".\"post_id\", \"post_tag\".\"tag_id\"", + "name": "posts_tag_avg", + "schema": "public", + "isExisting": false, + "materialized": false + } + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/atproto/db/drizzle/meta/0017_snapshot.json b/packages/atproto/db/drizzle/meta/0017_snapshot.json new file mode 100644 index 0000000..837164e --- /dev/null +++ b/packages/atproto/db/drizzle/meta/0017_snapshot.json @@ -0,0 +1,496 @@ +{ + "id": "dd8d2c35-b800-499d-a9fb-a542ffafb321", + "prevId": "fb08f85c-6265-4eef-876a-cecb7bc5c200", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.app": { + "name": "app", + "schema": "", + "columns": { + "k": { + "name": "k", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "v": { + "name": "v", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.follow": { + "name": "follow", + "schema": "", + "columns": { + "followed_by": { + "name": "followed_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "follows": { + "name": "follows", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "followedby_idx": { + "name": "followedby_idx", + "columns": [ + { + "expression": "followed_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "follow_followed_by_user_did_fk": { + "name": "follow_followed_by_user_did_fk", + "tableFrom": "follow", + "tableTo": "user", + "columnsFrom": [ + "followed_by" + ], + "columnsTo": [ + "did" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "follow_follows_followed_by_pk": { + "name": "follow_follows_followed_by_pk", + "columns": [ + "follows", + "followed_by" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_ranking": { + "name": "post_ranking", + "schema": "", + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "algo": { + "name": "algo", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "post_ranking_post_id_post_id_fk": { + "name": "post_ranking_post_id_post_id_fk", + "tableFrom": "post_ranking", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "post_ranking_tag_id_tag_id_fk": { + "name": "post_ranking_tag_id_tag_id_fk", + "tableFrom": "post_ranking", + "tableTo": "tag", + "columnsFrom": [ + "tag_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_record": { + "name": "post_record", + "schema": "", + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cid": { + "name": "cid", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "post_idx": { + "name": "post_idx", + "columns": [ + { + "expression": "post_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "post_record_post_id_post_id_fk": { + "name": "post_record_post_id_post_id_fk", + "tableFrom": "post_record", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post": { + "name": "post", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created": { + "name": "created", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "modified": { + "name": "modified", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_tag": { + "name": "post_tag", + "schema": "", + "columns": { + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "algo": { + "name": "algo", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tagpost_idx": { + "name": "tagpost_idx", + "columns": [ + { + "expression": "post_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tag_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "post_tag_tag_id_tag_id_fk": { + "name": "post_tag_tag_id_tag_id_fk", + "tableFrom": "post_tag", + "tableTo": "tag", + "columnsFrom": [ + "tag_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "post_tag_post_id_post_id_fk": { + "name": "post_tag_post_id_post_id_fk", + "tableFrom": "post_tag", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "post_tag_algo_post_id_tag_id_unique": { + "name": "post_tag_algo_post_id_tag_id_unique", + "nullsNotDistinct": false, + "columns": [ + "algo", + "post_id", + "tag_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tag": { + "name": "tag", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tfjs_model": { + "name": "tfjs_model", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "model_json": { + "name": "model_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "weights": { + "name": "weights", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "unique_words": { + "name": "unique_words", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "word_index": { + "name": "word_index", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "(CURRENT_TIMESTAMP)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "did": { + "name": "did", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "modified": { + "name": "modified", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "(CURRENT_TIMESTAMP)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": { + "public.posts_tag_avg": { + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "definition": "select \"post_id\", \"tag_id\", round(avg(\"score\"), 0) as \"avg_score\", ARRAY_AGG(DISTINCT \"algo\") as \"algos\" from \"post_tag\" group by \"post_tag\".\"post_id\", \"post_tag\".\"tag_id\"", + "name": "posts_tag_avg", + "schema": "public", + "isExisting": false, + "materialized": false + } + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/atproto/db/drizzle/meta/0018_snapshot.json b/packages/atproto/db/drizzle/meta/0018_snapshot.json new file mode 100644 index 0000000..4bafe18 --- /dev/null +++ b/packages/atproto/db/drizzle/meta/0018_snapshot.json @@ -0,0 +1,496 @@ +{ + "id": "4fb64707-3c78-4414-9921-2e88b826239d", + "prevId": "dd8d2c35-b800-499d-a9fb-a542ffafb321", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.app": { + "name": "app", + "schema": "", + "columns": { + "k": { + "name": "k", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "v": { + "name": "v", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.follow": { + "name": "follow", + "schema": "", + "columns": { + "followed_by": { + "name": "followed_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "follows": { + "name": "follows", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "followedby_idx": { + "name": "followedby_idx", + "columns": [ + { + "expression": "followed_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "follow_followed_by_user_did_fk": { + "name": "follow_followed_by_user_did_fk", + "tableFrom": "follow", + "tableTo": "user", + "columnsFrom": [ + "followed_by" + ], + "columnsTo": [ + "did" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "follow_follows_followed_by_pk": { + "name": "follow_follows_followed_by_pk", + "columns": [ + "follows", + "followed_by" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_ranking": { + "name": "post_ranking", + "schema": "", + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "algo": { + "name": "algo", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "post_ranking_post_id_post_id_fk": { + "name": "post_ranking_post_id_post_id_fk", + "tableFrom": "post_ranking", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "post_ranking_tag_id_tag_id_fk": { + "name": "post_ranking_tag_id_tag_id_fk", + "tableFrom": "post_ranking", + "tableTo": "tag", + "columnsFrom": [ + "tag_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_record": { + "name": "post_record", + "schema": "", + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cid": { + "name": "cid", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "post_idx": { + "name": "post_idx", + "columns": [ + { + "expression": "post_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "post_record_post_id_post_id_fk": { + "name": "post_record_post_id_post_id_fk", + "tableFrom": "post_record", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post": { + "name": "post", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created": { + "name": "created", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "modified": { + "name": "modified", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.post_tag": { + "name": "post_tag", + "schema": "", + "columns": { + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "algo": { + "name": "algo", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tagpost_idx": { + "name": "tagpost_idx", + "columns": [ + { + "expression": "post_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tag_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "post_tag_tag_id_tag_id_fk": { + "name": "post_tag_tag_id_tag_id_fk", + "tableFrom": "post_tag", + "tableTo": "tag", + "columnsFrom": [ + "tag_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "post_tag_post_id_post_id_fk": { + "name": "post_tag_post_id_post_id_fk", + "tableFrom": "post_tag", + "tableTo": "post", + "columnsFrom": [ + "post_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "post_tag_algo_post_id_tag_id_unique": { + "name": "post_tag_algo_post_id_tag_id_unique", + "nullsNotDistinct": false, + "columns": [ + "algo", + "post_id", + "tag_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tag": { + "name": "tag", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tfjs_model": { + "name": "tfjs_model", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "model_json": { + "name": "model_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "weights": { + "name": "weights", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "unique_words": { + "name": "unique_words", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "word_index": { + "name": "word_index", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "(CURRENT_TIMESTAMP)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "did": { + "name": "did", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "modified": { + "name": "modified", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "(CURRENT_TIMESTAMP)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": { + "public.posts_tag_avg": { + "columns": { + "post_id": { + "name": "post_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "definition": "select \"post_id\", \"tag_id\", round(avg(\"score\"), 0) as \"avg_score\", ARRAY_AGG(DISTINCT \"algo\") as \"algos\" from \"post_tag\" group by \"post_tag\".\"post_id\", \"post_tag\".\"tag_id\"", + "name": "posts_tag_avg", + "schema": "public", + "isExisting": false, + "materialized": false + } + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/atproto/db/drizzle/meta/_journal.json b/packages/atproto/db/drizzle/meta/_journal.json index 73e66af..73940f4 100644 --- a/packages/atproto/db/drizzle/meta/_journal.json +++ b/packages/atproto/db/drizzle/meta/_journal.json @@ -106,6 +106,34 @@ "when": 1735196841478, "tag": "0014_elite_goblin_queen", "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1735370837187, + "tag": "0015_gorgeous_santa_claus", + "breakpoints": true + }, + { + "idx": 16, + "version": "7", + "when": 1735371017061, + "tag": "0016_violet_namorita", + "breakpoints": true + }, + { + "idx": 17, + "version": "7", + "when": 1735371721014, + "tag": "0017_conscious_arachne", + "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1735371927796, + "tag": "0018_fair_iron_lad", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/atproto/db/schema.ts b/packages/atproto/db/schema.ts index 2958c8d..9de196b 100644 --- a/packages/atproto/db/schema.ts +++ b/packages/atproto/db/schema.ts @@ -45,12 +45,12 @@ export const postTable = pgTable( { id: text().primaryKey(), // bluesky post id authorId: text("author_id").notNull(), - created: text() - .default(sql`(CURRENT_TIMESTAMP)`) - .notNull(), - modified: text() - .default(sql`(CURRENT_TIMESTAMP)`) + created: timestamp({ + withTimezone: true, + }) + .defaultNow() .notNull(), + modified: timestamp({ withTimezone: true }).defaultNow().notNull(), }, () => [] ); @@ -61,7 +61,7 @@ export const postRecords = pgTable( { postId: text("post_id") .primaryKey() - .references(() => postTable.id), + .references(() => postTable.id, { onDelete: "cascade" }), type: text({ enum: ["AppBskyFeedPost.Record"], }).notNull(), @@ -88,7 +88,9 @@ export const postTags = pgTable( .references(() => tagTable.id), postId: text("post_id") .notNull() - .references(() => postTable.id), + .references(() => postTable.id, { + onDelete: "cascade", + }), // Allows an algorithm to give post a relevancy score for tag score: integer("score").notNull(), // 0-100 // popularity should be in its own table, regardless of algorithms diff --git a/packages/atproto/domain/cleanup-old-posts.ts b/packages/atproto/domain/cleanup-old-posts.ts new file mode 100644 index 0000000..2b4f83f --- /dev/null +++ b/packages/atproto/domain/cleanup-old-posts.ts @@ -0,0 +1,29 @@ +import { and, eq, inArray, isNull, lte } from "drizzle-orm"; +import type { AtContext } from "../context"; +import { postRecords, postTable, postTags } from "../db/schema"; + +const DAYS_IN_MS = 1000 * 60 * 60 * 24 * 7; + +export async function cleanupOldPosts(ctx: AtContext) { + // Delete anything older than X (currently 3 days) + const postLifeTime = new Date(new Date().getTime() - (DAYS_IN_MS / 7) * 3); + // Get all posts that don't have a manual tag + const res = await ctx.db + .select({ uri: postTable.id }) + .from(postTable) + .innerJoin(postRecords, eq(postTable.id, postRecords.postId)) + // This will populate the tags with null for posts that are not manually tagged + .leftJoin( + postTags, + and(eq(postTable.id, postTags.postId), eq(postTags.algo, "manual")) + ) + .where(and(lte(postTable.created, postLifeTime), isNull(postTags.tagId))); + + console.log(`[cleanup] Deleting ${res.length} old posts`); + await ctx.db.delete(postTable).where( + inArray( + postTable.id, + res.map((r) => r.uri) + ) + ); +} diff --git a/packages/atproto/feeds/index.ts b/packages/atproto/feeds/index.ts index f95060c..b8c7ca5 100644 --- a/packages/atproto/feeds/index.ts +++ b/packages/atproto/feeds/index.ts @@ -25,7 +25,7 @@ export const feeds: Array = [ description: "Posts from your following; filtered for anything that isn't considered tech related. \n\nNOTE: Under heavy development and without hosting atm", //avatar: avatarRef, - createdAt: new Date().toISOString(), + createdAt: new Date("2024-12-19").toISOString(), }, handler: getTechFollowingFeed, }, @@ -37,7 +37,7 @@ export const feeds: Array = [ description: "Curated tech posts from across the network.\n\nNOTE: Under heavy development and without hosting atm", //avatar: avatarRef, - createdAt: new Date().toISOString(), + createdAt: new Date("2024-12-19").toISOString(), }, handler: getTechAllFeed, }, diff --git a/packages/atproto/scripts/classifier.ts b/packages/atproto/scripts/classifier.ts index b417a52..74bfbb5 100644 --- a/packages/atproto/scripts/classifier.ts +++ b/packages/atproto/scripts/classifier.ts @@ -7,13 +7,18 @@ import type { ClassifierFn } from "../classifiers/types"; import { createAtContext } from "../context"; import { postRecords, postTable, postTags } from "../db/schema"; import { classifyPost } from "../domain/classify-manually"; +import { cleanupOldPosts } from "../domain/cleanup-old-posts"; import type { FeedPostWithUri } from "../domain/queue-for-classification"; export const LISTEN_NOTIFY_POSTQUEUE = "atproto.postqueue"; export async function classifier() { + const birthDay = new Date(); // We'll let it rest a bit after a while + // 1. Process existing records in batches const ctx = await createAtContext(); + await cleanupOldPosts(ctx); + const classifiers: Array = [ mutedWordsClassifier, techWordsRegExp, @@ -79,6 +84,10 @@ export async function classifier() { const t1 = performance.now(); console.log(`[classifier] ${p.uri} in ${(t1 - t0).toFixed(2)} ms`); + if (new Date().getTime() - birthDay.getTime() > 2 * 60 * 60 * 1000) { + // Time to rest. Fly will launch another one. + process.exit(0); + } }); //while (true) {} // listen doesn't wait } diff --git a/packages/atproto/update-feed.ts b/packages/atproto/update-feed.ts index d1d3eff..7a606d4 100644 --- a/packages/atproto/update-feed.ts +++ b/packages/atproto/update-feed.ts @@ -17,6 +17,13 @@ const run = async () => { }); } /* + await agent.api.com.atproto.repo.deleteRecord({ + repo: agent.session?.did ?? "", + collection: "app.bsky.feed.generator", + rkey: "testfeed", + }); + */ + /* await agent.api.com.atproto.repo.putRecord({ repo: agent.session?.did ?? "", collection: "app.bsky.feed.generator", -- 2.51.2