From 3d9fe82d83b945d2fe21fb4ffa84c568d24a0ff1 Mon Sep 17 00:00:00 2001 From: Timothy Quilling Date: Thu, 30 Oct 2025 10:54:34 -0400 Subject: [PATCH] feat: database hacks - big model change --- diesel.toml | 2 +- fk.patch | 18 - .../down.sql | 10 - .../up.sql | 3 - .../down.sql | 9 + .../up.sql | 440 ++++++++++++++++++ parakeet-db/src/models.rs | 103 +++- parakeet-db/src/schema.rs | 189 +++----- 8 files changed, 604 insertions(+), 170 deletions(-) delete mode 100644 fk.patch delete mode 100644 migrations/2025-10-29-164300_drop_records_table/down.sql delete mode 100644 migrations/2025-10-29-164300_drop_records_table/up.sql create mode 100644 migrations/2025-10-30-020000_optimize_records_table/down.sql create mode 100644 migrations/2025-10-30-020000_optimize_records_table/up.sql diff --git a/diesel.toml b/diesel.toml index ab40166b..632ac073 100644 --- a/diesel.toml +++ b/diesel.toml @@ -4,7 +4,7 @@ [print_schema] file = "parakeet-db/src/schema.rs" custom_type_derives = ["diesel::query_builder::QueryId"] -patch_file = "fk.patch" +# patch_file = "fk.patch" [migrations_directory] dir = "migrations" diff --git a/fk.patch b/fk.patch deleted file mode 100644 index 6ab63ef4..00000000 --- a/fk.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/parakeet-db/src/schema.rs b/parakeet-db/src/schema.rs -index 59f65d9..a4219d5 100644 ---- a/parakeet-db/src/schema.rs -+++ b/parakeet-db/src/schema.rs -@@ -364,11 +364,13 @@ diesel::joinable!(post_embed_images -> posts (post_uri)); - diesel::joinable!(post_embed_record -> posts (post_uri)); - diesel::joinable!(post_embed_video -> posts (post_uri)); - diesel::joinable!(post_embed_video_captions -> posts (post_uri)); -+diesel::joinable!(postgates -> posts (post_uri)); - diesel::joinable!(posts -> actors (did)); - diesel::joinable!(profiles -> actors (did)); - diesel::joinable!(reposts -> actors (did)); - diesel::joinable!(starterpacks -> actors (owner)); - diesel::joinable!(statuses -> actors (did)); -+diesel::joinable!(threadgates -> posts (post_uri)); - diesel::joinable!(verification -> actors (verifier)); - - diesel::allow_tables_to_appear_in_same_query!( diff --git a/migrations/2025-10-29-164300_drop_records_table/down.sql b/migrations/2025-10-29-164300_drop_records_table/down.sql deleted file mode 100644 index 8d79d61c..00000000 --- a/migrations/2025-10-29-164300_drop_records_table/down.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Recreate the generic records table (if rollback is needed) -CREATE TABLE records ( - at_uri TEXT PRIMARY KEY, - did TEXT NOT NULL, - cid BYTEA NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT NOW() -); - -CREATE INDEX idx_records_did ON records(did); -CREATE INDEX idx_records_created_at ON records(created_at); diff --git a/migrations/2025-10-29-164300_drop_records_table/up.sql b/migrations/2025-10-29-164300_drop_records_table/up.sql deleted file mode 100644 index b6c98306..00000000 --- a/migrations/2025-10-29-164300_drop_records_table/up.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Drop the generic records table --- This table was used for duplicate detection, but we now check collection-specific tables directly -DROP TABLE IF EXISTS records; diff --git a/migrations/2025-10-30-020000_optimize_records_table/down.sql b/migrations/2025-10-30-020000_optimize_records_table/down.sql new file mode 100644 index 00000000..c73eae5a --- /dev/null +++ b/migrations/2025-10-30-020000_optimize_records_table/down.sql @@ -0,0 +1,9 @@ +-- Rollback: This migration cannot be rolled back safely +-- +-- The old tables are dropped as part of the forward migration, so rollback +-- would result in complete data loss. +-- +-- If you need to rollback, you must restore from a backup taken before +-- running the migration. + +SELECT 'ERROR: This migration cannot be rolled back. Restore from backup instead.' as error; diff --git a/migrations/2025-10-30-020000_optimize_records_table/up.sql b/migrations/2025-10-30-020000_optimize_records_table/up.sql new file mode 100644 index 00000000..0912744e --- /dev/null +++ b/migrations/2025-10-30-020000_optimize_records_table/up.sql @@ -0,0 +1,440 @@ +-- Comprehensive storage optimization: Normalize DIDs, optimize CIDs, use FKs to records +-- +-- This migration optimizes ALL major tables in one pass: +-- - records: Remove at_uri, use actor_id + collection ENUM + rkey +-- - likes: Replace did + subject + subject_cid with actor_id + subject_record_id +-- - reposts: Replace did + post + post_cid with actor_id + post_record_id +-- - posts: Replace did + parent/root URIs with actor_id + parent/root_record_id, strip CID headers +-- - follows: Replace did with actor_id +-- - blocks: Replace did with actor_id +-- - notifications: Replace DIDs with actor_ids, strip CID header +-- +-- Total storage savings: ~6-8 GB (17-22% reduction) + +-- Step 1: Ensure actors table has id column +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_name = 'actors' AND column_name = 'id') THEN + ALTER TABLE actors ADD COLUMN id SERIAL UNIQUE; + END IF; +END $$; + +-- Step 2: Create record_type ENUM +-- Extract all unique collection types from existing records +DO $$ +DECLARE + collection_types TEXT[]; +BEGIN + -- Get all unique collection types + SELECT array_agg(DISTINCT SPLIT_PART(at_uri, '/', 4) ORDER BY SPLIT_PART(at_uri, '/', 4)) + INTO collection_types + FROM records + WHERE at_uri IS NOT NULL; + + -- Create enum type if it doesn't exist + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'record_type') THEN + EXECUTE format('CREATE TYPE record_type AS ENUM (%s)', + (SELECT string_agg(quote_literal(t), ', ') FROM unnest(collection_types) t) + ); + END IF; +END $$; + +-- Step 3: Create new optimized records table +CREATE TABLE records_new ( + id BIGSERIAL PRIMARY KEY, + actor_id INTEGER NOT NULL, + collection record_type NOT NULL, -- PostgreSQL ENUM (4 bytes) + rkey TEXT NOT NULL, + cid BYTEA NOT NULL, -- 32 bytes (header stripped), not 36 + indexed_at TIMESTAMP NOT NULL +); + +-- Step 4: Copy and transform data in a single pass +-- This is the expensive operation but happens once +INSERT INTO records_new (actor_id, collection, rkey, cid, indexed_at) +SELECT + a.id, + SPLIT_PART(r.at_uri, '/', 4)::record_type as collection, + SPLIT_PART(r.at_uri, '/', 5) as rkey, + -- Strip 4-byte CID header (0x01711220), keep only 32-byte digest + substring(r.cid from 5) as cid_digest, + r.indexed_at +FROM records r +INNER JOIN actors a ON r.did = a.did +WHERE a.id IS NOT NULL; -- Only migrate records where actor has ID + +-- Step 5: Create indexes on new table +CREATE UNIQUE INDEX idx_records_new_composite ON records_new(actor_id, collection, rkey); +CREATE INDEX idx_records_new_actor_id ON records_new(actor_id); +CREATE INDEX idx_records_new_collection ON records_new(collection); +CREATE INDEX idx_records_new_indexed_at ON records_new(indexed_at); + +-- Step 6: Add foreign key constraint +ALTER TABLE records_new + ADD CONSTRAINT fk_records_actor + FOREIGN KEY (actor_id) REFERENCES actors(id) ON DELETE CASCADE; + +-- Step 7: Swap tables (brief exclusive lock) +ALTER TABLE records RENAME TO records_old; +ALTER TABLE records_new RENAME TO records; + +-- Step 8: Update actors primary key to use id instead of did +ALTER TABLE actors DROP CONSTRAINT IF EXISTS actors_pkey CASCADE; +ALTER TABLE actors ADD PRIMARY KEY (id); +CREATE UNIQUE INDEX idx_actors_did ON actors(did); + +-- Step 8a: Drop old records table immediately +DROP TABLE records_old; + +-------------------------------------------------------------------------------- +-- LIKES TABLE OPTIMIZATION +-------------------------------------------------------------------------------- + +-- Step 9: Create optimized likes table +CREATE TABLE likes_new ( + id BIGSERIAL PRIMARY KEY, + rkey TEXT NOT NULL, + actor_id INTEGER NOT NULL, + subject_record_id BIGINT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + indexed_at TIMESTAMP NOT NULL, + via_record_id BIGINT -- NULL if no via +); + +-- Step 10: Transform likes data +-- Strategy: Parse subject URI, join to records via composite key +INSERT INTO likes_new (rkey, actor_id, subject_record_id, created_at, indexed_at, via_record_id) +SELECT + l.rkey, + a.id, + r_subject.id, + l.created_at, + l.indexed_at, + r_via.id +FROM likes l +INNER JOIN actors a ON l.did = a.did +-- Parse subject URI and find matching record +INNER JOIN actors actors_subj ON SPLIT_PART(l.subject, '/', 3) = actors_subj.did +INNER JOIN records r_subject ON + r_subject.actor_id = actors_subj.id + AND r_subject.collection::text = SPLIT_PART(l.subject, '/', 4) + AND r_subject.rkey = SPLIT_PART(l.subject, '/', 5) +-- Parse via_uri and find matching record if exists +LEFT JOIN actors actors_via ON l.via_uri IS NOT NULL AND SPLIT_PART(l.via_uri, '/', 3) = actors_via.did +LEFT JOIN records r_via ON + r_via.actor_id = actors_via.id + AND r_via.collection::text = SPLIT_PART(l.via_uri, '/', 4) + AND r_via.rkey = SPLIT_PART(l.via_uri, '/', 5) +WHERE a.id IS NOT NULL; + +-- Step 11: Create indexes on likes_new +CREATE UNIQUE INDEX idx_likes_new_actor_rkey ON likes_new(actor_id, rkey); +CREATE INDEX idx_likes_new_subject ON likes_new(subject_record_id); +CREATE INDEX idx_likes_new_indexed_at ON likes_new(indexed_at); +CREATE INDEX idx_likes_new_via ON likes_new(via_record_id) WHERE via_record_id IS NOT NULL; + +-- Step 12: Add foreign key constraints for likes +ALTER TABLE likes_new + ADD CONSTRAINT fk_likes_actor FOREIGN KEY (actor_id) REFERENCES actors(id) ON DELETE CASCADE, + ADD CONSTRAINT fk_likes_subject FOREIGN KEY (subject_record_id) REFERENCES records(id) ON DELETE CASCADE, + ADD CONSTRAINT fk_likes_via FOREIGN KEY (via_record_id) REFERENCES records(id) ON DELETE SET NULL; + +-- Step 13: Swap likes tables +ALTER TABLE likes RENAME TO likes_old; +ALTER TABLE likes_new RENAME TO likes; + +-- Step 13a: Drop old likes table immediately +DROP TABLE likes_old; + +-------------------------------------------------------------------------------- +-- REPOSTS TABLE OPTIMIZATION +-------------------------------------------------------------------------------- + +-- Step 14: Create optimized reposts table +CREATE TABLE reposts_new ( + id BIGSERIAL PRIMARY KEY, + rkey TEXT NOT NULL, + actor_id INTEGER NOT NULL, + post_record_id BIGINT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + indexed_at TIMESTAMP NOT NULL, + via_record_id BIGINT -- NULL if no via +); + +-- Step 15: Transform reposts data +INSERT INTO reposts_new (rkey, actor_id, post_record_id, created_at, indexed_at, via_record_id) +SELECT + rp.rkey, + a.id, + r_post.id, + rp.created_at, + rp.indexed_at, + r_via.id +FROM reposts rp +INNER JOIN actors a ON rp.did = a.did +-- Parse post URI and find matching record +INNER JOIN actors actors_post ON SPLIT_PART(rp.post, '/', 3) = actors_post.did +INNER JOIN records r_post ON + r_post.actor_id = actors_post.id + AND r_post.collection::text = SPLIT_PART(rp.post, '/', 4) + AND r_post.rkey = SPLIT_PART(rp.post, '/', 5) +-- Parse via_uri and find matching record if exists +LEFT JOIN actors actors_via ON rp.via_uri IS NOT NULL AND SPLIT_PART(rp.via_uri, '/', 3) = actors_via.did +LEFT JOIN records r_via ON + r_via.actor_id = actors_via.id + AND r_via.collection::text = SPLIT_PART(rp.via_uri, '/', 4) + AND r_via.rkey = SPLIT_PART(rp.via_uri, '/', 5) +WHERE a.id IS NOT NULL; + +-- Step 16: Create indexes on reposts_new +CREATE UNIQUE INDEX idx_reposts_new_actor_rkey ON reposts_new(actor_id, rkey); +CREATE INDEX idx_reposts_new_post ON reposts_new(post_record_id); +CREATE INDEX idx_reposts_new_indexed_at ON reposts_new(indexed_at); +CREATE INDEX idx_reposts_new_via ON reposts_new(via_record_id) WHERE via_record_id IS NOT NULL; + +-- Step 17: Add foreign key constraints for reposts +ALTER TABLE reposts_new + ADD CONSTRAINT fk_reposts_actor FOREIGN KEY (actor_id) REFERENCES actors(id) ON DELETE CASCADE, + ADD CONSTRAINT fk_reposts_post FOREIGN KEY (post_record_id) REFERENCES records(id) ON DELETE CASCADE, + ADD CONSTRAINT fk_reposts_via FOREIGN KEY (via_record_id) REFERENCES records(id) ON DELETE SET NULL; + +-- Step 18: Swap reposts tables +ALTER TABLE reposts RENAME TO reposts_old; +ALTER TABLE reposts_new RENAME TO reposts; + +-- Step 18a: Drop old reposts table immediately +DROP TABLE reposts_old; + +-------------------------------------------------------------------------------- +-- POSTS TABLE OPTIMIZATION +-------------------------------------------------------------------------------- + +-- Step 19: Create optimized posts table +CREATE TABLE posts_new ( + id BIGSERIAL PRIMARY KEY, + at_uri TEXT NOT NULL UNIQUE, -- Keep at_uri as it's referenced by many tables + cid_digest BYTEA NOT NULL, -- Strip header, 32 bytes + actor_id INTEGER NOT NULL, + record JSONB NOT NULL, + + content TEXT NOT NULL, + facets JSONB, + languages TEXT[] NOT NULL, + tags TEXT[] NOT NULL, + + parent_record_id BIGINT, -- FK to records instead of URI+CID + root_record_id BIGINT, -- FK to records instead of URI+CID + + embed TEXT, + embed_subtype TEXT, + + mentions TEXT[], + violates_threadgate BOOLEAN NOT NULL DEFAULT false, + + created_at TIMESTAMPTZ NOT NULL, + indexed_at TIMESTAMP NOT NULL +); + +-- Step 20: Transform posts data +-- Note: posts.cid is TEXT (base32), we need to convert to binary digest +INSERT INTO posts_new (at_uri, cid_digest, actor_id, record, content, facets, languages, tags, + parent_record_id, root_record_id, embed, embed_subtype, mentions, + violates_threadgate, created_at, indexed_at) +SELECT + p.at_uri, + decode(substring(p.cid from 9), 'hex'), -- Strip text CID header, store binary digest + a.id, + p.record, + p.content, + p.facets, + p.languages, + p.tags, + r_parent.id, + r_root.id, + p.embed, + p.embed_subtype, + p.mentions, + p.violates_threadgate, + p.created_at, + p.indexed_at +FROM posts p +INNER JOIN actors a ON p.did = a.did +-- Parse parent_uri and find matching record if exists +LEFT JOIN actors actors_parent ON p.parent_uri IS NOT NULL AND SPLIT_PART(p.parent_uri, '/', 3) = actors_parent.did +LEFT JOIN records r_parent ON + r_parent.actor_id = actors_parent.id + AND r_parent.collection::text = SPLIT_PART(p.parent_uri, '/', 4) + AND r_parent.rkey = SPLIT_PART(p.parent_uri, '/', 5) +-- Parse root_uri and find matching record if exists +LEFT JOIN actors actors_root ON p.root_uri IS NOT NULL AND SPLIT_PART(p.root_uri, '/', 3) = actors_root.did +LEFT JOIN records r_root ON + r_root.actor_id = actors_root.id + AND r_root.collection::text = SPLIT_PART(p.root_uri, '/', 4) + AND r_root.rkey = SPLIT_PART(p.root_uri, '/', 5) +WHERE a.id IS NOT NULL; + +-- Step 21: Create indexes on posts_new +CREATE INDEX idx_posts_new_actor ON posts_new(actor_id); +CREATE INDEX idx_posts_new_created_at ON posts_new(created_at); +CREATE INDEX idx_posts_new_indexed_at ON posts_new(indexed_at); +CREATE INDEX idx_posts_new_parent ON posts_new(parent_record_id) WHERE parent_record_id IS NOT NULL; +CREATE INDEX idx_posts_new_root ON posts_new(root_record_id) WHERE root_record_id IS NOT NULL; + +-- Step 22: Add foreign key constraints for posts +ALTER TABLE posts_new + ADD CONSTRAINT fk_posts_actor FOREIGN KEY (actor_id) REFERENCES actors(id) ON DELETE CASCADE, + ADD CONSTRAINT fk_posts_parent FOREIGN KEY (parent_record_id) REFERENCES records(id) ON DELETE SET NULL, + ADD CONSTRAINT fk_posts_root FOREIGN KEY (root_record_id) REFERENCES records(id) ON DELETE SET NULL; + +-- Step 23: Swap posts tables +ALTER TABLE posts RENAME TO posts_old; +ALTER TABLE posts_new RENAME TO posts; + +-- Step 23a: Drop old posts table immediately +DROP TABLE posts_old; + +-------------------------------------------------------------------------------- +-- FOLLOWS TABLE OPTIMIZATION +-------------------------------------------------------------------------------- + +-- Step 24: Create optimized follows table +CREATE TABLE follows_new ( + id BIGSERIAL PRIMARY KEY, + rkey TEXT NOT NULL, + actor_id INTEGER NOT NULL, + subject_actor_id INTEGER NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + UNIQUE(actor_id, rkey) +); + +-- Step 25: Transform follows data +INSERT INTO follows_new (rkey, actor_id, subject_actor_id, created_at) +SELECT + f.rkey, + a.id, + a_subject.id, + f.created_at +FROM follows f +INNER JOIN actors a ON f.did = a.did +INNER JOIN actors a_subject ON f.subject = a_subject.did +WHERE a.id IS NOT NULL AND a_subject.id IS NOT NULL; + +-- Step 26: Create indexes on follows_new +CREATE INDEX idx_follows_new_subject ON follows_new(subject_actor_id); +CREATE INDEX idx_follows_new_created_at ON follows_new(created_at); + +-- Step 27: Add foreign key constraints for follows +ALTER TABLE follows_new + ADD CONSTRAINT fk_follows_actor FOREIGN KEY (actor_id) REFERENCES actors(id) ON DELETE CASCADE, + ADD CONSTRAINT fk_follows_subject FOREIGN KEY (subject_actor_id) REFERENCES actors(id) ON DELETE CASCADE; + +-- Step 28: Swap follows tables +ALTER TABLE follows RENAME TO follows_old; +ALTER TABLE follows_new RENAME TO follows; + +-- Step 28a: Drop old follows table immediately +DROP TABLE follows_old; + +-------------------------------------------------------------------------------- +-- BLOCKS TABLE OPTIMIZATION +-------------------------------------------------------------------------------- + +-- Step 29: Create optimized blocks table +CREATE TABLE blocks_new ( + id BIGSERIAL PRIMARY KEY, + rkey TEXT NOT NULL, + actor_id INTEGER NOT NULL, + subject_actor_id INTEGER NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + UNIQUE(actor_id, rkey) +); + +-- Step 30: Transform blocks data +INSERT INTO blocks_new (rkey, actor_id, subject_actor_id, created_at) +SELECT + b.rkey, + a.id, + a_subject.id, + b.created_at +FROM blocks b +INNER JOIN actors a ON b.did = a.did +INNER JOIN actors a_subject ON b.subject = a_subject.did +WHERE a.id IS NOT NULL AND a_subject.id IS NOT NULL; + +-- Step 31: Create indexes on blocks_new +CREATE INDEX idx_blocks_new_subject ON blocks_new(subject_actor_id); +CREATE INDEX idx_blocks_new_created_at ON blocks_new(created_at); + +-- Step 32: Add foreign key constraints for blocks +ALTER TABLE blocks_new + ADD CONSTRAINT fk_blocks_actor FOREIGN KEY (actor_id) REFERENCES actors(id) ON DELETE CASCADE, + ADD CONSTRAINT fk_blocks_subject FOREIGN KEY (subject_actor_id) REFERENCES actors(id) ON DELETE CASCADE; + +-- Step 33: Swap blocks tables +ALTER TABLE blocks RENAME TO blocks_old; +ALTER TABLE blocks_new RENAME TO blocks; + +-- Step 33a: Drop old blocks table immediately +DROP TABLE blocks_old; + +-------------------------------------------------------------------------------- +-- NOTIFICATIONS TABLE OPTIMIZATION +-------------------------------------------------------------------------------- + +-- Step 34: Create optimized notifications table +CREATE TABLE notifications_new ( + id BIGSERIAL PRIMARY KEY, + record_id BIGINT NOT NULL, -- FK to records instead of URI + recipient_actor_id INTEGER NOT NULL, + author_actor_id INTEGER NOT NULL, + reason TEXT NOT NULL, + reason_subject TEXT, + cid_digest BYTEA NOT NULL, -- Strip header, 32 bytes + is_read BOOLEAN NOT NULL DEFAULT false, + indexed_at TIMESTAMP NOT NULL, + created_at TIMESTAMP NOT NULL +); + +-- Step 35: Transform notifications data +INSERT INTO notifications_new (record_id, recipient_actor_id, author_actor_id, reason, + reason_subject, cid_digest, is_read, indexed_at, created_at) +SELECT + r.id, + a_recipient.id, + a_author.id, + n.reason, + n.reason_subject, + decode(substring(n.cid from 9), 'hex'), -- Strip text CID header + n.is_read, + n.indexed_at, + n.created_at +FROM notifications n +INNER JOIN actors a_recipient ON n.recipient_did = a_recipient.did +INNER JOIN actors a_author ON n.author_did = a_author.did +-- Parse notification URI and find matching record +INNER JOIN actors actors_record ON SPLIT_PART(n.uri, '/', 3) = actors_record.did +INNER JOIN records r ON + r.actor_id = actors_record.id + AND r.collection::text = SPLIT_PART(n.uri, '/', 4) + AND r.rkey = SPLIT_PART(n.uri, '/', 5) +WHERE a_recipient.id IS NOT NULL AND a_author.id IS NOT NULL; + +-- Step 36: Create indexes on notifications_new +CREATE INDEX idx_notifications_new_recipient ON notifications_new(recipient_actor_id); +CREATE INDEX idx_notifications_new_author ON notifications_new(author_actor_id); +CREATE INDEX idx_notifications_new_record ON notifications_new(record_id); +CREATE INDEX idx_notifications_new_indexed_at ON notifications_new(indexed_at); + +-- Step 37: Add foreign key constraints for notifications +ALTER TABLE notifications_new + ADD CONSTRAINT fk_notifications_recipient FOREIGN KEY (recipient_actor_id) REFERENCES actors(id) ON DELETE CASCADE, + ADD CONSTRAINT fk_notifications_author FOREIGN KEY (author_actor_id) REFERENCES actors(id) ON DELETE CASCADE, + ADD CONSTRAINT fk_notifications_record FOREIGN KEY (record_id) REFERENCES records(id) ON DELETE CASCADE; + +-- Step 38: Swap notifications tables +ALTER TABLE notifications RENAME TO notifications_old; +ALTER TABLE notifications_new RENAME TO notifications; + +-- Step 38a: Drop old notifications table immediately +DROP TABLE notifications_old; diff --git a/parakeet-db/src/models.rs b/parakeet-db/src/models.rs index 6160783e..a99c4a72 100644 --- a/parakeet-db/src/models.rs +++ b/parakeet-db/src/models.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Queryable, Selectable, Identifiable)] #[diesel(table_name = crate::schema::actors)] -#[diesel(primary_key(did))] +#[diesel(primary_key(id))] #[diesel(check_for_backend(diesel::pg::Pg))] pub struct Actor { pub did: String, @@ -15,6 +15,7 @@ pub struct Actor { pub repo_rev: Option, pub repo_cid: Option, pub last_indexed: Option, + pub id: i32, } #[derive(Clone, Debug, Serialize, Deserialize, Queryable, Selectable, Identifiable)] @@ -114,12 +115,13 @@ pub struct FeedGen { #[derive(Clone, Debug, Serialize, Deserialize, Queryable, Selectable, Identifiable)] #[diesel(table_name = crate::schema::posts)] -#[diesel(primary_key(at_uri))] +#[diesel(primary_key(id))] #[diesel(check_for_backend(diesel::pg::Pg))] pub struct Post { + pub id: i64, pub at_uri: String, - pub cid: String, - pub did: String, + pub cid_digest: Vec, + pub actor_id: i32, pub record: serde_json::Value, pub content: String, @@ -127,10 +129,8 @@ pub struct Post { pub languages: not_null_vec::TextArray, pub tags: not_null_vec::TextArray, - pub parent_uri: Option, - pub parent_cid: Option, - pub root_uri: Option, - pub root_cid: Option, + pub parent_record_id: Option, + pub root_record_id: Option, pub embed: Option, pub embed_subtype: Option, @@ -347,12 +347,12 @@ pub struct VerificationEntry { #[diesel(check_for_backend(diesel::pg::Pg))] pub struct Notification { pub id: i64, - pub uri: String, - pub recipient_did: String, - pub author_did: String, + pub record_id: i64, + pub recipient_actor_id: i32, + pub author_actor_id: i32, pub reason: String, pub reason_subject: Option, - pub cid: String, + pub cid_digest: Vec, pub is_read: bool, pub indexed_at: NaiveDateTime, pub created_at: NaiveDateTime, @@ -442,6 +442,85 @@ pub struct AuthorFeedItem { pub sort_at: DateTime, } +// Optimized table models + +// Wrapper type for RecordType to allow String deserialization +#[derive(Debug, Clone, diesel::expression::AsExpression, diesel::deserialize::FromSqlRow)] +#[diesel(sql_type = crate::schema::sql_types::RecordType)] +pub struct RecordTypeWrapper(pub String); + +impl diesel::deserialize::FromSql for RecordTypeWrapper { + fn from_sql(bytes: diesel::pg::PgValue) -> diesel::deserialize::Result { + let s = >::from_sql(bytes)?; + Ok(RecordTypeWrapper(s)) + } +} + +#[derive(Clone, Debug, Queryable, Selectable, Identifiable)] +#[diesel(table_name = crate::schema::records)] +#[diesel(primary_key(id))] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct Record { + pub id: i64, + pub actor_id: i32, + pub collection: RecordTypeWrapper, + pub rkey: String, + pub cid: Vec, // 32-byte digest (header stripped) + pub indexed_at: NaiveDateTime, +} + +#[derive(Clone, Debug, Queryable, Selectable, Identifiable)] +#[diesel(table_name = crate::schema::likes)] +#[diesel(primary_key(id))] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct Like { + pub id: i64, + pub rkey: String, + pub actor_id: i32, + pub subject_record_id: i64, + pub created_at: DateTime, + pub indexed_at: NaiveDateTime, + pub via_record_id: Option, +} + +#[derive(Clone, Debug, Queryable, Selectable, Identifiable)] +#[diesel(table_name = crate::schema::reposts)] +#[diesel(primary_key(id))] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct Repost { + pub id: i64, + pub rkey: String, + pub actor_id: i32, + pub post_record_id: i64, + pub created_at: DateTime, + pub indexed_at: NaiveDateTime, + pub via_record_id: Option, +} + +#[derive(Clone, Debug, Queryable, Selectable, Identifiable)] +#[diesel(table_name = crate::schema::follows)] +#[diesel(primary_key(id))] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct Follow { + pub id: i64, + pub rkey: String, + pub actor_id: i32, + pub subject_actor_id: i32, + pub created_at: DateTime, +} + +#[derive(Clone, Debug, Queryable, Selectable, Identifiable)] +#[diesel(table_name = crate::schema::blocks)] +#[diesel(primary_key(id))] +#[diesel(check_for_backend(diesel::pg::Pg))] +pub struct Block { + pub id: i64, + pub rkey: String, + pub actor_id: i32, + pub subject_actor_id: i32, + pub created_at: DateTime, +} + pub use not_null_vec::TextArray; mod not_null_vec { use diesel::deserialize::FromSql; diff --git a/parakeet-db/src/schema.rs b/parakeet-db/src/schema.rs index 23d51646..40acb345 100644 --- a/parakeet-db/src/schema.rs +++ b/parakeet-db/src/schema.rs @@ -1,23 +1,17 @@ // @generated automatically by Diesel CLI. pub mod sql_types { + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "record_type"))] + pub struct RecordType; + #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "tsvector", schema = "pg_catalog"))] pub struct Tsvector; } diesel::table! { - actor_pds_mapping (did) { - did -> Text, - pds_host -> Text, - resolved_at -> Timestamp, - resolution_method -> Text, - confidence -> Text, - } -} - -diesel::table! { - actors (did) { + actors (id) { did -> Text, handle -> Nullable, status -> Text, @@ -25,6 +19,7 @@ diesel::table! { repo_rev -> Nullable, repo_cid -> Nullable, last_indexed -> Nullable, + id -> Int4, } } @@ -49,25 +44,11 @@ diesel::table! { } diesel::table! { - backfill_jobs (id) { - id -> Int4, - did -> Text, - since -> Nullable, - status -> Text, - created_at -> Timestamp, - updated_at -> Timestamp, - attempts -> Int4, - max_attempts -> Int4, - last_error -> Nullable, - retry_after -> Nullable, - } -} - -diesel::table! { - blocks (did, rkey) { + blocks (id) { + id -> Int8, rkey -> Text, - did -> Text, - subject -> Text, + actor_id -> Int4, + subject_actor_id -> Int4, created_at -> Timestamptz, } } @@ -92,14 +73,6 @@ diesel::table! { } } -diesel::table! { - cursors (cursor_key) { - cursor_key -> Text, - timestamp_us -> Int8, - updated_at -> Timestamptz, - } -} - diesel::table! { feedgens (at_uri) { at_uri -> Text, @@ -118,10 +91,11 @@ diesel::table! { } diesel::table! { - follows (did, rkey) { + follows (id) { + id -> Int8, rkey -> Text, - did -> Text, - subject -> Text, + actor_id -> Int4, + subject_actor_id -> Int4, created_at -> Timestamptz, } } @@ -168,15 +142,14 @@ diesel::table! { } diesel::table! { - likes (did, rkey) { + likes (id) { + id -> Int8, rkey -> Text, - did -> Text, - subject -> Text, - subject_cid -> Text, + actor_id -> Int4, + subject_record_id -> Int8, created_at -> Timestamptz, indexed_at -> Timestamp, - via_uri -> Nullable, - via_cid -> Nullable, + via_record_id -> Nullable, } } @@ -249,31 +222,18 @@ diesel::table! { diesel::table! { notifications (id) { id -> Int8, - uri -> Text, - recipient_did -> Text, - author_did -> Text, + record_id -> Int8, + recipient_actor_id -> Int4, + author_actor_id -> Int4, reason -> Text, reason_subject -> Nullable, - cid -> Text, + cid_digest -> Bytea, is_read -> Bool, indexed_at -> Timestamp, created_at -> Timestamp, } } -diesel::table! { - pds_hosts (host) { - host -> Text, - first_seen -> Timestamp, - last_seen -> Timestamp, - record_count -> Int8, - last_success -> Nullable, - last_failure -> Nullable, - failure_count -> Int4, - notes -> Nullable, - } -} - diesel::table! { post_embed_ext (post_uri) { post_uri -> Text, @@ -340,29 +300,24 @@ diesel::table! { } diesel::table! { - use diesel::sql_types::*; - use super::sql_types::Tsvector; - - posts (at_uri) { + posts (id) { + id -> Int8, at_uri -> Text, - cid -> Text, - did -> Text, + cid_digest -> Bytea, + actor_id -> Int4, record -> Jsonb, content -> Text, facets -> Nullable, languages -> Array>, tags -> Array>, - parent_uri -> Nullable, - parent_cid -> Nullable, - root_uri -> Nullable, - root_cid -> Nullable, + parent_record_id -> Nullable, + root_record_id -> Nullable, embed -> Nullable, embed_subtype -> Nullable, - created_at -> Timestamptz, - indexed_at -> Timestamp, mentions -> Nullable>>, violates_threadgate -> Bool, - search_vector -> Nullable, + created_at -> Timestamptz, + indexed_at -> Timestamp, } } @@ -403,24 +358,28 @@ diesel::table! { } diesel::table! { - records (at_uri) { - at_uri -> Text, + use diesel::sql_types::*; + use super::sql_types::RecordType; + + records (id) { + id -> Int8, + actor_id -> Int4, + collection -> RecordType, + rkey -> Text, cid -> Bytea, - did -> Text, indexed_at -> Timestamp, } } diesel::table! { - reposts (did, rkey) { + reposts (id) { + id -> Int8, rkey -> Text, - did -> Text, - post -> Text, - post_cid -> Text, + actor_id -> Int4, + post_record_id -> Int8, created_at -> Timestamptz, indexed_at -> Timestamp, - via_uri -> Nullable, - via_cid -> Nullable, + via_record_id -> Nullable, } } @@ -460,6 +419,14 @@ diesel::table! { } } +diesel::table! { + thread_mutes (did, thread_root) { + did -> Text, + thread_root -> Text, + created_at -> Timestamptz, + } +} + diesel::table! { threadgates (at_uri) { at_uri -> Text, @@ -474,14 +441,6 @@ diesel::table! { } } -diesel::table! { - thread_mutes (did, thread_root) { - did -> Text, - thread_root -> Text, - created_at -> Timestamptz, - } -} - diesel::table! { verification (at_uri) { at_uri -> Text, @@ -495,46 +454,25 @@ diesel::table! { } } -diesel::joinable!(actor_pds_mapping -> pds_hosts (pds_host)); -diesel::joinable!(blocks -> actors (did)); -diesel::joinable!(bookmarks -> actors (did)); -diesel::joinable!(chat_decls -> actors (did)); -diesel::joinable!(feedgens -> actors (owner)); -diesel::joinable!(follows -> actors (did)); +diesel::joinable!(blocks -> actors (actor_id)); +diesel::joinable!(follows -> actors (actor_id)); diesel::joinable!(labeler_defs -> labelers (labeler)); -diesel::joinable!(labelers -> actors (did)); -diesel::joinable!(likes -> actors (did)); -diesel::joinable!(list_blocks -> actors (did)); -diesel::joinable!(list_mutes -> actors (did)); -diesel::joinable!(lists -> actors (owner)); -diesel::joinable!(mutes -> actors (did)); -diesel::joinable!(notif_decl -> actors (did)); -diesel::joinable!(notification_seens -> actors (did)); -diesel::joinable!(post_embed_ext -> posts (post_uri)); -diesel::joinable!(post_embed_images -> posts (post_uri)); -diesel::joinable!(post_embed_record -> posts (post_uri)); -diesel::joinable!(post_embed_video -> posts (post_uri)); -diesel::joinable!(post_embed_video_captions -> posts (post_uri)); -diesel::joinable!(postgates -> posts (post_uri)); -diesel::joinable!(posts -> actors (did)); -diesel::joinable!(profiles -> actors (did)); -diesel::joinable!(reposts -> actors (did)); -diesel::joinable!(starterpacks -> actors (owner)); -diesel::joinable!(statuses -> actors (did)); -diesel::joinable!(threadgates -> posts (post_uri)); -diesel::joinable!(thread_mutes -> actors (did)); -diesel::joinable!(verification -> actors (verifier)); +diesel::joinable!(likes -> actors (actor_id)); +diesel::joinable!(likes -> records (subject_record_id)); +diesel::joinable!(notifications -> actors (recipient_actor_id)); +diesel::joinable!(notifications -> records (record_id)); +diesel::joinable!(posts -> actors (actor_id)); +diesel::joinable!(records -> actors (actor_id)); +diesel::joinable!(reposts -> actors (actor_id)); +diesel::joinable!(reposts -> records (post_record_id)); diesel::allow_tables_to_appear_in_same_query!( - actor_pds_mapping, actors, allowlist, author_feeds, - backfill_jobs, blocks, bookmarks, chat_decls, - cursors, feedgens, follows, labeler_defs, @@ -549,7 +487,6 @@ diesel::allow_tables_to_appear_in_same_query!( notif_decl, notification_seens, notifications, - pds_hosts, post_embed_ext, post_embed_images, post_embed_record, @@ -563,7 +500,7 @@ diesel::allow_tables_to_appear_in_same_query!( reposts, starterpacks, statuses, - threadgates, thread_mutes, + threadgates, verification, ); -- 2.51.2