From 7811a033d8b46259f093f1c5004754217316ecd4 Mon Sep 17 00:00:00 2001 From: Timothy Quilling Date: Thu, 27 Nov 2025 14:46:11 -0500 Subject: [PATCH] feat: consolidate actors table, wip --- .../down.sql | 57 +++ .../up.sql | 69 +++ .../down.sql | 325 +++++++++++++ .../up.sql | 432 ++++++++++++++++++ parakeet-db/src/models.rs | 10 +- parakeet-db/src/schema.rs | 136 ++---- parakeet/src/loaders/post.rs | 26 +- 7 files changed, 942 insertions(+), 113 deletions(-) create mode 100644 migrations/2025-11-27-192051_fix_posts_stats_integer_overflow/down.sql create mode 100644 migrations/2025-11-27-192051_fix_posts_stats_integer_overflow/up.sql create mode 100644 migrations/2025-11-27-193903_consolidate_actors_table/down.sql create mode 100644 migrations/2025-11-27-193903_consolidate_actors_table/up.sql diff --git a/migrations/2025-11-27-192051_fix_posts_stats_integer_overflow/down.sql b/migrations/2025-11-27-192051_fix_posts_stats_integer_overflow/down.sql new file mode 100644 index 00000000..1f204b5b --- /dev/null +++ b/migrations/2025-11-27-192051_fix_posts_stats_integer_overflow/down.sql @@ -0,0 +1,57 @@ +-- Revert posts stats columns from INTEGER back to SMALLINT +-- WARNING: This will fail if any values exceed 32,767 +-- Only safe to run if no production data has values > SMALLINT max + +-- TimescaleDB hypertable: Must disable compression before altering columns +-- First, decompress any compressed chunks (if any exist) +DO $$ +DECLARE + chunk_rec text; +BEGIN + FOR chunk_rec IN + SELECT format('%I.%I', c.chunk_schema, c.chunk_name)::regclass + FROM timescaledb_information.chunks c + WHERE c.hypertable_name = 'posts' AND c.is_compressed = true + LOOP + EXECUTE format('SELECT decompress_chunk(%L)', chunk_rec); + END LOOP; +END $$; + +-- Now disable compression +ALTER TABLE posts SET ( + timescaledb.compress = false +); + +-- Alter columns from INTEGER back to SMALLINT +ALTER TABLE posts ALTER COLUMN like_count TYPE SMALLINT; +ALTER TABLE posts ALTER COLUMN reply_count TYPE SMALLINT; +ALTER TABLE posts ALTER COLUMN repost_count TYPE SMALLINT; +ALTER TABLE posts ALTER COLUMN quote_count TYPE SMALLINT; + +-- Re-enable compression with updated column types +ALTER TABLE posts SET ( + timescaledb.compress = true, + timescaledb.compress_orderby = 'rkey DESC NULLS FIRST', + timescaledb.compress_segmentby = 'actor_id' +); + +-- Recompress chunks (async, will happen in background) +-- Only compress chunks that exist +DO $$ +DECLARE + chunk_rec text; +BEGIN + FOR chunk_rec IN + SELECT format('%I.%I', c.chunk_schema, c.chunk_name)::regclass + FROM timescaledb_information.chunks c + WHERE c.hypertable_name = 'posts' AND c.is_compressed = false + LOOP + EXECUTE format('SELECT compress_chunk(%L, if_not_compressed => true)', chunk_rec); + END LOOP; +END $$; + +-- Recreate engagement score index with SMALLINT +DROP INDEX IF EXISTS idx_posts_engagement_score; +CREATE INDEX idx_posts_engagement_score ON posts ( + (COALESCE(like_count, 0) + COALESCE(reply_count, 0) * 2 + COALESCE(repost_count, 0) * 3) DESC +); diff --git a/migrations/2025-11-27-192051_fix_posts_stats_integer_overflow/up.sql b/migrations/2025-11-27-192051_fix_posts_stats_integer_overflow/up.sql new file mode 100644 index 00000000..8fce5942 --- /dev/null +++ b/migrations/2025-11-27-192051_fix_posts_stats_integer_overflow/up.sql @@ -0,0 +1,69 @@ +-- Fix posts table integer overflow +-- Issue: SMALLINT (max 32,767) is too small for like_count, reply_count, repost_count, quote_count +-- Production data shows posts with 100k+ likes +-- Change to INTEGER to prevent overflow errors + +-- TimescaleDB hypertable: Must disable compression before altering columns +-- First, decompress any compressed chunks (if any exist) +DO $$ +DECLARE + chunk_rec text; +BEGIN + FOR chunk_rec IN + SELECT format('%I.%I', c.chunk_schema, c.chunk_name)::regclass + FROM timescaledb_information.chunks c + WHERE c.hypertable_name = 'posts' AND c.is_compressed = true + LOOP + EXECUTE format('SELECT decompress_chunk(%L)', chunk_rec); + END LOOP; +END $$; + +-- Now disable compression +ALTER TABLE posts SET ( + timescaledb.compress = false +); + +-- Alter columns from SMALLINT to INTEGER +ALTER TABLE posts ALTER COLUMN like_count TYPE INTEGER; +ALTER TABLE posts ALTER COLUMN reply_count TYPE INTEGER; +ALTER TABLE posts ALTER COLUMN repost_count TYPE INTEGER; +ALTER TABLE posts ALTER COLUMN quote_count TYPE INTEGER; + +-- Re-enable compression with updated column types +ALTER TABLE posts SET ( + timescaledb.compress = true, + timescaledb.compress_orderby = 'rkey DESC NULLS FIRST', + timescaledb.compress_segmentby = 'actor_id' +); + +-- Recompress chunks (async, will happen in background) +-- Only compress chunks that exist +DO $$ +DECLARE + chunk_rec text; +BEGIN + FOR chunk_rec IN + SELECT format('%I.%I', c.chunk_schema, c.chunk_name)::regclass + FROM timescaledb_information.chunks c + WHERE c.hypertable_name = 'posts' AND c.is_compressed = false + LOOP + EXECUTE format('SELECT compress_chunk(%L, if_not_compressed => true)', chunk_rec); + END LOOP; +END $$; + +-- Update engagement score index to use INTEGER +-- Drop and recreate the index with updated expression +DROP INDEX IF EXISTS idx_posts_engagement_score; +CREATE INDEX idx_posts_engagement_score ON posts ( + (COALESCE(like_count, 0) + COALESCE(reply_count, 0) * 2 + COALESCE(repost_count, 0) * 3) DESC +); + +-- Verification query +SELECT + 'Posts stats columns updated to INTEGER' as status, + COUNT(*) as total_posts, + MAX(like_count) as max_likes, + MAX(reply_count) as max_replies, + MAX(repost_count) as max_reposts, + MAX(quote_count) as max_quotes +FROM posts; diff --git a/migrations/2025-11-27-193903_consolidate_actors_table/down.sql b/migrations/2025-11-27-193903_consolidate_actors_table/down.sql new file mode 100644 index 00000000..2976638b --- /dev/null +++ b/migrations/2025-11-27-193903_consolidate_actors_table/down.sql @@ -0,0 +1,325 @@ +-- ============================================================================ +-- REVERSE ACTOR TABLE CONSOLIDATION +-- ============================================================================ +-- This migration reverses the consolidation by: +-- 1. Recreating the 6 original tables +-- 2. Backfilling data from actors table columns +-- 3. Dropping the consolidated columns from actors table +-- 4. Restoring indexes and constraints +-- ============================================================================ + +-- ============================================================================ +-- SECTION 1: RECREATE PROFILES TABLE +-- ============================================================================ + +CREATE TABLE profiles ( + actor_id INTEGER PRIMARY KEY, + cid BYTEA NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + avatar_cid BYTEA, + banner_cid BYTEA, + display_name TEXT, + description TEXT, + pinned_post_rkey BIGINT, + joined_sp_id BIGINT, + pronouns TEXT, + website TEXT, + search_vector TSVECTOR GENERATED ALWAYS AS ( + setweight(to_tsvector('english'::regconfig, COALESCE(display_name, ''::text)), 'A'::"char") || + setweight(to_tsvector('english'::regconfig, COALESCE(description, ''::text)), 'B'::"char") + ) STORED +); + +COMMENT ON TABLE profiles IS 'Actor profile records from app.bsky.actor.profile'; +COMMENT ON COLUMN profiles.actor_id IS 'Reference to actors table (also primary key)'; +COMMENT ON COLUMN profiles.cid IS 'Content identifier (CID) of profile record'; +COMMENT ON COLUMN profiles.created_at IS 'Profile record creation timestamp'; +COMMENT ON COLUMN profiles.avatar_cid IS 'Avatar image CID (blob reference)'; +COMMENT ON COLUMN profiles.banner_cid IS 'Banner image CID (blob reference)'; +COMMENT ON COLUMN profiles.display_name IS 'User-facing display name (e.g., "Alice Johnson")'; +COMMENT ON COLUMN profiles.description IS 'Profile bio/description text'; +COMMENT ON COLUMN profiles.pinned_post_rkey IS 'Pinned post rkey (natural key). No FK constraint due to hypertable limitation.'; +COMMENT ON COLUMN profiles.joined_sp_id IS 'Joined via starter pack ID (FK added in feeds migration)'; +COMMENT ON COLUMN profiles.pronouns IS 'User pronouns (e.g., "she/her", "they/them")'; +COMMENT ON COLUMN profiles.website IS 'Personal website URL'; +COMMENT ON COLUMN profiles.search_vector IS 'Full-text search vector: display_name (weight A) + description (weight B)'; + +CREATE INDEX idx_profiles_search ON profiles USING GIN (search_vector); + +ALTER TABLE profiles + ADD CONSTRAINT fk_profiles_joined_sp + FOREIGN KEY (joined_sp_id) + REFERENCES starterpacks(id) + ON DELETE SET NULL; + +-- Backfill from actors table +INSERT INTO profiles ( + actor_id, cid, created_at, avatar_cid, banner_cid, display_name, + description, pinned_post_rkey, joined_sp_id, pronouns, website +) +SELECT + id, profile_cid, profile_created_at, profile_avatar_cid, profile_banner_cid, + profile_display_name, profile_description, profile_pinned_post_rkey, + profile_joined_sp_id, profile_pronouns, profile_website +FROM actors +WHERE profile_cid IS NOT NULL; + +-- ============================================================================ +-- SECTION 2: RECREATE STATUSES TABLE +-- ============================================================================ + +CREATE TABLE statuses ( + actor_id INTEGER PRIMARY KEY, + cid BYTEA NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + status status_type NOT NULL, + duration INTEGER, + embed_post_actor_id INTEGER, + embed_post_rkey BIGINT, + thumb_mime_type image_mime_type, + thumb_cid BYTEA +); + +COMMENT ON TABLE statuses IS 'Actor status records (e.g., live streaming status)'; +COMMENT ON COLUMN statuses.actor_id IS 'Reference to actors table (also primary key)'; +COMMENT ON COLUMN statuses.cid IS 'Content identifier (CID) of status record'; +COMMENT ON COLUMN statuses.created_at IS 'Status record creation timestamp'; +COMMENT ON COLUMN statuses.status IS 'Status type (currently only app.bsky.actor.status#live)'; +COMMENT ON COLUMN statuses.duration IS 'Stream duration in seconds (nullable for ongoing streams)'; +COMMENT ON COLUMN statuses.embed_post_actor_id IS 'Referenced post actor_id (no FK due to hypertable limitation)'; +COMMENT ON COLUMN statuses.embed_post_rkey IS 'Referenced post rkey (natural key part 2)'; +COMMENT ON COLUMN statuses.thumb_mime_type IS 'Thumbnail image MIME type'; +COMMENT ON COLUMN statuses.thumb_cid IS 'Thumbnail image CID (blob reference)'; + +-- Backfill from actors table +INSERT INTO statuses ( + actor_id, cid, created_at, status, duration, embed_post_actor_id, + embed_post_rkey, thumb_mime_type, thumb_cid +) +SELECT + id, status_cid, status_created_at, status_type, status_duration, + status_embed_post_actor_id, status_embed_post_rkey, status_thumb_mime_type, + status_thumb_cid +FROM actors +WHERE status_cid IS NOT NULL; + +-- ============================================================================ +-- SECTION 3: RECREATE CHAT_DECLS TABLE +-- ============================================================================ + +CREATE TABLE chat_decls ( + actor_id INTEGER PRIMARY KEY, + allow_incoming chat_allow_incoming NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE chat_decls IS 'Chat preference declarations for actors'; +COMMENT ON COLUMN chat_decls.actor_id IS 'Reference to actors table (also primary key)'; +COMMENT ON COLUMN chat_decls.allow_incoming IS 'Who can message this actor: all, following, or none'; +COMMENT ON COLUMN chat_decls.created_at IS 'Timestamp when chat preference was set'; + +-- Backfill from actors table +INSERT INTO chat_decls (actor_id, allow_incoming, created_at) +SELECT id, chat_allow_incoming, chat_created_at +FROM actors +WHERE chat_allow_incoming IS NOT NULL; + +-- ============================================================================ +-- SECTION 4: RECREATE NOTIF_DECL TABLE +-- ============================================================================ + +CREATE TABLE notif_decl ( + actor_id INTEGER PRIMARY KEY, + allow_subscriptions notif_allow_subscriptions, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE notif_decl IS 'Notification preference declarations for actors'; +COMMENT ON COLUMN notif_decl.actor_id IS 'Reference to actors table'; +COMMENT ON COLUMN notif_decl.allow_subscriptions IS 'Who can subscribe to notifications'; +COMMENT ON COLUMN notif_decl.created_at IS 'Timestamp when notification preference was set'; + +-- Backfill from actors table +INSERT INTO notif_decl (actor_id, allow_subscriptions, created_at) +SELECT id, notif_decl_allow_subscriptions, notif_decl_created_at +FROM actors +WHERE notif_decl_allow_subscriptions IS NOT NULL; + +-- ============================================================================ +-- SECTION 5: RECREATE NOTIFICATION_STATE TABLE +-- ============================================================================ + +CREATE TABLE notification_state ( + actor_id INTEGER PRIMARY KEY, + seen_at TIMESTAMP WITH TIME ZONE, + unread_count INTEGER DEFAULT 0 +); + +COMMENT ON TABLE notification_state IS 'Notification read/unread state for actors'; +COMMENT ON COLUMN notification_state.actor_id IS 'Reference to actors table'; +COMMENT ON COLUMN notification_state.seen_at IS 'Timestamp when actor last saw notifications'; +COMMENT ON COLUMN notification_state.unread_count IS 'Count of unread notifications'; + +-- Backfill from actors table +INSERT INTO notification_state (actor_id, seen_at, unread_count) +SELECT id, notif_seen_at, COALESCE(notif_unread_count, 0) +FROM actors +WHERE notif_seen_at IS NOT NULL OR notif_unread_count IS NOT NULL; + +-- ============================================================================ +-- SECTION 6: RECREATE ACTOR_AGGREGATE_STATS TABLE +-- ============================================================================ + +CREATE TABLE actor_aggregate_stats ( + actor_id INTEGER NOT NULL, + stat_type actor_stat_type NOT NULL, + value INTEGER NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + PRIMARY KEY (actor_id, stat_type, updated_at) +); + +COMMENT ON TABLE actor_aggregate_stats IS 'Denormalized aggregate statistics for actors'; +COMMENT ON COLUMN actor_aggregate_stats.actor_id IS 'Reference to actors table'; +COMMENT ON COLUMN actor_aggregate_stats.stat_type IS 'Type of statistic (follow, follower, profile_post, etc.)'; +COMMENT ON COLUMN actor_aggregate_stats.value IS 'Count value for this stat type'; +COMMENT ON COLUMN actor_aggregate_stats.updated_at IS 'Timestamp when this stat was last updated'; + +CREATE INDEX actor_aggregate_stats_actor_id_idx ON actor_aggregate_stats(actor_id DESC); +CREATE INDEX idx_actor_aggregate_stats_type_value ON actor_aggregate_stats(stat_type, value DESC); +CREATE INDEX idx_actor_aggregate_stats_updated_at ON actor_aggregate_stats(updated_at DESC); + +-- Backfill from actors table (follower stats) +INSERT INTO actor_aggregate_stats (actor_id, stat_type, value, updated_at) +SELECT id, 'follower', followers_count, NOW() +FROM actors +WHERE followers_count IS NOT NULL AND followers_count > 0; + +-- Backfill from actors table (follow stats) +INSERT INTO actor_aggregate_stats (actor_id, stat_type, value, updated_at) +SELECT id, 'follow', following_count, NOW() +FROM actors +WHERE following_count IS NOT NULL AND following_count > 0; + +-- Backfill from actors table (post stats) +INSERT INTO actor_aggregate_stats (actor_id, stat_type, value, updated_at) +SELECT id, 'profile_post', posts_count, NOW() +FROM actors +WHERE posts_count IS NOT NULL AND posts_count > 0; + +-- Backfill from actors table (list stats) +INSERT INTO actor_aggregate_stats (actor_id, stat_type, value, updated_at) +SELECT id, 'profile_list', lists_count, NOW() +FROM actors +WHERE lists_count IS NOT NULL AND lists_count > 0; + +-- Backfill from actors table (feed stats) +INSERT INTO actor_aggregate_stats (actor_id, stat_type, value, updated_at) +SELECT id, 'profile_feed', feeds_count, NOW() +FROM actors +WHERE feeds_count IS NOT NULL AND feeds_count > 0; + +-- Backfill from actors table (starterpack stats) +INSERT INTO actor_aggregate_stats (actor_id, stat_type, value, updated_at) +SELECT id, 'profile_starterpack', starterpacks_count, NOW() +FROM actors +WHERE starterpacks_count IS NOT NULL AND starterpacks_count > 0; + +-- Convert to hypertable (matches original schema) +SELECT create_hypertable('actor_aggregate_stats', 'updated_at', + chunk_time_interval => INTERVAL '7 days', + if_not_exists => TRUE +); + +-- ============================================================================ +-- SECTION 7: DROP CONSOLIDATED COLUMNS FROM ACTORS TABLE +-- ============================================================================ + +-- Drop FK constraint first +ALTER TABLE actors DROP CONSTRAINT IF EXISTS fk_actors_profile_joined_sp; + +-- Drop indexes +DROP INDEX IF EXISTS idx_actors_profile_search; +DROP INDEX IF EXISTS idx_actors_profile_display_name; +DROP INDEX IF EXISTS idx_actors_profile_joined_sp; +DROP INDEX IF EXISTS idx_actors_status_type; +DROP INDEX IF EXISTS idx_actors_chat_allow_incoming; +DROP INDEX IF EXISTS idx_actors_notif_unread_count; +DROP INDEX IF EXISTS idx_actors_followers_count_desc; +DROP INDEX IF EXISTS idx_actors_posts_count_desc; +DROP INDEX IF EXISTS idx_actors_engagement_score; + +-- Drop profile columns +ALTER TABLE actors DROP COLUMN IF EXISTS profile_cid; +ALTER TABLE actors DROP COLUMN IF EXISTS profile_created_at; +ALTER TABLE actors DROP COLUMN IF EXISTS profile_avatar_cid; +ALTER TABLE actors DROP COLUMN IF EXISTS profile_banner_cid; +ALTER TABLE actors DROP COLUMN IF EXISTS profile_display_name; +ALTER TABLE actors DROP COLUMN IF EXISTS profile_description; +ALTER TABLE actors DROP COLUMN IF EXISTS profile_pinned_post_rkey; +ALTER TABLE actors DROP COLUMN IF EXISTS profile_joined_sp_id; +ALTER TABLE actors DROP COLUMN IF EXISTS profile_pronouns; +ALTER TABLE actors DROP COLUMN IF EXISTS profile_website; +ALTER TABLE actors DROP COLUMN IF EXISTS profile_search_vector; + +-- Drop status columns +ALTER TABLE actors DROP COLUMN IF EXISTS status_cid; +ALTER TABLE actors DROP COLUMN IF EXISTS status_created_at; +ALTER TABLE actors DROP COLUMN IF EXISTS status_type; +ALTER TABLE actors DROP COLUMN IF EXISTS status_duration; +ALTER TABLE actors DROP COLUMN IF EXISTS status_embed_post_actor_id; +ALTER TABLE actors DROP COLUMN IF EXISTS status_embed_post_rkey; +ALTER TABLE actors DROP COLUMN IF EXISTS status_thumb_mime_type; +ALTER TABLE actors DROP COLUMN IF EXISTS status_thumb_cid; + +-- Drop chat columns +ALTER TABLE actors DROP COLUMN IF EXISTS chat_allow_incoming; +ALTER TABLE actors DROP COLUMN IF EXISTS chat_created_at; + +-- Drop notif_decl columns +ALTER TABLE actors DROP COLUMN IF EXISTS notif_decl_allow_subscriptions; +ALTER TABLE actors DROP COLUMN IF EXISTS notif_decl_created_at; + +-- Drop notification_state columns +ALTER TABLE actors DROP COLUMN IF EXISTS notif_seen_at; +ALTER TABLE actors DROP COLUMN IF EXISTS notif_unread_count; + +-- Drop stats columns +ALTER TABLE actors DROP COLUMN IF EXISTS followers_count; +ALTER TABLE actors DROP COLUMN IF EXISTS following_count; +ALTER TABLE actors DROP COLUMN IF EXISTS posts_count; +ALTER TABLE actors DROP COLUMN IF EXISTS lists_count; +ALTER TABLE actors DROP COLUMN IF EXISTS feeds_count; +ALTER TABLE actors DROP COLUMN IF EXISTS starterpacks_count; + +-- ============================================================================ +-- SECTION 8: VERIFICATION +-- ============================================================================ + +DO $$ +DECLARE + profiles_count BIGINT; + statuses_count BIGINT; + chat_decls_count BIGINT; + notif_decl_count BIGINT; + notification_state_count BIGINT; + actor_aggregate_stats_count BIGINT; +BEGIN + SELECT COUNT(*) INTO profiles_count FROM profiles; + SELECT COUNT(*) INTO statuses_count FROM statuses; + SELECT COUNT(*) INTO chat_decls_count FROM chat_decls; + SELECT COUNT(*) INTO notif_decl_count FROM notif_decl; + SELECT COUNT(*) INTO notification_state_count FROM notification_state; + SELECT COUNT(*) INTO actor_aggregate_stats_count FROM actor_aggregate_stats; + + RAISE NOTICE '============================================================================'; + RAISE NOTICE 'ACTOR TABLE CONSOLIDATION ROLLBACK COMPLETE'; + RAISE NOTICE '============================================================================'; + RAISE NOTICE 'Restored tables:'; + RAISE NOTICE ' profiles: % rows', profiles_count; + RAISE NOTICE ' statuses: % rows', statuses_count; + RAISE NOTICE ' chat_decls: % rows', chat_decls_count; + RAISE NOTICE ' notif_decl: % rows', notif_decl_count; + RAISE NOTICE ' notification_state: % rows', notification_state_count; + RAISE NOTICE ' actor_aggregate_stats: % rows', actor_aggregate_stats_count; + RAISE NOTICE '============================================================================'; +END $$; diff --git a/migrations/2025-11-27-193903_consolidate_actors_table/up.sql b/migrations/2025-11-27-193903_consolidate_actors_table/up.sql new file mode 100644 index 00000000..f8d869ea --- /dev/null +++ b/migrations/2025-11-27-193903_consolidate_actors_table/up.sql @@ -0,0 +1,432 @@ +-- ============================================================================ +-- CONSOLIDATE ACTOR-RELATED TABLES INTO MAIN ACTORS TABLE +-- ============================================================================ +-- This migration consolidates 6 actor-related tables into the main actors table: +-- 1. profiles -> profile_* columns +-- 2. statuses -> status_* columns (prefix to avoid conflict with actors.status) +-- 3. chat_decls -> chat_* columns +-- 4. notif_decl -> notif_decl_* columns +-- 5. notification_state -> notif_* columns +-- 6. actor_aggregate_stats -> denormalized count columns +-- +-- Benefits: +-- - Single table lookup for complete actor data (eliminates JOINs) +-- - Better cache locality (all actor data together) +-- - Simpler queries for actor hydration +-- - TimescaleDB compression-friendly (NULL values compress to 1 bit) +-- +-- Pattern follows: migrations/2025-11-27-104604_add_post_aggregate_columns/up.sql +-- ============================================================================ + +-- ============================================================================ +-- SECTION 0: DECOMPRESS ACTORS HYPERTABLE (if compressed) +-- ============================================================================ +-- TimescaleDB hypertables must be decompressed before ALTER TABLE ADD COLUMN + +-- First, decompress any compressed chunks +DO $$ +DECLARE + chunk_rec text; +BEGIN + FOR chunk_rec IN + SELECT format('%I.%I', c.chunk_schema, c.chunk_name)::regclass + FROM timescaledb_information.chunks c + WHERE c.hypertable_name = 'actors' AND c.is_compressed = true + LOOP + EXECUTE format('SELECT decompress_chunk(%L)', chunk_rec); + END LOOP; +END $$; + +-- Disable compression temporarily +ALTER TABLE actors SET ( + timescaledb.compress = false +); + +-- ============================================================================ +-- SECTION 1: ADD COLUMNS FROM PROFILES TABLE (prefix: profile_) +-- ============================================================================ + +-- Core profile metadata +ALTER TABLE actors ADD COLUMN profile_cid BYTEA DEFAULT NULL; +ALTER TABLE actors ADD COLUMN profile_created_at TIMESTAMP WITH TIME ZONE DEFAULT NULL; +ALTER TABLE actors ADD COLUMN profile_avatar_cid BYTEA DEFAULT NULL; +ALTER TABLE actors ADD COLUMN profile_banner_cid BYTEA DEFAULT NULL; +ALTER TABLE actors ADD COLUMN profile_display_name TEXT DEFAULT NULL; +ALTER TABLE actors ADD COLUMN profile_description TEXT DEFAULT NULL; +ALTER TABLE actors ADD COLUMN profile_pinned_post_rkey BIGINT DEFAULT NULL; +ALTER TABLE actors ADD COLUMN profile_joined_sp_id BIGINT DEFAULT NULL; +ALTER TABLE actors ADD COLUMN profile_pronouns TEXT DEFAULT NULL; +ALTER TABLE actors ADD COLUMN profile_website TEXT DEFAULT NULL; +ALTER TABLE actors ADD COLUMN profile_search_vector TSVECTOR DEFAULT NULL; + +COMMENT ON COLUMN actors.profile_cid IS 'Content identifier (CID) of profile record'; +COMMENT ON COLUMN actors.profile_created_at IS 'Profile record creation timestamp'; +COMMENT ON COLUMN actors.profile_avatar_cid IS 'Avatar image CID (blob reference)'; +COMMENT ON COLUMN actors.profile_banner_cid IS 'Banner image CID (blob reference)'; +COMMENT ON COLUMN actors.profile_display_name IS 'User-facing display name (e.g., "Alice Johnson")'; +COMMENT ON COLUMN actors.profile_description IS 'Profile bio/description text'; +COMMENT ON COLUMN actors.profile_pinned_post_rkey IS 'Pinned post rkey (natural key). No FK constraint due to hypertable limitation.'; +COMMENT ON COLUMN actors.profile_joined_sp_id IS 'Joined via starter pack ID (FK to starterpacks)'; +COMMENT ON COLUMN actors.profile_pronouns IS 'User pronouns (e.g., "she/her", "they/them")'; +COMMENT ON COLUMN actors.profile_website IS 'Personal website URL'; +COMMENT ON COLUMN actors.profile_search_vector IS 'Full-text search vector: display_name (weight A) + description (weight B)'; + +-- ============================================================================ +-- SECTION 2: ADD COLUMNS FROM STATUSES TABLE (prefix: status_) +-- ============================================================================ +-- Note: "status" conflicts with actors.status (actor_status enum), so we use status_type + +ALTER TABLE actors ADD COLUMN status_cid BYTEA DEFAULT NULL; +ALTER TABLE actors ADD COLUMN status_created_at TIMESTAMP WITH TIME ZONE DEFAULT NULL; +ALTER TABLE actors ADD COLUMN status_type status_type DEFAULT NULL; +ALTER TABLE actors ADD COLUMN status_duration INTEGER DEFAULT NULL; +ALTER TABLE actors ADD COLUMN status_embed_post_actor_id INTEGER DEFAULT NULL; +ALTER TABLE actors ADD COLUMN status_embed_post_rkey BIGINT DEFAULT NULL; +ALTER TABLE actors ADD COLUMN status_thumb_mime_type image_mime_type DEFAULT NULL; +ALTER TABLE actors ADD COLUMN status_thumb_cid BYTEA DEFAULT NULL; + +COMMENT ON COLUMN actors.status_cid IS 'Content identifier (CID) of status record'; +COMMENT ON COLUMN actors.status_created_at IS 'Status record creation timestamp'; +COMMENT ON COLUMN actors.status_type IS 'Status type (currently only app.bsky.actor.status#live)'; +COMMENT ON COLUMN actors.status_duration IS 'Stream duration in seconds (nullable for ongoing streams)'; +COMMENT ON COLUMN actors.status_embed_post_actor_id IS 'Referenced post actor_id (no FK due to hypertable limitation)'; +COMMENT ON COLUMN actors.status_embed_post_rkey IS 'Referenced post rkey (natural key part 2)'; +COMMENT ON COLUMN actors.status_thumb_mime_type IS 'Thumbnail image MIME type'; +COMMENT ON COLUMN actors.status_thumb_cid IS 'Thumbnail image CID (blob reference)'; + +-- ============================================================================ +-- SECTION 3: ADD COLUMNS FROM CHAT_DECLS TABLE (prefix: chat_) +-- ============================================================================ + +ALTER TABLE actors ADD COLUMN chat_allow_incoming chat_allow_incoming DEFAULT NULL; +ALTER TABLE actors ADD COLUMN chat_created_at TIMESTAMP WITH TIME ZONE DEFAULT NULL; + +COMMENT ON COLUMN actors.chat_allow_incoming IS 'Who can message this actor: all, following, or none'; +COMMENT ON COLUMN actors.chat_created_at IS 'Timestamp when chat preference was set'; + +-- ============================================================================ +-- SECTION 4: ADD COLUMNS FROM NOTIF_DECL TABLE (prefix: notif_decl_) +-- ============================================================================ + +ALTER TABLE actors ADD COLUMN notif_decl_allow_subscriptions notif_allow_subscriptions DEFAULT NULL; +ALTER TABLE actors ADD COLUMN notif_decl_created_at TIMESTAMP WITH TIME ZONE DEFAULT NULL; + +COMMENT ON COLUMN actors.notif_decl_allow_subscriptions IS 'Who can subscribe to notifications: none, mutuals, or followers'; +COMMENT ON COLUMN actors.notif_decl_created_at IS 'Timestamp when notification preference was set'; + +-- ============================================================================ +-- SECTION 5: ADD COLUMNS FROM NOTIFICATION_STATE TABLE (prefix: notif_) +-- ============================================================================ + +ALTER TABLE actors ADD COLUMN notif_seen_at TIMESTAMP WITH TIME ZONE DEFAULT NULL; +ALTER TABLE actors ADD COLUMN notif_unread_count INTEGER DEFAULT NULL; + +COMMENT ON COLUMN actors.notif_seen_at IS 'Timestamp when actor last saw their notifications'; +COMMENT ON COLUMN actors.notif_unread_count IS 'Count of unread notifications (NULL = 0, optimized for compression)'; + +-- ============================================================================ +-- SECTION 6: ADD DENORMALIZED STATS COLUMNS FROM ACTOR_AGGREGATE_STATS +-- ============================================================================ +-- Rationale: +-- - INTEGER for high-volume counts (followers, following, posts): can exceed 32k +-- - SMALLINT for low-volume counts (lists, feeds, starterpacks): rarely exceed 100 +-- - NULL instead of 0: TimescaleDB Gorilla compression stores NULLs as 1 bit +-- - Denormalized for fast queries (no JOIN needed for counts) +-- +-- actor_stat_type enum values: +-- follow, follower, profile_post, profile_list, profile_feed, profile_starterpack + +ALTER TABLE actors ADD COLUMN followers_count INTEGER DEFAULT NULL; +ALTER TABLE actors ADD COLUMN following_count INTEGER DEFAULT NULL; +ALTER TABLE actors ADD COLUMN posts_count INTEGER DEFAULT NULL; +ALTER TABLE actors ADD COLUMN lists_count SMALLINT DEFAULT NULL; +ALTER TABLE actors ADD COLUMN feeds_count SMALLINT DEFAULT NULL; +ALTER TABLE actors ADD COLUMN starterpacks_count SMALLINT DEFAULT NULL; + +COMMENT ON COLUMN actors.followers_count IS 'Count of followers (NULL = 0, optimized for compression)'; +COMMENT ON COLUMN actors.following_count IS 'Count of following (NULL = 0, optimized for compression)'; +COMMENT ON COLUMN actors.posts_count IS 'Count of posts (NULL = 0, optimized for compression)'; +COMMENT ON COLUMN actors.lists_count IS 'Count of lists (NULL = 0, optimized for compression)'; +COMMENT ON COLUMN actors.feeds_count IS 'Count of feeds (NULL = 0, optimized for compression)'; +COMMENT ON COLUMN actors.starterpacks_count IS 'Count of starter packs (NULL = 0, optimized for compression)'; + +-- ============================================================================ +-- SECTION 7: BACKFILL DATA FROM EXISTING TABLES +-- ============================================================================ +-- Strategy: Only set non-NULL values (skip NULL/zero values for compression) + +-- Backfill from profiles table +UPDATE actors a +SET profile_cid = p.cid, + profile_created_at = p.created_at, + profile_avatar_cid = p.avatar_cid, + profile_banner_cid = p.banner_cid, + profile_display_name = p.display_name, + profile_description = p.description, + profile_pinned_post_rkey = p.pinned_post_rkey, + profile_joined_sp_id = p.joined_sp_id, + profile_pronouns = p.pronouns, + profile_website = p.website, + profile_search_vector = p.search_vector +FROM profiles p +WHERE p.actor_id = a.id; + +-- Backfill from statuses table +UPDATE actors a +SET status_cid = s.cid, + status_created_at = s.created_at, + status_type = s.status, + status_duration = s.duration, + status_embed_post_actor_id = s.embed_post_actor_id, + status_embed_post_rkey = s.embed_post_rkey, + status_thumb_mime_type = s.thumb_mime_type, + status_thumb_cid = s.thumb_cid +FROM statuses s +WHERE s.actor_id = a.id; + +-- Backfill from chat_decls table +UPDATE actors a +SET chat_allow_incoming = c.allow_incoming, + chat_created_at = c.created_at +FROM chat_decls c +WHERE c.actor_id = a.id; + +-- Backfill from notif_decl table +UPDATE actors a +SET notif_decl_allow_subscriptions = n.allow_subscriptions, + notif_decl_created_at = n.created_at +FROM notif_decl n +WHERE n.actor_id = a.id; + +-- Backfill from notification_state table (only set non-zero values) +UPDATE actors a +SET notif_seen_at = ns.seen_at, + notif_unread_count = ns.unread_count +FROM notification_state ns +WHERE ns.actor_id = a.id + AND (ns.seen_at IS NOT NULL OR ns.unread_count > 0); + +-- Backfill from actor_aggregate_stats table (only set non-zero values) +UPDATE actors a +SET followers_count = aas.value +FROM actor_aggregate_stats aas +WHERE aas.actor_id = a.id + AND aas.stat_type = 'follower' + AND aas.value > 0; + +UPDATE actors a +SET following_count = aas.value +FROM actor_aggregate_stats aas +WHERE aas.actor_id = a.id + AND aas.stat_type = 'follow' + AND aas.value > 0; + +UPDATE actors a +SET posts_count = aas.value +FROM actor_aggregate_stats aas +WHERE aas.actor_id = a.id + AND aas.stat_type = 'profile_post' + AND aas.value > 0; + +UPDATE actors a +SET lists_count = aas.value::smallint +FROM actor_aggregate_stats aas +WHERE aas.actor_id = a.id + AND aas.stat_type = 'profile_list' + AND aas.value > 0; + +UPDATE actors a +SET feeds_count = aas.value::smallint +FROM actor_aggregate_stats aas +WHERE aas.actor_id = a.id + AND aas.stat_type = 'profile_feed' + AND aas.value > 0; + +UPDATE actors a +SET starterpacks_count = aas.value::smallint +FROM actor_aggregate_stats aas +WHERE aas.actor_id = a.id + AND aas.stat_type = 'profile_starterpack' + AND aas.value > 0; + +-- ============================================================================ +-- SECTION 8: CREATE PARTIAL INDEXES ON NON-NULL VALUES +-- ============================================================================ +-- Partial indexes are smaller and more efficient (only index non-NULL values) + +-- Profile search index (full-text search) +CREATE INDEX idx_actors_profile_search + ON actors USING GIN (profile_search_vector) + WHERE profile_search_vector IS NOT NULL; + +-- Profile metadata indexes +CREATE INDEX idx_actors_profile_display_name + ON actors(profile_display_name) + WHERE profile_display_name IS NOT NULL; + +CREATE INDEX idx_actors_profile_joined_sp + ON actors(profile_joined_sp_id) + WHERE profile_joined_sp_id IS NOT NULL; + +-- Status indexes +CREATE INDEX idx_actors_status_type + ON actors(status_type) + WHERE status_type IS NOT NULL; + +-- Chat indexes +CREATE INDEX idx_actors_chat_allow_incoming + ON actors(chat_allow_incoming) + WHERE chat_allow_incoming IS NOT NULL; + +-- Notification indexes +CREATE INDEX idx_actors_notif_unread_count + ON actors(notif_unread_count DESC NULLS LAST) + WHERE notif_unread_count IS NOT NULL AND notif_unread_count > 0; + +-- Stats indexes (for leaderboards and popular actors) +CREATE INDEX idx_actors_followers_count_desc + ON actors(followers_count DESC NULLS LAST) + WHERE followers_count IS NOT NULL; + +CREATE INDEX idx_actors_posts_count_desc + ON actors(posts_count DESC NULLS LAST) + WHERE posts_count IS NOT NULL; + +-- Composite engagement index (total activity score) +CREATE INDEX idx_actors_engagement_score + ON actors((COALESCE(followers_count,0) + COALESCE(posts_count,0) + COALESCE(lists_count,0) + COALESCE(feeds_count,0) + COALESCE(starterpacks_count,0)) DESC) + WHERE (followers_count IS NOT NULL OR posts_count IS NOT NULL OR lists_count IS NOT NULL OR feeds_count IS NOT NULL OR starterpacks_count IS NOT NULL); + +-- ============================================================================ +-- SECTION 9: ADD FOREIGN KEY CONSTRAINT +-- ============================================================================ + +-- Add FK constraint for profile_joined_sp_id (matches original profiles table) +ALTER TABLE actors + ADD CONSTRAINT fk_actors_profile_joined_sp + FOREIGN KEY (profile_joined_sp_id) + REFERENCES starterpacks(id) + ON DELETE SET NULL; + +-- ============================================================================ +-- SECTION 10: DROP OLD TABLES +-- ============================================================================ + +DROP TABLE IF EXISTS profiles CASCADE; +DROP TABLE IF EXISTS statuses CASCADE; +DROP TABLE IF EXISTS chat_decls CASCADE; +DROP TABLE IF EXISTS notif_decl CASCADE; +DROP TABLE IF EXISTS notification_state CASCADE; +DROP TABLE IF EXISTS actor_aggregate_stats CASCADE; + +-- ============================================================================ +-- SECTION 10.5: RE-ENABLE COMPRESSION ON ACTORS HYPERTABLE +-- ============================================================================ + +-- Re-enable compression with original settings: segmentby=status, orderby=id DESC NULLS FIRST +ALTER TABLE actors SET ( + timescaledb.compress = true, + timescaledb.compress_segmentby = 'status', + timescaledb.compress_orderby = 'id DESC NULLS FIRST' +); + +-- Recompress chunks (async, will happen in background) +DO $$ +DECLARE + chunk_rec text; +BEGIN + FOR chunk_rec IN + SELECT format('%I.%I', c.chunk_schema, c.chunk_name)::regclass + FROM timescaledb_information.chunks c + WHERE c.hypertable_name = 'actors' AND c.is_compressed = false + LOOP + EXECUTE format('SELECT compress_chunk(%L, if_not_compressed => true)', chunk_rec); + END LOOP; +END $$; + +-- ============================================================================ +-- SECTION 11: VERIFICATION +-- ============================================================================ + +DO $$ +DECLARE + total_actors BIGINT; + actors_with_profiles BIGINT; + actors_with_statuses BIGINT; + actors_with_chat BIGINT; + actors_with_notif_decl BIGINT; + actors_with_notif_state BIGINT; + actors_with_followers BIGINT; + actors_with_following BIGINT; + actors_with_posts BIGINT; + actors_with_lists BIGINT; + actors_with_feeds BIGINT; + actors_with_starterpacks BIGINT; + actor_record RECORD; +BEGIN + -- Count total actors + SELECT COUNT(*) INTO total_actors FROM actors; + + -- Count actors with data from each consolidated table + SELECT COUNT(*) INTO actors_with_profiles FROM actors WHERE profile_cid IS NOT NULL; + SELECT COUNT(*) INTO actors_with_statuses FROM actors WHERE status_cid IS NOT NULL; + SELECT COUNT(*) INTO actors_with_chat FROM actors WHERE chat_allow_incoming IS NOT NULL; + SELECT COUNT(*) INTO actors_with_notif_decl FROM actors WHERE notif_decl_allow_subscriptions IS NOT NULL; + SELECT COUNT(*) INTO actors_with_notif_state FROM actors WHERE notif_seen_at IS NOT NULL OR notif_unread_count IS NOT NULL; + SELECT COUNT(*) INTO actors_with_followers FROM actors WHERE followers_count IS NOT NULL; + SELECT COUNT(*) INTO actors_with_following FROM actors WHERE following_count IS NOT NULL; + SELECT COUNT(*) INTO actors_with_posts FROM actors WHERE posts_count IS NOT NULL; + SELECT COUNT(*) INTO actors_with_lists FROM actors WHERE lists_count IS NOT NULL; + SELECT COUNT(*) INTO actors_with_feeds FROM actors WHERE feeds_count IS NOT NULL; + SELECT COUNT(*) INTO actors_with_starterpacks FROM actors WHERE starterpacks_count IS NOT NULL; + + RAISE NOTICE '============================================================================'; + RAISE NOTICE 'ACTOR TABLE CONSOLIDATION COMPLETE'; + RAISE NOTICE '============================================================================'; + RAISE NOTICE 'Total actors: %', total_actors; + RAISE NOTICE ''; + RAISE NOTICE 'Consolidated data from tables:'; + RAISE NOTICE ' Profiles: % actors (%.1f%%)', actors_with_profiles, (actors_with_profiles::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE ' Statuses: % actors (%.1f%%)', actors_with_statuses, (actors_with_statuses::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE ' Chat declarations: % actors (%.1f%%)', actors_with_chat, (actors_with_chat::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE ' Notif declarations: % actors (%.1f%%)', actors_with_notif_decl, (actors_with_notif_decl::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE ' Notification state: % actors (%.1f%%)', actors_with_notif_state, (actors_with_notif_state::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE ''; + RAISE NOTICE 'Denormalized stats:'; + RAISE NOTICE ' Actors with followers: % (%.1f%%)', actors_with_followers, (actors_with_followers::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE ' Actors with following: % (%.1f%%)', actors_with_following, (actors_with_following::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE ' Actors with posts: % (%.1f%%)', actors_with_posts, (actors_with_posts::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE ' Actors with lists: % (%.1f%%)', actors_with_lists, (actors_with_lists::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE ' Actors with feeds: % (%.1f%%)', actors_with_feeds, (actors_with_feeds::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE ' Actors with starterpacks: % (%.1f%%)', actors_with_starterpacks, (actors_with_starterpacks::float / NULLIF(total_actors, 0) * 100); + RAISE NOTICE '============================================================================'; + + -- Sample data from actors with rich profiles + RAISE NOTICE ''; + RAISE NOTICE 'Sample consolidated actor data (top 3 by followers):'; + RAISE NOTICE ''; + + FOR actor_record IN ( + SELECT + did, + handle, + profile_display_name, + COALESCE(followers_count, 0) as followers, + COALESCE(following_count, 0) as following, + COALESCE(posts_count, 0) as posts, + CASE WHEN profile_cid IS NOT NULL THEN 'Yes' ELSE 'No' END as has_profile, + CASE WHEN status_type IS NOT NULL THEN 'Yes' ELSE 'No' END as has_status, + CASE WHEN chat_allow_incoming IS NOT NULL THEN chat_allow_incoming::text ELSE 'None' END as chat_pref + FROM actors + WHERE profile_display_name IS NOT NULL + ORDER BY followers_count DESC NULLS LAST + LIMIT 3 + ) LOOP + RAISE NOTICE ' DID: %', actor_record.did; + RAISE NOTICE ' Handle: %', COALESCE(actor_record.handle, '(none)'); + RAISE NOTICE ' Display name: %', actor_record.profile_display_name; + RAISE NOTICE ' Stats: % followers, % following, % posts', actor_record.followers, actor_record.following, actor_record.posts; + RAISE NOTICE ' Profile: %, Status: %, Chat: %', actor_record.has_profile, actor_record.has_status, actor_record.chat_pref; + RAISE NOTICE ''; + END LOOP; +END $$; diff --git a/parakeet-db/src/models.rs b/parakeet-db/src/models.rs index 0fd03eca..62c3a56b 100644 --- a/parakeet-db/src/models.rs +++ b/parakeet-db/src/models.rs @@ -167,11 +167,11 @@ pub struct Post { pub facet_8: Option, pub mentions: Option>>, // Array of actor_ids // Aggregate stats (denormalized for performance, NULL = 0) - // SMALLINT in PostgreSQL = Option in Rust - pub like_count: Option, - pub reply_count: Option, - pub repost_count: Option, - pub quote_count: Option, + // INTEGER in PostgreSQL = Option in Rust (changed from SMALLINT to prevent overflow) + pub like_count: Option, + pub reply_count: Option, + pub repost_count: Option, + pub quote_count: Option, // Note: created_at derived from TID rkey via created_at() method } diff --git a/parakeet-db/src/schema.rs b/parakeet-db/src/schema.rs index d4a39a0a..43ab0e8f 100644 --- a/parakeet-db/src/schema.rs +++ b/parakeet-db/src/schema.rs @@ -1,10 +1,6 @@ // @generated automatically by Diesel CLI. pub mod sql_types { - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "actor_stat_type"))] - pub struct ActorStatType; - #[derive(diesel::query_builder::QueryId, diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "actor_status"))] pub struct ActorStatus; @@ -158,22 +154,15 @@ diesel::table! { } } -diesel::table! { - use diesel::sql_types::*; - use super::sql_types::ActorStatType; - - actor_aggregate_stats (actor_id, stat_type) { - actor_id -> Int4, - stat_type -> ActorStatType, - value -> Int4, - updated_at -> Timestamptz, - } -} - diesel::table! { use diesel::sql_types::*; use super::sql_types::ActorStatus; use super::sql_types::ActorSyncState; + use super::sql_types::Tsvector; + use super::sql_types::StatusType; + use super::sql_types::ImageMimeType; + use super::sql_types::ChatAllowIncoming; + use super::sql_types::NotifAllowSubscriptions; actors (id) { id -> Int4, @@ -185,6 +174,37 @@ diesel::table! { repo_cid -> Nullable, last_indexed -> Nullable, account_created_at -> Nullable, + profile_cid -> Nullable, + profile_created_at -> Nullable, + profile_avatar_cid -> Nullable, + profile_banner_cid -> Nullable, + profile_display_name -> Nullable, + profile_description -> Nullable, + profile_pinned_post_rkey -> Nullable, + profile_joined_sp_id -> Nullable, + profile_pronouns -> Nullable, + profile_website -> Nullable, + profile_search_vector -> Nullable, + status_cid -> Nullable, + status_created_at -> Nullable, + status_type -> Nullable, + status_duration -> Nullable, + status_embed_post_actor_id -> Nullable, + status_embed_post_rkey -> Nullable, + status_thumb_mime_type -> Nullable, + status_thumb_cid -> Nullable, + chat_allow_incoming -> Nullable, + chat_created_at -> Nullable, + notif_decl_allow_subscriptions -> Nullable, + notif_decl_created_at -> Nullable, + notif_seen_at -> Nullable, + notif_unread_count -> Nullable, + followers_count -> Nullable, + following_count -> Nullable, + posts_count -> Nullable, + lists_count -> Nullable, + feeds_count -> Nullable, + starterpacks_count -> Nullable, } } @@ -218,17 +238,6 @@ diesel::table! { } } -diesel::table! { - use diesel::sql_types::*; - use super::sql_types::ChatAllowIncoming; - - chat_decls (actor_id) { - actor_id -> Int4, - allow_incoming -> ChatAllowIncoming, - created_at -> Timestamptz, - } -} - diesel::table! { constellation_enrichment_queue (id) { id -> Int8, @@ -431,25 +440,6 @@ diesel::table! { } } -diesel::table! { - use diesel::sql_types::*; - use super::sql_types::NotifAllowSubscriptions; - - notif_decl (actor_id) { - actor_id -> Int4, - allow_subscriptions -> Nullable, - created_at -> Timestamptz, - } -} - -diesel::table! { - notification_state (actor_id) { - actor_id -> Int4, - seen_at -> Nullable, - unread_count -> Nullable, - } -} - diesel::table! { use diesel::sql_types::*; use super::sql_types::NotificationReason; @@ -561,30 +551,10 @@ diesel::table! { facet_7 -> Nullable, facet_8 -> Nullable, mentions -> Nullable>>, - like_count -> Nullable, - reply_count -> Nullable, - repost_count -> Nullable, - quote_count -> Nullable, - } -} - -diesel::table! { - use diesel::sql_types::*; - use super::sql_types::Tsvector; - - profiles (actor_id) { - actor_id -> Int4, - cid -> Bytea, - created_at -> Timestamptz, - avatar_cid -> Nullable, - banner_cid -> Nullable, - display_name -> Nullable, - description -> Nullable, - pinned_post_rkey -> Nullable, - joined_sp_id -> Nullable, - pronouns -> Nullable, - website -> Nullable, - search_vector -> Nullable, + like_count -> Nullable, + reply_count -> Nullable, + repost_count -> Nullable, + quote_count -> Nullable, } } @@ -645,24 +615,6 @@ diesel::table! { } } -diesel::table! { - use diesel::sql_types::*; - use super::sql_types::StatusType; - use super::sql_types::ImageMimeType; - - statuses (actor_id) { - actor_id -> Int4, - cid -> Bytea, - created_at -> Timestamptz, - status -> StatusType, - duration -> Nullable, - embed_post_actor_id -> Nullable, - embed_post_rkey -> Nullable, - thumb_mime_type -> Nullable, - thumb_cid -> Nullable, - } -} - diesel::table! { thread_mutes (actor_id, root_post_actor_id, root_post_rkey) { actor_id -> Int4, @@ -724,10 +676,10 @@ diesel::table! { } } +diesel::joinable!(actors -> starterpacks (profile_joined_sp_id)); diesel::joinable!(list_blocks -> lists (list_id)); diesel::joinable!(list_items -> lists (list_id)); diesel::joinable!(list_mutes -> lists (list_id)); -diesel::joinable!(profiles -> starterpacks (joined_sp_id)); diesel::joinable!(starterpack_feeds -> feedgens (feed_id)); diesel::joinable!(starterpack_feeds -> starterpacks (starterpack_id)); diesel::joinable!(starterpacks -> lists (list_id)); @@ -735,12 +687,10 @@ diesel::joinable!(threadgate_allowed_lists -> lists (list_id)); diesel::allow_tables_to_appear_in_same_query!( _diesel_schema_inference, - actor_aggregate_stats, actors, backfill_jobs, blocks, bookmarks, - chat_decls, constellation_enrichment_queue, feedgen_likes, feedgens, @@ -757,20 +707,16 @@ diesel::allow_tables_to_appear_in_same_query!( list_mutes, lists, mutes, - notif_decl, - notification_state, notifications, post_aggregate_stats, post_likes, postgate_detached, postgates, posts, - profiles, reposts, spatial_ref_sys, starterpack_feeds, starterpacks, - statuses, thread_mutes, threadgate_allowed_lists, threadgate_hidden_replies, diff --git a/parakeet/src/loaders/post.rs b/parakeet/src/loaders/post.rs index 6ae9b81a..0fe493bf 100644 --- a/parakeet/src/loaders/post.rs +++ b/parakeet/src/loaders/post.rs @@ -519,15 +519,15 @@ impl BatchFn for PostLoader { facet_7: Option, #[diesel(sql_type = diesel::sql_types::Nullable)] facet_8: Option, - // Aggregate stats (denormalized columns) - #[diesel(sql_type = diesel::sql_types::Nullable)] - like_count: Option, - #[diesel(sql_type = diesel::sql_types::Nullable)] - reply_count: Option, - #[diesel(sql_type = diesel::sql_types::Nullable)] - repost_count: Option, - #[diesel(sql_type = diesel::sql_types::Nullable)] - quote_count: Option, + // Aggregate stats (denormalized columns - changed from SmallInt to Integer) + #[diesel(sql_type = diesel::sql_types::Nullable)] + like_count: Option, + #[diesel(sql_type = diesel::sql_types::Nullable)] + reply_count: Option, + #[diesel(sql_type = diesel::sql_types::Nullable)] + repost_count: Option, + #[diesel(sql_type = diesel::sql_types::Nullable)] + quote_count: Option, // Note: author_did comes from posts_with_keys, not from query result } @@ -1256,10 +1256,10 @@ impl BatchFn for PostLoader { // Create PostStats from denormalized columns let stats = parakeet_db::models::PostStats { - likes: like_count.unwrap_or(0) as i32, - replies: reply_count.unwrap_or(0) as i32, - reposts: repost_count.unwrap_or(0) as i32, - quotes: quote_count.unwrap_or(0) as i32, + likes: like_count.unwrap_or(0), + replies: reply_count.unwrap_or(0), + reposts: repost_count.unwrap_or(0), + quotes: quote_count.unwrap_or(0), }; let hydrated_post = HydratedPost { -- 2.51.2