From 2df1ff6ad47bce37db396d337159228030f4f1a9 Mon Sep 17 00:00:00 2001 From: Timothy Quilling Date: Tue, 9 Dec 2025 10:45:32 -0500 Subject: [PATCH] feat: sort like rkeys for better columnar compression --- consumer/src/db/operations/feed/like.rs | 12 ++++++------ consumer/src/db/operations/feed/post_update.rs | 13 +++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/consumer/src/db/operations/feed/like.rs b/consumer/src/db/operations/feed/like.rs index c3ea47ab..60b98a08 100644 --- a/consumer/src/db/operations/feed/like.rs +++ b/consumer/src/db/operations/feed/like.rs @@ -170,12 +170,12 @@ pub async fn like_insert( let (feedgen_actor_id, feedgen_rkey) = subject_feedgen_key.expect("feedgen likes must have feedgen_key"); // Append like to feedgens table arrays (similar to posts) - // Use array concat operator || to append, with COALESCE for NULL handling + // Arrays are correlated (actor_id, rkey) pairs sorted by rkey for better columnstore compression // Note: like_count is maintained automatically by trigger conn.execute( "UPDATE feedgens - SET like_actor_ids = COALESCE(like_actor_ids, ARRAY[]::integer[]) || $3::integer, - like_rkeys = COALESCE(like_rkeys, ARRAY[]::bigint[]) || $4::bigint + SET like_actor_ids = (SELECT array_agg(actor_id ORDER BY rkey) FROM unnest(COALESCE(like_actor_ids, ARRAY[]::integer[]) || $3::integer, COALESCE(like_rkeys, ARRAY[]::bigint[]) || $4::bigint) AS t(actor_id, rkey)), + like_rkeys = (SELECT array_agg(rkey ORDER BY rkey) FROM unnest(COALESCE(like_actor_ids, ARRAY[]::integer[]) || $3::integer, COALESCE(like_rkeys, ARRAY[]::bigint[]) || $4::bigint) AS t(actor_id, rkey)) WHERE actor_id = $1 AND rkey = $2", &[&feedgen_actor_id, &feedgen_rkey, &actor_id, &rkey], ) @@ -186,12 +186,12 @@ pub async fn like_insert( let labeler_actor_id = subject_labeler_actor_id.expect("labeler likes must have labeler_actor_id"); // Append like to actors table labeler arrays (similar to feedgens/posts) - // Use array concat operator || to append, with COALESCE for NULL handling + // Arrays are correlated (actor_id, rkey) pairs sorted by rkey for better columnstore compression // Note: labeler_like_count is maintained automatically by trigger conn.execute( "UPDATE actors - SET labeler_like_actor_ids = COALESCE(labeler_like_actor_ids, ARRAY[]::integer[]) || $2::integer, - labeler_like_rkeys = COALESCE(labeler_like_rkeys, ARRAY[]::bigint[]) || $3::bigint + SET labeler_like_actor_ids = (SELECT array_agg(actor_id ORDER BY rkey) FROM unnest(COALESCE(labeler_like_actor_ids, ARRAY[]::integer[]) || $2::integer, COALESCE(labeler_like_rkeys, ARRAY[]::bigint[]) || $3::bigint) AS t(actor_id, rkey)), + labeler_like_rkeys = (SELECT array_agg(rkey ORDER BY rkey) FROM unnest(COALESCE(labeler_like_actor_ids, ARRAY[]::integer[]) || $2::integer, COALESCE(labeler_like_rkeys, ARRAY[]::bigint[]) || $3::bigint) AS t(actor_id, rkey)) WHERE id = $1", &[&labeler_actor_id, &actor_id, &rkey], ) diff --git a/consumer/src/db/operations/feed/post_update.rs b/consumer/src/db/operations/feed/post_update.rs index 0cb3c956..67e7f83b 100644 --- a/consumer/src/db/operations/feed/post_update.rs +++ b/consumer/src/db/operations/feed/post_update.rs @@ -259,19 +259,20 @@ impl PostUpdate { liker_rkey, via_repost, } => { - // Append to like_actor_ids array + // Append new like and sort by rkey (timestamp) for better columnstore compression + // IMPORTANT: Arrays are correlated (actor_id, rkey) pairs, so we must sort together + // by rkey to preserve the correlation while enabling delta encoding compression clauses.push(format!( - "like_actor_ids = COALESCE(like_actor_ids, ARRAY[]::integer[]) || ${}::integer", - param_idx + "like_actor_ids = (SELECT array_agg(actor_id ORDER BY rkey) FROM unnest(COALESCE(like_actor_ids, ARRAY[]::integer[]) || ${}::integer, COALESCE(like_rkeys, ARRAY[]::bigint[]) || ${}::bigint) AS t(actor_id, rkey))", + param_idx, param_idx + 1 )); params.push(Box::new(*liker_actor_id)); let actor_param_idx = param_idx; param_idx += 1; - // Append to like_rkeys array clauses.push(format!( - "like_rkeys = COALESCE(like_rkeys, ARRAY[]::bigint[]) || ${}::bigint", - param_idx + "like_rkeys = (SELECT array_agg(rkey ORDER BY rkey) FROM unnest(COALESCE(like_actor_ids, ARRAY[]::integer[]) || ${}::integer, COALESCE(like_rkeys, ARRAY[]::bigint[]) || ${}::bigint) AS t(actor_id, rkey))", + actor_param_idx, param_idx )); params.push(Box::new(*liker_rkey)); param_idx += 1; -- 2.51.2