From 57979979492d9c9b4836a5f3f3c52f83bcb7a600 Mon Sep 17 00:00:00 2001 From: Bretton Date: Sat, 8 Nov 2025 23:21:31 -0800 Subject: [PATCH] feat: add blob-to-URL transformation for feed API responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transform blob references to direct PDS URLs in feed responses, enabling clients to fetch thumbnails without complex blob resolution logic. **Blob Transform Module:** - TransformBlobRefsToURLs: Convert blob refs → PDS URLs in-place - transformThumbToURL: Extract CID and build getBlob URL - Handles external embeds only (social.coves.embed.external) - Graceful handling of missing/malformed data **Transform Logic:** ```go // Before (blob ref in database): "thumb": { "$type": "blob", "ref": {"$link": "bafyrei..."}, "mimeType": "image/jpeg", "size": 52813 } // After (URL string in API response): "thumb": "http://pds.example.com/xrpc/com.atproto.sync.getBlob?did=did:plc:community&cid=bafyrei..." ``` **Repository Updates:** - Add community_pds_url to all feed queries (feed_repo_base.go) - Include PDSURL in PostView.Community for transformation - Apply to: GetCommunityFeed, GetTimeline, GetDiscover **Handler Updates:** - Call TransformBlobRefsToURLs before returning posts - Applies to: social.coves.feed.getCommunityFeed - Applies to: social.coves.feed.getTimeline - Applies to: social.coves.feed.getDiscover **Comprehensive Tests** (13 test cases): - Valid blob ref → URL transformation - Missing thumb (no-op) - Already-transformed URL (no-op) - Nil post/community (no-op) - Missing/empty PDS URL (no-op) - Malformed blob refs (graceful) - Non-external embed types (ignored) **Why This Matters:** Clients receive ready-to-use image URLs instead of blob references, simplifying rendering and eliminating need for CID resolution logic. Works seamlessly with federated communities (each has own PDS URL). Fixes client-side rendering for external embeds with thumbnails. --- .../handlers/communityFeed/get_community.go | 8 + .../api/handlers/discover/get_discover.go | 8 + .../api/handlers/timeline/get_timeline.go | 8 + internal/core/posts/blob_transform.go | 81 +++++ internal/core/posts/blob_transform_test.go | 312 ++++++++++++++++++ internal/db/postgres/discover_repo.go | 4 +- internal/db/postgres/feed_repo.go | 4 +- internal/db/postgres/feed_repo_base.go | 6 +- internal/db/postgres/timeline_repo.go | 4 +- 9 files changed, 428 insertions(+), 7 deletions(-) create mode 100644 internal/core/posts/blob_transform.go create mode 100644 internal/core/posts/blob_transform_test.go diff --git a/internal/api/handlers/communityFeed/get_community.go b/internal/api/handlers/communityFeed/get_community.go index f81562e..015e1f2 100644 --- a/internal/api/handlers/communityFeed/get_community.go +++ b/internal/api/handlers/communityFeed/get_community.go @@ -2,6 +2,7 @@ package communityFeed import ( "Coves/internal/core/communityFeeds" + "Coves/internal/core/posts" "encoding/json" "log" "net/http" @@ -46,6 +47,13 @@ func (h *GetCommunityHandler) HandleGetCommunity(w http.ResponseWriter, r *http. return } + // Transform blob refs to URLs for all posts + for _, feedPost := range response.Feed { + if feedPost.Post != nil { + posts.TransformBlobRefsToURLs(feedPost.Post) + } + } + // Return feed w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) diff --git a/internal/api/handlers/discover/get_discover.go b/internal/api/handlers/discover/get_discover.go index eea484b..b648772 100644 --- a/internal/api/handlers/discover/get_discover.go +++ b/internal/api/handlers/discover/get_discover.go @@ -2,6 +2,7 @@ package discover import ( "Coves/internal/core/discover" + "Coves/internal/core/posts" "encoding/json" "log" "net/http" @@ -39,6 +40,13 @@ func (h *GetDiscoverHandler) HandleGetDiscover(w http.ResponseWriter, r *http.Re return } + // Transform blob refs to URLs for all posts + for _, feedPost := range response.Feed { + if feedPost.Post != nil { + posts.TransformBlobRefsToURLs(feedPost.Post) + } + } + // Return feed w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) diff --git a/internal/api/handlers/timeline/get_timeline.go b/internal/api/handlers/timeline/get_timeline.go index af23e76..e6b9675 100644 --- a/internal/api/handlers/timeline/get_timeline.go +++ b/internal/api/handlers/timeline/get_timeline.go @@ -2,6 +2,7 @@ package timeline import ( "Coves/internal/api/middleware" + "Coves/internal/core/posts" "Coves/internal/core/timeline" "encoding/json" "log" @@ -52,6 +53,13 @@ func (h *GetTimelineHandler) HandleGetTimeline(w http.ResponseWriter, r *http.Re return } + // Transform blob refs to URLs for all posts + for _, feedPost := range response.Feed { + if feedPost.Post != nil { + posts.TransformBlobRefsToURLs(feedPost.Post) + } + } + // Return feed w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) diff --git a/internal/core/posts/blob_transform.go b/internal/core/posts/blob_transform.go new file mode 100644 index 0000000..2e262c3 --- /dev/null +++ b/internal/core/posts/blob_transform.go @@ -0,0 +1,81 @@ +package posts + +import ( + "fmt" +) + +// TransformBlobRefsToURLs transforms all blob references in a PostView to PDS URLs +// This modifies the Embed field in-place, converting blob refs to direct URLs +// The transformation only affects external embeds with thumbnail blobs +func TransformBlobRefsToURLs(postView *PostView) { + if postView == nil || postView.Embed == nil { + return + } + + // Get community PDS URL from post view + if postView.Community == nil || postView.Community.PDSURL == "" { + return // Cannot transform without PDS URL + } + + communityDID := postView.Community.DID + pdsURL := postView.Community.PDSURL + + // Check if embed is a map (should be for external embeds) + embedMap, ok := postView.Embed.(map[string]interface{}) + if !ok { + return + } + + // Check embed type + embedType, ok := embedMap["$type"].(string) + if !ok { + return + } + + // Only transform external embeds + if embedType == "social.coves.embed.external" { + if external, ok := embedMap["external"].(map[string]interface{}); ok { + transformThumbToURL(external, communityDID, pdsURL) + } + } +} + +// transformThumbToURL converts a thumb blob ref to a PDS URL +// This modifies the external map in-place +func transformThumbToURL(external map[string]interface{}, communityDID, pdsURL string) { + // Check if thumb exists + thumb, ok := external["thumb"] + if !ok { + return + } + + // If thumb is already a string (URL), don't transform + if _, isString := thumb.(string); isString { + return + } + + // Try to parse as blob ref + thumbMap, ok := thumb.(map[string]interface{}) + if !ok { + return + } + + // Extract CID from blob ref + ref, ok := thumbMap["ref"].(map[string]interface{}) + if !ok { + return + } + + cid, ok := ref["$link"].(string) + if !ok || cid == "" { + return + } + + // Transform to PDS blob endpoint URL + // Format: {pds_url}/xrpc/com.atproto.sync.getBlob?did={community_did}&cid={cid} + blobURL := fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s", + pdsURL, communityDID, cid) + + // Replace blob ref with URL string + external["thumb"] = blobURL +} diff --git a/internal/core/posts/blob_transform_test.go b/internal/core/posts/blob_transform_test.go new file mode 100644 index 0000000..97aa199 --- /dev/null +++ b/internal/core/posts/blob_transform_test.go @@ -0,0 +1,312 @@ +package posts + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTransformBlobRefsToURLs(t *testing.T) { + t.Run("transforms external embed thumb from blob to URL", func(t *testing.T) { + post := &PostView{ + Community: &CommunityRef{ + DID: "did:plc:testcommunity", + PDSURL: "http://localhost:3001", + }, + Embed: map[string]interface{}{ + "$type": "social.coves.embed.external", + "external": map[string]interface{}{ + "uri": "https://example.com", + "thumb": map[string]interface{}{ + "$type": "blob", + "ref": map[string]interface{}{ + "$link": "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm", + }, + "mimeType": "image/jpeg", + "size": 52813, + }, + }, + }, + } + + TransformBlobRefsToURLs(post) + + // Verify embed is still a map + embedMap, ok := post.Embed.(map[string]interface{}) + require.True(t, ok, "embed should still be a map") + + // Verify external is still a map + external, ok := embedMap["external"].(map[string]interface{}) + require.True(t, ok, "external should be a map") + + // Verify thumb is now a URL string + thumbURL, ok := external["thumb"].(string) + require.True(t, ok, "thumb should be a string URL") + assert.Equal(t, + "http://localhost:3001/xrpc/com.atproto.sync.getBlob?did=did:plc:testcommunity&cid=bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm", + thumbURL) + }) + + t.Run("handles missing thumb gracefully", func(t *testing.T) { + post := &PostView{ + Community: &CommunityRef{ + DID: "did:plc:testcommunity", + PDSURL: "http://localhost:3001", + }, + Embed: map[string]interface{}{ + "$type": "social.coves.embed.external", + "external": map[string]interface{}{ + "uri": "https://example.com", + // No thumb field + }, + }, + } + + // Should not panic + TransformBlobRefsToURLs(post) + + // Verify external is unchanged + embedMap := post.Embed.(map[string]interface{}) + external := embedMap["external"].(map[string]interface{}) + _, hasThumb := external["thumb"] + assert.False(t, hasThumb, "thumb should not be added") + }) + + t.Run("handles already-transformed URL thumb", func(t *testing.T) { + expectedURL := "http://localhost:3001/xrpc/com.atproto.sync.getBlob?did=did:plc:test&cid=bafytest" + post := &PostView{ + Community: &CommunityRef{ + DID: "did:plc:testcommunity", + PDSURL: "http://localhost:3001", + }, + Embed: map[string]interface{}{ + "$type": "social.coves.embed.external", + "external": map[string]interface{}{ + "uri": "https://example.com", + "thumb": expectedURL, // Already a URL string + }, + }, + } + + // Should not error or change the URL + TransformBlobRefsToURLs(post) + + // Verify thumb is unchanged + embedMap := post.Embed.(map[string]interface{}) + external := embedMap["external"].(map[string]interface{}) + thumbURL, ok := external["thumb"].(string) + require.True(t, ok, "thumb should still be a string") + assert.Equal(t, expectedURL, thumbURL, "thumb URL should be unchanged") + }) + + t.Run("handles missing embed", func(t *testing.T) { + post := &PostView{ + Community: &CommunityRef{ + DID: "did:plc:testcommunity", + PDSURL: "http://localhost:3001", + }, + Embed: nil, + } + + // Should not panic + TransformBlobRefsToURLs(post) + + // Verify embed is still nil + assert.Nil(t, post.Embed, "embed should remain nil") + }) + + t.Run("handles nil post", func(t *testing.T) { + // Should not panic + TransformBlobRefsToURLs(nil) + }) + + t.Run("handles missing community", func(t *testing.T) { + post := &PostView{ + Community: nil, + Embed: map[string]interface{}{ + "$type": "social.coves.embed.external", + "external": map[string]interface{}{ + "uri": "https://example.com", + "thumb": map[string]interface{}{ + "$type": "blob", + "ref": map[string]interface{}{ + "$link": "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm", + }, + }, + }, + }, + } + + // Should not panic or transform + TransformBlobRefsToURLs(post) + + // Verify thumb is unchanged (still a blob) + embedMap := post.Embed.(map[string]interface{}) + external := embedMap["external"].(map[string]interface{}) + thumb, ok := external["thumb"].(map[string]interface{}) + require.True(t, ok, "thumb should still be a map (blob ref)") + assert.Equal(t, "blob", thumb["$type"], "blob type should be unchanged") + }) + + t.Run("handles missing PDS URL", func(t *testing.T) { + post := &PostView{ + Community: &CommunityRef{ + DID: "did:plc:testcommunity", + PDSURL: "", // Empty PDS URL + }, + Embed: map[string]interface{}{ + "$type": "social.coves.embed.external", + "external": map[string]interface{}{ + "uri": "https://example.com", + "thumb": map[string]interface{}{ + "$type": "blob", + "ref": map[string]interface{}{ + "$link": "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm", + }, + }, + }, + }, + } + + // Should not panic or transform + TransformBlobRefsToURLs(post) + + // Verify thumb is unchanged (still a blob) + embedMap := post.Embed.(map[string]interface{}) + external := embedMap["external"].(map[string]interface{}) + thumb, ok := external["thumb"].(map[string]interface{}) + require.True(t, ok, "thumb should still be a map (blob ref)") + assert.Equal(t, "blob", thumb["$type"], "blob type should be unchanged") + }) + + t.Run("handles malformed blob ref gracefully", func(t *testing.T) { + post := &PostView{ + Community: &CommunityRef{ + DID: "did:plc:testcommunity", + PDSURL: "http://localhost:3001", + }, + Embed: map[string]interface{}{ + "$type": "social.coves.embed.external", + "external": map[string]interface{}{ + "uri": "https://example.com", + "thumb": map[string]interface{}{ + "$type": "blob", + "ref": "invalid-ref-format", // Should be a map with $link + }, + }, + }, + } + + // Should not panic + TransformBlobRefsToURLs(post) + + // Verify thumb is unchanged (malformed blob) + embedMap := post.Embed.(map[string]interface{}) + external := embedMap["external"].(map[string]interface{}) + thumb, ok := external["thumb"].(map[string]interface{}) + require.True(t, ok, "thumb should still be a map") + assert.Equal(t, "invalid-ref-format", thumb["ref"], "malformed ref should be unchanged") + }) + + t.Run("ignores non-external embed types", func(t *testing.T) { + post := &PostView{ + Community: &CommunityRef{ + DID: "did:plc:testcommunity", + PDSURL: "http://localhost:3001", + }, + Embed: map[string]interface{}{ + "$type": "social.coves.embed.images", + "images": []interface{}{ + map[string]interface{}{ + "image": map[string]interface{}{ + "$type": "blob", + "ref": map[string]interface{}{ + "$link": "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm", + }, + }, + }, + }, + }, + } + + // Should not transform non-external embeds + TransformBlobRefsToURLs(post) + + // Verify images embed is unchanged + embedMap := post.Embed.(map[string]interface{}) + images := embedMap["images"].([]interface{}) + imageObj := images[0].(map[string]interface{}) + imageBlob := imageObj["image"].(map[string]interface{}) + assert.Equal(t, "blob", imageBlob["$type"], "image blob should be unchanged") + }) +} + +func TestTransformThumbToURL(t *testing.T) { + t.Run("transforms valid blob ref to URL", func(t *testing.T) { + external := map[string]interface{}{ + "uri": "https://example.com", + "thumb": map[string]interface{}{ + "$type": "blob", + "ref": map[string]interface{}{ + "$link": "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm", + }, + "mimeType": "image/jpeg", + "size": 52813, + }, + } + + transformThumbToURL(external, "did:plc:test", "http://localhost:3001") + + thumbURL, ok := external["thumb"].(string) + require.True(t, ok, "thumb should be a string URL") + assert.Equal(t, + "http://localhost:3001/xrpc/com.atproto.sync.getBlob?did=did:plc:test&cid=bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm", + thumbURL) + }) + + t.Run("does not transform if thumb is already string", func(t *testing.T) { + expectedURL := "http://localhost:3001/xrpc/com.atproto.sync.getBlob?did=did:plc:test&cid=bafytest" + external := map[string]interface{}{ + "uri": "https://example.com", + "thumb": expectedURL, + } + + transformThumbToURL(external, "did:plc:test", "http://localhost:3001") + + thumbURL, ok := external["thumb"].(string) + require.True(t, ok, "thumb should still be a string") + assert.Equal(t, expectedURL, thumbURL, "thumb should be unchanged") + }) + + t.Run("does not transform if thumb is missing", func(t *testing.T) { + external := map[string]interface{}{ + "uri": "https://example.com", + } + + transformThumbToURL(external, "did:plc:test", "http://localhost:3001") + + _, hasThumb := external["thumb"] + assert.False(t, hasThumb, "thumb should not be added") + }) + + t.Run("does not transform if CID is empty", func(t *testing.T) { + external := map[string]interface{}{ + "uri": "https://example.com", + "thumb": map[string]interface{}{ + "$type": "blob", + "ref": map[string]interface{}{ + "$link": "", // Empty CID + }, + }, + } + + transformThumbToURL(external, "did:plc:test", "http://localhost:3001") + + // Verify thumb is unchanged + thumb, ok := external["thumb"].(map[string]interface{}) + require.True(t, ok, "thumb should still be a map") + ref := thumb["ref"].(map[string]interface{}) + assert.Equal(t, "", ref["$link"], "empty CID should be unchanged") + }) +} diff --git a/internal/db/postgres/discover_repo.go b/internal/db/postgres/discover_repo.go index a601da9..39d8997 100644 --- a/internal/db/postgres/discover_repo.go +++ b/internal/db/postgres/discover_repo.go @@ -47,7 +47,7 @@ func (r *postgresDiscoverRepo) GetDiscover(ctx context.Context, req discover.Get SELECT p.uri, p.cid, p.rkey, p.author_did, u.handle as author_handle, - p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, + p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, c.pds_url as community_pds_url, p.title, p.content, p.content_facets, p.embed, p.content_labels, p.created_at, p.edited_at, p.indexed_at, p.upvote_count, p.downvote_count, p.score, p.comment_count, @@ -58,7 +58,7 @@ func (r *postgresDiscoverRepo) GetDiscover(ctx context.Context, req discover.Get SELECT p.uri, p.cid, p.rkey, p.author_did, u.handle as author_handle, - p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, + p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, c.pds_url as community_pds_url, p.title, p.content, p.content_facets, p.embed, p.content_labels, p.created_at, p.edited_at, p.indexed_at, p.upvote_count, p.downvote_count, p.score, p.comment_count, diff --git a/internal/db/postgres/feed_repo.go b/internal/db/postgres/feed_repo.go index eec5194..0ef88a9 100644 --- a/internal/db/postgres/feed_repo.go +++ b/internal/db/postgres/feed_repo.go @@ -53,7 +53,7 @@ func (r *postgresFeedRepo) GetCommunityFeed(ctx context.Context, req communityFe SELECT p.uri, p.cid, p.rkey, p.author_did, u.handle as author_handle, - p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, + p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, c.pds_url as community_pds_url, p.title, p.content, p.content_facets, p.embed, p.content_labels, p.created_at, p.edited_at, p.indexed_at, p.upvote_count, p.downvote_count, p.score, p.comment_count, @@ -64,7 +64,7 @@ func (r *postgresFeedRepo) GetCommunityFeed(ctx context.Context, req communityFe SELECT p.uri, p.cid, p.rkey, p.author_did, u.handle as author_handle, - p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, + p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, c.pds_url as community_pds_url, p.title, p.content, p.content_facets, p.embed, p.content_labels, p.created_at, p.edited_at, p.indexed_at, p.upvote_count, p.downvote_count, p.score, p.comment_count, diff --git a/internal/db/postgres/feed_repo_base.go b/internal/db/postgres/feed_repo_base.go index 71750dc..adbc36e 100644 --- a/internal/db/postgres/feed_repo_base.go +++ b/internal/db/postgres/feed_repo_base.go @@ -285,13 +285,14 @@ func (r *feedRepoBase) scanFeedPost(rows *sql.Rows) (*posts.PostView, float64, e editedAt sql.NullTime communityHandle sql.NullString communityAvatar sql.NullString + communityPDSURL sql.NullString hotRank sql.NullFloat64 ) err := rows.Scan( &postView.URI, &postView.CID, &postView.RKey, &authorView.DID, &authorView.Handle, - &communityRef.DID, &communityHandle, &communityRef.Name, &communityAvatar, + &communityRef.DID, &communityHandle, &communityRef.Name, &communityAvatar, &communityPDSURL, &title, &content, &facets, &embed, &labelsJSON, &postView.CreatedAt, &editedAt, &postView.IndexedAt, &postView.UpvoteCount, &postView.DownvoteCount, &postView.Score, &postView.CommentCount, @@ -309,6 +310,9 @@ func (r *feedRepoBase) scanFeedPost(rows *sql.Rows) (*posts.PostView, float64, e communityRef.Handle = communityHandle.String } communityRef.Avatar = nullStringPtr(communityAvatar) + if communityPDSURL.Valid { + communityRef.PDSURL = communityPDSURL.String + } postView.Community = &communityRef // Set optional fields diff --git a/internal/db/postgres/timeline_repo.go b/internal/db/postgres/timeline_repo.go index 0aa535b..4dcd9af 100644 --- a/internal/db/postgres/timeline_repo.go +++ b/internal/db/postgres/timeline_repo.go @@ -51,7 +51,7 @@ func (r *postgresTimelineRepo) GetTimeline(ctx context.Context, req timeline.Get SELECT p.uri, p.cid, p.rkey, p.author_did, u.handle as author_handle, - p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, + p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, c.pds_url as community_pds_url, p.title, p.content, p.content_facets, p.embed, p.content_labels, p.created_at, p.edited_at, p.indexed_at, p.upvote_count, p.downvote_count, p.score, p.comment_count, @@ -62,7 +62,7 @@ func (r *postgresTimelineRepo) GetTimeline(ctx context.Context, req timeline.Get SELECT p.uri, p.cid, p.rkey, p.author_did, u.handle as author_handle, - p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, + p.community_did, c.handle as community_handle, c.name as community_name, c.avatar_cid as community_avatar, c.pds_url as community_pds_url, p.title, p.content, p.content_facets, p.embed, p.content_labels, p.created_at, p.edited_at, p.indexed_at, p.upvote_count, p.downvote_count, p.score, p.comment_count, -- 2.51.2