From bb297394e0beea69608a1fd5f9cf9854f5dbeff2 Mon Sep 17 00:00:00 2001
From: scanash00 <scan@scanash.com>
Date: Mon, 9 Feb 2026 23:31:02 -0900
Subject: [PATCH] Retrieve and merge annotations, highlights, and bookmarks
 using both normalized and raw URL hashes

---
 backend/internal/api/handler.go    | 65 ++++++++++++++++++++++++++++++
 backend/internal/db/queries.go     |  6 +--
 lexicons/at/margin/annotation.json | 35 +++++++++-------
 3 files changed, 88 insertions(+), 18 deletions(-)

diff --git a/backend/internal/api/handler.go b/backend/internal/api/handler.go
index f0a85cb..3d8cf17 100644
--- a/backend/internal/api/handler.go
+++ b/backend/internal/api/handler.go
@@ -895,11 +895,22 @@ func (h *Handler) GetByTarget(w http.ResponseWriter, r *http.Request) {
 	offset := parseIntParam(r, "offset", 0)
 
 	urlHash := db.HashURL(source)
+	rawHash := db.HashString(source)
 
 	annotations, _ := h.db.GetAnnotationsByTargetHash(urlHash, limit, offset)
 	highlights, _ := h.db.GetHighlightsByTargetHash(urlHash, limit, offset)
 	bookmarks, _ := h.db.GetBookmarksByTargetHash(urlHash, limit, offset)
 
+	if rawHash != urlHash {
+		rawAnnotations, _ := h.db.GetAnnotationsByTargetHash(rawHash, limit, offset)
+		rawHighlights, _ := h.db.GetHighlightsByTargetHash(rawHash, limit, offset)
+		rawBookmarks, _ := h.db.GetBookmarksByTargetHash(rawHash, limit, offset)
+
+		annotations = mergeAnnotations(annotations, rawAnnotations)
+		highlights = mergeHighlights(highlights, rawHighlights)
+		bookmarks = mergeBookmarks(bookmarks, rawBookmarks)
+	}
+
 	enrichedAnnotations, _ := hydrateAnnotations(h.db, annotations, h.getViewerDID(r))
 	enrichedHighlights, _ := hydrateHighlights(h.db, highlights, h.getViewerDID(r))
 	enrichedBookmarks, _ := hydrateBookmarks(h.db, bookmarks, h.getViewerDID(r))
@@ -1569,3 +1580,57 @@ func (h *Handler) filterFeedByModeration(feed []interface{}, viewerDID string) [
 	}
 	return filtered
 }
+
+func mergeAnnotations(a, b []db.Annotation) []db.Annotation {
+	seen := make(map[string]bool)
+	var result []db.Annotation
+	for _, item := range a {
+		if !seen[item.URI] {
+			seen[item.URI] = true
+			result = append(result, item)
+		}
+	}
+	for _, item := range b {
+		if !seen[item.URI] {
+			seen[item.URI] = true
+			result = append(result, item)
+		}
+	}
+	return result
+}
+
+func mergeHighlights(a, b []db.Highlight) []db.Highlight {
+	seen := make(map[string]bool)
+	var result []db.Highlight
+	for _, item := range a {
+		if !seen[item.URI] {
+			seen[item.URI] = true
+			result = append(result, item)
+		}
+	}
+	for _, item := range b {
+		if !seen[item.URI] {
+			seen[item.URI] = true
+			result = append(result, item)
+		}
+	}
+	return result
+}
+
+func mergeBookmarks(a, b []db.Bookmark) []db.Bookmark {
+	seen := make(map[string]bool)
+	var result []db.Bookmark
+	for _, item := range a {
+		if !seen[item.URI] {
+			seen[item.URI] = true
+			result = append(result, item)
+		}
+	}
+	for _, item := range b {
+		if !seen[item.URI] {
+			seen[item.URI] = true
+			result = append(result, item)
+		}
+	}
+	return result
+}
diff --git a/backend/internal/db/queries.go b/backend/internal/db/queries.go
index 9aa2e2e..78c95d6 100644
--- a/backend/internal/db/queries.go
+++ b/backend/internal/db/queries.go
@@ -43,7 +43,7 @@ func (db *DB) AnnotationExists(uri string) bool {
 func HashURL(rawURL string) string {
 	parsed, err := url.Parse(rawURL)
 	if err != nil || parsed.Host == "" {
-		return hashString(rawURL)
+		return HashString(rawURL)
 	}
 
 	host := strings.ToLower(parsed.Host)
@@ -55,10 +55,10 @@ func HashURL(rawURL string) string {
 	}
 	normalized = strings.TrimSuffix(normalized, "/")
 
-	return hashString(normalized)
+	return HashString(normalized)
 }
 
-func hashString(s string) string {
+func HashString(s string) string {
 	h := sha256.New()
 	h.Write([]byte(s))
 	return hex.EncodeToString(h.Sum(nil))
diff --git a/lexicons/at/margin/annotation.json b/lexicons/at/margin/annotation.json
index 35fe890..9cc3e31 100644
--- a/lexicons/at/margin/annotation.json
+++ b/lexicons/at/margin/annotation.json
@@ -49,21 +49,9 @@
             "maxLength": 10
           },
           "generator": {
-            "type": "object",
-            "description": "The client/agent that created this record",
-            "properties": {
-              "id": {
-                "type": "string",
-                "format": "uri"
-              },
-              "name": {
-                "type": "string"
-              },
-              "homepage": {
-                "type": "string",
-                "format": "uri"
-              }
-            }
+            "type": "ref",
+            "ref": "#generator",
+            "description": "The client/agent that created this record"
           },
           "rights": {
             "type": "string",
@@ -82,6 +70,23 @@
         }
       }
     },
+    "generator": {
+      "type": "object",
+      "description": "The client/agent that created this record",
+      "properties": {
+        "id": {
+          "type": "string",
+          "format": "uri"
+        },
+        "name": {
+          "type": "string"
+        },
+        "homepage": {
+          "type": "string",
+          "format": "uri"
+        }
+      }
+    },
     "body": {
       "type": "object",
       "description": "Annotation body - the content of the annotation",
-- 
2.51.2