From 05719419450371bf2ff476ab963676b1dde136df Mon Sep 17 00:00:00 2001 From: Cameron Pfiffer Date: Sun, 8 Feb 2026 23:26:11 -0800 Subject: [PATCH] Add astral100's site.standard.document, fix leaflet content extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added site.standard.document to indexed collections (astral's essays) - Added leaflet content extraction: nested pub.leaflet.content blocks get flattened to searchable plaintext - Backfill running for astral100.bsky.social 🤖 Generated with [Letta Code](https://letta.com) Co-Authored-By: Letta --- indexer/indexer/backfill.py | 2 ++ indexer/indexer/embeddings.py | 10 +++++++++- indexer/indexer/worker.py | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/indexer/indexer/backfill.py b/indexer/indexer/backfill.py index d751fd5..4128ff1 100644 --- a/indexer/indexer/backfill.py +++ b/indexer/indexer/backfill.py @@ -33,6 +33,8 @@ COLLECTIONS = [ "systems.witchcraft.concept", "systems.witchcraft.memory", "systems.witchcraft.announcement", + # astral's essays + "site.standard.document", ] # Comind collective handles diff --git a/indexer/indexer/embeddings.py b/indexer/indexer/embeddings.py index 57f6af1..b8521a8 100644 --- a/indexer/indexer/embeddings.py +++ b/indexer/indexer/embeddings.py @@ -70,7 +70,15 @@ def extract_content(record: dict) -> Optional[str]: if description := record.get("description"): parts.append(description) if content := record.get("content"): - parts.append(content) + if isinstance(content, str): + parts.append(content) + elif isinstance(content, dict): + # Handle nested content (e.g., pub.leaflet.content) + for page in content.get("pages", []): + for block_wrapper in page.get("blocks", []): + block = block_wrapper.get("block", block_wrapper) + if plaintext := block.get("plaintext"): + parts.append(plaintext) if thought := record.get("thought"): parts.append(thought) if claim := record.get("claim"): diff --git a/indexer/indexer/worker.py b/indexer/indexer/worker.py index f3bba99..7258092 100644 --- a/indexer/indexer/worker.py +++ b/indexer/indexer/worker.py @@ -56,6 +56,8 @@ BASE_COLLECTIONS = [ "systems.witchcraft.concept", "systems.witchcraft.memory", "systems.witchcraft.announcement", + # site.standard.* - astral's essays + "site.standard.document", ] # Seed DIDs - always indexed, even without a profile record -- 2.51.2