diff --git a/at/news-cli.mjs b/at/news-cli.mjs index 175d3f337e..278fa4d876 100644 --- a/at/news-cli.mjs +++ b/at/news-cli.mjs @@ -18,6 +18,11 @@ import { MongoClient } from "mongodb"; import { AtpAgent } from "@atproto/api"; +import { + ingestAll, + ingestFromActor, + getConfiguredSources, +} from "../system/backend/news-bluesky-ingest.mjs"; import { config } from "dotenv"; import { execSync } from "child_process"; import { randomBytes } from "crypto"; @@ -432,6 +437,57 @@ async function commandDelete(args) { }); } +// --------------------------------------------------------------------------- +// Bluesky ingest (external headlines from trusted sources) +// --------------------------------------------------------------------------- + +async function commandPullBluesky(args) { + const actor = args._[1]; + const limit = parseInt(args.limit) || 30; + + await withDb(async (db) => { + const database = { db }; + const runOne = !!actor; + + if (runOne) { + console.log(`\n Pulling Bluesky feed: ${actor} (limit ${limit})\n`); + const result = await ingestFromActor(database, actor, { + limit, + log: (line) => console.log(` ${line}`), + }); + console.log( + `\n ${actor}: +${result.inserted} inserted, ${result.skipped} skipped, ${result.errors.length} errors`, + ); + if (result.errors.length) { + for (const err of result.errors) { + console.log(` ! ${err.uri || ""} ${err.message}`); + } + } + return; + } + + const sources = getConfiguredSources(); + console.log(`\n Pulling ${sources.length} Bluesky source(s):\n`); + for (const src of sources) console.log(` - ${src}`); + console.log(); + + const results = await ingestAll(database, { + limit, + log: (line) => console.log(` ${line}`), + }); + for (const r of results) { + console.log( + `\n ${r.actor}: +${r.inserted} inserted, ${r.skipped} skipped, ${r.errors.length} errors`, + ); + if (r.errors.length) { + for (const err of r.errors) { + console.log(` ! ${err.uri || ""} ${err.message}`); + } + } + } + }); +} + // --------------------------------------------------------------------------- // Screenshot (via oven) // --------------------------------------------------------------------------- @@ -502,6 +558,11 @@ Manage: edit ... --dry-run Preview without saving delete Delete a post (admin) +External (trusted third-party sources): + pull-bluesky Pull all configured Bluesky sources + pull-bluesky Pull a specific Bluesky account + pull-bluesky ... --limit 30 Override per-source post limit + Examples: ac-news commits --since "1 week ago" ac-news post "Dev Update" "The native OS build system got a major overhaul..." @@ -510,6 +571,8 @@ Examples: ac-news edit ncd2 --replace "https://aesthetic.computer)" --with "https://aesthetic.computer/chat)" ac-news screenshot notepat ac-news list + ac-news pull-bluesky # pull all trusted Bluesky sources + ac-news pull-bluesky artistnewsnetwork.bsky.social `); } @@ -524,6 +587,7 @@ const COMMANDS = { edit: commandEdit, delete: commandDelete, screenshot: commandScreenshot, + "pull-bluesky": commandPullBluesky, }; async function main() { diff --git a/silo/bluesky-ingest.mjs b/silo/bluesky-ingest.mjs new file mode 100644 index 0000000000..a80cce1e1b --- /dev/null +++ b/silo/bluesky-ingest.mjs @@ -0,0 +1,224 @@ +// bluesky-ingest.mjs +// Pulls headlines from trusted Bluesky accounts into news-posts. +// Self-contained: uses fetch + crypto, no extra deps beyond what silo ships. +// +// Mirrors the logic of ../system/backend/news-bluesky-ingest.mjs, but rewritten +// to avoid the @atproto/api and nanoid dependencies so it can run inside silo. + +import { randomBytes } from "node:crypto"; + +const BSKY_APPVIEW = "https://public.api.bsky.app"; +const DEFAULT_LIMIT = 30; +const MAX_TITLE = 200; +const MAX_TEXT = 5000; + +// ---- Config ---------------------------------------------------------------- + +export function getConfiguredSources(env = process.env) { + const raw = env.NEWS_EXTERNAL_SOURCES; + if (!raw) return ["artistnewsnetwork.bsky.social"]; + return String(raw).split(",").map((s) => s.trim()).filter(Boolean); +} + +// ---- Short-code generator (same alphabet as the rest of the system) -------- + +const CODE_ALPHABET = "bcdfghjklmnpqrstvwxyzaeiou23456789"; +const CODE_LEN = 3; + +function makeCandidate() { + const bytes = randomBytes(CODE_LEN); + return Array.from(bytes).map((b) => CODE_ALPHABET[b % CODE_ALPHABET.length]).join(""); +} + +async function generateUniqueCode(posts) { + for (let i = 0; i < 100; i++) { + const candidate = `n${makeCandidate()}`; + const existing = await posts.findOne({ code: candidate }); + if (!existing) return candidate; + } + throw new Error("Could not generate a unique news code after 100 attempts"); +} + +// ---- Pure helpers ---------------------------------------------------------- + +function truncate(value, max) { + if (!value) return ""; + const trimmed = String(value).trim(); + return trimmed.length > max ? trimmed.slice(0, max) : trimmed; +} + +function stripUrls(text) { + if (!text) return ""; + return text.replace(/https?:\/\/\S+/gi, "").replace(/\s+/g, " ").trim(); +} + +function extractExternalEmbed(post) { + const embed = post?.embed; + if (embed?.$type === "app.bsky.embed.external#view" && embed.external?.uri) { + return { uri: embed.external.uri, title: embed.external.title || "" }; + } + const recordEmbed = post?.record?.embed; + if (recordEmbed?.$type === "app.bsky.embed.external" && recordEmbed.external?.uri) { + return { uri: recordEmbed.external.uri, title: recordEmbed.external.title || "" }; + } + return null; +} + +function extractFirstFacetLink(post) { + const facets = post?.record?.facets; + if (!Array.isArray(facets)) return null; + for (const f of facets) { + const link = f?.features?.find((x) => x?.$type === "app.bsky.richtext.facet#link" && x.uri); + if (link) return link.uri; + } + return null; +} + +function projectPost(post) { + const record = post?.record; + if (!record) return null; + const rawText = record.text || ""; + const external = extractExternalEmbed(post); + if (external) { + return { + title: truncate(external.title || stripUrls(rawText) || "Untitled link", MAX_TITLE), + url: external.uri, + text: truncate(stripUrls(rawText), MAX_TEXT), + }; + } + const facetLink = extractFirstFacetLink(post); + if (facetLink) { + return { + title: truncate(stripUrls(rawText) || "Untitled link", MAX_TITLE), + url: facetLink, + text: truncate(stripUrls(rawText), MAX_TEXT), + }; + } + return null; +} + +function parseAtUri(uri) { + const match = /^at:\/\/([^/]+)\/([^/]+)\/(.+)$/.exec(uri || ""); + if (!match) return null; + return { did: match[1], collection: match[2], rkey: match[3] }; +} + +// ---- Bluesky appview calls (fetch, no auth) ------------------------------- + +async function xrpcGet(method, params) { + const qs = new URLSearchParams(params).toString(); + const url = `${BSKY_APPVIEW}/xrpc/${method}?${qs}`; + const res = await fetch(url, { + headers: { "User-Agent": "AestheticNewsBot/1.0 (+https://news.aesthetic.computer)" }, + }); + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error(`${method} ${res.status}: ${body.slice(0, 200)}`); + } + return res.json(); +} + +async function resolveProfile(actor) { + const data = await xrpcGet("app.bsky.actor.getProfile", { actor }); + if (!data?.did) throw new Error(`Could not resolve Bluesky actor: ${actor}`); + return { did: data.did, handle: data.handle || actor }; +} + +async function getAuthorFeed(did, limit) { + const data = await xrpcGet("app.bsky.feed.getAuthorFeed", { + actor: did, + filter: "posts_no_replies", + limit: String(limit), + }); + return data?.feed || []; +} + +// ---- Core ingest ---------------------------------------------------------- + +/** + * Pull posts from one Bluesky actor into db.collection("news-posts"). + * Returns { actor, did, handle, inserted, skipped, errors, createdCodes }. + */ +export async function ingestFromActor(db, actor, options = {}) { + const { limit = DEFAULT_LIMIT, now = () => new Date(), log = () => {} } = options; + const posts = db.collection("news-posts"); + + const profile = await resolveProfile(actor); + const feed = await getAuthorFeed(profile.did, limit); + + let inserted = 0; + let skipped = 0; + const errors = []; + const createdCodes = []; + + for (const item of feed) { + const post = item?.post; + if (!post?.uri) { skipped++; continue; } + if (item?.reason?.$type === "app.bsky.feed.defs#reasonRepost") { skipped++; continue; } + if (post?.record?.reply) { skipped++; continue; } + + const existing = await posts.findOne({ "external.postUri": post.uri }); + if (existing) { skipped++; continue; } + + const projected = projectPost(post); + if (!projected?.title) { skipped++; continue; } + + const parsed = parseAtUri(post.uri); + const postedAt = post.record?.createdAt ? new Date(post.record.createdAt) : now(); + const fetchedAt = now(); + + try { + const code = await generateUniqueCode(posts); + const doc = { + code, + title: projected.title, + url: projected.url, + text: projected.text, + user: null, + when: postedAt, + updated: fetchedAt, + score: 1, + commentCount: 0, + status: "live", + external: { + source: "bsky", + did: profile.did, + handle: profile.handle, + postUri: post.uri, + postCid: post.cid || null, + postedAt, + fetchedAt, + }, + atproto: parsed ? { did: parsed.did, uri: post.uri, rkey: parsed.rkey } : undefined, + }; + await posts.insertOne(doc); + inserted++; + createdCodes.push(code); + log(`✅ ${code} ← ${post.uri}`); + } catch (error) { + if (error?.code === 11000) { skipped++; continue; } + errors.push({ uri: post.uri, message: error.message }); + log(`❌ ${post.uri}: ${error.message}`); + } + } + + return { actor, did: profile.did, handle: profile.handle, inserted, skipped, errors, createdCodes }; +} + +/** + * Pull from every configured source. Returns an array of result summaries. + */ +export async function ingestAll(db, options = {}) { + const sources = options.sources || getConfiguredSources(); + const results = []; + for (const actor of sources) { + try { + results.push(await ingestFromActor(db, actor, options)); + } catch (error) { + results.push({ actor, inserted: 0, skipped: 0, errors: [{ message: error.message }] }); + } + } + return results; +} + +export const __testing = { projectPost, stripUrls, parseAtUri, extractExternalEmbed, extractFirstFacetLink }; diff --git a/silo/deploy.fish b/silo/deploy.fish index 37e94db4fe..531a9df71f 100755 --- a/silo/deploy.fish +++ b/silo/deploy.fish @@ -60,6 +60,7 @@ else echo -e "$GREEN-> Uploading silo files...$NC" scp -i $SSH_KEY -o StrictHostKeyChecking=no \ $SCRIPT_DIR/server.mjs \ + $SCRIPT_DIR/bluesky-ingest.mjs \ $SCRIPT_DIR/dashboard.html \ $SCRIPT_DIR/package.json \ $SCRIPT_DIR/package-lock.json \ diff --git a/silo/server.mjs b/silo/server.mjs index 87a23affcb..60b6e3dbc9 100644 --- a/silo/server.mjs +++ b/silo/server.mjs @@ -18,6 +18,7 @@ import { IgLoginTwoFactorRequiredError, IgLoginBadPasswordError, } from "instagram-private-api"; +import { ingestAll as ingestBluesky, ingestFromActor as ingestBlueskyActor, getConfiguredSources as getBlueskySources } from "./bluesky-ingest.mjs"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -1766,6 +1767,70 @@ app.get("/", (req, res) => { res.send(dashboardHtml); }); +// --- Bluesky ingest (external news headlines from trusted sources) --- +const BSKY_INGEST_INTERVAL_MS = parseInt(process.env.NEWS_BLUESKY_INTERVAL_MS || "", 10) || 10 * 60 * 1000; // default 10 min +const BSKY_INGEST_LIMIT = parseInt(process.env.NEWS_BLUESKY_LIMIT || "", 10) || 30; +let bskyIngestTimer = null; +let bskyLastRun = { when: null, results: [], error: null, runCount: 0 }; + +async function runBlueskyIngestOnce(actor) { + if (!db) { + const err = "mongo not connected"; + bskyLastRun = { when: new Date().toISOString(), results: [], error: err, runCount: bskyLastRun.runCount }; + return bskyLastRun; + } + try { + const results = actor + ? [await ingestBlueskyActor(db, actor, { limit: BSKY_INGEST_LIMIT, log: (line) => log("info", `bsky: ${line}`) })] + : await ingestBluesky(db, { limit: BSKY_INGEST_LIMIT, log: (line) => log("info", `bsky: ${line}`) }); + const totalInserted = results.reduce((n, r) => n + (r.inserted || 0), 0); + const totalErrors = results.reduce((n, r) => n + (r.errors?.length || 0), 0); + bskyLastRun = { + when: new Date().toISOString(), + results, + error: null, + runCount: (bskyLastRun.runCount || 0) + 1, + }; + if (totalInserted > 0 || totalErrors > 0) { + log("info", `bsky ingest: +${totalInserted} new, ${totalErrors} errors`); + } + return bskyLastRun; + } catch (err) { + bskyLastRun = { + when: new Date().toISOString(), + results: [], + error: err.message, + runCount: (bskyLastRun.runCount || 0) + 1, + }; + log("error", `bsky ingest failed: ${err.message}`); + return bskyLastRun; + } +} + +function startBlueskyIngestLoop() { + if (bskyIngestTimer) return; + const sources = getBlueskySources(); + log("info", `bsky ingest: ${sources.length} source(s), every ${Math.round(BSKY_INGEST_INTERVAL_MS / 60000)}m — ${sources.join(", ")}`); + // Kick off a first run ~30s after boot to avoid colliding with other startup work. + setTimeout(() => runBlueskyIngestOnce().catch(() => {}), 30_000); + bskyIngestTimer = setInterval(() => runBlueskyIngestOnce().catch(() => {}), BSKY_INGEST_INTERVAL_MS); +} + +app.get("/api/news/bluesky/status", (req, res) => { + res.json({ + sources: getBlueskySources(), + intervalMs: BSKY_INGEST_INTERVAL_MS, + limit: BSKY_INGEST_LIMIT, + lastRun: bskyLastRun, + }); +}); + +app.post("/api/news/bluesky/pull", async (req, res) => { + const actor = typeof req.body?.actor === "string" ? req.body.actor.trim() : ""; + const result = await runBlueskyIngestOnce(actor || undefined); + res.json(result); +}); + // --- 404 --- app.use((req, res) => res.status(404).json({ error: "Not found" })); @@ -1810,6 +1875,9 @@ await connectRedisSub(); // Restore TikTok session on startup loadTiktokSession().catch(() => {}); +// Start the Bluesky news ingest loop (pulls headlines from trusted sources). +startBlueskyIngestLoop(); + server.listen(PORT, () => { const proto = dev ? "https" : "http"; log("info", `silo running on ${proto}://localhost:${PORT}`); @@ -1818,6 +1886,7 @@ server.listen(PORT, () => { // --- Shutdown --- function shutdown(signal) { log("info", `received ${signal}, shutting down...`); + if (bskyIngestTimer) { clearInterval(bskyIngestTimer); bskyIngestTimer = null; } if (changeStream) changeStream.close().catch(() => {}); wss.clients.forEach((ws) => ws.close()); server.close(); diff --git a/system/backend/news-bluesky-ingest.mjs b/system/backend/news-bluesky-ingest.mjs new file mode 100644 index 0000000000..c0eac56bcc --- /dev/null +++ b/system/backend/news-bluesky-ingest.mjs @@ -0,0 +1,264 @@ +// news-bluesky-ingest.mjs +// Pulls posts from trusted Bluesky accounts into news-posts as "external" entries. +// External posts are attributed to a Bluesky DID rather than an AC @handle. + +import { AtpAgent } from "@atproto/api"; +import { generateUniqueCode } from "./generate-short-code.mjs"; + +const BSKY_APPVIEW = "https://public.api.bsky.app"; +const DEFAULT_LIMIT = 30; +const MAX_TITLE = 200; +const MAX_TEXT = 5000; + +// Comma-separated list of trusted Bluesky sources (DIDs or handles). +// Example: NEWS_EXTERNAL_SOURCES="artistnewsnetwork.bsky.social" +function parseTrustedSources(raw) { + if (!raw) return []; + return String(raw) + .split(",") + .map((s) => s.trim()) + .filter(Boolean); +} + +export function getConfiguredSources(env = process.env) { + const configured = parseTrustedSources(env.NEWS_EXTERNAL_SOURCES); + if (configured.length) return configured; + return ["artistnewsnetwork.bsky.social"]; +} + +function truncate(value, max) { + if (!value) return ""; + const trimmed = String(value).trim(); + return trimmed.length > max ? trimmed.slice(0, max) : trimmed; +} + +// Strip URLs from the prose body — they're already surfaced as the `url` field. +function stripUrls(text) { + if (!text) return ""; + return text.replace(/https?:\/\/\S+/gi, "").replace(/\s+/g, " ").trim(); +} + +function extractExternalEmbed(post) { + const embed = post?.embed; + if (!embed) return null; + // Hydrated appview shape + if (embed.$type === "app.bsky.embed.external#view" && embed.external?.uri) { + return { + uri: embed.external.uri, + title: embed.external.title || "", + description: embed.external.description || "", + }; + } + // Raw record shape (fallback) + const recordEmbed = post?.record?.embed; + if (recordEmbed?.$type === "app.bsky.embed.external" && recordEmbed.external?.uri) { + return { + uri: recordEmbed.external.uri, + title: recordEmbed.external.title || "", + description: recordEmbed.external.description || "", + }; + } + return null; +} + +// First link facet in the post record, if any. +function extractFirstFacetLink(post) { + const facets = post?.record?.facets; + if (!Array.isArray(facets)) return null; + for (const facet of facets) { + const feature = facet?.features?.find( + (f) => f?.$type === "app.bsky.richtext.facet#link" && f.uri, + ); + if (feature) return feature.uri; + } + return null; +} + +// Produce { title, url, text } for insertion, or null to skip. +function projectPost(post) { + const record = post?.record; + if (!record) return null; + const rawText = record.text || ""; + const external = extractExternalEmbed(post); + + if (external) { + const title = truncate(external.title || stripUrls(rawText) || "Untitled link", MAX_TITLE); + const text = truncate(stripUrls(rawText), MAX_TEXT); + return { title, url: external.uri, text }; + } + + const facetLink = extractFirstFacetLink(post); + if (facetLink) { + const title = truncate(stripUrls(rawText) || "Untitled link", MAX_TITLE); + const text = truncate(stripUrls(rawText), MAX_TEXT); + return { title, url: facetLink, text }; + } + + // No link at all — skip. We only ingest headline-style posts. + return null; +} + +async function resolveProfile(agent, actor) { + const result = await agent.app.bsky.actor.getProfile({ actor }); + const data = result?.data || result; + if (!data?.did) throw new Error(`Could not resolve Bluesky actor: ${actor}`); + return { did: data.did, handle: data.handle || actor }; +} + +async function fetchAuthorFeed(agent, did, limit) { + const result = await agent.app.bsky.feed.getAuthorFeed({ + actor: did, + filter: "posts_no_replies", + limit, + }); + const data = result?.data || result; + return data?.feed || []; +} + +function parseAtUri(uri) { + // at://did:plc:xxx/app.bsky.feed.post/abc + const match = /^at:\/\/([^/]+)\/([^/]+)\/(.+)$/.exec(uri || ""); + if (!match) return null; + return { did: match[1], collection: match[2], rkey: match[3] }; +} + +/** + * Ingest recent posts from one Bluesky actor into news-posts. + * Returns { inserted, skipped, errors, actor }. + */ +export async function ingestFromActor(database, actor, options = {}) { + const { + limit = DEFAULT_LIMIT, + agent: providedAgent, + now = () => new Date(), + generateCode = generateUniqueCode, + log = () => {}, + } = options; + + const posts = database.db.collection("news-posts"); + + const agent = providedAgent || new AtpAgent({ service: BSKY_APPVIEW }); + const profile = await resolveProfile(agent, actor); + const feed = await fetchAuthorFeed(agent, profile.did, limit); + + let inserted = 0; + let skipped = 0; + const errors = []; + const createdCodes = []; + + for (const item of feed) { + const post = item?.post; + if (!post?.uri) { + skipped++; + continue; + } + + // Skip reposts — we only want the actor's original posts. + if (item?.reason?.$type === "app.bsky.feed.defs#reasonRepost") { + skipped++; + continue; + } + // Skip replies + if (post?.record?.reply) { + skipped++; + continue; + } + + const existing = await posts.findOne({ "external.postUri": post.uri }); + if (existing) { + skipped++; + continue; + } + + const projected = projectPost(post); + if (!projected || !projected.title) { + skipped++; + continue; + } + + const parsed = parseAtUri(post.uri); + const postedAt = post.record?.createdAt + ? new Date(post.record.createdAt) + : now(); + const fetchedAt = now(); + + try { + const shortCode = await generateCode(posts, { mode: "random" }); + const code = `n${shortCode}`; + const doc = { + code, + title: projected.title, + url: projected.url, + text: projected.text, + user: null, + when: postedAt, + updated: fetchedAt, + score: 1, + commentCount: 0, + status: "live", + external: { + source: "bsky", + did: profile.did, + handle: profile.handle, + postUri: post.uri, + postCid: post.cid || null, + postedAt, + fetchedAt, + }, + // Mirror the Bluesky record into the `atproto` field so the + // existing AT permalink affordance works for external posts too. + atproto: parsed + ? { did: parsed.did, uri: post.uri, rkey: parsed.rkey } + : undefined, + }; + + await posts.insertOne(doc); + inserted++; + createdCodes.push(code); + log(`✅ ${code} ← ${post.uri}`); + } catch (error) { + // Unique-index collision means a concurrent ingest already created it. + if (error?.code === 11000) { + skipped++; + continue; + } + errors.push({ uri: post.uri, message: error.message }); + log(`❌ ${post.uri}: ${error.message}`); + } + } + + return { + actor, + did: profile.did, + handle: profile.handle, + inserted, + skipped, + errors, + createdCodes, + }; +} + +/** + * Ingest from every configured trusted source. + * Returns an array of per-actor result summaries. + */ +export async function ingestAll(database, options = {}) { + const sources = options.sources || getConfiguredSources(); + const results = []; + for (const actor of sources) { + try { + const result = await ingestFromActor(database, actor, options); + results.push(result); + } catch (error) { + results.push({ + actor, + inserted: 0, + skipped: 0, + errors: [{ message: error.message }], + }); + } + } + return results; +} + +export const __testing = { projectPost, extractExternalEmbed, extractFirstFacetLink, stripUrls, parseAtUri }; diff --git a/system/netlify/functions/news-api.mjs b/system/netlify/functions/news-api.mjs index c13b18e996..dbce1338e9 100644 --- a/system/netlify/functions/news-api.mjs +++ b/system/netlify/functions/news-api.mjs @@ -7,6 +7,7 @@ import { authorize, hasAdmin } from "../../backend/authorization.mjs"; import { generateUniqueCode } from "../../backend/generate-short-code.mjs"; import { ObjectId } from "mongodb"; import { createNewsOnAtproto } from "../../backend/news-atproto.mjs"; +import { ingestAll, ingestFromActor } from "../../backend/news-bluesky-ingest.mjs"; // Admin users who can delete/censor content const ADMIN_SUBS = [process.env.ADMIN_SUB].filter(Boolean); @@ -68,6 +69,10 @@ async function ensureIndexes(posts, comments, votes) { await posts.createIndex({ score: -1 }, { background: true }); await posts.createIndex({ user: 1 }, { background: true }); await posts.createIndex({ status: 1 }, { background: true }); + await posts.createIndex( + { "external.postUri": 1 }, + { unique: true, sparse: true, background: true }, + ); await comments.createIndex({ postCode: 1 }, { background: true }); await comments.createIndex({ parentId: 1 }, { background: true, sparse: true }); @@ -463,6 +468,22 @@ export function createHandler({ } return respondFn(200, { ok: true, deleted: itemId, redirect: redirectTo }); } + + // Admin-only: pull fresh headlines from trusted Bluesky sources. + if (route === "ingest-external") { + const user = await requireUserWith(event); + const isAdmin = await hasAdmin(user, "aesthetic"); + if (!isAdmin) { + return respondFn(403, { error: "Admin only" }); + } + const body = parseBody(event); + const actor = sanitizeText(body.actor, 256); + const limit = Math.min(parseInt(body.limit || "30", 10) || 30, 100); + const results = actor + ? [await ingestFromActor(database, actor, { limit })] + : await ingestAll(database, { limit }); + return respondFn(200, { ok: true, results }); + } } return respondFn(404, { error: "Not found" }); diff --git a/system/netlify/functions/news.mjs b/system/netlify/functions/news.mjs index bf3505c280..45dd848746 100644 --- a/system/netlify/functions/news.mjs +++ b/system/netlify/functions/news.mjs @@ -399,11 +399,20 @@ function atprotoPermalink(atprotoData) { return { pdsLs: pdsLsUrl, uri: atprotoData.uri }; } -function renderHandle(handle) { +function renderHandle(handle, options = {}) { const safeHandle = escapeHtml(handle || "@anon"); // Extract username without @ for the URL const username = safeHandle.startsWith("@") ? safeHandle.slice(1) : safeHandle; if (username === "anon") return safeHandle; + // External sources (e.g. Bluesky) point at their own profile and render + // as a plain external link rather than opening in an AC modal. + if (options.external) { + const href = escapeHtml(options.profileUrl || "#"); + const badge = options.sourceLabel + ? ` ${escapeHtml(options.sourceLabel)}` + : ""; + return `${safeHandle}${badge}`; + } const profileUrl = `https://aesthetic.computer/@${username}`; return `${safeHandle}`; } @@ -552,7 +561,11 @@ function renderPostRow(post, idx, basePath) { ${displayUrl ? `(${displayUrl})` : ""}
- by ${renderHandle(post.handle)} + by ${renderHandle(post.handle, post.externalAttribution ? { + external: true, + profileUrl: post.externalAttribution.profileUrl, + sourceLabel: post.externalAttribution.sourceLabel, + } : undefined)} ${formatDate(post.when)} ${post.commentCount || 0} comments
@@ -601,16 +614,44 @@ async function fetchPosts(database, { sort = "new", limit = 30 }) { return applyCommentCounts(database, docs); } +// Short display form of a Bluesky handle: "artistnewsnetwork.bsky.social" → "artistnewsnetwork" +function shortBskyHandle(handle) { + if (!handle) return "anon"; + return handle.replace(/\.bsky\.social$/i, ""); +} + async function hydrateHandles(database, docs) { const handles = database.db.collection("@handles"); const subs = docs.map((doc) => doc.user).filter(Boolean); - if (subs.length === 0) return docs; - const handleDocs = await handles.find({ _id: { $in: subs } }).toArray(); + const handleDocs = subs.length + ? await handles.find({ _id: { $in: subs } }).toArray() + : []; const map = new Map(handleDocs.map((h) => [h._id, h.handle])); - return docs.map((doc) => ({ - ...doc, - handle: doc.user ? `@${map.get(doc.user) || "anon"}` : "@anon", - })); + return docs.map((doc) => { + // External source takes precedence — attribute to the Bluesky handle/DID + // rather than an AC @handle, and mark the doc so renderers link externally. + if (doc.external?.source === "bsky" && doc.external?.handle) { + const short = shortBskyHandle(doc.external.handle); + return { + ...doc, + handle: `@${short}`, + externalAttribution: { + source: "bsky", + sourceLabel: "bsky", + did: doc.external.did, + fullHandle: doc.external.handle, + profileUrl: `https://bsky.app/profile/${doc.external.handle}`, + postUrl: doc.external.postUri + ? `https://bsky.app/profile/${doc.external.did}/post/${doc.external.postUri.split("/").pop()}` + : null, + }, + }; + } + return { + ...doc, + handle: doc.user ? `@${map.get(doc.user) || "anon"}` : "@anon", + }; + }); } async function renderFrontPage(database, basePath, sort) { @@ -674,6 +715,15 @@ async function renderItemPage(database, basePath, code) { 🔗 AT ` : ''; + // For external posts without body text, still show attribution so readers + // can see it came from a trusted third-party source. + const externalAttribHtml = (hydratedPost.externalAttribution && !hydratedPost.text) ? ` + via ${renderHandle(hydratedPost.handle, { + external: true, + profileUrl: hydratedPost.externalAttribution.profileUrl, + sourceLabel: hydratedPost.externalAttribution.sourceLabel, + })} ${formatDate(hydratedPost.when)}${hydratedPost.externalAttribution.postUrl ? ` · original` : ''}` : ''; + // Check for YouTube embed const youtubeId = parseYouTubeUrl(hydratedPost.url); const youtubeEmbedHtml = youtubeId ? ` @@ -831,6 +881,7 @@ async function renderItemPage(database, basePath, code) {
+ ${externalAttribHtml} ${atLinkHtml}
@@ -840,7 +891,11 @@ async function renderItemPage(database, basePath, code) { ${hydratedPost.text ? `
-
${renderHandle(hydratedPost.handle)} ${formatDate(hydratedPost.when)}
+
${renderHandle(hydratedPost.handle, hydratedPost.externalAttribution ? { + external: true, + profileUrl: hydratedPost.externalAttribution.profileUrl, + sourceLabel: hydratedPost.externalAttribution.sourceLabel, + } : undefined)} ${formatDate(hydratedPost.when)}
${renderMarkdown(hydratedPost.text)}
` : ""}
diff --git a/system/public/news.aesthetic.computer/main.css b/system/public/news.aesthetic.computer/main.css index 7c433b1a67..cbabd5bb1f 100644 --- a/system/public/news.aesthetic.computer/main.css +++ b/system/public/news.aesthetic.computer/main.css @@ -455,6 +455,27 @@ body { text-decoration: underline; } +/* External-source attribution (e.g. Bluesky) */ +.news-external-badge { + display: inline-block; + margin-left: 4px; + padding: 0 4px; + font-size: 0.7em; + font-weight: normal; + text-transform: uppercase; + letter-spacing: 0.04em; + border: 1px solid currentColor; + border-radius: 3px; + opacity: 0.65; + vertical-align: 1px; +} + +.news-external-attrib { + opacity: 0.75; + font-size: 0.9em; + margin-right: 8px; +} + /* ===== Item Page (HN-style table layout) ===== */ .news-item-table { margin-bottom: 10px; diff --git a/tests/news-bluesky.test.mjs b/tests/news-bluesky.test.mjs new file mode 100644 index 0000000000..3e33dcd91c --- /dev/null +++ b/tests/news-bluesky.test.mjs @@ -0,0 +1,433 @@ +import assert from "node:assert/strict"; +import { + ingestFromActor, + getConfiguredSources, + __testing, +} from "../system/backend/news-bluesky-ingest.mjs"; + +const { projectPost, stripUrls, parseAtUri } = __testing; + +// --- Fixtures --------------------------------------------------------------- + +const DID = "did:plc:testartist"; +const HANDLE = "artistnewsnetwork.bsky.social"; + +function makePostWithExternal(overrides = {}) { + return { + post: { + uri: `at://${DID}/app.bsky.feed.post/rkey${overrides.rkey || "a"}`, + cid: `bafyre${overrides.cid || "a"}`, + author: { did: DID, handle: HANDLE }, + record: { + $type: "app.bsky.feed.post", + text: overrides.text ?? "A great exhibition opening this weekend https://gallery.example/show", + createdAt: overrides.createdAt || "2026-04-10T12:00:00.000Z", + embed: { + $type: "app.bsky.embed.external", + external: { + uri: "https://gallery.example/show", + title: "Gallery Show: Weekend Opening", + description: "A new exhibition.", + }, + }, + }, + embed: { + $type: "app.bsky.embed.external#view", + external: { + uri: "https://gallery.example/show", + title: "Gallery Show: Weekend Opening", + description: "A new exhibition.", + }, + }, + }, + }; +} + +function makePostWithFacetLink() { + return { + post: { + uri: `at://${DID}/app.bsky.feed.post/facetpost`, + cid: "bafyrefacet", + author: { did: DID, handle: HANDLE }, + record: { + text: "Check this out https://museum.example/piece", + createdAt: "2026-04-10T13:00:00.000Z", + facets: [ + { + features: [ + { $type: "app.bsky.richtext.facet#link", uri: "https://museum.example/piece" }, + ], + }, + ], + }, + }, + }; +} + +function makePlainTextPost() { + return { + post: { + uri: `at://${DID}/app.bsky.feed.post/plain`, + cid: "bafyreplain", + author: { did: DID, handle: HANDLE }, + record: { + text: "Just thinking out loud today.", + createdAt: "2026-04-10T14:00:00.000Z", + }, + }, + }; +} + +function makeReply() { + return { + post: { + uri: `at://${DID}/app.bsky.feed.post/reply`, + cid: "bafyrereply", + author: { did: DID, handle: HANDLE }, + record: { + text: "Good point https://other.example/thing", + createdAt: "2026-04-10T15:00:00.000Z", + reply: { root: { uri: "at://x/y/z" }, parent: { uri: "at://x/y/z" } }, + embed: { + $type: "app.bsky.embed.external", + external: { uri: "https://other.example/thing", title: "Thing" }, + }, + }, + embed: { + $type: "app.bsky.embed.external#view", + external: { uri: "https://other.example/thing", title: "Thing" }, + }, + }, + }; +} + +function makeRepost() { + return { + reason: { $type: "app.bsky.feed.defs#reasonRepost" }, + post: { + uri: `at://${DID}/app.bsky.feed.post/reposted`, + cid: "bafyrerepost", + author: { did: "did:plc:someone-else", handle: "someone.bsky.social" }, + record: { + text: "Repost body https://x.example/reposted", + createdAt: "2026-04-10T16:00:00.000Z", + embed: { + $type: "app.bsky.embed.external", + external: { uri: "https://x.example/reposted", title: "Reposted" }, + }, + }, + }, + }; +} + +// --- Pure helpers ----------------------------------------------------------- + +function testStripUrls() { + assert.equal(stripUrls("before https://a.com after"), "before after"); + assert.equal(stripUrls("only text"), "only text"); + assert.equal(stripUrls(""), ""); +} + +function testParseAtUri() { + const parsed = parseAtUri("at://did:plc:abc/app.bsky.feed.post/rkey123"); + assert.deepEqual(parsed, { + did: "did:plc:abc", + collection: "app.bsky.feed.post", + rkey: "rkey123", + }); + assert.equal(parseAtUri("not-an-at-uri"), null); +} + +function testProjectPostExternalEmbed() { + const { post } = makePostWithExternal(); + const projected = projectPost(post); + assert.equal(projected.title, "Gallery Show: Weekend Opening"); + assert.equal(projected.url, "https://gallery.example/show"); + assert.equal(projected.text, "A great exhibition opening this weekend"); +} + +function testProjectPostFacetLinkFallback() { + const { post } = makePostWithFacetLink(); + const projected = projectPost(post); + assert.ok(projected, "should project a post with a link facet"); + assert.equal(projected.url, "https://museum.example/piece"); + assert.equal(projected.title, "Check this out"); +} + +function testProjectPostPlainTextSkipped() { + const { post } = makePlainTextPost(); + assert.equal(projectPost(post), null, "plain text posts with no link are skipped"); +} + +function testConfiguredSourcesDefault() { + const sources = getConfiguredSources({}); + assert.ok(sources.includes("artistnewsnetwork.bsky.social")); +} + +function testConfiguredSourcesOverride() { + const sources = getConfiguredSources({ + NEWS_EXTERNAL_SOURCES: "a.bsky.social, did:plc:xyz ,", + }); + assert.deepEqual(sources, ["a.bsky.social", "did:plc:xyz"]); +} + +// --- Integration: ingestFromActor ------------------------------------------ + +function makePostsCollection() { + const docs = []; + return { + docs, + async findOne(query) { + if (query?.["external.postUri"]) { + return docs.find((d) => d.external?.postUri === query["external.postUri"]) || null; + } + return docs.find((d) => { + return Object.entries(query).every(([k, v]) => d[k] === v); + }) || null; + }, + async insertOne(doc) { + const existing = docs.find((d) => d.external?.postUri === doc.external?.postUri); + if (existing) { + const err = new Error("dup"); + err.code = 11000; + throw err; + } + docs.push(doc); + return { insertedId: doc.code }; + }, + }; +} + +function makeFakeAgent(feed) { + return { + app: { + bsky: { + actor: { + async getProfile({ actor }) { + return { data: { did: DID, handle: HANDLE } }; + }, + }, + feed: { + async getAuthorFeed() { + return { data: { feed } }; + }, + }, + }, + }, + }; +} + +async function testIngestInsertsOnlyExternalAndFacetPosts() { + const posts = makePostsCollection(); + const database = { + db: { collection: (name) => (name === "news-posts" ? posts : null) }, + }; + const feed = [ + makePostWithExternal({ rkey: "a" }), + makePostWithFacetLink(), + makePlainTextPost(), + makeReply(), + makeRepost(), + ]; + let codeCounter = 0; + const result = await ingestFromActor(database, HANDLE, { + agent: makeFakeAgent(feed), + generateCode: async () => `gen${++codeCounter}`, + }); + assert.equal(result.inserted, 2, "should insert external + facet posts only"); + assert.equal(result.skipped, 3, "plain-text, reply, repost are skipped"); + assert.equal(posts.docs.length, 2); + const first = posts.docs[0]; + assert.equal(first.external.source, "bsky"); + assert.equal(first.external.did, DID); + assert.equal(first.external.handle, HANDLE); + assert.ok(first.external.postUri.startsWith("at://")); + assert.equal(first.user, null, "external posts have no AC user sub"); + assert.ok(first.atproto?.uri, "external posts mirror atproto fields"); +} + +async function testIngestDeduplicatesOnSecondRun() { + const posts = makePostsCollection(); + const database = { + db: { collection: (name) => (name === "news-posts" ? posts : null) }, + }; + const feed = [makePostWithExternal({ rkey: "dup" })]; + const agent = makeFakeAgent(feed); + let codeCounter = 0; + const genCode = async () => `dup${++codeCounter}`; + + const first = await ingestFromActor(database, HANDLE, { agent, generateCode: genCode }); + assert.equal(first.inserted, 1); + + const second = await ingestFromActor(database, HANDLE, { agent, generateCode: genCode }); + assert.equal(second.inserted, 0, "rerun should not insert duplicates"); + assert.equal(second.skipped, 1, "rerun should skip the already-ingested post"); + assert.equal(posts.docs.length, 1); +} + +// --- SSR rendering for external posts -------------------------------------- + +async function testExternalAttributionRendersOnFrontPage() { + const { createHandler } = await import("../system/netlify/functions/news.mjs"); + + const externalPost = { + code: "nextrnl", + title: "External Headline", + url: "https://gallery.example/show", + user: null, + when: new Date("2026-04-10T12:00:00Z"), + score: 1, + commentCount: 0, + status: "live", + external: { + source: "bsky", + did: DID, + handle: HANDLE, + postUri: `at://${DID}/app.bsky.feed.post/rkeyA`, + postedAt: new Date("2026-04-10T12:00:00Z"), + fetchedAt: new Date("2026-04-10T12:05:00Z"), + }, + }; + + const collections = new Map([ + ["news-posts", mockCollection([externalPost])], + ["news-comments", mockCollection([])], + ["@handles", mockCollection([])], + ]); + const database = { + db: { collection: (name) => collections.get(name) || mockCollection([]) }, + disconnect: async () => null, + }; + const handler = createHandler({ + connect: async () => database, + respond: (statusCode, body, headers = {}) => ({ statusCode, headers, body }), + }); + const res = await handler({ + httpMethod: "GET", + headers: { host: "localhost:8888" }, + queryStringParameters: { path: "" }, + }); + assert.equal(res.statusCode, 200); + assert.ok(res.body.includes("External Headline"), "title should render"); + assert.ok( + res.body.includes("bsky.app/profile/artistnewsnetwork.bsky.social"), + "handle should link to bluesky profile", + ); + assert.ok(res.body.includes("@artistnewsnetwork"), "short handle should render"); + assert.ok(res.body.includes("news-external-handle"), "external handle class should render"); +} + +async function testExternalAttributionRendersOnItemPage() { + const { createHandler } = await import("../system/netlify/functions/news.mjs"); + + const externalPost = { + code: "nexitem", + title: "External Item", + url: "https://gallery.example/item", + user: null, + when: new Date("2026-04-10T12:00:00Z"), + score: 1, + commentCount: 0, + status: "live", + external: { + source: "bsky", + did: DID, + handle: HANDLE, + postUri: `at://${DID}/app.bsky.feed.post/itemrkey`, + postedAt: new Date("2026-04-10T12:00:00Z"), + fetchedAt: new Date("2026-04-10T12:05:00Z"), + }, + }; + const collections = new Map([ + ["news-posts", mockCollection([externalPost])], + ["news-comments", mockCollection([])], + ["@handles", mockCollection([])], + ]); + const database = { + db: { collection: (name) => collections.get(name) || mockCollection([]) }, + disconnect: async () => null, + }; + const handler = createHandler({ + connect: async () => database, + respond: (statusCode, body, headers = {}) => ({ statusCode, headers, body }), + }); + const res = await handler({ + httpMethod: "GET", + headers: { host: "localhost:8888" }, + queryStringParameters: { path: "nexitem" }, + }); + assert.equal(res.statusCode, 200); + assert.ok(res.body.includes("External Item")); + assert.ok( + res.body.includes("bsky.app/profile/did:plc:testartist/post/itemrkey"), + "should link to original Bluesky post", + ); + assert.ok(res.body.includes("news-external-attrib"), "item page should tag external attribution"); +} + +// Tiny mock collection factory (same shape news.mjs expects). +function mockCollection(initial) { + const docs = [...initial]; + function matches(doc, query) { + if (!query) return true; + return Object.entries(query).every(([k, v]) => { + if (v && typeof v === "object" && !Array.isArray(v)) { + if (Object.hasOwn(v, "$ne")) return doc[k] !== v.$ne; + if (Object.hasOwn(v, "$in")) return v.$in.includes(doc[k]); + } + return doc[k] === v; + }); + } + return { + docs, + find(query) { + let sortSpec = null; + let limitValue = null; + const api = { + sort(spec) { sortSpec = spec; return api; }, + limit(v) { limitValue = v; return api; }, + async toArray() { + let out = docs.filter((d) => matches(d, query)); + if (sortSpec) { + const entries = Object.entries(sortSpec); + out = out.sort((a, b) => { + for (const [f, dir] of entries) { + if (a[f] === b[f]) continue; + return dir < 0 ? (b[f] ?? 0) - (a[f] ?? 0) : (a[f] ?? 0) - (b[f] ?? 0); + } + return 0; + }); + } + if (limitValue !== null) out = out.slice(0, limitValue); + return out; + }, + }; + return api; + }, + async findOne(query) { + return docs.find((d) => matches(d, query)) || null; + }, + aggregate() { + return { async toArray() { return []; } }; + }, + }; +} + +// --- Runner ---------------------------------------------------------------- + +async function run() { + testStripUrls(); + testParseAtUri(); + testProjectPostExternalEmbed(); + testProjectPostFacetLinkFallback(); + testProjectPostPlainTextSkipped(); + testConfiguredSourcesDefault(); + testConfiguredSourcesOverride(); + await testIngestInsertsOnlyExternalAndFacetPosts(); + await testIngestDeduplicatesOnSecondRun(); + await testExternalAttributionRendersOnFrontPage(); + await testExternalAttributionRendersOnItemPage(); + console.log("✅ news-bluesky tests passed"); +} + +run();