diff --git a/.env b/.env deleted file mode 100644 index 48782be..0000000 --- a/.env +++ /dev/null @@ -1,20 +0,0 @@ -# main indexers -JETSTREAM_URL="wss://jetstream1.us-east.bsky.network" -SPACEDUST_URL="wss://spacedust.whey.party" - -# for backfill (useless if you just started the instance right now) -CONSTELLATION_URL="https://constellation.microcosm.blue" -# i dont actually know why i need this -SLINGSHOT_URL="https://slingshot.whey.party" - -# bools -INDEX_SERVER_ENABLED=true -INDEX_SERVER_INVITES_REQUIRED=true - -VIEW_SERVER_ENABLED=true -INDEX_SERVER_INVITES_REQUIRED=true - -# this is for both index and view server btw -SERVICE_DID="did:web:local3768forumtest.whey.party" -SERVICE_ENDPOINT="https://local3768forumtest.whey.party" -SERVER_PORT="3768" \ No newline at end of file diff --git a/.gitignore b/.gitignore index aeea9f8..5af638b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ system.db docs/.vite/ .gitignore indexserver.ts -dbs/ \ No newline at end of file +dbs/ +config.jsonc +config.json \ No newline at end of file diff --git a/config.jsonc.example b/config.jsonc.example new file mode 100644 index 0000000..3c83fce --- /dev/null +++ b/config.jsonc.example @@ -0,0 +1,35 @@ +{ + // Main indexers + "jetstream": "wss://jetstream1.us-east.bsky.network", // you can self host it -> https://github.com/bluesky-social/jetstream + "spacedust": "wss://spacedust.your.site", // you can self host it -> https://www.microcosm.blue + + // For backfill (optional) + "constellation": "https://constellation.microcosm.blue", // (not useful on a new setup — requires pre-existing data to backfill) + + // Utility services + "slingshot": "https://slingshot.your.site", // you can self host it -> https://www.microcosm.blue + + // Index Server config + "indexServer": { + "inviteOnly": true, + "port": 3767, + "did": "did:web:skyliteindexserver.your.site", // should be the same domain as the endpoint + "host": "https://skyliteindexserver.your.site" + }, + + // View Server config + "viewServer": { + "inviteOnly": true, + "port": 3768, + "did": "did:web:skyliteviewserver.your.site", // should be the same domain as the endpoint + "host": "https://skyliteviewserver.your.site", + + // In order of which skylite index servers or bsky appviews to use first + "indexPriority": [ + "user#skylite_index", // user resolved skylite index server + "did:web:backupindexserver.your.site#skylite_index", // a specific skylite index server + "user#bsky_appview", // user resolved bsky appview + "did:web:api.bsky.app#bsky_appview" // a specific bsky appview + ] + } +} \ No newline at end of file diff --git a/config.ts b/config.ts new file mode 100644 index 0000000..6daeb91 --- /dev/null +++ b/config.ts @@ -0,0 +1,45 @@ +import { parse } from "jsr:@std/jsonc"; +import * as z from "npm:zod"; + +// configure these from the config.jsonc file (you can use config.jsonc.example as reference) +const indexTarget = z.string().refine( + (val) => { + const parts = val.split("#"); + if (parts.length !== 2) return false; + + const [prefix, suffix] = parts; + const validPrefix = prefix === "user" || prefix.startsWith("did:web:"); + const validSuffix = suffix === "skylite_index" || suffix === "bsky_appview"; + + return validPrefix && validSuffix; + }, + { + message: + "Each indexPriority entry must be in the form 'user#skylite_index', 'user#bsky_appview', 'did:web:...#skylite_index', or 'did:web:...#bsky_appview'", + } +); + +const ConfigSchema = z.object({ + jetstream: z.string(), + spacedust: z.string(), + constellation: z.string(), + slingshot: z.string(), + indexServer: z.object({ + inviteOnly: z.boolean(), + port: z.number(), + did: z.string(), + host: z.string(), + }), + viewServer: z.object({ + inviteOnly: z.boolean(), + port: z.number(), + did: z.string(), + host: z.string(), + indexPriority: z.array(indexTarget), + }), +}); + +const raw = await Deno.readTextFile("config.jsonc"); +const config = ConfigSchema.parse(parse(raw)); + +export { config }; diff --git a/deno.json b/deno.json index 9e1cd94..d3308e6 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,7 @@ { "tasks": { - "dev": "deno run --watch -A --env-file main.ts" + "index": "deno run --watch -A --env-file main-index.ts", + "view": "deno run --watch -A --env-file main-view.ts" }, "imports": { "@std/assert": "jsr:@std/assert@1" diff --git a/index/jetstream.ts b/index/jetstream.ts index f725862..a9b565f 100644 --- a/index/jetstream.ts +++ b/index/jetstream.ts @@ -1,5 +1,5 @@ import { Database } from "jsr:@db/sqlite@0.11"; -import { handleIndex } from "../main.ts"; +import { config } from "../config.ts"; import { resolveRecordFromURI } from "../utils/records.ts"; import { JetstreamManager } from "../utils/sharders.ts"; @@ -29,7 +29,7 @@ export function startJetstream(jetstreamManager: JetstreamManager) { }); } -export async function handleJetstream(msg: any) { +export async function handleJetstream(msg: any, handleIndex: Function) { console.log("Received Jetstream message: ", msg); const op = msg.commit.operation; diff --git a/index/onboardingBackfill.ts b/index/onboardingBackfill.ts index cdae011..6bf088d 100644 --- a/index/onboardingBackfill.ts +++ b/index/onboardingBackfill.ts @@ -1,5 +1,5 @@ -import { indexServerIndexer } from "../indexserver.ts"; -import { systemDB } from "../main.ts" +import { genericIndexServer } from "../main-index.ts"; +import { config } from "../config.ts" import { FINEPDSAndHandleFromDid } from "../utils/identity.ts"; @@ -68,15 +68,17 @@ function handleOnboardingBackfill( const doer = did; const rev = undefined; const aturi = uri; + const db = genericIndexServer.userManager.getDbForDid(doer); + if (!db) return; - indexServerIndexer({ + genericIndexServer.indexServerIndexer({ op, doer, rev, aturi, value, indexsrc: "onboarding_backfill", - userdbname: did, + db: db, }) return; // console.log(`[BACKFILL] ${collection} -> ${uri}`); diff --git a/index/spacedust.ts b/index/spacedust.ts index 0f47969..d1c5dae 100644 --- a/index/spacedust.ts +++ b/index/spacedust.ts @@ -1,5 +1,5 @@ import { Database } from "jsr:@db/sqlite@0.11"; -import { handleIndex } from "../main.ts"; +import { config } from "../config.ts"; import { parseAtUri } from "../utils/aturi.ts"; import { resolveRecordFromURI } from "../utils/records.ts"; import { SpacedustManager } from "../utils/sharders.ts"; diff --git a/indexserver.ts b/indexserver.ts index 0f81d42..a291df2 100644 --- a/indexserver.ts +++ b/indexserver.ts @@ -5,13 +5,14 @@ import { searchParamsToJson, withCors } from "./utils/server.ts"; import * as IndexServerTypes from "./utils/indexservertypes.ts"; import { Database } from "jsr:@db/sqlite@0.11"; import { setupUserDb } from "./utils/dbuser.ts"; -// import { systemDB } from "./main.ts"; +// import { systemDB } from "./env.ts"; import { JetstreamManager, SpacedustManager } from "./utils/sharders.ts"; import { handleSpacedust, SpacedustLinkMessage } from "./index/spacedust.ts"; import { handleJetstream } from "./index/jetstream.ts"; import * as ATPAPI from "npm:@atproto/api"; import { AtUri } from "npm:@atproto/api"; import * as IndexServerAPI from "./indexclient/index.ts"; +import * as IndexServerUtils from "./indexclient/util.ts" export interface IndexServerConfig { baseDbPath: string; @@ -79,6 +80,10 @@ export class IndexServer { return new Response("Not Found", { status: 404 }); } + public handlesDid(did: string): boolean { + return this.userManager.handlesDid(did); + } + // We will move all the global functions into this class as methods... indexServerHandler(req: Request): Response { const url = new URL(req.url); @@ -268,7 +273,7 @@ export class IndexServer { // TODO: not partial yet, currently skips refs - const qresult = this.queryActorLikes(jsonTyped.actor, jsonTyped.cursor); + const qresult = this.queryActorLikesPartial(jsonTyped.actor, jsonTyped.cursor); if (!qresult) { return new Response( JSON.stringify({ @@ -1027,6 +1032,16 @@ export class IndexServer { return post; } + + constructPostViewRef(uri: string): IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef { + const post: IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef = { + uri: uri, + cid: "cid.invalid", // oh shit we dont know the cid TODO: major design flaw + }; + + return post; + } + queryFeedViewPost( uri: string ): ATPAPI.AppBskyFeedDefs.FeedViewPost | undefined { @@ -1043,6 +1058,19 @@ export class IndexServer { return feedviewpost; } + constructFeedViewPostRef( + uri: string + ): IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef { + const post = this.constructPostViewRef(uri); + + const feedviewpostref: IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef = { + $type: "party.whey.app.bsky.feed.defs#feedViewPostRef", + post: post as IndexServerUtils.$Typed, + } + + return feedviewpostref + } + // user feedgens queryActorFeeds(did: string): ATPAPI.AppBskyFeedDefs.GeneratorView[] { @@ -1213,15 +1241,16 @@ export class IndexServer { return { items: [], cursor: undefined }; } - queryActorLikes( + queryActorLikesPartial( did: string, cursor?: string ): | { - items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; + items: (ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef)[]; cursor: string | undefined; } | undefined { + // early return only if the actor did is not registered if (!this.isRegisteredIndexUser(did)) return; const db = this.userManager.getDbForDid(did); if (!db) return; @@ -1249,8 +1278,16 @@ export class IndexServer { }[]; const items = rows - .map((row) => this.queryFeedViewPost(row.subject)) - .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost => !!p); + .map((row) => { + const subjectDid = new AtUri(row.subject).host; + + if (this.handlesDid(subjectDid)) { + return this.queryFeedViewPost(row.subject); + } else { + return this.constructFeedViewPostRef(row.subject); + } + }) + .filter((p): p is ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef => !!p); const lastItem = rows[rows.length - 1]; const nextCursor = lastItem @@ -1418,7 +1455,7 @@ export class IndexServer { return { thread: returned }; } - + /** * please do not use this, use openDbForDid() instead * @param did @@ -1431,7 +1468,11 @@ export class IndexServer { //await db.exec(/* CREATE IF NOT EXISTS statements */); return db; } - + /** + * @deprecated use handlesDid() instead + * @param did + * @returns + */ isRegisteredIndexUser(did: string): boolean { const stmt = this.systemDB.prepare(` SELECT 1 @@ -1453,7 +1494,10 @@ export class IndexServerUserManager { this.indexServer = indexServer; } - private users = new Map(); + public users = new Map(); + public handlesDid(did: string): boolean { + return this.users.has(did); + } /*async*/ addUser(did: string) { if (this.users.has(did)) return; @@ -1508,7 +1552,9 @@ class UserIndexServer { constructor(indexServerUserManager: IndexServerUserManager, did: string) { this.did = did; this.indexServerUserManager = indexServerUserManager; - this.db = this.indexServerUserManager.indexServer.internalCreateDbForDid(this.did); + this.db = this.indexServerUserManager.indexServer.internalCreateDbForDid( + this.did + ); // should probably put the params of exactly what were listening to here this.jetstream = new JetstreamManager((msg) => { console.log("Received Jetstream message: ", msg); diff --git a/main-index.ts b/main-index.ts new file mode 100644 index 0000000..25b2365 --- /dev/null +++ b/main-index.ts @@ -0,0 +1,125 @@ +import { setupAuth, getAuthenticatedDid, authVerifier } from "./utils/auth.ts"; +import { setupSystemDb } from "./utils/dbsystem.ts"; +import { didDocument } from "./utils/diddoc.ts"; +import { cachedFetch, searchParamsToJson, withCors } from "./utils/server.ts"; +import { IndexServer, IndexServerConfig } from "./indexserver.ts" +import { extractDid } from "./utils/identity.ts"; +import { config } from "./config.ts"; + +// ------------------------------------------ +// AppView Setup +// ------------------------------------------ + +const indexServerConfig: IndexServerConfig = { + baseDbPath: './dbs/registered-users', // The directory for user databases + systemDbPath: './dbs/registered-users/system.db', // The path for the main system database + jetstreamUrl: config.jetstream +}; +export const genericIndexServer = new IndexServer(indexServerConfig); +setupSystemDb(genericIndexServer.systemDB); + +// add me lol +genericIndexServer.systemDB.exec(` + INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus) + VALUES ( + 'did:plc:mn45tewwnse5btfftvd3powc', + 'admin', + datetime('now'), + 'ready' + ); + + INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus) + VALUES ( + 'did:web:did12.whey.party', + 'admin', + datetime('now'), + 'ready' + ); +`) + +genericIndexServer.start(); + +// ------------------------------------------ +// XRPC Method Implementations +// ------------------------------------------ + +// const indexServerRoutes = new Set([ +// "/xrpc/app.bsky.actor.getProfile", +// "/xrpc/app.bsky.actor.getProfiles", +// "/xrpc/app.bsky.feed.getActorFeeds", +// "/xrpc/app.bsky.feed.getFeedGenerator", +// "/xrpc/app.bsky.feed.getFeedGenerators", +// "/xrpc/app.bsky.feed.getPosts", +// "/xrpc/party.whey.app.bsky.feed.getActorLikesPartial", +// "/xrpc/party.whey.app.bsky.feed.getAuthorFeedPartial", +// "/xrpc/party.whey.app.bsky.feed.getLikesPartial", +// "/xrpc/party.whey.app.bsky.feed.getPostThreadPartial", +// "/xrpc/party.whey.app.bsky.feed.getQuotesPartial", +// "/xrpc/party.whey.app.bsky.feed.getRepostedByPartial", +// // more federated endpoints, not planned yet, lexicons will come later +// /* +// app.bsky.graph.getLists // doesnt need to because theres no items[], and its self ProfileViewBasic +// app.bsky.graph.getList // needs to be Partial-ed (items[] union with ProfileViewRef) +// app.bsky.graph.getActorStarterPacks // maybe doesnt need to be Partial-ed because its self ProfileViewBasic + +// app.bsky.feed.getListFeed // uhh actually already exists its getListFeedPartial +// */ +// "/xrpc/party.whey.app.bsky.feed.getListFeedPartial", +// ]); + +Deno.serve( + { port: config.indexServer.port }, + (req: Request): Response => { + const url = new URL(req.url); + const pathname = url.pathname; + const searchParams = searchParamsToJson(url.searchParams); + + if (pathname === "/.well-known/did.json") { + return new Response(JSON.stringify(didDocument("index",config.indexServer.did,config.indexServer.host,"whatever")), { + headers: withCors({ "Content-Type": "application/json" }), + }); + } + if (pathname === "/health") { + return new Response("OK", { + status: 200, + headers: withCors({ + "Content-Type": "text/plain", + }), + }); + } + if (req.method === "OPTIONS") { + return new Response(null, { + status: 204, + headers: { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "*", + }, + }); + } + console.log(`request for "${pathname}"`) + const constellation = pathname.startsWith("/links") + + if (constellation) { + const target = searchParams?.target as string + const safeDid = extractDid(target); + const targetserver = genericIndexServer.handlesDid(safeDid) + if (targetserver) { + return genericIndexServer.constellationAPIHandler(req); + } else { + return new Response( + JSON.stringify({ + error: "User not found", + }), + { + status: 404, + headers: withCors({ "Content-Type": "application/json" }), + } + ); + } + } else { + // indexServerRoutes.has(pathname) + return genericIndexServer.indexServerHandler(req); + } + } +); \ No newline at end of file diff --git a/main-view.ts b/main-view.ts new file mode 100644 index 0000000..3983a1b --- /dev/null +++ b/main-view.ts @@ -0,0 +1,59 @@ +import { setupAuth, getAuthenticatedDid, authVerifier } from "./utils/auth.ts"; +import { setupSystemDb } from "./utils/dbsystem.ts"; +import { didDocument } from "./utils/diddoc.ts"; +import { cachedFetch, searchParamsToJson, withCors } from "./utils/server.ts"; +import { IndexServer, IndexServerConfig } from "./indexserver.ts" +import { extractDid } from "./utils/identity.ts"; +import { config } from "./config.ts"; +import { viewServerHandler } from "./viewserver.ts"; + +// ------------------------------------------ +// AppView Setup +// ------------------------------------------ + +setupAuth({ + serviceDid: config.viewServer.did, + //keyCacheSize: 500, + //keyCacheTTL: 10 * 60 * 1000, +}); + +// ------------------------------------------ +// XRPC Method Implementations +// ------------------------------------------ + + +Deno.serve( + { port: config.viewServer.port }, + async (req: Request): Promise => { + const url = new URL(req.url); + const pathname = url.pathname; + const searchParams = searchParamsToJson(url.searchParams); + + if (pathname === "/.well-known/did.json") { + return new Response(JSON.stringify(didDocument), { + headers: withCors({ "Content-Type": "application/json" }), + }); + } + if (pathname === "/health") { + return new Response("OK", { + status: 200, + headers: withCors({ + "Content-Type": "text/plain", + }), + }); + } + if (req.method === "OPTIONS") { + return new Response(null, { + status: 204, + headers: { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "*", + }, + }); + } + console.log(`request for "${pathname}"`) + + return await viewServerHandler(req) + } +); \ No newline at end of file diff --git a/main.ts b/main.ts deleted file mode 100644 index c1fdfd0..0000000 --- a/main.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { setupAuth, getAuthenticatedDid, authVerifier } from "./utils/auth.ts"; -import { JetstreamManager, SpacedustManager } from "./utils/sharders.ts"; -import { resolveRecordFromURI, validateRecord } from "./utils/records.ts"; -import { setupSystemDb } from "./utils/dbsystem.ts"; -import { setupUserDb } from "./utils/dbuser.ts"; -import { handleSpacedust, startSpacedust } from "./index/spacedust.ts"; -import { handleJetstream, startJetstream } from "./index/jetstream.ts"; -import { Database } from "jsr:@db/sqlite@0.11"; -//import express from "npm:express"; -//import { createServer } from "./xrpc/index.ts"; -import { indexHandlerContext } from "./index/types.ts"; -import * as IndexServerTypes from "./utils/indexservertypes.ts"; -import * as ViewServerTypes from "./utils/viewservertypes.ts"; -import * as ATPAPI from "npm:@atproto/api"; -import { didDocument } from "./utils/diddoc.ts"; -import { cachedFetch, searchParamsToJson, withCors } from "./utils/server.ts"; -import { IndexServer, IndexServerConfig } from "./indexserver.ts" -import { viewServerHandler } from "./viewserver.ts"; - -export const jetstreamurl = Deno.env.get("JETSTREAM_URL"); -export const slingshoturl = Deno.env.get("SLINGSHOT_URL"); -export const constellationurl = Deno.env.get("CONSTELLATION_URL"); -export const spacedusturl = Deno.env.get("SPACEDUST_URL"); - -// ------------------------------------------ -// AppView Setup -// ------------------------------------------ - -const config: IndexServerConfig = { - baseDbPath: './dbs', // The directory for user databases - systemDbPath: './system.db', // The path for the main system database - jetstreamUrl: jetstreamurl || "" -}; -const registeredUsersIndexServer = new IndexServer(config); -setupSystemDb(registeredUsersIndexServer.systemDB); - -// add me lol -registeredUsersIndexServer.systemDB.exec(` - INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus) - VALUES ( - 'did:plc:mn45tewwnse5btfftvd3powc', - 'admin', - datetime('now'), - 'ready' - ); - - INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus) - VALUES ( - 'did:web:did12.whey.party', - 'admin', - datetime('now'), - 'ready' - ); -`) - -registeredUsersIndexServer.start(); - -// should do both of these per user actually, since now each user has their own db -// also the set of records and backlinks to listen should be seperate between index and view servers -// damn -// export const spacedustManager = new SpacedustManager((msg) => -// handleSpacedust(msg) -// ); -// export const jetstreamManager = new JetstreamManager((msg) => -// handleJetstream(msg) -// ); -// startSpacedust(); -// startJetstream(); - -// 1. connect to system db -// 2. get all registered users -// parse config (maybe some are only indexes and maybe some are only views) -// map all new jetstream and spacedust listeners -// call handleIndex with the specific db to use - -setupAuth({ - // local3768forumtest is just my tunnel from my dev env to the outside web that im reusing from forumtest - serviceDid: `${Deno.env.get("SERVICE_DID")}`, - //keyCacheSize: 500, - //keyCacheTTL: 10 * 60 * 1000, -}); - -// ------------------------------------------ -// XRPC Method Implementations -// ------------------------------------------ - -const indexServerRoutes = new Set([ - "/xrpc/app.bsky.actor.getProfile", - "/xrpc/app.bsky.actor.getProfiles", - "/xrpc/app.bsky.feed.getActorFeeds", - "/xrpc/app.bsky.feed.getFeedGenerator", - "/xrpc/app.bsky.feed.getFeedGenerators", - "/xrpc/app.bsky.feed.getPosts", - "/xrpc/party.whey.app.bsky.feed.getActorLikesPartial", - "/xrpc/party.whey.app.bsky.feed.getAuthorFeedPartial", - "/xrpc/party.whey.app.bsky.feed.getLikesPartial", - "/xrpc/party.whey.app.bsky.feed.getPostThreadPartial", - "/xrpc/party.whey.app.bsky.feed.getQuotesPartial", - "/xrpc/party.whey.app.bsky.feed.getRepostedByPartial", - // more federated endpoints, not planned yet, lexicons will come later - /* - app.bsky.graph.getLists // doesnt need to because theres no items[], and its self ProfileViewBasic - app.bsky.graph.getList // needs to be Partial-ed (items[] union with ProfileViewRef) - app.bsky.graph.getActorStarterPacks // maybe doesnt need to be Partial-ed because its self ProfileViewBasic - - app.bsky.feed.getListFeed // uhh actually already exists its getListFeedPartial - */ - "/xrpc/party.whey.app.bsky.feed.getListFeedPartial", -]); - -Deno.serve( - { port: Number(`${Deno.env.get("SERVER_PORT")}`) }, - async (req: Request): Promise => { - const url = new URL(req.url); - const pathname = url.pathname; - // const searchParams = searchParamsToJson(url.searchParams); - // let reqBody: undefined | string; - // let jsonbody: undefined | Record; - // try { - // const clone = req.clone(); - // jsonbody = await clone.json(); - // } catch (e) { - // console.warn("Request body is not valid JSON:", e); - // } - if (pathname === "/.well-known/did.json") { - return new Response(JSON.stringify(didDocument), { - headers: withCors({ "Content-Type": "application/json" }), - }); - } - if (pathname === "/health") { - return new Response("OK", { - status: 200, - headers: withCors({ - "Content-Type": "text/plain", - }), - }); - } - if (req.method === "OPTIONS") { - return new Response(null, { - status: 204, - headers: { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET, POST, OPTIONS", - "Access-Control-Allow-Headers": "*", - }, - }); - } - // const bskyUrl = `https://api.bsky.app${pathname}${url.search}`; - // const hasAuth = req.headers.has("authorization"); - // const xrpcMethod = pathname.startsWith("/xrpc/") - // ? pathname.slice("/xrpc/".length) - // : null; - console.log(`request for "${pathname}"`) - const constellation = pathname.startsWith("/links") - - // return await viewServerHandler(req) - - if (constellation) { - return registeredUsersIndexServer.constellationAPIHandler(req); - } - - if (indexServerRoutes.has(pathname)) { - return registeredUsersIndexServer.indexServerHandler(req); - } else { - return await viewServerHandler(req); - } - } -); - -// ------------------------------------------ -// Indexer -// ------------------------------------------ \ No newline at end of file diff --git a/readme.md b/readme.md index 70467d2..fb3e0ae 100644 --- a/readme.md +++ b/readme.md @@ -44,9 +44,9 @@ currently the state of the project is: this project is pre-alpha and not intended for general use yet. you are welcome to experiment if you dont mind errors or breaking changes. the project is split into two, the "Index Server" and the "View Server". -these currently run in a single process and share the same HTTP server and port. +despite both living in this repo, they run different http servers with different configs -configuration is in the `.env` file +example configuration is in the `config.jsonc.example` file expose your localhost to the web using a tunnel or something and use that url as the custom appview url diff --git a/utils/diddoc.ts b/utils/diddoc.ts index dcf4167..6c54c03 100644 --- a/utils/diddoc.ts +++ b/utils/diddoc.ts @@ -1,27 +1,42 @@ -export const didDocument = { - "@context": [ - "https://www.w3.org/ns/did/v1", - "https://w3id.org/security/multikey/v1", - ], - id: `${Deno.env.get("SERVICE_DID")}`, - verificationMethod: [ - { - id: `${Deno.env.get("SERVICE_DID")}#atproto`, - type: "Multikey", - controller: `${Deno.env.get("SERVICE_DID")}`, - publicKeyMultibase: "bullshit", +// type "both" should not be used +export function didDocument( + type: "view" | "index" | "both", + did: string, + endpoint: string, + publicKeyMultibase: string, +) { + const services = [ + (type === "view" || type === "both") && { + id: "#bsky_appview", + type: "BskyAppView", + serviceEndpoint: endpoint, }, - ], - service: [ - { + (type === "view" || type === "both") && { id: "#bsky_notif", type: "BskyNotificationService", - serviceEndpoint: `${Deno.env.get("SERVICE_ENDPOINT")}`, + serviceEndpoint: endpoint, }, - { - id: "#bsky_appview", - type: "BskyAppView", - serviceEndpoint: `${Deno.env.get("SERVICE_ENDPOINT")}`, + (type === "index" || type === "both") && { + id: "#skylite_index", + type: "SkyliteIndexServer", + serviceEndpoint: endpoint, }, - ], -}; + ].filter(Boolean); + + return { + "@context": [ + "https://www.w3.org/ns/did/v1", + "https://w3id.org/security/multikey/v1", + ], + id: did, + verificationMethod: [ + { + id: `${did}#atproto`, + type: "Multikey", + controller: did, + publicKeyMultibase: publicKeyMultibase, + }, + ], + service: services, + }; +} \ No newline at end of file diff --git a/utils/identity.ts b/utils/identity.ts index 8bf7026..b06dc7c 100644 --- a/utils/identity.ts +++ b/utils/identity.ts @@ -1,6 +1,7 @@ import { DidResolver, HandleResolver } from "npm:@atproto/identity"; import { Database } from "jsr:@db/sqlite@0.11"; +import { AtUri } from "npm:@atproto/api"; const systemDB = new Database("./system.db") // TODO: temporary shim. should seperate this to its own central system db instead of the now instantiated system dbs type DidMethod = "web" | "plc"; type DidDoc = { @@ -225,4 +226,17 @@ export async function FINEPDSAndHandleFromDid( console.error(`Failed to extract/store PDS and handle for '${did}':`, err); return null; } +} + +export function extractDid(input: string): string { + if (input.startsWith('did:')) { + return input + } + + try { + const uri = new AtUri(input) + return uri.host + } catch (e) { + throw new Error(`Invalid input: expected a DID or a valid AT URI, got "${input}"`) + } } \ No newline at end of file diff --git a/utils/server.ts b/utils/server.ts index 839324f..3afb98b 100644 --- a/utils/server.ts +++ b/utils/server.ts @@ -1,7 +1,7 @@ import ky from "npm:ky"; import QuickLRU from "npm:quick-lru"; import { createHash } from "node:crypto"; -import { slingshoturl, constellationurl } from "../main.ts"; +import { config } from "../config.ts"; import * as ATPAPI from "npm:@atproto/api"; const cache = new QuickLRU({ maxSize: 10000 }); @@ -57,7 +57,7 @@ export function searchParamsToJson( export async function resolveIdentity( actor: string ): Promise { - const url = `${slingshoturl}/xrpc/com.bad-example.identity.resolveMiniDoc?identifier=${actor}`; + const url = `${config.slingshot}/xrpc/com.bad-example.identity.resolveMiniDoc?identifier=${actor}`; return (await cachedFetch(url)) as SlingshotMiniDoc; } export async function getRecord({ @@ -155,7 +155,7 @@ export async function getSlingshotRecord( collection: string, rkey: string ): Promise { - const url = `${slingshoturl}/xrpc/com.atproto.repo.getRecord?repo=${did}&collection=${collection}&rkey=${rkey}`; + const url = `${config.slingshot}/xrpc/com.atproto.repo.getRecord?repo=${did}&collection=${collection}&rkey=${rkey}`; const result = (await cachedFetch(url)) as GetRecord; return result as GetRecord; } @@ -169,7 +169,7 @@ export async function getUniqueCount({ collection: string; path: string; }): Promise { - const url = `${constellationurl}/links/count/distinct-dids?target=${did}&collection=${collection}&path=${path}`; + const url = `${config.constellation}/links/count/distinct-dids?target=${did}&collection=${collection}&path=${path}`; const result = (await cachedFetch(url)) as ConstellationDistinctDids; return result.total; }