diff --git a/server/deno.json b/server/deno.json index 70d6b24..1a942b5 100644 --- a/server/deno.json +++ b/server/deno.json @@ -1,6 +1,6 @@ { "tasks": { - "dev": "deno run --watch --allow-net --allow-env --allow-sys --allow-read=/usr/bin/ldd,./blobs --allow-write=./blobs --allow-ffi --env-file src/index.ts", + "dev": "deno run --watch --allow-net --allow-env --allow-sys --allow-read=/usr/bin/ldd,./blobs,./src --allow-write=./blobs --allow-ffi --env-file src/index.ts", "lexgen": "deno run --allow-env --allow-sys --allow-read=.. --allow-write=./src/lexicons --no-prompt @atcute/lex-cli generate -c ./lex.config.js && cat ./src/lexicons/index.ts | sed \"s/.js/.ts/\" > ./src/lexicons/index.ts", "dk": "deno run -A --node-modules-dir npm:drizzle-kit" }, diff --git a/server/src/backfill.ts b/server/src/backfill.ts index 423e3de..151305b 100644 --- a/server/src/backfill.ts +++ b/server/src/backfill.ts @@ -1,7 +1,7 @@ import { drizzle } from "drizzle-orm/libsql"; import { routes } from "./db/schema.ts"; import * as schema from "./db/schema.ts"; -import { db as db_type } from "./types.ts"; +import { db as db_type } from "./utils.ts"; const db = drizzle(Deno.env.get("DB_FILE_NAME")!); diff --git a/server/src/index.ts b/server/src/index.ts index f11cc05..1bd30de 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,26 +1,7 @@ -import root from "./root.ts"; -import user from "./user.ts"; +import root from "./routes/root.ts"; +import user from "./routes/user.ts"; import backfill from "./backfill.ts"; -import { routes } from "./db/schema.ts"; - -const ROOT_DOMAIN = Deno.env.get("HOSTNAME") || "localhost"; -const PORT = Number(Deno.env.get("PORT")) || 80; - -const SUBDOMAIN_REGEX = new RegExp(`.+(?=\\.${ROOT_DOMAIN}$)`, "gm"); - -function clearCookies(req: Request): Headers { - const cookie_header = req.headers.get("Cookie"); - // cookies are unset so return empty headers - if (!cookie_header) return new Headers(); - // get each kv pair and extract the key - const cookies = cookie_header.split("; ").map((x) => x.split("=")[0]); - const head = new Headers(); - for (const key of cookies) { - // max-age <= 0 means instant expiry .: deleted instantly - head.append("Set-Cookie", `${key}=; Max-Age=-1`); - } - return head; -} +import { PORT, ROOT_DOMAIN, SUBDOMAIN_REGEX, clearCookies } from "./utils.ts"; const db = await backfill(); @@ -45,35 +26,27 @@ Deno.serve({ port: PORT, hostname: ROOT_DOMAIN }, async (req) => { // did:plc example: `sjkdgfjk.did-plc.ROOT_DOMAIN` // did:web example: `vielle.dev.did-web.ROOT_DOMAIN // last segment must be did-plc or did-web - if (subdomain.at(-1)?.match(/^did-(web|plc)+$/gm)) { - const res = await user(db, req, { - did: `did:${subdomain.at(-1) === "did-plc" ? "plc" : "web"}:${subdomain.slice(0, -1).join(".")}`, - }); - return new Response(res.body, { - ...res, - headers: { - ...Array.from(res.headers.entries()).reduce( - (acc, [k, v]) => ({ ...acc, [k]: v }), - {} - ), - ...clearCookies(req), - }, - }); - } - + const isDidSubdomain = !!subdomain.at(-1)?.match(/^did-(web|plc)+$/gm); // ex: vielle.dev.ROOT_DOMAIN // cannot contain hyphen in top level domain - if (!subdomain.at(-1)?.startsWith("did-") && subdomain.length > 1) { - const res = await user(db, req, { - handle: subdomain.join(".") as `${string}.${string}`, - }); + const isHandleSubdomain = !isDidSubdomain && subdomain.length > 1; + + if (isDidSubdomain || isHandleSubdomain) { + const res: Response = await user( + db, + req, + isDidSubdomain + ? { + did: `did:${subdomain.at(-1) === "did-plc" ? "plc" : "web"}:${subdomain.slice(0, -1).join(".")}`, + } + : { + handle: subdomain.join(".") as `${string}.${string}`, + } + ); return new Response(res.body, { ...res, headers: { - ...Array.from(res.headers.entries()).reduce( - (acc, [k, v]) => ({ ...acc, [k]: v }), - {} - ), + ...res.headers, ...clearCookies(req), }, }); diff --git a/server/src/root.ts b/server/src/root.ts deleted file mode 100644 index a5f7b5f..0000000 --- a/server/src/root.ts +++ /dev/null @@ -1,13 +0,0 @@ -import index from "./www/index.html" with { type: "text" }; - -export default function (req: Request) { - if (new URL(req.url).pathname === "/") - return new Response(index, { - headers: { - "Content-Type": "text/html; charset=utf8", - }, - }); - return new Response("404", { - status: 404, - }); -} diff --git a/server/src/routes/ascii.txt b/server/src/routes/ascii.txt new file mode 100644 index 0000000..c72d530 --- /dev/null +++ b/server/src/routes/ascii.txt @@ -0,0 +1,4 @@ +, | +|\ -+-- +|_\ | + \\ \_/ ://cities \ No newline at end of file diff --git a/server/src/routes/root.ts b/server/src/routes/root.ts new file mode 100644 index 0000000..521504d --- /dev/null +++ b/server/src/routes/root.ts @@ -0,0 +1,38 @@ +import { ROOT_DOMAIN } from "../utils.ts"; +import index from "../www/index.html#denoRawImport=text.ts" with { type: "text" }; +import ascii from "./ascii.txt" with { type: "text" }; + +function route( + path: string, + callback: (req: Request) => Response +): { + test: (path: string) => boolean; + fn: (req: Request) => Response; +} { + const pattern = new URLPattern(path, `https://www.${ROOT_DOMAIN}`); + return { + test: (path) => pattern.test(path, `https://www.${ROOT_DOMAIN}`), + fn: callback, + }; +} + +const routes = [ + route( + "/", + () => + new Response(index, { + headers: { + "Content-Type": "text/html; charset=utf8", + }, + }) + ), + route("/ascii.txt", () => new Response(ascii)), +]; + +export default function (req: Request) { + const path = new URL(req.url).pathname; + for (const r of routes) if (r.test(path)) return r.fn(req); + return new Response("404", { + status: 404, + }); +} diff --git a/server/src/routes/user.ts b/server/src/routes/user.ts new file mode 100644 index 0000000..788edd5 --- /dev/null +++ b/server/src/routes/user.ts @@ -0,0 +1,98 @@ +/// +import { + CompositeHandleResolver, + DohJsonHandleResolver, + WellKnownHandleResolver, +} from "@atcute/identity-resolver"; +import { and, eq } from "drizzle-orm"; +import { routes } from "../db/schema.ts"; +import { ROOT_DOMAIN, type db } from "../utils.ts"; +import ascii from "./ascii.txt" with { type: "text" }; + +const handleResolver = new CompositeHandleResolver({ + strategy: "race", + methods: { + dns: new DohJsonHandleResolver({ + dohUrl: "https://mozilla.cloudflare-dns.com/dns-query", + }), + http: new WellKnownHandleResolver(), + }, +}); + +export default async function ( + db: db, + req: Request, + user: + | { handle: `${string}.${string}` } + | { did: `did:plc:${string}` | `did:web:${string}` } +): Promise { + // if handle: resolve did + let did: `did:${"plc" | "web"}:${string}`; + if ("handle" in user) { + try { + // cast bc i know it will be `string.string` + did = await handleResolver.resolve(user.handle); + } catch { + return new Response( + `${ascii} + +This handle +`, + { + status: 500, + statusText: "Internal Server Error", + } + ); + } + } else did = user.did; + + // look up in db + const db_res = + ( + await db + .select() + .from(routes) + .where( + and( + eq(routes.did, did), + eq(routes.url_route, new URL(req.url).pathname) + ) + ) + ).at(0) ?? + ( + await db + .select() + .from(routes) + .where(and(eq(routes.did, did), eq(routes.url_route, "404"))) + ).at(0); + + if (!db_res) { + return new Response(`${ascii} + +404: The user has no atcities site or is missing a 404 page. + +If you're the owner of this account, head to https://atcities.dev/ for more information. +The index of this account is at https://${"handle" in user ? user.handle : user.did.split(":").at(-1) + ".did-" + user.did.split(":").at(1)}.${ROOT_DOMAIN}/ +`); + } + try { + const file = await Deno.readFile( + `./blobs/${db_res.did}/${db_res.blob_cid}` + ); + return new Response(file, { + headers: { + "Content-Type": db_res.mime, + }, + }); + } catch { + return new Response(`${ascii} + +This page isn't stored in the CDN. +TODO: + Fetch the content from the pds, + check its hash, + serve it if not known as illegal, + store in cdn if doesnt take account over fs limit +`); + } +} diff --git a/server/src/types.ts b/server/src/types.ts deleted file mode 100644 index 0b665b0..0000000 --- a/server/src/types.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { LibSQLDatabase } from "drizzle-orm/libsql"; -import { Client } from "@libsql/client"; -import * as schema from "./db/schema.ts"; - -export type db = LibSQLDatabase & { - $client: Client; -}; diff --git a/server/src/user.ts b/server/src/user.ts deleted file mode 100644 index 2de1055..0000000 --- a/server/src/user.ts +++ /dev/null @@ -1,342 +0,0 @@ -/// -import { Client, simpleFetchHandler } from "@atcute/client"; -import { is } from "@atcute/lexicons"; -import { - CompositeHandleResolver, - DohJsonHandleResolver, - WellKnownHandleResolver, - CompositeDidDocumentResolver, - PlcDidDocumentResolver, - WebDidDocumentResolver, -} from "@atcute/identity-resolver"; -import { DevAtcitiesRoute } from "./lexicons/index.ts"; -import { db } from "./types.ts"; -import { and, eq } from "drizzle-orm"; -import { routes } from "./db/schema.ts"; - -const handleResolver = new CompositeHandleResolver({ - strategy: "race", - methods: { - dns: new DohJsonHandleResolver({ - dohUrl: "https://mozilla.cloudflare-dns.com/dns-query", - }), - http: new WellKnownHandleResolver(), - }, -}); - -const docResolver = new CompositeDidDocumentResolver({ - methods: { - plc: new PlcDidDocumentResolver(), - web: new WebDidDocumentResolver(), - }, -}); - -/** - * given a valid url path string containing - * - `/` for seperating characters - * - a-zA-Z0-9 `-._~` as unreserved - * - `!$&'()*+,;=` as reserved but valid in paths - * - `:@` as neither reserved or unreserved but valid in paths - * - %XX where X are hex digits for percent encoding - * - * we need to consistently and bidirectionally convert it into a string containing the characters A-Z, a-z, 0-9, `.-_:~` for an atproto rkey - * A-Z a-z 0-9 are covered easily - * we can also take -._~ as they are also unreserved - * leaving : as a valid rkey character which looks nice for encoding - * the uppercase versions MUST be used to prevent ambiguity - * a colon which isnt followed by a valid character is an invalid rkey and should be ignored - * - `/` `::` - * - `%` `:~` - * - `!` `:21` - * - `$` `:24` - * - `&` `:26` - * - `'` `:27` - * - `(` `:28` - * - `)` `:29` - * - `*` `:2A` - * - `+` `:2B` - * - `,` `:2C` - * - `:` `:3A` - * - `;` `:3B` - * - `=` `:3D` - * - `@` `:40` - * @returns {string | undefined} undefined when input is invalid - */ -function urlToRkey(url: string): string | undefined { - // contains 0-9A-Za-z + special valid chars and / seperator. also can contain %XX with XX being hex - if (!url.match(/^([a-zA-Z0-9/\-._~!$&'()*+,;=:@]|(%[0-9a-fA-F]{2}))*$/gm)) - return; - return ( - url - // : replace is hoisted so it doesnt replace colons from elsewhere - .replaceAll(":", ":3A") - .replaceAll("/", "::") - .replaceAll("%", ":~") - .replaceAll("!", ":21") - .replaceAll("$", ":24") - .replaceAll("&", ":26") - .replaceAll("'", ":27") - .replaceAll("(", ":28") - .replaceAll(")", ":29") - .replaceAll("*", ":2A") - .replaceAll("+", ":2B") - .replaceAll(",", ":2C") - .replaceAll(";", ":3B") - .replaceAll("=", ":3D") - .replaceAll("@", ":40") - ); -} - -/** - * @see {@link urlToRkey} for rkey <=> url conversion syntax - * @returns {string | undefined} undefined when input is invalid - */ -function rkeyToUrl(rkey: string): string | undefined { - // contains 0-9A-Za-z .-_:~ - if (!rkey.match(/^[A-Za-z0-9.\-_:~]*$/gm)) return; - return rkey - .replaceAll("::", "/") - .replaceAll(":~", "%") - .replaceAll(":21", "!") - .replaceAll(":24", "$") - .replaceAll(":26", "&") - .replaceAll(":27", "'") - .replaceAll(":28", "(") - .replaceAll(":29", ")") - .replaceAll(":2A", "*") - .replaceAll(":2B", "+") - .replaceAll(":2C", ",") - .replaceAll(":3A", ":") - .replaceAll(":3B", ";") - .replaceAll(":3D", "=") - .replaceAll(":40", "@"); -} - -async function getRoute( - did: `did:${"plc" | "web"}:${string}`, - pds: string, - route: string -): Promise { - // get client to pds - const client = new Client({ - handler: simpleFetchHandler({ service: pds }), - }); - - try { - // note: / urls are reserved for special routes (404) - // any path should be prefixed with a / - // this is not enforced here so that this function can be used for special routes - const targetRkey = urlToRkey(route); - console.log("trying:", targetRkey, "for", route); - - if (!targetRkey) throw "invalid url"; - - const { ok: record_ok, data: record_data } = await client.get( - "com.atproto.repo.getRecord", - { - params: { - collection: "dev.atcities.route", - repo: did, - rkey: targetRkey, - }, - } - ); - - if (!record_ok) { - switch (record_data.error) { - case "RecordNotFound": { - // 404 error so try load 404 page - if (route !== "404") { - // try remove trailing / to see if that works - // if that fails it'll get a 404 anyway - if (route !== "/" && route.endsWith("/")) - return new Response(undefined, { - status: 308, - statusText: "Permanent Redirect", - headers: { - Location: route.slice(0, -1), - }, - }); - - const r404 = await getRoute(did, pds, "404"); - return new Response(r404.body, { - status: 404, - statusText: "Not Found", - headers: r404.headers, - }); - } - return new Response("Could not find page.", { - status: 404, - statusText: "Not Found", - }); - } - // unless its a 404, internal error - default: - throw ( - "Internal Error: got error fetching record: " + record_data.error - ); - } - } - - console.log(record_data.value); - if (is(DevAtcitiesRoute.mainSchema, record_data.value)) { - switch (record_data.value.page.$type) { - case "dev.atcities.route#blob": { - const { ok: blob_ok, data: blob_data } = await client.get( - "com.atproto.sync.getBlob", - { - params: { - did, - cid: - "ref" in record_data.value.page.blob - ? record_data.value.page.blob.ref.$link - : record_data.value.page.blob.cid, - }, - as: "stream", - } - ); - - // possible errors include: - // - request issue - // - 404 not found - // - account takedown - // in all cases thats not recoverable - // so throw out and take the happy path - if (!blob_ok) { - if (blob_data.error === "BlobNotFound") { - // 404 error so try load 404 page - if (route !== "404") { - const r404 = await getRoute(did, pds, "404"); - return new Response(r404.body, { - status: 404, - statusText: "Not Found", - headers: r404.headers, - }); - } - return new Response("Could not load page.", { - status: 404, - statusText: "Not Found", - }); - } else - throw "Internal Error: Error fetching blob: " + blob_data.error; - } - - return new Response(blob_data, { - headers: { - "Content-Type": record_data.value.page.blob.mimeType, - }, - }); - } - } - } - // isnt valid data so throw exception - return new Response( - "Malformed record for at://" + did + "/dev.atcities.route/" + targetRkey - ); - } catch (e) { - console.error(e); - return new Response("Something went wrong loading this route", { - status: 500, - statusText: "Internal Server Error", - }); - } -} - -export default async function ( - db: db, - req: Request, - user: - | { handle: `${string}.${string}` } - | { did: `did:plc:${string}` | `did:web:${string}` } -): Promise { - // if handle: resolve did - let did: `did:${"plc" | "web"}:${string}`; - if ("handle" in user) { - try { - // cast bc i know it will be `string.string` - did = await handleResolver.resolve(user.handle); - } catch { - return new Response("Failed to resolve handle", { - status: 500, - statusText: "Internal Server Error", - }); - } - } else did = user.did; - - // look up in db - const db_res = - ( - await db - .select() - .from(routes) - .where( - and( - eq(routes.did, did), - eq(routes.url_route, new URL(req.url).pathname) - ) - ) - ).at(0) ?? - ( - await db - .select() - .from(routes) - .where(and(eq(routes.did, did), eq(routes.url_route, "404"))) - ).at(0); - - if (db_res) { - try { - const file = await Deno.readFile( - `./blobs/${db_res.did}/${db_res.blob_cid}` - ); - return new Response(file, { - headers: { - "Content-Type": db_res.mime, - }, - }); - } catch { - return new Response( - "Could not find in CDN; TODO: fallback to fetch from pds && add to cdn if missing\nNB: should this be default? index routes in db but only cache used pages" - ); - } - } - - // resolve did doc - let doc; - try { - doc = await docResolver.resolve(did); - } catch { - return new Response("Could not resolve " + did + ".\n", { - status: 500, - statusText: "Internal Server Error", - }); - } - - // handle must be in did document - if ("handle" in user && !doc.alsoKnownAs?.includes(`at://${user.handle}`)) { - return new Response( - `Provided handle (at://${user.handle}) was not found in the did document for ${did}. Found handles:\n${doc.alsoKnownAs?.map((x) => "- " + x).join("\n") ?? "None"}`, - { - status: 500, - statusText: "Internal Server Error", - } - ); - } - - const pds = doc.service?.filter( - (x) => - x.id.endsWith("#atproto_pds") && - x.type === "AtprotoPersonalDataServer" && - typeof x.serviceEndpoint === "string" - // cast as we type checked it above but it didnt acc apply? - )[0].serviceEndpoint as string | undefined; - if (!pds) - return new Response( - `Could not find a valid pds for ${"handle" in user ? user.handle : user.did}`, - { - status: 500, - statusText: "Internal Server Error", - } - ); - - return await getRoute(did, pds, new URL(req.url).pathname); -} diff --git a/server/src/utils.ts b/server/src/utils.ts new file mode 100644 index 0000000..198eccd --- /dev/null +++ b/server/src/utils.ts @@ -0,0 +1,107 @@ +import { LibSQLDatabase } from "drizzle-orm/libsql"; +import { Client } from "@libsql/client"; +import * as schema from "./db/schema.ts"; + +export type db = LibSQLDatabase & { + $client: Client; +}; + +export const ROOT_DOMAIN = Deno.env.get("HOSTNAME") || "localhost"; +export const PORT = Number(Deno.env.get("PORT")) || 80; + +export const SUBDOMAIN_REGEX = new RegExp(`.+(?=\\.${ROOT_DOMAIN}$)`, "gm"); + +export function clearCookies(req: Request): Headers { + const cookie_header = req.headers.get("Cookie"); + // cookies are unset so return empty headers + if (!cookie_header) return new Headers(); + // get each kv pair and extract the key + const cookies = cookie_header.split("; ").map((x) => x.split("=")[0]); + const head = new Headers(); + for (const key of cookies) { + // max-age <= 0 means instant expiry .: deleted instantly + head.append("Set-Cookie", `${key}=; Max-Age=-1`); + } + return head; +} + +/** + * given a valid url path string containing + * - `/` for seperating characters + * - a-zA-Z0-9 `-._~` as unreserved + * - `!$&'()*+,;=` as reserved but valid in paths + * - `:@` as neither reserved or unreserved but valid in paths + * - %XX where X are hex digits for percent encoding + * + * we need to consistently and bidirectionally convert it into a string containing the characters A-Z, a-z, 0-9, `.-_:~` for an atproto rkey + * A-Z a-z 0-9 are covered easily + * we can also take -._~ as they are also unreserved + * leaving : as a valid rkey character which looks nice for encoding + * the uppercase versions MUST be used to prevent ambiguity + * a colon which isnt followed by a valid character is an invalid rkey and should be ignored + * - `/` `::` + * - `%` `:~` + * - `!` `:21` + * - `$` `:24` + * - `&` `:26` + * - `'` `:27` + * - `(` `:28` + * - `)` `:29` + * - `*` `:2A` + * - `+` `:2B` + * - `,` `:2C` + * - `:` `:3A` + * - `;` `:3B` + * - `=` `:3D` + * - `@` `:40` + * @returns {string | undefined} undefined when input is invalid + */ +export function urlToRkey(url: string): string | undefined { + // contains 0-9A-Za-z + special valid chars and / seperator. also can contain %XX with XX being hex + if (!url.match(/^([a-zA-Z0-9/\-._~!$&'()*+,;=:@]|(%[0-9a-fA-F]{2}))*$/gm)) + return; + return ( + url + // : replace is hoisted so it doesnt replace colons from elsewhere + .replaceAll(":", ":3A") + .replaceAll("/", "::") + .replaceAll("%", ":~") + .replaceAll("!", ":21") + .replaceAll("$", ":24") + .replaceAll("&", ":26") + .replaceAll("'", ":27") + .replaceAll("(", ":28") + .replaceAll(")", ":29") + .replaceAll("*", ":2A") + .replaceAll("+", ":2B") + .replaceAll(",", ":2C") + .replaceAll(";", ":3B") + .replaceAll("=", ":3D") + .replaceAll("@", ":40") + ); +} + +/** + * @see {@link urlToRkey} for rkey <=> url conversion syntax + * @returns {string | undefined} undefined when input is invalid + */ +export function rkeyToUrl(rkey: string): string | undefined { + // contains 0-9A-Za-z .-_:~ + if (!rkey.match(/^[A-Za-z0-9.\-_:~]*$/gm)) return; + return rkey + .replaceAll("::", "/") + .replaceAll(":~", "%") + .replaceAll(":21", "!") + .replaceAll(":24", "$") + .replaceAll(":26", "&") + .replaceAll(":27", "'") + .replaceAll(":28", "(") + .replaceAll(":29", ")") + .replaceAll(":2A", "*") + .replaceAll(":2B", "+") + .replaceAll(":2C", ",") + .replaceAll(":3A", ":") + .replaceAll(":3B", ";") + .replaceAll(":3D", "=") + .replaceAll(":40", "@"); +}