Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
TypeScript
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859import * as Effect from "effect/Effect";import type { WebSource } from "../shared/web";import { WebFailure } from "./web-errors";
// Conservative hostname/literal validation; this is not DNS rebinding protection.export function publicWebUrl(raw: string): URL { let url: URL; try { url = new URL(raw); } catch { throw new WebFailure("invalid_url"); } if ( raw.length > 4096 || !["http:", "https:"].includes(url.protocol) || url.username || url.password ) throw new WebFailure("invalid_url"); const host = url.hostname.toLowerCase().replace(/\.$/, ""); // Citation destinations do not need literal IPs. Blocking all literals also // covers mapped IPv6, alternate IPv4 encodings normalized by URL, and LANs. if ( !host.includes(".") || host.startsWith("[") || /^\d+\.\d+\.\d+\.\d+$/.test(host) || /(?:^|\.)(?:localhost|local|internal|test|invalid)$/.test(host) ) throw new WebFailure("blocked_url"); url.hash = ""; return url;}export function clip(text: string, max: number) { const value = text.slice(0, max); return /[\uD800-\uDBFF]$/.test(value) ? value.slice(0, -1) : value;}export const cleanText = (text: string, max: number) => clip( text .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, " ") .replace(/\s+/g, " ") .trim(), max, );export function webSource(source: Omit<WebSource, "id" | "fetchedAt">) { return Effect.gen(function* () { const digest = yield* Effect.promise(() => crypto.subtle.digest( "SHA-256", new TextEncoder().encode(source.finalUrl), ), ); const id = Array.from(new Uint8Array(digest)) .map((b) => b.toString(16).padStart(2, "0")) .join(""); return { ...source, id: `web-${id}`, fetchedAt: new Date().toISOString() }; });}