import { type Duration, Effect, Schedule } from "effect"; import type { z } from "zod"; import type { JsonValue } from "./types"; // "parse": body is not JSON (html, challenge page); "schema": valid JSON our // zod schema rejects — the split separates "page moved" from "our schema is stale". export type FetchErrorKind = | "http" | "parse" | "schema" | "network" | "timeout"; const KIND_LABELS: Record = { http: "fetch failed", parse: "non-JSON body", schema: "schema mismatch", network: "network error", timeout: "timeout", }; type FetchErrorInit = { url: string; fetcherName?: string; entryId?: string; httpStatus?: number; kind?: FetchErrorKind; cause?: Error; }; export class FetchError extends Error { readonly url: string; readonly fetcherName?: string; readonly entryId?: string; readonly httpStatus?: number; readonly kind?: FetchErrorKind; constructor(init: FetchErrorInit) { const ctx = [init.fetcherName, init.entryId && `(${init.entryId})`] .filter(Boolean) .join(" ") || "FetchError"; const status = init.httpStatus ? `HTTP ${init.httpStatus}` : init.kind ? KIND_LABELS[init.kind] : "fetch failed"; super(`[${ctx}] ${status}: ${init.url}`, { cause: init.cause }); this.name = "FetchError"; this.url = init.url; this.fetcherName = init.fetcherName; this.entryId = init.entryId; this.httpStatus = init.httpStatus; this.kind = init.kind; } } const DEFAULT_TIMEOUT: Duration.Input = "30000 millis"; export type FetchBaseOptions = { url: string; init?: Omit & { headers?: Record; }; timeout?: Duration.Input; fetcherName?: string; entryId?: string; }; const isRetryable = (err: FetchError): boolean => !( err.httpStatus !== undefined && err.httpStatus >= 400 && err.httpStatus < 500 ); const retryPolicy = { schedule: Schedule.exponential("100 millis").pipe(Schedule.jittered), times: 3, while: isRetryable, }; const buildFetchError = ( opts: FetchBaseOptions, extras: { httpStatus?: number; kind?: FetchErrorKind; cause?: Error }, ): FetchError => new FetchError({ url: opts.url, fetcherName: opts.fetcherName, entryId: opts.entryId, ...extras, }); const failWith = (opts: FetchBaseOptions, kind: FetchErrorKind) => (cause: unknown): FetchError => buildFetchError(opts, { kind, cause: cause instanceof Error ? cause : new Error(String(cause)), }); const doFetch = ( opts: FetchBaseOptions, defaultHeaders: Record, ): Effect.Effect => Effect.tryPromise({ try: (signal) => fetch(opts.url, { ...opts.init, headers: { ...defaultHeaders, ...opts.init?.headers }, signal, }), catch: failWith(opts, "network"), }).pipe( Effect.timeoutOrElse({ duration: opts.timeout ?? DEFAULT_TIMEOUT, orElse: () => Effect.fail( buildFetchError(opts, { kind: "timeout", cause: new Error( `timeout after ${String(opts.timeout ?? DEFAULT_TIMEOUT)}`, ), }), ), }), Effect.flatMap((response) => response.ok ? Effect.succeed(response) : Effect.fail( buildFetchError(opts, { httpStatus: response.status, kind: "http", }), ), ), ); // The body read gets its own timeout: `doFetch`'s only covers up to headers, // so a stalled body stream would otherwise hang callers forever. const fetchBody = ( opts: FetchBaseOptions, defaultHeaders: Record, read: (response: Response) => Effect.Effect, ): Effect.Effect => doFetch(opts, defaultHeaders).pipe( Effect.retry(retryPolicy), Effect.flatMap((response) => read(response).pipe( Effect.timeoutOrElse({ duration: opts.timeout ?? DEFAULT_TIMEOUT, orElse: () => Effect.fail( buildFetchError(opts, { kind: "timeout", cause: new Error( `body read timeout after ${String(opts.timeout ?? DEFAULT_TIMEOUT)}`, ), }), ), }), ), ), ); const JSON_HEADERS = { "User-Agent": "OpenStatus-Directory/1.0", Accept: "application/json", }; // browser-like UA: some status pages reject non-browser User-Agents const TEXT_HEADERS = { "User-Agent": "Mozilla/5.0 (compatible; OpenStatus-Bot/1.0)", }; export const fetchJson = ( opts: FetchBaseOptions & { schema: z.ZodType }, ): Effect.Effect => fetchBody(opts, JSON_HEADERS, (response) => Effect.tryPromise({ try: () => response.json(), catch: failWith(opts, "parse"), }).pipe( Effect.flatMap((json) => Effect.try({ try: () => opts.schema.parse(json), catch: failWith(opts, "schema"), }), ), ), ); // `response.json()` is typed `Promise` by the platform lib; the cast pins // the boundary to a concrete JsonValue so callers never touch `any`/`unknown`. export const fetchJsonWithRaw = ( opts: FetchBaseOptions & { schema: z.ZodType }, ): Effect.Effect<{ parsed: T; raw: JsonValue }, FetchError> => fetchBody(opts, JSON_HEADERS, (response) => Effect.tryPromise({ try: () => response.json() as Promise, catch: failWith(opts, "parse"), }).pipe( Effect.flatMap((raw) => Effect.try({ try: () => ({ parsed: opts.schema.parse(raw), raw }), catch: failWith(opts, "schema"), }), ), ), ); export const fetchText = ( opts: FetchBaseOptions, ): Effect.Effect => fetchBody(opts, TEXT_HEADERS, (response) => Effect.tryPromise({ try: () => response.text(), catch: failWith(opts, "network"), }), ); export const fetchTextWithUrl = ( opts: FetchBaseOptions, ): Effect.Effect<{ text: string; finalUrl: string }, FetchError> => fetchBody(opts, TEXT_HEADERS, (response) => Effect.tryPromise({ try: async () => ({ text: await response.text(), finalUrl: response.url, }), catch: failWith(opts, "network"), }), );