import type {Mapper} from "p-map"; import { sleep } from "../utils.ts"; import { nanoid } from "nanoid"; /** https://stackoverflow.com/a/63795192/1469797 */ export async function findAsyncSequential( array: T[], predicate: (t: T) => Promise, ): Promise { const i = await findIndexAsyncSequential(array, predicate); if(i === undefined) { return undefined; } return array[i]; } export async function findIndexAsyncSequential( array: T[], predicate: (t: T) => Promise, ): Promise { let index = 0; for (const t of array) { if (await predicate(t)) { return index; } index++; } return undefined; } /** https://stackoverflow.com/a/55601090/1469797 */ export async function findAsync( array: T[], predicate: (t: T) => Promise): Promise { const i = await findIndexAsync(array, predicate); if(i === undefined) { return undefined; } return array[i]; } export async function findIndexAsync( array: T[], predicate: (t: T) => Promise): Promise { const promises = array.map(predicate); const results = await Promise.all(promises); const index = results.findIndex(result => result); return index; } export interface StaggerOptions { maxRandomStagger?: number, initialInterval?: number, concurrency: number } export function staggerMapper(options: StaggerOptions) { const { initialInterval = 0, maxRandomStagger = 0, concurrency } = options; let initialStagger = 0; return (mapper: Mapper) => async (x: Element, index: number) => { if (index < concurrency) { await sleep(initialStagger); initialStagger += initialInterval; } else { const s = Math.min((Math.random() * 1000), maxRandomStagger) await sleep(s); } return await mapper(x, index); } } export const consumeQueueOnce = async (next: () => Promise, process: (item: T) => Promise, opts: { concurrency: number; signal: AbortSignal; onError?: (e: Error) => Promise, onSuccess?: () => void }): Promise => { const { concurrency, signal, onError } = opts; signal.throwIfAborted(); const inFlight = new Set>(); try { while (true) { signal.throwIfAborted(); if (inFlight.size >= concurrency) { await Promise.race(inFlight); continue; } const item = await next(); if (item === undefined) break; const task = (async () => { try { await process(item); } catch (err) { await onError?.(err); // swallow so one bad item doesn't kill the loop } })(); inFlight.add(task); void task.then(() => inFlight.delete(task)); } } finally { await Promise.allSettled(inFlight); // drain before sleeping or rethrowing } }; export const consumeQueue = async ( next: (queueId: string) => Promise, process: (item: T, queueId: string) => Promise, opts: { concurrency: number; idleMs: number; signal: AbortSignal; onError?: (e: Error, queueId: string) => Promise, onSuccess?: (item: T, queueId: string) => void, onEmpty?: () => void }, ): Promise => { const { concurrency, idleMs, signal, onError, onEmpty, onSuccess } = opts; while (true) { signal.throwIfAborted(); const inFlight = new Set>(); try { while (true) { signal.throwIfAborted(); if (inFlight.size >= concurrency) { await Promise.race(inFlight); continue; } const qId = nanoid(); const item = await next(qId); if (item === undefined) break; const task = (async () => { try { await process(item, qId); onSuccess?.(item, qId); } catch (err) { await onError?.(err, qId); // swallow so one bad item doesn't kill the loop } })(); inFlight.add(task); void task.then(() => inFlight.delete(task)); } } finally { await Promise.allSettled(inFlight); // drain before sleeping or rethrowing } onEmpty?.(); await sleep(idleMs, { signal }); } }