From b2b097ece29d0201315a40f7900a144d0b23a168 Mon Sep 17 00:00:00 2001 From: Philippe Serhal Date: Sun, 9 Aug 2026 15:23:43 -0400 Subject: [PATCH] refactor: remove refs to past codebase state --- AGENTS.md | 40 +- CONTRIBUTING.md | 4 +- PRODUCT.md | 5 +- .../functions/_shared/report-persistence.ts | 3 +- scripts/npm-audit.sh | 361 ------------------ scripts/npm-user-publishes.sh | 93 ----- src/App.svelte | 2 +- src/App.test.ts | 2 +- src/components/LogTerminal.svelte | 16 +- src/components/ResultsView.svelte | 4 +- src/lib/auditStream.ts | 2 +- src/lib/concurrency.ts | 7 +- src/lib/discovery.ts | 7 +- src/lib/downloads.ts | 2 +- src/lib/members.ts | 3 +- src/lib/npmClient.ts | 19 +- src/lib/reports.ts | 4 +- src/lib/runAudit.ts | 13 +- src/lib/schemas.ts | 5 +- src/lib/types.ts | 8 +- 20 files changed, 61 insertions(+), 539 deletions(-) delete mode 100644 scripts/npm-audit.sh delete mode 100644 scripts/npm-user-publishes.sh diff --git a/AGENTS.md b/AGENTS.md index 1757022..34cee92 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,10 +4,8 @@ Architecture and rules for agents editing this project. ## Project Shape -This is a web app for npm supply-chain audits. It is a TypeScript port of two -shell scripts in `scripts/`; those scripts are the reference specification for -audit behavior. When changing audit semantics, diff against the scripts and -preserve their documented behavior unless the user explicitly asks otherwise. +This is a web app for npm supply-chain audits. Preserve the documented audit +behavior and invariants unless the user explicitly asks otherwise. Audits run server-side and stream to the browser. When the user runs an audit the browser POSTs a validated request to a Netlify **edge function**, which runs @@ -53,10 +51,9 @@ The server-side entry points are: ## npm Access npm is fetched directly from the server. Because audits run in edge/background -functions, not the browser, there is no cross-origin restriction to work around, -so the CORS proxies (and their shared proxy core) were removed. There is no -host-generic proxy and no request-controlled upstream host — i.e. no SSRF -surface to defend. +functions, not the browser, there is no cross-origin restriction and no need for +a proxy. There is no host-generic proxy and no request-controlled upstream host +— i.e. no SSRF surface to defend. `src/lib/npmClient.ts` (`npmGet`/`npmGetJson`) fetches the upstream hosts directly, with the retry/backoff/`FailureLog` semantics under Invariants: @@ -66,7 +63,7 @@ directly, with the retry/backoff/`FailureLog` semantics under Invariants: - `npm.antfu.dev` (fast-npm-meta) — batched discovery metadata. When adding a new npm upstream, add the URL helper in `npmClient.ts` and fetch -it directly; do not reintroduce a proxy layer. +it directly; do not add a proxy layer. ## Edge Bundling (Deno) @@ -195,7 +192,6 @@ Two deliberate exceptions exist so they don't get "fixed": index.html Vite entry; loads IBM Plex fonts netlify.toml Netlify static publish of dist/, SPA redirect, security headers, edge import map import_map.json Deno import map: edge-only npm deps (valibot, packumeta) -> esm.sh -scripts/ Original shell scripts; behavior reference, not executed db/ Native Netlify Database connection and Valibot row contracts netlify/ edge-functions/ @@ -267,9 +263,9 @@ src/ with a 500 ms delay. Do not parallelize scoped downloads. A present-but-null entry is a real 0; only a failed/absent fetch stays unknown ("?"). - `trust` and `manual` share one discovery pass. `manual` scans the package set - from that cache: all packages under `-A`, otherwise only recency-filtered - packages. `external` ignores that cache and enumerates the full org list - because dormant packages can still have live maintainers. + from that cache: every package for an all-package audit, otherwise only + recency-filtered packages. `external` ignores that cache and enumerates the + full org list because dormant packages can still have live maintainers. - Do not hardcode a real org, user, or package. This is a generic tool and those values are user input. Generic automation-account defaults are allowed only when they are broadly applicable, such as the current `GitHub Actions` manual @@ -290,8 +286,8 @@ src/ resume — the first request runs the audit (and finishes it even if the client disconnects, so the report still saves); a reconnect only tails. Rows are throwaway (the durable report is in `reports`) and pruned hourly by - `audit-jobs-cleanup-background.ts`. Do not reintroduce a keepalive: it cannot - beat a total connection cut — reconnect + resume is the fix. + `audit-jobs-cleanup-background.ts`. Do not use a keepalive: it cannot beat a + total connection cut; reconnect + resume handles the connection lifecycle. - The report-creation endpoints are rate-limited per IP at the Netlify edge (the `rateLimit` field in each function's `config`): `audit-stream` and `user-publishes-stream` at 30/min, `reports.ts` at 120/min. This is the abuse @@ -323,10 +319,9 @@ src/ - No runtime dependency, no shell. The visible log is itself the screen-reader live region (`role="log"`, `aria-live="polite"`), so progress and the "results may be INCOMPLETE" warning are announced. -- It replaced a `ghostty-web` WASM terminal that inlined ~413 KB of base64 WASM - into the browser bundle (~85% of it) to render append-only text. Do not - reintroduce a WASM terminal for a display-only log; removing it also let the CSP - drop `'wasm-unsafe-eval'`. +- Keep the display-only log dependency-free. Append-only text does not warrant a + terminal emulator or WASM runtime, and the CSP must not require + `'wasm-unsafe-eval'`. ## Change Recipes @@ -345,11 +340,10 @@ Adding a report: Changing audit behavior: -1. Compare the relevant shell-script behavior in `scripts/`. -2. Preserve failure logging and partial-result warnings. -3. Keep data-source choices intentional: fast-npm-meta for discovery, +1. Preserve failure logging and partial-result warnings. +2. Keep data-source choices intentional: fast-npm-meta for discovery, per-version manifests for trust, full packuments only where needed. -4. Update README and CONTRIBUTING if user-visible behavior or required workflow +3. Update README and CONTRIBUTING if user-visible behavior or required workflow changes. Changing the audit request or SSE contract: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b718603..b92ccce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,8 +20,8 @@ package-trust tracking API, hooking up Netlify Database, etc. - Keep the app broadly usable for most developers. Do not hardcode a specific org, user, or package. Bot defaults should be broadly applicable automation accounts, not project-specific policy. - Keep audits server-side. The browser submits a request and renders the stream; it must not compute - the audit or POST a report. npm is fetched directly (no CORS proxy); do not reintroduce one. Moving - the report generation client-side precludes secure, trustable persisted reports. + the audit or POST a report. Fetch npm directly without a CORS proxy. Client-side report generation + would preclude secure, trustable persisted reports. - Keep scheduled reruns narrow: all-package package trust only, with direct npm fetches from the Netlify function runtime. diff --git a/PRODUCT.md b/PRODUCT.md index 5a92aeb..3fdb91f 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -8,9 +8,8 @@ publishes, and publish access held by non-members. ## Product Purpose -Audit npm organizations against public npm data, preserving the documented behavior of the original -shell scripts while making the results easier to run, inspect, export, and share as read-only -snapshots. +Audit npm organizations against public npm data, with results that are easy to run, inspect, export, +and share as read-only snapshots. ## Brand Personality diff --git a/netlify/functions/_shared/report-persistence.ts b/netlify/functions/_shared/report-persistence.ts index 278a315..628e924 100644 --- a/netlify/functions/_shared/report-persistence.ts +++ b/netlify/functions/_shared/report-persistence.ts @@ -40,8 +40,7 @@ export async function buildReportId( payload: unknown, now = new Date(), ): Promise { - // Web Crypto (not node:crypto) so this runs unchanged in edge/Deno as well as - // Node — the audit + save now runs in an edge function. + // Web Crypto keeps this module compatible with edge/Deno and Node runtimes. const bytes = new TextEncoder().encode(JSON.stringify(payload)); const digest = await crypto.subtle.digest("SHA-256", bytes); const hash = Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")) diff --git a/scripts/npm-audit.sh b/scripts/npm-audit.sh deleted file mode 100644 index b5132de..0000000 --- a/scripts/npm-audit.sh +++ /dev/null @@ -1,361 +0,0 @@ -#!/usr/bin/env bash -# npm-audit.sh — supply-chain audits over npm org packages (default: netlify gatsbyjs). -# -# Reports (comma-separated, or "all"): -# recent Packages whose latest release shipped within the window (or ALL of -# them with -A), with the supply-chain trust status of each package's -# `latest` release (logic ported from github.com/43081j/packumeta). -# Discovery (latest version + recency + deprecated) uses fast-npm-meta -# (npm.antfu.dev) in batch; trust fields come from the lightweight -# per-version registry manifest. Cache other reports build on; -# written to /recent-packages.tsv (cols: pkg, -# latest_publish_iso, latest_version, trust_level, provenance, -# trustedPublisher, stagedPublish, publisher, deprecated, -# downloads_last_week) plus a coverage summary in -# /trust-summary.txt. trust_level (packumeta): stagedPublish(3) -# > trustedPublisher(2, =trustedPublisher+provenance) > provenance(1) -# > none(0). -# manual Who published MANUALLY (non-CI/bot account) in the window, and what. -# external Users who can publish NOW (current maintainer) but aren't org members. -# Requires -M members.txt (npm org membership is not public). -# -# Note: per-user publish history is a separate tool, npm-user-publishes.sh — -# it shares nothing with these reports. -# -# Usage: -# npm-audit.sh -O OUTDIR [opts] REPORTS -# Options: -# -O DIR output directory (required) -# -m N window in months (default 12) -# -A analyze ALL org packages, ignoring the recency window -# -M FILE org members, one npm username per line (# comments ok) — for `external` -# -b LIST comma-separated CI/bot accounts to exclude in `manual` (default netlify-bot) -# -g LIST comma-separated orgs (default netlify,gatsbyjs) -# -j N parallel fetch jobs (default 12) -# -f force rebuild of the recent-packages cache -# -h this help -# Examples: -# npm-audit.sh -O /tmp/out all -M /tmp/members.txt -# npm-audit.sh -O /tmp/out manual,external -M /tmp/members.txt -m 6 - -set -euo pipefail -PROG=$(basename "$0") - -usage() { sed -n '2,/^set -euo/{/^set -euo/d;s/^# \{0,1\}//;p;}' "$0"; exit "${1:-0}"; } -[ $# -eq 0 ] && usage 0 - -MONTHS=12; JOBS=12; BOTS="netlify-bot"; MEMBERS=""; OUTDIR=""; FORCE=0; ALL=0 -ORGS_CSV="netlify,gatsbyjs"; REPORTS="" -# Hand-rolled parser so flags and the REPORTS positional may appear in any order -# (getopts stops at the first non-option, which bites with `... all -M file`). -while [ $# -gt 0 ]; do - case "$1" in - -O) OUTDIR=$2; shift 2 ;; - -m) MONTHS=$2; shift 2 ;; - -M) MEMBERS=$2; shift 2 ;; - -b) BOTS=$2; shift 2 ;; - -g) ORGS_CSV=$2; shift 2 ;; - -j) JOBS=$2; shift 2 ;; - -f) FORCE=1; shift ;; - -A) ALL=1; shift ;; - -h) usage 0 ;; - -*) echo "error: unknown option $1" >&2; usage 2 ;; - *) REPORTS=$1; shift ;; - esac -done -[ -z "$REPORTS" ] && { echo "error: no REPORTS given (recent,manual,external or all)" >&2; usage 2; } -[ -z "$OUTDIR" ] && { echo "error: -O OUTDIR is required" >&2; usage 2; } -mkdir -p "$OUTDIR" - -[ "$REPORTS" = "all" ] && REPORTS="recent,manual,external" -IFS=',' read -r -a WANT <<<"$REPORTS" -IFS=',' read -r -a ORGS <<<"$ORGS_CSV" - -# --- shared helpers ------------------------------------------------------- - -CACHE="$OUTDIR/recent-packages.tsv" -FAILURES="$OUTDIR/fetch-failures.log" -: > "$FAILURES" -export FAILURES - -# epoch seconds for an ISO-8601 timestamp (GNU and BSD date). -to_epoch() { - local iso="$1" - if date -u -d "$iso" +%s 2>/dev/null; then return; fi - date -u -j -f "%Y-%m-%dT%H:%M:%S" "${iso%%.*}" +%s 2>/dev/null -} -export -f to_epoch - -# Fetch a URL with retry + backoff. 2xx -> body on stdout; 404 -> empty (ok); -# 429/5xx/network -> retry up to 5x, then record the URL in $FAILURES and fail. -npm_get() { - local url="$1" tries=5 i=0 code tmp - tmp=$(mktemp) - while :; do - i=$((i + 1)) - code=$(curl -sS -m 60 -o "$tmp" -w '%{http_code}' "$url" 2>/dev/null) || code=000 - case "$code" in - 2*) cat "$tmp"; rm -f "$tmp"; return 0 ;; - 404) rm -f "$tmp"; return 0 ;; - 429|5*|000) - if [ "$i" -ge "$tries" ]; then - printf '%s\t(http %s)\n' "$url" "$code" >> "$FAILURES"; rm -f "$tmp"; return 1 - fi - sleep $((i * i)) ;; # 1,4,9,16s backoff - *) printf '%s\t(http %s)\n' "$url" "$code" >> "$FAILURES"; rm -f "$tmp"; return 1 ;; - esac - done -} -export -f npm_get - -# Registry doc URL for a package name (scoped names get '/' -> %2f). -pkg_url() { printf 'https://registry.npmjs.org/%s' "$(printf '%s' "$1" | sed 's:/:%2f:')"; } -export -f pkg_url - -cutoff_epoch() { - if date -u -d "$MONTHS months ago" +%s 2>/dev/null; then return; fi - date -u -v-"${MONTHS}"m +%s -} -CUTOFF=$(cutoff_epoch); export CUTOFF -CUTOFF_ISO=$( date -u -d "$MONTHS months ago" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-"${MONTHS}"m +%Y-%m-%dT%H:%M:%SZ ); export CUTOFF_ISO - -# All packages across the configured orgs (deduped). Caps at 250/org (registry -# limit); private/unlisted packages aren't visible unauthenticated. -list_org_packages() { - local org - for org in "${ORGS[@]}"; do - npm_get "https://registry.npmjs.org/-/org/${org}/package" | jq -r 'if type=="object" then keys[] else empty end' - done | sort -u -} - -# Append a weekly-downloads column to $CACHE in place. The downloads point API -# (api.npmjs.org) is a strict token bucket — ~5 requests then 429, regardless of -# concurrency — but refills fine when paced. So: unscoped names use the BULK -# endpoint (100/request, no per-package cost); scoped names (bulk rejects them) -# go sequential, paced ~2/s. Missing/failed lookups become "?". -add_downloads() { - local names map; names=$(cut -f1 "$CACHE"); map=$(mktemp) - local n_scoped; n_scoped=$(echo "$names" | grep -c '^@' || true) - echo "[recent] fetching weekly downloads (bulk unscoped + $n_scoped scoped paced ~2/s)..." >&2 - # Unscoped: bulk, 100 per request. Response is keyed by name (or flat for 1). - echo "$names" | grep -v '^@' | xargs -n100 2>/dev/null | tr ' ' ',' | while read -r batch; do - [ -z "$batch" ] && continue - npm_get "https://api.npmjs.org/downloads/point/last-week/$batch" \ - | jq -r 'if has("downloads") then "\(.package)\t\(.downloads)" else (to_entries[] | "\(.key)\t\(.value.downloads // 0)") end' 2>/dev/null - done >> "$map" - # Scoped: sequential + paced to stay under the token bucket (bursting => 429s). - echo "$names" | grep '^@' | while read -r p; do - [ -z "$p" ] && continue - local d; d=$(npm_get "https://api.npmjs.org/downloads/point/last-week/$p" | jq -r '.downloads // empty' 2>/dev/null) - printf '%s\t%s\n' "$p" "${d:-?}" - sleep 0.5 - done >> "$map" - # Join the map onto the cache as the final column (default "?" if missing). - local tmp; tmp=$(mktemp) - awk -F'\t' 'FNR==NR{d[$1]=$2;next}{print $0"\t"(($1 in d)?d[$1]:"?")}' "$map" "$CACHE" > "$tmp" - mv "$tmp" "$CACHE"; rm -f "$map" -} - -warn_failures() { - local n; n=$(grep -c . "$FAILURES" 2>/dev/null) || n=0 - if [ "$n" -gt 0 ]; then - echo "WARNING: $n fetch(es) failed after retries — results may be INCOMPLETE." >&2 - echo " see $FAILURES" >&2 - fi -} - -# --- report: recent (build cache) ----------------------------------------- - -build_recent() { - if [ -f "$CACHE" ] && [ "$FORCE" -ne 1 ]; then - echo "[recent] using existing $CACHE ($(wc -l < "$CACHE" | tr -d ' ') pkgs); -f to rebuild" >&2 - return - fi - local scope_label; [ "$ALL" -eq 1 ] && scope_label="ALL org packages" || scope_label="last $MONTHS months" - echo "[recent] listing packages in: ${ORGS[*]}" >&2 - local pkgs; pkgs=$(list_org_packages) - echo "[recent] $(echo "$pkgs" | wc -l | tr -d ' ') packages; resolving latest version + recency + deprecated via fast-npm-meta..." >&2 - - # Discovery via fast-npm-meta (npm.antfu.dev): batch-resolve latest version, - # publishedAt and deprecated for every package without pulling heavy - # packuments. Emits: nameversionpublishedAtdeprecated(yes|no). - local meta; meta=$(printf '%s\n' "$pkgs" | xargs -n100 2>/dev/null | while read -r grp; do - [ -z "$grp" ] && continue - npm_get "https://npm.antfu.dev/$(printf '%s' "$grp" | tr ' ' '+')?metadata=true" \ - | jq -r '(if type=="array" then .[] else . end) - | [ .name, (.version // ""), (.publishedAt // ""), - (if .deprecated then "yes" else "no" end) ] | @tsv' 2>/dev/null - done) - - # In-scope set: ALL packages, or only those whose latest release is within the - # window. publishedAt is ISO-8601 UTC, so lexical compare against the cutoff - # ISO works. Drop rows with no resolvable version. - local inscope; inscope=$(mktemp) - if [ "$ALL" -eq 1 ]; then - awk -F'\t' '$2!=""' <<<"$meta" > "$inscope" - else - awk -F'\t' -v c="$CUTOFF_ISO" '$2!="" && $3>=c' <<<"$meta" > "$inscope" - fi - echo "[recent] in scope ($scope_label): $(wc -l < "$inscope" | tr -d ' ') packages; fetching per-version manifests for trust status..." >&2 - - # Per package: pull the lightweight per-version manifest (has _npmUser + - # dist.attestations; ~KBs vs MBs for the full packument) and compute trust. - # Trust logic ported verbatim from github.com/43081j/packumeta: - # provenance=dist.attestations.provenance truthy; trustedPublisher= - # _npmUser.trustedPublisher truthy; stagedPublish=_npmUser.approver truthy; - # level: stagedPublish > (trustedPublisher && provenance) > provenance > none - local ROWS; ROWS=$(mktemp); export ROWS - trust_row() { - local pkg="$1" ver="$2" pub="$3" dep="$4" doc - [ -z "$ver" ] && return 0 - doc=$(npm_get "https://registry.npmjs.org/$(printf '%s' "$pkg" | sed 's:/:%2f:')/$ver") || return 0 - [ -z "$doc" ] && return 0 - local level prov tp staged who - IFS=$'\t' read -r level prov tp staged who < <(printf '%s' "$doc" | jq -r ' - def truthy: (. != null) and (. != false); - (.dist.attestations.provenance | truthy) as $prov - | (._npmUser.trustedPublisher | truthy) as $tp - | (._npmUser.approver | truthy) as $staged - | (if $staged then "stagedPublish" - elif ($tp and $prov) then "trustedPublisher" - elif $prov then "provenance" else "none" end) as $level - | [ $level, - (if $prov then "yes" else "no" end), - (if $tp then "yes" else "no" end), - (if $staged then "yes" else "no" end), - (._npmUser.name // "?") ] | @tsv') - # Append to a shared file (atomic for short lines) rather than a pipe to sort, - # which avoids EINTR-on-write under heavy parallelism. - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$pkg" "$pub" "$ver" "$level" "$prov" "$tp" "$staged" "$who" "$dep" >> "$ROWS" - return 0 - } - export -f trust_row - # -L1: each line's tab/space-split fields become $1..$4 (names have no spaces). - xargs -P "$JOBS" -L1 bash -c 'trust_row "$@"' _ < "$inscope" - sort -t$'\t' -k2 -r "$ROWS" > "$CACHE" - rm -f "$inscope" "$ROWS" - - # Append weekly downloads (col10). The downloads API (api.npmjs.org) rate-limits - # HARD, unlike the registry — so fetch gently: one bulk request per ~100 - # unscoped names, and scoped names individually at low concurrency. - add_downloads - - # Coverage by packumeta trust level (col4). Cols: 5=prov 6=tp 7=staged - # 9=deprecated 10=downloads. - local total prov_n tp_n staged_n dep_n - total=$(wc -l < "$CACHE" | tr -d ' ') - prov_n=$(awk -F'\t' '$5=="yes"' "$CACHE" | wc -l | tr -d ' ') - tp_n=$(awk -F'\t' '$6=="yes"' "$CACHE" | wc -l | tr -d ' ') - staged_n=$(awk -F'\t' '$7=="yes"' "$CACHE" | wc -l | tr -d ' ') - dep_n=$(awk -F'\t' '$9=="yes"' "$CACHE" | wc -l | tr -d ' ') - { - echo "=== Supply-chain trust coverage on latest release ($scope_label) ===" - echo "orgs: ${ORGS[*]} | trust logic: github.com/43081j/packumeta" - echo - echo "Provenance attestation: $prov_n / $total" - echo "Trusted publisher (OIDC): $tp_n / $total" - echo "Staged publish (approver): $staged_n / $total" - echo "Deprecated (latest): $dep_n / $total" - echo - echo "--- Counts by trust level ---" - awk -F'\t' 'NF{print $4}' "$CACHE" | sort | uniq -c | sort -rn - echo - # Within each level, most-downloaded first; flag deprecated packages. - for lvl in stagedPublish trustedPublisher provenance none; do - echo "--- $lvl (by weekly downloads) ---" - awk -F'\t' -v L="$lvl" '$4==L{printf "%12s %-45s %-12s (%s)%s\n", $10, $1, $3, $8, ($9=="yes"?" [DEPRECATED]":"")}' "$CACHE" | sort -rn - echo - done - } > "$OUTDIR/trust-summary.txt" - echo "[recent] wrote $total packages to $CACHE" >&2 - echo "[recent] trust: provenance=$prov_n trustedPublisher=$tp_n stagedPublish=$staged_n deprecated=$dep_n of $total (see $OUTDIR/trust-summary.txt)" >&2 -} - -# --- report: manual -------------------------------------------------------- - -run_manual() { - [ -f "$CACHE" ] || build_recent - local out="$OUTDIR/manual-publishes.txt" - echo "[manual] scanning $(wc -l < "$CACHE" | tr -d ' ') packages for publishes in last $MONTHS months..." >&2 - scan_manual() { - local pkg="$1" cutoff="$2" - npm_get "$(pkg_url "$pkg")" | jq -r ' - . as $d | (.versions // {}) | to_entries[] - | ($d.time[.key]) as $t | select($t != null) - | "\($t)\t\(.value._npmUser.name // "?")\t\($d.name)@\(.key)"' 2>/dev/null \ - | while IFS=$'\t' read -r iso who ref; do - local e; e=$(to_epoch "$iso") || continue - [ "$e" -ge "$cutoff" ] && printf '%s\t%s\t%s\n' "$iso" "$who" "$ref" - done - return 0 - } - export -f scan_manual - local rows; rows=$(mktemp) - cut -f1 "$CACHE" | xargs -P "$JOBS" -I {} bash -c 'scan_manual "$@"' _ {} "$CUTOFF" | sort -r > "$rows" - - local botre human total hn - botre=$(printf '%s' "$BOTS" | tr ',' '|') - human=$(awk -F'\t' -v b="^($botre)$" '$2 !~ b' "$rows") - total=$(wc -l < "$rows" | tr -d ' ') - hn=$(printf '%s' "$human" | grep -c . || true) - { - echo "=== Manual (non-CI) publishes — last $MONTHS months ===" - echo "Total publishes scanned: $total | excluded bots: $BOTS | manual: $hn" - echo - echo "--- By publisher (count) ---" - printf '%s\n' "$human" | awk -F'\t' 'NF{print $2}' | sort | uniq -c | sort -rn - echo - echo "--- Detail: ---" - printf '%s\n' "$human" | awk -F'\t' 'NF{printf "%s %-20s %s\n", $1, $2, $3}' - } > "$out" - rm -f "$rows" - echo "[manual] wrote $out ($hn manual publishes)" >&2 -} - -# --- report: external ------------------------------------------------------ - -run_external() { - [ -n "$MEMBERS" ] || { echo "[external] SKIPPED: -M members.txt required (org membership isn't public)" >&2; return 1; } - [ -f "$MEMBERS" ] || { echo "[external] error: members file not found: $MEMBERS" >&2; return 1; } - local out="$OUTDIR/external-maintainers.txt" - local mf; mf=$(mktemp) - grep -vE '^[[:space:]]*(#|$)' "$MEMBERS" | tr '[:upper:]' '[:lower:]' \ - | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | sort -u > "$mf" - - echo "[external] listing all packages in: ${ORGS[*]}" >&2 - local pkgs; pkgs=$(list_org_packages) - echo "[external] scanning $(echo "$pkgs" | wc -l | tr -d ' ') packages for current maintainers..." >&2 - scan_maint() { - npm_get "$(pkg_url "$1")" | jq -r '.name as $n | (.maintainers // [])[] | "\(.name)\t\($n)"' 2>/dev/null - return 0 - } - export -f scan_maint - local pairs; pairs=$(echo "$pkgs" | xargs -P "$JOBS" -I {} bash -c 'scan_maint "$@"' _ {} | sort -u) - local external; external=$(awk -F'\t' 'FNR==NR{m[$1]=1;next}{if(!(tolower($1) in m))print}' "$mf" <(printf '%s\n' "$pairs")) - local nu; nu=$(awk -F'\t' 'NF{print $1}' <<<"$external" | sort -u | grep -c . || true) - { - echo "=== External maintainers (publish access NOW, not an org member) ===" - echo "$nu distinct external maintainer account(s)." - echo - echo "--- By user (package count) ---" - awk -F'\t' 'NF{print $1}' <<<"$external" | sort | uniq -c | sort -rn - echo - echo "--- Detail: ---" - sort <<<"$external" | awk -F'\t' 'NF{printf "%-22s %s\n", $1, $2}' - } > "$out" - rm -f "$mf" - echo "[external] wrote $out ($nu external maintainers)" >&2 -} - -# --- dispatch -------------------------------------------------------------- - -for r in "${WANT[@]}"; do - case "$r" in - recent) build_recent ;; - manual) run_manual ;; - external) run_external || true ;; - *) echo "error: unknown report '$r' (want: recent,manual,external,all)" >&2; exit 2 ;; - esac -done -warn_failures -echo "Done. Output in $OUTDIR/" >&2 diff --git a/scripts/npm-user-publishes.sh b/scripts/npm-user-publishes.sh deleted file mode 100644 index 7a99268..0000000 --- a/scripts/npm-user-publishes.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash -# List versions that a specific npm user PERSONALLY published (recorded as the -# version's _npmUser) within the last N months. -# -# Package universe = the user's own maintained packages (/-/user//package, -# the authoritative, uncapped source) UNION an optional cache file (-i) of org -# packages, so nothing the user can touch is missed. -# -# _npmUser is the account whose auth ran `npm publish` — INCLUDING that account's -# automation/CI tokens. npm does not distinguish interactive logins from tokens. -# -# Usage: -# npm-user-publishes.sh [-m MONTHS] [-i CACHEFILE] [-j JOBS] USERNAME -# -i CACHEFILE optional TSV with a package name in column 1 (e.g. an -# npm-audit.sh recent-packages.tsv); merged into the scan set. -# Output TSV: "\t@" (newest first). - -set -euo pipefail - -MONTHS=12; JOBS=12; CACHE="" -while getopts "m:i:j:" opt; do - case "$opt" in - m) MONTHS=$OPTARG ;; - i) CACHE=$OPTARG ;; - j) JOBS=$OPTARG ;; - *) echo "usage: $0 [-m MONTHS] [-i CACHEFILE] [-j JOBS] USERNAME" >&2; exit 2 ;; - esac -done -shift $((OPTIND - 1)) - -USERNAME=${1:-} -[ -z "$USERNAME" ] && { echo "error: USERNAME required" >&2; exit 2; } - -FAILURES=$(mktemp); export FAILURES -UNIVERSE=$(mktemp) -trap 'rm -f "$UNIVERSE" "$FAILURES"' EXIT - -# Fetch with retry + backoff. 2xx -> body; 404 -> empty (ok); 429/5xx/network -> -# retry up to 5x then record the URL in $FAILURES and fail (so partial runs are -# visible rather than silently empty). -npm_get() { - local url="$1" tries=5 i=0 code tmp; tmp=$(mktemp) - while :; do - i=$((i + 1)) - code=$(curl -sS -m 60 -o "$tmp" -w '%{http_code}' "$url" 2>/dev/null) || code=000 - case "$code" in - 2*) cat "$tmp"; rm -f "$tmp"; return 0 ;; - 404) rm -f "$tmp"; return 0 ;; - 429|5*|000) - if [ "$i" -ge "$tries" ]; then printf '%s\t(http %s)\n' "$url" "$code" >> "$FAILURES"; rm -f "$tmp"; return 1; fi - sleep $((i * i)) ;; - *) printf '%s\t(http %s)\n' "$url" "$code" >> "$FAILURES"; rm -f "$tmp"; return 1 ;; - esac - done -} -export -f npm_get - -to_epoch() { - local iso="$1" - if date -u -d "$iso" +%s 2>/dev/null; then return; fi - date -u -j -f "%Y-%m-%dT%H:%M:%S" "${iso%%.*}" +%s 2>/dev/null -} -export -f to_epoch - -CUTOFF=$( date -u -d "$MONTHS months ago" +%s 2>/dev/null || date -u -v-"${MONTHS}"m +%s ) - -{ - npm_get "https://registry.npmjs.org/-/user/${USERNAME}/package" | jq -r 'if type=="object" then keys[] else empty end' 2>/dev/null - [ -n "$CACHE" ] && [ -f "$CACHE" ] && cut -f1 "$CACHE" -} | sort -u > "$UNIVERSE" - -scan_pkg() { - local pkg="$1" user="$2" cutoff="$3" url - url="https://registry.npmjs.org/$(printf '%s' "$pkg" | sed 's:/:%2f:')" - npm_get "$url" | jq -r --arg u "$user" ' - . as $d | (.versions // {}) | to_entries[] - | select(.value._npmUser.name == $u) - | ($d.time[.key]) as $t | select($t != null) - | "\($t)\t\($d.name)@\(.key)"' 2>/dev/null \ - | while IFS=$'\t' read -r iso ref; do - local e; e=$(to_epoch "$iso") || continue - [ "$e" -ge "$cutoff" ] && printf '%s\t%s\n' "$iso" "$ref" - done - return 0 -} -export -f scan_pkg - -echo "Scanning $(wc -l < "$UNIVERSE" | tr -d ' ') packages (user's own + cache) for versions published by '$USERNAME' (last $MONTHS months)..." >&2 -xargs -P "$JOBS" -I {} bash -c 'scan_pkg "$@"' _ {} "$USERNAME" "$CUTOFF" < "$UNIVERSE" | sort -r - -n=$(grep -c . "$FAILURES" 2>/dev/null) || n=0 -[ "$n" -gt 0 ] && { echo "WARNING: $n fetch(es) failed after retries — results may be INCOMPLETE:" >&2; cat "$FAILURES" >&2; } -echo "Done." >&2 diff --git a/src/App.svelte b/src/App.svelte index 8c0ad52..e8099f0 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -224,7 +224,7 @@ reportSaveError = null; terminal?.clear(); log(`→ audit ${orgs.join(", ")} | reports: ${selectedKinds.join(",")}`); - log(all ? "→ scope: ALL org packages (-A)" : `→ scope: last ${months} months`); + log(all ? "→ scope: ALL org packages" : `→ scope: last ${months} months`); try { // The audit runs server-side; we stream its progress into the terminal and diff --git a/src/App.test.ts b/src/App.test.ts index 0316205..d685cb5 100644 --- a/src/App.test.ts +++ b/src/App.test.ts @@ -6,7 +6,7 @@ import { auditResult } from "./test/fixtures"; import { streamAudit } from "./lib/auditStream"; import { streamUserPublishes } from "./lib/userPublishStream"; -// Both the audit and user-publishes now run server-side; the client streams them. +// Both server-side workflows stream through these client adapters. vi.mock("./lib/auditStream", () => ({ streamAudit: vi.fn() })); vi.mock("./lib/userPublishStream", () => ({ streamUserPublishes: vi.fn() })); diff --git a/src/components/LogTerminal.svelte b/src/components/LogTerminal.svelte index ef53fce..46b75d3 100644 --- a/src/components/LogTerminal.svelte +++ b/src/components/LogTerminal.svelte @@ -1,12 +1,9 @@