diff --git a/system/netlify/functions/oskiewar-stats.mjs b/system/netlify/functions/oskiewar-stats.mjs index a66456d48d..cf63a5020d 100644 --- a/system/netlify/functions/oskiewar-stats.mjs +++ b/system/netlify/functions/oskiewar-stats.mjs @@ -28,6 +28,12 @@ export { localDay }; const REPLAYS = "oskiewar-replays"; const POPS = "oskiewar-pops"; const POP_DAYS = "oskiewar-pop-days"; +// Reel figures reach here twice over. The ledger file is the record a human +// reads, but it only changes on the machine that publishes and only arrives on +// lith with a deploy — so a nightly insights pull would sit in git while the wall +// kept quoting whatever the last deploy happened to carry. The clockwork posts +// them here as well, and this collection wins when it has anything to say. +const REEL_INSIGHTS = "oskiewar-reel-insights"; const HOUR = 60 * 60 * 1000; const DAY = 24 * HOUR; // The wall shows a month of history and a day of detail. @@ -82,10 +88,30 @@ export function shapeHourly(rows, hours = HOURLY_HOURS) { return out; } -// Sum the reels the ledger says actually went out. `insights` is null until -// something pulls it, so a reel that has never been measured contributes its -// existence and nothing else — the wall says "2 live, 0 measured" rather than -// quietly reporting zero views as though that were a result. +// Stored rows and ledger rows are merged on id, with the stored insights winning +// because they are the ones that can be newer than the deploy. A reel the ledger +// has never heard of still counts — it was published, whatever this checkout +// knows — so the union is taken rather than the ledger's list alone. +export function mergeReelSources(ledger, stored = []) { + const posts = Array.isArray(ledger?.posts) ? ledger.posts : []; + const byId = new Map(); + for (const post of posts) { + if (post.mode === "live") byId.set(post.id, { ...post }); + } + for (const row of stored) { + const existing = byId.get(row._id) || { + mode: "live", id: row._id, day: row.day, segment: row.segment, + publishedAt: row.publishedAt, urls: { cover: row.cover }, + }; + byId.set(row._id, { ...existing, insights: row.insights, insightsAt: row.insightsAt }); + } + return { posts: [...byId.values()] }; +} + +// Sum the reels that actually went out. `insights` is null until something pulls +// it, so a reel that has never been measured contributes its existence and +// nothing else — the wall says "2 live, 0 measured" rather than quietly reporting +// zero views as though that were a result. export function shapeReels(ledger) { const posts = Array.isArray(ledger?.posts) ? ledger.posts : []; const live = posts.filter((post) => post.mode === "live"); @@ -146,12 +172,57 @@ function readLedger() { } } +// The clockwork's half of the loop: whatever it pulled from Meta is upserted per +// reel, keyed by the ledger's own id. Only the wall key opens this, since it is +// the same secret that is allowed to read the figures back out. +async function storeReelInsights(database, posts) { + const rows = (Array.isArray(posts) ? posts : []) + .filter((post) => post && typeof post.id === "string" && post.insights); + if (!rows.length) return 0; + const collection = database.db.collection(REEL_INSIGHTS); + await collection.bulkWrite(rows.map((post) => ({ + updateOne: { + filter: { _id: post.id }, + update: { + $set: { + day: post.day ?? null, + segment: post.segment ?? null, + publishedAt: post.publishedAt ?? null, + cover: post.urls?.cover ?? post.cover ?? null, + insights: post.insights, + insightsAt: post.insightsAt || new Date().toISOString(), + }, + }, + upsert: true, + }, + })), { ordered: false }); + return rows.length; +} + export async function handler(event) { if (event.httpMethod === "OPTIONS") return respond(204, ""); + + if (event.httpMethod === "POST") { + if (!reelsPermitted(event)) + return respond(403, { error: "Wall key required" }); + let database; + try { + const body = JSON.parse(event.body || "{}"); + database = await connect(); + const stored = await storeReelInsights(database, body.posts); + await database.disconnect(); + return respond(200, { ok: true, stored }); + } catch (error) { + if (database) await database.disconnect(); + return respond(500, { error: error.message || "Insight storage failed" }); + } + } + if (event.httpMethod !== "GET") return respond(405, { error: "Method not allowed" }); const timezone = event.queryStringParameters?.tz || WALL_TIMEZONE; + const wantsReels = reelsPermitted(event); let database; try { database = await connect(); @@ -160,7 +231,7 @@ export async function handler(event) { const key = dayKey("$recordedAt", timezone); const replays = database.db.collection(REPLAYS); - const [facets, popRows, popDays] = await Promise.all([ + const [facets, popRows, popDays, storedReels] = await Promise.all([ replays.aggregate([{ $facet: { // Everything ever recorded, in one pass. @@ -247,6 +318,10 @@ export async function handler(event) { // and the wall labels it rather than passing it off as all-time. database.db.collection(POP_DAYS).find({}) .sort({ _id: 1 }).limit(400).toArray(), + // Only worth fetching for a caller who may read them back out. + wantsReels + ? database.db.collection(REEL_INSIGHTS).find({}).limit(400).toArray() + : Promise.resolve([]), ]); await database.disconnect(); @@ -319,7 +394,9 @@ export async function handler(event) { }, hourly: shapeHourly(facet.hourly || []), daily, - reels: reelsPermitted(event) ? shapeReels(readLedger()) : null, + reels: wantsReels + ? shapeReels(mergeReelSources(readLedger(), storedReels)) + : null, }, { // The wall polls this; a minute stale costs nothing and keeps a reload // storm off the database. diff --git a/toolchain/instagram/oskiewar-clockwork.sh b/toolchain/instagram/oskiewar-clockwork.sh new file mode 100755 index 0000000000..2da2e4b20c --- /dev/null +++ b/toolchain/instagram/oskiewar-clockwork.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# The oskiewar reel clockwork, as cron should call it. +# +# oskiewar-clockwork publish 0 # bake slot 0 on the oven and post it +# oskiewar-clockwork insights # pull Meta's figures onto the ledger +# +# This exists because the crontab used to call `bash -lc '... node ...'` directly +# and every part of that was wrong in a way that failed silently: +# +# 1. node was not on the PATH. fnm is wired into fish, not into a bash login +# shell, so `node` simply did not exist and three publishes a day died +# before they started. Worse, `which node` in an interactive shell reports +# an fnm *multishell* path under a PID directory, which is per-session and +# would rot the moment it was pasted into a crontab. So node is resolved +# here, from the installations themselves. +# 2. Nothing updated the automation checkout, so it ran whatever it happened +# to be sitting at. It is fast-forwarded to origin/main first. +# 3. Publishing writes the ledger next to reel.mjs (publish.mjs resolves it +# from import.meta.url), so a checkout without a committed ledger records +# what it published into a file nobody reads. The update in step 2 brings +# the real one in. +# 4. A publish mutates that ledger and nothing pushed it, so lith — which +# serves the reel figures to the wall from the file on disk — never learned +# about a new reel. It is committed and pushed here. +# 5. None of the above announced itself. Every run now leaves a timestamped +# log line and a heartbeat with the exit code in it. +set -uo pipefail + +MODE="${1:-publish}" +INDEX="${2:-0}" +CHECKOUT="${OSKIEWAR_CLOCKWORK_CHECKOUT:-$HOME/.local/share/ac-automation}" +IG_ENV="${OSKIEWAR_IG_ENV:-$HOME/aesthetic-computer-vault/oskiewar/instagram.env}" +STATE="${OSKIEWAR_CLOCKWORK_STATE:-$HOME/.local/state}" +LOG="$STATE/oskiewar-reels.log" +BEAT="$STATE/oskiewar-clockwork.json" +LEDGER="xbox/live/marketing/ledger.json" + +mkdir -p "$STATE" +STARTED="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +say() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOG"; } + +# The heartbeat is written on every exit path, including the failures, because a +# run that dies without saying so is the whole reason this script exists. +STAGE="starting" +finish() { + local code=$? + printf '{"format":"ac.oskiewar.clockwork","mode":"%s","index":%s,"startedAt":"%s","finishedAt":"%s","exitCode":%d,"stage":"%s","ok":%s}\n' \ + "$MODE" "$INDEX" "$STARTED" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$code" "$STAGE" \ + "$([ $code -eq 0 ] && echo true || echo false)" > "$BEAT" + if [ $code -ne 0 ]; then + say "✗ FAILED at stage '$STAGE' (exit $code) · see $LOG" + # Best effort only: cron has no GUI session on every machine, and a missing + # notifier must not turn a reel failure into a script failure. + osascript -e "display notification \"oskiewar $MODE $INDEX failed at $STAGE\" with title \"reel clockwork\"" 2>/dev/null || true + fi + exit $code +} +trap finish EXIT + +# --- node, resolved from installations rather than from a shell's PATH --- +STAGE="resolving node" +NODE="" +for candidate in \ + $(ls -d "$HOME"/.local/share/fnm/node-versions/*/installation/bin/node 2>/dev/null | sort -V -r) \ + /opt/homebrew/bin/node /usr/local/bin/node /usr/bin/node; do + if [ -x "$candidate" ]; then NODE="$candidate"; break; fi +done +[ -n "$NODE" ] || { say "no usable node found"; exit 1; } + +# --- checkout, brought up to what was actually shipped --- +STAGE="updating checkout" +[ -d "$CHECKOUT/.git" ] || [ -f "$CHECKOUT/.git" ] || { say "no checkout at $CHECKOUT"; exit 1; } +# A detached automation checkout holds nothing worth keeping; a checkout sitting +# on a branch with real edits might, so that case stops rather than resetting. +if [ -n "$(git -C "$CHECKOUT" branch --show-current 2>/dev/null)" ]; then + say "checkout is on a branch, not detached — refusing to reset it"; exit 1 +fi +git -C "$CHECKOUT" fetch origin main --quiet || { say "fetch failed"; exit 1; } +git -C "$CHECKOUT" reset --hard origin/main --quiet || { say "reset failed"; exit 1; } +say "▸ $MODE $INDEX · node $("$NODE" --version) · $(git -C "$CHECKOUT" rev-parse --short HEAD)" + +[ -d "$CHECKOUT/node_modules" ] || say "warning: $CHECKOUT/node_modules is missing; deps may fail" + +# --- credentials --- +STAGE="loading credentials" +[ -f "$IG_ENV" ] || { say "no IG env at $IG_ENV"; exit 1; } +set -a +# shellcheck disable=SC1090 +. "$IG_ENV" +[ -f "$HOME/aesthetic-computer-vault/oskiewar/wall.env" ] && . "$HOME/aesthetic-computer-vault/oskiewar/wall.env" +set +a + +cd "$CHECKOUT" || exit 1 +BEFORE="$(git -C "$CHECKOUT" hash-object "$LEDGER" 2>/dev/null || echo none)" + +case "$MODE" in + publish) + STAGE="baking and publishing slot $INDEX" + "$NODE" toolchain/instagram/oskiewar-oven.mjs --index "$INDEX" --auto 2>&1 | tee -a "$LOG" + status=${PIPESTATUS[0]} + [ "$status" -eq 0 ] || exit "$status" + ;; + insights) + STAGE="pulling reel insights" + "$NODE" xbox/live/marketing/reel.mjs --insights 2>&1 | tee -a "$LOG" + status=${PIPESTATUS[0]} + [ "$status" -eq 0 ] || exit "$status" + ;; + *) + say "unknown mode '$MODE' (want: publish | insights)"; exit 2 ;; +esac + +# --- send the figures where the wall can see them now --- +# The ledger only reaches lith with a deploy, so git alone would leave the wall +# quoting the last deploy's numbers indefinitely. Posting them costs one request +# and makes the nightly refresh actually visible. +STAGE="posting insights to lith" +if [ -n "${OSKIEWAR_WALL_KEY:-}" ] && [ -f "$LEDGER" ]; then + posted=$("$NODE" -e ' + const { readFileSync } = await import("node:fs"); + const ledger = JSON.parse(readFileSync(process.argv[1], "utf8")); + const posts = (ledger.posts || []).filter((post) => post.insights); + if (!posts.length) { console.log("none"); process.exit(0); } + const host = process.env.OSKIEWAR_STATS_HOST || "https://oskiewar.com"; + const response = await fetch( + `${host}/api/oskiewar-stats?key=${encodeURIComponent(process.env.OSKIEWAR_WALL_KEY)}`, + { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ posts }) }); + const body = await response.json().catch(() => ({})); + if (!response.ok) { console.error(body.error || `HTTP ${response.status}`); process.exit(1); } + console.log(`stored ${body.stored}`); + ' "$LEDGER" 2>&1) && say "insights → lith: $posted" \ + || say "warning: could not post insights to lith ($posted)" +else + say "no wall key — skipping the post to lith" +fi + +# --- close the loop in git too: the ledger is the record a human reads --- +STAGE="publishing the ledger" +AFTER="$(git -C "$CHECKOUT" hash-object "$LEDGER" 2>/dev/null || echo none)" +if [ "$BEFORE" = "$AFTER" ]; then + say "ledger unchanged" +else + git -C "$CHECKOUT" add "$LEDGER" + git -C "$CHECKOUT" -c user.name="oskiewar clockwork" \ + -c user.email="clockwork@aesthetic.computer" \ + commit -q -m "Record oskiewar reel $MODE $INDEX" -- "$LEDGER" || { + say "nothing to commit"; exit 0; } + # The knot takes pushes from several machines, so losing a race here is normal + # and worth retrying rather than failing the whole run over. + pushed=0 + for attempt in 1 2 3; do + git -C "$CHECKOUT" fetch origin main --quiet + git -C "$CHECKOUT" rebase origin/main --quiet >/dev/null 2>&1 || { + git -C "$CHECKOUT" rebase --abort >/dev/null 2>&1 || true + say "rebase onto origin/main failed"; break; } + if git -C "$CHECKOUT" push origin HEAD:main --quiet 2>/dev/null; then + pushed=1; say "ledger pushed ($(git -C "$CHECKOUT" rev-parse --short HEAD))"; break + fi + say "push race on attempt $attempt, retrying" + done + [ "$pushed" -eq 1 ] || { say "could not push the ledger"; exit 1; } +fi + +STAGE="done" +say "✓ $MODE $INDEX complete"