Something went wrong. Try again.
notification manager for bsky noti.waow.tech
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275import {estimateUsd} from '../src/pricing'// Evals for the ask-noti propose path. These call the real model — that is the point. Testing the// pipeline with an injected model response tells you the plumbing works; it cannot tell you// whether "mute alex" becomes a mute of the right account, which is the product.//// bun run eval:propose # every case, 3 samples each// bun run eval:propose --samples 5 # more samples where a case is borderline// bun run eval:propose ambiguous-actor # one case by name//// Determinism does not come from the model. Sonnet 5 rejects non-default sampling parameters, so// temperature cannot be pinned to 0 — assertions are therefore written as invariants that hold// across samplings (which method, which actor, which kind), never as expected text. Where a case// is genuinely borderline, it is sampled N times and must clear a threshold; a case that cannot// clear its threshold is a finding about the prompt, not a flake to retry away.//// Cost: every case shares one system prefix, so the first call pays the cache write and the rest// read at 0.1x. A full run is reported in USD at the end and is well under a dollar.
import {existsSync, readFileSync} from 'node:fs'import {join} from 'node:path'
import {createRecommender} from '../src/recommend'import type {ActionProposal, ManagementTarget, UserList} from '../src/types'import {actor, EVAL_USER_DID, list} from '../test/fixtures/account'
type Check = {name: string; pass: (p: ActionProposal) => boolean}
type Case = { name: string what: string ask: string targets: ManagementTarget[] lists?: UserList[] thread?: {role: 'user' | 'noti'; text: string; kind?: string; createdAt: string}[] checks: Check[] /** fraction of samples that must pass. defaults to 1 — most invariants should always hold. */ threshold?: number}
// ── shared predicates ─────────────────────────────────────────────────────────────────────────const calls = (method: string) => (p: ActionProposal) => Boolean(p.recommendation?.code.includes(method))const kindIs = (kind: ActionProposal['kind']) => (p: ActionProposal) => p.kind === kindconst namesActor = (did: string) => (p: ActionProposal) => Boolean(p.recommendation?.actorDids?.includes(did))const mentionsNoActor = (did: string) => (p: ActionProposal) => !p.recommendation?.actorDids?.includes(did) && !p.message.includes(did)
// The failure that started this: noti told a user "list creation doesn't return a usable reference// right away, you'll need to ask again afterward". Any of these words in a user-facing message// means it is describing its own machinery instead of the change.const INTERNALS = /\b(proposal|tool call|sandbox|code|URI|reference|API|endpoint|schema|function|parameter)\b/iconst noInternals: Check = { name: 'message does not describe noti’s internals', pass: p => !INTERNALS.test(p.message),}
// Nothing in a proposal has happened yet. Claiming otherwise is the difference between "this will// mute them" and "I muted them", and the user cannot tell which state they are in.const NOT_YET_DONE = /\b(I (?:have |'ve )?(?:created|muted|blocked|added|removed|turned off)|has been (?:created|added|muted)|is now (?:created|added|muted))\b/iconst futureTense: Check = { name: 'does not claim the action already happened', pass: p => !NOT_YET_DONE.test(p.message),}
// ── the cases ─────────────────────────────────────────────────────────────────────────────────const LOUD = actor({handle: 'loud.example', name: 'Loud', recent24h: 30, recent7d: 90, replies: 4, unread: 12})const ALEX_ONE = actor({handle: 'alex.bsky.team', name: 'Alex', recent7d: 5})const ALEX_TWO = actor({handle: 'alex.example.com', name: 'Alex Two', recent7d: 3})const MUTUAL = actor({handle: 'friend.example', name: 'Friend', follows: true, followedBy: true, recent7d: 8, subscribedPosts: true, subscribedReplies: true})const CLOSED = actor({handle: 'closed.example', name: 'Closed', subscriptionPolicy: 'mutuals', subscriptionEligible: false, recent7d: 2})
const CASES: Case[] = [ { name: 'mute-by-handle', what: 'an unambiguous mute names the right actor and calls muteActor', ask: 'mute @loud.example', targets: [LOUD, MUTUAL], checks: [ {name: 'kind is action', pass: kindIs('action')}, {name: 'calls muteActor', pass: calls('muteActor')}, {name: 'names loud.example', pass: namesActor(LOUD.actorDid)}, noInternals, futureTense, ], }, { name: 'ambiguous-actor', what: 'two accounts match the name, so noti asks instead of guessing', ask: 'mute alex', targets: [ALEX_ONE, ALEX_TWO, LOUD], checks: [ {name: 'kind is clarify', pass: kindIs('clarify')}, {name: 'offers options to pick from', pass: p => (p.examples?.length ?? 0) >= 2}, {name: 'mutes nobody yet', pass: p => !p.recommendation}, noInternals, ], // Guessing one of two identically-named accounts is the failure this case exists to catch, but // a model that instead refuses is wrong in a less dangerous way. Threshold, not absolute. threshold: 0.8, }, { name: 'answers-its-own-question', what: 'given the earlier turns, a bare answer resolves to an action without re-asking', ask: 'the first one', targets: [ALEX_ONE, ALEX_TWO], thread: [ {role: 'user', text: 'mute alex', createdAt: ''}, {role: 'noti', text: 'Which account? @alex.bsky.team or @alex.example.com?', kind: 'clarify', createdAt: ''}, ], checks: [ {name: 'kind is action', pass: kindIs('action')}, {name: 'does not ask again', pass: p => p.kind !== 'clarify'}, {name: 'names alex.bsky.team', pass: namesActor(ALEX_ONE.actorDid)}, noInternals, ], threshold: 0.8, }, { name: 'create-list', what: 'a list request produces an applicable action, not prose about how lists work', ask: 'can you make a core atproto dev list', targets: [LOUD], lists: [], checks: [ {name: 'kind is action', pass: kindIs('action')}, {name: 'calls createList', pass: calls('createList')}, {name: 'produces something to apply', pass: p => Boolean(p.recommendation?.code)}, noInternals, futureTense, ], }, { name: 'add-to-existing-list', what: 'adding to a list the user keeps targets that list, and does not mint a new one', ask: 'add @loud.example to zig people', targets: [LOUD], lists: [list('zig people', 4)], checks: [ {name: 'kind is action', pass: kindIs('action')}, {name: 'calls addToList', pass: calls('addToList')}, {name: 'does not create a list', pass: p => !p.recommendation?.code.includes('createList')}, {name: 'uses the real list uri', pass: p => Boolean(p.recommendation?.code.includes('app.bsky.graph.list/'))}, noInternals, ], }, { name: 'ineligible-subscription', what: 'an actor who only allows mutuals cannot be subscribed to, and noti says so', ask: 'subscribe to posts from @closed.example', targets: [CLOSED], checks: [ {name: 'does not propose a subscription', pass: p => !p.recommendation?.code.includes('putActivitySubscription')}, {name: 'explains rather than acts', pass: p => p.kind === 'refusal' || p.kind === 'clarify'}, noInternals, ], }, { name: 'help-is-not-an-action', what: 'asking what noti does returns help, never a mutation', ask: 'what can you do?', targets: [LOUD], checks: [ {name: 'kind is help', pass: kindIs('help')}, {name: 'proposes nothing', pass: p => !p.recommendation}, {name: 'suggests something to try', pass: p => (p.examples?.length ?? 0) > 0}, {name: 'leaves the loud account alone', pass: mentionsNoActor(LOUD.actorDid)}, ], },]
// ── runner ────────────────────────────────────────────────────────────────────────────────────function devVar(name: string): string | undefined { const path = join(import.meta.dir, '..', '.dev.vars') if (!existsSync(path)) return undefined for (const line of readFileSync(path, 'utf8').split('\n')) { const match = line.match(/^\s*([A-Z_]+)\s*=\s*(.*)$/) if (match?.[1] === name) return match[2].trim().replace(/^["']|["']$/g, '') } return undefined}
async function main() { const apiKey = (process.env.ANTHROPIC_API_KEY || devVar('ANTHROPIC_API_KEY'))?.trim() if (!apiKey) { console.error('ANTHROPIC_API_KEY not found in the environment or .dev.vars') process.exit(1) }
const argv = process.argv.slice(2) let samples = 3 const names: string[] = [] for (let i = 0; i < argv.length; i++) { if (argv[i] === '--samples') { samples = Number(argv[++i]) || 3 continue } names.push(argv[i]) } const cases = names.length ? CASES.filter(c => names.includes(c.name)) : CASES if (!cases.length) { console.error(`no case matched ${names.join(', ')}. known: ${CASES.map(c => c.name).join(', ')}`) process.exit(1) }
let costUsd = 0 let totalMs = 0 let callCount = 0 const recommender = createRecommender({ apiKey, model: process.env.ANTHROPIC_MODEL, onUsage: event => { callCount += 1 totalMs += event.durationMs costUsd += estimateUsd(event.model, event.usage?.input_tokens ?? 0, event.usage?.cache_creation_input_tokens ?? 0, event.usage?.cache_read_input_tokens ?? 0, event.usage?.output_tokens ?? 0) }, })
console.log(`propose evals: ${cases.length} case(s) × ${samples} sample(s), model ${process.env.ANTHROPIC_MODEL || 'claude-sonnet-5'}\n`)
const failures: string[] = [] for (const testCase of cases) { const threshold = testCase.threshold ?? 1 const passes = new Map<string, number>() const notes: string[] = []
for (let sample = 0; sample < samples; sample++) { let proposal: ActionProposal try { proposal = await recommender.proposeAction( EVAL_USER_DID, testCase.ask, [], testCase.targets, testCase.lists ?? [], testCase.thread ?? [], ) } catch (error) { notes.push(`sample ${sample + 1} threw: ${error instanceof Error ? error.message : String(error)}`) continue } for (const check of testCase.checks) { const ok = check.pass(proposal) passes.set(check.name, (passes.get(check.name) ?? 0) + (ok ? 1 : 0)) // One failing sample is the interesting artifact — print what it actually said, so a // failure is diagnosable without a rerun. if (!ok && !notes.some(n => n.startsWith(`✗ ${check.name}`))) { notes.push(`✗ ${check.name} — kind=${proposal.kind} msg=${JSON.stringify(proposal.message.slice(0, 120))}`) } } }
const failed = testCase.checks.filter(check => (passes.get(check.name) ?? 0) / samples < threshold) const mark = failed.length ? '✗' : '✓' console.log(`${mark} ${testCase.name} — ${testCase.what}`) for (const check of testCase.checks) { const rate = (passes.get(check.name) ?? 0) / samples const bad = rate < threshold console.log(` ${bad ? '✗' : '·'} ${check.name} ${Math.round(rate * 100)}%${threshold < 1 ? ` (need ${Math.round(threshold * 100)}%)` : ''}`) } for (const note of notes) console.log(` ${note}`) if (failed.length) failures.push(`${testCase.name}: ${failed.map(f => f.name).join(', ')}`) console.log() }
console.log(`${callCount} calls · ${(totalMs / Math.max(callCount, 1) / 1000).toFixed(1)}s mean · $${costUsd.toFixed(3)} total`) if (failures.length) { console.error(`\n${failures.length} case(s) failed:`) for (const failure of failures) console.error(` ${failure}`) process.exit(1) } console.log('\nall cases passed')}
await main()