diff --git a/package.json b/package.json index aceaabc..21be203 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "assets": "npm run icons && npm run wordmark && npm run cws && npm run og && npm run shots", "package": "npm run build && npm run verify:dist", "rc": "node scripts/rc-load.mjs", + "smoke": "node scripts/smoke-test.mjs", "package:dev": "npm run build:dev && npm run verify:dist", "bump": "node scripts/version-bump.mjs", "test": "vitest run", diff --git a/scripts/smoke-test.mjs b/scripts/smoke-test.mjs new file mode 100644 index 0000000..671a1a1 --- /dev/null +++ b/scripts/smoke-test.mjs @@ -0,0 +1,194 @@ +// Does the built extension actually detect publications? +// +// The unit tests cover every module in isolation and the whole of v1.2.1 +// still shipped with sign-in dead, because what broke was the wiring between +// them. This is the other end of that: load the real build into a real +// Chrome, visit real publications over the real network, and assert the state +// the worker ends up in. +// +// Each site is here because it reaches detection by a different route, so a +// pass means the routes work, not just that one lucky page resolved: +// +// standard.site no hint at all; found by the origin well-known +// atproto.com/blog a hint, and an origin well-known that 404s, so it +// verifies through the path-scoped probe +// permadeath.com both: the hint and the origin well-known +// example.com no publication; the control, and the case that catches a +// build that "detects" everything +// +// Sites and projects only, never a person's personal blog (see CLAUDE.md); +// the one that belongs to an individual is the author's own. +// +// This talks to the live internet, so it is not a prek hook and not part of +// `npm test`. It is a release gate: run it on the tree that is about to be +// packaged. +// +// npm run smoke # against dist/ +// node scripts/smoke-test.mjs --dist some/other/dist +// node scripts/smoke-test.mjs --headful # watch it happen + +import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { poll, withChrome } from './cdp.mjs' + +/** + * `did` is asserted where the publication is stable and public; permadeath.com + * is left to `verified` alone so rotating the record does not fail the build. + */ +export const CASES = [ + { + name: 'standard.site', + url: 'https://standard.site/', + route: 'origin well-known, no link tag', + expect: { publication: true, verified: true, did: 'did:plc:re3ebnp5v7ffagz6rb6xfei4' }, + }, + { + name: 'atproto.com/blog', + url: 'https://atproto.com/blog', + route: 'link tag + path-scoped well-known (origin 404s)', + expect: { publication: true, verified: true, did: 'did:plc:ewvi7nxzyoun6zhxrhs64oiz' }, + }, + { + name: 'permadeath.com', + url: 'https://permadeath.com/', + route: 'link tag and origin well-known', + expect: { publication: true, verified: true }, + }, + { + name: 'example.com', + url: 'https://example.com/', + route: 'control: no publication anywhere', + expect: { publication: false }, + }, +] + +/** + * Compare one tab's detection state against what the case expected. Pure, so + * the interesting half is testable without a browser. + */ +export function checkCase(testCase, state, iconState) { + const problems = [] + const want = testCase.expect + + if (!want.publication) { + if (state?.pub) { + problems.push(`expected no publication, got "${state.pub.record?.name}" (${state.pub.uri})`) + } + // A control page must not merely fail to find a publication — it must have + // finished looking. `warning` here would mean detection errored. + if (iconState !== 'none') { + problems.push(`expected icon state "none", got "${iconState}"`) + } + return problems + } + + if (!state?.pub) { + problems.push(`no publication detected${state?.error ? ` (error: ${state.error})` : ''}`) + return problems + } + if (want.verified && !state.pub.verified) { + problems.push(`publication found but not verified: ${state.pub.uri}`) + } + if (want.did && state.pub.did !== want.did) { + problems.push(`expected did ${want.did}, got ${state.pub.did}`) + } + if (!state.pub.record?.name) { + problems.push('publication record has no name; the record fetch did not resolve') + } + // Signed out, a verified publication reads as "detected, subscription + // unknown". Anything else means the badge would be wrong on a real toolbar. + if (want.verified && iconState !== 'signedout') { + problems.push(`expected icon state "signedout" while signed out, got "${iconState}"`) + } + return problems +} + +async function run(argv) { + const root = resolve(import.meta.dirname, '..') + const distFlag = argv.indexOf('--dist') + const dist = resolve(distFlag === -1 ? join(root, 'dist') : argv[distFlag + 1]) + const headless = !argv.includes('--headful') + + if (!existsSync(join(dist, 'manifest.json'))) { + console.error(`smoke-test: ${dist} is not a built extension — run \`npm run build\` first`) + process.exit(1) + } + + const profile = mkdtempSync(join(tmpdir(), 'substandard-smoke-')) + console.log(`smoke-test: ${dist}`) + + const failures = [] + try { + await withChrome({ extensionDir: dist, profile, headless }, async ({ inWorker }) => { + for (const testCase of CASES) { + let tab + try { + tab = await inWorker( + `chrome.tabs.create({ url: ${JSON.stringify(testCase.url)}, active: true })`, + ) + // Best effort only. Detection runs from the content script at + // document_idle, so a page that keeps a connection open never + // reaches `complete` and does not need to: the settled state below + // is the signal being waited on. + await poll( + () => inWorker(`chrome.tabs.get(${tab.id}).then((t) => t.status === 'complete')`), + `${testCase.name} to finish loading`, + 10_000, + ).catch(() => {}) + + // Detection is asynchronous and driven by the content script's + // report, so wait for the worker to record a settled answer rather + // than reading whatever is there the instant the page loads. + const key = `tab:${tab.id}` + const state = await poll( + () => + inWorker( + `chrome.storage.session.get(${JSON.stringify(key)}).then((r) => { + const s = r[${JSON.stringify(key)}] + return s && !s.error ? s : false + })`, + ), + `a settled detection state for ${testCase.name}`, + 25_000, + ).catch(async () => { + // Report the failed state rather than only the timeout. + return await inWorker( + `chrome.storage.session.get(${JSON.stringify(key)}).then((r) => r[${JSON.stringify(key)}] ?? null)`, + ) + }) + + const iconState = await inWorker( + `__substandard.iconStateFor(${JSON.stringify(state)})`, + ) + const problems = checkCase(testCase, state, iconState) + if (problems.length === 0) { + const what = state?.pub ? `"${state.pub.record.name}"` : 'no publication' + console.log(` ok ${testCase.name.padEnd(18)} ${what} [${testCase.route}]`) + } else { + failures.push({ name: testCase.name, problems }) + console.log(` FAIL ${testCase.name.padEnd(18)} [${testCase.route}]`) + for (const p of problems) console.log(` ${p}`) + } + } catch (err) { + // One unreachable site must not abandon the rest of the gate. + failures.push({ name: testCase.name, problems: [String(err.message ?? err)] }) + console.log(` FAIL ${testCase.name.padEnd(18)} [${testCase.route}]`) + console.log(` ${err.message ?? err}`) + } finally { + if (tab) await inWorker(`chrome.tabs.remove(${tab.id})`).catch(() => {}) + } + } + }) + } finally { + rmSync(profile, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }) + } + + if (failures.length > 0) { + console.error(`\nsmoke-test: FAILED — ${failures.length} of ${CASES.length} case(s)`) + process.exit(1) + } + console.log(`\nsmoke-test: OK — ${CASES.length} cases, every detection route reached`) +} + +if (process.argv[1] === import.meta.filename) await run(process.argv.slice(2)) diff --git a/scripts/smoke-test.test.mjs b/scripts/smoke-test.test.mjs new file mode 100644 index 0000000..0e6e369 --- /dev/null +++ b/scripts/smoke-test.test.mjs @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest' +import { CASES, checkCase } from './smoke-test.mjs' + +const pub = (over = {}) => ({ + uri: 'at://did:plc:abc/site.standard.publication/self', + did: 'did:plc:abc', + verified: true, + record: { name: 'Example Publication', url: 'https://example.test' }, + ...over, +}) +const detected = { name: 't', route: 'r', expect: { publication: true, verified: true } } +const control = { name: 'c', route: 'r', expect: { publication: false } } + +describe('CASES', () => { + it('covers every detection route plus a negative control', () => { + expect(CASES.filter((c) => c.expect.publication).length).toBeGreaterThanOrEqual(3) + expect(CASES.some((c) => !c.expect.publication)).toBe(true) + }) + + it('gives every case a url and a stated route', () => { + for (const c of CASES) { + expect(c.url).toMatch(/^https:\/\//) + expect(c.route).toBeTruthy() + } + }) +}) + +describe('checkCase: a publication is expected', () => { + it('passes a verified publication seen while signed out', () => { + expect(checkCase(detected, { pub: pub() }, 'signedout')).toEqual([]) + }) + + // The shape the broken-build control produced: the hint found the record, + // but the well-known check that proves the site owns it did not pass. + it('fails a publication that was found but not verified', () => { + const problems = checkCase(detected, { pub: pub({ verified: false }) }, 'warning') + expect(problems).toEqual([ + expect.stringContaining('not verified'), + expect.stringContaining('signedout'), + ]) + }) + + it('fails when nothing was detected, and says so with the error', () => { + expect(checkCase(detected, { error: 'fetch-failed' }, 'warning')).toEqual([ + expect.stringContaining('fetch-failed'), + ]) + }) + + it('fails a did that is not the one the case pinned', () => { + const c = { ...detected, expect: { ...detected.expect, did: 'did:plc:expected' } } + expect(checkCase(c, { pub: pub() }, 'signedout')).toEqual([ + expect.stringContaining('did:plc:expected'), + ]) + }) + + // A record that resolved to nothing still renders an empty card, so an + // unnamed publication is a failure rather than a pass with a blank. + it('fails a record that carries no name', () => { + const problems = checkCase(detected, { pub: pub({ record: {} }) }, 'signedout') + expect(problems).toEqual([expect.stringContaining('no name')]) + }) +}) + +describe('checkCase: the control', () => { + it('passes a page with no publication that finished looking', () => { + expect(checkCase(control, { url: 'https://example.test/' }, 'none')).toEqual([]) + }) + + it('fails if a publication turns up on the control page', () => { + expect(checkCase(control, { pub: pub() }, 'signedout')).toEqual([ + expect.stringContaining('expected no publication'), + expect.stringContaining('none'), + ]) + }) + + // "Found nothing" and "could not look" are different answers, and only one + // of them means the control passed. + it('fails a control page where detection errored rather than concluded', () => { + expect(checkCase(control, { error: 'fetch-failed' }, 'warning')).toEqual([ + expect.stringContaining('expected icon state "none"'), + ]) + }) +})