diff --git a/docs/api/advanced/reporters.md b/docs/api/advanced/reporters.md index e9213488d..2ae7f1303 100644 --- a/docs/api/advanced/reporters.md +++ b/docs/api/advanced/reporters.md @@ -21,6 +21,7 @@ Vitest has its own test run lifecycle. These are represented by reporter's metho - [`onHookEnd(beforeEach)`](#onhookend) - [`onHookStart(afterEach)`](#onhookstart) - [`onHookEnd(afterEach)`](#onhookend) + - [`onTestCaseAttempt`](#ontestcaseattempt) 5.1.0 - [`onTestCaseResult`](#ontestcaseresult) - [`onHookStart(afterAll)`](#onhookstart) - [`onHookEnd(afterAll)`](#onhookend) @@ -309,6 +310,28 @@ This method is called before the test starts to run or it was skipped. Note that Notice that it's possible to have [`testCase.result()`](/api/advanced/test-case#result) with `passed` or `failed` state already when `onTestCaseReady` is called. This can happen if test was running too fast and both `onTestCaseReady` and `onTestCaseResult` were scheduled to run in the same microtask. ::: +## onTestCaseAttempt 5.1.0 {#ontestcaseattempt} + +```ts +function onTestCaseAttempt(testCase: TestCase, attempt: TestAttempt): Awaitable +``` + +This method is called every time a run of the test finishes, including every [`retry`](/config/retry) and [`repeats`](/api/test#repeats) run. It is called after the `afterEach` hooks of that run, and before `onTestCaseResult` for the last run. Tests that are skipped without running, for example with `test.skip`, do not call it. + +The `attempt` argument has the same data as the matching entry in [`testCase.attempts()`](/api/advanced/test-case#attempts). Use it instead of reading the last item of `testCase.attempts()`, because `attempts()` can already include later runs by the time the hook is called, for example when running with `--merge-reports`. + +```ts +import type { Reporter, TestAttempt, TestCase } from 'vitest/node' + +export default { + onTestCaseAttempt(testCase: TestCase, attempt: TestAttempt) { + if (attempt.state === 'failed') { + console.log(`${testCase.fullName} failed on retry ${attempt.retryIndex}`) + } + }, +} satisfies Reporter +``` + ## onTestCaseResult ```ts diff --git a/docs/api/advanced/test-case.md b/docs/api/advanced/test-case.md index c07e47fb6..98ca63f36 100644 --- a/docs/api/advanced/test-case.md +++ b/docs/api/advanced/test-case.md @@ -327,7 +327,7 @@ For example, a test with `retry: 2` that passes on its third run returns: ``` ::: info -`attempts()` returns an empty array if the test has not run yet. +`attempts()` returns an empty array if the test has not run yet. While the test is running, it only includes the runs that have finished. Use the [`onTestCaseAttempt`](/api/advanced/reporters#ontestcaseattempt) reporter hook to be notified when each run finishes. ::: ## annotations diff --git a/packages/vitest/src/node/reporters/reported-tasks.ts b/packages/vitest/src/node/reporters/reported-tasks.ts index 260d8fcc2..834bb7552 100644 --- a/packages/vitest/src/node/reporters/reported-tasks.ts +++ b/packages/vitest/src/node/reporters/reported-tasks.ts @@ -8,6 +8,7 @@ import type { Suite as RunnerTestSuite, SerializableRetry, TaskMeta, + TaskResultAttempt, TestAnnotation, TestArtifact, TestBenchmark, @@ -198,14 +199,7 @@ export class TestCase extends ReportedTaskImplementation { * Individual results for every retry and repeat attempt. */ public attempts(): ReadonlyArray { - return (this.task.result?.attempts || []).map((attempt) => ({ - state: attempt.state === 'pass' ? 'passed' : attempt.state === 'fail' ? 'failed' : 'skipped', - errors: attempt.errors as TestError[] | undefined, - duration: attempt.duration, - startTime: attempt.startTime, - retryIndex: attempt.retryIndex, - repeatIndex: attempt.repeatIndex, - })) + return (this.task.result?.attempts || []).map(toTestAttempt) } /** @@ -804,6 +798,17 @@ function getReportedTask( return reportedTask } +export function toTestAttempt(attempt: TaskResultAttempt): TestAttempt { + return { + state: attempt.state === 'pass' ? 'passed' : attempt.state === 'fail' ? 'failed' : 'skipped', + errors: attempt.errors as TestError[] | undefined, + duration: attempt.duration, + startTime: attempt.startTime, + retryIndex: attempt.retryIndex, + repeatIndex: attempt.repeatIndex, + } +} + function getSuiteState(task: RunnerTestSuite | RunnerTestFile): TestSuiteState { const mode = task.mode const state = task.result?.state diff --git a/packages/vitest/src/node/test-run.ts b/packages/vitest/src/node/test-run.ts index 3740a6abd..551e169a1 100644 --- a/packages/vitest/src/node/test-run.ts +++ b/packages/vitest/src/node/test-run.ts @@ -30,6 +30,7 @@ import { parseErrorStacktrace } from '@vitest/utils/source-map' import { extractSourcemapFromFile } from '@vitest/utils/source-map/node' import mime from 'mime/lite' import { basename, extname, resolve } from 'pathe' +import { toTestAttempt } from './reporters/reported-tasks' export class TestRun { constructor(private vitest: Vitest) {} @@ -271,6 +272,11 @@ export class TestRun { return await this.vitest.report('onTestCaseReady', entity) } + if (event === 'test-attempt-finished' && entity.type === 'test') { + assert(data?.attempt, 'Attempt must be provided for test-attempt-finished event') + return await this.vitest.report('onTestCaseAttempt', entity, toTestAttempt(data.attempt)) + } + if (event === 'test-finished' && entity.type === 'test') { return await this.vitest.report('onTestCaseResult', entity) } diff --git a/packages/vitest/src/node/types/reporter.ts b/packages/vitest/src/node/types/reporter.ts index d05012dea..33d4949eb 100644 --- a/packages/vitest/src/node/types/reporter.ts +++ b/packages/vitest/src/node/types/reporter.ts @@ -12,6 +12,7 @@ import type { Vitest } from '../core' import type { TestProject } from '../project' import type { ReportedHookContext, + TestAttempt, TestCase, TestModule, TestSuite, @@ -76,6 +77,11 @@ export interface Reporter { * The `result()` cannot be `pending`. */ onTestCaseResult?: (testCase: TestCase) => Awaitable + /** + * Called after each run of the test finishes, including every retry and repeat. + * Called before `onTestCaseResult` for the final attempt. + */ + onTestCaseAttempt?: (testCase: TestCase, attempt: TestAttempt) => Awaitable /** * Called when annotation is added via the `task.annotate` API. diff --git a/packages/vitest/src/runtime/runner/run.ts b/packages/vitest/src/runtime/runner/run.ts index 0edc13a5e..b26975375 100644 --- a/packages/vitest/src/runtime/runner/run.ts +++ b/packages/vitest/src/runtime/runner/run.ts @@ -10,6 +10,8 @@ import type { Suite, SuiteHooks, Task, + TaskEventData, + TaskEventPack, TaskMeta, TaskResult, TaskResultAttempt, @@ -456,7 +458,7 @@ async function callAroundEachHooks( } const packs = new Map() -const eventsPacks: [string, TaskUpdateEvent, undefined][] = [] +const eventsPacks: TaskEventPack[] = [] const pendingTasksUpdates: Promise[] = [] function sendTasksUpdate(runner: VitestRunner): void { @@ -507,8 +509,13 @@ function throttle void>(fn: T, ms: number): T { // throttle based on summary reporter's DURATION_UPDATE_INTERVAL_MS const sendTasksUpdateThrottled = throttle(sendTasksUpdate, 100) -function updateTask(event: TaskUpdateEvent, task: Task, runner: VitestRunner): void { - eventsPacks.push([task.id, event, undefined]) +function updateTask( + event: TaskUpdateEvent, + task: Task, + runner: VitestRunner, + data?: TaskEventData, +): void { + eventsPacks.push([task.id, event, data]) packs.set(task.id, [task.result, task.meta]) sendTasksUpdateThrottled(runner) } @@ -609,14 +616,20 @@ async function runTest(test: Test, runner: VitestRunner): Promise { const attemptErrorsStart = test.result.errors?.length ?? 0 const recordAttempt = (state: TaskResultAttempt['state']) => { const errors = test.result!.errors?.slice(attemptErrorsStart) - test.result!.attempts!.push({ + const attempt: TaskResultAttempt = { state, errors: errors?.length ? errors : undefined, duration: now() - attemptStart, startTime: attemptStartTime, retryIndex: retryCount, repeatIndex: repeatCount, - }) + } + test.result!.attempts!.push(attempt) + updateTask('test-attempt-finished', test, runner, { attempt }) + // tests that run once don't need a dedicated flush per attempt + if (retry || repeats) { + sendTasksUpdate(runner) + } } let beforeEachCleanups: unknown[] = [] // fixtureCheckpoint is passed by callAroundEachHooks - it represents the count diff --git a/packages/vitest/src/runtime/runner/types.ts b/packages/vitest/src/runtime/runner/types.ts index a6d0f848c..fdb4ed9b7 100644 --- a/packages/vitest/src/runtime/runner/types.ts +++ b/packages/vitest/src/runtime/runner/types.ts @@ -267,6 +267,7 @@ export type TaskResultPack = [ export interface TaskEventData { annotation?: TestAnnotation | undefined artifact?: TestArtifact | undefined + attempt?: TaskResultAttempt | undefined } export type TaskEventPack = [ @@ -289,7 +290,11 @@ export type TaskUpdateEvent = | 'suite-failed-early' | 'test-prepare' | 'test-finished' + /** + * @deprecated Use `test-attempt-finished` instead. + */ | 'test-retried' + | 'test-attempt-finished' | 'test-cancel' | 'suite-prepare' | 'suite-finished' diff --git a/packages/vitest/src/utils/tasks.ts b/packages/vitest/src/utils/tasks.ts index 15fed5ee2..811e51445 100644 --- a/packages/vitest/src/utils/tasks.ts +++ b/packages/vitest/src/utils/tasks.ts @@ -133,6 +133,9 @@ export function convertTasksToEvents( task.artifacts.forEach((artifact) => { events.push([task.id, 'test-artifact', { artifact }]) }) + task.result?.attempts?.forEach((attempt) => { + events.push([task.id, 'test-attempt-finished', { attempt }]) + }) events.push([task.id, 'test-finished', undefined]) } } diff --git a/test/browser/specs/test-attempts.test.ts b/test/browser/specs/test-attempts.test.ts new file mode 100644 index 000000000..bbc94b892 --- /dev/null +++ b/test/browser/specs/test-attempts.test.ts @@ -0,0 +1,47 @@ +import { expect, test } from 'vitest' +import { instances, runInlineBrowserTests } from './utils' + +test('reports every attempt through onTestCaseAttempt', async () => { + const events: string[] = [] + + const { stderr } = await runInlineBrowserTests( + { + 'basic.test.ts': ` + import { expect, test } from 'vitest' + + let runs = 0 + test('flaky', { retry: 1, repeats: 1 }, () => { + expect(++runs % 2).toBe(0) + }) + `, + }, + { + browser: { + instances: [instances[0]], + }, + reporters: [ + { + onTestCaseAttempt(testCase, attempt) { + events.push( + `attempt |${testCase.name}| [repeat ${attempt.repeatIndex}, retry ${attempt.retryIndex}, ${attempt.state}, attempts() ${testCase.attempts().length}]`, + ) + }, + onTestCaseResult(testCase) { + events.push(`result |${testCase.name}| ${testCase.result().state}`) + }, + }, + ], + }, + ) + + expect(stderr).toBe('') + expect(events).toMatchInlineSnapshot(` + [ + "attempt |flaky| [repeat 0, retry 0, failed, attempts() 1]", + "attempt |flaky| [repeat 0, retry 1, passed, attempts() 2]", + "attempt |flaky| [repeat 1, retry 0, failed, attempts() 3]", + "attempt |flaky| [repeat 1, retry 1, passed, attempts() 4]", + "result |flaky| passed", + ] + `) +}) diff --git a/test/e2e/test/reporters/test-run.test.ts b/test/e2e/test/reporters/test-run.test.ts index 7db8de9f9..8cb557777 100644 --- a/test/e2e/test/reporters/test-run.test.ts +++ b/test/e2e/test/reporters/test-run.test.ts @@ -3,6 +3,7 @@ import type { ReportedHookContext, Reporter, SerializedError, + TestAttempt, TestCase, TestModule, TestRunEndReason, @@ -973,6 +974,148 @@ describe('merge reports', () => { }) }) +describe('attempts', () => { + test('single attempt', async () => { + const report = await run( + { + 'example.test.ts': ts` + test('passes', () => {}); + test('fails', () => { expect(1).toBe(2) }); + `, + }, + undefined, + { printAttempts: true, failed: true }, + ) + + expect(report).toMatchInlineSnapshot(` + " + onTestModuleQueued (example.test.ts) + onTestModuleCollected (example.test.ts) + onTestModuleStart (example.test.ts) + onTestCaseReady (example.test.ts) |passes| + onTestCaseAttempt (example.test.ts) |passes| [repeat 0, retry 0, passed, 0 errors, attempts() 1] + onTestCaseResult (example.test.ts) |passes| + onTestCaseReady (example.test.ts) |fails| + onTestCaseAttempt (example.test.ts) |fails| [repeat 0, retry 0, failed, 1 errors, attempts() 1] + onTestCaseResult (example.test.ts) |fails| + onTestModuleEnd (example.test.ts)" + `) + }) + + test('retries and repeats', async () => { + const report = await run( + { + 'example.test.ts': ts` + let runs = 0 + beforeEach(() => {}) + test('flaky', { retry: 1, repeats: 1 }, () => { + expect(++runs % 2).toBe(0) + }); + `, + }, + undefined, + { printAttempts: true }, + ) + + expect(report).toMatchInlineSnapshot(` + " + onTestModuleQueued (example.test.ts) + onTestModuleCollected (example.test.ts) + onTestModuleStart (example.test.ts) + onTestCaseReady (example.test.ts) |flaky| + onHookStart (example.test.ts) |flaky| [beforeEach] + onHookEnd (example.test.ts) |flaky| [beforeEach] + onTestCaseAttempt (example.test.ts) |flaky| [repeat 0, retry 0, failed, 1 errors, attempts() 1] + onHookStart (example.test.ts) |flaky| [beforeEach] + onHookEnd (example.test.ts) |flaky| [beforeEach] + onTestCaseAttempt (example.test.ts) |flaky| [repeat 0, retry 1, passed, 0 errors, attempts() 2] + onHookStart (example.test.ts) |flaky| [beforeEach] + onHookEnd (example.test.ts) |flaky| [beforeEach] + onTestCaseAttempt (example.test.ts) |flaky| [repeat 1, retry 0, failed, 1 errors, attempts() 3] + onHookStart (example.test.ts) |flaky| [beforeEach] + onHookEnd (example.test.ts) |flaky| [beforeEach] + onTestCaseAttempt (example.test.ts) |flaky| [repeat 1, retry 1, passed, 0 errors, attempts() 4] + onTestCaseResult (example.test.ts) |flaky| + onTestModuleEnd (example.test.ts)" + `) + }) + + test('expected failures and skips', async () => { + const report = await run( + { + 'example.test.ts': ts` + test.fails('fails', { retry: 1 }, () => { expect(1).toBe(2) }); + test.fails('passes', { retry: 1 }, () => {}); + test('skips', ({ skip }) => { skip() }); + `, + }, + undefined, + { printAttempts: true, failed: true }, + ) + + expect(report).toMatchInlineSnapshot(` + " + onTestModuleQueued (example.test.ts) + onTestModuleCollected (example.test.ts) + onTestModuleStart (example.test.ts) + onTestCaseReady (example.test.ts) |fails| + onTestCaseAttempt (example.test.ts) |fails| [repeat 0, retry 0, passed, 0 errors, attempts() 1] + onTestCaseResult (example.test.ts) |fails| + onTestCaseReady (example.test.ts) |passes| + onTestCaseAttempt (example.test.ts) |passes| [repeat 0, retry 0, failed, 1 errors, attempts() 1] + onTestCaseAttempt (example.test.ts) |passes| [repeat 0, retry 1, failed, 1 errors, attempts() 2] + onTestCaseResult (example.test.ts) |passes| + onTestCaseReady (example.test.ts) |skips| + onTestCaseAttempt (example.test.ts) |skips| [repeat 0, retry 0, skipped, 0 errors, attempts() 1] + onTestCaseResult (example.test.ts) |skips| + onTestModuleEnd (example.test.ts)" + `) + }) + + test('merge reports replays attempts', async () => { + const blobsOutputDirectory = resolve(import.meta.dirname, 'fixtures-blobs-attempts') + const blobOutputFile = resolve(blobsOutputDirectory, 'blob.json') + onTestFinished(() => { + rmSync(blobsOutputDirectory, { recursive: true, force: true }) + }) + + const { root } = await runInlineTests( + { + 'example.test.ts': ts` + let runs = 0 + test('flaky', { retry: 1, repeats: 1 }, () => { + expect(++runs % 2).toBe(0) + }); + `, + }, + { + globals: true, + reporters: [['blob', { outputFile: blobOutputFile }]], + }, + ) + + const report = await run( + {}, + { mergeReports: blobsOutputDirectory }, + { roots: [root], printAttempts: true }, + ) + + expect(report).toMatchInlineSnapshot(` + " + onTestModuleQueued (example.test.ts) + onTestModuleCollected (example.test.ts) + onTestModuleStart (example.test.ts) + onTestCaseReady (example.test.ts) |flaky| + onTestCaseAttempt (example.test.ts) |flaky| [repeat 0, retry 0, failed, 1 errors, attempts() 4] + onTestCaseAttempt (example.test.ts) |flaky| [repeat 0, retry 1, passed, 0 errors, attempts() 4] + onTestCaseAttempt (example.test.ts) |flaky| [repeat 1, retry 0, failed, 1 errors, attempts() 4] + onTestCaseAttempt (example.test.ts) |flaky| [repeat 1, retry 1, passed, 0 errors, attempts() 4] + onTestCaseResult (example.test.ts) |flaky| + onTestModuleEnd (example.test.ts)" + `) + }) +}) + describe('type checking', () => { test('typechecking is reported correctly', async () => { const report = await run( @@ -1139,6 +1282,7 @@ describe('test run result', () => { interface ReporterOptions { printTestRunEvents?: boolean + printAttempts?: boolean roots?: string[] failed?: boolean } @@ -1251,6 +1395,14 @@ class CustomReporter implements Reporter { ) } + onTestCaseAttempt(test: TestCase, attempt: TestAttempt) { + if (this.options.printAttempts) { + this.calls.push( + `${padded(test, 'onTestCaseAttempt')} (${this.normalizeFilename(test.module)}) |${test.name}| [repeat ${attempt.repeatIndex}, retry ${attempt.retryIndex}, ${attempt.state}, ${attempt.errors?.length ?? 0} errors, attempts() ${test.attempts().length}]`, + ) + } + } + onUserConsoleLog(log: UserConsoleLog) { const task = this.ctx.state.idMap.get(log.taskId!) const test = task && (this.ctx.state.getReportedEntity(task) as TestCase)