diff --git a/docs/api/advanced/reporters.md b/docs/api/advanced/reporters.md
index e9213488d..2ae7f1303 100644
--- a/docs/api/advanced/reporters.md
+++ b/docs/api/advanced/reporters.md
@@ -21,6 +21,7 @@ Vitest has its own test run lifecycle. These are represented by reporter's metho
- [`onHookEnd(beforeEach)`](#onhookend)
- [`onHookStart(afterEach)`](#onhookstart)
- [`onHookEnd(afterEach)`](#onhookend)
+ - [`onTestCaseAttempt`](#ontestcaseattempt) 5.1.0
- [`onTestCaseResult`](#ontestcaseresult)
- [`onHookStart(afterAll)`](#onhookstart)
- [`onHookEnd(afterAll)`](#onhookend)
@@ -309,6 +310,28 @@ This method is called before the test starts to run or it was skipped. Note that
Notice that it's possible to have [`testCase.result()`](/api/advanced/test-case#result) with `passed` or `failed` state already when `onTestCaseReady` is called. This can happen if test was running too fast and both `onTestCaseReady` and `onTestCaseResult` were scheduled to run in the same microtask.
:::
+## onTestCaseAttempt 5.1.0 {#ontestcaseattempt}
+
+```ts
+function onTestCaseAttempt(testCase: TestCase, attempt: TestAttempt): Awaitable
+```
+
+This method is called every time a run of the test finishes, including every [`retry`](/config/retry) and [`repeats`](/api/test#repeats) run. It is called after the `afterEach` hooks of that run, and before `onTestCaseResult` for the last run. Tests that are skipped without running, for example with `test.skip`, do not call it.
+
+The `attempt` argument has the same data as the matching entry in [`testCase.attempts()`](/api/advanced/test-case#attempts). Use it instead of reading the last item of `testCase.attempts()`, because `attempts()` can already include later runs by the time the hook is called, for example when running with `--merge-reports`.
+
+```ts
+import type { Reporter, TestAttempt, TestCase } from 'vitest/node'
+
+export default {
+ onTestCaseAttempt(testCase: TestCase, attempt: TestAttempt) {
+ if (attempt.state === 'failed') {
+ console.log(`${testCase.fullName} failed on retry ${attempt.retryIndex}`)
+ }
+ },
+} satisfies Reporter
+```
+
## onTestCaseResult
```ts
diff --git a/docs/api/advanced/test-case.md b/docs/api/advanced/test-case.md
index c07e47fb6..98ca63f36 100644
--- a/docs/api/advanced/test-case.md
+++ b/docs/api/advanced/test-case.md
@@ -327,7 +327,7 @@ For example, a test with `retry: 2` that passes on its third run returns:
```
::: info
-`attempts()` returns an empty array if the test has not run yet.
+`attempts()` returns an empty array if the test has not run yet. While the test is running, it only includes the runs that have finished. Use the [`onTestCaseAttempt`](/api/advanced/reporters#ontestcaseattempt) reporter hook to be notified when each run finishes.
:::
## annotations
diff --git a/packages/vitest/src/node/reporters/reported-tasks.ts b/packages/vitest/src/node/reporters/reported-tasks.ts
index 260d8fcc2..834bb7552 100644
--- a/packages/vitest/src/node/reporters/reported-tasks.ts
+++ b/packages/vitest/src/node/reporters/reported-tasks.ts
@@ -8,6 +8,7 @@ import type {
Suite as RunnerTestSuite,
SerializableRetry,
TaskMeta,
+ TaskResultAttempt,
TestAnnotation,
TestArtifact,
TestBenchmark,
@@ -198,14 +199,7 @@ export class TestCase extends ReportedTaskImplementation {
* Individual results for every retry and repeat attempt.
*/
public attempts(): ReadonlyArray {
- return (this.task.result?.attempts || []).map((attempt) => ({
- state: attempt.state === 'pass' ? 'passed' : attempt.state === 'fail' ? 'failed' : 'skipped',
- errors: attempt.errors as TestError[] | undefined,
- duration: attempt.duration,
- startTime: attempt.startTime,
- retryIndex: attempt.retryIndex,
- repeatIndex: attempt.repeatIndex,
- }))
+ return (this.task.result?.attempts || []).map(toTestAttempt)
}
/**
@@ -804,6 +798,17 @@ function getReportedTask(
return reportedTask
}
+export function toTestAttempt(attempt: TaskResultAttempt): TestAttempt {
+ return {
+ state: attempt.state === 'pass' ? 'passed' : attempt.state === 'fail' ? 'failed' : 'skipped',
+ errors: attempt.errors as TestError[] | undefined,
+ duration: attempt.duration,
+ startTime: attempt.startTime,
+ retryIndex: attempt.retryIndex,
+ repeatIndex: attempt.repeatIndex,
+ }
+}
+
function getSuiteState(task: RunnerTestSuite | RunnerTestFile): TestSuiteState {
const mode = task.mode
const state = task.result?.state
diff --git a/packages/vitest/src/node/test-run.ts b/packages/vitest/src/node/test-run.ts
index 3740a6abd..551e169a1 100644
--- a/packages/vitest/src/node/test-run.ts
+++ b/packages/vitest/src/node/test-run.ts
@@ -30,6 +30,7 @@ import { parseErrorStacktrace } from '@vitest/utils/source-map'
import { extractSourcemapFromFile } from '@vitest/utils/source-map/node'
import mime from 'mime/lite'
import { basename, extname, resolve } from 'pathe'
+import { toTestAttempt } from './reporters/reported-tasks'
export class TestRun {
constructor(private vitest: Vitest) {}
@@ -271,6 +272,11 @@ export class TestRun {
return await this.vitest.report('onTestCaseReady', entity)
}
+ if (event === 'test-attempt-finished' && entity.type === 'test') {
+ assert(data?.attempt, 'Attempt must be provided for test-attempt-finished event')
+ return await this.vitest.report('onTestCaseAttempt', entity, toTestAttempt(data.attempt))
+ }
+
if (event === 'test-finished' && entity.type === 'test') {
return await this.vitest.report('onTestCaseResult', entity)
}
diff --git a/packages/vitest/src/node/types/reporter.ts b/packages/vitest/src/node/types/reporter.ts
index d05012dea..33d4949eb 100644
--- a/packages/vitest/src/node/types/reporter.ts
+++ b/packages/vitest/src/node/types/reporter.ts
@@ -12,6 +12,7 @@ import type { Vitest } from '../core'
import type { TestProject } from '../project'
import type {
ReportedHookContext,
+ TestAttempt,
TestCase,
TestModule,
TestSuite,
@@ -76,6 +77,11 @@ export interface Reporter {
* The `result()` cannot be `pending`.
*/
onTestCaseResult?: (testCase: TestCase) => Awaitable
+ /**
+ * Called after each run of the test finishes, including every retry and repeat.
+ * Called before `onTestCaseResult` for the final attempt.
+ */
+ onTestCaseAttempt?: (testCase: TestCase, attempt: TestAttempt) => Awaitable
/**
* Called when annotation is added via the `task.annotate` API.
diff --git a/packages/vitest/src/runtime/runner/run.ts b/packages/vitest/src/runtime/runner/run.ts
index 0edc13a5e..b26975375 100644
--- a/packages/vitest/src/runtime/runner/run.ts
+++ b/packages/vitest/src/runtime/runner/run.ts
@@ -10,6 +10,8 @@ import type {
Suite,
SuiteHooks,
Task,
+ TaskEventData,
+ TaskEventPack,
TaskMeta,
TaskResult,
TaskResultAttempt,
@@ -456,7 +458,7 @@ async function callAroundEachHooks(
}
const packs = new Map()
-const eventsPacks: [string, TaskUpdateEvent, undefined][] = []
+const eventsPacks: TaskEventPack[] = []
const pendingTasksUpdates: Promise[] = []
function sendTasksUpdate(runner: VitestRunner): void {
@@ -507,8 +509,13 @@ function throttle void>(fn: T, ms: number): T {
// throttle based on summary reporter's DURATION_UPDATE_INTERVAL_MS
const sendTasksUpdateThrottled = throttle(sendTasksUpdate, 100)
-function updateTask(event: TaskUpdateEvent, task: Task, runner: VitestRunner): void {
- eventsPacks.push([task.id, event, undefined])
+function updateTask(
+ event: TaskUpdateEvent,
+ task: Task,
+ runner: VitestRunner,
+ data?: TaskEventData,
+): void {
+ eventsPacks.push([task.id, event, data])
packs.set(task.id, [task.result, task.meta])
sendTasksUpdateThrottled(runner)
}
@@ -609,14 +616,20 @@ async function runTest(test: Test, runner: VitestRunner): Promise {
const attemptErrorsStart = test.result.errors?.length ?? 0
const recordAttempt = (state: TaskResultAttempt['state']) => {
const errors = test.result!.errors?.slice(attemptErrorsStart)
- test.result!.attempts!.push({
+ const attempt: TaskResultAttempt = {
state,
errors: errors?.length ? errors : undefined,
duration: now() - attemptStart,
startTime: attemptStartTime,
retryIndex: retryCount,
repeatIndex: repeatCount,
- })
+ }
+ test.result!.attempts!.push(attempt)
+ updateTask('test-attempt-finished', test, runner, { attempt })
+ // tests that run once don't need a dedicated flush per attempt
+ if (retry || repeats) {
+ sendTasksUpdate(runner)
+ }
}
let beforeEachCleanups: unknown[] = []
// fixtureCheckpoint is passed by callAroundEachHooks - it represents the count
diff --git a/packages/vitest/src/runtime/runner/types.ts b/packages/vitest/src/runtime/runner/types.ts
index a6d0f848c..fdb4ed9b7 100644
--- a/packages/vitest/src/runtime/runner/types.ts
+++ b/packages/vitest/src/runtime/runner/types.ts
@@ -267,6 +267,7 @@ export type TaskResultPack = [
export interface TaskEventData {
annotation?: TestAnnotation | undefined
artifact?: TestArtifact | undefined
+ attempt?: TaskResultAttempt | undefined
}
export type TaskEventPack = [
@@ -289,7 +290,11 @@ export type TaskUpdateEvent =
| 'suite-failed-early'
| 'test-prepare'
| 'test-finished'
+ /**
+ * @deprecated Use `test-attempt-finished` instead.
+ */
| 'test-retried'
+ | 'test-attempt-finished'
| 'test-cancel'
| 'suite-prepare'
| 'suite-finished'
diff --git a/packages/vitest/src/utils/tasks.ts b/packages/vitest/src/utils/tasks.ts
index 15fed5ee2..811e51445 100644
--- a/packages/vitest/src/utils/tasks.ts
+++ b/packages/vitest/src/utils/tasks.ts
@@ -133,6 +133,9 @@ export function convertTasksToEvents(
task.artifacts.forEach((artifact) => {
events.push([task.id, 'test-artifact', { artifact }])
})
+ task.result?.attempts?.forEach((attempt) => {
+ events.push([task.id, 'test-attempt-finished', { attempt }])
+ })
events.push([task.id, 'test-finished', undefined])
}
}
diff --git a/test/browser/specs/test-attempts.test.ts b/test/browser/specs/test-attempts.test.ts
new file mode 100644
index 000000000..bbc94b892
--- /dev/null
+++ b/test/browser/specs/test-attempts.test.ts
@@ -0,0 +1,47 @@
+import { expect, test } from 'vitest'
+import { instances, runInlineBrowserTests } from './utils'
+
+test('reports every attempt through onTestCaseAttempt', async () => {
+ const events: string[] = []
+
+ const { stderr } = await runInlineBrowserTests(
+ {
+ 'basic.test.ts': `
+ import { expect, test } from 'vitest'
+
+ let runs = 0
+ test('flaky', { retry: 1, repeats: 1 }, () => {
+ expect(++runs % 2).toBe(0)
+ })
+ `,
+ },
+ {
+ browser: {
+ instances: [instances[0]],
+ },
+ reporters: [
+ {
+ onTestCaseAttempt(testCase, attempt) {
+ events.push(
+ `attempt |${testCase.name}| [repeat ${attempt.repeatIndex}, retry ${attempt.retryIndex}, ${attempt.state}, attempts() ${testCase.attempts().length}]`,
+ )
+ },
+ onTestCaseResult(testCase) {
+ events.push(`result |${testCase.name}| ${testCase.result().state}`)
+ },
+ },
+ ],
+ },
+ )
+
+ expect(stderr).toBe('')
+ expect(events).toMatchInlineSnapshot(`
+ [
+ "attempt |flaky| [repeat 0, retry 0, failed, attempts() 1]",
+ "attempt |flaky| [repeat 0, retry 1, passed, attempts() 2]",
+ "attempt |flaky| [repeat 1, retry 0, failed, attempts() 3]",
+ "attempt |flaky| [repeat 1, retry 1, passed, attempts() 4]",
+ "result |flaky| passed",
+ ]
+ `)
+})
diff --git a/test/e2e/test/reporters/test-run.test.ts b/test/e2e/test/reporters/test-run.test.ts
index 7db8de9f9..8cb557777 100644
--- a/test/e2e/test/reporters/test-run.test.ts
+++ b/test/e2e/test/reporters/test-run.test.ts
@@ -3,6 +3,7 @@ import type {
ReportedHookContext,
Reporter,
SerializedError,
+ TestAttempt,
TestCase,
TestModule,
TestRunEndReason,
@@ -973,6 +974,148 @@ describe('merge reports', () => {
})
})
+describe('attempts', () => {
+ test('single attempt', async () => {
+ const report = await run(
+ {
+ 'example.test.ts': ts`
+ test('passes', () => {});
+ test('fails', () => { expect(1).toBe(2) });
+ `,
+ },
+ undefined,
+ { printAttempts: true, failed: true },
+ )
+
+ expect(report).toMatchInlineSnapshot(`
+ "
+ onTestModuleQueued (example.test.ts)
+ onTestModuleCollected (example.test.ts)
+ onTestModuleStart (example.test.ts)
+ onTestCaseReady (example.test.ts) |passes|
+ onTestCaseAttempt (example.test.ts) |passes| [repeat 0, retry 0, passed, 0 errors, attempts() 1]
+ onTestCaseResult (example.test.ts) |passes|
+ onTestCaseReady (example.test.ts) |fails|
+ onTestCaseAttempt (example.test.ts) |fails| [repeat 0, retry 0, failed, 1 errors, attempts() 1]
+ onTestCaseResult (example.test.ts) |fails|
+ onTestModuleEnd (example.test.ts)"
+ `)
+ })
+
+ test('retries and repeats', async () => {
+ const report = await run(
+ {
+ 'example.test.ts': ts`
+ let runs = 0
+ beforeEach(() => {})
+ test('flaky', { retry: 1, repeats: 1 }, () => {
+ expect(++runs % 2).toBe(0)
+ });
+ `,
+ },
+ undefined,
+ { printAttempts: true },
+ )
+
+ expect(report).toMatchInlineSnapshot(`
+ "
+ onTestModuleQueued (example.test.ts)
+ onTestModuleCollected (example.test.ts)
+ onTestModuleStart (example.test.ts)
+ onTestCaseReady (example.test.ts) |flaky|
+ onHookStart (example.test.ts) |flaky| [beforeEach]
+ onHookEnd (example.test.ts) |flaky| [beforeEach]
+ onTestCaseAttempt (example.test.ts) |flaky| [repeat 0, retry 0, failed, 1 errors, attempts() 1]
+ onHookStart (example.test.ts) |flaky| [beforeEach]
+ onHookEnd (example.test.ts) |flaky| [beforeEach]
+ onTestCaseAttempt (example.test.ts) |flaky| [repeat 0, retry 1, passed, 0 errors, attempts() 2]
+ onHookStart (example.test.ts) |flaky| [beforeEach]
+ onHookEnd (example.test.ts) |flaky| [beforeEach]
+ onTestCaseAttempt (example.test.ts) |flaky| [repeat 1, retry 0, failed, 1 errors, attempts() 3]
+ onHookStart (example.test.ts) |flaky| [beforeEach]
+ onHookEnd (example.test.ts) |flaky| [beforeEach]
+ onTestCaseAttempt (example.test.ts) |flaky| [repeat 1, retry 1, passed, 0 errors, attempts() 4]
+ onTestCaseResult (example.test.ts) |flaky|
+ onTestModuleEnd (example.test.ts)"
+ `)
+ })
+
+ test('expected failures and skips', async () => {
+ const report = await run(
+ {
+ 'example.test.ts': ts`
+ test.fails('fails', { retry: 1 }, () => { expect(1).toBe(2) });
+ test.fails('passes', { retry: 1 }, () => {});
+ test('skips', ({ skip }) => { skip() });
+ `,
+ },
+ undefined,
+ { printAttempts: true, failed: true },
+ )
+
+ expect(report).toMatchInlineSnapshot(`
+ "
+ onTestModuleQueued (example.test.ts)
+ onTestModuleCollected (example.test.ts)
+ onTestModuleStart (example.test.ts)
+ onTestCaseReady (example.test.ts) |fails|
+ onTestCaseAttempt (example.test.ts) |fails| [repeat 0, retry 0, passed, 0 errors, attempts() 1]
+ onTestCaseResult (example.test.ts) |fails|
+ onTestCaseReady (example.test.ts) |passes|
+ onTestCaseAttempt (example.test.ts) |passes| [repeat 0, retry 0, failed, 1 errors, attempts() 1]
+ onTestCaseAttempt (example.test.ts) |passes| [repeat 0, retry 1, failed, 1 errors, attempts() 2]
+ onTestCaseResult (example.test.ts) |passes|
+ onTestCaseReady (example.test.ts) |skips|
+ onTestCaseAttempt (example.test.ts) |skips| [repeat 0, retry 0, skipped, 0 errors, attempts() 1]
+ onTestCaseResult (example.test.ts) |skips|
+ onTestModuleEnd (example.test.ts)"
+ `)
+ })
+
+ test('merge reports replays attempts', async () => {
+ const blobsOutputDirectory = resolve(import.meta.dirname, 'fixtures-blobs-attempts')
+ const blobOutputFile = resolve(blobsOutputDirectory, 'blob.json')
+ onTestFinished(() => {
+ rmSync(blobsOutputDirectory, { recursive: true, force: true })
+ })
+
+ const { root } = await runInlineTests(
+ {
+ 'example.test.ts': ts`
+ let runs = 0
+ test('flaky', { retry: 1, repeats: 1 }, () => {
+ expect(++runs % 2).toBe(0)
+ });
+ `,
+ },
+ {
+ globals: true,
+ reporters: [['blob', { outputFile: blobOutputFile }]],
+ },
+ )
+
+ const report = await run(
+ {},
+ { mergeReports: blobsOutputDirectory },
+ { roots: [root], printAttempts: true },
+ )
+
+ expect(report).toMatchInlineSnapshot(`
+ "
+ onTestModuleQueued (example.test.ts)
+ onTestModuleCollected (example.test.ts)
+ onTestModuleStart (example.test.ts)
+ onTestCaseReady (example.test.ts) |flaky|
+ onTestCaseAttempt (example.test.ts) |flaky| [repeat 0, retry 0, failed, 1 errors, attempts() 4]
+ onTestCaseAttempt (example.test.ts) |flaky| [repeat 0, retry 1, passed, 0 errors, attempts() 4]
+ onTestCaseAttempt (example.test.ts) |flaky| [repeat 1, retry 0, failed, 1 errors, attempts() 4]
+ onTestCaseAttempt (example.test.ts) |flaky| [repeat 1, retry 1, passed, 0 errors, attempts() 4]
+ onTestCaseResult (example.test.ts) |flaky|
+ onTestModuleEnd (example.test.ts)"
+ `)
+ })
+})
+
describe('type checking', () => {
test('typechecking is reported correctly', async () => {
const report = await run(
@@ -1139,6 +1282,7 @@ describe('test run result', () => {
interface ReporterOptions {
printTestRunEvents?: boolean
+ printAttempts?: boolean
roots?: string[]
failed?: boolean
}
@@ -1251,6 +1395,14 @@ class CustomReporter implements Reporter {
)
}
+ onTestCaseAttempt(test: TestCase, attempt: TestAttempt) {
+ if (this.options.printAttempts) {
+ this.calls.push(
+ `${padded(test, 'onTestCaseAttempt')} (${this.normalizeFilename(test.module)}) |${test.name}| [repeat ${attempt.repeatIndex}, retry ${attempt.retryIndex}, ${attempt.state}, ${attempt.errors?.length ?? 0} errors, attempts() ${test.attempts().length}]`,
+ )
+ }
+ }
+
onUserConsoleLog(log: UserConsoleLog) {
const task = this.ctx.state.idMap.get(log.taskId!)
const test = task && (this.ctx.state.getReportedEntity(task) as TestCase)