diff --git a/docs/api/advanced/runner.md b/docs/api/advanced/runner.md index bec83f9bb..f4f6561ad 100644 --- a/docs/api/advanced/runner.md +++ b/docs/api/advanced/runner.md @@ -261,13 +261,11 @@ export interface TaskResult { */ hooks?: Partial> /** - * The amount of times the task was retried. The task is retried only if it - * failed and `retry` option is set. + * The number of times the task has been retried across all repeats. */ retryCount?: number /** - * The amount of times the task was repeated. The task is repeated only if - * `repeats` option is set. This number also contains `retryCount`. + * The zero-based index of the current repeat. */ repeatCount?: number } diff --git a/packages/vitest/src/runtime/runner/run.ts b/packages/vitest/src/runtime/runner/run.ts index 26a2c72d5..3fcf0910f 100644 --- a/packages/vitest/src/runtime/runner/run.ts +++ b/packages/vitest/src/runtime/runner/run.ts @@ -596,7 +596,10 @@ async function runTest(test: Test, runner: VitestRunner): Promise { const $ = runner.trace! const repeats = test.repeats ?? 0 + let hasFailedRepeat = false for (let repeatCount = 0; repeatCount <= repeats; repeatCount++) { + // Force widening to TaskState because TypeScript cannot track mutations made by hooks and the test. + test.result.state = 'run' as TaskState const retry = getRetryCount(test.retry) for (let retryCount = 0; retryCount <= retry; retryCount++) { let beforeEachCleanups: unknown[] = [] @@ -726,6 +729,14 @@ async function runTest(test: Test, runner: VitestRunner): Promise { // update retry info updateTask('test-retried', test, runner) } + + if (test.result.state === 'fail') { + hasFailedRepeat = true + } + } + + if (hasFailedRepeat) { + test.result.state = 'fail' } // if test is marked to be failed, flip the result unless `TestSyntaxError` is present diff --git a/packages/vitest/src/runtime/runner/types.ts b/packages/vitest/src/runtime/runner/types.ts index 8ca16e737..d57028ead 100644 --- a/packages/vitest/src/runtime/runner/types.ts +++ b/packages/vitest/src/runtime/runner/types.ts @@ -192,13 +192,11 @@ export interface TaskResult { */ hooks?: Partial> /** - * The amount of times the task was retried. The task is retried only if it - * failed and `retry` option is set. + * The number of times the task has been retried across all repeats. */ retryCount?: number /** - * The amount of times the task was repeated. The task is repeated only if - * `repeats` option is set. This number also contains `retryCount`. + * The zero-based index of the current repeat. */ repeatCount?: number /** @internal */ diff --git a/test/e2e/test/repeats.test.ts b/test/e2e/test/repeats.test.ts index a1edb69b8..297916168 100644 --- a/test/e2e/test/repeats.test.ts +++ b/test/e2e/test/repeats.test.ts @@ -1,7 +1,7 @@ import type { TestModule } from 'vitest/node' import { resolve } from 'pathe' import { expect, test } from 'vitest' -import { runVitest } from '../../test-utils' +import { runInlineTests, runVitest } from '../../test-utils' const root = resolve(__dirname, '..', 'fixtures', 'repeats') @@ -24,3 +24,113 @@ test('repeats config option is exposed to tests and repeats execution', async () expect(overridden.options.repeats).toBe(1) expect(overridden.diagnostic()!.repeatCount).toBe(1) }) + +test('retry count and errors are retained across repeats', async () => { + const { errorTree, results } = await runInlineTests({ + 'repeats.test.js': ` + import { afterAll, expect, it } from 'vitest' + + const runs = [] + + it('retries each repeat once', { repeats: 2, retry: 1 }, ({ task }) => { + const repeatCount = task.result.repeatCount + const retryCount = task.result.retryCount + runs.push([repeatCount, retryCount]) + if (repeatCount === retryCount) { + throw new Error('repeat ' + repeatCount + ', retry ' + retryCount + ' failed') + } + }) + + afterAll(() => { + expect(runs).toEqual([ + [0, 0], + [0, 1], + [1, 1], + [1, 2], + [2, 2], + [2, 3], + ]) + }) + `, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "repeats.test.js": { + "retries each repeat once": "passed", + }, + } + `) + const [test] = results[0].children.allTests() + expect(test.result().errors?.map((error) => error.message)).toMatchInlineSnapshot(` + [ + "repeat 0, retry 0 failed", + "repeat 1, retry 1 failed", + "repeat 2, retry 2 failed", + ] + `) +}) + +test('failed repeats are retained after a successful repeat', async () => { + const { errorTree, results } = await runInlineTests({ + 'repeats.test.js': ` + import { it } from 'vitest' + + it('fails twice then passes', { repeats: 2, retry: 1 }, ({ task }) => { + if (task.result.repeatCount < 2) { + throw new Error('repeat ' + task.result.repeatCount + ', retry ' + task.result.retryCount + ' failed') + } + }) + `, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "repeats.test.js": { + "fails twice then passes": [ + "repeat 0, retry 0 failed", + "repeat 0, retry 1 failed", + "repeat 1, retry 1 failed", + "repeat 1, retry 2 failed", + ], + }, + } + `) + const [test] = results[0].children.allTests() + expect(test.diagnostic()!.retryCount).toBe(2) + expect(test.diagnostic()!.repeatCount).toBe(2) +}) + +test('onTestFailed runs only for failed repeats', async () => { + const { errorTree } = await runInlineTests({ + 'repeats.test.js': ` + import { expect, it } from 'vitest' + + const failedRepeats = [] + + it('alternates passing and failing repeats', { repeats: 4 }, ({ task, onTestFailed }) => { + const repeatCount = task.result.repeatCount + onTestFailed(() => failedRepeats.push(repeatCount)) + if (repeatCount % 2 === 1) { + throw new Error('repeat ' + repeatCount + ' failed') + } + }) + + it('records failed repeats', () => { + expect(failedRepeats).toEqual([1, 3]) + }) + `, + }) + + expect(errorTree()).toMatchInlineSnapshot(` + { + "repeats.test.js": { + "alternates passing and failing repeats": [ + "repeat 1 failed", + "repeat 3 failed", + ], + "records failed repeats": "passed", + }, + } + `) +}) diff --git a/test/unit/test/on-finished.test.ts b/test/unit/test/on-finished.test.ts index 57ebc1690..99c1e88cc 100644 --- a/test/unit/test/on-finished.test.ts +++ b/test/unit/test/on-finished.test.ts @@ -120,7 +120,6 @@ describe('repeats fail', () => { "(0, 1) fail", "(0, 2) run", "(0, 2) finish", - "(0, 2) fail", ] `) }) diff --git a/test/unit/test/repeats.test.ts b/test/unit/test/repeats.test.ts index 0b369008d..14f44e6af 100644 --- a/test/unit/test/repeats.test.ts +++ b/test/unit/test/repeats.test.ts @@ -1,4 +1,4 @@ -import { afterAll, describe, expect, test, TestRunner } from 'vitest' +import { afterAll, describe, expect, test } from 'vitest' const testNumbers: number[] = [] @@ -50,10 +50,6 @@ describe('testing repeats with retry', () => { expect(retryNumbers).toStrictEqual(result) }) }) - - test('should not reset retry count', { repeats: 2, retry: 1 }, () => { - expect(TestRunner.getCurrentTest()!.result?.retryCount).toBe(3) - }) }) const nestedDescribeNumbers: number[] = []