diff --git a/docs/.vitepress/scripts/cli-generator.ts b/docs/.vitepress/scripts/cli-generator.ts index 717b037b5..50055c7a9 100644 --- a/docs/.vitepress/scripts/cli-generator.ts +++ b/docs/.vitepress/scripts/cli-generator.ts @@ -25,6 +25,7 @@ const skipConfig = new Set([ 'coverage.thresholds.functions', 'coverage.thresholds.lines', 'standalone', + 'lastFailed', 'clearScreen', 'configLoader', 'color', diff --git a/docs/api/advanced/test-specification.md b/docs/api/advanced/test-specification.md index adeeb398e..e542769e4 100644 --- a/docs/api/advanced/test-specification.md +++ b/docs/api/advanced/test-specification.md @@ -86,6 +86,10 @@ A regexp that matches the name of the test in this module. This value will overr The ids of tasks inside of this specification to run. +## testNames 5.1.0 {#testnames} + +The full names of the tests inside of this specification to run, including the parent suites separated with `>`. + ## testTagsFilter 4.1.0 {#testtagsfilter} The [tags filter](/guide/test-tags#syntax) that a test must pass in order to be included in the run. Multiple filters are treated as `AND`. diff --git a/docs/api/advanced/vitest.md b/docs/api/advanced/vitest.md index 2e781faa6..d3dcbee18 100644 --- a/docs/api/advanced/vitest.md +++ b/docs/api/advanced/vitest.md @@ -44,7 +44,7 @@ You can get the latest summary of snapshots via the `vitest.snapshot.summary` pr ## cache -Cache manager that stores the results of the previous test runs. In Vitest itself this is only used by the default sequencer to sort tests. +Cache manager that stores the results of the previous test runs. Vitest uses it to sort tests in the default sequencer and to select tests with [`--last-failed`](/guide/cli#last-failed). ### cache.getTestSpecificationResult 5.0.4 {#cache-gettestspecificationresult} @@ -58,7 +58,7 @@ function getTestSpecificationResult( interface CachedTestFileResult { /** * The file has a known failure. - * Only a complete run of the file clears it. + * A complete run of the file clears it, or a run in which every known failed test passes. */ failed: boolean /** @@ -70,6 +70,12 @@ interface CachedTestFileResult { * It is not set if the file never ran completely. */ lastRun?: number + /** + * Full names of the tests that failed in the last run of the file. + * `--last-failed` runs only these tests. + * It is not set if the failure happened outside of tests, for example in a hook or during collection. + */ + failedTests?: string[] } ``` diff --git a/docs/config/cache.md b/docs/config/cache.md index 95da25190..f0555d890 100644 --- a/docs/config/cache.md +++ b/docs/config/cache.md @@ -9,9 +9,9 @@ outline: deep - **Default:** `true` - **CLI:** `--cache`, `--no-cache` -Store the results of test runs on the file system. Vitest uses them to run failed and longer test files first. +Store the results of test runs on the file system. Vitest uses them to run failed and longer test files first and to select tests with [`--last-failed`](/guide/cli#last-failed). -For every test file, Vitest stores whether the file failed, how long it ran and when it ran last. A run that did not execute the whole file (for example, a run filtered with [`--testNamePattern`](/config/testnamepattern) or cancelled by [`bail`](/config/bail)) can mark the file as failed, but it cannot mark it as passed. +For every test file, Vitest stores whether the file failed, which tests failed, how long it ran and when it ran last. A run that did not execute the whole file (for example, a run filtered with [`--testNamePattern`](/config/testnamepattern) or cancelled by [`bail`](/config/bail)) can mark the file as failed, but it cannot mark it as passed. You can delete the cache by running [`vitest --clearCache`](/guide/cli#clearcache). diff --git a/docs/guide/cli-generated.md b/docs/guide/cli-generated.md index 9ec6f1d97..98a170790 100644 --- a/docs/guide/cli-generated.md +++ b/docs/guide/cli-generated.md @@ -514,6 +514,12 @@ Ignore any unhandled errors that occur Run tests that are affected by the changed files (default: `false`) +### lastFailed + +- **CLI:** `--lastFailed` + +Run only the tests that failed in the last run (default: `false`) + ### sequence.shuffle.files - **CLI:** `--sequence.shuffle.files` diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 14d6bcc25..d63373e43 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -271,3 +271,18 @@ Merges every blob report located in the specified folder (`.vitest/blob/` by def ```sh vitest --merge-reports --reporter=junit ``` + +### last-failed + +- **Type:** `boolean` +- **Default:** `false` + +Run only the tests that failed in the last run. Vitest reads the failures from the [results cache](/config/cache) and skips every other test file and test: + +```sh +vitest run --last-failed +``` + +A test file runs completely if it failed outside of a test, for example in a `beforeAll` hook or during collection. Vitest selects the tests by their full names, so a renamed test does not run until the file runs again without the flag. If no test failed in the last run, Vitest prints `No failed tests found` and exits with code 1 unless [`--passWithNoTests`](/config/passwithnotests) is set. + +The flag can be combined with other filters. For example, `vitest run --last-failed -t login` runs only the failed tests whose names match `login`. In watch mode, only the first run is limited to the failed tests. diff --git a/packages/vitest/src/node/cache/results.ts b/packages/vitest/src/node/cache/results.ts index c6ac2eb2e..f1b8bfe74 100644 --- a/packages/vitest/src/node/cache/results.ts +++ b/packages/vitest/src/node/cache/results.ts @@ -1,5 +1,6 @@ import type { File } from '../../runtime/runner/types' import type { Vitest } from '../core' +import type { TestModule, TestModuleState, TestSuite } from '../reporters/reported-tasks' import type { TestSpecification } from '../test-specification' import { existsSync } from 'node:fs' import { mkdir, readFile, rm, stat } from 'node:fs/promises' @@ -14,7 +15,8 @@ const RESULTS_VERSION = 1 export interface CachedTestFileResult { /** - * The file has a known failure. Only a complete run of the file clears it. + * The file has a known failure. + * A complete run of the file clears it, or a run in which every known failed test passes. */ failed: boolean /** @@ -27,6 +29,11 @@ export interface CachedTestFileResult { * It is not set if the file never ran completely. */ lastRun?: number + /** + * Full names of the tests that failed in the last run of the file. `--last-failed` runs only these tests. + * It is not set if the failure happened outside of tests, for example in a hook or during collection. + */ + failedTests?: string[] } interface ProjectResults { @@ -48,7 +55,7 @@ interface SerializedResults { export class ResultsCache { private projects = new Map() private legacyResults: Map | undefined - private recordedTasks = new WeakSet() + private recordedModules = new WeakSet() private path: string | null // the version of the file that this process read or wrote last private fileStamp: string | undefined @@ -110,30 +117,43 @@ export class ResultsCache { let changed = false for (const specification of specifications) { - const task = specification.testModule?.task - const state = task?.result?.state - // a file that did not start in this run still has the task of a previous run - if (!task || this.recordedTasks.has(task) || !isFinalState(state)) { + const testModule = specification.testModule + const state = testModule?.state() + // a file that did not start in this run still has the module of a previous run + if (!testModule || this.recordedModules.has(testModule) || !isFinalState(state)) { continue } - this.recordedTasks.add(task) + this.recordedModules.add(testModule) const project = this.getProjectResults(specification.project.name) - const results = task.meta.typecheck ? project.typecheck : project.files + const results = testModule.meta().typecheck ? project.typecheck : project.files const file = this.getFilePath(specification) const previous = results.get(file) - const failed = state === 'fail' - const duration = Math.max(task.result?.duration ?? 0, 0) + const failed = state === 'failed' + const duration = Math.max(testModule.diagnostic().duration, 0) - if (!isPartialRun(this.vitest, specification, task)) { - results.set(file, { failed, duration, lastRun: startTime }) + let record: CachedTestFileResult + if (!isPartialRun(this.vitest, specification, testModule.task)) { + record = { failed, duration, lastRun: startTime } + if (failed) { + setFailedTests(record, getFailedTests(testModule)) + } } else if (failed) { - results.set(file, { duration, ...previous, failed }) + record = { duration, ...previous, failed } + // a file that failed outside of tests stays a whole-file failure until a complete run + if (!previous?.failed || previous.failedTests) { + setFailedTests(record, mergeFailedTests(previous?.failedTests, testModule)) + } + } else if (previous?.failedTests) { + record = { ...previous } + setFailedTests(record, mergeFailedTests(previous.failedTests, testModule)) + record.failed = !!record.failedTests } else if (!previous) { - results.set(file, { failed, duration }) + record = { failed, duration } } else { continue } + results.set(file, record) changed = true } @@ -191,8 +211,70 @@ export class ResultsCache { } } -function isFinalState(state: string | undefined): boolean { - return state === 'pass' || state === 'fail' || state === 'skip' || state === 'todo' +function isFinalState(state: TestModuleState | undefined): boolean { + return state === 'passed' || state === 'failed' || state === 'skipped' +} + +function setFailedTests(record: CachedTestFileResult, failedTests: string[] | undefined): void { + if (failedTests) { + record.failedTests = failedTests + } else { + delete record.failedTests + } +} + +interface CollectedTests { + failed: string[] + passed: string[] +} + +// returns true if a failure happened outside of tests, for example in a hook +function collectTests(suite: TestModule | TestSuite, collected: CollectedTests): boolean { + if (suite.errors().length) { + return true + } + for (const child of suite.children) { + if (child.type === 'suite') { + if (collectTests(child, collected)) { + return true + } + continue + } + const state = child.result().state + if (state === 'failed') { + collected.failed.push(child.fullName) + } else if (state === 'passed') { + collected.passed.push(child.fullName) + } + } + return false +} + +function getFailedTests(testModule: TestModule): string[] | undefined { + const collected: CollectedTests = { failed: [], passed: [] } + if (collectTests(testModule, collected) || !collected.failed.length) { + return undefined + } + return collected.failed +} + +// the known failures after a partial run; a test that did not run keeps its state +function mergeFailedTests( + previous: string[] | undefined, + testModule: TestModule, +): string[] | undefined { + const collected: CollectedTests = { failed: [], passed: [] } + if (collectTests(testModule, collected)) { + return undefined + } + const names = new Set(previous) + for (const name of collected.passed) { + names.delete(name) + } + for (const name of collected.failed) { + names.add(name) + } + return names.size ? Array.from(names) : undefined } // a partial run does not prove that the whole file passes @@ -206,6 +288,7 @@ export function isPartialRun( task.containsOnly || specification.testLines?.length || specification.testIds?.length || + specification.testNames?.length || specification.testNamePattern || specification.testTagsFilter?.length || vitest.getGlobalTestNamePattern() || diff --git a/packages/vitest/src/node/cli/cli-api.ts b/packages/vitest/src/node/cli/cli-api.ts index 9802cda63..a65804d54 100644 --- a/packages/vitest/src/node/cli/cli-api.ts +++ b/packages/vitest/src/node/cli/cli-api.ts @@ -23,6 +23,10 @@ export interface CliOptions extends UserConfig { * Override the watch mode */ run?: boolean + /** + * Run only the tests that failed in the last run + */ + lastFailed?: boolean /** * Removes colors from the console output */ diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index 0218263e1..aeab91bd7 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -505,6 +505,9 @@ export const cliOptionsConfig: VitestCLIOptions = { description: 'Run tests that are affected by the changed files (default: `false`)', argument: '[since]', }, + lastFailed: { + description: 'Run only the tests that failed in the last run (default: `false`)', + }, sequence: { description: 'Options for how tests should be sorted', argument: '', @@ -1030,4 +1033,7 @@ export const collectCliOptionsConfig: VitestCLIOptions = { description: 'Print only tests that are affected by the changed files (default: `false`)', argument: '[since]', }, + lastFailed: { + description: 'Print only the tests that failed in the last run (default: `false`)', + }, } diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index a264c6b53..92e7965ec 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -150,6 +150,7 @@ export class Vitest { /** @internal */ configOverride: Partial = {} /** @internal */ filenamePattern?: string[] /** @internal */ _sourceFilterResult?: { affected: number; total: number } + /** @internal */ _failedFilterResult?: { selected: number; total: number } /** @internal */ runningPromise?: Promise /** @internal */ closingPromise?: Promise /** @internal */ cancelPromise?: Promise @@ -886,7 +887,7 @@ export class Vitest { }) } - // if run with --changed, don't exit if no tests are found + // if run with --changed or --last-failed, don't exit if no tests are found if (!specifications.length) { await this._traces.$('vitest.test_run', async () => { await this._testRun.start([]) @@ -898,7 +899,9 @@ export class Vitest { await this.reportCoverage(coverage, true) }) - if (!this.config.watch || !(this.config.changed || this.config.related?.length)) { + const narrowed = + this.config.changed || this.config.related?.length || this.config.cliOptions.lastFailed + if (!this.config.watch || !narrowed) { throw new FilesNotFoundError() } } @@ -1091,6 +1094,7 @@ export class Vitest { this.config.changed = false this.config.related = undefined this._sourceFilterResult = undefined + this._failedFilterResult = undefined }) return await this.runningPromise @@ -1200,6 +1204,7 @@ export class Vitest { config.testNamePattern, specification.testLines, specification.testIds, + specification.testNames, tagsFilter, globalHasOnly, false, @@ -1229,6 +1234,7 @@ export class Vitest { config.testNamePattern, specification.testLines, specification.testIds, + specification.testNames, tagsFilter, hasOnly, false, @@ -1291,6 +1297,7 @@ export class Vitest { this.config.changed = false this.config.related = undefined this._sourceFilterResult = undefined + this._failedFilterResult = undefined }) return await this.runningPromise diff --git a/packages/vitest/src/node/logger.ts b/packages/vitest/src/node/logger.ts index 200c99afb..5c490871a 100644 --- a/packages/vitest/src/node/logger.ts +++ b/packages/vitest/src/node/logger.ts @@ -168,13 +168,17 @@ export class Logger { const config = this.ctx.config // test files were found, but none of them depend on the changed files const noAffectedTests = !!this.ctx._sourceFilterResult?.total - const message = !noAffectedTests - ? 'No test files found' - : config.related?.length - ? 'No affected test files found' - : 'No changed files found' - - if (config.watch && noAffectedTests) { + // test files were found, but none of them failed in the last run + const noFailedTests = !!this.ctx._failedFilterResult?.total + const message = noFailedTests + ? 'No failed tests found' + : !noAffectedTests + ? 'No test files found' + : config.related?.length + ? 'No affected test files found' + : 'No changed files found' + + if (config.watch && (noAffectedTests || noFailedTests)) { this.log(`${message}\n`) } else if (config.watch) { this.log(c.red(`${message}. You can change the file name pattern by pressing "p"\n`)) @@ -184,7 +188,7 @@ export class Logger { this.error(c.red(`${message}, exiting with code 1\n`)) } - if (noAffectedTests) { + if (noAffectedTests || noFailedTests) { return } diff --git a/packages/vitest/src/node/pool.ts b/packages/vitest/src/node/pool.ts index b8607c699..7931ffffa 100644 --- a/packages/vitest/src/node/pool.ts +++ b/packages/vitest/src/node/pool.ts @@ -159,6 +159,7 @@ export function createPool(ctx: Vitest): ProcessPool { testLocations: spec.testLines, testNamePattern: spec.testNamePattern, testIds: spec.testIds, + testNames: spec.testNames, testTagsFilter: spec.testTagsFilter, })), invalidates, diff --git a/packages/vitest/src/node/pools/browser.ts b/packages/vitest/src/node/pools/browser.ts index 524596bd1..47738547f 100644 --- a/packages/vitest/src/node/pools/browser.ts +++ b/packages/vitest/src/node/pools/browser.ts @@ -306,12 +306,14 @@ async function groupSpecifications( const docblocks = await Promise.all(specs.map((spec) => getSpecificationDocblock(spec, code))) specs.forEach((spec, index) => { - const { project, moduleId, testLines, testIds, testNamePattern, testTagsFilter } = spec + const { project, moduleId, testLines, testIds, testNames, testNamePattern, testTagsFilter } = + spec const files = groupedFiles.get(project) || [] files.push({ filepath: moduleId, testLocations: testLines, testIds, + testNames, testNamePattern, testTagsFilter, fileTags: docblocks[index].tags, diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 5c1da68e3..3e8d7f4a5 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -678,6 +678,14 @@ export abstract class BaseReporter implements Reporter { ) } + private printLastFailedSummary(): void { + const result = this.ctx._failedFilterResult + if (!result) { + return + } + this.log(padSummaryTitle('Last Failed'), `${result.selected} of ${result.total} test files`) + } + reportTestSummary(files: File[], errors: unknown[], leakCount: number): void { this.log() @@ -697,6 +705,7 @@ export abstract class BaseReporter implements Reporter { this.log(padSummaryTitle('Test Files'), getStateString(affectedFiles)) this.printSourceFilterSummary() + this.printLastFailedSummary() this.log(padSummaryTitle('Tests'), getStateString(tests)) if (this.ctx.projects.some((c) => c.config.typecheck.enabled)) { diff --git a/packages/vitest/src/node/specifications.ts b/packages/vitest/src/node/specifications.ts index 72fe605ce..79edc9897 100644 --- a/packages/vitest/src/node/specifications.ts +++ b/packages/vitest/src/node/specifications.ts @@ -32,7 +32,13 @@ export class VitestSpecifications { } public async getRelevantTestSpecifications(filters: string[] = []): Promise { - return this.filterTestsBySource(await this.globTestSpecifications(filters)) + const specifications = await this.filterTestsBySource( + await this.globTestSpecifications(filters), + ) + if (this.vitest.config.cliOptions.lastFailed) { + return this.filterFailedTests(specifications) + } + return specifications } public async globTestSpecifications(filters: string[] = []): Promise { @@ -156,4 +162,25 @@ export class VitestSpecifications { this.vitest._sourceFilterResult = { affected: affectedSpecs.length, total: specs.length } return affectedSpecs } + + private filterFailedTests(specs: TestSpecification[]): TestSpecification[] { + const failed: TestSpecification[] = [] + for (const spec of specs) { + const result = this.vitest.cache.getTestSpecificationResult(spec) + if (!result?.failed) { + continue + } + failed.push( + result.failedTests + ? spec.project.createSpecification( + spec.moduleId, + { testNames: result.failedTests, testLines: spec.testLines }, + spec.pool, + ) + : spec, + ) + } + this.vitest._failedFilterResult = { selected: failed.length, total: specs.length } + return failed + } } diff --git a/packages/vitest/src/node/test-specification.ts b/packages/vitest/src/node/test-specification.ts index 9dc9b6021..f196dd820 100644 --- a/packages/vitest/src/node/test-specification.ts +++ b/packages/vitest/src/node/test-specification.ts @@ -9,6 +9,7 @@ import { generateFileHash } from '../utils/tasks' export interface TestSpecificationOptions { testNamePattern?: RegExp testIds?: string[] + testNames?: string[] testLines?: number[] testTagsFilter?: string[] } @@ -48,6 +49,10 @@ export class TestSpecification { * The ids of tasks inside of this specification to run. */ public testIds: string[] | undefined + /** + * The full names of the tests inside of this specification to run, including the parent suites separated with `>`. + */ + public testNames: string[] | undefined /** * The tags of tests to run. */ @@ -84,6 +89,7 @@ export class TestSpecification { this.testLines = testLinesOrOptions.testLines this.testNamePattern = testLinesOrOptions.testNamePattern this.testIds = testLinesOrOptions.testIds + this.testNames = testLinesOrOptions.testNames this.testTagsFilter = testLinesOrOptions.testTagsFilter } } @@ -110,6 +116,7 @@ export class TestSpecification { pool: this.pool, testLines: this.testLines, testIds: this.testIds, + testNames: this.testNames, testNamePattern: this.testNamePattern, testTagsFilter: this.testTagsFilter, }, diff --git a/packages/vitest/src/runtime/runner/collect.ts b/packages/vitest/src/runtime/runner/collect.ts index c1d90bc1b..31ac4e75e 100644 --- a/packages/vitest/src/runtime/runner/collect.ts +++ b/packages/vitest/src/runtime/runner/collect.ts @@ -30,6 +30,7 @@ export async function collectTests( const testLocations = typeof spec === 'string' ? undefined : spec.testLocations const testNamePattern = typeof spec === 'string' ? undefined : spec.testNamePattern const testIds = typeof spec === 'string' ? undefined : spec.testIds + const testNames = typeof spec === 'string' ? undefined : spec.testNames const testTagsFilter = typeof spec === 'object' && spec.testTagsFilter ? createTagsFilter(spec.testTagsFilter, config.tags) @@ -142,6 +143,7 @@ export async function collectTests( testNamePattern ?? config.testNamePattern, testLocations, testIds, + testNames, testTagsFilter ?? defaultTagsFilter, hasOnlyTasks, false, diff --git a/packages/vitest/src/runtime/runner/types.ts b/packages/vitest/src/runtime/runner/types.ts index d57028ead..0ccc66583 100644 --- a/packages/vitest/src/runtime/runner/types.ts +++ b/packages/vitest/src/runtime/runner/types.ts @@ -1595,6 +1595,7 @@ export interface FileSpecification { testNamePattern?: RegExp | undefined testTagsFilter?: string[] | undefined testIds?: string[] | undefined + testNames?: string[] | undefined } export type VitestRunnerImportSource = 'collect' | 'setup' diff --git a/packages/vitest/src/runtime/types/utils.ts b/packages/vitest/src/runtime/types/utils.ts index b20580119..c904deecf 100644 --- a/packages/vitest/src/runtime/types/utils.ts +++ b/packages/vitest/src/runtime/types/utils.ts @@ -5,6 +5,7 @@ export type SerializedTestSpecification = [ pool: string testLines?: number[] | undefined testIds?: string[] | undefined + testNames?: string[] | undefined testNamePattern?: RegExp | undefined testTagsFilter?: string[] | undefined }, diff --git a/packages/vitest/src/utils/tasks.ts b/packages/vitest/src/utils/tasks.ts index 15fed5ee2..9099a4cff 100644 --- a/packages/vitest/src/utils/tasks.ts +++ b/packages/vitest/src/utils/tasks.ts @@ -234,6 +234,7 @@ export function interpretTaskModes( namePattern?: string | RegExp, testLocations?: number[] | undefined, testIds?: string[] | undefined, + testNames?: string[] | undefined, testTagsFilter?: ((testTags: string[]) => boolean) | undefined, onlyMode?: boolean, parentIsOnly?: boolean, @@ -243,6 +244,7 @@ export function interpretTaskModes( const testLocationsSet = testLocations !== undefined && testLocations.length !== 0 ? new Set(testLocations) : undefined const testIdsSet = testIds ? new Set(testIds) : undefined + const testNamesSet = testNames ? new Set(testNames) : undefined const traverseSuite = ( suite: Suite, @@ -300,6 +302,9 @@ export function interpretTaskModes( if (testIdsSet && !testIdsSet.has(t.id)) { t.mode = 'skip' } + if (testNamesSet && !testNamesSet.has(t.fullTestName)) { + t.mode = 'skip' + } if (testTagsFilter && !testTagsFilter(t.tags || [])) { t.mode = 'skip' } diff --git a/test/e2e/test/cache.test.ts b/test/e2e/test/cache.test.ts index e0dae23be..51100b61e 100644 --- a/test/e2e/test/cache.test.ts +++ b/test/e2e/test/cache.test.ts @@ -3,7 +3,7 @@ import { createHash } from 'node:crypto' import { existsSync, readFileSync, writeFileSync } from 'node:fs' import { resolve } from 'pathe' import { expect, test } from 'vitest' -import { replaceRoot, runInlineTests, runVitest } from '../../test-utils' +import { replaceRoot, runInlineTests, runVitest, runVitestCli } from '../../test-utils' const cacheDir = `node_modules/.vite/vitest/${createHash('sha1').update('').digest('hex')}` const resultsFile = `${cacheDir}/results.json` @@ -36,6 +36,40 @@ function readResults(ctx: Vitest | undefined) { }) } +const mixedTest = /* js */ ` +import { describe, expect, test } from 'vitest' +import { value } from './value.js' +test('passes', () => {}) +test('fails', () => { + expect(value).toBe(1) +}) +describe('nested', () => { + test('passes too', () => {}) + test('fails too', () => { + expect(value).toBe(1) + }) +}) +` + +const hookTest = /* js */ ` +import { beforeAll, test } from 'vitest' +beforeAll(() => { + throw new Error('hook failed') +}) +test('never runs', () => {}) +` + +const failingProject = { + 'mixed.test.js': mixedTest, + 'hook.test.js': hookTest, + 'passing.test.js': passingTest, + 'value.js': 'export const value = 2', +} + +function lastFailedSummary(stdout: string) { + return stdout.split('\n').find((line) => line.includes('Last Failed')) +} + function recordOrder(order: string[]) { return { onTestModuleStart(testModule: TestModule) { @@ -69,6 +103,9 @@ test('stores the results of test files by project', async () => { "api/basic.test.js": { "duration": "", "failed": true, + "failedTests": [ + "fails", + ], "lastRun": "", }, }, @@ -103,6 +140,7 @@ test('returns the cached result of a specification', async () => { failed: true, duration: expect.any(Number), lastRun: expect.any(Number), + failedTests: ['fails'], }) expect(result).toEqual(readRawResults(ctx).projects['unit:node'].files['basic.test.js']) @@ -121,6 +159,7 @@ test('a partial run cannot mark a failed file as passed', async () => { failed: true, duration: expect.any(Number), lastRun: expect.any(Number), + failedTests: ['fails'], }) const filteredRun = await runVitest({ root: fs.root, cache: true, testNamePattern: 'passes' }) @@ -159,6 +198,9 @@ test('a partial run records a failure without the time of the run', async () => "basic.test.js": { "duration": "", "failed": true, + "failedTests": [ + "fails", + ], }, }, }, @@ -427,3 +469,305 @@ test('the deprecated methods of the cache still work in a custom sequencer', asy `) expect(order).toEqual(['unknown.test.js', 'long.test.js', 'short.test.js']) }) + +test('stores the names of the failed tests', async () => { + const { ctx, fs } = await runInlineTests(failingProject, { cache: true }) + + expect(readResults(ctx)).toMatchInlineSnapshot(` + { + "projects": { + "": { + "files": { + "hook.test.js": { + "duration": "", + "failed": true, + "lastRun": "", + }, + "mixed.test.js": { + "duration": "", + "failed": true, + "failedTests": [ + "fails", + "nested > fails too", + ], + "lastRun": "", + }, + "passing.test.js": { + "duration": "", + "failed": false, + "lastRun": "", + }, + }, + }, + }, + "version": 1, + } + `) + + const [specification] = ctx!.getModuleSpecifications(fs.resolveFile('mixed.test.js')) + const failedTests = Array.from( + specification.testModule!.children.allTests('failed'), + (test) => test.fullName, + ) + expect(ctx!.cache.getTestSpecificationResult(specification)!.failedTests).toEqual(failedTests) +}) + +test('--last-failed runs only the tests that failed in the last run', async () => { + const { fs } = await runInlineTests(failingProject, { cache: true }) + + const rerun = await runVitest({ root: fs.root, cache: true, $cliOptions: { lastFailed: true } }) + + expect(rerun.testTree()).toMatchInlineSnapshot(` + { + "hook.test.js": { + "never runs": "skipped", + }, + "mixed.test.js": { + "fails": "failed", + "nested": { + "fails too": "failed", + "passes too": "skipped", + }, + "passes": "skipped", + }, + } + `) + expect(lastFailedSummary(rerun.stdout)).toMatchInlineSnapshot(`"Last Failed 2 of 3 test files"`) +}) + +test('--last-failed respects the test name pattern', async () => { + const { fs } = await runInlineTests(failingProject, { cache: true }) + + const rerun = await runVitest({ + root: fs.root, + cache: true, + testNamePattern: 'fails too', + $cliOptions: { lastFailed: true }, + }) + + expect(rerun.testTree()).toMatchInlineSnapshot(` + { + "hook.test.js": { + "never runs": "skipped", + }, + "mixed.test.js": { + "fails": "skipped", + "nested": { + "fails too": "failed", + "passes too": "skipped", + }, + "passes": "skipped", + }, + } + `) +}) + +test('a --last-failed run in which the failed tests pass clears the failure', async () => { + const { fs } = await runInlineTests(failingProject, { cache: true }) + fs.editFile('value.js', () => 'export const value = 1') + + const rerun = await runVitest({ root: fs.root, cache: true, $cliOptions: { lastFailed: true } }) + + expect(rerun.testTree()).toMatchInlineSnapshot(` + { + "hook.test.js": { + "never runs": "skipped", + }, + "mixed.test.js": { + "fails": "passed", + "nested": { + "fails too": "passed", + "passes too": "skipped", + }, + "passes": "skipped", + }, + } + `) + expect(readResults(rerun.ctx)).toMatchInlineSnapshot(` + { + "projects": { + "": { + "files": { + "hook.test.js": { + "duration": "", + "failed": true, + "lastRun": "", + }, + "mixed.test.js": { + "duration": "", + "failed": false, + "lastRun": "", + }, + "passing.test.js": { + "duration": "", + "failed": false, + "lastRun": "", + }, + }, + }, + }, + "version": 1, + } + `) + + fs.editFile('hook.test.js', (content) => content.replace("throw new Error('hook failed')", '')) + const hookRerun = await runVitest({ + root: fs.root, + cache: true, + $cliOptions: { lastFailed: true }, + }) + + expect(hookRerun.stderr).toBe('') + expect(hookRerun.testTree()).toMatchInlineSnapshot(` + { + "hook.test.js": { + "never runs": "passed", + }, + } + `) + + const emptyRerun = await runVitest({ + root: fs.root, + cache: true, + $cliOptions: { lastFailed: true }, + }) + + expect(emptyRerun.stderr).toMatchInlineSnapshot(` + "No failed tests found, exiting with code 1 + + " + `) + expect(emptyRerun.exitCode).toBe(1) +}) + +test('a changed test file still runs only the failed tests', async () => { + const { fs } = await runInlineTests(failingProject, { cache: true }) + fs.editFile('mixed.test.js', (content) => + content.replace("test('passes'", "test('added', () => {})\ntest('passes'"), + ) + + const rerun = await runVitest({ root: fs.root, cache: true, $cliOptions: { lastFailed: true } }) + + expect(rerun.testTree()).toMatchInlineSnapshot(` + { + "hook.test.js": { + "never runs": "skipped", + }, + "mixed.test.js": { + "added": "skipped", + "fails": "failed", + "nested": { + "fails too": "failed", + "passes too": "skipped", + }, + "passes": "skipped", + }, + } + `) + expect(readResults(rerun.ctx).projects[''].files['mixed.test.js']).toMatchInlineSnapshot(` + { + "duration": "", + "failed": true, + "failedTests": [ + "fails", + "nested > fails too", + ], + "lastRun": "", + } + `) +}) + +test('a renamed failed test does not run', async () => { + const { fs } = await runInlineTests(failingProject, { cache: true }) + fs.editFile('mixed.test.js', (content) => content.replace("test('fails'", "test('still fails'")) + + const rerun = await runVitest({ root: fs.root, cache: true, $cliOptions: { lastFailed: true } }) + + expect(rerun.testTree()).toMatchInlineSnapshot(` + { + "hook.test.js": { + "never runs": "skipped", + }, + "mixed.test.js": { + "nested": { + "fails too": "failed", + "passes too": "skipped", + }, + "passes": "skipped", + "still fails": "skipped", + }, + } + `) +}) + +test('a partial run updates the failed tests', async () => { + const { fs } = await runInlineTests(failingProject, { cache: true }) + fs.editFile('value.js', () => 'export const value = 1') + + const partialRun = await runVitest({ root: fs.root, cache: true, testNamePattern: '^fails$' }) + + expect(partialRun.stderr).toBe('') + expect(readResults(partialRun.ctx).projects[''].files['mixed.test.js']).toMatchInlineSnapshot(` + { + "duration": "", + "failed": true, + "failedTests": [ + "nested > fails too", + ], + "lastRun": "", + } + `) + + const rerun = await runVitest({ root: fs.root, cache: true, $cliOptions: { lastFailed: true } }) + + expect(rerun.testTree()).toMatchInlineSnapshot(` + { + "hook.test.js": { + "never runs": "skipped", + }, + "mixed.test.js": { + "fails": "skipped", + "nested": { + "fails too": "passed", + "passes too": "skipped", + }, + "passes": "skipped", + }, + } + `) +}) + +test('a rerun in watch mode runs the whole file', async () => { + const { fs } = await runInlineTests(failingProject, { cache: true }) + + const { vitest } = await runVitest({ + root: fs.root, + cache: true, + watch: true, + $cliOptions: { lastFailed: true }, + }) + await vitest.waitForStdout('Tests 2 failed | 3 skipped (5)') + + fs.editFile('value.js', () => 'export const value = 1') + await vitest.waitForStdout('RERUN') + await vitest.waitForStdout('Tests 4 passed | 1 skipped (5)') +}) + +test('vitest list --last-failed prints only the failed tests', async () => { + const { fs } = await runInlineTests(failingProject, { cache: true }) + + const { stdout, stderr, exitCode } = await runVitestCli( + 'list', + `--root=${fs.root}`, + '--last-failed', + ) + + expect(stderr).toBe('') + expect(stdout).toMatchInlineSnapshot(` + "hook.test.js > never runs + mixed.test.js > fails + mixed.test.js > nested > fails too + " + `) + expect(exitCode).toBe(0) +})