diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index dd00153ab..c5021809e 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -1096,6 +1096,10 @@ export default ({ mode }: { mode: string }) => { text: 'Parallel and Sequential Test Files', link: '/guide/recipes/parallel-sequential', }, + { + text: 'Running Only Affected Tests', + link: '/guide/recipes/affected-tests', + }, ], }, { diff --git a/docs/.vitepress/scripts/cli-generator.ts b/docs/.vitepress/scripts/cli-generator.ts index 50055c7a9..b0931fb6f 100644 --- a/docs/.vitepress/scripts/cli-generator.ts +++ b/docs/.vitepress/scripts/cli-generator.ts @@ -9,7 +9,7 @@ const cliTablePath = resolve(docsDir, './guide/cli-generated.md') const nonNullable = (value: T): value is NonNullable => value !== null && value !== undefined -const skipCli = new Set(['mergeReports', 'shard']) +const skipCli = new Set(['mergeReports', 'shard', 'sinceLastRun']) const skipConfig = new Set([ 'config', diff --git a/docs/config/experimental.md b/docs/config/experimental.md index b2f7010eb..d2ecfc65f 100644 --- a/docs/config/experimental.md +++ b/docs/config/experimental.md @@ -316,7 +316,7 @@ If you are running tests in Deno, TypeScript files are processed by the runtime ## experimental.vcsProvider 4.1.1 {#experimental-vcsprovider} -- **Type:** `VCSProvider | string` +- **Type:** `'git' | 'mtime' | VCSProvider | string` ```ts interface VCSProvider { @@ -326,14 +326,55 @@ interface VCSProvider { interface VCSProviderOptions { root: string changedSince?: string | boolean + /** Resolves the dependencies of the test files of the run. */ + resolver: ModulesResolver +} + +interface ModulesResolver { + /** + * Every file whose change can affect the test files of the run. + */ + getDependencies(): Promise +} + +interface ModuleDependency { + /** Absolute path of the file. */ + file: string + /** + * Start time of the oldest run that recorded the file, in milliseconds. + * Not set when the dependency comes from the static module graph. + */ + recordedAt?: number } ``` - **Default:** `'git'` -Custom provider for detecting changed files. Used with the [`--changed`](/guide/cli#changed) flag to determine which files have been modified. +Provider for detecting changed files. Used with the [`--changed`](/guide/cli#changed) flag to determine which files have been modified. + +Vitest has two built-in providers: + +- `'git'` returns the files that differ from the `--changed` reference, plus staged and untracked files. +- `'mtime'` 5.1.0 returns the recorded dependencies whose modification time is newer than the run that recorded them, and deleted dependencies. It needs [`experimental.recordDependencies`](#experimental-recorddependencies): a dependency without a record is always reported as changed, so every test runs until the records exist. It does not need Git and ignores the `--changed` reference. A `--changed` run updates the records of the tests it ran, so the next `--changed` run selects only the tests whose dependencies changed since, and the test files that failed in their last run. + +```ts [vitest.config.ts] +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + experimental: { + recordDependencies: true, + vcsProvider: 'mtime', + }, + }, +}) +``` + +::: warning +The `mtime` provider compares file times with the clock of the machine that ran the tests. A checkout or a copy that sets file times in the past is not detected. +::: -By default, Vitest uses Git to detect changed files. You can provide a custom implementation of the `VCSProvider` interface to use a different version control system: +You can provide a custom implementation of the `VCSProvider` interface to use a different version control system. `options.resolver.getDependencies()` returns every file the test files of the run depend on, so a provider can limit its work to these files: ```ts [vitest.config.ts] import { defineConfig } from 'vitest/config' @@ -342,8 +383,9 @@ export default defineConfig({ test: { experimental: { vcsProvider: { - async findChangedFiles({ root, changedSince }) { - // return paths of changed files + async findChangedFiles({ root, changedSince, resolver }) { + const dependencies = await resolver.getDependencies() + // return the paths of the changed files return [] }, }, @@ -375,7 +417,7 @@ export default { } ``` -## experimental.recordDependencies 5.0.4 {#experimental-recorddependencies} +## experimental.recordDependencies 5.1.0 {#experimental-recorddependencies} - **Type:** `boolean` - **Default:** `false` diff --git a/docs/guide/cli-generated.md b/docs/guide/cli-generated.md index 1c6b5dd08..799a387e8 100644 --- a/docs/guide/cli-generated.md +++ b/docs/guide/cli-generated.md @@ -1018,7 +1018,7 @@ Controls whether Vitest will use Node.js Loader API to process in-source or mock - **CLI:** `--experimental.vcsProvider ` - **Config:** [experimental.vcsProvider](/config/experimental#experimental-vcsprovider) -Custom provider for detecting changed files. (default: `git`) +Provider for detecting changed files: `git`, `mtime` (requires `experimental.recordDependencies`) or a path to a custom provider. (default: `git`) ### experimental.recordDependencies diff --git a/docs/guide/cli.md b/docs/guide/cli.md index d63373e43..f9b80c997 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -272,6 +272,27 @@ Merges every blob report located in the specified folder (`.vitest/blob/` by def vitest --merge-reports --reporter=junit ``` +### since-last-run 5.1.0 {#since-last-run} + +- **Type:** `boolean` +- **Default:** `false` + +Run only the tests whose dependencies were modified since the last run of the test. A test is selected when one of the files it loaded during its last run has a newer modification time than that run, or was deleted. A test that never ran with this flag has no record and always runs. A test file that failed in its last run also runs, until it passes, because the cause of a failure can be outside the recorded files, for example an environment variable or a server that was down. + +```sh +vitest run --since-last-run +``` + +The flag enables [`changed`](/config/changed), [`experimental.recordDependencies`](/config/experimental#experimental-recorddependencies) and the `mtime` [provider](/config/experimental#experimental-vcsprovider). Every run with this flag updates the records of the tests it ran, so the next run selects only the tests affected by changes made in between. To run every test again, run Vitest without the flag, or delete the cache with [`--clearCache`](#clearcache). + +The flag requires [`cache`](/config/cache). Vitest throws an error if the cache is disabled. + +::: warning +The modification time comes from the file system. A `git checkout` or an unpacked archive that sets old modification times is not detected. Files that a test did not load during its last run, for example a new `.env` file or a new `__mocks__` file, are not detected either. See [`experimental.recordDependencies`](/config/experimental#experimental-recorddependencies) for the full list of recorded files. +::: + +See [Running Only Affected Tests](/guide/recipes/affected-tests) for a comparison with `--changed`. + ### last-failed - **Type:** `boolean` diff --git a/docs/guide/recipes/affected-tests.md b/docs/guide/recipes/affected-tests.md new file mode 100644 index 000000000..ac7973e5e --- /dev/null +++ b/docs/guide/recipes/affected-tests.md @@ -0,0 +1,66 @@ +--- +title: Running Only Affected Tests | Recipes +--- + +# Running Only Affected Tests + +A full run of a large suite is the right check before a merge, but it is slow feedback for a small change. If you edited one module, only the tests that load that module can change their result. Vitest has two flags that select those tests. They differ in one question: changed compared to what? + +- [`--changed`](/config/changed) compares the working tree with a Git reference. +- [`--since-last-run`](/guide/cli#since-last-run) compares every file a test loaded with the time of the test's last run. + +## `--changed`: what differs from a Git reference + +```sh +vitest run --changed # uncommitted changes, staged and unstaged +vitest run --changed HEAD~1 # the last commit and uncommitted changes +vitest run --changed origin/main # everything on this branch +``` + +Vitest asks Git for the changed files, then finds the test files that import them, directly or through other modules. A test runs when any file in its import graph is in the list. + +The reference is the baseline, so the same command gives the same selection until the baseline moves. This fits checks that are defined in terms of Git: a pull request against `origin/main`, or a pre-push hook that covers the commits you are about to push. It requires a Git repository, and a change that is committed and merged stops being "changed", even if the tests never ran against it on your machine. + +## `--since-last-run`: what changed since the test last ran + +```sh +vitest run --since-last-run +``` + +The first run with `--since-last-run` runs every test and records the files each test loaded, including dynamic imports, mocks, setup files and snapshot files. Every later run compares the modification time of those files with the time of the run that recorded them. A test runs again when one of its files is newer or was deleted, or when the test has no record yet. A test file that failed in its last run also runs again, until it passes, even if the fix was outside the recorded files. + +Each run updates the records of the tests it ran, so the baseline moves with you: after a run, only the next edit selects tests again. Git is not involved, so the flag works in a fresh container, in a worktree with an unrelated history, or in a directory that is not a repository. A new test file, or one that was never recorded, always runs. + +The records live in the [cache](/config/cache). `vitest run --clearCache` deletes them, and the next `--since-last-run` run is a full run again. + +## Which one to pick + +| You want to run the tests affected by… | Use | +| ---------------------------------------------------------------------- | --------------------------------- | +| a branch, before opening or updating a pull request | `--changed origin/main` | +| the commits you are about to push | `--changed ` | +| your uncommitted edits, in a Git repository | `--changed` or `--since-last-run` | +| your edits since you last ran the tests, committed or not | `--since-last-run` | +| edits in a directory without Git, or with a history you do not control | `--since-last-run` | + +`--changed` is the better fit when the baseline is shared with other people: the reference is the same on every machine and in CI. `--since-last-run` is the better fit when the baseline is your own last run: it does not matter whether you committed in between, or whether Git is there at all. + +## Agents + +AI coding agents run tests after almost every edit. They often work in a container or a temporary worktree, commit in the middle of a task, and start a new process for every command. A Git baseline fits none of that: `--changed` is empty right after a commit, and it needs a repository. `--since-last-run` tracks what the agent actually ran, so each `vitest run --since-last-run` covers exactly the edits since the previous one, and the first command in a fresh environment is a full run that builds the records. + +Tell the agent about it in the instructions file of your project: + +```md [AGENTS.md] +- After a change, run `vitest run --since-last-run` to test only what the change affects. +- Before you finish, run `vitest run` once for the whole suite. +``` + +Two things the records cannot see: a file that no test loaded during the last run (a new `.env` file, a new `__mocks__` file), and modification times set in the past by a checkout or an unpacked archive. The full run at the end covers both. + +## See also + +- [`changed`](/config/changed) +- [`--since-last-run`](/guide/cli#since-last-run) +- [`experimental.recordDependencies`](/config/experimental#experimental-recorddependencies) +- [Test Filtering](/guide/filtering) diff --git a/packages/vitest/src/node/affected-modules.ts b/packages/vitest/src/node/affected-modules.ts index c6a5917ed..25de3910e 100644 --- a/packages/vitest/src/node/affected-modules.ts +++ b/packages/vitest/src/node/affected-modules.ts @@ -2,6 +2,7 @@ import type { DevEnvironment } from 'vite' import type { Vitest } from './core' import type { TestProject } from './project' import type { TestSpecification } from './test-specification' +import type { ModuleDependency, ModulesResolver } from './vcs/vcs' import { existsSync } from 'node:fs' import os from 'node:os' import { cleanUrl } from '@vitest/utils/helpers' @@ -25,18 +26,32 @@ interface ModuleNode { /** * Finds test specifications that statically depend on any of the changed files. */ -export class AffectedModulesResolver { +export class AffectedModulesResolver implements ModulesResolver { private existsCache = new Map() private transformConcurrency = os.availableParallelism() private activeTransforms = 0 private transformQueue: Array<() => void> = [] + private related: string[] = [] + // set while `getDependencies` walks the graph + walked: Set | undefined constructor( private vitest: Vitest, - private related: string[], + private specs: TestSpecification[], ) {} - async resolve(specs: TestSpecification[]): Promise { + // the graph is only known after transforming it, so it is walked once without changes + async getDependencies(): Promise { + this.walked = new Set() + await this.getAffectedSpecifications([]) + const files = new Set(Array.from(this.walked, (id) => cleanUrl(id))) + this.walked = undefined + return Array.from(files, (file) => ({ file })) + } + + async getAffectedSpecifications(related: string[]): Promise { + this.related = related + const specs = this.specs if (this.dependsOnConfig(this.vitest.vite.config.configFileDependencies)) { return specs } @@ -286,7 +301,7 @@ class ProjectGraph { constructor( private project: TestProject, private environment: DevEnvironment, - private resolver: AffectedModulesResolver, + public resolver: AffectedModulesResolver, ) {} getModule(id: string): Promise { @@ -443,6 +458,8 @@ class GraphWalk { if (!node) { return } + this.graph.resolver.walked?.add(id) + node.watchedFiles.forEach((file) => this.graph.resolver.walked?.add(file)) // a file imported with a query is a separate module if (node.failed || this.changed.has(cleanUrl(id))) { this.changed.add(id) diff --git a/packages/vitest/src/node/cache/dependencies.ts b/packages/vitest/src/node/cache/dependencies.ts index 94c69955b..62bdb25e5 100644 --- a/packages/vitest/src/node/cache/dependencies.ts +++ b/packages/vitest/src/node/cache/dependencies.ts @@ -6,21 +6,31 @@ import { mkdir, readFile, rm } from 'node:fs/promises' import { cleanUrl } from '@vitest/utils/helpers' import { dirname, isAbsolute, resolve } from 'pathe' import { createDebugger } from '../../utils/debugger' +import { getEnvFiles } from '../affected-modules' import { tagCacheDir } from './cachedirTag' import { atomicWriteFile } from './fsModuleCache' import { getFileStamp, getRootRelativePath, isPartialRun } from './results' const debug = createDebugger('vitest:cache:dependencies') -const DEPENDENCIES_VERSION = 1 +const DEPENDENCIES_VERSION = 2 + +export interface DependencyRecord { + /** Start time of the run that recorded the dependencies. */ + at: number + /** Indices into `modules`. */ + deps: number[] +} export interface ProjectDependencies { /** Absolute paths without a query, referenced by index. */ modules: string[] moduleIndex: Map - globalSetup?: number[] - /** Root-relative test file path to the indices of every module it loaded. */ - files: Map + /** The config file, its imports and the `.env` files. */ + config?: DependencyRecord + globalSetup?: DependencyRecord + /** Keyed by the root-relative test file path. */ + files: Map } interface SerializedDependencies { @@ -29,8 +39,9 @@ interface SerializedDependencies { string, { modules: string[] - globalSetup?: number[] - files: Record + config?: DependencyRecord + globalSetup?: DependencyRecord + files: Record } > } @@ -92,11 +103,12 @@ export class DependenciesCache { const root = this.vitest.config.root const result = new Map() for (const name in projects) { - const { modules, globalSetup, files } = projects[name] + const { modules, config, globalSetup, files } = projects[name] const absoluteModules = modules.map((file) => resolve(root, file)) result.set(name, { modules: absoluteModules, moduleIndex: new Map(absoluteModules.map((file, index) => [file, index])), + config, globalSetup, files: new Map(Object.entries(files)), }) @@ -107,15 +119,25 @@ export class DependenciesCache { } } - async update(specifications: TestSpecification[]): Promise { + async update(specifications: TestSpecification[], startTime: number): Promise { if (!this.recordedFiles.size && !this.recordedGlobalSetup.size) { return } await this.read() + const projects = new Set(specifications.map((spec) => spec.project)) + projects.add(this.vitest.getRootProject()) + for (const project of projects) { + const dependencies = this.getProjectDependencies(project.name) + dependencies.config = { + at: startTime, + deps: this.intern(dependencies, getConfigFiles(project)), + } + } + for (const [name, files] of this.recordedGlobalSetup) { const project = this.getProjectDependencies(name) - project.globalSetup = this.intern(project, files) + project.globalSetup = { at: startTime, deps: this.intern(project, files) } } for (const specification of specifications) { @@ -130,9 +152,12 @@ export class DependenciesCache { // a partial run skips tests, so it can miss their dynamic imports, // without a previous record the file keeps having none and always runs if (!isPartialRun(this.vitest, specification, task)) { - project.files.set(file, this.intern(project, files)) + project.files.set(file, { at: startTime, deps: this.intern(project, files) }) } else if (previous) { - project.files.set(file, Array.from(new Set([...previous, ...this.intern(project, files)]))) + project.files.set(file, { + at: previous.at, + deps: Array.from(new Set([...previous.deps, ...this.intern(project, files)])), + }) } } @@ -209,9 +234,10 @@ export class DependenciesCache { } const root = this.vitest.config.root const projects: SerializedDependencies['projects'] = Object.create(null) - for (const [name, { modules, globalSetup, files }] of this.projects) { + for (const [name, { modules, config, globalSetup, files }] of this.projects) { projects[name] = { modules: modules.map((file) => getRootRelativePath(root, file)), + config, globalSetup, files: Object.fromEntries(files), } @@ -228,3 +254,13 @@ export class DependenciesCache { } } } + +// the files that run every test when they change, only the ones that exist +function getConfigFiles(project: TestProject): string[] { + const files = new Set([ + ...(project.vitest.vite.config.configFileDependencies || []), + ...(project.vite.config.configFileDependencies || []), + ...getEnvFiles(project), + ]) + return Array.from(files).filter((file) => existsSync(file)) +} diff --git a/packages/vitest/src/node/cache/index.ts b/packages/vitest/src/node/cache/index.ts index 7f8ba09e4..d083e1f9f 100644 --- a/packages/vitest/src/node/cache/index.ts +++ b/packages/vitest/src/node/cache/index.ts @@ -60,7 +60,7 @@ export class VitestCache { async _update(specifications: TestSpecification[], startTime: number): Promise { await Promise.all([ this._results.update(specifications, startTime), - this._dependencies.update(specifications), + this._dependencies.update(specifications, startTime), ]) } diff --git a/packages/vitest/src/node/cli/cli-api.ts b/packages/vitest/src/node/cli/cli-api.ts index a65804d54..a05b596df 100644 --- a/packages/vitest/src/node/cli/cli-api.ts +++ b/packages/vitest/src/node/cli/cli-api.ts @@ -58,6 +58,13 @@ export interface CliOptions extends UserConfig { */ configLoader?: ViteInlineConfig extends { configLoader?: infer T } ? T : never + /** + * Run only the tests whose recorded dependencies were modified since the last run of the test, + * and the tests that failed in their last run. Requires `cache`. + * @experimental + */ + sinceLastRun?: boolean + /** * Only run benchmark projects, filtering out all other projects. * Set automatically by `vitest bench`. diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index 9f1d8d621..43811a431 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -512,6 +512,11 @@ export const cliOptionsConfig: VitestCLIOptions = { description: 'Run tests that are affected by the changed files (default: `false`)', argument: '[since]', }, + sinceLastRun: { + alias: 'since-last-run', + description: + 'Run only the tests whose recorded dependencies were modified since the last run of the test, and the tests that failed in their last run. Enables `--changed`, `--experimental.recordDependencies` and the `mtime` provider (default: `false`)', + }, lastFailed: { description: 'Run only the tests that failed in the last run (default: `false`)', }, @@ -943,7 +948,8 @@ export const cliOptionsConfig: VitestCLIOptions = { }, vcsProvider: { argument: '', - description: 'Custom provider for detecting changed files. (default: `git`)', + description: + 'Provider for detecting changed files: `git`, `mtime` (requires `experimental.recordDependencies`) or a path to a custom provider. (default: `git`)', subcommands: null, }, recordDependencies: { @@ -1040,6 +1046,11 @@ export const collectCliOptionsConfig: VitestCLIOptions = { description: 'Print only tests that are affected by the changed files (default: `false`)', argument: '[since]', }, + sinceLastRun: { + alias: 'since-last-run', + description: + 'Print only the tests whose recorded dependencies were modified since the last run of the test, and the tests that failed in their last run (default: `false`)', + }, lastFailed: { description: 'Print only the tests that failed in the last run (default: `false`)', }, diff --git a/packages/vitest/src/node/config/resolveConfig.ts b/packages/vitest/src/node/config/resolveConfig.ts index 5f002c8fc..470754342 100644 --- a/packages/vitest/src/node/config/resolveConfig.ts +++ b/packages/vitest/src/node/config/resolveConfig.ts @@ -4,6 +4,7 @@ import type { Plugin as VitePlugin, UserConfig as ViteUserConfig, } from 'vite' +import type { CliOptions } from '../cli/cli-api' import type { Logger } from '../logger' import type { ApiConfig, @@ -37,6 +38,8 @@ import { RandomSequencer } from '../sequencers/RandomSequencer' import { resolveApiToken } from './apiToken' import { PluginHarness } from './pluginHarness' +const builtinVCSProviders = new Set(['git', 'mtime']) + function resolvePath(path: string, root: string) { // local-pkg (mlly)'s resolveModule("./file", { paths: ["/some/root"] }) tries // /some/file @@ -511,6 +514,18 @@ export function resolveTestConfig( } } } + // CLI options are merged into the test config + if ((options as CliOptions).sinceLastRun) { + if (resolved.cache === false) { + throw new Error( + 'The "--since-last-run" flag requires the "cache" option to store the dependencies of the test files.', + ) + } + resolved.changed ||= true + resolved.experimental ??= {} as any + resolved.experimental.recordDependencies = true + resolved.experimental.vcsProvider = 'mtime' + } if (resolved.coverage.changed === undefined && resolved.changed !== undefined) { resolved.coverage.changed = resolved.changed } @@ -1041,7 +1056,7 @@ export function resolveTestConfig( if ( typeof resolved.experimental.vcsProvider === 'string' && - resolved.experimental.vcsProvider !== 'git' + !builtinVCSProviders.has(resolved.experimental.vcsProvider) ) { resolved.experimental.vcsProvider = resolvePath( resolved.experimental.vcsProvider, diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index b52426b0c..53c328467 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -149,7 +149,7 @@ export class Vitest { /** @internal */ configOverride: Partial = {} /** @internal */ filenamePattern?: string[] - /** @internal */ _sourceFilterResult?: { affected: number; total: number } + /** @internal */ _sourceFilterResult?: { affected: number; total: number; failed: number } /** @internal */ _failedFilterResult?: { selected: number; total: number } /** @internal */ runningPromise?: Promise /** @internal */ closingPromise?: Promise @@ -720,7 +720,7 @@ export class Vitest { } await this._testRun.start(specifications) - await this.coverageProvider?.onTestRunStart?.() + await this.coverageProvider?.onTestRunStart?.(specifications) for (const file of files) { await this._reportFileTask(file) @@ -891,7 +891,7 @@ export class Vitest { if (!specifications.length) { await this._traces.$('vitest.test_run', async () => { await this._testRun.start([]) - await this.coverageProvider?.onTestRunStart?.() + await this.coverageProvider?.onTestRunStart?.([]) const coverage = await this.coverageProvider?.generateCoverage?.({ allTestsRun: true }) await this._testRun.end([], [], coverage) @@ -1038,7 +1038,7 @@ export class Vitest { private async runFiles(specs: TestSpecification[], allTestsRun: boolean): Promise { return this._traces.$('vitest.test_run', async () => { await this._testRun.start(specs) - await this.coverageProvider?.onTestRunStart?.() + await this.coverageProvider?.onTestRunStart?.(specs) // previous run await this.cancelPromise diff --git a/packages/vitest/src/node/coverage.ts b/packages/vitest/src/node/coverage.ts index 77be48fe8..154d5c730 100644 --- a/packages/vitest/src/node/coverage.ts +++ b/packages/vitest/src/node/coverage.ts @@ -11,6 +11,7 @@ import type { import type { SerializedCoverageConfig } from '../runtime/config' import type { AfterSuiteRunMeta } from '../types/general' import type { TestProject } from './project' +import type { TestSpecification } from './test-specification' import { createHash } from 'node:crypto' import { existsSync, promises as fs, readdirSync, writeFileSync } from 'node:fs' import module from 'node:module' @@ -24,6 +25,7 @@ import { glob } from 'tinyglobby' import c from 'tinyrainbow' import { coverageConfigDefaults } from '../defaults' import { resolveCoverageReporters } from '../node/config/resolveConfig' +import { createModulesResolver } from '../node/modules-resolver' import { getCoverageFilesDirectory, resolveCoverageProviderModule } from '../utils/coverage' type Threshold = 'lines' | 'functions' | 'statements' | 'branches' @@ -375,12 +377,13 @@ export class BaseCoverageProvider { } } - async onTestRunStart(): Promise { + async onTestRunStart(specifications: ReadonlyArray): Promise { if (this.options.changed) { try { const changedFiles = await this.ctx.vcs.findChangedFiles({ root: this.ctx.config.root, changedSince: this.options.changed, + resolver: createModulesResolver(this.ctx, specifications), }) this.changedFiles = changedFiles diff --git a/packages/vitest/src/node/modules-resolver.ts b/packages/vitest/src/node/modules-resolver.ts new file mode 100644 index 000000000..6fab3ddef --- /dev/null +++ b/packages/vitest/src/node/modules-resolver.ts @@ -0,0 +1,14 @@ +import type { Vitest } from './core' +import type { TestSpecification } from './test-specification' +import { AffectedModulesResolver } from './affected-modules' +import { RecordedModulesResolver } from './recorded-modules' + +export function createModulesResolver( + vitest: Vitest, + specifications: ReadonlyArray, +): AffectedModulesResolver | RecordedModulesResolver { + const specs = Array.from(specifications) + return vitest.config.experimental.recordDependencies + ? new RecordedModulesResolver(vitest, specs) + : new AffectedModulesResolver(vitest, specs) +} diff --git a/packages/vitest/src/node/recorded-modules.ts b/packages/vitest/src/node/recorded-modules.ts index 40884a510..22101bf72 100644 --- a/packages/vitest/src/node/recorded-modules.ts +++ b/packages/vitest/src/node/recorded-modules.ts @@ -1,6 +1,8 @@ +import type { DependencyRecord, ProjectDependencies } from './cache/dependencies' import type { Vitest } from './core' import type { TestProject } from './project' import type { TestSpecification } from './test-specification' +import type { ModuleDependency, ModulesResolver } from './vcs/vcs' import { getEnvFiles } from './affected-modules' import { getRootRelativePath } from './cache/results' @@ -8,41 +10,79 @@ import { getRootRelativePath } from './cache/results' * Finds test specifications that loaded any of the changed files during their last recorded run. * A specification without a record is always affected. */ -export class RecordedModulesResolver { +export class RecordedModulesResolver implements ModulesResolver { + private specsByProject = new Map() + constructor( private vitest: Vitest, - private related: string[], - ) {} - - async resolve(specs: TestSpecification[]): Promise { - if (this.dependsOnConfig(this.vitest.vite.config.configFileDependencies)) { - return specs - } - await this.vitest.cache._dependencies.read() - - const specsByProject = new Map() + private specs: TestSpecification[], + ) { for (const spec of specs) { - let projectSpecs = specsByProject.get(spec.project) + let projectSpecs = this.specsByProject.get(spec.project) if (!projectSpecs) { - specsByProject.set(spec.project, (projectSpecs = [])) + this.specsByProject.set(spec.project, (projectSpecs = [])) } projectSpecs.push(spec) } + } + + async getDependencies(): Promise { + await this.vitest.cache._dependencies.read() + const root = this.vitest.config.root + const recordedAt = new Map() + const add = (dependencies: ProjectDependencies, record: DependencyRecord) => { + for (const index of record.deps) { + const file = dependencies.modules[index] + const previous = recordedAt.get(file) + if (!recordedAt.has(file) || (previous !== undefined && record.at < previous)) { + recordedAt.set(file, record.at) + } + } + } + for (const [project, specs] of this.specsByProject) { + const dependencies = this.vitest.cache._dependencies.get(project) + if (dependencies?.config) { + add(dependencies, dependencies.config) + } + if (dependencies?.globalSetup) { + add(dependencies, dependencies.globalSetup) + } + for (const spec of specs) { + const record = dependencies?.files.get(getRootRelativePath(root, spec.moduleId)) + if (record && dependencies) { + add(dependencies, record) + } else if (!recordedAt.has(spec.moduleId)) { + // a test file without a record has no known time, so it always runs + recordedAt.set(spec.moduleId, undefined) + } + } + } + return Array.from(recordedAt, ([file, at]) => ({ file, recordedAt: at })) + } + + async getAffectedSpecifications(related: string[]): Promise { + if (dependsOnConfig(this.vitest.vite.config.configFileDependencies, related)) { + return this.specs + } + await this.vitest.cache._dependencies.read() const affected = new Set() - for (const [project, projectSpecs] of specsByProject) { - this.findAffectedInProject(project, projectSpecs).forEach((spec) => affected.add(spec)) + for (const [project, projectSpecs] of this.specsByProject) { + this.findAffectedInProject(project, projectSpecs, related).forEach((spec) => + affected.add(spec), + ) } - return specs.filter((spec) => affected.has(spec)) + return this.specs.filter((spec) => affected.has(spec)) } private findAffectedInProject( project: TestProject, specs: TestSpecification[], + related: string[], ): TestSpecification[] { if ( - this.dependsOnConfig(project.vite.config.configFileDependencies) || - this.dependsOnConfig(getEnvFiles(project)) + dependsOnConfig(project.vite.config.configFileDependencies, related) || + dependsOnConfig(getEnvFiles(project), related) ) { return specs } @@ -53,7 +93,7 @@ export class RecordedModulesResolver { } const changed = new Set() - for (const file of this.related) { + for (const file of related) { const index = dependencies.moduleIndex.get(file) if (index !== undefined) { changed.add(index) @@ -61,19 +101,19 @@ export class RecordedModulesResolver { } if (project.config.globalSetup.length) { const globalSetup = dependencies.globalSetup - if (!globalSetup || globalSetup.some((index) => changed.has(index))) { + if (!globalSetup || globalSetup.deps.some((index) => changed.has(index))) { return specs } } const root = this.vitest.config.root return specs.filter((spec) => { - const loaded = dependencies.files.get(getRootRelativePath(root, spec.moduleId)) - return !loaded || loaded.some((index) => changed.has(index)) + const record = dependencies.files.get(getRootRelativePath(root, spec.moduleId)) + return !record || record.deps.some((index) => changed.has(index)) }) } +} - private dependsOnConfig(configDependencies: string[] | undefined): boolean { - return !!configDependencies?.some((file) => this.related.includes(file)) - } +function dependsOnConfig(configDependencies: string[] | undefined, related: string[]): boolean { + return !!configDependencies?.some((file) => related.includes(file)) } diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 3e8d7f4a5..97c739b9e 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -670,11 +670,12 @@ export abstract class BaseReporter implements Reporter { related.length === 1 && !this.ctx.config.changed ? relative(this.ctx.config.root, related[0]) : `${related.length} ${this.ctx.config.changed ? 'changed ' : ''}${noun(related.length, 'file', 'files')}` + const failed = result.failed ? `, ${result.failed} previously failed` : '' this.log( padSummaryTitle('Affected'), `${result.affected} of ${result.total} test files`, - c.dim(`(related to ${sources})`), + c.dim(`(related to ${sources}${failed})`), ) } diff --git a/packages/vitest/src/node/specifications.ts b/packages/vitest/src/node/specifications.ts index 5707760ab..81b1266f1 100644 --- a/packages/vitest/src/node/specifications.ts +++ b/packages/vitest/src/node/specifications.ts @@ -2,10 +2,10 @@ import type { Vitest } from './core' import type { TestSpecification } from './test-specification' import { relative, resolve } from 'pathe' import pm from 'picomatch' -import { AffectedModulesResolver } from './affected-modules' import { groupFilters, parseFilter } from './cli/filter' import { IncludeTaskLocationDisabledError, LocationFilterFileNotFoundError } from './errors' -import { RecordedModulesResolver } from './recorded-modules' +import { createModulesResolver } from './modules-resolver' +import { MtimeVCSProvider } from './vcs/mtime' export class VitestSpecifications { private readonly _cachedSpecs = new Map() @@ -129,10 +129,13 @@ export class VitestSpecifications { private async filterTestsBySource(specs: TestSpecification[]): Promise { this.vitest._sourceFilterResult = undefined + const resolver = createModulesResolver(this.vitest, specs) + if (this.vitest.config.changed && !this.vitest.config.related) { const related = await this.vitest.vcs.findChangedFiles({ root: this.vitest.config.root, changedSince: this.vitest.config.changed, + resolver, }) this.vitest.config.related = Array.from(new Set(related)) } @@ -148,21 +151,40 @@ export class VitestSpecifications { return specs } + const failedSpecs = this.getFailedSpecifications(specs) + // don't run anything if no related sources are found // if we are in watch mode, we want to process all tests if (!this.vitest.config.watch && !related.length) { - this.vitest._sourceFilterResult = { affected: 0, total: specs.length } - return [] + this.vitest._sourceFilterResult = { + affected: failedSpecs.length, + total: specs.length, + failed: failedSpecs.length, + } + return failedSpecs } - const resolver = this.vitest.config.experimental.recordDependencies - ? new RecordedModulesResolver(this.vitest, related) - : new AffectedModulesResolver(this.vitest, related) - const affectedSpecs = await resolver.resolve(specs) - this.vitest._sourceFilterResult = { affected: affectedSpecs.length, total: specs.length } + const affected = new Set([ + ...(await resolver.getAffectedSpecifications(related)), + ...failedSpecs, + ]) + const affectedSpecs = specs.filter((spec) => affected.has(spec)) + this.vitest._sourceFilterResult = { + affected: affectedSpecs.length, + total: specs.length, + failed: failedSpecs.length, + } return affectedSpecs } + // a failure can have a cause outside the recorded files, so the file runs until it passes + private getFailedSpecifications(specs: TestSpecification[]): TestSpecification[] { + if (!this.vitest.config.changed || !(this.vitest.vcs instanceof MtimeVCSProvider)) { + return [] + } + return specs.filter((spec) => this.vitest.cache.getTestSpecificationResult(spec)?.failed) + } + private filterFailedTests(specs: TestSpecification[]): TestSpecification[] { const failed: TestSpecification[] = [] for (const spec of specs) { diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index e56dd7816..cc2d7ad2c 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -1026,13 +1026,17 @@ export interface InlineConfig { nodeLoader?: boolean /** - * Custom provider for detecting changed files. Used with the `--changed` flag + * Provider for detecting changed files. Used with the `--changed` flag * to determine which files have been modified. * - * By default, Vitest uses Git to detect changed files. You can provide a custom - * implementation of the `VCSProvider` interface to use a different version control system. + * - `'git'` (default) compares the working tree with the `--changed` reference. + * - `'mtime'` compares the modification time of every recorded dependency with the run + * that recorded it. It requires `experimental.recordDependencies`. The test files that + * failed in their last run also run. + * + * You can also provide a custom implementation of the `VCSProvider` interface. */ - vcsProvider?: VCSProvider | string + vcsProvider?: 'git' | 'mtime' | VCSProvider | (string & {}) /** * Record the modules that every test file loads in the cache and use the records diff --git a/packages/vitest/src/node/types/coverage.ts b/packages/vitest/src/node/types/coverage.ts index d3780dd5e..f25d3ba74 100644 --- a/packages/vitest/src/node/types/coverage.ts +++ b/packages/vitest/src/node/types/coverage.ts @@ -7,6 +7,7 @@ import type { RuntimeCoverageProviderModule, } from '../../utils/coverage' import type { Vitest } from '../core' +import type { TestSpecification } from '../test-specification' type TransformResult = string | Partial | undefined | null | void type CoverageResults = unknown @@ -27,7 +28,7 @@ export interface CoverageProvider { onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void | Promise /** Callback called when test run starts */ - onTestRunStart?: () => void | Promise + onTestRunStart?: (specifications: ReadonlyArray) => void | Promise /** Callback called when test run fails due to test failures */ onTestFailure?: () => void | Promise diff --git a/packages/vitest/src/node/vcs/mtime.ts b/packages/vitest/src/node/vcs/mtime.ts new file mode 100644 index 000000000..a1bea63c2 --- /dev/null +++ b/packages/vitest/src/node/vcs/mtime.ts @@ -0,0 +1,22 @@ +import type { VCSProvider, VCSProviderOptions } from './vcs' +import { stat } from 'node:fs/promises' + +/** + * Reports a dependency as changed when its modification time is newer than the run that recorded it. + * A dependency without a recorded time is always changed. + */ +export class MtimeVCSProvider implements VCSProvider { + async findChangedFiles(options: VCSProviderOptions): Promise { + const dependencies = await options.resolver.getDependencies() + const changed = await Promise.all( + dependencies.map(async ({ file, recordedAt }) => { + if (recordedAt == null) { + return file + } + const stats = await stat(file).catch(() => null) + return !stats || stats.mtimeMs > recordedAt ? file : null + }), + ) + return changed.filter((file) => file != null) + } +} diff --git a/packages/vitest/src/node/vcs/vcs.ts b/packages/vitest/src/node/vcs/vcs.ts index 32fea5270..df2283263 100644 --- a/packages/vitest/src/node/vcs/vcs.ts +++ b/packages/vitest/src/node/vcs/vcs.ts @@ -1,10 +1,36 @@ import type { ModuleRunner } from 'vite/module-runner' import { resolve } from 'pathe' import { GitVCSProvider } from './git' +import { MtimeVCSProvider } from './mtime' + +export interface ModuleDependency { + /** + * Absolute path of the file. + */ + file: string + /** + * Start time of the oldest run that recorded the file, in milliseconds since the epoch. + * A change after this time was not seen by every test file that depends on the file. + * It is not set when the dependency comes from the static module graph. + */ + recordedAt?: number +} + +export interface ModulesResolver { + /** + * Every file whose change can affect the test files of the run. + */ + // oxlint-disable-next-line typescript/method-signature-style + getDependencies(): Promise +} export interface VCSProviderOptions { root: string changedSince?: string | boolean + /** + * Resolves the dependencies of the test files of the run. + */ + resolver: ModulesResolver } export interface VCSProvider { @@ -22,6 +48,9 @@ export async function loadVCSProvider( if (!vcsProvider || vcsProvider === 'git') { return new GitVCSProvider() } + if (vcsProvider === 'mtime') { + return new MtimeVCSProvider() + } const module = (await runner.import(vcsProvider)) as { default: VCSProvider } if ( !module.default || diff --git a/packages/vitest/src/public/node.ts b/packages/vitest/src/public/node.ts index 67724c46e..28806e9fd 100644 --- a/packages/vitest/src/public/node.ts +++ b/packages/vitest/src/public/node.ts @@ -33,6 +33,14 @@ export { resolveFsAllow } from '../node/plugins/utils' export type { ProcessPool } from '../node/pool' export { getFilePoolName } from '../node/pool' export { createMethodsRPC } from '../node/pools/rpc' +export type { + ModuleDependency, + ModulesResolver, + VCSProvider, + VCSProviderOptions, +} from '../node/vcs/vcs' +export { GitVCSProvider } from '../node/vcs/git' +export { MtimeVCSProvider } from '../node/vcs/mtime' export type { PoolOptions, PoolRunnerInitializer, diff --git a/test/e2e/test/recorded-deps.test.ts b/test/e2e/test/recorded-deps.test.ts index 1d60b9702..674d41868 100644 --- a/test/e2e/test/recorded-deps.test.ts +++ b/test/e2e/test/recorded-deps.test.ts @@ -23,18 +23,21 @@ function readDependencies(ctx: Vitest | undefined) { const { projects } = JSON.parse(readFileSync(getDependenciesPath(ctx), 'utf-8')) const result: Record> = {} for (const name in projects) { - const { modules, files, globalSetup } = projects[name] + const { modules, files, globalSetup, config } = projects[name] const project: Record = (result[name] = {}) const toFiles = (indices: number[]) => indices .map((index: number) => modules[index]) .filter((file: string) => !file.startsWith('../')) .sort() + if (config.deps.length) { + project[''] = toFiles(config.deps) + } if (globalSetup) { - project[''] = toFiles(globalSetup) + project[''] = toFiles(globalSetup.deps) } for (const file in files) { - project[file] = toFiles(files[file]) + project[file] = toFiles(files[file].deps) } } return result @@ -60,6 +63,9 @@ test('only the tests that loaded a changed file run', async () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/helper.js", @@ -114,6 +120,9 @@ test.for([ expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/inner.js", @@ -159,6 +168,9 @@ test('a mocked module does not load its own imports', async () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/dep.js", @@ -206,6 +218,9 @@ test('a file loaded by a setup file or a global setup runs every test', async () expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "": [ "global-setup.js", "src/global-helper.js", @@ -300,7 +315,11 @@ test('a partial run without a previous record does not create one', async () => expect(stderr).toBe('') expect(readDependencies(ctx)).toMatchInlineSnapshot(` { - "": {}, + "": { + "": [ + "vitest.config.js", + ], + }, } `) @@ -321,6 +340,9 @@ test('a partial run without a previous record does not create one', async () => expect(readDependencies(first.ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/dynamic.js", @@ -372,6 +394,9 @@ test('a mock loaded from a file is recorded', async () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "automock.test.js": [ "automock.test.js", "src/__mocks__/dep.js", @@ -438,6 +463,9 @@ test('a file imported with a query and a file added by a plugin are recorded', a expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "raw.test.js": [ "raw.test.js", "src/data.txt", @@ -485,21 +513,24 @@ test.for([{ pool: 'forks' }, { pool: 'vmForks' }])( ) expect(stderr).toBe('') expect(readDependencies(ctx)).toMatchInlineSnapshot(` - { - "": { - "a.test.js": [ - "a.test.js", - "environment.js", - "src/environment-helper.js", - ], - "b.test.js": [ - "b.test.js", - "environment.js", - "src/environment-helper.js", - ], - }, - } - `) + { + "": { + "": [ + "vitest.config.js", + ], + "a.test.js": [ + "a.test.js", + "environment.js", + "src/environment-helper.js", + ], + "b.test.js": [ + "b.test.js", + "environment.js", + "src/environment-helper.js", + ], + }, + } + `) const changed = await runVitest({ root: fs.root, @@ -530,6 +561,9 @@ test('a file loaded by a custom runner runs every test', async () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "runner.js", @@ -599,6 +633,9 @@ exports[\`snap 1\`] = \`1\`; expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "file.test.js": [ "file.test.js", "fixtures/file.txt", @@ -648,13 +685,24 @@ test('records are kept per project', async () => { expect(stderr).toBe('') expect(readDependencies(ctx)).toMatchInlineSnapshot(` { + "": { + "": [ + "vitest.config.js", + ], + }, "first": { + "": [ + "vitest.config.js", + ], "first/a.test.js": [ "first/a.test.js", "src/shared.js", ], }, "second": { + "": [ + "vitest.config.js", + ], "second/b.test.js": [ "second/b.test.js", ], @@ -746,6 +794,9 @@ test('a complete run replaces the record of the file and keeps the others', asyn expect(readDependencies(filtered.ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/other.js", @@ -849,6 +900,9 @@ test('a rerun of one file in watch mode keeps the records of the other files', a expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/other.js", @@ -911,6 +965,9 @@ describe('without the module runner', () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/helper.js", @@ -944,6 +1001,9 @@ describe('without the module runner', () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/inner.js", @@ -1004,6 +1064,9 @@ describe('without the module runner', () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "": [ "global-setup.js", "src/global-helper.js", @@ -1066,6 +1129,9 @@ describe('without the module runner', () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "automock.test.js": [ "automock.test.js", "src/__mocks__/dep.js", diff --git a/test/e2e/test/vcs-mtime.test.ts b/test/e2e/test/vcs-mtime.test.ts new file mode 100644 index 000000000..b40f58ce8 --- /dev/null +++ b/test/e2e/test/vcs-mtime.test.ts @@ -0,0 +1,250 @@ +import { readFileSync, rmSync } from 'node:fs' +import { relative } from 'pathe' +import { expect, test } from 'vitest' +import { runInlineTests, runVitest } from '../../test-utils' + +const recorded = { cache: true, experimental: { recordDependencies: true, vcsProvider: 'mtime' } } +const sinceLastRun = { cache: true, $cliOptions: { sinceLastRun: true } } + +function testFile(name: string, imports = '') { + return ` + ${imports} + import { test } from 'vitest' + test('${name}', () => {}) + ` +} + +function affectedSummary(stdout: string) { + return stdout.split('\n').find((line) => line.includes('Affected')) +} + +const files = { + 'src/helper.js': 'export {}', + 'src/other.js': 'export {}', + 'a.test.js': testFile('a', `import './src/helper.js'`), + 'b.test.js': testFile('b', `import './src/other.js'`), +} + +test('runs the tests that loaded a file modified after their last run', async () => { + const { fs, stderr } = await runInlineTests(files, recorded) + expect(stderr).toBe('') + + fs.editFile('src/helper.js', () => 'export const helper = 1') + const changed = await runVitest({ root: fs.root, ...recorded, changed: true }) + expect(changed.stderr).toBe('') + expect(changed.testTree()).toMatchInlineSnapshot(` + { + "a.test.js": { + "a": "passed", + }, + } + `) + + // the run above recorded the file again, so nothing is newer than the records + const unchanged = await runVitest({ root: fs.root, ...recorded, changed: true }) + expect(unchanged.stderr).toBe('') + expect(unchanged.ctx!._sourceFilterResult).toEqual({ affected: 0, total: 2, failed: 0 }) +}) + +test('a deleted dependency and a modified config file are detected', async () => { + const { fs, stderr } = await runInlineTests( + { ...files, 'vitest.config.js': { test: recorded } }, + recorded, + ) + expect(stderr).toBe('') + + rmSync(fs.resolveFile('src/other.js')) + const deleted = await runVitest({ root: fs.root, ...recorded, changed: true }) + expect(deleted.errorTree()).toMatchInlineSnapshot(` + { + "b.test.js": { + "__module_errors__": [ + "Cannot find module './src/other.js' imported from /b.test.js", + ], + }, + } + `) + + fs.editFile('vitest.config.js', (content) => `${content}\n`) + const configChanged = await runVitest({ root: fs.root, ...recorded, changed: true }) + expect(Object.keys(configChanged.testTree()).sort()).toEqual(['a.test.js', 'b.test.js']) +}) + +test('a test file without a record runs', async () => { + const { fs, stderr } = await runInlineTests(files, recorded) + expect(stderr).toBe('') + + fs.createFile('c.test.js', testFile('c')) + const changed = await runVitest({ root: fs.root, ...recorded, changed: true }) + expect(changed.stderr).toBe('') + expect(Object.keys(changed.testTree())).toEqual(['c.test.js']) +}) + +test('every test runs without recorded dependencies', async () => { + const { testTree, stderr } = await runInlineTests(files, { + cache: true, + experimental: { vcsProvider: 'mtime' }, + changed: true, + }) + expect(stderr).toBe('') + expect(Object.keys(testTree()).sort()).toEqual(['a.test.js', 'b.test.js']) +}) + +test('a custom provider receives the dependencies of the test files', async () => { + const { fs, stderr } = await runInlineTests( + { + ...files, + 'vitest.config.js': ` + import { writeFileSync } from 'node:fs' + import { relative, sep } from 'node:path' + export default { + test: { + cache: true, + experimental: { + recordDependencies: true, + vcsProvider: { + async findChangedFiles({ root, resolver }) { + const dependencies = await resolver.getDependencies() + const files = dependencies + .map(({ file, recordedAt }) => [relative(root, file).split(sep).join('/'), typeof recordedAt]) + .filter(([file]) => !file.startsWith('..')) + .sort() + writeFileSync(new URL('./dependencies.json', import.meta.url), JSON.stringify(files)) + return [] + }, + }, + }, + }, + } + `, + }, + { cache: true }, + ) + expect(stderr).toBe('') + + const changed = await runVitest({ root: fs.root, cache: true, changed: true }) + expect(changed.stderr).toBe('') + expect(changed.ctx!._sourceFilterResult).toEqual({ affected: 0, total: 2, failed: 0 }) + expect(JSON.parse(readFileSync(fs.resolveFile('dependencies.json'), 'utf-8'))) + .toMatchInlineSnapshot(` + [ + [ + "a.test.js", + "number", + ], + [ + "b.test.js", + "number", + ], + [ + "src/helper.js", + "number", + ], + [ + "src/other.js", + "number", + ], + [ + "vitest.config.js", + "number", + ], + ] + `) +}) + +test('coverage of changed files uses the dependencies of the run', async () => { + const coverage = { enabled: true, provider: 'v8', reporter: ['json'], changed: true } as const + const { fs, stderr } = await runInlineTests(files, recorded) + expect(stderr).toBe('') + + fs.editFile('src/helper.js', () => 'export const helper = 1') + const changed = await runVitest({ root: fs.root, ...recorded, coverage }) + expect(changed.stderr).toBe('') + const report = JSON.parse(readFileSync(fs.resolveFile('coverage/coverage-final.json'), 'utf-8')) + expect(Object.keys(report).map((file) => relative(fs.root, file))).toEqual(['src/helper.js']) +}) + +test('--since-last-run records the dependencies and runs the affected tests', async () => { + // the first run has no records, so every test runs + const { fs, stderr, testTree } = await runInlineTests(files, sinceLastRun) + expect(stderr).toBe('') + expect(Object.keys(testTree()).sort()).toEqual(['a.test.js', 'b.test.js']) + + fs.editFile('src/other.js', () => 'export const other = 1') + const changed = await runVitest({ root: fs.root, ...sinceLastRun }) + expect(changed.stderr).toBe('') + expect(Object.keys(changed.testTree())).toEqual(['b.test.js']) + + const unchanged = await runVitest({ root: fs.root, ...sinceLastRun }) + expect(unchanged.stderr).toBe('') + expect(unchanged.ctx!._sourceFilterResult).toEqual({ affected: 0, total: 2, failed: 0 }) +}) + +test('--since-last-run runs a failed test file until it passes', async () => { + const database = { DATABASE_URL: 'postgres://localhost:5432/test' } + const { fs, errorTree } = await runInlineTests( + { + ...files, + 'b.test.js': ` + import { expect, test } from 'vitest' + test('b', () => { + expect(process.env.DATABASE_URL).toBeDefined() + }) + `, + }, + sinceLastRun, + ) + expect(errorTree()).toMatchInlineSnapshot(` + { + "a.test.js": { + "a": "passed", + }, + "b.test.js": { + "b": [ + "expected undefined to be defined", + ], + }, + } + `) + + const failed = await runVitest({ root: fs.root, ...sinceLastRun }) + expect(failed.exitCode).toBe(1) + expect(failed.errorTree()).toMatchInlineSnapshot(` + { + "b.test.js": { + "b": [ + "expected undefined to be defined", + ], + }, + } + `) + expect(affectedSummary(failed.stdout)).toMatchInlineSnapshot( + `" Affected 1 of 2 test files (related to 0 changed files, 1 previously failed)"`, + ) + + // --related does not compare with the last run, so the failed file does not run + const related = await runVitest({ root: fs.root, ...recorded, related: ['src/helper.js'] }) + expect(related.stderr).toBe('') + expect(Object.keys(related.testTree())).toEqual(['a.test.js']) + + const fixed = await runVitest({ root: fs.root, ...sinceLastRun, env: database }) + expect(fixed.stderr).toBe('') + expect(fixed.testTree()).toMatchInlineSnapshot(` + { + "b.test.js": { + "b": "passed", + }, + } + `) + + const passed = await runVitest({ root: fs.root, ...sinceLastRun, env: database }) + expect(passed.stderr).toBe('') + expect(passed.ctx!._sourceFilterResult).toEqual({ affected: 0, total: 2, failed: 0 }) +}) + +test('--since-last-run requires the cache', async () => { + const { stderr } = await runInlineTests(files, { ...sinceLastRun, cache: false }) + expect(stderr.split('\n')[0]).toMatchInlineSnapshot( + `"Error: The "--since-last-run" flag requires the "cache" option to store the dependencies of the test files."`, + ) +}) diff --git a/test/unit/test/exports.test.ts b/test/unit/test/exports.test.ts index aad856da5..7fd8434e0 100644 --- a/test/unit/test/exports.test.ts +++ b/test/unit/test/exports.test.ts @@ -88,12 +88,14 @@ it('exports snapshot', async ({ skip, task }) => { "DotReporter": "function", "ForksPoolWorker": "function", "GitNotFoundError": "function", + "GitVCSProvider": "function", "GithubActionsReporter": "function", "HangingProcessReporter": "function", "JUnitReporter": "function", "JsonReporter": "function", "Logger": "function", "MinimalReporter": "function", + "MtimeVCSProvider": "function", "PluginHarness": "function", "ReportersMap": "object", "TapFlatReporter": "function",