diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 6e3ea4d9c..12043cc47 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -620,6 +620,10 @@ export default ({ mode }: { mode: string }) => { text: 'changed', link: '/config/changed', }, + { + text: 'stale', + link: '/config/stale', + }, { text: 'experimental', link: '/config/experimental', @@ -1092,6 +1096,10 @@ export default ({ mode }: { mode: string }) => { text: 'Parallel and Sequential Test Files', link: '/guide/recipes/parallel-sequential', }, + { + text: 'Running Only Affected Tests', + link: '/guide/recipes/affected-tests', + }, ], }, { diff --git a/docs/config/experimental.md b/docs/config/experimental.md index 75e862163..17ee9c093 100644 --- a/docs/config/experimental.md +++ b/docs/config/experimental.md @@ -316,7 +316,7 @@ If you are running tests in Deno, TypeScript files are processed by the runtime ## experimental.vcsProvider 4.1.1 {#experimental-vcsprovider} -- **Type:** `VCSProvider | string` +- **Type:** `'git' | 'mtime' | VCSProvider | string` ```ts interface VCSProvider { @@ -326,14 +326,55 @@ interface VCSProvider { interface VCSProviderOptions { root: string changedSince?: string | boolean + /** Resolves the dependencies of the test files of the run. */ + resolver: ModulesResolver +} + +interface ModulesResolver { + /** + * Every file whose change can affect the test files of the run. + */ + getDependencies(): Promise +} + +interface ModuleDependency { + /** Absolute path of the file. */ + file: string + /** + * Start time of the oldest run that recorded the file, in milliseconds. + * Not set when the dependency comes from the static module graph. + */ + recordedAt?: number } ``` - **Default:** `'git'` -Custom provider for detecting changed files. Used with the [`--changed`](/guide/cli#changed) flag to determine which files have been modified. +Provider for detecting changed files. Used with the [`--changed`](/guide/cli#changed) flag to determine which files have been modified. + +Vitest has two built-in providers: + +- `'git'` returns the files that differ from the `--changed` reference, plus staged and untracked files. +- `'mtime'` 5.0.4 returns the recorded dependencies whose modification time is newer than the run that recorded them, and deleted dependencies. It needs [`experimental.recordDependencies`](#experimental-recorddependencies): a dependency without a record is always reported as changed, so every test runs until the records exist. It does not need Git and ignores the `--changed` reference. A `--changed` run updates the records of the tests it ran, so the next `--changed` run selects only the tests whose dependencies changed since. + +```ts [vitest.config.ts] +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + experimental: { + recordDependencies: true, + vcsProvider: 'mtime', + }, + }, +}) +``` + +::: warning +The `mtime` provider compares file times with the clock of the machine that ran the tests. A checkout or a copy that sets file times in the past is not detected. +::: -By default, Vitest uses Git to detect changed files. You can provide a custom implementation of the `VCSProvider` interface to use a different version control system: +You can provide a custom implementation of the `VCSProvider` interface to use a different version control system. `options.resolver.getDependencies()` returns every file the test files of the run depend on, so a provider can limit its work to these files: ```ts [vitest.config.ts] import { defineConfig } from 'vitest/config' @@ -342,8 +383,9 @@ export default defineConfig({ test: { experimental: { vcsProvider: { - async findChangedFiles({ root, changedSince }) { - // return paths of changed files + async findChangedFiles({ root, changedSince, resolver }) { + const dependencies = await resolver.getDependencies() + // return the paths of the changed files return [] }, }, diff --git a/docs/config/stale.md b/docs/config/stale.md new file mode 100644 index 000000000..6823d5352 --- /dev/null +++ b/docs/config/stale.md @@ -0,0 +1,24 @@ +--- +title: stale | Config +outline: deep +--- + +### stale 5.1.0 {#stale} + +- **Type:** `boolean` +- **Default:** `false` +- **CLI:** `--stale` + +Run only the tests whose dependencies were modified since the last run of the test. A test is stale when one of the files it loaded during its last run has a newer modification time than that run, or was deleted. A test that never ran with this option has no record and always runs. + +```sh +vitest run --stale +``` + +Every run with this option updates the records of the tests it ran, so the next run selects only the tests affected by changes made in between. To run every test again, run Vitest without the flag, or delete the cache with [`--clearCache`](/guide/cli#clearcache). + +The option requires [`cache`](/config/cache). Vitest throws an error if the cache is disabled. + +::: warning +The modification time comes from the file system. A `git checkout` or an unpacked archive that sets old modification times is not detected. Files that a test did not load during its last run, for example a new `.env` file or a new `__mocks__` file, are not detected either. See [`experimental.recordDependencies`](/config/experimental#experimental-recorddependencies) for the full list of recorded files. +::: diff --git a/docs/guide/cli-generated.md b/docs/guide/cli-generated.md index 9ec6f1d97..4dc71b823 100644 --- a/docs/guide/cli-generated.md +++ b/docs/guide/cli-generated.md @@ -514,6 +514,13 @@ Ignore any unhandled errors that occur Run tests that are affected by the changed files (default: `false`) +### stale + +- **CLI:** `--stale` +- **Config:** [stale](/config/stale) + +Run only the tests whose recorded dependencies were modified since the last run of the test. Enables `--changed`, `--experimental.recordDependencies` and the `mtime` provider (default: `false`) + ### sequence.shuffle.files - **CLI:** `--sequence.shuffle.files` @@ -998,7 +1005,7 @@ Controls whether Vitest will use Node.js Loader API to process in-source or mock - **CLI:** `--experimental.vcsProvider ` - **Config:** [experimental.vcsProvider](/config/experimental#experimental-vcsprovider) -Custom provider for detecting changed files. (default: `git`) +Provider for detecting changed files: `git`, `mtime` (requires `experimental.recordDependencies`) or a path to a custom provider. (default: `git`) ### experimental.recordDependencies diff --git a/docs/guide/recipes/affected-tests.md b/docs/guide/recipes/affected-tests.md new file mode 100644 index 000000000..93e8e630d --- /dev/null +++ b/docs/guide/recipes/affected-tests.md @@ -0,0 +1,66 @@ +--- +title: Running Only Affected Tests | Recipes +--- + +# Running Only Affected Tests + +A full run of a large suite is the right check before a merge, but it is slow feedback for a small change. If you edited one module, only the tests that load that module can change their result. Vitest has two flags that select those tests. They differ in one question: changed compared to what? + +- [`--changed`](/config/changed) compares the working tree with a Git reference. +- [`--stale`](/config/stale) compares every file a test loaded with the time of the test's last run. + +## `--changed`: what differs from a Git reference + +```sh +vitest run --changed # uncommitted changes, staged and unstaged +vitest run --changed HEAD~1 # the last commit and uncommitted changes +vitest run --changed origin/main # everything on this branch +``` + +Vitest asks Git for the changed files, then finds the test files that import them, directly or through other modules. A test runs when any file in its import graph is in the list. + +The reference is the baseline, so the same command gives the same selection until the baseline moves. This fits checks that are defined in terms of Git: a pull request against `origin/main`, or a pre-push hook that covers the commits you are about to push. It requires a Git repository, and a change that is committed and merged stops being "changed", even if the tests never ran against it on your machine. + +## `--stale`: what changed since the test last ran + +```sh +vitest run --stale +``` + +The first run with `--stale` runs every test and records the files each test loaded, including dynamic imports, mocks, setup files and snapshot files. Every later run compares the modification time of those files with the time of the run that recorded them. A test is stale when one of its files is newer or was deleted, or when the test has no record yet. + +Each run updates the records of the tests it ran, so the baseline moves with you: after a stale run, only the next edit makes tests stale again. Git is not involved, so the flag works in a fresh container, in a worktree with an unrelated history, or in a directory that is not a repository. A new test file, or one that was never recorded, always runs. + +The records live in the [cache](/config/cache). `vitest run --clearCache` deletes them, and the next `--stale` run is a full run again. + +## Which one to pick + +| You want to run the tests affected by… | Use | +| ---------------------------------------------------------------------- | ------------------------ | +| a branch, before opening or updating a pull request | `--changed origin/main` | +| the commits you are about to push | `--changed ` | +| your uncommitted edits, in a Git repository | `--changed` or `--stale` | +| your edits since you last ran the tests, committed or not | `--stale` | +| edits in a directory without Git, or with a history you do not control | `--stale` | + +`--changed` is the better fit when the baseline is shared with other people: the reference is the same on every machine and in CI. `--stale` is the better fit when the baseline is your own last run: it does not matter whether you committed in between, or whether Git is there at all. + +## Agents + +AI coding agents run tests after almost every edit. They often work in a container or a temporary worktree, commit in the middle of a task, and start a new process for every command. A Git baseline fits none of that: `--changed` is empty right after a commit, and it needs a repository. `--stale` tracks what the agent actually ran, so each `vitest run --stale` covers exactly the edits since the previous one, and the first command in a fresh environment is a full run that builds the records. + +Tell the agent about it in the instructions file of your project: + +```md [AGENTS.md] +- After a change, run `vitest run --stale` to test only what the change affects. +- Before you finish, run `vitest run` once for the whole suite. +``` + +Two things the records cannot see: a file that no test loaded during the last run (a new `.env` file, a new `__mocks__` file), and modification times set in the past by a checkout or an unpacked archive. The full run at the end covers both. + +## See also + +- [`changed`](/config/changed) +- [`stale`](/config/stale) +- [`experimental.recordDependencies`](/config/experimental#experimental-recorddependencies) +- [Test Filtering](/guide/filtering) diff --git a/packages/vitest/src/node/affected-modules.ts b/packages/vitest/src/node/affected-modules.ts index 5c337a850..107448bde 100644 --- a/packages/vitest/src/node/affected-modules.ts +++ b/packages/vitest/src/node/affected-modules.ts @@ -2,6 +2,7 @@ import type { DevEnvironment } from 'vite' import type { Vitest } from './core' import type { TestProject } from './project' import type { TestSpecification } from './test-specification' +import type { ModuleDependency, ModulesResolver } from './vcs/vcs' import { existsSync } from 'node:fs' import os from 'node:os' import { cleanUrl } from '@vitest/utils/helpers' @@ -25,18 +26,32 @@ interface ModuleNode { /** * Finds test specifications that statically depend on any of the changed files. */ -export class AffectedModulesResolver { +export class AffectedModulesResolver implements ModulesResolver { private existsCache = new Map() private transformConcurrency = os.availableParallelism?.() ?? os.cpus().length private activeTransforms = 0 private transformQueue: Array<() => void> = [] + private related: string[] = [] + // set while `getDependencies` walks the graph + walked: Set | undefined constructor( private vitest: Vitest, - private related: string[], + private specs: TestSpecification[], ) {} - async resolve(specs: TestSpecification[]): Promise { + // the graph is only known after transforming it, so it is walked once without changes + async getDependencies(): Promise { + this.walked = new Set() + await this.getAffectedSpecifications([]) + const files = new Set(Array.from(this.walked, (id) => cleanUrl(id))) + this.walked = undefined + return Array.from(files, (file) => ({ file })) + } + + async getAffectedSpecifications(related: string[]): Promise { + this.related = related + const specs = this.specs if (this.dependsOnConfig(this.vitest.vite.config.configFileDependencies)) { return specs } @@ -286,7 +301,7 @@ class ProjectGraph { constructor( private project: TestProject, private environment: DevEnvironment, - private resolver: AffectedModulesResolver, + public resolver: AffectedModulesResolver, ) {} getModule(id: string): Promise { @@ -443,6 +458,8 @@ class GraphWalk { if (!node) { return } + this.graph.resolver.walked?.add(id) + node.watchedFiles.forEach((file) => this.graph.resolver.walked?.add(file)) // a file imported with a query is a separate module if (node.failed || this.changed.has(cleanUrl(id))) { this.changed.add(id) diff --git a/packages/vitest/src/node/cache/dependencies.ts b/packages/vitest/src/node/cache/dependencies.ts index b05c3aa00..5895e03f3 100644 --- a/packages/vitest/src/node/cache/dependencies.ts +++ b/packages/vitest/src/node/cache/dependencies.ts @@ -6,21 +6,31 @@ import { mkdir, readFile, rm } from 'node:fs/promises' import { cleanUrl } from '@vitest/utils/helpers' import { dirname, isAbsolute, resolve } from 'pathe' import { createDebugger } from '../../utils/debugger' +import { getEnvFiles } from '../affected-modules' import { tagCacheDir } from './cachedirTag' import { atomicWriteFile } from './fsModuleCache' import { getFileStamp, getRootRelativePath, isPartialRun } from './results' const debug = createDebugger('vitest:cache:dependencies') -const DEPENDENCIES_VERSION = 1 +const DEPENDENCIES_VERSION = 2 + +export interface DependencyRecord { + /** Start time of the run that recorded the dependencies. */ + at: number + /** Indices into `modules`. */ + deps: number[] +} export interface ProjectDependencies { /** Absolute paths without a query, referenced by index. */ modules: string[] moduleIndex: Map - globalSetup?: number[] - /** Root-relative test file path to the indices of every module it loaded. */ - files: Map + /** The config file, its imports and the `.env` files. */ + config?: DependencyRecord + globalSetup?: DependencyRecord + /** Keyed by the root-relative test file path. */ + files: Map } interface SerializedDependencies { @@ -29,8 +39,9 @@ interface SerializedDependencies { string, { modules: string[] - globalSetup?: number[] - files: Record + config?: DependencyRecord + globalSetup?: DependencyRecord + files: Record } > } @@ -92,11 +103,12 @@ export class DependenciesCache { const root = this.vitest.config.root const result = new Map() for (const name in projects) { - const { modules, globalSetup, files } = projects[name] + const { modules, config, globalSetup, files } = projects[name] const absoluteModules = modules.map((file) => resolve(root, file)) result.set(name, { modules: absoluteModules, moduleIndex: new Map(absoluteModules.map((file, index) => [file, index])), + config, globalSetup, files: new Map(Object.entries(files)), }) @@ -107,15 +119,25 @@ export class DependenciesCache { } } - async update(specifications: TestSpecification[]): Promise { + async update(specifications: TestSpecification[], startTime: number): Promise { if (!this.recordedFiles.size && !this.recordedGlobalSetup.size) { return } await this.read() + const projects = new Set(specifications.map((spec) => spec.project)) + projects.add(this.vitest.getRootProject()) + for (const project of projects) { + const dependencies = this.getProjectDependencies(project.name) + dependencies.config = { + at: startTime, + deps: this.intern(dependencies, getConfigFiles(project)), + } + } + for (const [name, files] of this.recordedGlobalSetup) { const project = this.getProjectDependencies(name) - project.globalSetup = this.intern(project, files) + project.globalSetup = { at: startTime, deps: this.intern(project, files) } } for (const specification of specifications) { @@ -126,13 +148,16 @@ export class DependenciesCache { } const project = this.getProjectDependencies(specification.project.name) const file = getRootRelativePath(this.vitest.config.root, specification.moduleId) - const indices = this.intern(project, files) + const deps = this.intern(project, files) const previous = project.files.get(file) // a partial run skips tests, so it can miss their dynamic imports if (previous && isPartialRun(this.vitest, specification, task)) { - project.files.set(file, Array.from(new Set([...previous, ...indices]))) + project.files.set(file, { + at: previous.at, + deps: Array.from(new Set([...previous.deps, ...deps])), + }) } else { - project.files.set(file, indices) + project.files.set(file, { at: startTime, deps }) } } @@ -204,9 +229,10 @@ export class DependenciesCache { } const root = this.vitest.config.root const projects: SerializedDependencies['projects'] = Object.create(null) - for (const [name, { modules, globalSetup, files }] of this.projects) { + for (const [name, { modules, config, globalSetup, files }] of this.projects) { projects[name] = { modules: modules.map((file) => getRootRelativePath(root, file)), + config, globalSetup, files: Object.fromEntries(files), } @@ -223,3 +249,13 @@ export class DependenciesCache { } } } + +// the files that run every test when they change, only the ones that exist +function getConfigFiles(project: TestProject): string[] { + const files = new Set([ + ...(project.vitest.vite.config.configFileDependencies || []), + ...(project.vite.config.configFileDependencies || []), + ...getEnvFiles(project), + ]) + return Array.from(files).filter((file) => existsSync(file)) +} diff --git a/packages/vitest/src/node/cache/index.ts b/packages/vitest/src/node/cache/index.ts index 7f8ba09e4..d083e1f9f 100644 --- a/packages/vitest/src/node/cache/index.ts +++ b/packages/vitest/src/node/cache/index.ts @@ -60,7 +60,7 @@ export class VitestCache { async _update(specifications: TestSpecification[], startTime: number): Promise { await Promise.all([ this._results.update(specifications, startTime), - this._dependencies.update(specifications), + this._dependencies.update(specifications, startTime), ]) } diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index 0218263e1..d4d9a3557 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -505,6 +505,10 @@ export const cliOptionsConfig: VitestCLIOptions = { description: 'Run tests that are affected by the changed files (default: `false`)', argument: '[since]', }, + stale: { + description: + 'Run only the tests whose recorded dependencies were modified since the last run of the test. Enables `--changed`, `--experimental.recordDependencies` and the `mtime` provider (default: `false`)', + }, sequence: { description: 'Options for how tests should be sorted', argument: '', @@ -933,7 +937,8 @@ export const cliOptionsConfig: VitestCLIOptions = { }, vcsProvider: { argument: '', - description: 'Custom provider for detecting changed files. (default: `git`)', + description: + 'Provider for detecting changed files: `git`, `mtime` (requires `experimental.recordDependencies`) or a path to a custom provider. (default: `git`)', subcommands: null, }, recordDependencies: { @@ -1030,4 +1035,8 @@ export const collectCliOptionsConfig: VitestCLIOptions = { description: 'Print only tests that are affected by the changed files (default: `false`)', argument: '[since]', }, + stale: { + description: + 'Print only the tests whose recorded dependencies were modified since the last run of the test (default: `false`)', + }, } diff --git a/packages/vitest/src/node/config/resolveConfig.ts b/packages/vitest/src/node/config/resolveConfig.ts index 9b41728ed..94f71875a 100644 --- a/packages/vitest/src/node/config/resolveConfig.ts +++ b/packages/vitest/src/node/config/resolveConfig.ts @@ -37,6 +37,8 @@ import { RandomSequencer } from '../sequencers/RandomSequencer' import { API_TOKEN_FILE, resolveApiToken } from './apiToken' import { PluginHarness } from './pluginHarness' +const builtinVCSProviders = new Set(['git', 'mtime']) + function resolvePath(path: string, root: string) { // local-pkg (mlly)'s resolveModule("./file", { paths: ["/some/root"] }) tries // /some/file @@ -518,6 +520,17 @@ export function resolveTestConfig( } } } + if (resolved.stale) { + if (resolved.cache === false) { + throw new Error( + 'The "stale" option requires the "cache" option to store the dependencies of the test files.', + ) + } + resolved.changed ||= true + resolved.experimental ??= {} as any + resolved.experimental.recordDependencies = true + resolved.experimental.vcsProvider = 'mtime' + } if (resolved.coverage.changed === undefined && resolved.changed !== undefined) { resolved.coverage.changed = resolved.changed } @@ -1045,7 +1058,7 @@ export function resolveTestConfig( if ( typeof resolved.experimental.vcsProvider === 'string' && - resolved.experimental.vcsProvider !== 'git' + !builtinVCSProviders.has(resolved.experimental.vcsProvider) ) { resolved.experimental.vcsProvider = resolvePath( resolved.experimental.vcsProvider, diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index 294ae1513..ba9a59a96 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -718,7 +718,7 @@ export class Vitest { } await this._testRun.start(specifications) - await this.coverageProvider?.onTestRunStart?.() + await this.coverageProvider?.onTestRunStart?.(specifications) for (const file of files) { await this._reportFileTask(file) @@ -889,7 +889,7 @@ export class Vitest { if (!specifications.length) { await this._traces.$('vitest.test_run', async () => { await this._testRun.start([]) - await this.coverageProvider?.onTestRunStart?.() + await this.coverageProvider?.onTestRunStart?.([]) const coverage = await this.coverageProvider?.generateCoverage?.({ allTestsRun: true }) await this._testRun.end([], [], coverage) @@ -1034,7 +1034,7 @@ export class Vitest { private async runFiles(specs: TestSpecification[], allTestsRun: boolean): Promise { return this._traces.$('vitest.test_run', async () => { await this._testRun.start(specs) - await this.coverageProvider?.onTestRunStart?.() + await this.coverageProvider?.onTestRunStart?.(specs) // previous run await this.cancelPromise diff --git a/packages/vitest/src/node/coverage.ts b/packages/vitest/src/node/coverage.ts index 8a57bbd17..245735a49 100644 --- a/packages/vitest/src/node/coverage.ts +++ b/packages/vitest/src/node/coverage.ts @@ -11,6 +11,7 @@ import type { import type { SerializedCoverageConfig } from '../runtime/config' import type { AfterSuiteRunMeta } from '../types/general' import type { TestProject } from './project' +import type { TestSpecification } from './test-specification' import { createHash } from 'node:crypto' import { existsSync, promises as fs, readdirSync, writeFileSync } from 'node:fs' import module from 'node:module' @@ -24,6 +25,7 @@ import { glob } from 'tinyglobby' import c from 'tinyrainbow' import { coverageConfigDefaults } from '../defaults' import { resolveCoverageReporters } from '../node/config/resolveConfig' +import { createModulesResolver } from '../node/modules-resolver' import { getCoverageFilesDirectory, resolveCoverageProviderModule } from '../utils/coverage' type Threshold = 'lines' | 'functions' | 'statements' | 'branches' @@ -369,12 +371,13 @@ export class BaseCoverageProvider { } } - async onTestRunStart(): Promise { + async onTestRunStart(specifications: ReadonlyArray): Promise { if (this.options.changed) { try { const changedFiles = await this.ctx.vcs.findChangedFiles({ root: this.ctx.config.root, changedSince: this.options.changed, + resolver: createModulesResolver(this.ctx, specifications), }) this.changedFiles = changedFiles diff --git a/packages/vitest/src/node/modules-resolver.ts b/packages/vitest/src/node/modules-resolver.ts new file mode 100644 index 000000000..6fab3ddef --- /dev/null +++ b/packages/vitest/src/node/modules-resolver.ts @@ -0,0 +1,14 @@ +import type { Vitest } from './core' +import type { TestSpecification } from './test-specification' +import { AffectedModulesResolver } from './affected-modules' +import { RecordedModulesResolver } from './recorded-modules' + +export function createModulesResolver( + vitest: Vitest, + specifications: ReadonlyArray, +): AffectedModulesResolver | RecordedModulesResolver { + const specs = Array.from(specifications) + return vitest.config.experimental.recordDependencies + ? new RecordedModulesResolver(vitest, specs) + : new AffectedModulesResolver(vitest, specs) +} diff --git a/packages/vitest/src/node/recorded-modules.ts b/packages/vitest/src/node/recorded-modules.ts index 40884a510..22101bf72 100644 --- a/packages/vitest/src/node/recorded-modules.ts +++ b/packages/vitest/src/node/recorded-modules.ts @@ -1,6 +1,8 @@ +import type { DependencyRecord, ProjectDependencies } from './cache/dependencies' import type { Vitest } from './core' import type { TestProject } from './project' import type { TestSpecification } from './test-specification' +import type { ModuleDependency, ModulesResolver } from './vcs/vcs' import { getEnvFiles } from './affected-modules' import { getRootRelativePath } from './cache/results' @@ -8,41 +10,79 @@ import { getRootRelativePath } from './cache/results' * Finds test specifications that loaded any of the changed files during their last recorded run. * A specification without a record is always affected. */ -export class RecordedModulesResolver { +export class RecordedModulesResolver implements ModulesResolver { + private specsByProject = new Map() + constructor( private vitest: Vitest, - private related: string[], - ) {} - - async resolve(specs: TestSpecification[]): Promise { - if (this.dependsOnConfig(this.vitest.vite.config.configFileDependencies)) { - return specs - } - await this.vitest.cache._dependencies.read() - - const specsByProject = new Map() + private specs: TestSpecification[], + ) { for (const spec of specs) { - let projectSpecs = specsByProject.get(spec.project) + let projectSpecs = this.specsByProject.get(spec.project) if (!projectSpecs) { - specsByProject.set(spec.project, (projectSpecs = [])) + this.specsByProject.set(spec.project, (projectSpecs = [])) } projectSpecs.push(spec) } + } + + async getDependencies(): Promise { + await this.vitest.cache._dependencies.read() + const root = this.vitest.config.root + const recordedAt = new Map() + const add = (dependencies: ProjectDependencies, record: DependencyRecord) => { + for (const index of record.deps) { + const file = dependencies.modules[index] + const previous = recordedAt.get(file) + if (!recordedAt.has(file) || (previous !== undefined && record.at < previous)) { + recordedAt.set(file, record.at) + } + } + } + for (const [project, specs] of this.specsByProject) { + const dependencies = this.vitest.cache._dependencies.get(project) + if (dependencies?.config) { + add(dependencies, dependencies.config) + } + if (dependencies?.globalSetup) { + add(dependencies, dependencies.globalSetup) + } + for (const spec of specs) { + const record = dependencies?.files.get(getRootRelativePath(root, spec.moduleId)) + if (record && dependencies) { + add(dependencies, record) + } else if (!recordedAt.has(spec.moduleId)) { + // a test file without a record has no known time, so it always runs + recordedAt.set(spec.moduleId, undefined) + } + } + } + return Array.from(recordedAt, ([file, at]) => ({ file, recordedAt: at })) + } + + async getAffectedSpecifications(related: string[]): Promise { + if (dependsOnConfig(this.vitest.vite.config.configFileDependencies, related)) { + return this.specs + } + await this.vitest.cache._dependencies.read() const affected = new Set() - for (const [project, projectSpecs] of specsByProject) { - this.findAffectedInProject(project, projectSpecs).forEach((spec) => affected.add(spec)) + for (const [project, projectSpecs] of this.specsByProject) { + this.findAffectedInProject(project, projectSpecs, related).forEach((spec) => + affected.add(spec), + ) } - return specs.filter((spec) => affected.has(spec)) + return this.specs.filter((spec) => affected.has(spec)) } private findAffectedInProject( project: TestProject, specs: TestSpecification[], + related: string[], ): TestSpecification[] { if ( - this.dependsOnConfig(project.vite.config.configFileDependencies) || - this.dependsOnConfig(getEnvFiles(project)) + dependsOnConfig(project.vite.config.configFileDependencies, related) || + dependsOnConfig(getEnvFiles(project), related) ) { return specs } @@ -53,7 +93,7 @@ export class RecordedModulesResolver { } const changed = new Set() - for (const file of this.related) { + for (const file of related) { const index = dependencies.moduleIndex.get(file) if (index !== undefined) { changed.add(index) @@ -61,19 +101,19 @@ export class RecordedModulesResolver { } if (project.config.globalSetup.length) { const globalSetup = dependencies.globalSetup - if (!globalSetup || globalSetup.some((index) => changed.has(index))) { + if (!globalSetup || globalSetup.deps.some((index) => changed.has(index))) { return specs } } const root = this.vitest.config.root return specs.filter((spec) => { - const loaded = dependencies.files.get(getRootRelativePath(root, spec.moduleId)) - return !loaded || loaded.some((index) => changed.has(index)) + const record = dependencies.files.get(getRootRelativePath(root, spec.moduleId)) + return !record || record.deps.some((index) => changed.has(index)) }) } +} - private dependsOnConfig(configDependencies: string[] | undefined): boolean { - return !!configDependencies?.some((file) => this.related.includes(file)) - } +function dependsOnConfig(configDependencies: string[] | undefined, related: string[]): boolean { + return !!configDependencies?.some((file) => related.includes(file)) } diff --git a/packages/vitest/src/node/specifications.ts b/packages/vitest/src/node/specifications.ts index 72fe605ce..d08f47109 100644 --- a/packages/vitest/src/node/specifications.ts +++ b/packages/vitest/src/node/specifications.ts @@ -2,10 +2,9 @@ import type { Vitest } from './core' import type { TestSpecification } from './test-specification' import { relative, resolve } from 'pathe' import pm from 'picomatch' -import { AffectedModulesResolver } from './affected-modules' import { groupFilters, parseFilter } from './cli/filter' import { IncludeTaskLocationDisabledError, LocationFilterFileNotFoundError } from './errors' -import { RecordedModulesResolver } from './recorded-modules' +import { createModulesResolver } from './modules-resolver' export class VitestSpecifications { private readonly _cachedSpecs = new Map() @@ -123,10 +122,13 @@ export class VitestSpecifications { private async filterTestsBySource(specs: TestSpecification[]): Promise { this.vitest._sourceFilterResult = undefined + const resolver = createModulesResolver(this.vitest, specs) + if (this.vitest.config.changed && !this.vitest.config.related) { const related = await this.vitest.vcs.findChangedFiles({ root: this.vitest.config.root, changedSince: this.vitest.config.changed, + resolver, }) this.vitest.config.related = Array.from(new Set(related)) } @@ -149,10 +151,7 @@ export class VitestSpecifications { return [] } - const resolver = this.vitest.config.experimental.recordDependencies - ? new RecordedModulesResolver(this.vitest, related) - : new AffectedModulesResolver(this.vitest, related) - const affectedSpecs = await resolver.resolve(specs) + const affectedSpecs = await resolver.getAffectedSpecifications(related) this.vitest._sourceFilterResult = { affected: affectedSpecs.length, total: specs.length } return affectedSpecs } diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index e71a7ca71..b1ae9725b 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -1026,13 +1026,16 @@ export interface InlineConfig { nodeLoader?: boolean /** - * Custom provider for detecting changed files. Used with the `--changed` flag + * Provider for detecting changed files. Used with the `--changed` flag * to determine which files have been modified. * - * By default, Vitest uses Git to detect changed files. You can provide a custom - * implementation of the `VCSProvider` interface to use a different version control system. + * - `'git'` (default) compares the working tree with the `--changed` reference. + * - `'mtime'` compares the modification time of every recorded dependency with the run + * that recorded it. It requires `experimental.recordDependencies`. + * + * You can also provide a custom implementation of the `VCSProvider` interface. */ - vcsProvider?: VCSProvider | string + vcsProvider?: 'git' | 'mtime' | VCSProvider | (string & {}) /** * Record the modules that every test file loads in the cache and use the records @@ -1100,6 +1103,13 @@ export interface InlineConfig { * @default false */ changed?: boolean | string + + /** + * Runs only the tests whose recorded dependencies were modified since the last run of the test. + * Requires `cache`. + * @default false + */ + stale?: boolean } export interface TypecheckConfig { diff --git a/packages/vitest/src/node/types/coverage.ts b/packages/vitest/src/node/types/coverage.ts index 48fbdc3c9..8aef855fc 100644 --- a/packages/vitest/src/node/types/coverage.ts +++ b/packages/vitest/src/node/types/coverage.ts @@ -7,6 +7,7 @@ import type { RuntimeCoverageProviderModule, } from '../../utils/coverage' import type { Vitest } from '../core' +import type { TestSpecification } from '../test-specification' type TransformResult = string | Partial | undefined | null | void type CoverageResults = unknown @@ -27,7 +28,7 @@ export interface CoverageProvider { onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void | Promise /** Callback called when test run starts */ - onTestRunStart?: () => void | Promise + onTestRunStart?: (specifications: ReadonlyArray) => void | Promise /** Callback called when test run fails due to test failures */ onTestFailure?: () => void | Promise diff --git a/packages/vitest/src/node/vcs/mtime.ts b/packages/vitest/src/node/vcs/mtime.ts new file mode 100644 index 000000000..a1bea63c2 --- /dev/null +++ b/packages/vitest/src/node/vcs/mtime.ts @@ -0,0 +1,22 @@ +import type { VCSProvider, VCSProviderOptions } from './vcs' +import { stat } from 'node:fs/promises' + +/** + * Reports a dependency as changed when its modification time is newer than the run that recorded it. + * A dependency without a recorded time is always changed. + */ +export class MtimeVCSProvider implements VCSProvider { + async findChangedFiles(options: VCSProviderOptions): Promise { + const dependencies = await options.resolver.getDependencies() + const changed = await Promise.all( + dependencies.map(async ({ file, recordedAt }) => { + if (recordedAt == null) { + return file + } + const stats = await stat(file).catch(() => null) + return !stats || stats.mtimeMs > recordedAt ? file : null + }), + ) + return changed.filter((file) => file != null) + } +} diff --git a/packages/vitest/src/node/vcs/vcs.ts b/packages/vitest/src/node/vcs/vcs.ts index 32fea5270..df2283263 100644 --- a/packages/vitest/src/node/vcs/vcs.ts +++ b/packages/vitest/src/node/vcs/vcs.ts @@ -1,10 +1,36 @@ import type { ModuleRunner } from 'vite/module-runner' import { resolve } from 'pathe' import { GitVCSProvider } from './git' +import { MtimeVCSProvider } from './mtime' + +export interface ModuleDependency { + /** + * Absolute path of the file. + */ + file: string + /** + * Start time of the oldest run that recorded the file, in milliseconds since the epoch. + * A change after this time was not seen by every test file that depends on the file. + * It is not set when the dependency comes from the static module graph. + */ + recordedAt?: number +} + +export interface ModulesResolver { + /** + * Every file whose change can affect the test files of the run. + */ + // oxlint-disable-next-line typescript/method-signature-style + getDependencies(): Promise +} export interface VCSProviderOptions { root: string changedSince?: string | boolean + /** + * Resolves the dependencies of the test files of the run. + */ + resolver: ModulesResolver } export interface VCSProvider { @@ -22,6 +48,9 @@ export async function loadVCSProvider( if (!vcsProvider || vcsProvider === 'git') { return new GitVCSProvider() } + if (vcsProvider === 'mtime') { + return new MtimeVCSProvider() + } const module = (await runner.import(vcsProvider)) as { default: VCSProvider } if ( !module.default || diff --git a/packages/vitest/src/public/node.ts b/packages/vitest/src/public/node.ts index 6f80d5db1..c4f194a5d 100644 --- a/packages/vitest/src/public/node.ts +++ b/packages/vitest/src/public/node.ts @@ -33,6 +33,14 @@ export { resolveFsAllow } from '../node/plugins/utils' export type { ProcessPool } from '../node/pool' export { getFilePoolName } from '../node/pool' export { createMethodsRPC } from '../node/pools/rpc' +export type { + ModuleDependency, + ModulesResolver, + VCSProvider, + VCSProviderOptions, +} from '../node/vcs/vcs' +export { GitVCSProvider } from '../node/vcs/git' +export { MtimeVCSProvider } from '../node/vcs/mtime' export type { PoolOptions, PoolRunnerInitializer, diff --git a/test/e2e/test/recorded-deps.test.ts b/test/e2e/test/recorded-deps.test.ts index 106c8eb79..66e4ae933 100644 --- a/test/e2e/test/recorded-deps.test.ts +++ b/test/e2e/test/recorded-deps.test.ts @@ -23,18 +23,21 @@ function readDependencies(ctx: Vitest | undefined) { const { projects } = JSON.parse(readFileSync(getDependenciesPath(ctx), 'utf-8')) const result: Record> = {} for (const name in projects) { - const { modules, files, globalSetup } = projects[name] + const { modules, files, globalSetup, config } = projects[name] const project: Record = (result[name] = {}) const toFiles = (indices: number[]) => indices .map((index: number) => modules[index]) .filter((file: string) => !file.startsWith('../')) .sort() + if (config.deps.length) { + project[''] = toFiles(config.deps) + } if (globalSetup) { - project[''] = toFiles(globalSetup) + project[''] = toFiles(globalSetup.deps) } for (const file in files) { - project[file] = toFiles(files[file]) + project[file] = toFiles(files[file].deps) } } return result @@ -60,6 +63,9 @@ test('only the tests that loaded a changed file run', async () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/helper.js", @@ -114,6 +120,9 @@ test.for([ expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/inner.js", @@ -159,6 +168,9 @@ test('a mocked module does not load its own imports', async () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/dep.js", @@ -206,6 +218,9 @@ test('a file loaded by a setup file or a global setup runs every test', async () expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "": [ "global-setup.js", "src/global-helper.js", @@ -317,6 +332,9 @@ test('a mock loaded from a file is recorded', async () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "automock.test.js": [ "automock.test.js", "src/__mocks__/dep.js", @@ -383,6 +401,9 @@ test('a file imported with a query and a file added by a plugin are recorded', a expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "raw.test.js": [ "raw.test.js", "src/data.txt", @@ -430,21 +451,24 @@ test.for([{ pool: 'forks' }, { pool: 'vmForks' }])( ) expect(stderr).toBe('') expect(readDependencies(ctx)).toMatchInlineSnapshot(` - { - "": { - "a.test.js": [ - "a.test.js", - "environment.js", - "src/environment-helper.js", - ], - "b.test.js": [ - "b.test.js", - "environment.js", - "src/environment-helper.js", - ], - }, - } - `) + { + "": { + "": [ + "vitest.config.js", + ], + "a.test.js": [ + "a.test.js", + "environment.js", + "src/environment-helper.js", + ], + "b.test.js": [ + "b.test.js", + "environment.js", + "src/environment-helper.js", + ], + }, + } + `) const changed = await runVitest({ root: fs.root, @@ -475,6 +499,9 @@ test('a file loaded by a custom runner runs every test', async () => { expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "runner.js", @@ -544,6 +571,9 @@ exports[\`snap 1\`] = \`1\`; expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "file.test.js": [ "file.test.js", "fixtures/file.txt", @@ -593,13 +623,24 @@ test('records are kept per project', async () => { expect(stderr).toBe('') expect(readDependencies(ctx)).toMatchInlineSnapshot(` { + "": { + "": [ + "vitest.config.js", + ], + }, "first": { + "": [ + "vitest.config.js", + ], "first/a.test.js": [ "first/a.test.js", "src/shared.js", ], }, "second": { + "": [ + "vitest.config.js", + ], "second/b.test.js": [ "second/b.test.js", ], @@ -691,6 +732,9 @@ test('a complete run replaces the record of the file and keeps the others', asyn expect(readDependencies(filtered.ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", "src/other.js", @@ -791,9 +835,12 @@ test('a rerun of one file in watch mode keeps the records of the other files', a expect(readDependencies(ctx)).toMatchInlineSnapshot(` { "": { + "": [ + "vitest.config.js", + ], "a.test.js": [ "a.test.js", - "src/helper.js", + "src/other.js", ], "b.test.js": [ "b.test.js", diff --git a/test/e2e/test/vcs-mtime.test.ts b/test/e2e/test/vcs-mtime.test.ts new file mode 100644 index 000000000..6387a15e9 --- /dev/null +++ b/test/e2e/test/vcs-mtime.test.ts @@ -0,0 +1,184 @@ +import { readFileSync, rmSync } from 'node:fs' +import { expect, test } from 'vitest' +import { runInlineTests, runVitest } from '../../test-utils' + +const recorded = { cache: true, experimental: { recordDependencies: true, vcsProvider: 'mtime' } } + +function testFile(name: string, imports = '') { + return ` + ${imports} + import { test } from 'vitest' + test('${name}', () => {}) + ` +} + +const files = { + 'src/helper.js': 'export {}', + 'src/other.js': 'export {}', + 'a.test.js': testFile('a', `import './src/helper.js'`), + 'b.test.js': testFile('b', `import './src/other.js'`), +} + +test('runs the tests that loaded a file modified after their last run', async () => { + const { fs, stderr } = await runInlineTests(files, recorded) + expect(stderr).toBe('') + + fs.editFile('src/helper.js', () => 'export const helper = 1') + const changed = await runVitest({ root: fs.root, ...recorded, changed: true }) + expect(changed.stderr).toBe('') + expect(changed.testTree()).toMatchInlineSnapshot(` + { + "a.test.js": { + "a": "passed", + }, + } + `) + + // the run above recorded the file again, so nothing is newer than the records + const unchanged = await runVitest({ root: fs.root, ...recorded, changed: true }) + expect(unchanged.stderr).toBe('') + expect(unchanged.ctx!._sourceFilterResult).toEqual({ affected: 0, total: 2 }) +}) + +test('a deleted dependency and a modified config file are detected', async () => { + const { fs, stderr } = await runInlineTests( + { ...files, 'vitest.config.js': { test: recorded } }, + recorded, + ) + expect(stderr).toBe('') + + rmSync(fs.resolveFile('src/other.js')) + const deleted = await runVitest({ root: fs.root, ...recorded, changed: true }) + expect(deleted.errorTree()).toMatchInlineSnapshot(` + { + "b.test.js": { + "__module_errors__": [ + "Cannot find module './src/other.js' imported from /b.test.js", + ], + }, + } + `) + + fs.editFile('vitest.config.js', (content) => `${content}\n`) + const configChanged = await runVitest({ root: fs.root, ...recorded, changed: true }) + expect(Object.keys(configChanged.testTree()).sort()).toEqual(['a.test.js', 'b.test.js']) +}) + +test('a test file without a record runs', async () => { + const { fs, stderr } = await runInlineTests(files, recorded) + expect(stderr).toBe('') + + fs.createFile('c.test.js', testFile('c')) + const changed = await runVitest({ root: fs.root, ...recorded, changed: true }) + expect(changed.stderr).toBe('') + expect(Object.keys(changed.testTree())).toEqual(['c.test.js']) +}) + +test('every test runs without recorded dependencies', async () => { + const { testTree, stderr } = await runInlineTests(files, { + cache: true, + experimental: { vcsProvider: 'mtime' }, + changed: true, + }) + expect(stderr).toBe('') + expect(Object.keys(testTree()).sort()).toEqual(['a.test.js', 'b.test.js']) +}) + +test('a custom provider receives the dependencies of the test files', async () => { + const { fs, stderr } = await runInlineTests( + { + ...files, + 'vitest.config.js': ` + import { writeFileSync } from 'node:fs' + import { relative } from 'node:path' + export default { + test: { + cache: true, + experimental: { + recordDependencies: true, + vcsProvider: { + async findChangedFiles({ root, resolver }) { + const dependencies = await resolver.getDependencies() + const files = dependencies + .map(({ file, recordedAt }) => [relative(root, file), typeof recordedAt]) + .filter(([file]) => !file.startsWith('..')) + .sort() + writeFileSync(new URL('./dependencies.json', import.meta.url), JSON.stringify(files)) + return [] + }, + }, + }, + }, + } + `, + }, + { cache: true }, + ) + expect(stderr).toBe('') + + const changed = await runVitest({ root: fs.root, cache: true, changed: true }) + expect(changed.stderr).toBe('') + expect(changed.ctx!._sourceFilterResult).toEqual({ affected: 0, total: 2 }) + expect(JSON.parse(readFileSync(fs.resolveFile('dependencies.json'), 'utf-8'))) + .toMatchInlineSnapshot(` + [ + [ + "a.test.js", + "number", + ], + [ + "b.test.js", + "number", + ], + [ + "src/helper.js", + "number", + ], + [ + "src/other.js", + "number", + ], + [ + "vitest.config.js", + "number", + ], + ] + `) +}) + +test('coverage of changed files uses the dependencies of the run', async () => { + const coverage = { enabled: true, provider: 'v8', reporter: ['json'], changed: true } as const + const { fs, stderr } = await runInlineTests(files, recorded) + expect(stderr).toBe('') + + fs.editFile('src/helper.js', () => 'export const helper = 1') + const changed = await runVitest({ root: fs.root, ...recorded, coverage }) + expect(changed.stderr).toBe('') + const report = JSON.parse(readFileSync(fs.resolveFile('coverage/coverage-final.json'), 'utf-8')) + expect(Object.keys(report).map((file) => file.slice(fs.root.length + 1))).toEqual([ + 'src/helper.js', + ]) +}) + +test('--stale records the dependencies and runs the stale tests', async () => { + // the first run has no records, so every test runs + const { fs, stderr, testTree } = await runInlineTests(files, { cache: true, stale: true }) + expect(stderr).toBe('') + expect(Object.keys(testTree()).sort()).toEqual(['a.test.js', 'b.test.js']) + + fs.editFile('src/other.js', () => 'export const other = 1') + const stale = await runVitest({ root: fs.root, cache: true, stale: true }) + expect(stale.stderr).toBe('') + expect(Object.keys(stale.testTree())).toEqual(['b.test.js']) + + const fresh = await runVitest({ root: fs.root, cache: true, stale: true }) + expect(fresh.stderr).toBe('') + expect(fresh.ctx!._sourceFilterResult).toEqual({ affected: 0, total: 2 }) +}) + +test('--stale requires the cache', async () => { + const { stderr } = await runInlineTests(files, { cache: false, stale: true }) + expect(stderr.split('\n')[0]).toMatchInlineSnapshot( + `"Error: The "stale" option requires the "cache" option to store the dependencies of the test files."`, + ) +})