diff --git a/docs/config/index.md b/docs/config/index.md index 8a62a620bdd2..4b5b2693f7ee 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -1229,7 +1229,7 @@ Clean coverage results before running tests - **Available for providers:** `'v8' | 'istanbul'` - **CLI:** `--coverage.cleanOnRerun`, `--coverage.cleanOnRerun=false` -Clean coverage report on watch rerun +Clean coverage report on watch rerun. Set to `false` to preserve coverage results from previous run in watch mode. #### coverage.reportsDirectory diff --git a/packages/browser/src/client/tester/runner.ts b/packages/browser/src/client/tester/runner.ts index 5a2012fac025..83bf799cf1b7 100644 --- a/packages/browser/src/client/tester/runner.ts +++ b/packages/browser/src/client/tester/runner.ts @@ -91,6 +91,7 @@ export function createBrowserRunner( if (coverage) { await rpc().onAfterSuiteRun({ coverage, + testFiles: files.map(file => file.name), transformMode: 'browser', projectName: this.config.name, }) diff --git a/packages/coverage-istanbul/src/provider.ts b/packages/coverage-istanbul/src/provider.ts index 34769995522b..bde4ddd84850 100644 --- a/packages/coverage-istanbul/src/provider.ts +++ b/packages/coverage-istanbul/src/provider.ts @@ -35,14 +35,30 @@ import { version } from '../package.json' with { type: 'json' } import { COVERAGE_STORE_KEY } from './constants' type Options = ResolvedCoverageOptions<'istanbul'> -type Filename = string -type CoverageFilesByTransformMode = Record< - AfterSuiteRunMeta['transformMode'], - Filename[] + +/** + * Holds info about raw coverage results that are stored on file system: + * + * ```json + * "project-a": { + * "web": { + * "tests/math.test.ts": "coverage-1.json", + * "tests/utils.test.ts": "coverage-2.json", + * // ^^^^^^^^^^^^^^^ Raw coverage on file system + * }, + * "ssr": { ... }, + * "browser": { ... }, + * }, + * "project-b": ... + * ``` + */ +type CoverageFiles = Map< + NonNullable | typeof DEFAULT_PROJECT, + Record< + AfterSuiteRunMeta['transformMode'], + { [TestFilenames: string]: string } + > > -type ProjectName = - | NonNullable - | typeof DEFAULT_PROJECT interface TestExclude { new (opts: { @@ -70,7 +86,7 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co instrumenter!: Instrumenter testExclude!: InstanceType - coverageFiles: Map = new Map() + coverageFiles: CoverageFiles = new Map() coverageFilesDirectory!: string pendingPromises: Promise[] = [] @@ -188,7 +204,7 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co * Note that adding new entries here and requiring on those without * backwards compatibility is a breaking change. */ - onAfterSuiteRun({ coverage, transformMode, projectName }: AfterSuiteRunMeta): void { + onAfterSuiteRun({ coverage, transformMode, projectName, testFiles }: AfterSuiteRunMeta): void { if (!coverage) { return } @@ -200,15 +216,18 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co let entry = this.coverageFiles.get(projectName || DEFAULT_PROJECT) if (!entry) { - entry = { web: [], ssr: [], browser: [] } + entry = { web: {}, ssr: {}, browser: {} } this.coverageFiles.set(projectName || DEFAULT_PROJECT, entry) } + const testFilenames = testFiles.join() const filename = resolve( this.coverageFilesDirectory, `coverage-${uniqueId++}.json`, ) - entry[transformMode].push(filename) + + // If there's a result from previous run, overwrite it + entry[transformMode][testFilenames] = filename const promise = fs.writeFile(filename, JSON.stringify(coverage), 'utf-8') this.pendingPromises.push(promise) @@ -246,12 +265,13 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co this.pendingPromises = [] for (const coveragePerProject of this.coverageFiles.values()) { - for (const filenames of [ + for (const coverageByTestfiles of [ coveragePerProject.ssr, coveragePerProject.web, coveragePerProject.browser, ]) { const coverageMapByTransformMode = libCoverage.createCoverageMap({}) + const filenames = Object.values(coverageByTestfiles) for (const chunk of this.toSlices( filenames, @@ -281,7 +301,9 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co } } - if (this.options.all && allTestsRun) { + // Include untested files when all tests were run (not a single file re-run) + // or if previous results are preserved by "cleanOnRerun: false" + if (this.options.all && (allTestsRun || !this.options.cleanOnRerun)) { const coveredFiles = coverageMap.files() const uncoveredCoverage = await this.getCoverageMapForUncoveredFiles( coveredFiles, diff --git a/packages/coverage-v8/src/provider.ts b/packages/coverage-v8/src/provider.ts index 2921c26b6014..bb84e712f6b8 100644 --- a/packages/coverage-v8/src/provider.ts +++ b/packages/coverage-v8/src/provider.ts @@ -53,15 +53,31 @@ interface TestExclude { type Options = ResolvedCoverageOptions<'v8'> type TransformResults = Map -type Filename = string type RawCoverage = Profiler.TakePreciseCoverageReturnType -type CoverageFilesByTransformMode = Record< - AfterSuiteRunMeta['transformMode'], - Filename[] + +/** + * Holds info about raw coverage results that are stored on file system: + * + * ```json + * "project-a": { + * "web": { + * "tests/math.test.ts": "coverage-1.json", + * "tests/utils.test.ts": "coverage-2.json", + * // ^^^^^^^^^^^^^^^ Raw coverage on file system + * }, + * "ssr": { ... }, + * "browser": { ... }, + * }, + * "project-b": ... + * ``` + */ +type CoverageFiles = Map< + NonNullable | typeof DEFAULT_PROJECT, + Record< + AfterSuiteRunMeta['transformMode'], + { [TestFilenames: string]: string } + > > -type ProjectName = - | NonNullable - | typeof DEFAULT_PROJECT type Entries = [keyof T, T[keyof T]][] @@ -86,7 +102,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage options!: Options testExclude!: InstanceType - coverageFiles: Map = new Map() + coverageFiles: CoverageFiles = new Map() coverageFilesDirectory!: string pendingPromises: Promise[] = [] @@ -181,7 +197,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage * Note that adding new entries here and requiring on those without * backwards compatibility is a breaking change. */ - onAfterSuiteRun({ coverage, transformMode, projectName }: AfterSuiteRunMeta): void { + onAfterSuiteRun({ coverage, transformMode, projectName, testFiles }: AfterSuiteRunMeta): void { if (transformMode !== 'web' && transformMode !== 'ssr' && transformMode !== 'browser') { throw new Error(`Invalid transform mode: ${transformMode}`) } @@ -189,15 +205,18 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage let entry = this.coverageFiles.get(projectName || DEFAULT_PROJECT) if (!entry) { - entry = { web: [], ssr: [], browser: [] } + entry = { web: { }, ssr: { }, browser: { } } this.coverageFiles.set(projectName || DEFAULT_PROJECT, entry) } + const testFilenames = testFiles.join() const filename = resolve( this.coverageFilesDirectory, `coverage-${uniqueId++}.json`, ) - entry[transformMode].push(filename) + + // If there's a result from previous run, overwrite it + entry[transformMode][testFilenames] = filename const promise = fs.writeFile(filename, JSON.stringify(coverage), 'utf-8') this.pendingPromises.push(promise) @@ -212,9 +231,10 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage this.pendingPromises = [] for (const [projectName, coveragePerProject] of this.coverageFiles.entries()) { - for (const [transformMode, filenames] of Object.entries(coveragePerProject) as Entries) { + for (const [transformMode, coverageByTestfiles] of Object.entries(coveragePerProject) as Entries) { let merged: RawCoverage = { result: [] } + const filenames = Object.values(coverageByTestfiles) const project = this.ctx.projects.find(p => p.getName() === projectName) || this.ctx.getCoreWorkspaceProject() for (const chunk of this.toSlices(filenames, this.options.processingConcurrency)) { @@ -245,7 +265,9 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage } } - if (this.options.all && allTestsRun) { + // Include untested files when all tests were run (not a single file re-run) + // or if previous results are preserved by "cleanOnRerun: false" + if (this.options.all && (allTestsRun || !this.options.cleanOnRerun)) { const coveredFiles = coverageMap.files() const untestedCoverage = await this.getUntestedFiles(coveredFiles) @@ -519,7 +541,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage private async convertCoverage( coverage: RawCoverage, project: WorkspaceProject = this.ctx.getCoreWorkspaceProject(), - transformMode?: keyof CoverageFilesByTransformMode, + transformMode?: AfterSuiteRunMeta['transformMode'], ): Promise { let fetchCache = project.vitenode.fetchCache diff --git a/packages/vitest/src/public/index.ts b/packages/vitest/src/public/index.ts index f644c819b92c..42f2c031670b 100644 --- a/packages/vitest/src/public/index.ts +++ b/packages/vitest/src/public/index.ts @@ -264,7 +264,7 @@ export type { } from '../integrations/spy' export type { BrowserUI } from '../types/ui' -/** @deprecated import from `vitest/node` instead */ +/** @deprecated import from `vitest/reporter` instead */ export type Reporter = Reporter_ /** @deprecated import from `vitest/node` instead */ export type Vitest = Vitest_ diff --git a/packages/vitest/src/runtime/runners/index.ts b/packages/vitest/src/runtime/runners/index.ts index 803721a743d8..6c5c61fc156c 100644 --- a/packages/vitest/src/runtime/runners/index.ts +++ b/packages/vitest/src/runtime/runners/index.ts @@ -90,6 +90,7 @@ export async function resolveTestRunner( if (coverage) { rpc().onAfterSuiteRun({ coverage, + testFiles: files.map(file => file.name).sort(), transformMode: state.environment.transformMode, projectName: state.ctx.projectName, }) diff --git a/packages/vitest/src/types/general.ts b/packages/vitest/src/types/general.ts index 74b7f2e27be9..99cba19ca50b 100644 --- a/packages/vitest/src/types/general.ts +++ b/packages/vitest/src/types/general.ts @@ -24,6 +24,7 @@ export interface ModuleCache { export interface AfterSuiteRunMeta { coverage?: unknown + testFiles: string[] transformMode: TransformMode | 'browser' projectName?: string } diff --git a/test/coverage-test/fixtures/test/clean-on-rerun-fixture.test.ts b/test/coverage-test/fixtures/test/clean-on-rerun-fixture.test.ts new file mode 100644 index 000000000000..b19cab1faf7b --- /dev/null +++ b/test/coverage-test/fixtures/test/clean-on-rerun-fixture.test.ts @@ -0,0 +1,9 @@ +import { expect, test } from 'vitest' +import * as math from '../src/math' + +// This line will be changed by clean-on-rerun.test.ts +const methodToTest = 'sum' + +test(`run ${methodToTest}`, () => { + expect(() => math[methodToTest](1, 2)).not.toThrow() +}) diff --git a/test/coverage-test/test/changed.test.ts b/test/coverage-test/test/changed.test.ts index 78083beec26c..a8d3d53c6c5e 100644 --- a/test/coverage-test/test/changed.test.ts +++ b/test/coverage-test/test/changed.test.ts @@ -1,6 +1,6 @@ import { readFileSync, rmSync, writeFileSync } from 'node:fs' import { resolve } from 'node:path' -import { afterAll, beforeAll, expect } from 'vitest' +import { beforeAll, expect } from 'vitest' import { readCoverageMap, runVitest, test } from '../utils' // Note that this test may fail if you have new files in "vitest/test/coverage/src" @@ -11,9 +11,9 @@ const FILE_TO_CHANGE = resolve('./fixtures/src/file-to-change.ts') const NEW_UNCOVERED_FILE = resolve('./fixtures/src/new-uncovered-file.ts') beforeAll(() => { - let content = readFileSync(FILE_TO_CHANGE, 'utf8') - content = content.replace('This file will be modified by test cases', 'Changed!') - writeFileSync(FILE_TO_CHANGE, content, 'utf8') + const original = readFileSync(FILE_TO_CHANGE, 'utf8') + const changed = original.replace('This file will be modified by test cases', 'Changed!') + writeFileSync(FILE_TO_CHANGE, changed, 'utf8') writeFileSync(NEW_UNCOVERED_FILE, ` // This file is not covered by any tests but should be picked by --changed @@ -21,13 +21,11 @@ beforeAll(() => { return 'Hello world' } `.trim(), 'utf8') -}) -afterAll(() => { - let content = readFileSync(FILE_TO_CHANGE, 'utf8') - content = content.replace('Changed!', 'This file will be modified by test cases') - writeFileSync(FILE_TO_CHANGE, content, 'utf8') - rmSync(NEW_UNCOVERED_FILE) + return function restore() { + writeFileSync(FILE_TO_CHANGE, original, 'utf8') + rmSync(NEW_UNCOVERED_FILE) + } }) test('{ changed: "HEAD" }', async () => { diff --git a/test/coverage-test/test/clean-on-rerun.test.ts b/test/coverage-test/test/clean-on-rerun.test.ts new file mode 100644 index 000000000000..334a56140ca4 --- /dev/null +++ b/test/coverage-test/test/clean-on-rerun.test.ts @@ -0,0 +1,163 @@ +import { readFileSync, writeFileSync } from 'node:fs' +import { beforeEach, expect } from 'vitest' +import { readCoverageMap, runVitest, test } from '../utils' + +const FIXTURE = 'fixtures/test/clean-on-rerun-fixture.test.ts' + +beforeEach(() => { + const original = readFileSync(FIXTURE, 'utf8') + return () => writeFileSync(FIXTURE, original, 'utf8') +}) + +test('{ cleanOnReRun: false } should invalidate and preserve previous coverage', async () => { + const { waitForRun } = await startWatchMode({ cleanOnRerun: false }) + + // Initially only "sum" should be covered + expect(await getFunctionCoverageCounts('math.ts')).toMatchInlineSnapshot(` + { + "sum": 1, + } + `) + expect(await getFunctionCoverageCounts('even.ts')).toMatchInlineSnapshot(` + { + "isEven": 1, + } + `) + expect(await getReportedFiles()).toMatchInlineSnapshot(` + [ + "/fixtures/src/even.ts", + "/fixtures/src/math.ts", + "/fixtures/src/untested-file.ts", + ] + `) + + // Change test file to cover "multiply" only + await waitForRun(() => editTestFile('multiply')) + + // Sum should not be covered. Multiply should be. + expect(await getFunctionCoverageCounts('math.ts')).toMatchInlineSnapshot(` + { + "multiply": 1, + } + `) + // Results of non-changed file should preserve + expect(await getFunctionCoverageCounts('even.ts')).toMatchInlineSnapshot(` + { + "isEven": 1, + } + `) + // Untested file should still be in the report + expect(await getReportedFiles()).toMatchInlineSnapshot(` + [ + "/fixtures/src/even.ts", + "/fixtures/src/math.ts", + "/fixtures/src/untested-file.ts", + ] + `) + + // Change test file to cover "subtract" only + await waitForRun(() => editTestFile('subtract')) + + // Sum and multiply should not be covered. Subtract should be. + expect(await getFunctionCoverageCounts('math.ts')).toMatchInlineSnapshot(` + { + "subtract": 1, + } + `) + // Results of non-changed file should preserve + expect(await getFunctionCoverageCounts('even.ts')).toMatchInlineSnapshot(` + { + "isEven": 1, + } + `) + // Untested file should still be in the report + expect(await getReportedFiles()).toMatchInlineSnapshot(` + [ + "/fixtures/src/even.ts", + "/fixtures/src/math.ts", + "/fixtures/src/untested-file.ts", + ] + `) +}) + +test('{ cleanOnReRun: true } remove previous coverage results', async () => { + const { waitForRun } = await startWatchMode({ cleanOnRerun: true }) + + // Initially only "sum" should be covered + expect(await getFunctionCoverageCounts('math.ts')).toMatchInlineSnapshot(` + { + "sum": 1, + } + `) + // All files should be in report + expect(await getReportedFiles()).toMatchInlineSnapshot(` + [ + "/fixtures/src/even.ts", + "/fixtures/src/math.ts", + "/fixtures/src/untested-file.ts", + ] + `) + + // Change test file to cover "multiply" only + await waitForRun(() => editTestFile('multiply')) + + // Sum should not be covered. Multiply should be. + expect(await getFunctionCoverageCounts('math.ts')).toMatchInlineSnapshot(` + { + "multiply": 1, + } + `) + // Previous results should be removed, only math.ts should be present in report + expect(await getReportedFiles()).toMatchInlineSnapshot(` + [ + "/fixtures/src/math.ts", + ] + `) +}) + +async function startWatchMode(options: { cleanOnRerun: boolean }) { + const { vitest, ctx } = await runVitest({ + watch: true, + include: [FIXTURE, 'fixtures/test/even.test.ts'], + coverage: { + include: [ + 'fixtures/src/math.ts', + 'fixtures/src/even.ts', + 'fixtures/src/untested-file.ts', + ], + reporter: 'json', + ...options, + }, + }) + + async function waitForRun(method: () => void) { + vitest.resetOutput() + method() + await vitest.waitForStdout('1 passed') + await ctx?.runningPromise + } + + return { waitForRun } +} + +async function getFunctionCoverageCounts(file: 'math.ts' | 'even.ts') { + const coverageMap = await readCoverageMap() + const fileCoverage = coverageMap.fileCoverageFor(`/fixtures/src/${file}`) + + return Object.entries(fileCoverage.fnMap).reduce((total, [key, data]) => ({ + ...total, + ...(fileCoverage.f[key] ? { [data.name]: fileCoverage.f[key] } : {}), + + }), {} as Record<'sum' | 'subtract' | 'multiply' | 'remainder', number>) +} + +async function getReportedFiles() { + const coverageMap = await readCoverageMap() + return coverageMap.files() +} + +function editTestFile(method: 'sum' | 'subtract' | 'multiply' | 'remainder') { + let content = readFileSync(FIXTURE, 'utf8') + content = content.replace(/(const methodToTest = )'(.*)'/, `$1'${method}'`) + writeFileSync(FIXTURE, content, 'utf8') +} diff --git a/test/coverage-test/test/custom-provider.custom.test.ts b/test/coverage-test/test/custom-provider.custom.test.ts index eed7c3d135da..7674e06b823e 100644 --- a/test/coverage-test/test/custom-provider.custom.test.ts +++ b/test/coverage-test/test/custom-provider.custom.test.ts @@ -19,7 +19,8 @@ test('custom provider', async () => { "reportCoverage with {\\"allTestsRun\\":true}" ], "coverageReports": [ - "{\\"coverage\\":{\\"customCoverage\\":\\"Coverage report passed from workers to main thread\\"},\\"transformMode\\":\\"ssr\\",\\"projectName\\":\\"\\"}" + "{\\"coverage\\":{\\"customCoverage\\":\\"Coverage report passed from workers to main thread\\"},\\"testFiles\\":[\\"fixtures/test/even.test.ts\\"],\\"transformMode\\":\\"ssr\\",\\"projectName\\":\\"\\"}", + "{\\"coverage\\":{\\"customCoverage\\":\\"Coverage report passed from workers to main thread\\"},\\"testFiles\\":[\\"fixtures/test/math.test.ts\\"],\\"transformMode\\":\\"ssr\\",\\"projectName\\":\\"\\"}" ], "transformedFiles": [ "/fixtures/src/even.ts",