From 91ba6f95e4bea54c6e732161d05cd3e49c7712a3 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 2 Jul 2024 14:29:26 +0200 Subject: [PATCH] fix(vitest): show all failed tests when rerunning a test (#6022) --- packages/vitest/src/api/setup.ts | 2 +- packages/vitest/src/node/pools/typecheck.ts | 2 +- packages/vitest/src/node/reporters/base.ts | 121 ++++++++++-------- packages/vitest/src/node/reporters/basic.ts | 5 +- packages/vitest/src/node/reporters/default.ts | 10 ++ .../src/node/reporters/renderers/utils.ts | 13 +- packages/vitest/src/types/reporter.ts | 4 +- test/core/vite.config.ts | 1 - test/reporters/tests/merge-reports.test.ts | 4 +- .../fixtures/single-failed/basic.test.ts | 6 + .../fixtures/single-failed/failed.test.ts | 6 + .../fixtures/single-failed/vitest.config.ts | 11 ++ test/watch/fixtures/vitest.config.ts | 6 +- test/watch/test/reporter-failed.test.ts | 66 ++++++++++ 14 files changed, 194 insertions(+), 63 deletions(-) create mode 100644 test/watch/fixtures/single-failed/basic.test.ts create mode 100644 test/watch/fixtures/single-failed/failed.test.ts create mode 100644 test/watch/fixtures/single-failed/vitest.config.ts create mode 100644 test/watch/test/reporter-failed.test.ts diff --git a/packages/vitest/src/api/setup.ts b/packages/vitest/src/api/setup.ts index bb04c48c60de..69ee138e9b3a 100644 --- a/packages/vitest/src/api/setup.ts +++ b/packages/vitest/src/api/setup.ts @@ -201,7 +201,7 @@ export class WebSocketReporter implements Reporter { }) } - onFinished(files?: File[], errors?: unknown[]) { + onFinished(files: File[], errors: unknown[]) { this.clients.forEach((client) => { client.onFinished?.(files, errors)?.catch?.(noop) }) diff --git a/packages/vitest/src/node/pools/typecheck.ts b/packages/vitest/src/node/pools/typecheck.ts index 9edb034b9993..3c86225e5db6 100644 --- a/packages/vitest/src/node/pools/typecheck.ts +++ b/packages/vitest/src/node/pools/typecheck.ts @@ -40,7 +40,7 @@ export function createTypecheckPool(ctx: Vitest): ProcessPool { // triggered by TSC watcher, not Vitest watcher, so we need to emulate what Vitest does in this case if (ctx.config.watch && !ctx.runningPromise) { - await ctx.report('onFinished', files) + await ctx.report('onFinished', files, []) await ctx.report('onWatcherStart', files, [ ...(project.config.typecheck.ignoreSourceErrors ? [] : sourceErrors), ...ctx.state.getUnhandledErrors(), diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 5fcb7f6b69e7..70590fb208f5 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -14,6 +14,7 @@ import { getFullName, getSafeTimers, getSuites, + getTestName, getTests, hasFailed, hasFailedSnapshot, @@ -33,8 +34,8 @@ import { formatTimeString, getStateString, getStateSymbol, - pointer, renderSnapshotSummary, + taskFail, } from './renderers/utils' const BADGE_PADDING = ' ' @@ -63,6 +64,7 @@ export abstract class BaseReporter implements Reporter { start = 0 end = 0 watchFilters?: string[] + failedUnwatchedFiles: Task[] = [] isTTY: boolean ctx: Vitest = undefined! @@ -115,59 +117,65 @@ export abstract class BaseReporter implements Reporter { if (this.isTTY) { return } - const logger = this.ctx.logger for (const pack of packs) { const task = this.ctx.state.idMap.get(pack[0]) - if ( - task - && 'filepath' in task - && task.result?.state - && task.result?.state !== 'run' - ) { - const tests = getTests(task) - const failed = tests.filter(t => t.result?.state === 'fail') - const skipped = tests.filter( - t => t.mode === 'skip' || t.mode === 'todo', - ) - let state = c.dim(`${tests.length} test${tests.length > 1 ? 's' : ''}`) - if (failed.length) { - state += ` ${c.dim('|')} ${c.red(`${failed.length} failed`)}` - } - if (skipped.length) { - state += ` ${c.dim('|')} ${c.yellow(`${skipped.length} skipped`)}` - } - let suffix = c.dim(' (') + state + c.dim(')') - if (task.result.duration) { - const color - = task.result.duration > this.ctx.config.slowTestThreshold - ? c.yellow - : c.gray - suffix += color(` ${Math.round(task.result.duration)}${c.dim('ms')}`) - } - if (this.ctx.config.logHeapUsage && task.result.heap != null) { - suffix += c.magenta( - ` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`, - ) - } - - let title = ` ${getStateSymbol(task)} ` - if (task.projectName) { - title += formatProjectName(task.projectName) - } - title += `${task.name} ${suffix}` - logger.log(title) - - // print short errors, full errors will be at the end in summary - for (const test of failed) { - logger.log(c.red(` ${pointer} ${getFullName(test, c.dim(' > '))}`)) - test.result?.errors?.forEach((e) => { - logger.log(c.red(` ${F_RIGHT} ${(e as any)?.message}`)) - }) - } + if (task) { + this.printTask(task) } } } + protected printTask(task: Task) { + if ( + !('filepath' in task) + || !task.result?.state + || task.result?.state === 'run') { + return + } + const logger = this.ctx.logger + + const tests = getTests(task) + const failed = tests.filter(t => t.result?.state === 'fail') + const skipped = tests.filter( + t => t.mode === 'skip' || t.mode === 'todo', + ) + let state = c.dim(`${tests.length} test${tests.length > 1 ? 's' : ''}`) + if (failed.length) { + state += ` ${c.dim('|')} ${c.red(`${failed.length} failed`)}` + } + if (skipped.length) { + state += ` ${c.dim('|')} ${c.yellow(`${skipped.length} skipped`)}` + } + let suffix = c.dim(' (') + state + c.dim(')') + if (task.result.duration) { + const color + = task.result.duration > this.ctx.config.slowTestThreshold + ? c.yellow + : c.gray + suffix += color(` ${Math.round(task.result.duration)}${c.dim('ms')}`) + } + if (this.ctx.config.logHeapUsage && task.result.heap != null) { + suffix += c.magenta( + ` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`, + ) + } + + let title = ` ${getStateSymbol(task)} ` + if (task.projectName) { + title += formatProjectName(task.projectName) + } + title += `${task.name} ${suffix}` + logger.log(title) + + // print short errors, full errors will be at the end in summary + for (const test of failed) { + logger.log(c.red(` ${taskFail} ${getTestName(test, c.dim(' > '))}`)) + test.result?.errors?.forEach((e) => { + logger.log(c.red(` ${F_RIGHT} ${(e as any)?.message}`)) + }) + } + } + onWatcherStart( files = this.ctx.state.getFiles(), errors = this.ctx.state.getUnhandledErrors(), @@ -233,6 +241,9 @@ export abstract class BaseReporter implements Reporter { onWatcherRerun(files: string[], trigger?: string) { this.resetLastRunLog() this.watchFilters = files + this.failedUnwatchedFiles = this.ctx.state.getFiles().filter((file) => { + return !files.includes(file.filepath) && hasFailed(file) + }) files.forEach((filepath) => { let reruns = this._filesInWatchMode.get(filepath) ?? 0 @@ -274,6 +285,12 @@ export abstract class BaseReporter implements Reporter { ) } + if (!this.isTTY) { + for (const task of this.failedUnwatchedFiles) { + this.printTask(task) + } + } + this._timeStart = new Date() this.start = performance.now() } @@ -375,7 +392,11 @@ export abstract class BaseReporter implements Reporter { } reportTestSummary(files: File[], errors: unknown[]) { - const tests = getTests(files) + const affectedFiles = [ + ...this.failedUnwatchedFiles, + ...files, + ] + const tests = getTests(affectedFiles) const logger = this.ctx.logger const executionTime = this.end - this.start @@ -437,7 +458,7 @@ export abstract class BaseReporter implements Reporter { } } - logger.log(padTitle('Test Files'), getStateString(files)) + logger.log(padTitle('Test Files'), getStateString(affectedFiles)) logger.log(padTitle('Tests'), getStateString(tests)) if (this.ctx.projects.some(c => c.config.typecheck.enabled)) { const failed = tests.filter( diff --git a/packages/vitest/src/node/reporters/basic.ts b/packages/vitest/src/node/reporters/basic.ts index a6b2b3632a90..9f4438102bf8 100644 --- a/packages/vitest/src/node/reporters/basic.ts +++ b/packages/vitest/src/node/reporters/basic.ts @@ -2,7 +2,10 @@ import type { File } from '../../types/tasks' import { BaseReporter } from './base' export class BasicReporter extends BaseReporter { - isTTY = false + constructor() { + super() + this.isTTY = false + } reportSummary(files: File[], errors: unknown[]) { // non-tty mode doesn't add a new line diff --git a/packages/vitest/src/node/reporters/default.ts b/packages/vitest/src/node/reporters/default.ts index 1d19b46f2d35..a6b3dc513665 100644 --- a/packages/vitest/src/node/reporters/default.ts +++ b/packages/vitest/src/node/reporters/default.ts @@ -56,6 +56,16 @@ export class DefaultReporter extends BaseReporter { files = this.ctx.state.getFiles(), errors = this.ctx.state.getUnhandledErrors(), ) { + // print failed tests without their errors to keep track of previously failed tests + // this can happen if there are multiple test errors, and user changed a file + // that triggered a rerun of unrelated tests - in that case they want to see + // the error for the test they are currently working on, but still keep track of + // the other failed tests + this.renderer?.update([ + ...this.failedUnwatchedFiles, + ...files, + ]) + this.stopListRender() this.ctx.logger.log() super.onFinished(files, errors) diff --git a/packages/vitest/src/node/reporters/renderers/utils.ts b/packages/vitest/src/node/reporters/renderers/utils.ts index 4686c1b1abf5..ea8cab848137 100644 --- a/packages/vitest/src/node/reporters/renderers/utils.ts +++ b/packages/vitest/src/node/reporters/renderers/utils.ts @@ -19,6 +19,12 @@ export const hookSpinnerMap = new WeakMap string>>() export const pointer = c.yellow(F_POINTER) export const skipped = c.dim(c.gray(F_DOWN)) +export const benchmarkPass = c.green(F_DOT) +export const testPass = c.green(F_CHECK) +export const taskFail = c.red(F_CROSS) +export const suiteFail = c.red(F_POINTER) +export const pending = c.gray('·') + export function getCols(delta = 0) { let length = process.stdout?.columns if (!length || Number.isNaN(length)) { @@ -154,10 +160,9 @@ export function getStateSymbol(task: Task) { } if (!task.result) { - return c.gray('·') + return pending } - // pending if (task.result.state === 'run') { if (task.type === 'suite') { return pointer @@ -171,11 +176,11 @@ export function getStateSymbol(task: Task) { } if (task.result.state === 'pass') { - return task.meta?.benchmark ? c.green(F_DOT) : c.green(F_CHECK) + return task.meta?.benchmark ? benchmarkPass : testPass } if (task.result.state === 'fail') { - return task.type === 'suite' ? pointer : c.red(F_CROSS) + return task.type === 'suite' ? suiteFail : taskFail } return ' ' diff --git a/packages/vitest/src/types/reporter.ts b/packages/vitest/src/types/reporter.ts index ff2518aa7171..968eb85da159 100644 --- a/packages/vitest/src/types/reporter.ts +++ b/packages/vitest/src/types/reporter.ts @@ -8,8 +8,8 @@ export interface Reporter { onSpecsCollected?: (specs?: SerializableSpec[]) => Awaitable onCollected?: (files?: File[]) => Awaitable onFinished?: ( - files?: File[], - errors?: unknown[], + files: File[], + errors: unknown[], coverage?: unknown ) => Awaitable onTaskUpdate?: (packs: TaskResultPack[]) => Awaitable diff --git a/test/core/vite.config.ts b/test/core/vite.config.ts index 4ebb0cb03994..cd0bf996e990 100644 --- a/test/core/vite.config.ts +++ b/test/core/vite.config.ts @@ -50,7 +50,6 @@ export default defineConfig({ port: 3022, }, test: { - reporters: ['dot'], api: { port: 3023, }, diff --git a/test/reporters/tests/merge-reports.test.ts b/test/reporters/tests/merge-reports.test.ts index be73e8dc03db..70de1b63fba0 100644 --- a/test/reporters/tests/merge-reports.test.ts +++ b/test/reporters/tests/merge-reports.test.ts @@ -89,13 +89,13 @@ test('merge reports', async () => { test 1-2 ❯ first.test.ts (2 tests | 1 failed)