Skip to content

Commit 47fb992

Browse files
authored
fix(reporter): report crashed test files as failed in json (#11965)
1 parent cab34bb commit 47fb992

2 files changed

Lines changed: 102 additions & 6 deletions

File tree

‎packages/vitest/src/node/reporters/json.ts‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { CoverageMap } from '@vitest/istanbul-lib-coverage'
22
import type { SnapshotSummary } from '@vitest/snapshot'
3-
import type { Suite, TaskMeta, TaskState, TestBenchmark } from '../../runtime/runner/types'
3+
import type { File, Suite, TaskMeta, TaskState, TestBenchmark } from '../../runtime/runner/types'
44
import type { Vitest } from '../core'
55
import type { Reporter } from '../types/reporter'
66
import type { TestModule } from './reported-tasks'
@@ -113,9 +113,15 @@ export class JsonReporter implements Reporter {
113113
const tests = getTests(files)
114114
const numTotalTests = tests.length
115115

116-
const numFailedTestSuites = suites.filter((s) => s.result?.state === 'fail').length
116+
const unfinishedFiles = new Set<Suite>(files.filter(isUnfinished))
117+
118+
const numFailedTestSuites = suites.filter(
119+
(s) => s.result?.state === 'fail' || unfinishedFiles.has(s),
120+
).length
117121
const numPendingTestSuites = suites.filter(
118-
(s) => s.result?.state === 'run' || s.result?.state === 'queued' || s.mode === 'todo',
122+
(s) =>
123+
!unfinishedFiles.has(s) &&
124+
(s.result?.state === 'run' || s.result?.state === 'queued' || s.mode === 'todo'),
119125
).length
120126
const numPassedTestSuites = numTotalTestSuites - numFailedTestSuites - numPendingTestSuites
121127

@@ -200,7 +206,10 @@ export class JsonReporter implements Reporter {
200206
assertionResults,
201207
startTime,
202208
endTime,
203-
status: file.result?.state === 'fail' || hasFailedTests ? 'failed' : 'passed',
209+
status:
210+
unfinishedFiles.has(file) || file.result?.state === 'fail' || hasFailedTests
211+
? 'failed'
212+
: 'passed',
204213
message: file.result?.errors?.[0]?.message ?? '',
205214
name: file.filepath,
206215
})
@@ -245,3 +254,14 @@ export class JsonReporter implements Reporter {
245254
}
246255
}
247256
}
257+
258+
// the file did not finish, for example because its worker crashed
259+
// (same as `pending` or `queued` in `TestModule.state()`)
260+
function isUnfinished(file: File): boolean {
261+
const state = file.result?.state
262+
return (
263+
file.mode !== 'skip' &&
264+
file.mode !== 'todo' &&
265+
(state == null || state === 'run' || state === 'only' || state === 'queued')
266+
)
267+
}

‎test/e2e/test/reporters/json.test.ts‎

Lines changed: 78 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { readFileSync } from 'node:fs'
2-
import { resolve } from 'pathe'
2+
import { basename, resolve } from 'pathe'
33
import { describe, expect, it } from 'vitest'
4-
import { runVitest } from '#test-utils'
4+
import { runInlineTests, runVitest } from '#test-utils'
55

66
describe('json reporter', async () => {
77
const root = resolve(import.meta.dirname, '..', '..', 'fixtures', 'reporters')
@@ -202,4 +202,80 @@ describe('json reporter', async () => {
202202
success: true,
203203
})
204204
})
205+
206+
describe('when a worker crashes', () => {
207+
async function runWithCrash(crashTest: string) {
208+
const { stdout, exitCode } = await runInlineTests(
209+
{
210+
'ok.test.ts': `
211+
import { expect, test } from 'vitest'
212+
213+
test('ok', () => {
214+
expect(1).toBe(1)
215+
})
216+
`,
217+
'crash.test.ts': crashTest,
218+
},
219+
{
220+
pool: 'forks',
221+
dangerouslyIgnoreUnhandledErrors: true,
222+
reporters: [['json', { stdout: true }]],
223+
},
224+
)
225+
226+
const data = JSON.parse(stdout)
227+
return {
228+
exitCode,
229+
data,
230+
statuses: Object.fromEntries(
231+
data.testResults.map((r: any) => [basename(r.name), r.status]),
232+
),
233+
}
234+
}
235+
236+
it('reports the file as failed if the worker crashes during collection', async () => {
237+
const { exitCode, data, statuses } = await runWithCrash(`
238+
import { test } from 'vitest'
239+
240+
// the worker dies while Vitest collects the tests of this file
241+
process.kill(process.pid, 'SIGKILL')
242+
243+
test('never collected', () => {})
244+
`)
245+
246+
expect(exitCode).toBe(1)
247+
expect(statuses).toEqual({ 'crash.test.ts': 'failed', 'ok.test.ts': 'passed' })
248+
expect(data).toMatchObject({
249+
numTotalTestSuites: 2,
250+
numFailedTestSuites: 1,
251+
numPassedTestSuites: 1,
252+
numPendingTestSuites: 0,
253+
success: false,
254+
})
255+
})
256+
257+
it('reports the file as failed if the worker crashes during a test', async () => {
258+
const { exitCode, data, statuses } = await runWithCrash(`
259+
import { test } from 'vitest'
260+
261+
test('before', () => {})
262+
263+
test('kills the worker', () => {
264+
process.kill(process.pid, 'SIGKILL')
265+
})
266+
267+
test('after', () => {})
268+
`)
269+
270+
expect(exitCode).toBe(1)
271+
expect(statuses).toEqual({ 'crash.test.ts': 'failed', 'ok.test.ts': 'passed' })
272+
expect(data).toMatchObject({
273+
numTotalTestSuites: 2,
274+
numFailedTestSuites: 1,
275+
numPassedTestSuites: 1,
276+
numPendingTestSuites: 0,
277+
success: false,
278+
})
279+
})
280+
})
205281
})

0 commit comments

Comments
 (0)