From bfcfe72bf629eb9b3d1399776d171be166d88c47 Mon Sep 17 00:00:00 2001 From: pulkit-30 Date: Tue, 19 Dec 2023 18:36:06 +0530 Subject: [PATCH] test_runner: feat add coverage threshold for tests --- index.mjs | 11 +++ index.test.mjs | 6 ++ lib/internal/test_runner/coverage.js | 23 +++++- lib/internal/test_runner/harness.js | 17 +++- lib/internal/test_runner/runner.js | 12 ++- lib/internal/test_runner/test.js | 20 ++++- lib/internal/test_runner/utils.js | 11 ++- test/parallel/test-runner-coverage.js | 114 ++++++++++++++++---------- test/parallel/test-runner-run.mjs | 38 +++++++++ 9 files changed, 197 insertions(+), 55 deletions(-) create mode 100644 index.mjs create mode 100644 index.test.mjs diff --git a/index.mjs b/index.mjs new file mode 100644 index 00000000000000..1ed03cad67cdbd --- /dev/null +++ b/index.mjs @@ -0,0 +1,11 @@ +import { spec } from 'node:test/reporters'; +import { run } from 'node:test'; +import process from 'node:process'; + +run({ files: ['./index.test.mjs'], coverage: { + lines: 100, + branches: 100, + functions: 100, +} }) + .compose(spec) + .pipe(process.stdout); \ No newline at end of file diff --git a/index.test.mjs b/index.test.mjs new file mode 100644 index 00000000000000..0936b46baf3e4a --- /dev/null +++ b/index.test.mjs @@ -0,0 +1,6 @@ +import assert from 'node:assert'; +import test from 'node:test'; + +test('hello test', () => { + // throw new Error("error"); +}); diff --git a/lib/internal/test_runner/coverage.js b/lib/internal/test_runner/coverage.js index 7727ab006052ba..61cba6ea36ea80 100644 --- a/lib/internal/test_runner/coverage.js +++ b/lib/internal/test_runner/coverage.js @@ -31,6 +31,12 @@ const kIgnoreRegex = /\/\* node:coverage ignore next (?\d+ )?\*\//; const kLineEndingRegex = /\r?\n$/u; const kLineSplitRegex = /(?<=\r?\n)/u; const kStatusRegex = /\/\* node:coverage (?enable|disable) \*\//; +const kDefaultMinimumThreshold = { + __proto__: null, + lines: 0, + functions: 0, + branches: 0, +}; class CoverageLine { #covered; @@ -63,10 +69,11 @@ class CoverageLine { } class TestCoverage { - constructor(coverageDirectory, originalCoverageDirectory, workingDirectory) { + constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, minimumThreshold) { this.coverageDirectory = coverageDirectory; this.originalCoverageDirectory = originalCoverageDirectory; this.workingDirectory = workingDirectory; + this.minimumThreshold = minimumThreshold || kDefaultMinimumThreshold; } summary() { @@ -260,6 +267,12 @@ class TestCoverage { ); coverageSummary.files.sort(sortCoverageFiles); + coverageSummary.threshold = { + __proto__: null, + lines: doesThresholdPass(this.minimumThreshold.lines, coverageSummary.totals.coveredLinePercent), + functions: doesThresholdPass(this.minimumThreshold.functions, coverageSummary.totals.coveredFunctionPercent), + branches: doesThresholdPass(this.minimumThreshold.branches, coverageSummary.totals.coveredBranchPercent), + }; return coverageSummary; } @@ -299,7 +312,7 @@ function sortCoverageFiles(a, b) { return StringPrototypeLocaleCompare(a.path, b.path); } -function setupCoverage() { +function setupCoverage(minimumThreshold = null) { let originalCoverageDirectory = process.env.NODE_V8_COVERAGE; const cwd = process.cwd(); @@ -323,7 +336,7 @@ function setupCoverage() { // child processes. process.env.NODE_V8_COVERAGE = coverageDirectory; - return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd); + return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd, minimumThreshold); } function mapRangeToLines(range, lines) { @@ -538,4 +551,8 @@ function doesRangeContainOtherRange(range, otherRange) { range.endOffset >= otherRange.endOffset; } +function doesThresholdPass(compareValue, actualValue) { + return compareValue ? actualValue >= compareValue : true; +} + module.exports = { setupCoverage, TestCoverage }; diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 2f18b0bcf091ac..c045413b00f25e 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -1,9 +1,12 @@ 'use strict'; const { + ArrayPrototypeFilter, ArrayPrototypeForEach, FunctionPrototypeBind, PromiseResolve, SafeMap, + StringPrototypeSplit, + StringPrototypeStartsWith, } = primordials; const { getCallerLocation } = internalBinding('util'); const { @@ -78,14 +81,20 @@ function createProcessEventHandler(eventName, rootTest) { } function configureCoverage(rootTest, globalOptions) { - if (!globalOptions.coverage) { - return null; + let coverageThreshold = rootTest.coverage; + if (!coverageThreshold) { + if (!globalOptions.coverage) { + return null; + } + const { 1: value } = StringPrototypeSplit( + ArrayPrototypeFilter(process.execArgv, (arg) => + StringPrototypeStartsWith(arg, '--experimental-test-coverage')), '='); + coverageThreshold = { __proto__: null, lines: value, functions: value, branches: value }; } - const { setupCoverage } = require('internal/test_runner/coverage'); try { - return setupCoverage(); + return setupCoverage(coverageThreshold); } catch (err) { const msg = `Warning: Code coverage could not be enabled. ${err}`; diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index 34100213ebd935..ecccc3192c0acc 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -439,7 +439,7 @@ function run(options = kEmptyObject) { validateObject(options, 'options'); let { testNamePatterns, shard } = options; - const { concurrency, timeout, signal, files, inspectPort, watch, setup, only } = options; + const { concurrency, timeout, signal, files, inspectPort, watch, setup, only, coverage } = options; if (files != null) { validateArray(files, 'options.files'); @@ -486,7 +486,15 @@ function run(options = kEmptyObject) { }); } - const root = createTestTree({ __proto__: null, concurrency, timeout, signal }); + if (coverage != null) { + if (typeof coverage === 'boolean') { + validateBoolean(coverage, 'options.coverage'); + } else { + validateObject(coverage, 'options.coverage'); + } + } + + const root = createTestTree({ __proto__: null, concurrency, timeout, signal, coverage }); if (process.env.NODE_TEST_CONTEXT !== undefined) { return root.reporter; } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 19ae283ed9ad78..76e9bf42c8e860 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -1,5 +1,7 @@ 'use strict'; const { + ArrayPrototypeFilter, + ArrayPrototypeJoin, ArrayPrototypePush, ArrayPrototypePushApply, ArrayPrototypeReduce, @@ -10,6 +12,8 @@ const { FunctionPrototype, MathMax, Number, + ObjectDefineProperty, + ObjectKeys, ObjectSeal, PromisePrototypeThen, PromiseResolve, @@ -21,7 +25,6 @@ const { SafePromiseAll, SafePromiseRace, SymbolDispose, - ObjectDefineProperty, Symbol, } = primordials; const { getCallerLocation } = internalBinding('util'); @@ -49,6 +52,7 @@ const { once: runOnce, } = require('internal/util'); const { isPromise } = require('internal/util/types'); +const console = require('internal/console/global'); const { validateAbortSignal, validateNumber, @@ -209,7 +213,7 @@ class Test extends AsyncResource { super('Test'); let { fn, name, parent, skip } = options; - const { concurrency, loc, only, timeout, todo, signal } = options; + const { concurrency, loc, only, timeout, todo, signal, coverage } = options; if (typeof fn !== 'function') { fn = noop; @@ -239,6 +243,7 @@ class Test extends AsyncResource { beforeEach: [], afterEach: [], }; + this.coverage = coverage; } else { const nesting = parent.parent === null ? parent.nesting : parent.nesting + 1; @@ -258,6 +263,7 @@ class Test extends AsyncResource { beforeEach: ArrayPrototypeSlice(parent.hooks.beforeEach), afterEach: ArrayPrototypeSlice(parent.hooks.afterEach), }; + this.coverage = parent.coverage; } switch (typeof concurrency) { @@ -670,7 +676,9 @@ class Test extends AsyncResource { // postRun() method is called when the process is getting ready to exit. // This helps catch any asynchronous activity that occurs after the tests // have finished executing. - this.postRun(); + try { + this.postRun(); + } catch { /* ignore error */ } } } @@ -754,6 +762,12 @@ class Test extends AsyncResource { if (coverage) { reporter.coverage(nesting, loc, coverage); + const failedCoverage = ArrayPrototypeFilter( + ObjectKeys(coverage.threshold), (key) => !coverage.threshold[key]); + if (failedCoverage.length > 0) { + console.error(`test coverage failed for ${ArrayPrototypeJoin(failedCoverage, ', ')}`); + process.exit(1); + } } reporter.end(); diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 6b4663f14302c3..66021b4dcaa280 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -3,17 +3,19 @@ const { ArrayPrototypeJoin, ArrayPrototypeMap, ArrayPrototypeFlatMap, + ArrayPrototypeForEach, ArrayPrototypePush, ArrayPrototypeReduce, ObjectGetOwnPropertyDescriptor, + ObjectKeys, MathFloor, MathMax, MathMin, NumberPrototypeToFixed, - SafePromiseAllReturnArrayLike, RegExp, RegExpPrototypeExec, SafeMap, + SafePromiseAllReturnArrayLike, StringPrototypePadStart, StringPrototypePadEnd, StringPrototypeRepeat, @@ -316,6 +318,7 @@ const kSeparator = ' | '; function getCoverageReport(pad, summary, symbol, color, table) { const prefix = `${pad}${symbol}`; + color ||= white; let report = `${color}${prefix}start of coverage report\n`; let filePadLength; @@ -405,6 +408,12 @@ function getCoverageReport(pad, summary, symbol, color, table) { `${ArrayPrototypeJoin(ArrayPrototypeMap(kColumnsKeys, (columnKey, j) => getCell(NumberPrototypeToFixed(summary.totals[columnKey], 2), columnPadLengths[j], StringPrototypePadStart, false, summary.totals[columnKey])), kSeparator)} |\n`; if (table) report += addTableLine(prefix, tableWidth); + ArrayPrototypeForEach(ObjectKeys(summary.threshold), (key) => { + if (!summary.threshold[key]) { + report += `${red}${prefix}coverage threshold for ${key} not met\n${color}`; + } + }); + report += `${prefix}end of coverage report\n`; if (color) { report += white; diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index d8181417205b46..17ae43019f7ecf 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -21,46 +21,24 @@ function findCoverageFileForPid(pid) { }); } -function getTapCoverageFixtureReport() { - /* eslint-disable max-len */ - const report = [ - '# start of coverage report', - '# -------------------------------------------------------------------------------------------------------------------', - '# file | line % | branch % | funcs % | uncovered lines', - '# -------------------------------------------------------------------------------------------------------------------', - '# test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72', - '# test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ', - '# test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5-6', - '# -------------------------------------------------------------------------------------------------------------------', - '# all files | 78.35 | 43.75 | 60.00 |', - '# -------------------------------------------------------------------------------------------------------------------', - '# end of coverage report', - ].join('\n'); - /* eslint-enable max-len */ - - if (common.isWindows) { - return report.replaceAll('/', '\\'); - } +function getCoverageFixtureReport(reporter = 'tap', extraReport = []) { + const symbol = reporter === 'spec' ? '\u2139' : '#'; - return report; -} - -function getSpecCoverageFixtureReport() { - /* eslint-disable max-len */ const report = [ - '\u2139 start of coverage report', - '\u2139 -------------------------------------------------------------------------------------------------------------------', - '\u2139 file | line % | branch % | funcs % | uncovered lines', - '\u2139 -------------------------------------------------------------------------------------------------------------------', - '\u2139 test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72', - '\u2139 test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ', - '\u2139 test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5-6', - '\u2139 -------------------------------------------------------------------------------------------------------------------', - '\u2139 all files | 78.35 | 43.75 | 60.00 |', - '\u2139 -------------------------------------------------------------------------------------------------------------------', - '\u2139 end of coverage report', + `${symbol} start of coverage report`, + `${symbol} -------------------------------------------------------------------------------------------------------------------`, + `${symbol} file | line % | branch % | funcs % | uncovered lines`, + `${symbol} -------------------------------------------------------------------------------------------------------------------`, + `${symbol} test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72`, + `${symbol} test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | `, + `${symbol} test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5-6`, + `${symbol} -------------------------------------------------------------------------------------------------------------------`, + `${symbol} all files | 78.35 | 43.75 | 60.00 |`, + `${symbol} -------------------------------------------------------------------------------------------------------------------`, + ...extraReport, + `${symbol} end of coverage report`, ].join('\n'); - /* eslint-enable max-len */ + if (common.isWindows) { return report.replaceAll('/', '\\'); @@ -69,6 +47,7 @@ function getSpecCoverageFixtureReport() { return report; } + test('test coverage report', async (t) => { await t.test('handles the inspector not being available', (t) => { if (process.features.inspector) { @@ -92,7 +71,7 @@ test('test tap coverage reporter', skipIfNoInspector, async (t) => { const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture]; const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } }; const result = spawnSync(process.execPath, args, options); - const report = getTapCoverageFixtureReport(); + const report = getCoverageFixtureReport(); assert(result.stdout.toString().includes(report)); assert.strictEqual(result.stderr.toString(), ''); assert.strictEqual(result.status, 0); @@ -103,7 +82,7 @@ test('test tap coverage reporter', skipIfNoInspector, async (t) => { const fixture = fixtures.path('test-runner', 'coverage.js'); const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture]; const result = spawnSync(process.execPath, args); - const report = getTapCoverageFixtureReport(); + const report = getCoverageFixtureReport(); assert(result.stdout.toString().includes(report)); assert.strictEqual(result.stderr.toString(), ''); @@ -118,7 +97,7 @@ test('test spec coverage reporter', skipIfNoInspector, async (t) => { const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture]; const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } }; const result = spawnSync(process.execPath, args, options); - const report = getSpecCoverageFixtureReport(); + const report = getCoverageFixtureReport('spec'); assert(result.stdout.toString().includes(report)); assert.strictEqual(result.stderr.toString(), ''); @@ -130,7 +109,7 @@ test('test spec coverage reporter', skipIfNoInspector, async (t) => { const fixture = fixtures.path('test-runner', 'coverage.js'); const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture]; const result = spawnSync(process.execPath, args); - const report = getSpecCoverageFixtureReport(); + const report = getCoverageFixtureReport('spec'); assert(result.stdout.toString().includes(report)); assert.strictEqual(result.stderr.toString(), ''); @@ -145,7 +124,7 @@ test('single process coverage is the same with --test', skipIfNoInspector, () => '--test', '--experimental-test-coverage', '--test-reporter', 'tap', fixture, ]; const result = spawnSync(process.execPath, args); - const report = getTapCoverageFixtureReport(); + const report = getCoverageFixtureReport(); assert.strictEqual(result.stderr.toString(), ''); assert(result.stdout.toString().includes(report)); @@ -242,3 +221,54 @@ test('coverage reports on lines, functions, and branches', skipIfNoInspector, as }); }); }); + +test('test coverage for tap reporter with min 100% threshold value', async (t) => { + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--experimental-test-coverage=100', '--test-reporter', 'tap', fixture]; + const options = { env: { ...process.env } }; + const result = spawnSync(process.execPath, args, options); + const report = getCoverageFixtureReport('tap', + ['# coverage threshold for lines not met', + '# coverage threshold for branches not met', + '# coverage threshold for functions not met']); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.stderr.toString(), ''); + assert.strictEqual(result.status, 0); +}); + +test('test coverage for spec reporter with min 100% threshold value', async () => { + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--test', '--experimental-test-coverage=100', '--test-reporter', 'spec', fixture]; + const options = { env: { ...process.env } }; + const result = spawnSync(process.execPath, args, options); + const report = getCoverageFixtureReport('spec', + ['\u2139 coverage threshold for lines not met', + '\u2139 coverage threshold for branches not met', + '\u2139 coverage threshold for functions not met']); + assert.strictEqual(result.stderr.toString(), ''); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.status, 0); +}); + + +test('test coverage for tap reporter with min 10% threshold value', async (t) => { + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--experimental-test-coverage=10', '--test-reporter', 'tap', fixture]; + const options = { env: { ...process.env } }; + const result = spawnSync(process.execPath, args, options); + const report = getCoverageFixtureReport('tap'); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.stderr.toString(), ''); + assert.strictEqual(result.status, 0); +}); + +test('test coverage for spec reporter with min 10% threshold value', async () => { + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--test', '--experimental-test-coverage=10', '--test-reporter', 'spec', fixture]; + const options = { env: { ...process.env } }; + const result = spawnSync(process.execPath, args, options); + const report = getCoverageFixtureReport('spec'); + assert.strictEqual(result.stderr.toString(), ''); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.status, 0); +}); diff --git a/test/parallel/test-runner-run.mjs b/test/parallel/test-runner-run.mjs index c712f500153b42..1d41f0d7706668 100644 --- a/test/parallel/test-runner-run.mjs +++ b/test/parallel/test-runner-run.mjs @@ -500,3 +500,41 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { for await (const _ of stream); }); }); + +describe('coverage report for test stream', () => { + it('should run coverage report', async () => { + const stream = run({ files: [join(testFixtures, 'default-behavior/test/random.cjs')], + coverage: { + lines: 100, + functions: 100, + branches: 100 + } }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall()); + stream.on('test:coverage', common.mustCall()); + + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + }); + it('should run coverage report', async () => { + const stream = run({ files: [join(testFixtures, 'default-behavior/test/random.cjs')], + coverage: false }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall()); + stream.on('test:coverage', common.mustNotCall()); + + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + }); + + it('should run coverage report', async () => { + const stream = run({ files: [join(testFixtures, 'default-behavior/test/random.cjs')], + coverage: true }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall()); + stream.on('test:coverage', common.mustCall()); + + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + }); +});