feat: add thresholds for enforcing coverage percentage (bcoe#59)

README.md

@@ -20,6 +20,30 @@ The above example will output coverage metrics for `foo.js`.
 
 run `c8 report` to regenerate reports after `c8` has already been run.
 
+## Checking coverage
+
+c8 can fail tests if coverage falls below a threshold.
+After running your tests with c8, simply run:
+
+```shell
+c8 check-coverage --lines 95 --functions 95 --branches 95
+```
+
+c8 also accepts a `--check-coverage` shorthand, which can be used to
+both run tests and check that coverage falls within the threshold provided:
+
+```shell
+c8 --check-coverage --lines 100 npm test
+```
+
+The above check fails if coverage falls below 100%.
+
+To check thresholds on a per-file basis run:
+
+```shell
+c8 check-coverage --lines 95 --per-file
+```
+
 ## Supported Node.js Versions
 
 c8 uses

bin/c8.js

@@ -2,48 +2,35 @@
 'use strict'
 
 const fs = require('fs')
-const util = require('util')
-
 const foreground = require('foreground-child')
-const report = require('../lib/report')
+const { outputReport } = require('../lib/commands/report')
+const { checkCoverages } = require('../lib/commands/check-coverage')
+const { promisify } = require('util')
 const rimraf = require('rimraf')
 const {
+  buildYargs,
   hideInstrumenteeArgs,
-  hideInstrumenterArgs,
-  yargs
+  hideInstrumenterArgs
 } = require('../lib/parse-args')
 
 const instrumenterArgs = hideInstrumenteeArgs()
-let argv = yargs.parse(instrumenterArgs)
-
-const _p = util.promisify
-
-function outputReport () {
-  report({
-    include: argv.include,
-    exclude: argv.exclude,
-    reporter: Array.isArray(argv.reporter) ? argv.reporter : [argv.reporter],
-    tempDirectory: argv.tempDirectory,
-    watermarks: argv.watermarks,
-    resolve: argv.resolve,
-    omitRelative: argv.omitRelative,
-    wrapperLength: argv.wrapperLength
-  })
-}
+let argv = buildYargs().parse(instrumenterArgs)
 
-(async function run () {
-  if (argv._[0] === 'report') {
-    argv = yargs.parse(process.argv) // support flag arguments after "report".
-    outputReport()
+;(async function run () {
+  if ([
+    'check-coverage', 'report'
+  ].indexOf(argv._[0]) !== -1) {
+    argv = buildYargs(true).parse(process.argv.slice(2))
   } else {
     if (argv.clean) {
-      await _p(rimraf)(argv.tempDirectory)
-      await _p(fs.mkdir)(argv.tempDirectory, { recursive: true })
+      await promisify(rimraf)(argv.tempDirectory)
+      await promisify(fs.mkdir)(argv.tempDirectory, { recursive: true })
     }
-    process.env.NODE_V8_COVERAGE = argv.tempDirectory
 
+    process.env.NODE_V8_COVERAGE = argv.tempDirectory
     foreground(hideInstrumenterArgs(argv), () => {
-      outputReport()
+      const report = outputReport(argv)
+      if (argv.checkCoverage) checkCoverages(argv, report)
     })
   }
 })()

lib/commands/check-coverage.js

@@ -0,0 +1,59 @@
+const { relative } = require('path')
+const Report = require('../report')
+
+exports.command = 'check-coverage'
+
+exports.describe = 'check whether coverage is within thresholds provided'
+
+exports.builder = function (yargs) {
+  yargs
+    .example('$0 check-coverage --lines 95', "check whether the JSON in c8's output folder meets the thresholds provided")
+}
+
+exports.handler = function (argv) {
+  const report = Report({
+    include: argv.include,
+    exclude: argv.exclude,
+    reporter: Array.isArray(argv.reporter) ? argv.reporter : [argv.reporter],
+    tempDirectory: argv.tempDirectory,
+    watermarks: argv.watermarks,
+    resolve: argv.resolve,
+    omitRelative: argv.omitRelative,
+    wrapperLength: argv.wrapperLength
+  })
+  exports.checkCoverages(argv, report)
+}
+
+exports.checkCoverages = function (argv, report) {
+  const thresholds = {
+    lines: argv.lines,
+    functions: argv.functions,
+    branches: argv.branches,
+    statements: argv.statements
+  }
+  const map = report.getCoverageMapFromAllCoverageFiles()
+  if (argv.perFile) {
+    map.files().forEach(file => {
+      checkCoverage(map.fileCoverageFor(file).toSummary(), thresholds, file)
+    })
+  } else {
+    checkCoverage(map.getCoverageSummary(), thresholds)
+  }
+}
+
+function checkCoverage (summary, thresholds, file) {
+  Object.keys(thresholds).forEach(key => {
+    const coverage = summary[key].pct
+    if (coverage < thresholds[key]) {
+      process.exitCode = 1
+      if (file) {
+        console.error(
+          'ERROR: Coverage for ' + key + ' (' + coverage + '%) does not meet threshold (' + thresholds[key] + '%) for ' +
+            relative('./', file).replace(/\\/g, '/') // standardize path for Windows.
+        )
+      } else {
+        console.error('ERROR: Coverage for ' + key + ' (' + coverage + '%) does not meet global threshold (' + thresholds[key] + '%)')
+      }
+    }
+  })
+}

lib/commands/report.js

@@ -0,0 +1,24 @@
+const Report = require('../report')
+
+exports.command = 'report'
+
+exports.describe = 'read V8 coverage data from temp and output report'
+
+exports.handler = function (argv) {
+  exports.outputReport(argv)
+}
+
+exports.outputReport = function (argv) {
+  const report = Report({
+    include: argv.include,
+    exclude: argv.exclude,
+    reporter: Array.isArray(argv.reporter) ? argv.reporter : [argv.reporter],
+    tempDirectory: argv.tempDirectory,
+    watermarks: argv.watermarks,
+    resolve: argv.resolve,
+    omitRelative: argv.omitRelative,
+    wrapperLength: argv.wrapperLength
+  })
+  report.run()
+  return report
+}

lib/parse-args.js

@@ -1,56 +1,94 @@
 const Exclude = require('test-exclude')
 const findUp = require('find-up')
 const { readFileSync } = require('fs')
-const yargs = require('yargs')
+const Yargs = require('yargs/yargs')
 const parser = require('yargs-parser')
 
 const configPath = findUp.sync(['.c8rc', '.c8rc.json'])
 const config = configPath ? JSON.parse(readFileSync(configPath)) : {}
 
-yargs()
-  .usage('$0 [opts] [script] [opts]')
-  .option('reporter', {
-    alias: 'r',
-    describe: 'coverage reporter(s) to use',
-    default: 'text'
-  })
-  .option('exclude', {
-    alias: 'x',
-    default: Exclude.defaultExclude,
-    describe: 'a list of specific files and directories that should be excluded from coverage (glob patterns are supported)'
-  })
-  .option('include', {
-    alias: 'n',
-    default: [],
-    describe: 'a list of specific files that should be covered (glob patterns are supported)'
-  })
-  .option('temp-directory', {
-    default: './coverage/tmp',
-    describe: 'directory V8 coverage data is written to and read from'
-  })
-  .option('resolve', {
-    default: '',
-    describe: 'resolve paths to alternate base directory'
-  })
-  .option('wrapper-length', {
-    describe: 'how many bytes is the wrapper prefix on executed JavaScript',
-    type: 'number'
-  })
-  .option('omit-relative', {
-    default: true,
-    type: 'boolean',
-    describe: 'omit any paths that are not absolute, e.g., internal/net.js'
-  })
-  .option('clean', {
-    default: true,
-    type: 'boolean',
-    describe: 'should temp files be deleted before script execution'
-  })
-  .command('report', 'read V8 coverage data from temp and output report')
-  .pkgConf('c8')
-  .config(config)
-  .demandCommand(1)
-  .epilog('visit https://git.io/vHysA for list of available reporters')
+function buildYargs (withCommands = false) {
+  const yargs = Yargs([])
+    .usage('$0 [opts] [script] [opts]')
+    .option('reporter', {
+      alias: 'r',
+      describe: 'coverage reporter(s) to use',
+      default: 'text'
+    })
+    .option('exclude', {
+      alias: 'x',
+      default: Exclude.defaultExclude,
+      describe: 'a list of specific files and directories that should be excluded from coverage (glob patterns are supported)'
+    })
+    .option('include', {
+      alias: 'n',
+      default: [],
+      describe: 'a list of specific files that should be covered (glob patterns are supported)'
+    })
+    .option('check-coverage', {
+      default: false,
+      type: 'boolean',
+      description: 'check whether coverage is within thresholds provided'
+    })
+    .option('branches', {
+      default: 0,
+      description: 'what % of branches must be covered?'
+    })
+    .option('functions', {
+      default: 0,
+      description: 'what % of functions must be covered?'
+    })
+    .option('lines', {
+      default: 90,
+      description: 'what % of lines must be covered?'
+    })
+    .option('statements', {
+      default: 0,
+      description: 'what % of statements must be covered?'
+    })
+    .option('per-file', {
+      default: false,
+      description: 'check thresholds per file'
+    })
+    .option('temp-directory', {
+      default: './coverage/tmp',
+      describe: 'directory V8 coverage data is written to and read from'
+    })
+    .option('resolve', {
+      default: '',
+      describe: 'resolve paths to alternate base directory'
+    })
+    .option('wrapper-length', {
+      describe: 'how many bytes is the wrapper prefix on executed JavaScript',
+      type: 'number'
+    })
+    .option('omit-relative', {
+      default: true,
+      type: 'boolean',
+      describe: 'omit any paths that are not absolute, e.g., internal/net.js'
+    })
+    .option('clean', {
+      default: true,
+      type: 'boolean',
+      describe: 'should temp files be deleted before script execution'
+    })
+    .pkgConf('c8')
+    .config(config)
+    .demandCommand(1)
+    .epilog('visit https://git.io/vHysA for list of available reporters')
+
+  const checkCoverage = require('./commands/check-coverage')
+  const report = require('./commands/report')
+  if (withCommands) {
+    yargs.command(checkCoverage)
+    yargs.command(report)
+  } else {
+    yargs.command(checkCoverage.command, checkCoverage.describe)
+    yargs.command(report.command, report.describe)
+  }
+
+  return yargs
+}
 
 function hideInstrumenterArgs (yargv) {
   var argv = process.argv.slice(1)
@@ -76,7 +114,7 @@ function hideInstrumenteeArgs () {
 }
 
 module.exports = {
-  yargs,
+  buildYargs,
   hideInstrumenterArgs,
   hideInstrumenteeArgs
 }

lib/report.js

@@ -32,7 +32,7 @@ class Report {
     this.wrapperLength = wrapperLength
   }
   run () {
-    const map = this._getCoverageMapFromAllCoverageFiles()
+    const map = this.getCoverageMapFromAllCoverageFiles()
     var context = libReport.createContext({
       dir: './coverage',
       watermarks: this.watermarks
@@ -45,7 +45,13 @@ class Report {
     })
   }
 
-  _getCoverageMapFromAllCoverageFiles () {
+  getCoverageMapFromAllCoverageFiles () {
+    // the merge process can be very expensive, and it's often the case that
+    // check-coverage is called immediately after a report. We memoize the
+    // result from getCoverageMapFromAllCoverageFiles() to address this
+    // use-case.
+    if (this._allCoverageFiles) return this._allCoverageFiles
+
     const v8ProcessCov = this._getMergedProcessCov()
 
     const map = libCoverage.createCoverageMap({})
@@ -61,7 +67,8 @@ class Report {
       }
     }
 
-    return map
+    this._allCoverageFiles = map
+    return this._allCoverageFiles
   }
 
   /**
@@ -138,6 +145,5 @@ class Report {
 }
 
 module.exports = function (opts) {
-  const report = new Report(opts)
-  report.run()
+  return new Report(opts)
 }