Add per-file checks, configuration support, and CI fixes.

.travis.yml

@@ -16,6 +16,5 @@ after_script:
   - if [[ `node --version` == *v0.10* ]]; then cat ./build/coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js; fi
 
 before_install:
-  - if [[ `node --version` == *v0.8* ]]; then npm install -g npm; fi
-
-
+  # Node v0.8 needs specific NPM version: https://github.com/npm/npm/issues/6246#issuecomment-57911124
+  - if [[ `node --version` == *v0.8* ]]; then npm install -g npm@1.4.28; fi

README.md

@@ -73,8 +73,8 @@ Usage: istanbul help config | <command>
 Available commands are:
 
       check-coverage
-              checks overall coverage against thresholds from coverage JSON
-              files. Exits 1 if thresholds are not met, 0 otherwise
+              checks overall/per-file coverage against thresholds from coverage
+              JSON files. Exits 1 if thresholds are not met, 0 otherwise
 
 
       cover   transparently adds coverage information to a node command. Saves
@@ -109,8 +109,8 @@ The `cover` command can be used to get a coverage object and reports for any arb
 node script. By default, coverage information is written under `./coverage` - this
 can be changed using command-line options.
 
-The `cover` command can also be passed an optional `--handle-sigint` flag to 
-enable writing reports when a user triggers a manual SIGINT of the process that is 
+The `cover` command can also be passed an optional `--handle-sigint` flag to
+enable writing reports when a user triggers a manual SIGINT of the process that is
 being covered. This can be useful when you are generating coverage for a long lived process.
 
 #### The `test` command
@@ -123,13 +123,13 @@ set the `npm_config_coverage` variable.
 
 #### The `instrument` command
 
-Instruments a single JS file or an entire directory tree and produces an output 
-directory tree with instrumented code. This should not be required for running node 
+Instruments a single JS file or an entire directory tree and produces an output
+directory tree with instrumented code. This should not be required for running node
 unit tests but is useful for tests to be run on the browser.
 
 #### The `report` command
 
-Writes reports using `coverage*.json` files as the source of coverage information. 
+Writes reports using `coverage*.json` files as the source of coverage information.
 Reports are available in multiple formats and can be individually configured
 using the istanbul config file. See `istanbul help report` for more details.
 
@@ -151,20 +151,20 @@ See [ignoring-code-for-coverage.md](ignoring-code-for-coverage.md) for the spec.
 ### API
 
 All the features of istanbul can be accessed as a library.
- 
+
 #### Instrument code
 
 ```javascript
     var instrumenter = new require('istanbul').Instrumenter();
-    
+
     var generatedCode = instrumenter.instrumentSync('function meaningOfLife() { return 42; }',
         'filename.js');
 ```
 
 #### Generate reports given a bunch of coverage JSON objects
 
 ```javascript
-    var istanbul = require('istanbul'), 
+    var istanbul = require('istanbul'),
         collector = new istanbul.Collector(),
         reporter = new istanbul.Reporter(),
         sync = false;
@@ -219,6 +219,6 @@ The following third-party libraries are used by this module:
 
 ### Why the funky name?
 
-Since all the good ones are taken. Comes from the loose association of ideas across 
+Since all the good ones are taken. Comes from the loose association of ideas across
 coverage, carpet-area coverage, the country that makes good carpets and so on...
 

lib/command/check-coverage.js

@@ -11,27 +11,46 @@ var nopt = require('nopt'),
     util = require('util'),
     utils = require('../object-utils'),
     filesFor = require('../util/file-matcher').filesFor,
-    Command = require('./index');
+    Command = require('./index'),
+    configuration = require('../config');
 
 function CheckCoverageCommand() {
     Command.call(this);
 }
 
+function removeFiles(covObj, root, files) {
+    var filesObj = {},
+        obj = {};
+
+    // Create lookup table.
+    files.forEach(function (file) {
+        filesObj[path.join(root, file)] = true;
+    });
+
+    Object.keys(covObj).forEach(function (key) {
+        if (filesObj[key] !== true) {
+            obj[key] = covObj[key];
+        }
+    });
+
+    return obj;
+}
+
 CheckCoverageCommand.TYPE = 'check-coverage';
 util.inherits(CheckCoverageCommand, Command);
 
 Command.mix(CheckCoverageCommand, {
     synopsis: function () {
-        return "checks overall coverage against thresholds from coverage JSON files. Exits 1 if thresholds are not met, 0 otherwise";
+        return "checks overall/per-file coverage against thresholds from coverage JSON files. Exits 1 if thresholds are not met, 0 otherwise";
     },
 
     usage: function () {
         console.error('\nUsage: ' + this.toolName() + ' ' + this.type() + ' <options> [<include-pattern>]\n\nOptions are:\n\n' +
             [
-                formatOption('--statements <threshold>', 'statement coverage threshold'),
-                formatOption('--functions <threshold>', 'function coverage threshold'),
-                formatOption('--branches <threshold>', 'branch coverage threshold'),
-                formatOption('--lines <threshold>', 'line coverage threshold')
+                formatOption('--statements <threshold>', 'global statement coverage threshold'),
+                formatOption('--functions <threshold>', 'global function coverage threshold'),
+                formatOption('--branches <threshold>', 'global branch coverage threshold'),
+                formatOption('--lines <threshold>', 'global line coverage threshold')
             ].join('\n\n') + '\n');
 
         console.error('\n\n');
@@ -40,6 +59,7 @@ Command.mix(CheckCoverageCommand, {
         console.error('When a threshold is specified as a negative number it represents the maximum number of uncovered entities allowed.\n');
         console.error('For example, --statements 90 implies minimum statement coverage is 90%.');
         console.error('             --statements -10 implies that no more than 10 uncovered statements are allowed\n');
+        console.error('Per-file thresholds can be specified via a configuration file.\n');
         console.error('<include-pattern> is a fileset pattern that can be used to select one or more coverage files ' +
             'for merge. This defaults to "**/coverage*.json"');
 
@@ -48,16 +68,28 @@ Command.mix(CheckCoverageCommand, {
 
     run: function (args, callback) {
 
-        var config = {
+        var template = {
+                config: path,
                 root: path,
-                dir: path,
                 statements: Number,
                 lines: Number,
                 branches: Number,
                 functions: Number,
                 verbose: Boolean
             },
-            opts = nopt(config, { v : '--verbose' }, args, 0),
+            opts = nopt(template, { v : '--verbose' }, args, 0),
+            // Translate to config opts.
+            config = configuration.loadFile(opts.config, {
+                verbose: opts.verbose,
+                check: {
+                    global: {
+                        statements: opts.statements,
+                        lines: opts.lines,
+                        branches: opts.branches,
+                        functions: opts.functions
+                    }
+                }
+            }),
             includePattern = '**/coverage*.json',
             root,
             collector = new Collector(),
@@ -74,37 +106,70 @@ Command.mix(CheckCoverageCommand, {
         }, function (err, files) {
             if (err) { throw err; }
             files.forEach(function (file) {
-                var coverageObject =  JSON.parse(fs.readFileSync(file, 'utf8'));
+                var coverageObject = JSON.parse(fs.readFileSync(file, 'utf8'));
                 collector.add(coverageObject);
             });
             var thresholds = {
-                statements: opts.statements || 0,
-                branches: opts.branches || 0,
-                lines: opts.lines || 0,
-                functions: opts.functions || 0
+                global: {
+                    statements: config.check.global.statements || 0,
+                    branches: config.check.global.branches || 0,
+                    lines: config.check.global.lines || 0,
+                    functions: config.check.global.functions || 0,
+                    excludes: config.check.global.excludes || []
+                },
+                each: {
+                    statements: config.check.each.statements || 0,
+                    branches: config.check.each.branches || 0,
+                    lines: config.check.each.lines || 0,
+                    functions: config.check.each.functions || 0,
+                    excludes: config.check.each.excludes || []
+                }
             },
-                actuals = utils.summarizeCoverage(collector.getFinalCoverage());
+                rawCoverage = collector.getFinalCoverage(),
+                globalResults = utils.summarizeCoverage(removeFiles(rawCoverage, root, thresholds.global.excludes)),
+                eachResults = removeFiles(rawCoverage, root, thresholds.each.excludes);
 
-            if (opts.verbose) {
+            // Summarize per-file results and mutate original results.
+            Object.keys(eachResults).forEach(function (key) {
+                eachResults[key] = utils.summarizeFileCoverage(eachResults[key]);
+            });
+
+            if (config.verbose) {
                 console.log('Compare actuals against thresholds');
-                console.log(JSON.stringify({ actuals: actuals, thresholds: thresholds }, undefined, 4));
+                console.log(JSON.stringify({ global: globalResults, each: eachResults, thresholds: thresholds }, undefined, 4));
             }
 
-            Object.keys(thresholds).forEach(function (key) {
-                var actual = actuals[key].pct,
-                    actualUncovered = actuals[key].total - actuals[key].covered,
-                    threshold = thresholds[key];
-
-                if (threshold < 0) {
-                    if (threshold * -1 < actualUncovered) {
-                        errors.push('ERROR: Uncovered count for ' + key + ' (' + actualUncovered + ') exceeds threshold (' + -1 * threshold + ')');
+            function check(name, thresholds, actuals) {
+                [
+                    "statements",
+                    "branches",
+                    "lines",
+                    "functions"
+                ].forEach(function (key) {
+                    var actual = actuals[key].pct,
+                        actualUncovered = actuals[key].total - actuals[key].covered,
+                        threshold = thresholds[key];
+
+                    if (threshold < 0) {
+                        if (threshold * -1 < actualUncovered) {
+                            errors.push('ERROR: Uncovered count for ' + key + ' (' + actualUncovered +
+                                ') exceeds ' + name + ' threshold (' + -1 * threshold + ')');
+                        }
+                    } else {
+                        if (actual < threshold) {
+                            errors.push('ERROR: Coverage for ' + key + ' (' + actual +
+                                '%) does not meet ' + name + ' threshold (' + threshold + '%)');
+                        }
                     }
-                } else {
-                    if (actual < threshold) {
-                        errors.push('ERROR: Coverage for ' + key + ' (' + actual + '%) does not meet threshold (' + threshold + '%)');
-                    }
-                }
+                });
+            }
+
+            check("global", thresholds.global, globalResults);
+
+            Object.keys(eachResults).forEach(function (key) {
+                check("per-file" + " (" + key + ") ", thresholds.each, eachResults[key]);
             });
+
             return callback(errors.length === 0 ? null : errors.join("\n"));
         });
     }

lib/command/help.js

@@ -21,8 +21,8 @@ function showConfigHelp(toolName) {
             'customize its location per command. The alternate config file can be in YAML, JSON or node.js ' +
             '(exporting the config object).'));
     console.error('\n' +
-        formatPara('The config file currently has three sections for instrumentation, reporting and hooks. ' +
-            'Note that certain commands (like `cover`) use information from multiple sections.'));
+        formatPara('The config file currently has four sections for instrumentation, reporting, hooks, ' +
+            'and checking. Note that certain commands (like `cover`) use information from multiple sections.'));
     console.error('\n' +
         formatPara('Keys in the config file usually correspond to command line parameters with the same name. ' +
             'The verbose option for every command shows you the exact configuration used. See the api ' +
@@ -40,6 +40,10 @@ function showConfigHelp(toolName) {
     console.error('\n' +
         formatPara('The `reportConfig` section allows you to configure each report format independently ' +
             'and has no command-line equivalent either.'));
+    console.error('\n' +
+        formatPara('The `check` section configures minimum threshold enforcement for coverage results. ' +
+            '`global` applies to all files together and `each` on a per-file basis. A list of files can ' +
+            'be excluded from enforcement relative to root via the `exclude` property.'));
     console.error('');
 }
 

lib/config.js

@@ -35,6 +35,22 @@ function defaultConfig() {
             'hook-run-in-context': false,
             'post-require-hook': null,
             'handle-sigint': false
+        },
+        check: {
+            global: {
+                statements: 0,
+                lines: 0,
+                branches: 0,
+                functions: 0,
+                excludes: [] // Currently list of files (root + path). For future, extend to patterns.
+            },
+            each: {
+                statements: 0,
+                lines: 0,
+                branches: 0,
+                functions: 0,
+                excludes: []
+            }
         }
     };
     ret.reporting.watermarks = defaults.watermarks();
@@ -350,7 +366,7 @@ function Configuration(obj, overrides) {
     this.instrumentation = new InstrumentOptions(config.instrumentation);
     this.reporting = new ReportingOptions(config.reporting);
     this.hooks = new HookOptions(config.hooks);
-    //this.thresholds = new ThresholdOptions(config.thresholds);
+    this.check = config.check; // Pass raw config sub-object.
 }
 
 /**

test/cli-helper.js

@@ -36,13 +36,13 @@ var path = require('path'),
  *  3. In the child process,
  *      a. hooks the module loader to set up coverage for our library code
  *      b. Ensures that the `hook` module is not loaded until all this happens
-  *         so that the hook module sees _our_ module loader hook as the original
-  *         loader. This ensures that our hook will be used to instrument this
-  *         library's code. Note that the hook set up by the `cover` command that
-  *         is executed only instruments the modules of the sample test library.
-  *     c. Calls Module.runMain on the command that it was asked to invoke
-  *     d. Sets up an exit handler to write the coverage information for our
-  *         library calls
+ *         so that the hook module sees _our_ module loader hook as the original
+ *         loader. This ensures that our hook will be used to instrument this
+ *         library's code. Note that the hook set up by the `cover` command that
+ *         is executed only instruments the modules of the sample test library.
+ *      c. Calls Module.runMain on the command that it was asked to invoke
+ *      d. Sets up an exit handler to write the coverage information for our
+ *         library calls
  *   4. The exit handler is also set up in special ways because in order to
  *      instrument the `cover` command's exit handler, our exit handler has
  *      to be added later so as to be able to track coverage for the cover

test/cli/sample-project/config-check-each.istanbul.yml

@@ -0,0 +1,8 @@
+check:
+  each:
+    statements: 72
+    functions: 50
+    branches: 72
+    lines: 72
+    excludes:
+        - lib/bar.js # 40, 0, 40, 0

test/cli/sample-project/config-check-global.istanbul.yml

@@ -0,0 +1,12 @@
+check:
+  global:
+    statements: 72
+    functions: 50
+    branches: 72
+    lines: 72
+    # Example: Could exclude all files covered (would pass checks and fail the tests).
+    # excludes:
+    #     - lib/foo.js
+    #     - vendor/dummy_vendor_lib.js
+    #     - lib/util/generate-names.js
+    #     - lib/bar.js

test/cli/sample-project/config-check-mixed.istanbul.yml

@@ -0,0 +1,13 @@
+check:
+  global:
+    statements: 80
+    functions: 80
+    branches: 40
+    lines: 40
+  each:
+    statements: 72
+    functions: 50
+    branches: 72
+    lines: 72
+    excludes:
+        - lib/bar.js # 40, 0, 40, 0

test/cli/sample-project/config.istanbul.yml

@@ -6,4 +6,3 @@ reporting:
   report-config:
     cobertura:
       file: 'foo.xml'
-