diff --git a/README.md b/README.md index 67bb69ba..5eb41396 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ module.exports = { | client | *Object* | `require("ganache-core")` | Useful if you need a specific ganache version. | | providerOptions | *Object* | `{ }` | [ganache-core options][1] | | skipFiles | *Array* | `['Migrations.sol']` | Array of contracts or folders (with paths expressed relative to the `contracts` directory) that should be skipped when doing instrumentation. | +| istanbulFolder | *String* | `./coverage` | Folder location for Istanbul coverage reports. | | istanbulReporter | *Array* | `['html', 'lcov', 'text']` | [Istanbul coverage reporters][2] | | mocha | *Object* | `{ }` | [Mocha options][3] to merge into existing mocha config. `grep` and `invert` are useful for skipping certain tests under coverage using tags in the test descriptions.| | onServerReady[*][14] | *Function* | | Hook run *after* server is launched, *before* the tests execute. Useful if you need to use the Oraclize bridge or have setup scripts which rely on the server's availability. [More...][23] | diff --git a/dist/plugin-assets/buidler.ui.js b/dist/plugin-assets/buidler.ui.js index 35f35ff5..7624ab53 100644 --- a/dist/plugin-assets/buidler.ui.js +++ b/dist/plugin-assets/buidler.ui.js @@ -1,7 +1,7 @@ const UI = require('./../../lib/ui').UI; /** - * Truffle Plugin logging + * Buidler Plugin logging */ class PluginUI extends UI { constructor(log){ diff --git a/dist/plugin-assets/buidler.utils.js b/dist/plugin-assets/buidler.utils.js index 9ef5dffa..05de70d3 100644 --- a/dist/plugin-assets/buidler.utils.js +++ b/dist/plugin-assets/buidler.utils.js @@ -7,7 +7,7 @@ const { createProvider } = require("@nomiclabs/buidler/internal/core/providers/c // ============================= -// Buidler Specific Plugin Utils +// Buidler Plugin Utils // ============================= /** diff --git a/dist/plugin-assets/plugin.utils.js b/dist/plugin-assets/plugin.utils.js index 1dc50a93..002a51f2 100644 --- a/dist/plugin-assets/plugin.utils.js +++ b/dist/plugin-assets/plugin.utils.js @@ -1,9 +1,6 @@ /** * A collection of utilities for common tasks plugins will need in the course * of composing a workflow using the solidity-coverage API - * - * TODO: Sweep back through here and make all `config.truffle_variable` plugin - * platform neutral... */ const PluginUI = require('./truffle.ui'); diff --git a/lib/api.js b/lib/api.js index 4ed9d4c3..14ed170c 100644 --- a/lib/api.js +++ b/lib/api.js @@ -6,6 +6,7 @@ const istanbul = require('istanbul'); const util = require('util'); const assert = require('assert'); const detect = require('detect-port'); +const _ = require('lodash/lang'); const ConfigValidator = require('./validator'); const Instrumenter = require('./instrumenter'); @@ -17,7 +18,7 @@ const AppUI = require('./ui').AppUI; * Coverage Runner */ class API { - constructor(config) { + constructor(config={}) { this.coverage = new Coverage(); this.instrumenter = new Instrumenter(); this.validator = new ConfigValidator() @@ -54,7 +55,8 @@ class API { this.gasLimitString = "0xfffffffffff"; // block gas limit for ganache (higher than "gas sent") this.gasPrice = 0x01; - this.istanbulReporter = config.istanbulReporter || ['html', 'lcov', 'text']; + this.istanbulFolder = config.istanbulFolder || false; + this.istanbulReporter = config.istanbulReporter || ['html', 'lcov', 'text', 'json']; this.setLoggingLevel(config.silent); this.ui = new AppUI(this.log); @@ -65,21 +67,12 @@ class API { * Instruments a set of sources to prepare them for running under coverage * @param {Object[]} targets (see below) * @return {Object[]} (see below) - * @example: - * - * targets: - * [{ - * canonicalPath: - * relativePath: - * source: - * - * },...] - * - * outputs: - * [{ - * canonicalPath: - * source: - * }...] + * @example of input/output array: + * [{ + * source: (required) , + * canonicalPath: (required) + * relativePath: (optional) + * }] */ instrument(targets=[]) { let currentFile; // Keep track of filename in case we crash... @@ -95,7 +88,7 @@ class API { this.ui.report('instr-start'); } - this.ui.report('instr-item', [target.relativePath]); + this.ui.report('instr-item', [currentFile]); const instrumented = this.instrumenter.instrument( target.source, @@ -119,22 +112,35 @@ class API { return outputs; } + /** + * Returns a copy of the hit map created during instrumentation. + * Useful if you'd like to delegate coverage collection to multiple processes. + * @return {Object} instrumentationData + */ + getInstrumentationData(){ + return _.cloneDeep(this.instrumenter.instrumentationData) + } + + /** + * Sets the hit map object generated during instrumentation. Useful if you'd like + * to collect data for a pre-existing instrumentation. + * @param {Object} data + */ + setInstrumentationData(data={}){ + this.instrumenter.instrumentationData = _.cloneDeep(data); + } + /** * Launches an in-process ethereum client server, hooking the DataCollector to its VM. * @param {Object} client ganache client * @return {String} address of server to connect to */ async ganache(client){ - let retry = false; - let address = `http://${this.host}:${this.port}`; - // Check for port-in-use if (await detect(this.port) !== this.port){ throw new Error(this.ui.generate('server-fail', [this.port])) } - if(!this.client) this.client = client; // Prefer client from options - this.collector = new DataCollector(this.instrumenter.instrumentationData); this.providerOptions.gasLimit = this.gasLimitString; @@ -143,16 +149,17 @@ class API { // Launch server and attach to vm step of supplied client try { if (this.config.forceBackupServer) throw new Error() - await this.attachToVM() + await this.attachToVM(client) } - // Fallback to ganache-core-sc (eq: ganache-core 2.7.0) + // Fallback to ganache-cli) catch(err) { - this.ui.report('vm-fail', []); - this.client = require('ganache-core-sc'); - await this.attachToVM(); + const _ganache = require('ganache-cli'); + this.ui.report('vm-fail', [_ganache.version]); + await this.attachToVM(_ganache); } + const address = `http://${this.host}:${this.port}`; this.ui.report('server', [address]); return address; } @@ -162,7 +169,7 @@ class API { */ async report() { const collector = new istanbul.Collector(); - const reporter = new istanbul.Reporter(); + const reporter = new istanbul.Reporter(false, this.istanbulFolder); return new Promise((resolve, reject) => { try { @@ -177,7 +184,8 @@ class API { // Pify doesn't like this one... reporter.write(collector, true, (err) => { - if (err) throw err; + if (err) return reject(err); + this.ui.report('istanbul'); resolve(); }); @@ -204,9 +212,11 @@ class API { // ======== // Provider // ======== - async attachToVM(){ + async attachToVM(client){ const self = this; + // Prefer client from options + if(!this.client) this.client = client; this.server = this.client.server(this.providerOptions); this.assertHasBlockchain(this.server.provider); @@ -225,7 +235,6 @@ class API { return vm; } - // NB: EADDRINUSE errors are uncatch-able? await pify(this.server.listen)(this.port); } diff --git a/lib/ui.js b/lib/ui.js index 97b534a0..cbad7772 100644 --- a/lib/ui.js +++ b/lib/ui.js @@ -56,9 +56,9 @@ class AppUI extends UI { const w = ":warning:"; const kinds = { - 'vm-fail': `${w} ${c.red('There was a problem attaching to the ganache-core VM.')} `+ - `${c.red('Check the provider option syntax in solidity-coverage docs.')}\n`+ - `${w} ${c.red('Using ganache-core-sc (eq. core v2.7.0) instead.')}\n`, + 'vm-fail': `${w} ${c.red('There was a problem attaching to the ganache VM.')}\n` + + `${w} ${c.red('For help, see the "client" & "providerOptions" syntax in solidity-coverage docs.')}\n`+ + `${w} ${c.red(`Using ganache-cli (v${args[0]}) instead.`)}\n`, 'instr-start': `\n${c.bold('Instrumenting for coverage...')}` + diff --git a/package.json b/package.json index 9f411918..3d634697 100644 --- a/package.json +++ b/package.json @@ -12,9 +12,9 @@ }, "scripts": { "nyc": "SILENT=true nyc --exclude '**/sc_temp/**' --exclude '**/test/**'", - "test": "npm run nyc -- mocha test/units/* --timeout 100000 --no-warnings --exit", - "test:ci": "SILENT=true node --max-old-space-size=3072 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit", - "test:debug": "mocha test/units/* --timeout 100000 --no-warnings --exit" + "test": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc -- mocha test/units/* --timeout 100000 --no-warnings --exit", + "test:ci": "SILENT=true node --max-old-space-size=4096 ./node_modules/.bin/nyc --reporter=lcov --exclude '**/sc_temp/**' --exclude '**/test/**/' -- mocha test/units/* --timeout 100000 --no-warnings --exit", + "test:debug": "node --max-old-space-size=4096 ./node_modules/.bin/mocha test/units/* --timeout 100000 --no-warnings --exit" }, "homepage": "https://github.com/sc-forks/solidity-coverage", "repository": { @@ -35,6 +35,7 @@ "globby": "^10.0.1", "istanbul": "^0.4.5", "jsonschema": "^1.2.4", + "lodash": "^4.17.15", "node-dir": "^0.1.17", "node-emoji": "^1.10.0", "pify": "^4.0.1", diff --git a/test/integration/projects/ganache-solcoverjs/.gitignore b/test/integration/projects/ganache-solcoverjs/.gitignore new file mode 100644 index 00000000..736e8ae5 --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/.gitignore @@ -0,0 +1 @@ +!node_modules \ No newline at end of file diff --git a/test/integration/projects/ganache-solcoverjs/.solcover.js b/test/integration/projects/ganache-solcoverjs/.solcover.js new file mode 100644 index 00000000..6e2f5326 --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/.solcover.js @@ -0,0 +1,5 @@ +module.exports = { + client: require('ganache-cli'), + silent: process.env.SILENT ? true : false, + istanbulReporter: ['json-summary', 'text'], +} diff --git a/test/integration/projects/ganache-solcoverjs/buidler.config.js b/test/integration/projects/ganache-solcoverjs/buidler.config.js new file mode 100644 index 00000000..084a9f23 --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/buidler.config.js @@ -0,0 +1,8 @@ +const { loadPluginFile } = require("@nomiclabs/buidler/plugins-testing"); +loadPluginFile(__dirname + "/../dist/buidler.plugin"); +usePlugin("@nomiclabs/buidler-truffle5"); + +module.exports={ + defaultNetwork: "buidlerevm", + logger: process.env.SILENT ? { log: () => {} } : console, +}; \ No newline at end of file diff --git a/test/integration/projects/ganache-solcoverjs/contracts/ContractA.sol b/test/integration/projects/ganache-solcoverjs/contracts/ContractA.sol new file mode 100644 index 00000000..9d8d1344 --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/contracts/ContractA.sol @@ -0,0 +1,17 @@ +pragma solidity ^0.5.0; + + +contract ContractA { + uint x; + constructor() public { + } + + function sendFn() public { + x = 5; + } + + function callFn() public pure returns (uint){ + uint y = 5; + return y; + } +} diff --git a/test/integration/projects/ganache-solcoverjs/contracts/ContractB.sol b/test/integration/projects/ganache-solcoverjs/contracts/ContractB.sol new file mode 100644 index 00000000..daa42f7d --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/contracts/ContractB.sol @@ -0,0 +1,17 @@ +pragma solidity ^0.5.0; + + +contract ContractB { + uint x; + constructor() public { + } + + function sendFn() public { + x = 5; + } + + function callFn() public pure returns (uint){ + uint y = 5; + return y; + } +} diff --git a/test/integration/projects/ganache-solcoverjs/contracts/ContractC.sol b/test/integration/projects/ganache-solcoverjs/contracts/ContractC.sol new file mode 100644 index 00000000..454c86cd --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/contracts/ContractC.sol @@ -0,0 +1,17 @@ +pragma solidity ^0.5.0; + + +contract ContractC { + uint x; + constructor() public { + } + + function sendFn() public { + x = 5; + } + + function callFn() public pure returns (uint){ + uint y = 5; + return y; + } +} diff --git a/test/integration/projects/ganache-solcoverjs/contracts/Migrations.sol b/test/integration/projects/ganache-solcoverjs/contracts/Migrations.sol new file mode 100644 index 00000000..c378ffb0 --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/contracts/Migrations.sol @@ -0,0 +1,23 @@ +pragma solidity >=0.4.21 <0.6.0; + +contract Migrations { + address public owner; + uint public last_completed_migration; + + constructor() public { + owner = msg.sender; + } + + modifier restricted() { + if (msg.sender == owner) _; + } + + function setCompleted(uint completed) public restricted { + last_completed_migration = completed; + } + + function upgrade(address new_address) public restricted { + Migrations upgraded = Migrations(new_address); + upgraded.setCompleted(last_completed_migration); + } +} diff --git a/test/integration/projects/ganache-solcoverjs/test/contracta.js b/test/integration/projects/ganache-solcoverjs/test/contracta.js new file mode 100644 index 00000000..cc778027 --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/test/contracta.js @@ -0,0 +1,15 @@ +const ContractA = artifacts.require("ContractA"); + +contract("contracta", function(accounts) { + let instance; + + before(async () => instance = await ContractA.new()) + + it('sends [ @skipForCoverage ]', async function(){ + await instance.sendFn(); + }); + + it('calls [ @skipForCoverage ]', async function(){ + await instance.callFn(); + }) +}); diff --git a/test/integration/projects/ganache-solcoverjs/test/contractb.js b/test/integration/projects/ganache-solcoverjs/test/contractb.js new file mode 100644 index 00000000..71ebfb7a --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/test/contractb.js @@ -0,0 +1,15 @@ +const ContractB = artifacts.require("ContractB"); + +contract("contractB [ @skipForCoverage ]", function(accounts) { + let instance; + + before(async () => instance = await ContractB.new()) + + it('sends', async function(){ + await instance.sendFn(); + }); + + it('calls', async function(){ + await instance.callFn(); + }) +}); diff --git a/test/integration/projects/ganache-solcoverjs/test/contractc.js b/test/integration/projects/ganache-solcoverjs/test/contractc.js new file mode 100644 index 00000000..9b3d950d --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/test/contractc.js @@ -0,0 +1,20 @@ +const ContractC = artifacts.require("ContractC"); + +contract("contractc", function(accounts) { + let instance; + + before(async () => instance = await ContractC.new()) + + it('sends', async function(){ + await instance.sendFn(); + }); + + it('calls', async function(){ + await instance.callFn(); + }) + + it('sends', async function(){ + await instance.sendFn(); + }); + +}); diff --git a/test/integration/projects/ganache-solcoverjs/truffle-config.js b/test/integration/projects/ganache-solcoverjs/truffle-config.js new file mode 100644 index 00000000..b398b071 --- /dev/null +++ b/test/integration/projects/ganache-solcoverjs/truffle-config.js @@ -0,0 +1,7 @@ +module.exports = { + networks: {}, + mocha: {}, + compilers: { + solc: {} + } +} diff --git a/test/units/api.js b/test/units/api.js new file mode 100644 index 00000000..c6fc90c0 --- /dev/null +++ b/test/units/api.js @@ -0,0 +1,54 @@ +const assert = require('assert'); +const util = require('./../util/util.js'); +const API = require('./../../lib/api.js'); + +describe('api', () => { + const opts = {silent: true}; + + it('getInstrumentationData', function(){ + const api = new API(opts); + const canonicalPath = 'statements/single.sol' + const source = util.getCode(canonicalPath); + + api.instrument([{ + source: source, + canonicalPath: canonicalPath + }]); + + const data = api.getInstrumentationData(); + + const hash = Object.keys(data)[0]; + assert(data[hash].hits === 0); + }); + + it('setInstrumentationData', function(){ + let api = new API(opts); + + const canonicalPath = 'statements/single.sol' + const source = util.getCode(canonicalPath); + + api.instrument([{ + source: source, + canonicalPath: canonicalPath + }]); + + const cloneA = api.getInstrumentationData(); + const hash = Object.keys(cloneA)[0]; + + // Verify cloning + cloneA[hash].hits = 5; + const cloneB = api.getInstrumentationData(); + assert(cloneB[hash].hits === 0); + + // Verify setting + api = new API(opts); + api.instrument([{ + source: source, + canonicalPath: canonicalPath + }]); + + api.setInstrumentationData(cloneA); + const cloneC = api.getInstrumentationData(); + assert(cloneC[hash].hits === 5); + }); +}) diff --git a/test/units/buidler/flags.js b/test/units/buidler/flags.js index 435e2970..8e451583 100644 --- a/test/units/buidler/flags.js +++ b/test/units/buidler/flags.js @@ -10,9 +10,6 @@ const plugin = require('../../../dist/buidler.plugin'); // ======================= // CLI Options / Flags // ======================= -async function delay(){ - return new Promise(res => setTimeout(() => res(), 1000)) -} describe('Buidler Plugin: command line options', function() { let buidlerConfig; diff --git a/test/units/truffle/flags.js b/test/units/truffle/flags.js index 5cef07c4..f3c63cc0 100644 --- a/test/units/truffle/flags.js +++ b/test/units/truffle/flags.js @@ -22,6 +22,7 @@ describe('Truffle Plugin: command line options', function() { solcoverConfig = {}; truffleConfig = mock.getDefaultTruffleConfig(); verify.cleanInitialState(); + }) afterEach(() => mock.clean()); diff --git a/test/units/truffle/standard.js b/test/units/truffle/standard.js index 1597a385..5d0824bf 100644 --- a/test/units/truffle/standard.js +++ b/test/units/truffle/standard.js @@ -47,7 +47,7 @@ describe('Truffle Plugin: standard use cases', function() { }); // Instrumentation speed is fine - but this takes solc almost a minute to compile - // so annoying. Unskip whenever modifying the instrumentation files though..... + // Unskip whenever modifying the instrumentation files though..... it.skip('with many unbracketed statements (time check)', async function() { truffleConfig.compilers.solc.version = "0.4.24"; @@ -156,7 +156,10 @@ describe('Truffle Plugin: standard use cases', function() { }); // Truffle test asserts deployment cost is greater than 20,000,000 gas - it.skip('deployment cost > block gasLimit', async function() { + // Test times out on CircleCI @ 100000 ms. Fine locally though. + it('deployment cost > block gasLimit', async function() { + if (process.env.CI) return; + mock.install('Expensive', 'block-gas-limit.js', solcoverConfig); await plugin(truffleConfig); }); @@ -181,6 +184,24 @@ describe('Truffle Plugin: standard use cases', function() { assert(output[path].fnMap['2'].name === 'getX', 'cov missing "getX"'); }); + // This test tightly coupled to the ganache version in truffle dev dep + it('uses the server from truffle by default', async function(){ + truffleConfig.logger = mock.testLogger; + truffleConfig.version = true; + + // Baseline inequality check + const truffleClientVersion = "v2.5.7"; + + // Truffle client + mock.install('Simple', 'simple.js', solcoverConfig); + await plugin(truffleConfig); + + assert( + mock.loggerOutput.val.includes(truffleClientVersion), + `Should use truffles ganache: ${mock.loggerOutput.val}` + ); + }); + it('uses the fallback server', async function(){ truffleConfig.logger = mock.testLogger; solcoverConfig.forceBackupServer = true; @@ -189,11 +210,39 @@ describe('Truffle Plugin: standard use cases', function() { await plugin(truffleConfig); assert( - mock.loggerOutput.val.includes("Using ganache-core-sc"), + mock.loggerOutput.val.includes("Using ganache-cli"), `Should notify about backup server module: ${mock.loggerOutput.val}` ); }); + // This test tightly coupled to the ganache version in production deps + // "test-files" project solcoverjs includes `client: require('ganache-cli')` + it('config: client', async function(){ + truffleConfig.logger = mock.testLogger; + truffleConfig.version = true; + + const configClientVersion = "v2.8.0"; + + // Config client + mock.installFullProject('ganache-solcoverjs'); + await plugin(truffleConfig); + + assert( + mock.loggerOutput.val.includes(configClientVersion), + `Should use solcover provided ganache: ${mock.loggerOutput.val}` + ); + }); + + it('config: istanbulFolder', async function(){ + solcoverConfig.istanbulFolder = mock.pathToTemp('specialFolder'); + + // Truffle client + mock.install('Simple', 'simple.js', solcoverConfig); + await plugin(truffleConfig); + + assert(verify.pathExists(solcoverConfig.istanbulFolder)); + }); + // This project has [ @skipForCoverage ] tags in the test descriptions // at selected 'contract' and 'it' blocks. it('config: mocha options', async function() { diff --git a/test/util/integration.js b/test/util/integration.js index 8f37e5e6..dbc6d420 100644 --- a/test/util/integration.js +++ b/test/util/integration.js @@ -55,6 +55,10 @@ function pathToContract(config, file) { return path.join('contracts', file); } +function pathToTemp(_path) { + return path.join(temp, _path); +} + function getOutput(config){ const workingDir = config.working_directory || config.paths.root; const jsonPath = path.join(workingDir, "coverage.json"); @@ -303,6 +307,7 @@ const testLogger = { module.exports = { + pathToTemp: pathToTemp, testLogger: testLogger, loggerOutput: loggerOutput, getDefaultTruffleConfig: getDefaultTruffleConfig, diff --git a/test/util/util.js b/test/util/util.js index 85874055..ff101db3 100644 --- a/test/util/util.js +++ b/test/util/util.js @@ -118,11 +118,11 @@ function initializeProvider(ganache){ } module.exports = { + getCode: getCode, pathPrefix: pathPrefix, filePath: filePath, report: report, instrumentAndCompile: instrumentAndCompile, bootstrapCoverage: bootstrapCoverage, initializeProvider: initializeProvider, - } diff --git a/test/util/verifiers.js b/test/util/verifiers.js index 38bbb5cb..b5a25d77 100644 --- a/test/util/verifiers.js +++ b/test/util/verifiers.js @@ -45,6 +45,7 @@ function coverageNotGenerated(config){ } module.exports = { + pathExists: pathExists, lineCoverage: lineCoverage, coverageMissing: coverageMissing, cleanInitialState: cleanInitialState,