From 7a9b618e4d151c7b2340a392c989182b7010113f Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 17 Nov 2017 09:57:47 -0800 Subject: [PATCH] unit test for save file and dir walking --- bin/run.js | 66 +++--- package.json | 1 + tap-snapshots/test-run.js-TAP.test.js | 305 ++++++++++++++++++++++++++ test/clean-stacks.js | 4 + test/run.js | 211 ++++++++++++++++-- 5 files changed, 542 insertions(+), 45 deletions(-) diff --git a/bin/run.js b/bin/run.js index fa35509c5..a1bacb1c7 100755 --- a/bin/run.js +++ b/bin/run.js @@ -25,10 +25,13 @@ const coverageServiceTest = process.env.COVERAGE_SERVICE_TEST === 'true' /* istanbul ignore next */ if (process.env._TAP_COVERAGE_ === '1') global.__coverage__ = global.__coverage__ || {} +else if (process.env._TAP_COVERAGE_ === '0') { + global.__coverage__ = null + Object.keys(process.env).filter(k => /NYC/.test(k)).forEach(k => + process.env[k] = '') +} -// console.error(process.argv.join(' ')) -// console.error('CST=%j', process.env.COVERAGE_SERVICE_TEST) -// console.error('CRT=%j', process.env.COVERALLS_REPO_TOKEN) +/* istanbul ignore next */ if (coverageServiceTest) console.log('COVERAGE_SERVICE_TEST') @@ -46,7 +49,6 @@ const coverageServices = [ } ] - const main = _ => { const args = process.argv.slice(2) @@ -75,6 +77,7 @@ const main = _ => { return process.stdout.on('error', er => { + /* istanbul ignore else */ if (er.code === 'EPIPE') process.exit() else @@ -83,6 +86,8 @@ const main = _ => { options.files = globFiles(options.files) + // this is only testable by escaping from the covered environment + /* istanbul ignore next */ if ((options.coverageReport || options.checkCoverage) && options.files.length === 0) return runCoverageReport(options) @@ -102,10 +107,10 @@ const main = _ => { // By definition, the next block cannot be covered, because // they are only relevant when coverage is turned off. - - /* istanbul ignore if */ - if (options.coverage && !global.__coverage__) + /* istanbul ignore next */ + if (options.coverage && !global.__coverage__) { return respawnWithCoverage(options) + } setupTapEnv(options) runTests(options) @@ -420,7 +425,6 @@ const parseArgs = (args, options) => { // it only runs when we DON'T have coverage enabled, to enable it. /* istanbul ignore next */ const respawnWithCoverage = options => { - // console.error('respawn with coverage') // Re-spawn with coverage const args = [nycBin].concat( '--silent', @@ -437,11 +441,10 @@ const respawnWithCoverage = options => { runCoverageReport(options, code, signal)) } +/* istanbul ignore next */ const pipeToCoverageServices = (options, child) => { - // console.error('pipe to services') let piped = false for (let i = 0; i < coverageServices.length; i++) { - // console.error('pipe to service?', coverageServices[i].env) if (process.env[coverageServices[i].env]) { pipeToCoverageService(coverageServices[i], options, child) piped = true @@ -452,12 +455,11 @@ const pipeToCoverageServices = (options, child) => { throw new Error('unknown service, internal error') } +/* istanbul ignore next */ const pipeToCoverageService = (service, options, child) => { - // console.error('pipe to service yes', service.env) let bin = service.bin if (coverageServiceTest) { - // console.error('use fakey test bin') // test scaffolding. // don't actually send stuff to the service bin = require.resolve('../test/fixtures/cat.js') @@ -476,6 +478,7 @@ const pipeToCoverageService = (service, options, child) => { : console.log('Successfully piped to ' + service.name)) } +/* istanbul ignore next */ const runCoverageReport = (options, code, signal) => { if (options.checkCoverage) runCoverageCheck(options, code, signal) @@ -483,6 +486,7 @@ const runCoverageReport = (options, code, signal) => { runCoverageReportOnly(options, code, signal) } +/* istanbul ignore next */ const runCoverageReportOnly = (options, code, signal) => { const close = (s, c) => { if (signal || s) { @@ -503,12 +507,10 @@ const runCoverageReportOnly = (options, code, signal) => { } const args = [nycBin, 'report', '--reporter', options.coverageReport] - // console.error('run coverage report', args) let child // automatically hook into coveralls if (options.coverageReport === 'text-lcov' && options.pipeToService) { - // console.error('pipeToService') child = spawn(node, args, { stdio: [ 0, 'pipe', 2 ] }) pipeToCoverageServices(options, child) } else { @@ -525,6 +527,7 @@ const runCoverageReportOnly = (options, code, signal) => { } } +/* istanbul ignore next */ const coverageCheckArgs = options => { const args = [] if (options.branches) @@ -539,6 +542,7 @@ const coverageCheckArgs = options => { return args } +/* istanbul ignore next */ const runCoverageCheck = (options, code, signal) => { const args = [nycBin, 'check-coverage'].concat(coverageCheckArgs(options)) @@ -589,20 +593,8 @@ const setupTapEnv = options => { process.env.TAP_ONLY = '1' } -const globFiles = files => { - return files.reduce((acc, f) => { - if (f === '-') { - acc.push(f) - return acc - } - - // glob claims patterns MUST not include any '\'s - if (!/\\/.test(f)) - f = glob.sync(f) || f - - return acc.concat(f) - }, []) -} +const globFiles = files => files.reduce((acc, f) => + acc.concat(f === '-' ? f : glob.sync(f, { nonull: true })), []) const makeReporter = options => new (require('tap-mocha-reporter'))(options.reporter) @@ -670,16 +662,26 @@ const saveFails = (options, tap) => { tap.on('end', save) } -const filterFiles = (files, saved, parallelOk) => { - return files.filter(file => { +const filterFiles = (files, saved, parallelOk) => + files.filter(file => { if (path.basename(file) === 'tap-parallel-ok') parallelOk[path.resolve(path.dirname(file))] = true else if (path.basename(file) === 'tap-parallel-not-ok') parallelOk[path.resolve(path.dirname(file))] = false else - return saved === null || saved.indexOf(file) !== -1 + return saved === null || onSavedList(saved, file) }) -} + +// check if the file is on the list, or if it's a parent dir of +// any items that are on the list. +const onSavedList = (saved, file) => + !saved || !saved.length ? true + : file === path.dirname(file) ? false + : saved.indexOf(file) !== -1 ? true + : onSavedList(saved.filter( + f => f !== path.dirname(f)).map( + f => path.dirname(f) + ), file) const isParallelOk = (parallelOk, file) => { const dir = path.resolve(path.dirname(file)) diff --git a/package.json b/package.json index 4b6567f0f..8040a7055 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "regen-fixtures": "node scripts/generate-test-test.js test/test/*.js", "snap": "TAP_SNAPSHOT=1 node bin/run.js test/*.js", "test": "node bin/run.js test/*.js --100 -J", + "unit": "bash scripts/unit.sh", "test-legacy": "node bin/run.js test-legacy/*.* --coverage -t3600 -sfails", "smoke": "node bin/run.js --node-arg=test-legacy/test.js test-legacy/test/*.js -j2", "posttest": "standard lib test", diff --git a/tap-snapshots/test-run.js-TAP.test.js b/tap-snapshots/test-run.js-TAP.test.js index 418bab29d..78e09a4db 100644 --- a/tap-snapshots/test-run.js-TAP.test.js +++ b/tap-snapshots/test-run.js-TAP.test.js @@ -33,3 +33,308 @@ ok 2 - /dev/stdin # {time} { # {time} ` + +exports[`run.js TAP coverage --100 pass > 100 pass 1`] = ` +TAP version 13 +# Subtest: 1.test.js + ok 1 - should be equal + 1..1 + # {time} +ok 1 - 1.test.js # {time} + +# Subtest: 2.test.js + ok 1 - should be equal + 1..1 + # {time} +ok 2 - 2.test.js # {time} + +# Subtest: 3.test.js + ok 1 - should be equal + 1..1 + # {time} +ok 3 - 3.test.js # {time} + +1..3 +# {time} +----------|----------|----------|----------|----------|----------------| +File | % Stmts | % Branch | % Funcs | % Lines |Uncovered Lines | +----------|----------|----------|----------|----------|----------------| +All files | 100 | 100 | 100 | 100 | | + ok.js | 100 | 100 | 100 | 100 | | +----------|----------|----------|----------|----------|----------------| + +` + +exports[`run.js TAP coverage --100 fail > 100 fail 1`] = ` +TAP version 13 +# Subtest: 1.test.js + ok 1 - should be equal + 1..1 + # {time} +ok 1 - 1.test.js # {time} + +# Subtest: 2.test.js + ok 1 - should be equal + 1..1 + # {time} +ok 2 - 2.test.js # {time} + +1..2 +# {time} +----------|----------|----------|----------|----------|----------------| +File | % Stmts | % Branch | % Funcs | % Lines |Uncovered Lines | +----------|----------|----------|----------|----------|----------------| +All files | 75 | 75 | 100 | 75 | | + ok.js | 75 | 75 | 100 | 75 | 6 | +----------|----------|----------|----------|----------|----------------| + +` + +exports[`run.js TAP coverage report only > lcov output 1`] = ` +TN: +SF:{CWD}/ok.js +FN:2,(anonymous_0) +FNF:1 +FNH:1 +FNDA:2,(anonymous_0) +DA:2,2 +DA:3,2 +DA:4,2 +DA:6,0 +LF:4 +LH:3 +BRDA:3,0,0,2 +BRDA:3,0,1,0 +BRDA:4,1,0,2 +BRDA:4,1,1,1 +BRF:4 +BRH:3 +end_of_record + +` + +exports[`run.js TAP coverage report with checks > lcov output and 100 check 1`] = ` +TN: +SF:{CWD}/ok.js +FN:2,(anonymous_0) +FNF:1 +FNH:1 +FNDA:2,(anonymous_0) +DA:2,2 +DA:3,2 +DA:4,2 +DA:6,0 +LF:4 +LH:3 +BRDA:3,0,0,2 +BRDA:3,0,1,0 +BRDA:4,1,0,2 +BRDA:4,1,1,1 +BRF:4 +BRH:3 +end_of_record + +` + +exports[`run.js TAP coverage pipe to service > piped to coverage service cat 1`] = ` +COVERAGE_SERVICE_TEST +TN: +SF:{CWD}/ok.js +FN:2,(anonymous_0) +FNF:1 +FNH:1 +FNDA:2,(anonymous_0) +DA:2,2 +DA:3,2 +DA:4,2 +DA:6,0 +LF:4 +LH:3 +BRDA:3,0,0,2 +BRDA:3,0,1,0 +BRDA:4,1,0,2 +BRDA:4,1,1,1 +BRF:4 +BRH:3 +end_of_record + +` + +exports[`run.js TAP save file with bailout, should save all untested > stdout 1`] = ` +TAP version 13 +# Subtest: a/b/2.js + ok 1 - 2 + 1..1 + # {time} +ok 1 - a/b/2.js # {time} + +# Subtest: a/b/f1.js + not ok 1 - a/b + --- + at: + line: # + column: # + file: a/b/f1.js + source: | + require("{TAPDIR}/").fail('a/b') + ... + + Bail out! # a/b +Bail out! # a/b + +` + +exports[`run.js TAP save file with bailout, should save all untested > savefile 1`] = ` +a +x +a/b +x/y +a/b/f1.js +a/b/f2.js +x/y/1.js + +` + +exports[`run.js TAP save file without bailout, run untested, save failures > stdout 1`] = ` +TAP version 13 +# Subtest: a/b/2.js + ok 1 - 2 + 1..1 + # {time} +ok 1 - a/b/2.js # {time} + +# Subtest: a/b/f1.js + not ok 1 - a/b + --- + at: + line: # + column: # + file: a/b/f1.js + source: | + require("{TAPDIR}/").fail('a/b') + ... + + 1..1 + # failed 1 test + # {time} +not ok 2 - a/b/f1.js # {time} + --- + args: + - a/b/f1.js + command: /usr/local/bin/node + cwd: {CWD} + exitCode: 1 + file: a/b/f1.js + stdio: + - 0 + - pipe + - 2 timeout: {default} + ... + +# Subtest: a/b/f2.js + not ok 1 - c/d + --- + at: + line: # + column: # + file: a/b/f2.js + source: | + require("{TAPDIR}/").fail('c/d') + ... + + 1..1 + # failed 1 test + # {time} +not ok 3 - a/b/f2.js # {time} + --- + args: + - a/b/f2.js + command: /usr/local/bin/node + cwd: {CWD} + exitCode: 1 + file: a/b/f2.js + stdio: + - 0 + - pipe + - 2 timeout: {default} + ... + +# Subtest: x/y/1.js + ok 1 - one + 1..1 + # {time} +ok 4 - x/y/1.js # {time} + +1..4 +# failed 2 of 4 tests +# {time} + +` + +exports[`run.js TAP save file without bailout, run untested, save failures > savefile 1`] = ` +a/b/f1.js +a/b/f2.js + +` + +exports[`run.js TAP save file pass, empty save file > stdout 1`] = ` +TAP version 13 +# Subtest: a/b/2.js + ok 1 - 2 + 1..1 + # {time} +ok 1 - a/b/2.js # {time} + +# Subtest: a/b/f1.js + ok 1 - fine now + 1..1 + # {time} +ok 2 - a/b/f1.js # {time} + +# Subtest: a/b/f2.js + ok 1 - fine now too + 1..1 + # {time} +ok 3 - a/b/f2.js # {time} + +# Subtest: x/y/1.js + ok 1 - one + 1..1 + # {time} +ok 4 - x/y/1.js # {time} + +1..4 +# {time} + +` + +exports[`run.js TAP save file empty save file, run all tests > stdout 1`] = ` +TAP version 13 +# Subtest: a/b/2.js + ok 1 - 2 + 1..1 + # {time} +ok 1 - a/b/2.js # {time} + +# Subtest: a/b/f1.js + ok 1 - fine now + 1..1 + # {time} +ok 2 - a/b/f1.js # {time} + +# Subtest: a/b/f2.js + ok 1 - fine now too + 1..1 + # {time} +ok 3 - a/b/f2.js # {time} + +# Subtest: x/y/1.js + ok 1 - one + 1..1 + # {time} +ok 4 - x/y/1.js # {time} + +1..4 +# {time} + +` diff --git a/test/clean-stacks.js b/test/clean-stacks.js index e09eb5959..64d7b7161 100644 --- a/test/clean-stacks.js +++ b/test/clean-stacks.js @@ -45,8 +45,12 @@ module.exports = out => out .replace(/\n( +)method: .*(\n\1 .*)*\n/g, '\n') .replace(/\n( +)type: .*\n/g, '\n') + // timeout values are different when coverage is present + .replace(/\n( *)timeout: (30000|240000)(\n|$)/g, '$1timeout: {default}$3') + // fix references to cwd .split(process.cwd()).join('{CWD}') + .split(require('path').resolve(__dirname, '..')).join('{TAPDIR}') // nothing to see here if (module === require.main) diff --git a/test/run.js b/test/run.js index bae7abc1c..1e3084872 100644 --- a/test/run.js +++ b/test/run.js @@ -3,7 +3,8 @@ const fs = require('fs') const mkdirp = require('mkdirp') const rimraf = require('rimraf') const path = require('path') -const execFile = require('child_process').execFile +const cp = require('child_process') +const execFile = cp.execFile const node = process.execPath const bin = require.resolve('../bin/run.js') const tap = JSON.stringify(path.join(__dirname, '..') + '/') @@ -11,6 +12,13 @@ const t = require('../') const dir = path.join(__dirname, 'cli-tests') mkdirp.sync(dir) +t.teardown(() => rimraf.sync(dir)) + +// set this forcibly so it doesn't interfere with other tests. +delete process.env.TAP_DIAG +delete process.env.TAP_BAIL +delete process.env.TAP_COLORS +delete process.env.TAP_TIMEOUT const clean = require('./clean-stacks.js') @@ -26,8 +34,12 @@ const run = (args, options, cb) => { } const tmpfile = (t, filename, content) => { + const parts = filename.split('/') + // make any necessary dirs + if (parts.length > 1) + mkdirp.sync(path.join(dir, parts.slice(0, -1).join('/'))) + t.teardown(() => rimraf.sync(path.join(dir, parts[0]))) filename = path.join(dir, filename) - t.teardown(() => rimraf.sync(filename)) fs.writeFileSync(filename, content) return path.relative('', filename) } @@ -77,7 +89,7 @@ t.test('usage and other basics', t => { t.test('basic', t => { const ok = tmpfile(t, 'ok.js', `require(${tap}).pass('this is fine')`) - run(['-Cb', '--', ok], null, (err, stdout, stderr) => { + run(['-Cbt0', '--', ok], null, (err, stdout, stderr) => { t.matchSnapshot(clean(stdout), 'ok.js output') t.end() }) @@ -229,7 +241,7 @@ t.test('stdin', t => { const tapcode = 'TAP version 13\n1..1\nok\n' t.test('with output file', t => { - const c = run(['-', '-C', '-Rspec', '-ofoo.txt', '--cov'], { env: { + const c = run(['-', '-c', '-Rspec', '-ofoo.txt', '--cov'], { env: { _TAP_IS_TTY: '1', TAP: '0' }}, (er, o, e) => { @@ -244,12 +256,12 @@ t.test('stdin', t => { }) t.test('no output file', t => { - const c = run(['-', '--only', '-gx', '-iC', '-Rclassic'], { env: { + const c = run(['--only', '-gx', '-iC', '-Rclassic'], { env: { _TAP_IS_TTY: '1', TAP: '0' }}, (er, o, e) => { t.equal(er, null) - t.equal(e, '') + t.equal(e, 'Reading TAP data from stdin (use "-" argument to suppress)\n') t.match(o, /total \.+ 1\/1/) t.throws(() => fs.statSync('foo.txt')) t.end() @@ -279,6 +291,21 @@ t.test('stdin', t => { t.end() }) +t.test('epipe on stdout', t => { + const c = run(['-', '-C'], { stdio: 'pipe' }, (er, o, e) => { + t.equal(er, null) + t.equal(o, 'TAP version 13\n1..9\nok\n') + t.equal(e, '') + t.end() + }) + c.stdin.write('TAP version 13\n1..9\nok\n') + c.stdout.on('data', chunk => { + c.stdout.destroy() + c.stdin.write('ok\nok\nok\nok\n') + c.stdin.write('ok\nok\nok\nok\n') + }) +}) + t.test('unknown arg throws', t => { run(['--blerg'], null, (er, o, e) => { t.match(er, { code: 1 }) @@ -287,10 +314,168 @@ t.test('unknown arg throws', t => { }) }) -t.test('save file') -t.test('coverage service piping') -t.test('coverage report only') -t.test('color, -c -C TAP_COLOR=1') -t.test('timeouts incl --no-timeout') -t.test('epipe') -t.test('parallel sigil files') +t.test('coverage', t => { + const cwd = process.cwd() + process.chdir(dir) + const ok = tmpfile(t, 'ok.js', `'use strict' + module.exports = (x, y) => { + if (x) + return y || x + else + return y + }`) + + tmpfile(t, 'package.json', '{"name":"just a placeholder"}') + + const t1 = tmpfile(t, '1.test.js', `'use strict' + const ok = require('./ok.js') + require(${tap}).equal(ok(1), 1)`) + + const t2 = tmpfile(t, '2.test.js', `'use strict' + const ok = require('./ok.js') + require(${tap}).equal(ok(1, 2), 2)`) + + const t3 = tmpfile(t, '3.test.js', `'use strict' + const ok = require('./ok.js') + require(${tap}).equal(ok(0, 3), 3)`) + + // escape from new york + const esc = tmpfile(t, 'runtest.sh', +`"${node}" "${bin}" "\$@" --cov --nyc-arg=--include="${ok}" +`) + + const escape = (args, options, cb) => { + options = options || {} + const env = Object.keys(process.env).filter( + k => !/TAP|NYC|SW_ORIG/.test(k) + ).reduce((env, k) => { + if (!env.hasOwnProperty(k)) + env[k] = process.env[k] + return env + }, options.env || {}) + options.env = env + return execFile('bash', [esc].concat(args), options, cb) + } + + t.test('--100', t => { + t.test('pass', t => { + escape([t1, t2, t3, '--100'], null, (er, o, e) => { + t.equal(er, null) + t.matchSnapshot(clean(o), '100 pass') + t.end() + }) + }) + t.test('fail', t => { + escape([t1, t2, '--100'], null, (er, o, e) => { + t.match(er, { code: 1 }) + t.matchSnapshot(clean(o), '100 fail') + t.end() + }) + }) + t.end() + }) + + t.test('report only', t => { + escape(['--coverage-report=text-lcov'], null, (er, o, e) => { + t.equal(er, null) + t.matchSnapshot(clean(o), 'lcov output') + t.end() + }) + }) + + t.test('report with checks', t => { + escape(['--100', '--coverage-report=text-lcov'], null, (er, o, e) => { + t.match(er, { code: 1 }) + t.matchSnapshot(clean(o), 'lcov output and 100 check') + t.end() + }) + }) + + t.test('pipe to service', t => { + escape(['--coverage-report=text-lcov'], { env: { + COVERAGE_SERVICE_TEST: 'true' + }}, (er, o, e) => { + t.equal(er, null) + t.matchSnapshot(clean(o), 'piped to coverage service cat') + t.end() + }) + }) + + t.end() +}) + +t.test('save file', t => { + const xy1 = tmpfile(t, 'x/y/1.js', `'use strict' + const t = require(${tap}) + t.pass('one') + `) + + const ab2 = tmpfile(t, 'a/b/2.js', `'use strict' + const t = require(${tap}) + t.pass('2') + `) + + const abf1 = tmpfile(t, 'a/b/f1.js', `'use strict' + require(${tap}).fail('a/b') + `) + + const abf2 = tmpfile(t, 'a/b/f2.js', `'use strict' + require(${tap}).fail('c/d') + `) + + const savefile = path.resolve(tmpfile(t, 'fails.txt', '')) + + t.test('with bailout, should save all untested', t => { + run(['a', 'x', '-s', savefile, '-b'], { cwd: dir }, (er, o, e) => { + t.match(er, { code: 1 }) + t.matchSnapshot(clean(o), 'stdout') + t.equal(e, '') + t.matchSnapshot(clean(fs.readFileSync(savefile, 'utf8')), 'savefile') + t.end() + }) + }) + + t.test('without bailout, run untested, save failures', t => { + run(['a', 'x', '-s', savefile], { cwd: dir }, (er, o, e) => { + t.match(er, { code: 1 }) + t.matchSnapshot(clean(o), 'stdout') + t.equal(e, '') + t.matchSnapshot(clean(fs.readFileSync(savefile, 'utf8')), 'savefile') + t.end() + }) + }) + + t.test('make fails pass', t => { + fs.writeFileSync(abf1, `'use strict' + require(${tap}).pass('fine now') + `) + fs.writeFileSync(abf2, `'use strict' + require(${tap}).pass('fine now too') + `) + t.end() + }) + + t.test('pass, empty save file', t => { + run(['a', 'x', '-s', savefile], { cwd: dir }, (er, o, e) => { + t.equal(er, null) + t.matchSnapshot(clean(o), 'stdout') + t.equal(e, '') + t.throws(() => fs.statSync(savefile), 'save file is gone') + t.end() + }) + }) + + t.test('empty save file, run all tests', t => { + run(['a', 'x', '-s', savefile], { cwd: dir }, (er, o, e) => { + t.equal(er, null) + t.matchSnapshot(clean(o), 'stdout') + t.equal(e, '') + t.throws(() => fs.statSync(savefile), 'save file is gone') + t.end() + }) + }) + + t.end() +}) + +t.test('parallel')