diff --git a/package.json b/package.json index 10bdc493a..d2aeac664 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,8 @@ "stoppable": "^1.1.0", "tap": "^14.4.1", "tsd": "^0.11.0", - "workq": "^2.1.0" + "workq": "^2.1.0", + "xmlbuilder2": "^2.1.2" }, "dependencies": { "debug": "^4.1.1", diff --git a/test/integration/index.js b/test/integration/index.js index 507273aff..1dbf82d6b 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -4,14 +4,15 @@ 'use strict' -const { readFileSync, accessSync, mkdirSync, readdirSync, statSync } = require('fs') +const { writeFileSync, readFileSync, accessSync, mkdirSync, readdirSync, statSync } = require('fs') const { join, sep } = require('path') const yaml = require('js-yaml') const Git = require('simple-git') +const ms = require('ms') const { Client } = require('../../index') const build = require('./test-runner') const { sleep } = require('./helper') -const ms = require('ms') +const createJunitReporter = require('./reporter') const esRepo = 'https://github.com/elastic/elasticsearch.git' const esFolder = join(__dirname, '..', '..', 'elasticsearch') @@ -133,6 +134,8 @@ async function start ({ client, isXPack }) { await withSHA(sha) log(`Testing ${isXPack ? 'XPack' : 'oss'} api...`) + const junit = createJunitReporter() + const junitTestSuites = junit.testsuites(`Integration test for ${isXPack ? 'XPack' : 'oss'} api`) const stats = { total: 0, @@ -196,31 +199,43 @@ async function start ({ client, isXPack }) { const cleanPath = file.slice(file.lastIndexOf(apiName)) log(' ' + cleanPath) + const junitTestSuite = junitTestSuites.testsuite(apiName.slice(1) + ' - ' + cleanPath) for (const test of tests) { const testTime = now() const name = Object.keys(test)[0] if (name === 'setup' || name === 'teardown') continue + const junitTestCase = junitTestSuite.testcase(name) + stats.total += 1 if (shouldSkip(isXPack, file, name)) { stats.skip += 1 + junitTestCase.skip('This test is in the skip list of the client') + junitTestCase.end() continue } log(' - ' + name) try { - await testRunner.run(setupTest, test[name], teardownTest, stats) + await testRunner.run(setupTest, test[name], teardownTest, stats, junitTestCase) stats.pass += 1 } catch (err) { + junitTestCase.failure(err) + junitTestCase.end() + junitTestSuite.end() + junitTestSuites.end() + generateJunitXmlReport(junit, isXPack ? 'xpack' : 'oss') console.error(err) process.exit(1) } const totalTestTime = now() - testTime + junitTestCase.end() if (totalTestTime > MAX_TEST_TIME) { log(' took too long: ' + ms(totalTestTime)) } else { log(' took: ' + ms(totalTestTime)) } } + junitTestSuite.end() const totalFileTime = now() - fileTime if (totalFileTime > MAX_FILE_TIME) { log(` ${cleanPath} took too long: ` + ms(totalFileTime)) @@ -235,6 +250,8 @@ async function start ({ client, isXPack }) { log(`${apiName} took: ` + ms(totalApiTime)) } } + junitTestSuites.end() + generateJunitXmlReport(junit, isXPack ? 'xpack' : 'oss') log(`Total testing time: ${ms(now() - totalTime)}`) log(`Test stats: - Total: ${stats.total} @@ -359,6 +376,13 @@ function createFolder (name) { } } +function generateJunitXmlReport (junit, suite) { + writeFileSync( + join(__dirname, '..', '..', `${suite}-report-junit.xml`), + junit.prettyPrint() + ) +} + if (require.main === module) { const node = process.env.TEST_ES_SERVER || 'http://localhost:9200' const opts = { diff --git a/test/integration/reporter.js b/test/integration/reporter.js new file mode 100644 index 000000000..0d3621de7 --- /dev/null +++ b/test/integration/reporter.js @@ -0,0 +1,109 @@ +'use strict' + +const assert = require('assert') +const { create } = require('xmlbuilder2') + +function createJunitReporter () { + const report = {} + + return { testsuites, prettyPrint } + + function prettyPrint () { + return create(report).end({ prettyPrint: true }) + } + + function testsuites (name) { + assert(name, 'The testsuites name is required') + assert(report.testsuites === undefined, 'Cannot set more than one testsuites block') + const startTime = Date.now() + + report.testsuites = { + '@id': new Date().toISOString(), + '@name': name + } + + const testsuiteList = [] + + return { + testsuite: createTestSuite(testsuiteList), + end () { + report.testsuites['@time'] = Math.round((Date.now() - startTime) / 1000) + report.testsuites['@tests'] = testsuiteList.reduce((acc, val) => { + acc += val['@tests'] + return acc + }, 0) + report.testsuites['@failures'] = testsuiteList.reduce((acc, val) => { + acc += val['@failures'] + return acc + }, 0) + report.testsuites['@skipped'] = testsuiteList.reduce((acc, val) => { + acc += val['@skipped'] + return acc + }, 0) + if (testsuiteList.length) { + report.testsuites.testsuite = testsuiteList + } + } + } + } + + function createTestSuite (testsuiteList) { + return function testsuite (name) { + assert(name, 'The testsuite name is required') + const startTime = Date.now() + const suite = { + '@id': new Date().toISOString(), + '@name': name + } + const testcaseList = [] + testsuiteList.push(suite) + return { + testcase: createTestCase(testcaseList), + end () { + suite['@time'] = Math.round((Date.now() - startTime) / 1000) + suite['@tests'] = testcaseList.length + suite['@failures'] = testcaseList.filter(t => t.failure).length + suite['@skipped'] = testcaseList.filter(t => t.skipped).length + if (testcaseList.length) { + suite.testcase = testcaseList + } + } + } + } + } + + function createTestCase (testcaseList) { + return function testcase (name) { + assert(name, 'The testcase name is required') + const startTime = Date.now() + const tcase = { + '@id': new Date().toISOString(), + '@name': name + } + testcaseList.push(tcase) + return { + failure (error) { + assert(error, 'The failure error object is required') + tcase.failure = { + '#': error.stack, + '@message': error.message, + '@type': error.code + } + }, + skip (reason) { + if (typeof reason !== 'string') { + reason = JSON.stringify(reason, null, 2) + } + tcase.skipped = { + '#': reason + } + }, + end () { + tcase['@time'] = Math.round((Date.now() - startTime) / 1000) + } + } + } + } +} + +module.exports = createJunitReporter diff --git a/test/integration/test-runner.js b/test/integration/test-runner.js index 96443f9b2..acfd29143 100644 --- a/test/integration/test-runner.js +++ b/test/integration/test-runner.js @@ -218,11 +218,12 @@ function build (opts = {}) { * @oaram {object} teardown (null if not needed) * @returns {Promise} */ - async function run (setup, test, teardown, stats) { + async function run (setup, test, teardown, stats, junit) { // if we should skip a feature in the setup/teardown section // we should skip the entire test file const skip = getSkip(setup) || getSkip(teardown) if (skip && shouldSkip(esVersion, skip)) { + junit.skip(skip) logSkip(skip) return } @@ -240,11 +241,11 @@ function build (opts = {}) { } } - if (setup) await exec('Setup', setup, stats) + if (setup) await exec('Setup', setup, stats, junit) - await exec('Test', test, stats) + await exec('Test', test, stats, junit) - if (teardown) await exec('Teardown', teardown, stats) + if (teardown) await exec('Teardown', teardown, stats, junit) if (isXPack) await cleanupXPack() @@ -451,11 +452,12 @@ function build (opts = {}) { * @param {object} the actions to perform * @returns {Promise} */ - async function exec (name, actions, stats) { + async function exec (name, actions, stats, junit) { // tap.comment(name) for (const action of actions) { if (action.skip) { if (shouldSkip(esVersion, action.skip)) { + junit.skip(fillStashedValues(action.skip)) logSkip(fillStashedValues(action.skip)) break }