From 23ccc152aa859ba0bb540b5eaf0d8423027aa571 Mon Sep 17 00:00:00 2001 From: delvedor Date: Mon, 20 Apr 2020 11:08:52 +0200 Subject: [PATCH 1/7] Enable junit report --- .ci/jobs/defaults.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml index 1cddafc36..0d6632c4d 100644 --- a/.ci/jobs/defaults.yml +++ b/.ci/jobs/defaults.yml @@ -69,6 +69,6 @@ publishers: - email: recipients: infra-root+build@elastic.co - # - junit: - # results: "*-junit.xml" - # allow-empty-results: true + - junit: + results: "*-junit.xml" + allow-empty-results: true From 833af2b7a4a5733568737d73211c360adc013550 Mon Sep 17 00:00:00 2001 From: delvedor Date: Mon, 20 Apr 2020 11:09:15 +0200 Subject: [PATCH 2/7] Added junit reporter utility --- package.json | 3 +- test/integration/reporter.js | 104 +++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 test/integration/reporter.js diff --git a/package.json b/package.json index 10bdc493a..d2aeac664 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,8 @@ "stoppable": "^1.1.0", "tap": "^14.4.1", "tsd": "^0.11.0", - "workq": "^2.1.0" + "workq": "^2.1.0", + "xmlbuilder2": "^2.1.2" }, "dependencies": { "debug": "^4.1.1", diff --git a/test/integration/reporter.js b/test/integration/reporter.js new file mode 100644 index 000000000..4ecb60902 --- /dev/null +++ b/test/integration/reporter.js @@ -0,0 +1,104 @@ +'use strict' + +const assert = require('assert') +const { create } = require('xmlbuilder2') + +function createJunitReporter () { + const report = {} + + return { testsuites, prettyPrint } + + function prettyPrint () { + return create(report).end({ prettyPrint: true }) + } + + function testsuites (name) { + assert(name, 'The testsuites name is required') + assert(report.testsuites === undefined, 'Cannot set more than one testsuites block') + const startTime = Date.now() + + report.testsuites = { + '@id': new Date().toISOString(), + '@name': name + } + + const testsuiteList = [] + + return { + testsuite: createTestSuite(testsuiteList), + end () { + report.testsuites['@time'] = Date.now() - startTime + report.testsuites['@tests'] = testsuiteList.reduce((acc, val) => { + acc += val['@tests'] + return acc + }, 0) + report.testsuites['@failures'] = testsuiteList.reduce((acc, val) => { + acc += val['@failures'] + return acc + }, 0) + report.testsuites['@skipped'] = testsuiteList.reduce((acc, val) => { + acc += val['@skipped'] + return acc + }, 0) + if (testsuiteList.length) { + report.testsuites.testsuite = testsuiteList + } + } + } + } + + function createTestSuite (testsuiteList) { + return function testsuite (name) { + assert(name, 'The testsuite name is required') + const startTime = Date.now() + const suite = { + '@id': new Date().toISOString(), + '@name': name + } + const testcaseList = [] + testsuiteList.push(suite) + return { + testcase: createTestCase(testcaseList), + end () { + suite['@time'] = Date.now() - startTime + suite['@tests'] = testcaseList.length + suite['@failures'] = testcaseList.filter(t => t.failure).length + suite['@skipped'] = testcaseList.filter(t => t.skipped).length + if (testcaseList.length) { + suite.testcase = testcaseList + } + } + } + } + } + + function createTestCase (testcaseList) { + return function testcase (name) { + assert(name, 'The testcase name is required') + const startTime = Date.now() + const tcase = { + '@id': new Date().toISOString(), + '@name': name + } + testcaseList.push(tcase) + return { + failure (error) { + assert(error, 'The failure error object is required') + tcase.failure = { + '#': error.stack, + '@message': error.message, + '@type': error.code + } + }, + skip () { + tcase.skipped = {} + }, + end () { + tcase['@time'] = Date.now() - startTime + } + } + } + } +} + +module.exports = createJunitReporter From 535abb3da8cd12a2455bd5deb0b2ee57ce7f595a Mon Sep 17 00:00:00 2001 From: delvedor Date: Mon, 20 Apr 2020 11:09:42 +0200 Subject: [PATCH 3/7] Generate junit report --- test/integration/index.js | 29 ++++++++++++++++++++++++++--- test/integration/test-runner.js | 12 +++++++----- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/test/integration/index.js b/test/integration/index.js index 507273aff..02d4ac5f7 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -4,14 +4,15 @@ 'use strict' -const { readFileSync, accessSync, mkdirSync, readdirSync, statSync } = require('fs') +const { writeFileSync, readFileSync, accessSync, mkdirSync, readdirSync, statSync } = require('fs') const { join, sep } = require('path') const yaml = require('js-yaml') const Git = require('simple-git') +const ms = require('ms') const { Client } = require('../../index') const build = require('./test-runner') const { sleep } = require('./helper') -const ms = require('ms') +const createJunitReporter = require('./reporter') const esRepo = 'https://github.com/elastic/elasticsearch.git' const esFolder = join(__dirname, '..', '..', 'elasticsearch') @@ -133,6 +134,8 @@ async function start ({ client, isXPack }) { await withSHA(sha) log(`Testing ${isXPack ? 'XPack' : 'oss'} api...`) + const junit = createJunitReporter() + const junitTestSuites = junit.testsuites(`Integration test for ${isXPack ? 'XPack' : 'oss'} api`) const stats = { total: 0, @@ -196,31 +199,42 @@ async function start ({ client, isXPack }) { const cleanPath = file.slice(file.lastIndexOf(apiName)) log(' ' + cleanPath) + const junitTestSuite = junitTestSuites.testsuite(apiName.slice(1) + ' - ' + cleanPath) for (const test of tests) { const testTime = now() const name = Object.keys(test)[0] if (name === 'setup' || name === 'teardown') continue + const junitTestCase = junitTestSuite.testcase(name) + stats.total += 1 if (shouldSkip(isXPack, file, name)) { stats.skip += 1 + junitTestCase.skip() continue } log(' - ' + name) try { - await testRunner.run(setupTest, test[name], teardownTest, stats) + await testRunner.run(setupTest, test[name], teardownTest, stats, junitTestCase) stats.pass += 1 } catch (err) { + junitTestCase.failure(err) + junitTestCase.end() + junitTestSuite.end() + junitTestSuites.end() + generateJunitXmlReport(junit, isXPack ? 'xpack' : 'oss') console.error(err) process.exit(1) } const totalTestTime = now() - testTime + junitTestCase.end() if (totalTestTime > MAX_TEST_TIME) { log(' took too long: ' + ms(totalTestTime)) } else { log(' took: ' + ms(totalTestTime)) } } + junitTestSuite.end() const totalFileTime = now() - fileTime if (totalFileTime > MAX_FILE_TIME) { log(` ${cleanPath} took too long: ` + ms(totalFileTime)) @@ -235,6 +249,8 @@ async function start ({ client, isXPack }) { log(`${apiName} took: ` + ms(totalApiTime)) } } + junitTestSuites.end() + generateJunitXmlReport(junit, isXPack ? 'xpack' : 'oss') log(`Total testing time: ${ms(now() - totalTime)}`) log(`Test stats: - Total: ${stats.total} @@ -359,6 +375,13 @@ function createFolder (name) { } } +function generateJunitXmlReport (junit, suite) { + writeFileSync( + join(__dirname, '..', '..', `${suite}-report-junit.xml`), + junit.prettyPrint() + ) +} + if (require.main === module) { const node = process.env.TEST_ES_SERVER || 'http://localhost:9200' const opts = { diff --git a/test/integration/test-runner.js b/test/integration/test-runner.js index 96443f9b2..9fa579a0e 100644 --- a/test/integration/test-runner.js +++ b/test/integration/test-runner.js @@ -218,11 +218,12 @@ function build (opts = {}) { * @oaram {object} teardown (null if not needed) * @returns {Promise} */ - async function run (setup, test, teardown, stats) { + async function run (setup, test, teardown, stats, junit) { // if we should skip a feature in the setup/teardown section // we should skip the entire test file const skip = getSkip(setup) || getSkip(teardown) if (skip && shouldSkip(esVersion, skip)) { + junit.skip() logSkip(skip) return } @@ -240,11 +241,11 @@ function build (opts = {}) { } } - if (setup) await exec('Setup', setup, stats) + if (setup) await exec('Setup', setup, stats, junit) - await exec('Test', test, stats) + await exec('Test', test, stats, junit) - if (teardown) await exec('Teardown', teardown, stats) + if (teardown) await exec('Teardown', teardown, stats, junit) if (isXPack) await cleanupXPack() @@ -451,11 +452,12 @@ function build (opts = {}) { * @param {object} the actions to perform * @returns {Promise} */ - async function exec (name, actions, stats) { + async function exec (name, actions, stats, junit) { // tap.comment(name) for (const action of actions) { if (action.skip) { if (shouldSkip(esVersion, action.skip)) { + junit.skip() logSkip(fillStashedValues(action.skip)) break } From 62d5b377146f02a5c6bae20fff203461df019569 Mon Sep 17 00:00:00 2001 From: delvedor Date: Wed, 22 Apr 2020 10:47:58 +0200 Subject: [PATCH 4/7] junit reporter: store time in seconds --- test/integration/reporter.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/reporter.js b/test/integration/reporter.js index 4ecb60902..173b444dc 100644 --- a/test/integration/reporter.js +++ b/test/integration/reporter.js @@ -27,7 +27,7 @@ function createJunitReporter () { return { testsuite: createTestSuite(testsuiteList), end () { - report.testsuites['@time'] = Date.now() - startTime + report.testsuites['@time'] = Math.round((Date.now() - startTime) / 1000) report.testsuites['@tests'] = testsuiteList.reduce((acc, val) => { acc += val['@tests'] return acc @@ -60,7 +60,7 @@ function createJunitReporter () { return { testcase: createTestCase(testcaseList), end () { - suite['@time'] = Date.now() - startTime + suite['@time'] = Math.round((Date.now() - startTime) / 1000) suite['@tests'] = testcaseList.length suite['@failures'] = testcaseList.filter(t => t.failure).length suite['@skipped'] = testcaseList.filter(t => t.skipped).length @@ -94,7 +94,7 @@ function createJunitReporter () { tcase.skipped = {} }, end () { - tcase['@time'] = Date.now() - startTime + tcase['@time'] = Math.round((Date.now() - startTime) / 1000) } } } From 24d8b390194505cf03a37b05961fef9f928d8156 Mon Sep 17 00:00:00 2001 From: delvedor Date: Wed, 22 Apr 2020 14:03:39 +0200 Subject: [PATCH 5/7] Added skip reason in junit report --- test/integration/index.js | 2 +- test/integration/reporter.js | 9 +++++++-- test/integration/test-runner.js | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/integration/index.js b/test/integration/index.js index 02d4ac5f7..e06fc2f0b 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -210,7 +210,7 @@ async function start ({ client, isXPack }) { stats.total += 1 if (shouldSkip(isXPack, file, name)) { stats.skip += 1 - junitTestCase.skip() + junitTestCase.skip('This test is in the skip list of the client') continue } log(' - ' + name) diff --git a/test/integration/reporter.js b/test/integration/reporter.js index 173b444dc..40bd299b7 100644 --- a/test/integration/reporter.js +++ b/test/integration/reporter.js @@ -90,8 +90,13 @@ function createJunitReporter () { '@type': error.code } }, - skip () { - tcase.skipped = {} + skip (reason) { + if (typeof reason !== 'string') { + reason = JSON.stringify(reason) + } + tcase.skipped = { + '#': reason + } }, end () { tcase['@time'] = Math.round((Date.now() - startTime) / 1000) diff --git a/test/integration/test-runner.js b/test/integration/test-runner.js index 9fa579a0e..acfd29143 100644 --- a/test/integration/test-runner.js +++ b/test/integration/test-runner.js @@ -223,7 +223,7 @@ function build (opts = {}) { // we should skip the entire test file const skip = getSkip(setup) || getSkip(teardown) if (skip && shouldSkip(esVersion, skip)) { - junit.skip() + junit.skip(skip) logSkip(skip) return } @@ -457,7 +457,7 @@ function build (opts = {}) { for (const action of actions) { if (action.skip) { if (shouldSkip(esVersion, action.skip)) { - junit.skip() + junit.skip(fillStashedValues(action.skip)) logSkip(fillStashedValues(action.skip)) break } From 983f78339dbb50d70ccfbb759d7577985d0e1e4a Mon Sep 17 00:00:00 2001 From: delvedor Date: Wed, 22 Apr 2020 14:37:20 +0200 Subject: [PATCH 6/7] Added missing end --- test/integration/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/index.js b/test/integration/index.js index e06fc2f0b..1dbf82d6b 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -211,6 +211,7 @@ async function start ({ client, isXPack }) { if (shouldSkip(isXPack, file, name)) { stats.skip += 1 junitTestCase.skip('This test is in the skip list of the client') + junitTestCase.end() continue } log(' - ' + name) From a6fef0fdc42b4cc1a417d8f8743c50017e3472b1 Mon Sep 17 00:00:00 2001 From: delvedor Date: Wed, 22 Apr 2020 14:38:08 +0200 Subject: [PATCH 7/7] Fix skip serialize indentation --- test/integration/reporter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/reporter.js b/test/integration/reporter.js index 40bd299b7..0d3621de7 100644 --- a/test/integration/reporter.js +++ b/test/integration/reporter.js @@ -92,7 +92,7 @@ function createJunitReporter () { }, skip (reason) { if (typeof reason !== 'string') { - reason = JSON.stringify(reason) + reason = JSON.stringify(reason, null, 2) } tcase.skipped = { '#': reason