Generate junit report (#1165)

* Enable junit report

* Added junit reporter utility

* Generate junit report

* junit reporter: store time in seconds

* Added skip reason in junit report

* Added missing end

* Fix skip serialize indentation

Author: delvedor

package.json

@@ -62,7 +62,8 @@
     "stoppable": "^1.1.0",
     "tap": "^14.4.1",
     "tsd": "^0.11.0",
-    "workq": "^2.1.0"
+    "workq": "^2.1.0",
+    "xmlbuilder2": "^2.1.2"
   },
   "dependencies": {
     "debug": "^4.1.1",

test/integration/index.js

@@ -4,14 +4,15 @@
 
 'use strict'
 
-const { readFileSync, accessSync, mkdirSync, readdirSync, statSync } = require('fs')
+const { writeFileSync, readFileSync, accessSync, mkdirSync, readdirSync, statSync } = require('fs')
 const { join, sep } = require('path')
 const yaml = require('js-yaml')
 const Git = require('simple-git')
+const ms = require('ms')
 const { Client } = require('../../index')
 const build = require('./test-runner')
 const { sleep } = require('./helper')
-const ms = require('ms')
+const createJunitReporter = require('./reporter')
 
 const esRepo = 'https://github.com/elastic/elasticsearch.git'
 const esFolder = join(__dirname, '..', '..', 'elasticsearch')
@@ -133,6 +134,8 @@ async function start ({ client, isXPack }) {
   await withSHA(sha)
 
   log(`Testing ${isXPack ? 'XPack' : 'oss'} api...`)
+  const junit = createJunitReporter()
+  const junitTestSuites = junit.testsuites(`Integration test for ${isXPack ? 'XPack' : 'oss'} api`)
 
   const stats = {
     total: 0,
@@ -196,31 +199,43 @@ async function start ({ client, isXPack }) {
 
       const cleanPath = file.slice(file.lastIndexOf(apiName))
       log('    ' + cleanPath)
+      const junitTestSuite = junitTestSuites.testsuite(apiName.slice(1) + ' - ' + cleanPath)
 
       for (const test of tests) {
         const testTime = now()
         const name = Object.keys(test)[0]
         if (name === 'setup' || name === 'teardown') continue
+        const junitTestCase = junitTestSuite.testcase(name)
+
         stats.total += 1
         if (shouldSkip(isXPack, file, name)) {
           stats.skip += 1
+          junitTestCase.skip('This test is in the skip list of the client')
+          junitTestCase.end()
           continue
         }
         log('        - ' + name)
         try {
-          await testRunner.run(setupTest, test[name], teardownTest, stats)
+          await testRunner.run(setupTest, test[name], teardownTest, stats, junitTestCase)
           stats.pass += 1
         } catch (err) {
+          junitTestCase.failure(err)
+          junitTestCase.end()
+          junitTestSuite.end()
+          junitTestSuites.end()
+          generateJunitXmlReport(junit, isXPack ? 'xpack' : 'oss')
           console.error(err)
           process.exit(1)
         }
         const totalTestTime = now() - testTime
+        junitTestCase.end()
         if (totalTestTime > MAX_TEST_TIME) {
           log('          took too long: ' + ms(totalTestTime))
         } else {
           log('          took: ' + ms(totalTestTime))
         }
       }
+      junitTestSuite.end()
       const totalFileTime = now() - fileTime
       if (totalFileTime > MAX_FILE_TIME) {
         log(`    ${cleanPath} took too long: ` + ms(totalFileTime))
@@ -235,6 +250,8 @@ async function start ({ client, isXPack }) {
       log(`${apiName} took: ` + ms(totalApiTime))
     }
   }
+  junitTestSuites.end()
+  generateJunitXmlReport(junit, isXPack ? 'xpack' : 'oss')
   log(`Total testing time: ${ms(now() - totalTime)}`)
   log(`Test stats:
   - Total: ${stats.total}
@@ -359,6 +376,13 @@ function createFolder (name) {
   }
 }
 
+function generateJunitXmlReport (junit, suite) {
+  writeFileSync(
+    join(__dirname, '..', '..', `${suite}-report-junit.xml`),
+    junit.prettyPrint()
+  )
+}
+
 if (require.main === module) {
   const node = process.env.TEST_ES_SERVER || 'http://localhost:9200'
   const opts = {

test/integration/reporter.js

@@ -0,0 +1,109 @@
+'use strict'
+
+const assert = require('assert')
+const { create } = require('xmlbuilder2')
+
+function createJunitReporter () {
+  const report = {}
+
+  return { testsuites, prettyPrint }
+
+  function prettyPrint () {
+    return create(report).end({ prettyPrint: true })
+  }
+
+  function testsuites (name) {
+    assert(name, 'The testsuites name is required')
+    assert(report.testsuites === undefined, 'Cannot set more than one testsuites block')
+    const startTime = Date.now()
+
+    report.testsuites = {
+      '@id': new Date().toISOString(),
+      '@name': name
+    }
+
+    const testsuiteList = []
+
+    return {
+      testsuite: createTestSuite(testsuiteList),
+      end () {
+        report.testsuites['@time'] = Math.round((Date.now() - startTime) / 1000)
+        report.testsuites['@tests'] = testsuiteList.reduce((acc, val) => {
+          acc += val['@tests']
+          return acc
+        }, 0)
+        report.testsuites['@failures'] = testsuiteList.reduce((acc, val) => {
+          acc += val['@failures']
+          return acc
+        }, 0)
+        report.testsuites['@skipped'] = testsuiteList.reduce((acc, val) => {
+          acc += val['@skipped']
+          return acc
+        }, 0)
+        if (testsuiteList.length) {
+          report.testsuites.testsuite = testsuiteList
+        }
+      }
+    }
+  }
+
+  function createTestSuite (testsuiteList) {
+    return function testsuite (name) {
+      assert(name, 'The testsuite name is required')
+      const startTime = Date.now()
+      const suite = {
+        '@id': new Date().toISOString(),
+        '@name': name
+      }
+      const testcaseList = []
+      testsuiteList.push(suite)
+      return {
+        testcase: createTestCase(testcaseList),
+        end () {
+          suite['@time'] = Math.round((Date.now() - startTime) / 1000)
+          suite['@tests'] = testcaseList.length
+          suite['@failures'] = testcaseList.filter(t => t.failure).length
+          suite['@skipped'] = testcaseList.filter(t => t.skipped).length
+          if (testcaseList.length) {
+            suite.testcase = testcaseList
+          }
+        }
+      }
+    }
+  }
+
+  function createTestCase (testcaseList) {
+    return function testcase (name) {
+      assert(name, 'The testcase name is required')
+      const startTime = Date.now()
+      const tcase = {
+        '@id': new Date().toISOString(),
+        '@name': name
+      }
+      testcaseList.push(tcase)
+      return {
+        failure (error) {
+          assert(error, 'The failure error object is required')
+          tcase.failure = {
+            '#': error.stack,
+            '@message': error.message,
+            '@type': error.code
+          }
+        },
+        skip (reason) {
+          if (typeof reason !== 'string') {
+            reason = JSON.stringify(reason, null, 2)
+          }
+          tcase.skipped = {
+            '#': reason
+          }
+        },
+        end () {
+          tcase['@time'] = Math.round((Date.now() - startTime) / 1000)
+        }
+      }
+    }
+  }
+}
+
+module.exports = createJunitReporter

test/integration/test-runner.js

@@ -218,11 +218,12 @@ function build (opts = {}) {
    * @oaram {object} teardown (null if not needed)
    * @returns {Promise}
    */
-  async function run (setup, test, teardown, stats) {
+  async function run (setup, test, teardown, stats, junit) {
     // if we should skip a feature in the setup/teardown section
     // we should skip the entire test file
     const skip = getSkip(setup) || getSkip(teardown)
     if (skip && shouldSkip(esVersion, skip)) {
+      junit.skip(skip)
       logSkip(skip)
       return
     }
@@ -240,11 +241,11 @@ function build (opts = {}) {
       }
     }
 
-    if (setup) await exec('Setup', setup, stats)
+    if (setup) await exec('Setup', setup, stats, junit)
 
-    await exec('Test', test, stats)
+    await exec('Test', test, stats, junit)
 
-    if (teardown) await exec('Teardown', teardown, stats)
+    if (teardown) await exec('Teardown', teardown, stats, junit)
 
     if (isXPack) await cleanupXPack()
 
@@ -451,11 +452,12 @@ function build (opts = {}) {
    * @param {object} the actions to perform
    * @returns {Promise}
    */
-  async function exec (name, actions, stats) {
+  async function exec (name, actions, stats, junit) {
     // tap.comment(name)
     for (const action of actions) {
       if (action.skip) {
         if (shouldSkip(esVersion, action.skip)) {
+          junit.skip(fillStashedValues(action.skip))
           logSkip(fillStashedValues(action.skip))
           break
         }