From 68a76a78e148cac6ef3fe0538ed96bf6b6a6fa76 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 17 Mar 2020 14:23:49 -0400 Subject: [PATCH] [FTR] Add support for --include and --exclude files via tags (#60123) (#60383) --- .../src/functional_test_runner/cli.ts | 18 +++++-- .../lib/config/schema.ts | 13 +++-- .../lib/mocha/decorate_mocha_ui.js | 7 ++- .../lib/mocha/load_test_files.js | 27 +--------- .../lib/mocha/setup_mocha.js | 14 +++++- .../run_tests/__snapshots__/args.test.js.snap | 50 +++++++++++++++++++ .../run_tests/__snapshots__/cli.test.js.snap | 2 + .../functional_tests/cli/run_tests/args.js | 15 ++++++ .../src/functional_tests/lib/run_ftr.js | 6 ++- 9 files changed, 117 insertions(+), 35 deletions(-) diff --git a/packages/kbn-test/src/functional_test_runner/cli.ts b/packages/kbn-test/src/functional_test_runner/cli.ts index 36412961ce75b4..e384936bc6c8ef 100644 --- a/packages/kbn-test/src/functional_test_runner/cli.ts +++ b/packages/kbn-test/src/functional_test_runner/cli.ts @@ -48,12 +48,15 @@ export function runFtrCli() { kbnTestServer: { installDir: parseInstallDir(flags), }, + suiteFiles: { + include: toArray(flags.include as string | string[]).map(makeAbsolutePath), + exclude: toArray(flags.exclude as string | string[]).map(makeAbsolutePath), + }, suiteTags: { include: toArray(flags['include-tag'] as string | string[]), exclude: toArray(flags['exclude-tag'] as string | string[]), }, updateBaselines: flags.updateBaselines, - excludeTestFiles: flags.exclude || undefined, } ); @@ -96,7 +99,15 @@ export function runFtrCli() { }, { flags: { - string: ['config', 'grep', 'exclude', 'include-tag', 'exclude-tag', 'kibana-install-dir'], + string: [ + 'config', + 'grep', + 'include', + 'exclude', + 'include-tag', + 'exclude-tag', + 'kibana-install-dir', + ], boolean: ['bail', 'invert', 'test-stats', 'updateBaselines'], default: { config: 'test/functional/config.js', @@ -107,7 +118,8 @@ export function runFtrCli() { --bail stop tests after the first failure --grep pattern used to select which tests to run --invert invert grep to exclude tests - --exclude=file path to a test file that should not be loaded + --include=file a test file to be included, pass multiple times for multiple files + --exclude=file a test file to be excluded, pass multiple times for multiple files --include-tag=tag a tag to be included, pass multiple times for multiple tags --exclude-tag=tag a tag to be excluded, pass multiple times for multiple tags --test-stats print the number of tests (included and excluded) to STDERR diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index 75623d6c088905..28e8396d0bebae 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -64,9 +64,16 @@ export const schema = Joi.object() testFiles: Joi.array().items(Joi.string()), testRunner: Joi.func(), - excludeTestFiles: Joi.array() - .items(Joi.string()) - .default([]), + suiteFiles: Joi.object() + .keys({ + include: Joi.array() + .items(Joi.string()) + .default([]), + exclude: Joi.array() + .items(Joi.string()) + .default([]), + }) + .default(), suiteTags: Joi.object() .keys({ diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js index 64fc51a04aac9b..1cac852a7e7130 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js @@ -16,7 +16,8 @@ * specific language governing permissions and limitations * under the License. */ - +import { relative } from 'path'; +import { REPO_ROOT } from '@kbn/dev-utils'; import { createAssignmentProxy } from './assignment_proxy'; import { wrapFunction } from './wrap_function'; import { wrapRunnableArgs } from './wrap_runnable_args'; @@ -65,6 +66,10 @@ export function decorateMochaUi(lifecycle, context) { this._tags = [].concat(this._tags || [], tags); }; + const relativeFilePath = relative(REPO_ROOT, this.file); + this.tags(relativeFilePath); + this.suiteTag = relativeFilePath; // The tag that uniquely targets this suite/file + provider.call(this); after(async () => { diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js index 70b0c0874e5e91..6ee65b1b7e3941 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js @@ -31,28 +31,12 @@ import { decorateMochaUi } from './decorate_mocha_ui'; * @param {String} path * @return {undefined} - mutates mocha, no return value */ -export const loadTestFiles = ({ - mocha, - log, - lifecycle, - providers, - paths, - excludePaths, - updateBaselines, -}) => { - const pendingExcludes = new Set(excludePaths.slice(0)); - +export const loadTestFiles = ({ mocha, log, lifecycle, providers, paths, updateBaselines }) => { const innerLoadTestFile = path => { if (typeof path !== 'string' || !isAbsolute(path)) { throw new TypeError('loadTestFile() only accepts absolute paths'); } - if (pendingExcludes.has(path)) { - pendingExcludes.delete(path); - log.warning('Skipping test file %s', path); - return; - } - loadTracer(path, `testFile[${path}]`, () => { log.verbose('Loading test file %s', path); @@ -94,13 +78,4 @@ export const loadTestFiles = ({ }; paths.forEach(innerLoadTestFile); - - if (pendingExcludes.size) { - throw new Error( - `After loading all test files some exclude paths were not consumed:${[ - '', - ...pendingExcludes, - ].join('\n -')}` - ); - } }; diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js index 326877919d9855..61851cece0e8ff 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js @@ -18,6 +18,8 @@ */ import Mocha from 'mocha'; +import { relative } from 'path'; +import { REPO_ROOT } from '@kbn/dev-utils'; import { loadTestFiles } from './load_test_files'; import { filterSuitesByTags } from './filter_suites_by_tags'; @@ -50,10 +52,20 @@ export async function setupMocha(lifecycle, log, config, providers) { lifecycle, providers, paths: config.get('testFiles'), - excludePaths: config.get('excludeTestFiles'), updateBaselines: config.get('updateBaselines'), }); + // Each suite has a tag that is the path relative to the root of the repo + // So we just need to take input paths, make them relative to the root, and use them as tags + // Also, this is a separate filterSuitesByTags() call so that the test suites will be filtered first by + // files, then by tags. This way, you can target tags (like smoke) in a specific file. + filterSuitesByTags({ + log, + mocha, + include: config.get('suiteFiles.include').map(file => relative(REPO_ROOT, file)), + exclude: config.get('suiteFiles.exclude').map(file => relative(REPO_ROOT, file)), + }); + filterSuitesByTags({ log, mocha, diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap index bbf8b38712ac17..434c374d5d23da 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap @@ -16,6 +16,8 @@ Options: --bail Stop the test run at the first failure. --grep Pattern to select which tests to run. --updateBaselines Replace baseline screenshots with whatever is generated from the test. + --include Files that must included to be run, can be included multiple times. + --exclude Files that must NOT be included to be run, can be included multiple times. --include-tag Tags that suites must include to be run, can be included multiple times. --exclude-tag Tags that suites must NOT include to be run, can be included multiple times. --assert-none-excluded Exit with 1/0 based on if any test is excluded with the current set of tags. @@ -34,6 +36,10 @@ Object { "createLogger": [Function], "esFrom": "snapshot", "extraKbnOpts": undefined, + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -52,6 +58,10 @@ Object { "debug": true, "esFrom": "snapshot", "extraKbnOpts": undefined, + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -69,6 +79,10 @@ Object { "createLogger": [Function], "esFrom": "snapshot", "extraKbnOpts": undefined, + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -90,6 +104,10 @@ Object { "extraKbnOpts": Object { "server.foo": "bar", }, + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -107,6 +125,10 @@ Object { "esFrom": "snapshot", "extraKbnOpts": undefined, "quiet": true, + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -124,6 +146,10 @@ Object { "esFrom": "snapshot", "extraKbnOpts": undefined, "silent": true, + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -140,6 +166,10 @@ Object { "createLogger": [Function], "esFrom": "source", "extraKbnOpts": undefined, + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -156,6 +186,10 @@ Object { "createLogger": [Function], "esFrom": "source", "extraKbnOpts": undefined, + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -173,6 +207,10 @@ Object { "esFrom": "snapshot", "extraKbnOpts": undefined, "installDir": "foo", + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -190,6 +228,10 @@ Object { "esFrom": "snapshot", "extraKbnOpts": undefined, "grep": "management", + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -206,6 +248,10 @@ Object { "createLogger": [Function], "esFrom": "snapshot", "extraKbnOpts": undefined, + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], @@ -223,6 +269,10 @@ Object { "createLogger": [Function], "esFrom": "snapshot", "extraKbnOpts": undefined, + "suiteFiles": Object { + "exclude": Array [], + "include": Array [], + }, "suiteTags": Object { "exclude": Array [], "include": Array [], diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap index b12739b3b5df5b..6ede71a6c39402 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap @@ -16,6 +16,8 @@ Options: --bail Stop the test run at the first failure. --grep Pattern to select which tests to run. --updateBaselines Replace baseline screenshots with whatever is generated from the test. + --include Files that must included to be run, can be included multiple times. + --exclude Files that must NOT be included to be run, can be included multiple times. --include-tag Tags that suites must include to be run, can be included multiple times. --exclude-tag Tags that suites must NOT include to be run, can be included multiple times. --assert-none-excluded Exit with 1/0 based on if any test is excluded with the current set of tags. diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js index b34006a38a45d8..7d2414305de8e7 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js @@ -46,6 +46,14 @@ const options = { updateBaselines: { desc: 'Replace baseline screenshots with whatever is generated from the test.', }, + include: { + arg: '', + desc: 'Files that must included to be run, can be included multiple times.', + }, + exclude: { + arg: '', + desc: 'Files that must NOT be included to be run, can be included multiple times.', + }, 'include-tag': { arg: '', desc: 'Tags that suites must include to be run, can be included multiple times.', @@ -115,6 +123,13 @@ export function processOptions(userOptions, defaultConfigPaths) { delete userOptions['kibana-install-dir']; } + userOptions.suiteFiles = { + include: [].concat(userOptions.include || []), + exclude: [].concat(userOptions.exclude || []), + }; + delete userOptions.include; + delete userOptions.exclude; + userOptions.suiteTags = { include: [].concat(userOptions['include-tag'] || []), exclude: [].concat(userOptions['exclude-tag'] || []), diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-test/src/functional_tests/lib/run_ftr.js index 9b631e33f3b24c..14883ac977c431 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_ftr.js +++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.js @@ -22,7 +22,7 @@ import { CliError } from './run_cli'; async function createFtr({ configPath, - options: { installDir, log, bail, grep, updateBaselines, suiteTags }, + options: { installDir, log, bail, grep, updateBaselines, suiteFiles, suiteTags }, }) { const config = await readConfigFile(log, configPath); @@ -37,6 +37,10 @@ async function createFtr({ installDir, }, updateBaselines, + suiteFiles: { + include: [...suiteFiles.include, ...config.get('suiteFiles.include')], + exclude: [...suiteFiles.exclude, ...config.get('suiteFiles.exclude')], + }, suiteTags: { include: [...suiteTags.include, ...config.get('suiteTags.include')], exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')],