From e239b141146168d9dcf6daad620b96eac3011366 Mon Sep 17 00:00:00 2001 From: Stacey Gammon Date: Mon, 15 Jul 2019 17:42:07 -0400 Subject: [PATCH 1/3] turn on filtering tests --- test/functional/apps/dashboard/dashboard_filtering.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/functional/apps/dashboard/dashboard_filtering.js b/test/functional/apps/dashboard/dashboard_filtering.js index af865226465681..fb32de7832b4d3 100644 --- a/test/functional/apps/dashboard/dashboard_filtering.js +++ b/test/functional/apps/dashboard/dashboard_filtering.js @@ -217,8 +217,7 @@ export default function ({ getService, getPageObjects }) { await dashboardExpect.tsvbMarkdownWithValuesExists(['7,209.286']); }); - // FLAKY: https://github.com/elastic/kibana/issues/41087 - it.skip('saved searches', async () => { + it('saved searches', async () => { await dashboardExpect.savedSearchRowCount(1); }); From 5a4a590e1053f03e668224d82984212497b256b5 Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Jul 2019 06:25:32 -0700 Subject: [PATCH 2/3] run x-pack-firefoxSmoke ciGroup 40 times, run dashboard tests 20 times per job --- .ci/jobs.yml | 70 +++++++++++-------- .../lib/mocha/setup_mocha.js | 6 +- test/functional/config.js | 41 ++++++++--- 3 files changed, 74 insertions(+), 43 deletions(-) diff --git a/.ci/jobs.yml b/.ci/jobs.yml index fc3e80064fa6fa..f6394c1fe779f4 100644 --- a/.ci/jobs.yml +++ b/.ci/jobs.yml @@ -1,34 +1,44 @@ JOB: - - kibana-intake - - x-pack-intake - - kibana-firefoxSmoke - - kibana-ciGroup1 - - kibana-ciGroup2 - - kibana-ciGroup3 - - kibana-ciGroup4 - - kibana-ciGroup5 - - kibana-ciGroup6 - - kibana-ciGroup7 - - kibana-ciGroup8 - - kibana-ciGroup9 - - kibana-ciGroup10 - - kibana-ciGroup11 - - kibana-ciGroup12 - # - kibana-visualRegression - - # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh - - x-pack-firefoxSmoke - - x-pack-ciGroup1 - - x-pack-ciGroup2 - - x-pack-ciGroup3 - - x-pack-ciGroup4 - - x-pack-ciGroup5 - - x-pack-ciGroup6 - - x-pack-ciGroup7 - - x-pack-ciGroup8 - - x-pack-ciGroup9 - - x-pack-ciGroup10 - # - x-pack-visualRegression + - x-pack-firefoxSmoke-0 + - x-pack-firefoxSmoke-1 + - x-pack-firefoxSmoke-2 + - x-pack-firefoxSmoke-3 + - x-pack-firefoxSmoke-4 + - x-pack-firefoxSmoke-5 + - x-pack-firefoxSmoke-6 + - x-pack-firefoxSmoke-7 + - x-pack-firefoxSmoke-8 + - x-pack-firefoxSmoke-9 + - x-pack-firefoxSmoke-10 + - x-pack-firefoxSmoke-11 + - x-pack-firefoxSmoke-12 + - x-pack-firefoxSmoke-13 + - x-pack-firefoxSmoke-14 + - x-pack-firefoxSmoke-15 + - x-pack-firefoxSmoke-16 + - x-pack-firefoxSmoke-17 + - x-pack-firefoxSmoke-18 + - x-pack-firefoxSmoke-19 + - x-pack-firefoxSmoke-20 + - x-pack-firefoxSmoke-21 + - x-pack-firefoxSmoke-22 + - x-pack-firefoxSmoke-23 + - x-pack-firefoxSmoke-24 + - x-pack-firefoxSmoke-25 + - x-pack-firefoxSmoke-26 + - x-pack-firefoxSmoke-27 + - x-pack-firefoxSmoke-28 + - x-pack-firefoxSmoke-29 + - x-pack-firefoxSmoke-30 + - x-pack-firefoxSmoke-31 + - x-pack-firefoxSmoke-32 + - x-pack-firefoxSmoke-33 + - x-pack-firefoxSmoke-34 + - x-pack-firefoxSmoke-35 + - x-pack-firefoxSmoke-36 + - x-pack-firefoxSmoke-37 + - x-pack-firefoxSmoke-38 + - x-pack-firefoxSmoke-39 # `~` is yaml for `null` exclude: ~ diff --git a/src/functional_test_runner/lib/mocha/setup_mocha.js b/src/functional_test_runner/lib/mocha/setup_mocha.js index c39db58cf9a5de..d8ba2721041b5d 100644 --- a/src/functional_test_runner/lib/mocha/setup_mocha.js +++ b/src/functional_test_runner/lib/mocha/setup_mocha.js @@ -60,8 +60,10 @@ export async function setupMocha(lifecycle, log, config, providers) { filterSuitesByTags({ log, mocha, - include: config.get('suiteTags.include'), - exclude: config.get('suiteTags.exclude'), + include: config.get('suiteTags.include') + .map(tag => tag.replace(/-\d+$/, '')), + exclude: config.get('suiteTags.exclude') + .map(tag => tag.replace(/-\d+$/, '')) }); return mocha; diff --git a/test/functional/config.js b/test/functional/config.js index 1835808763ae18..5a271b95012b17 100644 --- a/test/functional/config.js +++ b/test/functional/config.js @@ -25,17 +25,36 @@ export default async function ({ readConfigFile }) { return { testFiles: [ - require.resolve('./apps/console'), - require.resolve('./apps/getting_started'), - require.resolve('./apps/context'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/discover'), - require.resolve('./apps/home'), - require.resolve('./apps/management'), - require.resolve('./apps/status_page'), - require.resolve('./apps/timelion'), - require.resolve('./apps/visualize'), - require.resolve('./apps/xpack'), + // require.resolve('./apps/console'), + // require.resolve('./apps/getting_started'), + // require.resolve('./apps/context'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/dashboard'), + // require.resolve('./apps/discover'), + // require.resolve('./apps/home'), + // require.resolve('./apps/management'), + // require.resolve('./apps/status_page'), + // require.resolve('./apps/timelion'), + // require.resolve('./apps/visualize'), + // require.resolve('./apps/xpack'), ], pageObjects, services, From 2de4d86f2826a2b4b901e98281e293bb15a99a13 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 18 Jul 2019 07:01:14 -0700 Subject: [PATCH 3/3] Revert "run x-pack-firefoxSmoke ciGroup 40 times, run dashboard tests 20 times per job" This reverts commit 5ef02cc53ba5085c3f74431cb6ef20be2d876cde. --- .ci/jobs.yml | 70 ++++++++----------- .../lib/mocha/setup_mocha.js | 6 +- test/functional/config.js | 41 +++-------- 3 files changed, 43 insertions(+), 74 deletions(-) diff --git a/.ci/jobs.yml b/.ci/jobs.yml index f6394c1fe779f4..fc3e80064fa6fa 100644 --- a/.ci/jobs.yml +++ b/.ci/jobs.yml @@ -1,44 +1,34 @@ JOB: - - x-pack-firefoxSmoke-0 - - x-pack-firefoxSmoke-1 - - x-pack-firefoxSmoke-2 - - x-pack-firefoxSmoke-3 - - x-pack-firefoxSmoke-4 - - x-pack-firefoxSmoke-5 - - x-pack-firefoxSmoke-6 - - x-pack-firefoxSmoke-7 - - x-pack-firefoxSmoke-8 - - x-pack-firefoxSmoke-9 - - x-pack-firefoxSmoke-10 - - x-pack-firefoxSmoke-11 - - x-pack-firefoxSmoke-12 - - x-pack-firefoxSmoke-13 - - x-pack-firefoxSmoke-14 - - x-pack-firefoxSmoke-15 - - x-pack-firefoxSmoke-16 - - x-pack-firefoxSmoke-17 - - x-pack-firefoxSmoke-18 - - x-pack-firefoxSmoke-19 - - x-pack-firefoxSmoke-20 - - x-pack-firefoxSmoke-21 - - x-pack-firefoxSmoke-22 - - x-pack-firefoxSmoke-23 - - x-pack-firefoxSmoke-24 - - x-pack-firefoxSmoke-25 - - x-pack-firefoxSmoke-26 - - x-pack-firefoxSmoke-27 - - x-pack-firefoxSmoke-28 - - x-pack-firefoxSmoke-29 - - x-pack-firefoxSmoke-30 - - x-pack-firefoxSmoke-31 - - x-pack-firefoxSmoke-32 - - x-pack-firefoxSmoke-33 - - x-pack-firefoxSmoke-34 - - x-pack-firefoxSmoke-35 - - x-pack-firefoxSmoke-36 - - x-pack-firefoxSmoke-37 - - x-pack-firefoxSmoke-38 - - x-pack-firefoxSmoke-39 + - kibana-intake + - x-pack-intake + - kibana-firefoxSmoke + - kibana-ciGroup1 + - kibana-ciGroup2 + - kibana-ciGroup3 + - kibana-ciGroup4 + - kibana-ciGroup5 + - kibana-ciGroup6 + - kibana-ciGroup7 + - kibana-ciGroup8 + - kibana-ciGroup9 + - kibana-ciGroup10 + - kibana-ciGroup11 + - kibana-ciGroup12 + # - kibana-visualRegression + + # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh + - x-pack-firefoxSmoke + - x-pack-ciGroup1 + - x-pack-ciGroup2 + - x-pack-ciGroup3 + - x-pack-ciGroup4 + - x-pack-ciGroup5 + - x-pack-ciGroup6 + - x-pack-ciGroup7 + - x-pack-ciGroup8 + - x-pack-ciGroup9 + - x-pack-ciGroup10 + # - x-pack-visualRegression # `~` is yaml for `null` exclude: ~ diff --git a/src/functional_test_runner/lib/mocha/setup_mocha.js b/src/functional_test_runner/lib/mocha/setup_mocha.js index d8ba2721041b5d..c39db58cf9a5de 100644 --- a/src/functional_test_runner/lib/mocha/setup_mocha.js +++ b/src/functional_test_runner/lib/mocha/setup_mocha.js @@ -60,10 +60,8 @@ export async function setupMocha(lifecycle, log, config, providers) { filterSuitesByTags({ log, mocha, - include: config.get('suiteTags.include') - .map(tag => tag.replace(/-\d+$/, '')), - exclude: config.get('suiteTags.exclude') - .map(tag => tag.replace(/-\d+$/, '')) + include: config.get('suiteTags.include'), + exclude: config.get('suiteTags.exclude'), }); return mocha; diff --git a/test/functional/config.js b/test/functional/config.js index 5a271b95012b17..1835808763ae18 100644 --- a/test/functional/config.js +++ b/test/functional/config.js @@ -25,36 +25,17 @@ export default async function ({ readConfigFile }) { return { testFiles: [ - // require.resolve('./apps/console'), - // require.resolve('./apps/getting_started'), - // require.resolve('./apps/context'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - require.resolve('./apps/dashboard'), - // require.resolve('./apps/discover'), - // require.resolve('./apps/home'), - // require.resolve('./apps/management'), - // require.resolve('./apps/status_page'), - // require.resolve('./apps/timelion'), - // require.resolve('./apps/visualize'), - // require.resolve('./apps/xpack'), + require.resolve('./apps/console'), + require.resolve('./apps/getting_started'), + require.resolve('./apps/context'), + require.resolve('./apps/dashboard'), + require.resolve('./apps/discover'), + require.resolve('./apps/home'), + require.resolve('./apps/management'), + require.resolve('./apps/status_page'), + require.resolve('./apps/timelion'), + require.resolve('./apps/visualize'), + require.resolve('./apps/xpack'), ], pageObjects, services,