move apps lower in tree, add metricbeat dashboard screenshot test (#7…

…9001)

test/functional/services/common/failure_debugging.ts

@@ -38,7 +38,7 @@ export async function FailureDebuggingProvider({ getService }: FtrProviderContex
   const log = getService('log');
   const browser = getService('browser');
 
-  if (process.env.CI !== 'true') {
+  if (process.env.CI !== 'true' && !process.env.stack_functional_integration) {
     await del(config.get('failureDebugging.htmlDirectory'));
   }
 

test/functional/services/common/screenshots.ts

@@ -40,7 +40,7 @@ export async function ScreenshotsProvider({ getService }: FtrProviderContext) {
   const FAILURE_DIRECTORY = resolve(config.get('screenshots.directory'), 'failure');
   const BASELINE_DIRECTORY = resolve(config.get('screenshots.directory'), 'baseline');
 
-  if (process.env.CI !== 'true') {
+  if (process.env.CI !== 'true' && !process.env.stack_functional_integration) {
     await del([SESSION_DIRECTORY, FAILURE_DIRECTORY]);
   }
 

test/functional/services/common/snapshots.ts

@@ -35,7 +35,7 @@ export async function SnapshotsProvider({ getService }: FtrProviderContext) {
   const SESSION_DIRECTORY = resolve(config.get('snapshots.directory'), 'session');
   const BASELINE_DIRECTORY = resolve(config.get('snapshots.directory'), 'baseline');
 
-  if (process.env.CI !== 'true') {
+  if (process.env.CI !== 'true' && !process.env.stack_functional_integration) {
     await del([SESSION_DIRECTORY]);
   }
 

x-pack/test/stack_functional_integration/apps/metricbeat/_metricbeat_dashboard.js

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import { REPO_ROOT } from '@kbn/dev-utils';
+
+export default function ({ getService, getPageObjects, updateBaselines }) {
+  const screenshot = getService('screenshots');
+  const browser = getService('browser');
+  const esArchiver = getService('esArchiver');
+  const PageObjects = getPageObjects(['common', 'dashboard', 'timePicker']);
+
+  describe('check metricbeat Dashboard', function () {
+    before(async function () {
+      await esArchiver.load(`${REPO_ROOT}/../integration-test/test/es_archives/metricbeat`);
+
+      // this navigateToActualURL takes the place of navigating to the dashboard landing page,
+      // filtering on the dashboard name, selecting it, setting the timepicker, and going to full screen
+      await PageObjects.common.navigateToActualUrl(
+        'dashboard',
+        'view/Metricbeat-system-overview-ecs?_g=(filters:!(),refreshInterval:(pause:!t,value:0),' +
+          'time:(from:%272020-09-29T19:02:37.902Z%27,to:%272020-09-29T19:06:43.218Z%27))&_a=' +
+          '(description:%27Overview%20of%20system%20metrics%27,filters:!(),fullScreenMode:!t,' +
+          'options:(darkTheme:!f),query:(language:kuery,query:%27%27),timeRestore:!f,' +
+          'title:%27%5BMetricbeat%20System%5D%20Overview%20ECS%27,viewMode:view)',
+        {
+          ensureCurrentUrl: false,
+          shouldLoginIfPrompted: true,
+        }
+      );
+      // await PageObjects.common.navigateToApp('dashboard', { insertTimestamp: false });
+      // await PageObjects.dashboard.loadSavedDashboard('[Metricbeat System] Overview ECS');
+      // await PageObjects.timePicker.setAbsoluteRange(
+      //   'Sep 29, 2020 @ 14:02:37.902',
+      //   'Sep 29, 2020 @ 14:06:43.218'
+      // );
+      // await PageObjects.dashboard.clickFullScreenMode();
+
+      await PageObjects.common.sleep(2000);
+      await PageObjects.dashboard.waitForRenderComplete();
+      await browser.setScreenshotSize(1000, 1000);
+    });
+
+    it('[Metricbeat System] Overview ECS should match snapshot', async function () {
+      try {
+        const percentDifference = await screenshot.compareAgainstBaseline(
+          'metricbeat_dashboard',
+          updateBaselines
+        );
+        expect(percentDifference).to.be.lessThan(0.01);
+      } finally {
+        await PageObjects.dashboard.clickExitFullScreenLogoButton();
+      }
+    });
+  });
+}

x-pack/test/stack_functional_integration/test/functional/apps/metricbeat/index.js → x-pack/test/stack_functional_integration/apps/metricbeat/index.js

@@ -7,5 +7,6 @@
 export default function ({ loadTestFile }) {
   describe('metricbeat app', function () {
     loadTestFile(require.resolve('./_metricbeat'));
+    loadTestFile(require.resolve('./_metricbeat_dashboard'));
   });
 }

x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js

@@ -6,26 +6,27 @@
 
 import { resolve } from 'path';
 import buildState from './build_state';
-import { ToolingLog } from '@kbn/dev-utils';
+import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils';
 import chalk from 'chalk';
 import { esTestConfig, kbnTestConfig } from '@kbn/test';
 
 const reportName = 'Stack Functional Integration Tests';
-const testsFolder = '../test/functional/apps';
+const testsFolder = '../apps';
 const log = new ToolingLog({
   level: 'info',
   writeTo: process.stdout,
 });
 log.info(`WORKSPACE in config file ${process.env.WORKSPACE}`);
 const stateFilePath = process.env.WORKSPACE
   ? `${process.env.WORKSPACE}/qa/envvars.sh`
-  : '../../../../../integration-test/qa/envvars.sh';
+  : `${REPO_ROOT}/../integration-test/qa/envvars.sh`;
 
 const prepend = (testFile) => require.resolve(`${testsFolder}/${testFile}`);
 
 export default async ({ readConfigFile }) => {
   const defaultConfigs = await readConfigFile(require.resolve('../../functional/config'));
   const { tests, ...provisionedConfigs } = buildState(resolve(__dirname, stateFilePath));
+  process.env.stack_functional_integration = true;
 
   const servers = {
     kibana: kbnTestConfig.getUrlParts(),
@@ -43,6 +44,14 @@ export default async ({ readConfigFile }) => {
     // If we need to do things like disable animations, we can do it in configure_start_kibana.sh, in the provisioner...which lives in the integration-test private repo
     uiSettings: {},
     security: { disableTestUser: true },
+    // choose where screenshots should be saved
+    screenshots: {
+      directory: resolve(`${REPO_ROOT}/../integration-test`, 'test/screenshots'),
+    },
+    // choose where esArchiver should load archives from
+    esArchiver: {
+      directory: resolve(`${REPO_ROOT}/../integration-test`, 'test/es_archives'),
+    },
   };
   return settings;
 };
@@ -55,7 +64,7 @@ function truncate(testPath) {
   return dropKibanaPath(testPath);
 }
 function highLight(testPath) {
-  const dropTestsPath = splitRight(/^.+test[\\/]functional[\\/]apps[\\/](.*)[\\/]/gm);
+  const dropTestsPath = splitRight(/^.+apps[\\/](.*)[\\/]/gm);
   const cleaned = dropTestsPath(testPath);
   const colored = chalk.greenBright.bold(cleaned);
   return testPath.replace(cleaned, colored);

x-pack/test/stack_functional_integration/configs/tests_list.js

@@ -20,7 +20,11 @@ export default (envObj) => {
   }
 
   if (envObj.BEATS.includes('metricbeat')) {
-    xs.push('metricbeat');
+    xs.push('metricbeat/_metricbeat');
+    if (envObj.XPACK === 'YES') {
+      // the esArchive and dashboard png are specific to the default distribution (with XPACK)
+      xs.push('metricbeat/_metricbeat_dashboard');
+    }
   }
   if (envObj.BEATS.includes('filebeat')) {
     xs.push('filebeat');