feat: added test suites (#22591)

feat: added test suites

Signed-off-by: mdolhalo <mdolhalo@redhat.com>

tests/e2e/CODE_STYLE.md

@@ -117,7 +117,7 @@ Full set of rules can be found:
 
 2. Mocha framework
 
-    - ✔ TDD framework (`suit()`, `test()`)
+    - ✔ TDD framework (`suite()`, `test()`)
     - ✔ Inject class instances, declare all test data inside test `suit()` function to avoid unnecessary code execution if test suit will not be run
 
     ```

tests/e2e/configs/inversify.config.ts

@@ -9,7 +9,7 @@
  **********************************************************************/
 
 import 'reflect-metadata';
-import { Container, decorate, injectable, unmanaged } from 'inversify';
+import { Container } from 'inversify';
 import { IDriver } from '../driver/IDriver';
 import { ChromeDriver } from '../driver/ChromeDriver';
 import { CLASSES, EXTERNAL_CLASSES, TYPES } from './inversify.types';
@@ -52,12 +52,6 @@ import { Main as Generator } from '@eclipse-che/che-devworkspace-generator/lib/m
 import { ContainerTerminal, KubernetesCommandLineToolsExecutor } from '../utils/KubernetesCommandLineToolsExecutor';
 import { ShellExecutor } from '../utils/ShellExecutor';
 
-decorate(injectable(), Generator);
-decorate(injectable(), LocatorLoader);
-decorate(unmanaged(), LocatorLoader, 0);
-decorate(unmanaged(), LocatorLoader, 1);
-decorate(unmanaged(), LocatorLoader, 2);
-
 const e2eContainer: Container = new Container({ defaultScope: 'Transient', skipBaseClassChecks: true });
 
 e2eContainer.bind<IDriver>(TYPES.Driver).to(ChromeDriver).inSingletonScope();

tests/e2e/configs/mocharc.ts

@@ -11,23 +11,27 @@
 'use strict';
 
 import { MOCHA_CONSTANTS } from '../constants/MOCHA_CONSTANTS';
+import { BASE_TEST_CONSTANTS } from '../constants/BASE_TEST_CONSTANTS';
 
 /**
  * full available options list https://mochajs.org/api/mocha
  */
 
 module.exports = {
 	timeout: MOCHA_CONSTANTS.MOCHA_DEFAULT_TIMEOUT,
+	slow: 60000,
 	reporter: 'mocha-multi-reporters',
 	reporterOptions: 'configFile=configs/reporters-config.json',
 	ui: 'tdd',
 	require: ['dist/specs/MochaHooks.js', 'ts-node/register'],
 	bail: MOCHA_CONSTANTS.MOCHA_BAIL,
 	'full-trace': true,
 	spec:
-		// variable MOCHA_DIRECTORY uses in command "test-all-devfiles" and sets up automatically.
-		// you can set it up to run files from specific directory with export environmental variable.
-		MOCHA_CONSTANTS.MOCHA_DIRECTORY
+		BASE_TEST_CONSTANTS.TEST_ENVIRONMENT !== ''
+			? `dist/suites/${MOCHA_CONSTANTS.MOCHA_DIRECTORY}/${MOCHA_CONSTANTS.MOCHA_SUITE}.suite.js`
+			: // variable MOCHA_DIRECTORY uses in command "test-all-devfiles" and sets up automatically.
+			// you can set it up to run files from specific directory with export environmental variable.
+			MOCHA_CONSTANTS.MOCHA_DIRECTORY
 			? // to run one file (name without extension). uses in "test", "test-all-devfiles".
 			  MOCHA_CONSTANTS.MOCHA_USERSTORY
 				? `dist/specs/${MOCHA_CONSTANTS.MOCHA_DIRECTORY}/${MOCHA_CONSTANTS.MOCHA_USERSTORY}.spec.js`

tests/e2e/configs/sh-scripts/runFunctionalTests.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+# shellcheck source=/dev/null
+
+validateParameters(){
+# Validate required parameters
+  if [ -z "$OCP_VERSION" ] || [ -z "$ARCH_VERSION" ] || [ -z "$TS_SELENIUM_BASE_URL" ]; then
+      echo "The ARCH_VERSION, OCP_VERSION or TS_SELENIUM_BASE_URL is not set!";
+      echo "Please, set all required environment variable parameters"
+      exit 1
+  fi
+}
+
+########################################
+############# Methods ##################
+########################################
+
+launchAPITests() {
+  export MOCHA_SUITE="APITest"
+  echo "MOCHA_SUITE = ${MOCHA_SUITE}"
+  export RP_LAUNCH_NAME="API tests suite"
+  echo "suites/$MOCHA_DIRECTORY/$MOCHA_SUITE"
+  npm run driver-less-test
+}
+
+launchDynamicallyGeneratingAPITests() {
+  export MOCHA_SUITE="DynamicallyGeneratingAPITest"
+  export RP_LAUNCH_NAME="Application inbuilt DevWorkspaces API tests suite"
+  echo "MOCHA_SUITE = ${MOCHA_SUITE}"
+  echo "suites/$MOCHA_DIRECTORY/$MOCHA_SUITE"
+  npm run delayed-test
+}
+
+launchUITests() {
+  export MOCHA_SUITE="UITest"
+  export RP_LAUNCH_NAME="UI tests suite"
+  echo "MOCHA_SUITE = ${MOCHA_SUITE}"
+  echo "suites/$MOCHA_DIRECTORY/$MOCHA_SUITE"
+  npm run test
+}
+
+launchAllTests() {
+  validateParameters
+  initTestValues
+  echo ""
+  echo "Launching all tests for $TEST_ENVIRONMENT"
+  echo ""
+  launchDynamicallyGeneratingAPITests
+  launchAPITests
+  launchUITests
+}
+
+initTestValues() {
+  if [[ "$TS_SELENIUM_BASE_URL" =~ "airgap" || (-n "$IS_CLUSTER_DISCONNECTED" && "$IS_CLUSTER_DISCONNECTED" == "true") ]]
+  then
+     echo "Disconnected environment"
+     export MOCHA_DIRECTORY="disconnected-ocp"
+  else
+     echo "Online environment"
+     export MOCHA_DIRECTORY="online-ocp"
+  fi
+
+  export TEST_ENVIRONMENT="$ARCH_VERSION $MOCHA_DIRECTORY $OCP_VERSION"
+  export DELETE_WORKSPACE_ON_FAILED_TEST=${DELETE_WORKSPACE_ON_FAILED_TEST:-'false'}
+  export DELETE_SCREENCAST_IF_TEST_PASS=${DELETE_SCREENCAST_IF_TEST_PASS:-'true'}
+  export NODE_TLS_REJECT_UNAUTHORIZED=${NODE_TLS_REJECT_UNAUTHORIZED:-'0'}
+  export TS_OCP_LOGIN_PAGE_PROVIDER_TITLE=${TS_OCP_LOGIN_PAGE_PROVIDER_TITLE:-'htpasswd'}
+  export TS_SELENIUM_DELAY_BETWEEN_SCREENSHOTS=${TS_SELENIUM_DELAY_BETWEEN_SCREENSHOTS:-'1000'}
+  export TS_SELENIUM_EDITOR=${TS_SELENIUM_EDITOR:-'che-code'}
+  export TS_SELENIUM_EXECUTION_SCREENCAST=${TS_SELENIUM_EXECUTION_SCREENCAST:-'false'}
+  export TS_SELENIUM_HEADLESS=${TS_SELENIUM_HEADLESS:-'false'}
+  export TS_SELENIUM_LAUNCH_FULLSCREEN=${TS_SELENIUM_LAUNCH_FULLSCREEN:-'true'}
+  export TS_SELENIUM_LOG_LEVEL=${TS_SELENIUM_LOG_LEVEL:-'TRACE'}
+  export TS_SELENIUM_OCP_PASSWORD=${TS_SELENIUM_OCP_PASSWORD:-''}
+  export TS_SELENIUM_OCP_USERNAME=${TS_SELENIUM_OCP_USERNAME:-'admin'}
+  export TS_SELENIUM_VALUE_OPENSHIFT_OAUTH=${TS_SELENIUM_VALUE_OPENSHIFT_OAUTH:-'true'}
+  export TS_SELENIUM_REPORT_FOLDER=${TS_SELENIUM_REPORT_FOLDER:-'./report'}
+  export MOCHA_BAIL=${MOCHA_BAIL:-'false'}
+  export MOCHA_RETRIES=${MOCHA_RETRIES:-'1'}
+
+  echo "TS_SELENIUM_BASE_URL=${TS_SELENIUM_BASE_URL}"
+  echo "TEST_ENVIRONMENT=${TEST_ENVIRONMENT}"
+  echo "DELETE_WORKSPACE_ON_FAILED_TEST=${DELETE_WORKSPACE_ON_FAILED_TEST}"
+  echo "DELETE_SCREENCAST_IF_TEST_PASS=${DELETE_SCREENCAST_IF_TEST_PASS}"
+  echo "NODE_TLS_REJECT_UNAUTHORIZED=${NODE_TLS_REJECT_UNAUTHORIZED}"
+  echo "TS_OCP_LOGIN_PAGE_PROVIDER_TITLE=${TS_OCP_LOGIN_PAGE_PROVIDER_TITLE}"
+  echo "TS_SELENIUM_DELAY_BETWEEN_SCREENSHOTS=${TS_SELENIUM_DELAY_BETWEEN_SCREENSHOTS}"
+  echo "TS_SELENIUM_EDITOR=${TS_SELENIUM_EDITOR}"
+  echo "TS_SELENIUM_EXECUTION_SCREENCAST=${TS_SELENIUM_EXECUTION_SCREENCAST}"
+  echo "TS_SELENIUM_HEADLESS=${TS_SELENIUM_HEADLESS}"
+  echo "TS_SELENIUM_LAUNCH_FULLSCREEN=${TS_SELENIUM_LAUNCH_FULLSCREEN}"
+  echo "TS_SELENIUM_LOG_LEVEL=${TS_SELENIUM_LOG_LEVEL}"
+  echo "TS_SELENIUM_OCP_USERNAME=${TS_SELENIUM_OCP_USERNAME}"
+  echo "TS_SELENIUM_VALUE_OPENSHIFT_OAUTH=${TS_SELENIUM_VALUE_OPENSHIFT_OAUTH}"
+  echo "TS_SELENIUM_REPORT_FOLDER=${TS_SELENIUM_REPORT_FOLDER}"
+  echo "MOCHA_BAIL=${MOCHA_BAIL}"
+  echo "MOCHA_RETRIES=${MOCHA_RETRIES}"
+}
+
+########################################
+############# Launching ################
+########################################
+
+launchAllTests
+

tests/e2e/constants/API_TEST_CONSTANTS.ts

@@ -23,6 +23,7 @@ export const SUPPORTED_DEVFILE_REGISTRIES: {
 	GIT_HUB_CHE_DEVFILE_REGISTRY_URL: 'https://api.github.com/repos/eclipse-che/che-devfile-registry/contents/devfiles/'
 };
 export const API_TEST_CONSTANTS: {
+	TS_API_TEST_DEV_WORKSPACE_LIST: string | undefined;
 	TS_API_TEST_KUBERNETES_COMMAND_LINE_TOOL: string;
 	TS_API_TEST_PLUGIN_REGISTRY_URL: string | undefined;
 	TS_API_TEST_CHE_CODE_EDITOR_DEVFILE_URI: string | undefined;
@@ -62,6 +63,8 @@ export const API_TEST_CONSTANTS: {
 	 * to run all devfile from registry. used in DevfileAcceptanceTestAPI.suite.ts
 	 */
 	TS_API_ACCEPTANCE_TEST_REGISTRY_URL(): string {
-		return process.env.TS_API_ACCEPTANCE_TEST_REGISTRY_URL || SUPPORTED_DEVFILE_REGISTRIES.INBUILT_APPLICATION_DEVFILE_REGISTRY_URL();
-	}
+		return process.env.TS_API_ACCEPTANCE_TEST_REGISTRY_URL || '';
+	},
+
+	TS_API_TEST_DEV_WORKSPACE_LIST: process.env.TS_API_TEST_DEV_WORKSPACE_LIST || undefined
 };

tests/e2e/constants/BASE_TEST_CONSTANTS.ts

@@ -13,26 +13,44 @@ export enum Platform {
 }
 
 export const BASE_TEST_CONSTANTS: {
+	ARCH_VERSION: string;
+	DELETE_WORKSPACE_ON_FAILED_TEST: boolean;
+	IS_CLUSTER_DISCONNECTED: () => boolean;
 	IS_PRODUCT_DOCUMENTATION_RELEASED: any;
+	OCP_VERSION: string;
 	TESTING_APPLICATION_VERSION: string;
+	TEST_ENVIRONMENT: string;
 	TS_DEBUG_MODE: boolean;
+	TS_LOAD_TESTS: string;
 	TS_PLATFORM: string;
-	TS_SELENIUM_RESPONSE_INTERCEPTOR: boolean;
 	TS_SELENIUM_BASE_URL: string;
-	DELETE_WORKSPACE_ON_FAILED_TEST: boolean;
-	TS_SELENIUM_EDITOR: string;
-	TS_LOAD_TESTS: string;
-	TS_SELENIUM_REQUEST_INTERCEPTOR: boolean;
-	TS_SELENIUM_PROJECT_ROOT_FILE_NAME: string;
 	TS_SELENIUM_DASHBOARD_SAMPLE_NAME: string;
+	TS_SELENIUM_EDITOR: string;
 	TS_SELENIUM_HAPPY_PATH_WORKSPACE_NAME: string;
-	IS_CLUSTER_DISCONNECTED: () => boolean;
+	TS_SELENIUM_PROJECT_ROOT_FILE_NAME: string;
+	TS_SELENIUM_REQUEST_INTERCEPTOR: boolean;
+	TS_SELENIUM_RESPONSE_INTERCEPTOR: boolean;
 } = {
 	/**
 	 * base URL of the application which should be checked
 	 */
 	TS_SELENIUM_BASE_URL: !process.env.TS_SELENIUM_BASE_URL ? 'http://sample-url' : process.env.TS_SELENIUM_BASE_URL.replace(/\/$/, ''),
 
+	/**
+	 * system arch type
+	 */
+	ARCH_VERSION: process.env.ARCH_VERSION || '',
+
+	/**
+	 * openShift version
+	 */
+	OCP_VERSION: process.env.OCP_VERSION || '',
+
+	/**
+	 * test environment (used as prefix in suite name)
+	 */
+	TEST_ENVIRONMENT: process.env.TEST_ENVIRONMENT || '',
+
 	/**
 	 * testing application version
 	 */

tests/e2e/constants/MOCHA_CONSTANTS.ts

@@ -7,8 +7,11 @@
  *
  * SPDX-License-Identifier: EPL-2.0
  **********************************************************************/
+import { BASE_TEST_CONSTANTS } from './BASE_TEST_CONSTANTS';
 
 export const MOCHA_CONSTANTS: {
+	MOCHA_SUITE: string | undefined;
+	MOCHA_DELAYED_SUITE: boolean;
 	MOCHA_DEFAULT_TIMEOUT: string | number;
 	MOCHA_DIRECTORY: string | undefined;
 	MOCHA_USERSTORY: undefined | string;
@@ -21,7 +24,11 @@ export const MOCHA_CONSTANTS: {
 
 	MOCHA_BAIL: process.env.MOCHA_BAIL !== 'false',
 
-	MOCHA_DEFAULT_TIMEOUT: process.env.MOCHA_DEFAULT_TIMEOUT || 1200000,
+	MOCHA_DELAYED_SUITE: process.env.MOCHA_DELAYED_SUITE === 'true',
 
-	MOCHA_RETRIES: process.env.MOCHA_RETRIES || 3
+	MOCHA_DEFAULT_TIMEOUT: process.env.MOCHA_DEFAULT_TIMEOUT || 420000,
+
+	MOCHA_RETRIES: process.env.MOCHA_RETRIES || BASE_TEST_CONSTANTS.TEST_ENVIRONMENT === '' ? 0 : 2,
+
+	MOCHA_SUITE: process.env.MOCHA_SUITE || undefined
 };

tests/e2e/package.json

@@ -9,11 +9,13 @@
 		"tsc": "rm -rf ./dist && ./configs/sh-scripts/generateIndex.sh && tsc -p .",
 		"test": "./configs/sh-scripts/initDefaultValues.sh npm run lint && npm run tsc && export USERSTORY=$USERSTORY && mocha --config dist/configs/mocharc.js",
 		"driver-less-test": "export TS_USE_WEB_DRIVER_FOR_TEST=false && npm run test",
+		"delayed-test": "npm run lint && npm run tsc && export TS_USE_WEB_DRIVER_FOR_TEST=false && export MOCHA_DELAYED_SUITE=true && mocha --config dist/configs/mocharc.js --delay",
 		"open-allure-dasboard": "allure generate .allure-results --clean -o .allure-report && allure open .allure-report",
 		"cleanup-docker": "if [ $(docker ps -a | grep -c selenium-e2e) -gt 0 ]; then docker rm -f $(docker ps --filter \"name=selenium-e2e\" -aq); fi;",
 		"test-docker": "npm run cleanup-docker && docker run -it --shm-size=2g -p 5920:5920 --name selenium-e2e -e TS_SELENIUM_BASE_URL=$TS_SELENIUM_BASE_URL eclipse/che-e2e:nightly",
 		"test-docker-mount-e2e": "npm run cleanup-docker && docker run -it --shm-size=2g -p 5920:5920 --name selenium-e2e -e TS_SELENIUM_BASE_URL=$TS_SELENIUM_BASE_URL -v $(pwd):/tmp/e2e:Z eclipse/che-e2e:nightly",
 		"test-all-devfiles": " ./configs/sh-scripts/initDefaultValues.sh && ./configs/sh-scripts/initDevfileTests.sh",
+		"functional-test-suite": "configs/sh-scripts/runFunctionalTests.sh",
 		"devfile-acceptance-test-suite": "./configs/sh-scripts/initDefaultValues.sh npm run lint && npm run tsc && export TS_USE_WEB_DRIVER_FOR_TEST=false && mocha 'dist/specs/api/*.js' --config dist/configs/mocharc.js --delay --grep 'Devfile acceptance test suite'"
 	},
 	"author": "Ihor Okhrimenko (iokhrime@redhat.com)",

tests/e2e/specs/MochaHooks.ts

@@ -22,6 +22,9 @@ import { allure } from 'allure-mocha/runtime';
 import { BASE_TEST_CONSTANTS } from '../constants/BASE_TEST_CONSTANTS';
 import { MONACO_CONSTANTS } from '../constants/MONACO_CONSTANTS';
 import { CHROME_DRIVER_CONSTANTS } from '../constants/CHROME_DRIVER_CONSTANTS';
+import { decorate, injectable, unmanaged } from 'inversify';
+import { Main } from '@eclipse-che/che-devworkspace-generator/lib/main';
+import { LocatorLoader } from 'monaco-page-objects/out/locators/loader';
 
 const driverHelper: DriverHelper = e2eContainer.get(CLASSES.DriverHelper);
 let latestWorkspace: string = '';
@@ -34,6 +37,14 @@ export function registerRunningWorkspace(workspaceName: string): void {
 
 exports.mochaHooks = {
 	beforeAll: [
+		function decorateExternalClasses(): void {
+			decorate(injectable(), Main);
+			decorate(injectable(), LocatorLoader);
+			decorate(unmanaged(), LocatorLoader, 0);
+			decorate(unmanaged(), LocatorLoader, 1);
+			decorate(unmanaged(), LocatorLoader, 2);
+		},
+
 		function enableRequestInterceptor(): void {
 			if (BASE_TEST_CONSTANTS.TS_SELENIUM_REQUEST_INTERCEPTOR) {
 				CheApiRequestHandler.enableRequestInterceptor();

tests/e2e/specs/api/ContainerOverridesAPI.spec.ts

@@ -15,8 +15,9 @@ import { expect } from 'chai';
 import { ShellExecutor } from '../../utils/ShellExecutor';
 import { e2eContainer } from '../../configs/inversify.config';
 import { CLASSES } from '../../configs/inversify.types';
+import { BASE_TEST_CONSTANTS } from '../../constants/BASE_TEST_CONSTANTS';
 
-suite('Test defining container overrides via attribute.', function (): void {
+suite(`Test defining container overrides via attribute ${BASE_TEST_CONSTANTS.TEST_ENVIRONMENT}`, function (): void {
 	const pathToSampleFile: string = path.resolve('resources/container-overrides.yaml');
 	const workspaceName: string = YAML.parse(fs.readFileSync(pathToSampleFile, 'utf8')).metadata.name;
 	const kubernetesCommandLineToolsExecutor: KubernetesCommandLineToolsExecutor = e2eContainer.get(

tests/e2e/specs/api/DevfileAcceptanceTestAPI.spec.ts

@@ -16,21 +16,23 @@ import { StringUtil } from '../../utils/StringUtil';
 import { Logger } from '../../utils/Logger';
 import { e2eContainer } from '../../configs/inversify.config';
 import { CLASSES } from '../../configs/inversify.types';
+import { DevfilesRegistryHelper } from '../../utils/DevfilesRegistryHelper';
+import { MOCHA_CONSTANTS } from '../../constants/MOCHA_CONSTANTS';
+import { API_TEST_CONSTANTS } from '../../constants/API_TEST_CONSTANTS';
 
 /**
  * dynamically generating tests
  * info: https://mochajs.org/#delayed-root-suite
  */
-// todo: skipped while don`t use to avoid sending useless requests
-// eslint-disable-next-line @typescript-eslint/require-await
 void (async function (): Promise<void> {
-	// const devfilesRegistryHelper: DevfilesRegistryHelper = e2eContainer.get(CLASSES.DevfilesRegistryHelper);
-
-	const devfileSamples: any = [];
-	// devfileSamples = await devfilesRegistryHelper.collectPathsToDevfilesFromRegistry();
+	const devfilesRegistryHelper: DevfilesRegistryHelper = e2eContainer.get(CLASSES.DevfilesRegistryHelper);
 
+	let devfileSamples: any = [];
+	if (MOCHA_CONSTANTS.MOCHA_DELAYED_SUITE && API_TEST_CONSTANTS.TS_API_ACCEPTANCE_TEST_REGISTRY_URL()) {
+		devfileSamples = await devfilesRegistryHelper.collectPathsToDevfilesFromRegistry(false);
+	}
 	for (const devfileSample of devfileSamples) {
-		suite.skip(`Devfile acceptance test suite for ${devfileSample.name}`, function (): void {
+		suite(`Devfile acceptance test suite for ${devfileSample.name}`, function (): void {
 			this.bail(false);
 			this.timeout(1500000); // 25 minutes because build of Quarkus sample takes 20+ minutes
 			let devWorkspaceConfigurationHelper: DevWorkspaceConfigurationHelper;
@@ -105,7 +107,7 @@ void (async function (): Promise<void> {
 						);
 						Logger.info(`Full build command to be executed: ${commandString}`);
 
-						const output: ShellString = containerTerminal.executeCommand(commandString, command.exec.component);
+						const output: ShellString = containerTerminal.execInContainerCommand(commandString, command.exec.component);
 						expect(output.code).eqls(0);
 					});
 				}

tests/e2e/specs/api/EmptyWorkspaceAPI.spec.ts

@@ -31,7 +31,7 @@ suite('Empty workspace API test', function (): void {
 
 	const gitRepository: string = 'https://github.com/crw-qe/web-nodejs-sample';
 
-	suiteSetup('Create empty workspace with OC client', async function (): Promise<void> {
+	suiteSetup(`Create empty workspace with OC client ${BASE_TEST_CONSTANTS.TEST_ENVIRONMENT}`, async function (): Promise<void> {
 		const workspaceName: string = 'empty-' + Math.floor(Math.random() * 1000);
 		const devfileContent: string = 'schemaVersion: 2.2.0\n' + 'metadata:\n' + `  name: ${workspaceName}\n`;
 
@@ -54,7 +54,7 @@ suite('Empty workspace API test', function (): void {
 		expect(output.stdout).contains('condition met');
 	});
 
-	suite('Clone public repo without previous setup', function (): void {
+	suite(`Clone public repo without previous setup ${BASE_TEST_CONSTANTS.TEST_ENVIRONMENT}`, function (): void {
 		test('Check if public repo can be cloned', function (): void {
 			containerWorkDir = containerTerminal.pwd().stdout.replace('\n', '');
 			const cloneOutput: ShellString = containerTerminal.gitClone(gitRepository);