[ISSUE-1206]: E2E tests CI/CD job for master (#1205)

Makefile

@@ -7,6 +7,8 @@ include Makefile.validation
 
 .PHONY: version test build
 
+E2E_VM_SERVICE_NODE_IP := $(shell echo $(CLUSTER_IPS) | cut -d',' -f1)
+
 # print version
 version:
 	@printf $(TAG)
@@ -134,3 +136,54 @@ generate-api: compile-proto generate-baremetal-crds generate-deepcopy generate-s
 # Used for UT. Need to regenerate after updating k8s API version
 generate-mocks: install-mockery
 	mockery --dir=/usr/local/go/pkg/mod/k8s.io/client-go\@$(CLIENT_GO_VER)/kubernetes/typed/core/v1/ --name=EventInterface --output=pkg/events/mocks
+
+
+run-csi-baremetal-functional-tests:
+	@echo "Configuring functional tests for csi-baremetal..."; \
+	edited_list=$$(echo ${CLUSTER_IPS} | sed 's/, /", "/g; s/^/"/; s/$$/"/'); \
+	echo "edited_list: $$edited_list"; \
+	sed -i '/parser.addoption("--login", action="store", default=""/s/default=""/default="${USERNAME}"/' ${PROJECT_DIR}/tests/e2e-test-framework/conftest.py; \
+	sed -i '/parser.addoption("--password", action="store", default=""/s/default=""/default="${PASSWORD}"/' ${PROJECT_DIR}/tests/e2e-test-framework/conftest.py; \
+	sed -i '/parser.addoption("--hosts", action="store", default=\[\], help="Hosts")/s/default=\[\],/default=\['"$$edited_list"'\],/' ${PROJECT_DIR}/tests/e2e-test-framework/conftest.py; \
+	sed -i '/parser.addoption("--qtest_token", action="store", default=""/s/default=""/default="${QTEST_API_KEY}"/' ${PROJECT_DIR}/tests/e2e-test-framework/conftest.py; \
+	sed -i '/parser.addoption("--qtest_test_suite", action="store", default=""/s/default=""/default="${QTEST_SUITE_ID}"/' ${PROJECT_DIR}/tests/e2e-test-framework/conftest.py; \
+	sed -i '/parser.addoption("--ansible_server", action="store", default=""/s/default="",/default="${ANSIBLE_SERVER_IP}",/' ${PROJECT_DIR}/tests/e2e-test-framework/conftest.py; \
+	if [ ${REGRESSION_JOB_ENABLE} == "true" && ${SKIP_UPGRADE} == "false" ]; then \
+		sed -i '/parser.addoption("--cmo_bundle_version", action="store", default=""/s/default=""/default="${BUNDLE_VERSION}"/' ${PROJECT_DIR}/tests/e2e-test-framework/conftest.py; \
+	fi; \
+	echo "conftest.py:"; \
+	cat ${PROJECT_DIR}/tests/e2e-test-framework/conftest.py; \
+	echo "Copying test files to remote server..."; \
+	sshpass -p '${PASSWORD}' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${USERNAME}@${E2E_VM_SERVICE_NODE_IP} "mkdir -p /root/tests/e2e"; \
+	sshpass -p '${PASSWORD}' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r ${PROJECT_DIR}/tests/e2e-test-framework ${USERNAME}@${E2E_VM_SERVICE_NODE_IP}:/root/tests/; \
+	echo "Installing dependencies and running tests on remote server..."; \
+	sshpass -p '${PASSWORD}' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${USERNAME}@${E2E_VM_SERVICE_NODE_IP} '. /root/venv/python3.12.2/bin/activate && cd /root/tests/e2e-test-framework && pip3 install -r requirements.txt && pytest -m hal --junitxml=test_results_csi_baremetal.xml ${TEST_FILTER_SMART_INFO}'; \
+	echo "Copying test results back to local machine..."; \
+	sshpass -p '${PASSWORD}' scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r ${USERNAME}@${E2E_VM_SERVICE_NODE_IP}:/root/tests/e2e-test-framework/test_results_csi_baremetal.xml ${PROJECT_DIR}/test_results_csi_baremetal.xml; \
+	TEST_EXIT_CODE=$$?; \
+	echo "Test exit code: $$TEST_EXIT_CODE"; \
+	if [ -e "${PROJECT_DIR}/test_results_csi_baremetal.xml" ]; then \
+		echo "Test results for csi-baremetal copied successfully."; \
+	else \
+		echo "Error: Failed to copy test results for csi-baremetal."; \
+	fi; \
+	if [ $$TEST_EXIT_CODE -eq 0 ]; then \
+		echo "All tests for csi-baremetal passed successfully."; \
+		echo "SUCCESS" > build_status.txt; \
+	else \
+		echo "Functional tests for csi-baremetal failed."; \
+		echo "FAILURE" > build_status.txt; \
+	fi; \
+
+#cleanup test files on remote server
+functional-tests-cleanup:
+	@echo "Cleaning up functional test files on remote server..."; \
+	sshpass -p '${PASSWORD}' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${USERNAME}@${E2E_VM_SERVICE_NODE_IP} 'rm -rf /root/tests/*'; \
+	echo "Functional test cleanup completed."
+
+.PHONY: csi-baremetal-functional-tests
+csi-baremetal-functional-tests: \
+	functional-tests-cleanup \
+	run-csi-baremetal-functional-tests \
+	functional-tests-cleanup
+	@echo "Functional tests for csi-baremetal completed."

tests/e2e-test-framework/conftest.py

@@ -1,15 +1,19 @@
 import logging
 from datetime import datetime
 import pytest
-from framework.description_plugin import DescriptionPlugin
+from framework.test_description_plugin import TestDescriptionPlugin
+from wiremock.testing.testcontainer import wiremock_container
+from wiremock.constants import Config
 from framework.qtest_helper import QTestHelper
-from framework.propagating_thread import PropagatingThread
+from framework.docker_helper import Docker
 import re
+from datetime import datetime
+from framework.propagating_thread import PropagatingThread
 
-@pytest.hookimpl(trylast=True)
+@pytest.mark.trylast
 def pytest_configure(config):
     terminal_reporter = config.pluginmanager.getplugin('terminalreporter')
-    config.pluginmanager.register(DescriptionPlugin(terminal_reporter), 'testdescription')
+    config.pluginmanager.register(TestDescriptionPlugin(terminal_reporter), 'testdescription')
 
     # Configure log file logging
     log_file_suffix = '{:%Y_%m_%d_%H%M%S}.log'.format(datetime.now())
@@ -20,7 +24,7 @@ def pytest_configure(config):
     file_handler.setFormatter(file_formatter)
     logging.getLogger().addHandler(file_handler)
 
-    pytest.qtest_helper = QTestHelper(config.getoption("--qtest_token")) if config.getoption("--qtest_token") else None
+    pytest.qtest_helper = QTestHelper(config.getoption("--qtest_token"), config.getoption("--cmo_bundle_version")) if config.getoption("--qtest_token") else None
 
     pytest.tests_in_suite = {}
     pytest.threads = []
@@ -29,8 +33,11 @@ def pytest_addoption(parser):
     parser.addoption("--login", action="store", default="", help="Login")
     parser.addoption("--password", action="store", default="", help="Password")
     parser.addoption("--namespace", action="store", default="atlantic", help="Namespace")
+    parser.addoption("--hosts", action="store", default=[], help="Hosts")
     parser.addoption("--qtest_token", action="store", default="", help="qTest Token")
+    parser.addoption("--ansible_server", action="store", default="", help="Server")
     parser.addoption("--qtest_test_suite", action="store", default="", help="qTest Test Suite ID")
+    parser.addoption("--cmo_bundle_version", action="store", default="", help="Version of CMO bundle")
 
 def pytest_collection_modifyitems(config):
     qtest_token = config.getoption("--qtest_token")
@@ -51,22 +58,16 @@ def pytest_sessionfinish():
     if len(pytest.threads) == 0:
         return
 
-    suite_failed = False
     for thread in pytest.threads:
-        try:
-            thread.join()
-        except Exception:
-            suite_failed = True
+        thread.join()
 
     logging.info("[qTest] Summary")
     for thread in pytest.threads:
         if thread.has_failed():
             logging.error(f"[qTest] {thread.test_name} {thread.get_target_name()} failed: {thread.exc}")
-        if not thread.has_failed():
+        else:
             logging.info(f"[qTest] {thread.test_name} {thread.get_target_name()} success.")
 
-    assert not suite_failed, "One or more threads failed"
-
 @pytest.fixture(scope="session")
 def vm_user(request):
     return request.config.getoption("--login")
@@ -79,26 +80,52 @@ def vm_cred(request):
 def namespace(request):
     return request.config.getoption("--namespace")
 
+@pytest.fixture(scope="session")
+def hosts(request):
+    return request.config.getoption("--hosts")
+
+@pytest.fixture(scope="session")
+def ansible_server(request):
+    return request.config.getoption("--ansible_server")
+
+@pytest.fixture(scope="session")
+def wire_mock():
+    if not Docker.is_docker_running():
+        pytest.skip('Docker is not running. Please start docker.')
+    with wiremock_container(image="asdrepo.isus.emc.com:9042/wiremock:2.35.1-1", verify_ssl_certs=False) as wire_mock:
+        Config.base_url = wire_mock.get_url("__admin")
+        Config.requests_verify = False
+        yield wire_mock
+
 @pytest.fixture(scope="function", autouse=True)
 def link_requirements_in_background(request):
     if pytest.qtest_helper is not None:
         requirements_thread = PropagatingThread(target=link_requirements, args=(request,), test_name=request.node.name)
         requirements_thread.start()
         pytest.threads.append(requirements_thread)
-
+    
 def link_requirements(request):
     for marker in request.node.iter_markers():
-            if marker.name == "requirements":
-                logging.info(f" [qTest] Test function {request.node.name} is associated with requirement: {marker.args}.")
-                test_case_pid, requirement_ids = marker.args
-                for requirement_id in requirement_ids:
-                    pytest.qtest_helper.link_test_case_to_requirement(requirement_id, test_case_pid)
-                return
+        if marker.name == "requirements":
+            logging.info(f" [qTest] Test function {request.node.name} is associated with requirement: {marker.args}.")
+            test_case_pid, requirement_ids = marker.args
+            for requirement_id in requirement_ids:
+                pytest.qtest_helper.link_test_case_to_requirement(requirement_id, test_case_pid)
+            return
     logging.info(f"[qTest] Test function {request.node.name} is missing requirements marker.")
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
 def pytest_runtest_makereport(item):
     report = (yield).get_result()
+
+    if report.outcome == 'skipped' and pytest.qtest_helper is not None and item.config.getoption("--qtest_test_suite") != '':
+        update_thread = PropagatingThread(target=update_test_result,
+                                      args=(item.name, item.config.getoption("--qtest_test_suite"), report.outcome, datetime.now(), datetime.now()),
+                                      test_name=item.name)
+        update_thread.start()
+        pytest.threads.append(update_thread)
+        return
+
     setattr(item, 'report', report)
 
 
@@ -115,16 +142,17 @@ def update_test_results_in_background(request):
     yield
     test_end_date = datetime.now()
 
+    test_name = request.node.name
+    test_suite_id = request.config.getoption("--qtest_test_suite")
+    outcome = request.node.report.outcome
+
     update_thread = PropagatingThread(target=update_test_result,
-                                      args=(request, test_start_date, test_end_date),
+                                      args=(test_name, test_suite_id, outcome, test_start_date, test_end_date),
                                       test_name=request.node.name)
     update_thread.start()
     pytest.threads.append(update_thread)
 
-def update_test_result(request, test_start_date, test_end_date):
-    test_name = request.node.name
-    test_suite_id = request.config.getoption("--qtest_test_suite")
-
+def update_test_result(test_name, test_suite_id, outcome, test_start_date, test_end_date):
     match = re.match(r"test_(\d+)", test_name)
 
     if not match:
@@ -136,11 +164,7 @@ def update_test_result(request, test_start_date, test_end_date):
 
     if test_case_id not in pytest.tests_in_suite:
         test_run_id = pytest.qtest_helper.add_test_run_to_test_suite(test_case_id, test_suite_id)
-        logging.info(f"[qTest] Added test case {test_case} to test suite {test_suite_id}")
     else:
         test_run_id = pytest.tests_in_suite[test_case_id]
 
-    pytest.qtest_helper.update_test_run_status_in_test_suite(test_run_id, request.node.report.outcome, test_start_date, test_end_date)
-
-    logging.info(f"[qTest] Updated test run {test_run_id} with status {request.node.report.outcome}")
-
+    pytest.qtest_helper.update_test_run_status_in_test_suite(test_run_id, outcome, test_start_date, test_end_date)

tests/e2e-test-framework/framework/docker_helper.py

@@ -0,0 +1,16 @@
+import logging
+import docker
+
+
+class Docker:
+    @classmethod
+    def is_docker_running(cls):
+        try:
+            client = docker.from_env()
+            client.ping()
+
+            logging.info("\nDocker is running.")
+            return True
+        except Exception as exc:
+            logging.error(f"Error: {exc}")
+            return False

tests/e2e-test-framework/framework/propagating_thread.py

@@ -28,8 +28,3 @@ def has_failed(self):
 
     def get_target_name(self):
         return self.target.__name__
-
-    def join(self, timeout=None):
-        super(PropagatingThread, self).join(timeout)
-        if self.exc is not None:
-            raise RuntimeError(f"{self.test_name} {self.exc}")