remove redundant tests and add wait_and_assert (#2984)

.github/actions/smoke-tests/action.yaml

@@ -112,7 +112,8 @@ runs:
         --service=nodeport --node-ip=${{ steps.k8s.outputs.cluster_ip }} \
         --html=tests-${{ steps.k8s.outputs.cluster }}.html \
         --self-contained-html \
+        --durations=10 \
         --show-ic-logs=yes \
-        -m ${{ inputs.marker != '' && inputs.marker || '""' }}
+        -m ${{ inputs.marker != '' && inputs.marker || '""' }} 
       working-directory: ./tests
       shell: bash

tests/suite/test_app_protect_integration.py

@@ -291,73 +291,6 @@ def test_ap_enable_false_policy_incorrect(
         delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
         assert_valid_responses(response)
 
-    @pytest.mark.flaky(max_runs=3)
-    def test_ap_sec_logs_on(
-        self,
-        request,
-        kube_apis,
-        ingress_controller_prerequisites,
-        crd_ingress_controller_with_ap,
-        appprotect_setup,
-        test_namespace,
-    ):
-        """
-        Test corresponding log entries with correct policy (includes setting up a syslog server as defined in syslog.yaml)
-        """
-        log_loc = "/var/log/messages"
-        syslog_dst = f"syslog-svc.{test_namespace}"
-        syslog_pod = get_pod_name_that_contains(kube_apis.v1, test_namespace, "syslog-")
-
-        create_ingress_with_ap_annotations(
-            kube_apis, src_ing_yaml, test_namespace, ap_policy, "True", "True", f"{syslog_dst}:514"
-        )
-        ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
-
-        print("--------- Run test while AppProtect module is enabled with correct policy ---------")
-
-        ensure_response_from_backend(appprotect_setup.req_url, ingress_host, check404=True)
-
-        print("----------------------- Send invalid request ----------------------")
-        response_block = requests.get(
-            appprotect_setup.req_url + "/<script>", headers={"host": ingress_host}, verify=False
-        )
-        print(response_block.text)
-        log_contents_block = ""
-        retry = 0
-        while "ASM:attack_type" not in log_contents_block and retry <= 30:
-            log_contents_block = get_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
-            retry += 1
-            wait_before_test(1)
-            print(f"Security log not updated, retrying... #{retry}")
-
-        print("----------------------- Send valid request ----------------------")
-        headers = {
-            "Host": ingress_host,
-            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0",
-        }
-        response = requests.get(appprotect_setup.req_url, headers=headers, verify=False)
-        print(response.text)
-        wait_before_test(10)
-        log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
-
-        delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
-        clear_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
-
-        assert_invalid_responses(response_block)
-        assert (
-            'ASM:attack_type="Non-browser Client,Abuse of Functionality,Cross Site Scripting (XSS)"'
-            in log_contents_block
-        )
-        assert 'severity="Critical"' in log_contents_block
-        assert 'request_status="blocked"' in log_contents_block
-        assert 'outcome="REJECTED"' in log_contents_block
-
-        assert_valid_responses(response)
-        assert 'ASM:attack_type="N/A"' in log_contents
-        assert 'severity="Informational"' in log_contents
-        assert 'request_status="passed"' in log_contents
-        assert 'outcome="PASSED"' in log_contents
-
     @pytest.mark.startup
     def test_ap_pod_startup(
         self,
@@ -500,9 +433,10 @@ def test_ap_enable_true_policy_correct_uds(
         wait_before_test(120)
         ensure_response_from_backend(appprotect_setup.req_url, ingress_host, check404=True)
         print("----------------------- Send request ----------------------")
-        response = requests.get(appprotect_setup.req_url, headers={"host": ingress_host}, verify=False, data="kic")
-        print(response.text)
-
+        response1 = requests.get(appprotect_setup.req_url, headers={"host": ingress_host}, verify=False, data="kic")
+        print(response1.text)
+        response2 = requests.get(appprotect_setup.req_url + "/<script>", headers={"host": ingress_host}, verify=False)
+        print(response2.text)
         reload_ms = get_last_reload_time(appprotect_setup.metrics_url, "nginx")
         print(f"last reload duration: {reload_ms} ms")
         reload_times[f"{request.node.name}"] = f"last reload duration: {reload_ms} ms"
@@ -517,4 +451,5 @@ def test_ap_enable_true_policy_correct_uds(
         delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
 
         assert_ap_crd_info(ap_crd_info, ap_policy)
-        assert_invalid_responses(response)
+        assert_invalid_responses(response1)
+        assert_invalid_responses(response2)

tests/suite/test_app_protect_waf_policies.py

@@ -291,71 +291,6 @@ def test_ap_waf_policy_allow(
         assert_valid_responses(response1)
         assert_valid_responses(response2)
 
-    @pytest.mark.flaky(max_runs=3)
-    def test_ap_waf_policy_logs(
-        self,
-        kube_apis,
-        crd_ingress_controller_with_ap,
-        virtual_server_setup,
-        appprotect_setup,
-        test_namespace,
-    ):
-        """
-        Test waf policy logs
-        """
-        src_syslog_yaml = f"{TEST_DATA}/ap-waf/syslog.yaml"
-        log_loc = f"/var/log/messages"
-        create_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
-        syslog_dst = f"syslog-svc.{test_namespace}"
-        syslog_pod = kube_apis.v1.list_namespaced_pod(test_namespace).items[-1].metadata.name
-        print(f"Create waf policy")
-        create_ap_waf_policy_from_yaml(
-            kube_apis.custom_objects,
-            waf_pol_dataguard_src,
-            test_namespace,
-            test_namespace,
-            True,
-            True,
-            ap_pol_name,
-            log_name,
-            f"syslog:server={syslog_dst}:514",
-        )
-        wait_before_test()
-        print(f"Patch vs with policy: {waf_spec_vs_src}")
-        patch_virtual_server_from_yaml(
-            kube_apis.custom_objects,
-            virtual_server_setup.vs_name,
-            waf_spec_vs_src,
-            virtual_server_setup.namespace,
-        )
-        wait_before_test()
-        ap_crd_info = read_ap_custom_resource(kube_apis.custom_objects, test_namespace, "appolicies", ap_policy_uds)
-        assert_ap_crd_info(ap_crd_info, ap_policy_uds)
-        wait_before_test(120)
-
-        print("----------------------- Send request with embedded malicious script----------------------")
-        response = requests.get(
-            virtual_server_setup.backend_1_url + "</script>",
-            headers={"host": virtual_server_setup.vs_host},
-        )
-        print(response.text)
-        log_contents = ""
-        retry = 0
-        while "ASM:attack_type" not in log_contents and retry <= 30:
-            log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
-            retry += 1
-            wait_before_test(1)
-            print(f"Security log not updated, retrying... #{retry}")
-
-        delete_policy(kube_apis.custom_objects, "waf-policy", test_namespace)
-        self.restore_default_vs(kube_apis, virtual_server_setup)
-        delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
-        assert_invalid_responses(response)
-        assert f'ASM:attack_type="Non-browser Client,Abuse of Functionality,Cross Site Scripting (XSS)"' in log_contents
-        assert f'severity="Critical"' in log_contents
-        assert f'request_status="blocked"' in log_contents
-        assert f'outcome="REJECTED"' in log_contents
-
     def test_ap_waf_policy_multi_logs(
         self,
         kube_apis,

tests/suite/test_smoke.py

@@ -5,6 +5,7 @@
 import requests
 import yaml
 from settings import TEST_DATA
+from suite.custom_assertions import wait_and_assert_status_code
 from suite.fixtures import PublicEndpoint
 from suite.resources_utils import (
     create_example_app,
@@ -91,12 +92,10 @@ def test_response_code_200_and_server_name(self, request, ingress_controller, sm
             f"http://{smoke_setup.public_endpoint.public_ip}:{smoke_setup.public_endpoint.metrics_port}/metrics"
         )
         ensure_response_from_backend(req_url, smoke_setup.ingress_host)
-        resp = requests.get(req_url, headers={"host": smoke_setup.ingress_host}, verify=False)
         reload_ms = get_last_reload_time(metrics_url, "nginx")
         print(f"last reload duration: {reload_ms} ms")
         reload_times[f"{request.node.name}"] = f"last reload duration: {reload_ms} ms"
-        assert resp.status_code == 200
-        assert f"Server name: {path}" in resp.text
+        wait_and_assert_status_code(200, req_url, smoke_setup.ingress_host, verify=False)
 
     @pytest.mark.parametrize(
         "ingress_controller",

tests/suite/test_v_s_route_upstream_tls.py

@@ -12,6 +12,7 @@
 from suite.resources_utils import (
     create_items_from_yaml,
     delete_items_from_yaml,
+    ensure_response_from_backend,
     get_events,
     get_first_pod_name,
     wait_before_test,
@@ -86,8 +87,15 @@ def test_responses_and_config_after_setup(
             ingress_controller_prerequisites.namespace,
         )
         req_url = f"http://{v_s_route_setup.public_endpoint.public_ip}:{v_s_route_setup.public_endpoint.port}"
+        ensure_response_from_backend(
+            f"{req_url}{v_s_route_setup.route_m.paths[0]}", v_s_route_setup.vs_host, check404=True
+        )
         resp_1 = requests.get(f"{req_url}{v_s_route_setup.route_m.paths[0]}", headers={"host": v_s_route_setup.vs_host})
+        ensure_response_from_backend(
+            f"{req_url}{v_s_route_setup.route_s.paths[0]}", v_s_route_setup.vs_host, check404=True
+        )
         resp_2 = requests.get(f"{req_url}{v_s_route_setup.route_s.paths[0]}", headers={"host": v_s_route_setup.vs_host})
+
         vs_line = f"vs_{v_s_route_setup.namespace}_{v_s_route_setup.vs_name}"
         proxy_host_s = f"{vs_line}_vsr_{v_s_route_setup.route_s.namespace}_{v_s_route_setup.route_s.name}"
         proxy_host_m = f"{vs_line}_vsr_{v_s_route_setup.route_m.namespace}_{v_s_route_setup.route_m.name}"
@@ -149,7 +157,13 @@ def test_validation_flow(
             ingress_controller_prerequisites.namespace,
         )
         req_url = f"http://{v_s_route_setup.public_endpoint.public_ip}:{v_s_route_setup.public_endpoint.port}"
+        ensure_response_from_backend(
+            f"{req_url}{v_s_route_setup.route_m.paths[0]}", v_s_route_setup.vs_host, check404=True
+        )
         resp_1 = requests.get(f"{req_url}{v_s_route_setup.route_m.paths[0]}", headers={"host": v_s_route_setup.vs_host})
+        ensure_response_from_backend(
+            f"{req_url}{v_s_route_setup.route_s.paths[0]}", v_s_route_setup.vs_host, check404=True
+        )
         resp_2 = requests.get(f"{req_url}{v_s_route_setup.route_s.paths[0]}", headers={"host": v_s_route_setup.vs_host})
         new_events_ns_m = get_events(kube_apis.v1, v_s_route_setup.route_m.namespace)
         new_events_ns_s = get_events(kube_apis.v1, v_s_route_setup.route_s.namespace)
@@ -198,7 +212,13 @@ def test_responses_and_config_after_disable_tls(
             ingress_controller_prerequisites.namespace,
         )
         req_url = f"http://{v_s_route_setup.public_endpoint.public_ip}:{v_s_route_setup.public_endpoint.port}"
+        ensure_response_from_backend(
+            f"{req_url}{v_s_route_setup.route_m.paths[0]}", v_s_route_setup.vs_host, check404=True
+        )
         resp_1 = requests.get(f"{req_url}{v_s_route_setup.route_m.paths[0]}", headers={"host": v_s_route_setup.vs_host})
+        ensure_response_from_backend(
+            f"{req_url}{v_s_route_setup.route_s.paths[0]}", v_s_route_setup.vs_host, check404=True
+        )
         resp_2 = requests.get(f"{req_url}{v_s_route_setup.route_s.paths[0]}", headers={"host": v_s_route_setup.vs_host})
         new_events_ns_m = get_events(kube_apis.v1, v_s_route_setup.route_m.namespace)
         new_events_ns_s = get_events(kube_apis.v1, v_s_route_setup.route_s.namespace)