Add automated tests for -watch-secret-namespace (#3245)

tests/data/watch-secret-namespace/route-multiple.yaml

@@ -0,0 +1,20 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServerRoute
+metadata:
+  name: backends
+spec:
+  host: virtual-server-route.example.com
+  upstreams:
+  - name: backend1
+    service: backend1-svc
+    port: 80
+  - name: backend3
+    service: backend3-svc
+    port: 80
+  subroutes:
+  - path: "/backends/backend1"
+    action:
+      pass: backend1
+  - path: "/backends/backend3"
+    action:
+      pass: backend3

tests/data/watch-secret-namespace/route-single.yaml

@@ -0,0 +1,14 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServerRoute
+metadata:
+  name: backend2
+spec:
+  host: virtual-server-route.example.com
+  upstreams:
+  - name: backend2
+    service: backend2-svc
+    port: 80
+  subroutes:
+  - path: "/backend2"
+    action:
+      pass: backend2

tests/data/watch-secret-namespace/standard/virtual-server.yaml

@@ -0,0 +1,13 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+  name: virtual-server-route
+spec:
+  host: virtual-server-route.example.com
+  tls:
+    secret: virtual-server-tls-secret
+  routes:
+  - path: "/backends"
+    route: backends # implicit namespace
+  - path: "/backend2"
+    route: backend2-namespace/backend2

tests/data/watch-secret-namespace/tls-secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: virtual-server-tls-secret
+type: kubernetes.io/tls
+data:
+  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ4RENDQXRpZ0F3SUJBZ0lKQU9jbHdCelprYmlhTUEwR0NTcUdTSWIzRFFFQkJRVUFNRmd4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFaE1COEdBMVVFQ2hNWVNXNTBaWEp1WlhRZ1YybGtaMmwwY3lCUQpkSGtnVEhSa01Sa3dGd1lEVlFRREV4QmpZV1psTG1WNFlXMXdiR1V1WTI5dE1CNFhEVEUzTURnek1URXdNVGN5Ck1Gb1hEVEU0TURnek1URXdNVGN5TUZvd1dERUxNQWtHQTFVRUJoTUNWVk14Q3pBSkJnTlZCQWdUQWtOQk1TRXcKSHdZRFZRUUtFeGhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXhHVEFYQmdOVkJBTVRFR05oWm1VdQpaWGhoYlhCc1pTNWpiMjB3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRQzdIT0xJCm5oZjE1aUcxOE16RXBzN0lvZmxHQmovMi9NVjA0OWtBS0hrTnZOem1XaXRXWDV2QU1yRkF5THY0dXBIWDI5b0IKa3l6YUhlYyt2TlFibEh1bStINUtkUWZXWHFXNkJ6UnVhMzBreEkrcG91cnhpNy9jaDJORS92djBhRGtvaTJ0RAovOUI2aHAyVkoxWXFJdm9hQ2wwSmFYWDd0WEc4SGU1S1BSZzBYMm1Mblcwa29tay9ZVGRPbS9xOVRjUDRnUmhrCms3bUhJMDlSME5vNUhTbURydmVBWFEyY3lGWVJQVUNjWkNPd0h6UUdVUVB1UU0wNVArWUNnVlNRcElKWFV0b0kKbGdEMHhZZUw4UU1rZjZ0TWpYcXpTVVRhQlhzNkRKU2x1YWN1aHpkV212UnFPUVNNYlVpZ3dVUEZCRDVLRUFIcwozM0hHeVZ5dkI4cVlrYUczQWdNQkFBR2pnYnd3Z2Jrd0hRWURWUjBPQkJZRUZOdTQvMTdpSituRGxPMkoyVisvCitqM2x5SzVZTUlHSkJnTlZIU01FZ1lFd2Y0QVUyN2ovWHVJbjZjT1U3WW5aWDcvNlBlWElybGloWEtSYU1GZ3gKQ3pBSkJnTlZCQVlUQWxWVE1Rc3dDUVlEVlFRSUV3SkRRVEVoTUI4R0ExVUVDaE1ZU1c1MFpYSnVaWFFnVjJsawpaMmwwY3lCUWRIa2dUSFJrTVJrd0Z3WURWUVFERXhCallXWmxMbVY0WVcxd2JHVXVZMjl0Z2drQTV5WEFITm1SCnVKb3dEQVlEVlIwVEJBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkFRVUZBQU9DQVFFQUtGUHJBcXA3a3lzTDVGNnMKWFhWdXZkZzAyc0srUlpzb2F3QWVxbHlSRmpJeUlQL2VTajBhQjQwQmNOcWFyRHhwNjhBd1pZNG4yQk9EVmo5WgphOFlvV1YyOFpwamloaThxNnBPSElOa0MrOXpCY1hsZ2lvVUZBTERCcXFPTXFUZkw1cjNGejNUTGN1clozajhuCnUzL2hRVHNXZG5TZENWbmN0aXhaUHJ5cnhJSFlWSERiVHF4ZWdTQUN6WkU1MHMwdlRpMFJkNUkrcVdubVpIUloKL0hLNVZnNWlNS2E1clBPRTFaT2M3L2VnVjZ6R2p4THJiNEdlQ2JyTjBBb01tazNpL2d2K2kzL0N6aTlXOVhNNApwa2hQSjJUcEtMSjVaOHgxUVhjUW5Dem5yOEdtL2FuVzV4b3lDdWhjZzlXMlVSYzRKVTZ1UXh0WU9tczYrc0RxCjBBN1Y3UT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+  tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdXh6aXlKNFg5ZVlodGZETXhLYk95S0g1UmdZLzl2ekZkT1BaQUNoNURiemM1bG9yClZsK2J3REt4UU1pNytMcVIxOXZhQVpNczJoM25QcnpVRzVSN3B2aCtTblVIMWw2bHVnYzBibXQ5Sk1TUHFhTHEKOFl1LzNJZGpSUDc3OUdnNUtJdHJRLy9RZW9hZGxTZFdLaUw2R2dwZENXbDErN1Z4dkIzdVNqMFlORjlwaTUxdApKS0pwUDJFM1RwdjZ2VTNEK0lFWVpKTzVoeU5QVWREYU9SMHBnNjczZ0YwTm5NaFdFVDFBbkdRanNCODBCbEVECjdrRE5PVC9tQW9GVWtLU0NWMUxhQ0pZQTlNV0hpL0VESkgrclRJMTZzMGxFMmdWN09neVVwYm1uTG9jM1ZwcjAKYWprRWpHMUlvTUZEeFFRK1NoQUI3Tjl4eHNsY3J3ZkttSkdodHdJREFRQUJBb0lCQUVuZHpXbUZmOUFEV2F1Sgp0RXl0elZSSEhURVhwb2pLb09qVVNnWlY4L1FJYXV4RkRIYThwNi9vVXpGUURXVFR3bCtFMnp0ajdvRHM3UzFIClBqVGxHU3VCVGRuMitYRVhURFYwUXE2VW9JS3pWa09SblU1ZDdSQVNJbzVLV3d6UldEODVTczg5WGdBQXhKVHQKUW9hLzZCdi9tMXJyMXpmWEdWODZNYWY5Rm1FVjI5bmpCcHZ3cUpzOFlUaFU5VE5BRUdnbkxBWGFJYTJZSUJTSgozcVhJd1RrdFBHUUZyZUowdnBiOVlaTkRxWk0rMmI1K3BBVEVXclpyeTQxTlpvdzhNUnVOYnoxVU5sWXkyazZKCjlXclUxUDVYM2l3dEZmcXA3TzREakNOMFR2Y1Y1Nm5QczJoYldDeHp4RmxFalJwNW5KQ3Bsdi9yNCt4eHJVRWQKd0NjU0x3RUNnWUVBOGVDUGM2S2JpSXVFUWxwVjcxTXRQMjdBZzhkMzN5aWZKajAvQ3R3RlF6dlhWcHZCSHFRagphUE1ZaEswZ3o0MzFHMGtqVUpTeUU2STJYVWhFU3gxZUw2TU5sVS9xc29Uc0JaZEV3N0cvMFRhdW1TQUJWemZPCkIvckdtbEZkZitxWVpiR3pIdGtvbVJWb1JqVWtPcGRJL3RnL1ZlSmJ3K1RIS2dYS01NU2V5cjhDZ1lFQXhnbTkKWXN5dVZVUGlEdDhuL1JXeEJwZ1Zqazk0M3BFOVFjK2FVK0VPcGROK2NNQjkvaGJiWUpMZEZ2WUUwd2s1cEQ1SQpibktscjZycndEVUJKNWY2TXM0L3gyS3JISjlCQ3VNT3JyTEVTVm42ZG9NdDhWdEtpQkVLTVFuSnIzYmM4UDV3CnpvVmNhRGl1dCtISjcySG83cG5QQTFhaS9QV1ZwM2pZUTlCSXZ3a0NnWUJRNzkzUXlmYlZxQ25uc2liVFlMZmgKWkFRVGxLbXVDUC9JWWZJNGhndFV4aTkya2NQN3B0MGFmMDRUQjRQVk1DRjJzZkNaUkVpYWZVdEh4Nmppb2I4awpuYUVyOTRRSG5LY0Y3K3BZdWFBQU9CWVFzejcvbW5MZEJMTjBiQW1uaGk3Y3lLdXhoT1VxNUpqeDlWSmNNTWVDClQ0WlNETjY4SEUvdzVlTVVrcGE0TFFLQmdGM0piUXhtUE1XYW9XdERtYytNdjBxTktlQThtTlJtMmlqWnBZL0YKek1jUnN4YTR3ckpicHNkRXBqbmlod1Jlb1JLOGdGYjJLcXRYK2RBTUNpRHpJNFYrRWN4ZVdRVDBFcnlTTFhqawpwbnJLaHdnck5jM1EyeW8zVDZsTHBsMVhvR2p0UndVM09UME9Zd2dvZ1JiQ09xc000bklGVEtrWnNTY2YzdU8yCnQwenBBb0dCQU5Fc1V4Yit6UHpKbEgwL2hOTlhteisvNGx4aXlJb1ZQQXgzaEtBemtjOGtkZzhoS25XWkZhK0YKTjZMOXZjTDhNbnRjNHpmVDdDanNxWGJ2cGUzRUFOeGk4TWRSZzd1Z093L2NiZWdLVklZY3hTdXdRKzYrNUZYOApKRzlhbGxzdXovTk40b2RpMzRvblpEVmRZcURnY2xaZnZucXZKMHZWSzc1VXhQZ3F3aUJQCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==

tests/suite/test_watch_secret_namespace.py

@@ -0,0 +1,112 @@
+from unittest import mock
+
+import pytest
+import requests
+from kubernetes.client.rest import ApiException
+from settings import TEST_DATA
+from suite.utils.resources_utils import create_secret_from_yaml, ensure_response_from_backend, wait_before_test
+from suite.utils.ssl_utils import create_sni_session
+
+
+@pytest.mark.vsr
+@pytest.mark.parametrize(
+    "crd_ingress_controller, v_s_route_setup",
+    [
+        (
+            {
+                "type": "complete",
+                "extra_args": [
+                    f"-enable-custom-resources",
+                    f"-watch-namespace=nginx-ingress,backends,backend2-namespace",
+                    f"-watch-secret-namespace=backends",
+                ],
+            },
+            {"example": "watch-secret-namespace"},
+        )
+    ],
+    indirect=True,
+)
+class TestVSRWatchSecretNamespacesValid:
+    def test_responses(
+        self,
+        kube_apis,
+        ingress_controller_prerequisites,
+        crd_ingress_controller,
+        v_s_route_setup,
+        v_s_route_app_setup,
+    ):
+        """Creates 3 tests 1). watching specific ns and secret ns, 2). watching only specific secret ns, 3). watching all ns"""
+        src_vs_sec_yaml = f"{TEST_DATA}/watch-secret-namespace/tls-secret.yaml"
+        create_secret_from_yaml(kube_apis.v1, "backends", src_vs_sec_yaml)
+        req_url = f"https://{v_s_route_setup.public_endpoint.public_ip}:{v_s_route_setup.public_endpoint.port_ssl}"
+        session = create_sni_session()
+        exception = ""
+        resp = mock.Mock()
+        resp.status_code = "None"
+        resp.text = "None"
+        retry = 0
+        while resp.status_code == "None" and retry < 10:
+            wait_before_test()
+            try:
+                resp = session.get(
+                    f"{req_url}{v_s_route_setup.route_m.paths[0]}",
+                    headers={"host": v_s_route_setup.vs_host},
+                    allow_redirects=False,
+                    verify=False,
+                )
+            except requests.exceptions.SSLError as e:
+                exception = str(e)
+                print(f"SSL certificate exception: {exception}")
+            retry = +1
+
+        assert resp.status_code == 200
+
+
+@pytest.mark.vsr
+@pytest.mark.parametrize(
+    "crd_ingress_controller, v_s_route_setup",
+    [
+        (
+            {
+                "type": "complete",
+                "extra_args": [
+                    f"-enable-custom-resources",
+                    f"-watch-namespace=nginx-ingress,backends,backend2-namespace",
+                    f"-watch-secret-namespace=invalid",
+                ],
+            },
+            {"example": "watch-secret-namespace"},
+        )
+    ],
+    indirect=True,
+)
+class TestVSRWatchSecretNamespacesInvalid:
+    def test_responses(
+        self,
+        kube_apis,
+        ingress_controller_prerequisites,
+        crd_ingress_controller,
+        v_s_route_setup,
+        v_s_route_app_setup,
+    ):
+        """Creates 2 tests 1). watching specific ns and invalid secret ns, 2). watching only invalid secret ns"""
+        src_vs_sec_yaml = f"{TEST_DATA}/watch-secret-namespace/tls-secret.yaml"
+        create_secret_from_yaml(kube_apis.v1, "backends", src_vs_sec_yaml)
+        req_url = f"https://{v_s_route_setup.public_endpoint.public_ip}:{v_s_route_setup.public_endpoint.port_ssl}"
+        session = create_sni_session()
+        exception = ""
+        wait_before_test()
+        try:
+            resp = session.get(
+                f"{req_url}{v_s_route_setup.route_m.paths[0]}",
+                headers={"host": v_s_route_setup.vs_host},
+                allow_redirects=False,
+                verify=False,
+            )
+        except requests.exceptions.SSLError as e:
+            exception = str(e)
+            print(f"SSL certificate exception: {exception}")
+            resp = mock.Mock()
+            resp.status_code = "None"
+            resp.text = "None"
+        assert "[SSL: TLSV1_UNRECOGNIZED_NAME]" in exception and "None" in resp.status_code

tests/suite/utils/resources_utils.py

@@ -3,6 +3,7 @@
 import os
 import re
 import time
+from unittest import mock
 
 import pytest
 import requests
@@ -12,6 +13,7 @@
 from kubernetes.stream import stream
 from more_itertools import first
 from settings import DEPLOYMENTS, PROJECT_ROOT, RECONFIGURATION_DELAY, TEST_DATA
+from suite.utils.ssl_utils import create_sni_session
 
 
 class RBACAuthorization:
@@ -1429,7 +1431,7 @@ def get_events(v1: CoreV1Api, namespace) -> []:
     return res.items
 
 
-def ensure_response_from_backend(req_url, host, additional_headers=None, check404=False) -> None:
+def ensure_response_from_backend(req_url, host, additional_headers=None, check404=False, sni=False) -> None:
     """
     Wait for 502|504|404 to disappear.
 
@@ -1442,6 +1444,29 @@ def ensure_response_from_backend(req_url, host, additional_headers=None, check40
     if additional_headers:
         headers.update(additional_headers)
 
+    if sni and check404:
+        session = create_sni_session()
+        for _ in range(60):
+            try:
+                resp = session.get(
+                    req_url,
+                    headers=headers,
+                    allow_redirects=False,
+                    verify=False,
+                )
+                if resp.status_code != 502 and resp.status_code != 504 and resp.status_code != 404:
+                    print(
+                        f"After {_} retries at 1 second interval, got {resp.status_code} response. Continue with tests..."
+                    )
+                    return
+                time.sleep(1)
+            except requests.exceptions.SSLError as e:
+                exception = str(e)
+                print(f"SSL certificate exception: {exception}")
+                resp = mock.Mock()
+                resp.status_code = "None"
+        pytest.fail(f"Keep getting {resp.status_code} from {req_url} after 60 seconds. Exiting...")
+
     if check404:
         for _ in range(60):
             resp = requests.get(req_url, headers=headers, verify=False)