Add tests for TransportServer TCP load balancing (#1543)

tests/data/common/service/nodeport-with-additional-ports.yaml

@@ -22,5 +22,9 @@ spec:
     targetPort: 9113
     protocol: TCP
     name: exporter
+  - port: 3333
+    targetPort: 3333
+    protocol: TCP
+    name: tcp-server
   selector:
     app: nginx-ingress

tests/data/transport-server-status/standard/dns.yaml → tests/data/transport-server-status/standard/service_deployment.yaml

@@ -32,9 +32,6 @@ spec:
           mountPath: /etc/coredns
           readOnly: true
         ports:
-        - containerPort: 5353
-          name: dns
-          protocol: UDP
         - containerPort: 5353
           name: dns-tcp
           protocol: TCP
@@ -54,7 +51,7 @@ metadata:
   name: coredns 
 spec:
   selector:
-   app: coredns 
+    app: coredns
   ports:
   - name: dns
     port: 5353

tests/data/transport-server-tcp-load-balance/missing-service-transport-server.yaml

@@ -0,0 +1,14 @@
+apiVersion: k8s.nginx.org/v1alpha1
+kind: TransportServer
+metadata:
+  name: transport-server
+spec:
+  listener:
+    name: tcp-server
+    protocol: TCP
+  upstreams:
+    - name: tcp-app
+      service: missing-service
+      port: 3333
+  action:
+    pass: tcp-app

tests/data/transport-server-tcp-load-balance/second-transport-server.yaml

@@ -0,0 +1,14 @@
+apiVersion: k8s.nginx.org/v1alpha1
+kind: TransportServer
+metadata:
+  name: transport-server-two
+spec:
+  listener:
+    name: tcp-server
+    protocol: TCP
+  upstreams:
+    - name: tcp-app
+      service: tcp-service
+      port: 3333
+  action:
+    pass: tcp-app

tests/data/transport-server-tcp-load-balance/standard/global-configuration.yaml

@@ -0,0 +1,9 @@
+apiVersion: k8s.nginx.org/v1alpha1
+kind: GlobalConfiguration 
+metadata:
+  name: nginx-configuration
+spec:
+  listeners:
+  - name: tcp-server
+    port: 3333
+    protocol: TCP

tests/data/transport-server-tcp-load-balance/standard/service_deployment.yaml

@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tcp-service
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: tcp-service
+  template:
+    metadata:
+      labels:
+        app: tcp-service
+    spec:
+      containers:
+        - name: tcp-service
+          image: seanoneillf5/tcp-server:1.0
+          ports:
+            - containerPort: 3333
+              name: tcp-server
+              protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tcp-service
+spec:
+  selector:
+    app: tcp-service
+  ports:
+  - name: tcp-server
+    targetPort: 3333
+    port: 3333
+    protocol: TCP

tests/data/transport-server-tcp-load-balance/standard/transport-server.yaml

@@ -0,0 +1,14 @@
+apiVersion: k8s.nginx.org/v1alpha1
+kind: TransportServer
+metadata:
+  name: transport-server
+spec:
+  listener:
+    name: tcp-server
+    protocol: TCP
+  upstreams:
+    - name: tcp-app
+      service: tcp-service
+      port: 3333
+  action:
+    pass: tcp-app

tests/data/transport-server-tcp-load-balance/wrong-port-transport-server.yaml

@@ -0,0 +1,14 @@
+apiVersion: k8s.nginx.org/v1alpha1
+kind: TransportServer
+metadata:
+  name: transport-server
+spec:
+  listener:
+    name: tcp-server
+    protocol: TCP
+  upstreams:
+    - name: tcp-app
+      service: tcp-service
+      port: 2222
+  action:
+    pass: tcp-app

tests/suite/fixtures.py

@@ -45,6 +45,7 @@
     delete_service,
     replace_configmap_from_yaml,
     delete_testing_namespaces,
+    get_first_pod_name,
 )
 from suite.resources_utils import (
     create_ingress_controller,
@@ -120,12 +121,13 @@ class PublicEndpoint:
         port_ssl (int):
     """
 
-    def __init__(self, public_ip, port=80, port_ssl=443, api_port=8080, metrics_port=9113):
+    def __init__(self, public_ip, port=80, port_ssl=443, api_port=8080, metrics_port=9113, tcp_server_port=3333):
         self.public_ip = public_ip
         self.port = port
         self.port_ssl = port_ssl
         self.api_port = api_port
         self.metrics_port = metrics_port
+        self.tcp_server_port = tcp_server_port
 
 
 class IngressControllerPrerequisites:
@@ -252,10 +254,10 @@ def ingress_controller_endpoint(
             namespace,
             f"{TEST_DATA}/common/service/nodeport-with-additional-ports.yaml",
         )
-        port, port_ssl, api_port, metrics_port = get_service_node_ports(
+        port, port_ssl, api_port, metrics_port, tcp_server_port = get_service_node_ports(
             kube_apis.v1, service_name, namespace
         )
-        return PublicEndpoint(public_ip, port, port_ssl, api_port, metrics_port)
+        return PublicEndpoint(public_ip, port, port_ssl, api_port, metrics_port, tcp_server_port)
     else:
         create_service_from_yaml(
             kube_apis.v1,
@@ -754,16 +756,24 @@ class TransportServerSetup:
         namespace (str):
     """
 
-    def __init__(self, name, namespace):
+    def __init__(self, name, namespace, ingress_pod_name, ic_namespace, public_endpoint: PublicEndpoint, resource):
         self.name = name
         self.namespace = namespace
+        self.ingress_pod_name = ingress_pod_name
+        self.ic_namespace = ic_namespace
+        self.public_endpoint = public_endpoint
+        self.resource = resource
 
 
 @pytest.fixture(scope="class")
-def transport_server_setup(request, kube_apis, test_namespace) -> TransportServerSetup:
+def transport_server_setup(
+        request, kube_apis, ingress_controller_prerequisites, test_namespace, ingress_controller_endpoint
+) -> TransportServerSetup:
     """
     Prepare Transport Server Example.
 
+    :param ingress_controller_endpoint:
+    :param ingress_controller_prerequisites:
     :param request: internal pytest fixture to parametrize this method
     :param kube_apis: client apis
     :param test_namespace:
@@ -779,9 +789,9 @@ def transport_server_setup(request, kube_apis, test_namespace) -> TransportServe
     )
     gc_resource = create_gc_from_yaml(kube_apis.custom_objects, global_config_file, "nginx-ingress")
 
-    # deploy dns
-    dns_file = f"{TEST_DATA}/{request.param['example']}/standard/dns.yaml"
-    create_items_from_yaml(kube_apis, dns_file, test_namespace)
+    # deploy service_file
+    service_file = f"{TEST_DATA}/{request.param['example']}/standard/service_deployment.yaml"
+    create_items_from_yaml(kube_apis, service_file, test_namespace)
 
     # deploy transport server
     transport_server_file = f"{TEST_DATA}/{request.param['example']}/standard/transport-server.yaml"
@@ -794,12 +804,22 @@ def transport_server_setup(request, kube_apis, test_namespace) -> TransportServe
     def fin():
         print("Clean up TransportServer Example:")
         delete_ts(kube_apis.custom_objects, ts_resource, test_namespace)
-        delete_items_from_yaml(kube_apis, dns_file, test_namespace)
+        delete_items_from_yaml(kube_apis, service_file, test_namespace)
         delete_gc(kube_apis.custom_objects, gc_resource, "nginx-ingress")
 
     request.addfinalizer(fin)
 
-    return TransportServerSetup(ts_resource["metadata"]["name"], test_namespace)
+    ic_pod_name = get_first_pod_name(kube_apis.v1, ingress_controller_prerequisites.namespace)
+    ic_namespace = ingress_controller_prerequisites.namespace
+
+    return TransportServerSetup(
+        ts_resource['metadata']['name'],
+        test_namespace,
+        ic_pod_name,
+        ic_namespace,
+        ingress_controller_endpoint,
+        ts_resource,
+    )
 
 
 @pytest.fixture(scope="class")

tests/suite/resources_utils.py

@@ -131,6 +131,36 @@ def create_deployment_from_yaml(apps_v1_api: AppsV1Api, namespace, yaml_manifest
     return create_deployment(apps_v1_api, namespace, dep)
 
 
+def patch_deployment_from_yaml(apps_v1_api: AppsV1Api, namespace, yaml_manifest) -> str:
+    """
+    Create a deployment based on yaml file.
+
+    :param apps_v1_api: AppsV1Api
+    :param namespace: namespace name
+    :param yaml_manifest: absolute path to file
+    :return: str
+    """
+    print(f"Load {yaml_manifest}")
+    with open(yaml_manifest) as f:
+        dep = yaml.safe_load(f)
+    return patch_deployment(apps_v1_api, namespace, dep)
+
+
+def patch_deployment(apps_v1_api: AppsV1Api, namespace, body) -> str:
+    """
+    Create a deployment based on a dict.
+
+    :param apps_v1_api: AppsV1Api
+    :param namespace: namespace name
+    :param body: dict
+    :return: str
+    """
+    print("Patch a deployment:")
+    apps_v1_api.patch_namespaced_deployment(body['metadata']['name'], namespace, body)
+    print(f"Deployment patched with name '{body['metadata']['name']}'")
+    return body['metadata']['name']
+
+
 def create_deployment(apps_v1_api: AppsV1Api, namespace, body) -> str:
     """
     Create a deployment based on a dict.
@@ -165,21 +195,23 @@ def create_deployment_with_name(apps_v1_api: AppsV1Api, namespace, name) -> str:
         return create_deployment(apps_v1_api, namespace, dep)
 
 
-def scale_deployment(apps_v1_api: AppsV1Api, name, namespace, value) -> None:
+def scale_deployment(apps_v1_api: AppsV1Api, name, namespace, value) -> int:
     """
     Scale a deployment.
 
     :param apps_v1_api: AppsV1Api
     :param namespace: namespace name
     :param name: deployment name
     :param value: int
-    :return:
+    :return: original: int the original amount of replicas
     """
     print(f"Scale a deployment '{name}'")
     body = apps_v1_api.read_namespaced_deployment_scale(name, namespace)
+    original = body.spec.replicas
     body.spec.replicas = value
     apps_v1_api.patch_namespaced_deployment_scale(name, namespace, body)
     print(f"Scale a deployment '{name}': complete")
+    return original
 
 
 def create_daemon_set(apps_v1_api: AppsV1Api, namespace, body) -> str:
@@ -310,7 +342,7 @@ def create_service_with_name(v1: CoreV1Api, namespace, name) -> str:
         return create_service(v1, namespace, dep)
 
 
-def get_service_node_ports(v1: CoreV1Api, name, namespace) -> (int, int, int, int):
+def get_service_node_ports(v1: CoreV1Api, name, namespace) -> (int, int, int, int, int):
     """
     Get service allocated node_ports.
 
@@ -320,11 +352,11 @@ def get_service_node_ports(v1: CoreV1Api, name, namespace) -> (int, int, int, in
     :return: (plain_port, ssl_port, api_port, exporter_port)
     """
     resp = v1.read_namespaced_service(name, namespace)
-    assert len(resp.spec.ports) == 4, "An unexpected amount of ports in a service. Check the configuration"
+    assert len(resp.spec.ports) == 5, "An unexpected amount of ports in a service. Check the configuration"
     print(f"Service with an API port: {resp.spec.ports[2].node_port}")
     print(f"Service with an Exporter port: {resp.spec.ports[3].node_port}")
     return resp.spec.ports[0].node_port, resp.spec.ports[1].node_port,\
-        resp.spec.ports[2].node_port, resp.spec.ports[3].node_port
+        resp.spec.ports[2].node_port, resp.spec.ports[3].node_port, resp.spec.ports[4].node_port
 
 
 def wait_for_public_ip(v1: CoreV1Api, namespace: str) -> str:
@@ -750,6 +782,21 @@ def get_ingress_nginx_template_conf(v1: CoreV1Api, ingress_namespace, ingress_na
     return get_file_contents(v1, file_path, pod_name, pod_namespace)
 
 
+def get_ts_nginx_template_conf(v1: CoreV1Api, resource_namespace, resource_name, pod_name, pod_namespace) -> str:
+    """
+    Get contents of /etc/nginx/stream-conf.d/ts_{namespace}-{resource_name}.conf in the pod.
+
+    :param v1: CoreV1Api
+    :param resource_namespace:
+    :param resource_name:
+    :param pod_name:
+    :param pod_namespace:
+    :return: str
+    """
+    file_path = f"/etc/nginx/stream-conf.d/ts_{resource_namespace}_{resource_name}.conf"
+    return get_file_contents(v1, file_path, pod_name, pod_namespace)
+
+
 def create_example_app(kube_apis, app_type, namespace) -> None:
     """
     Create a backend application.

tests/suite/test_transport_server_status.py

@@ -17,7 +17,6 @@
                 "type": "complete",
                 "extra_args":
                     [
-                        "-enable-custom-resources",
                         "-global-configuration=nginx-ingress/nginx-configuration",
                         "-enable-leader-election=false"
                     ]