Add test for TransportServer load balancing config

tests/data/transport-server-status/standard/dns.yaml → tests/data/transport-server-status/standard/service.yaml

@@ -32,9 +32,6 @@ spec:
           mountPath: /etc/coredns
           readOnly: true
         ports:
-        - containerPort: 5353
-          name: dns
-          protocol: UDP
         - containerPort: 5353
           name: dns-tcp
           protocol: TCP
@@ -54,7 +51,7 @@ metadata:
   name: coredns 
 spec:
   selector:
-   app: coredns 
+    app: coredns
   ports:
   - name: dns
     port: 5353

tests/data/transport-server-tcp-load-balance/standard/global-configuration.yaml

@@ -0,0 +1,9 @@
+apiVersion: k8s.nginx.org/v1alpha1
+kind: GlobalConfiguration 
+metadata:
+  name: nginx-configuration
+spec:
+  listeners:
+  - name: tcp-server
+    port: 3333
+    protocol: TCP

tests/data/transport-server-tcp-load-balance/standard/node-port.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: tcp-nodeport
+spec:
+  selector:
+    app: tcp-service
+  type: NodePort
+  ports:
+    - port: 3333
+      targetPort: 3333
+      protocol: TCP

tests/data/transport-server-tcp-load-balance/standard/service.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tcp-service
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: tcp-service
+  template:
+    metadata:
+      labels:
+        app: tcp-service
+    spec:
+      containers:
+        - name: tcp-service
+          image: tcp-server:latest
+          ports:
+            - containerPort: 3333
+              name: tcp-server
+              protocol: TCP
+          imagePullPolicy: Never
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tcp-service
+spec:
+  selector:
+    app: tcp-service
+  ports:
+  - name: tcp-server
+    targetPort: 3333
+    port: 3333
+    protocol: TCP

tests/data/transport-server-tcp-load-balance/standard/transport-server.yaml

@@ -0,0 +1,14 @@
+apiVersion: k8s.nginx.org/v1alpha1
+kind: TransportServer
+metadata:
+  name: transport-server
+spec:
+  listener:
+    name: tcp-server
+    protocol: TCP
+  upstreams:
+    - name: tcp-app
+      service: tcp-service
+      port: 3333
+  action:
+    pass: tcp-app

tests/suite/fixtures.py

@@ -45,6 +45,7 @@
     delete_service,
     replace_configmap_from_yaml,
     delete_testing_namespaces,
+    get_first_pod_name,
 )
 from suite.resources_utils import (
     create_ingress_controller,
@@ -742,18 +743,23 @@ class TransportServerSetup:
         namespace (str):
     """
 
-    def __init__(self, name, namespace):
+    def __init__(self, name, namespace, ingress_pod_name, ic_namespace, public_endpoint: PublicEndpoint):
         self.name = name
         self.namespace = namespace
+        self.ingress_pod_name = ingress_pod_name
+        self.ic_namespace = ic_namespace
+        self.public_endpoint = public_endpoint
 
 
 @pytest.fixture(scope="class")
 def transport_server_setup(
-        request, kube_apis, test_namespace
+        request, kube_apis, ingress_controller_prerequisites, test_namespace, ingress_controller_endpoint
 ) -> TransportServerSetup:
     """
     Prepare Transport Server Example.
 
+    :param ingress_controller_endpoint:
+    :param ingress_controller_prerequisites:
     :param request: internal pytest fixture to parametrize this method
     :param kube_apis: client apis
     :param test_namespace:
@@ -767,9 +773,9 @@ def transport_server_setup(
     global_config_file = f"{TEST_DATA}/{request.param['example']}/standard/global-configuration.yaml"
     gc_resource = create_gc_from_yaml(kube_apis.custom_objects, global_config_file, "nginx-ingress")
 
-    # deploy dns
-    dns_file = f"{TEST_DATA}/{request.param['example']}/standard/dns.yaml"
-    create_items_from_yaml(kube_apis, dns_file, test_namespace)
+    # deploy service_file
+    service_file = f"{TEST_DATA}/{request.param['example']}/standard/service.yaml"
+    create_items_from_yaml(kube_apis, service_file, test_namespace)
 
     # deploy transport server
     transport_server_file = f"{TEST_DATA}/{request.param['example']}/standard/transport-server.yaml"
@@ -780,12 +786,21 @@ def transport_server_setup(
     def fin():
         print("Clean up TransportServer Example:")
         delete_ts(kube_apis.custom_objects, ts_resource, test_namespace)
-        delete_items_from_yaml(kube_apis, dns_file, test_namespace)
+        delete_items_from_yaml(kube_apis, service_file, test_namespace)
         delete_gc(kube_apis.custom_objects, gc_resource, "nginx-ingress")
 
     request.addfinalizer(fin)
 
-    return TransportServerSetup(ts_resource['metadata']['name'], test_namespace)
+    ic_pod_name = get_first_pod_name(kube_apis.v1, ingress_controller_prerequisites.namespace)
+    ic_namespace = ingress_controller_prerequisites.namespace
+
+    return TransportServerSetup(
+        ts_resource['metadata']['name'],
+        test_namespace,
+        ic_pod_name,
+        ic_namespace,
+        ingress_controller_endpoint,
+    )
 
 
 @pytest.fixture(scope="class")

tests/suite/resources_utils.py

@@ -131,6 +131,36 @@ def create_deployment_from_yaml(apps_v1_api: AppsV1Api, namespace, yaml_manifest
     return create_deployment(apps_v1_api, namespace, dep)
 
 
+def patch_deployment_from_yaml(apps_v1_api: AppsV1Api, namespace, yaml_manifest) -> str:
+    """
+    Create a deployment based on yaml file.
+
+    :param apps_v1_api: AppsV1Api
+    :param namespace: namespace name
+    :param yaml_manifest: absolute path to file
+    :return: str
+    """
+    print(f"Load {yaml_manifest}")
+    with open(yaml_manifest) as f:
+        dep = yaml.safe_load(f)
+    return patch_deployment(apps_v1_api, namespace, dep)
+
+
+def patch_deployment(apps_v1_api: AppsV1Api, namespace, body) -> str:
+    """
+    Create a deployment based on a dict.
+
+    :param apps_v1_api: AppsV1Api
+    :param namespace: namespace name
+    :param body: dict
+    :return: str
+    """
+    print("Patch a deployment:")
+    apps_v1_api.patch_namespaced_deployment(body['metadata']['name'], namespace, body)
+    print(f"Deployment patched with name '{body['metadata']['name']}'")
+    return body['metadata']['name']
+
+
 def create_deployment(apps_v1_api: AppsV1Api, namespace, body) -> str:
     """
     Create a deployment based on a dict.
@@ -165,21 +195,23 @@ def create_deployment_with_name(apps_v1_api: AppsV1Api, namespace, name) -> str:
         return create_deployment(apps_v1_api, namespace, dep)
 
 
-def scale_deployment(apps_v1_api: AppsV1Api, name, namespace, value) -> None:
+def scale_deployment(apps_v1_api: AppsV1Api, name, namespace, value) -> int:
     """
     Scale a deployment.
 
     :param apps_v1_api: AppsV1Api
     :param namespace: namespace name
     :param name: deployment name
     :param value: int
-    :return:
+    :return: original: int the original amount of replicas
     """
     print(f"Scale a deployment '{name}'")
     body = apps_v1_api.read_namespaced_deployment_scale(name, namespace)
+    original = body.spec.replicas
     body.spec.replicas = value
     apps_v1_api.patch_namespaced_deployment_scale(name, namespace, body)
     print(f"Scale a deployment '{name}': complete")
+    return original
 
 
 def create_daemon_set(apps_v1_api: AppsV1Api, namespace, body) -> str:
@@ -750,6 +782,21 @@ def get_ingress_nginx_template_conf(v1: CoreV1Api, ingress_namespace, ingress_na
     return get_file_contents(v1, file_path, pod_name, pod_namespace)
 
 
+def get_ts_nginx_template_conf(v1: CoreV1Api, resource_namespace, resource_name, ingress_name, ingress_namespace) -> str:
+    """
+    Get contents of /etc/nginx/stream-conf.d/ts_{namespace}-{resource_name}.conf in the pod.
+
+    :param v1: CoreV1Api
+    :param resource_namespace:
+    :param resource_name:
+    :param ingress_name:
+    :param ingress_namespace:
+    :return: str
+    """
+    file_path = f"/etc/nginx/stream-conf.d/ts_{resource_namespace}_{resource_name}.conf"
+    return get_file_contents(v1, file_path, ingress_name, ingress_namespace)
+
+
 def create_example_app(kube_apis, app_type, namespace) -> None:
     """
     Create a backend application.

tests/suite/test_transport_server_status.py

@@ -17,7 +17,6 @@
                 "type": "complete",
                 "extra_args":
                     [
-                        "-enable-custom-resources",
                         "-global-configuration=nginx-ingress/nginx-configuration",
                         "-enable-leader-election=false"
                     ]

tests/suite/test_transport_server_tcp_load_balance.py

@@ -0,0 +1,114 @@
+import pytest
+import re
+import time
+import socket
+
+from settings import TEST_DATA
+from suite.resources_utils import (
+    wait_before_test,
+    patch_deployment_from_yaml,
+    create_items_from_yaml,
+    get_ingress_nginx_template_conf,
+    get_ts_nginx_template_conf,
+    create_service_from_yaml,
+    delete_service,
+    scale_deployment
+)
+from suite.custom_resources_utils import (
+    patch_ts
+)
+
+
+@pytest.mark.ts
+@pytest.mark.parametrize(
+    "crd_ingress_controller, transport_server_setup",
+    [
+        (
+            {
+                "type": "complete",
+                "extra_args":
+                    [
+                        "-global-configuration=nginx-ingress/nginx-configuration",
+                        "-enable-leader-election=false"
+                    ]
+            },
+            {"example": "transport-server-tcp-load-balance", "app_type": "simple"},
+        )
+    ],
+    indirect=True,
+)
+class TestTransportServerTcpLoadBalance:
+
+    def restore_transport_server(self, kube_apis, transport_server_setup) -> None:
+        """
+        Function to revert a TransportServer resource to a valid state.
+        """
+        transport_server_file = f"{TEST_DATA}/transport-server-status/standard/transport-server.yaml"
+        patch_ts(kube_apis.custom_objects, transport_server_setup.name, transport_server_file, transport_server_setup.namespace)
+
+    def test_number_of_replicas(
+        self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites
+    ):
+        """
+        The load balancing of TCP should result in 4 servers to match the 4 replicas of a service.
+        """
+        original = scale_deployment(kube_apis.apps_v1_api, "tcp-service", transport_server_setup.namespace, 4)
+
+        wait_before_test()
+
+        result_conf = get_ts_nginx_template_conf(
+            kube_apis.v1,
+            transport_server_setup.namespace,
+            transport_server_setup.name,
+            transport_server_setup.ingress_pod_name,
+            ingress_controller_prerequisites.namespace
+        )
+
+        pattern = 'server.*max_fails=1 fail_timeout=10s;'
+        num_servers = len(re.findall(pattern, result_conf))
+
+        assert num_servers is 4
+
+        scale_deployment(kube_apis.apps_v1_api, "tcp-service", transport_server_setup.namespace, original)
+
+    @pytest.mark.ts
+    def test_tcp_request_load_balanced(
+            self, kube_apis, crd_ingress_controller, transport_server_setup
+    ):
+        """
+        Requests to the load balanced TCP service should result in responses from 3 different endpoints.
+        """
+        node_port_file = f"{TEST_DATA}/transport-server-tcp-load-balance/standard/node-port.yaml"
+        node_port_name = create_service_from_yaml(
+            kube_apis.v1,
+            transport_server_setup.namespace,
+            node_port_file,
+        )
+
+        wait_before_test(3)
+
+        node_port_service = kube_apis.v1.read_namespaced_service(node_port_name, transport_server_setup.namespace)
+        port = node_port_service.spec.ports[0].node_port
+        host = transport_server_setup.public_endpoint.public_ip
+
+        print(f"sending tcp requests to: {host}:{port}")
+
+        endpoints = {}
+        for i in range(20):
+            client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            client.connect((host, port))
+            response = client.recv(4096)
+            endpoint = response.decode()
+            print(f' req number {i}; response: {endpoint}')
+            if endpoint not in endpoints:
+                endpoints[endpoint] = 1
+            else:
+                endpoints[endpoint] = endpoints[endpoint] + 1
+
+        assert len(endpoints) is 3
+
+        delete_service(kube_apis.v1, node_port_name, transport_server_setup.namespace)
+
+        self.restore_transport_server(kube_apis, transport_server_setup)
+
+

tests/tcp-server/Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:1.16-alpine as builder
+
+RUN mkdir /app
+
+COPY main.go /app
+
+WORKDIR /app
+
+RUN GO111MODULE=off CGO_ENABLED=0 GOOS=linux go build -o main
+
+FROM scratch
+
+COPY --from=builder /app/main /app/
+
+EXPOSE 3333
+
+ENTRYPOINT ["/app/main"]