diff --git a/common/k8s/base.py b/common/k8s/base.py index 6dc532b53..e581ec7d2 100644 --- a/common/k8s/base.py +++ b/common/k8s/base.py @@ -71,6 +71,7 @@ def setup_http_service(self, type=None, external_ips=None, frontend_port=80, + nodePort=None, backend_port=80): ''' A simple helper method to create a service @@ -84,14 +85,11 @@ def setup_http_service(self, metadata.update({'name': name}) selector_dict = {} labels = labels or {} + d1 = {'protocol': 'TCP','port': int(frontend_port),'targetPort': int(backend_port) } + if nodePort: + d1['nodePort'] = int(nodePort) spec.update({ - 'ports': [ - { - 'protocol': 'TCP', - 'port': int(frontend_port), - 'targetPort': int(backend_port) - } - ] + 'ports': [d1] }) if labels: selector_dict = {'selector': labels} @@ -102,7 +100,6 @@ def setup_http_service(self, if external_ips: external_ips_dict = {'external_i_ps': external_ips} spec.update(external_ips_dict) - return self.useFixture(ServiceFixture( connections=self.connections, name=name, @@ -249,7 +246,7 @@ def verify_nginx_pod(self, pod, path=None): result = pod.verify_on_setup() if result: if path: - pod.run_cmd('echo %s > /usr/share/nginx/html/index.html' % (pod.name)) + pod.run_cmd('echo %s > /usr/share/nginx/html/index.html' % (pod.name)) cmd = "cp /usr/share/nginx/html/index.html /usr/share/nginx/html/%s" %(path) pod.run_cmd(cmd) else: @@ -375,6 +372,7 @@ def validate_nginx_lb(self, host=None, path='', port='80', + nodePort=None, barred_pods=None, protocol=None, cert=None, @@ -396,7 +394,8 @@ def validate_nginx_lb(self, hit[x.name] = 0 for x in barred_pods: hit_me_not[x.name] = 0 - + if nodePort: + port = nodePort link = '%s://%s:%s/%s' % (protocol, service_ip, port, path) for i in range(0, attempts): (ret_val, output) = self.do_wget(link, pod=test_pod, host=host, @@ -465,11 +464,11 @@ def setup_update_policy(self, ... ] egress = [ - { 'to': + { 'to': [ { 'pod_selector': {'role': 'temp' } }, - {'ip_block': + {'ip_block': {"cidr" : "1.2.3.4/24"}, }, ], @@ -507,7 +506,7 @@ def setup_update_policy(self, ingress_pod_selector = None ingress_ns_selector = None ingress_ip_block = None - + from_item_dict = from_item.get('pod_selector') or {} for k, v in from_item_dict.iteritems(): if not ingress_pod_dict: @@ -534,7 +533,7 @@ def setup_update_policy(self, ingress_ip_block_dict.update({k: v}) ingress_ip_block = { 'ip_block': ingress_ip_block_dict} - + from_entries.append(ingress_pod_selector or ingress_ns_selector or ingress_ip_block) @@ -582,7 +581,7 @@ def setup_update_policy(self, egress_ns_dict['match_labels'].update({k: v}) egress_ns_selector = { 'namespace_selector': egress_ns_dict} - + to_item_dict = to_item.get('ip_block') or {} for k, v in to_item_dict.iteritems(): if not egress_ip_block_dict: @@ -592,9 +591,9 @@ def setup_update_policy(self, if k == "_except": egress_ip_block_dict.update({k: v}) egress_ip_block = { - 'ip_block': egress_ip_block_dict} - - to_entries.append(egress_pod_selector or + 'ip_block': egress_ip_block_dict} + + to_entries.append(egress_pod_selector or egress_ns_selector or egress_ip_block) # end for to_item @@ -611,7 +610,7 @@ def setup_update_policy(self, egress_list.append(egress_item_dict) # end for egress_item # end of egress - + if policy_types: spec['policy_types'] = policy_types if ingress: @@ -668,7 +667,7 @@ def setup_update_simple_policy(self, ''' metadata = metadata or {} spec = spec or {} - + ingress_pods = ingress_pods ingress_namespaces = ingress_namespaces ingress_ipblock = ingress_ipblock @@ -677,7 +676,7 @@ def setup_update_simple_policy(self, egress_ipblock = egress_ipblock ports = ports egress_ports = egress_ports - + ingress_pod_selector = None ingress_ns_selector = None ingress_ipblock_selector = None @@ -686,7 +685,7 @@ def setup_update_simple_policy(self, egress_ipblock_selector = None port_list = [] egress_port_list = [] - + name = name or get_random_name('np-') metadata.update({'name': name}) selector_dict = {} @@ -710,7 +709,7 @@ def setup_update_simple_policy(self, if ingress_ipblock is not None: ingress_ipblock_selector = {'ip_block': ingress_ipblock} - + if egress_pods is not None: egress_pod_dict = {'match_labels': {}} for k, v in egress_pods.iteritems(): @@ -722,7 +721,7 @@ def setup_update_simple_policy(self, for k, v in egress_namespaces.iteritems(): egress_ns_dict['match_labels'].update({k: v}) egress_ns_selector = {'namespace_selector': egress_ns_dict} - + if egress_ipblock is not None: egress_ipblock_selector = {'ip_block': egress_ipblock} @@ -730,7 +729,7 @@ def setup_update_simple_policy(self, for port_str in ports: protocol, port = port_str.split('/') port_list.append({'protocol': protocol, 'port': int(port)}) - + if egress_ports is not None: for port_str in egress_ports: protocol, port = port_str.split('/') @@ -942,11 +941,11 @@ def restart_pod(self, pod_fixture): def create_snat_router(self, name): - obj = self.connections.vnc_lib_fixture.vnc_h.create_router(name=name, + obj = self.connections.vnc_lib_fixture.vnc_h.create_router(name=name, project_obj=self.connections.vnc_lib_fixture.get_project_obj()) self.addCleanup(self.connections.vnc_lib_fixture.vnc_h.delete_router, obj) - return obj + return obj def connect_vn_with_router(self, router_obj, vn_fq_name): @@ -977,16 +976,16 @@ def _remove_namespace_from_router(self, router_obj, vmi_obj): router_obj.del_virtual_machine_interface(vmi_obj) # Update logical router object self.vnc_lib.logical_router_update(router_obj) - + def configure_snat_for_pod (self, pod): - - # Create logical router + + # Create logical router router_obj = self.create_snat_router("snat_router") - # Connect router with virtual network associated to pod + # Connect router with virtual network associated to pod self.connect_vn_with_router(router_obj, pod.vn_fq_names[0]) - + # Configure external_gateway self.connections.vnc_lib_fixture.vnc_h.connect_gateway_with_router(router_obj,\ self.public_vn.public_vn_fixture.obj) @@ -1021,8 +1020,8 @@ def setup_tls_secret(self, data=data, **kwargs)) # end setup_tls_secret - - def setup_vn(self, + + def setup_vn(self, project_name = None, connections = None, inputs = None, @@ -1034,7 +1033,7 @@ def setup_vn(self, vn_name = vn_name or get_random_name('vn_test') return self.useFixture(VNFixture(project_name=project_name, connections=connections, - inputs=inputs, + inputs=inputs, vn_name=vn_name, option=option)) @@ -1061,9 +1060,9 @@ def modify_cluster_project(self, project_name = None): "Setting it to default project for few tests") cmd = r'crudini --set /entrypoint.sh KUBERNETES cluster_project \\${KUBERNETES_CLUSTER_PROJECT:-\\"{\'domain\':\'default-domain\'\,\'project\':\'default\'}\\"}' operation = "set" - project = "default" + project = "default" for kube_manager in self.inputs.kube_manager_ips: - self.inputs.run_cmd_on_server(kube_manager, cmd, + self.inputs.run_cmd_on_server(kube_manager, cmd, container='contrail-kube-manager', shell_prefix = None) self.restart_kube_manager() @@ -1072,10 +1071,10 @@ def modify_cluster_project(self, project_name = None): operation = operation) return operation #end modify_cluster_project - + def revert_cluster_project(self, project_name = None, operation = None): """ - This method reverts the value of cluster_project after performing few + This method reverts the value of cluster_project after performing few sanity tests. """ if operation =="set": diff --git a/fixtures/k8s/service.py b/fixtures/k8s/service.py index 6734b7dd9..4aa634f0b 100644 --- a/fixtures/k8s/service.py +++ b/fixtures/k8s/service.py @@ -55,11 +55,12 @@ def _populate_attr(self): self.metadata_obj = self.obj.metadata self.kind = self.obj.kind self.type = self.obj.spec.type - # While creating the object it is not getting updated with external IP # So reading it again . Will try to read it couple of times if self.type == 'LoadBalancer' or 'external_i_ps' in self.spec: self.get_external_ips() + if self.type == 'NodePort' or 'nodePort' in self.spec: + self.nodePort = self.obj.spec.ports[0].node_port def read(self): try: @@ -138,4 +139,4 @@ def get_external_ips(self): % (self.name)) return False return True - # end verify_service_in_contrail_api + # end verify_service_in_contrail_api diff --git a/scripts/k8s_scripts/test_service.py b/scripts/k8s_scripts/test_service.py index 4083ca524..bf4676724 100644 --- a/scripts/k8s_scripts/test_service.py +++ b/scripts/k8s_scripts/test_service.py @@ -1,11 +1,13 @@ from tcutils.util import skip_because -import random +#import random from common.k8s.base import BaseK8sTest from k8s.namespace import NamespaceFixture from k8s.service import ServiceFixture from tcutils.wrappers import preposttest_wrapper from tcutils.util import get_lock import test +from tcutils.util import get_random_name + class TestService(BaseK8sTest): @@ -50,6 +52,118 @@ def test_service_1(self): test_pod=pod3) # end test_service_1 + @preposttest_wrapper + def test_service_type_nodeport_without_namespace_isolation(self): + ''' Create a service with 2 pods running nginx + Create a third busybox pod and validate that webservice is + load-balanced using NodePort service from with in the cluster + outside of the cluster + ''' + app = 'http_nodeport_test' + labels = {'app': app} + namespace = self.setup_namespace("ns1") + assert namespace.verify_on_setup() + + service = self.setup_http_service(namespace=namespace.name, + labels=labels, type="NodePort") + pod1 = self.setup_nginx_pod(namespace=namespace.name, labels=labels) + pod2 = self.setup_nginx_pod(namespace=namespace.name, labels=labels) + pod3 = self.setup_busybox_pod(namespace=namespace.name) + + assert self.verify_nginx_pod(pod1) + assert self.verify_nginx_pod(pod2) + assert pod3.verify_on_setup() + # validate Service Nodeport functionality with load-balancing on the service + for node_ip in self.inputs.k8s_slave_ips: + assert self.validate_nginx_lb([pod1, pod2], node_ip, + test_pod=pod3, + nodePort=service.nodePort) + assert self.validate_nginx_lb([pod1, pod2], node_ip, + nodePort=service.nodePort) + # end test_service_type_nodeport__without_namespace_isolation + + @preposttest_wrapper + def test_service_type_nodeport_with_namespace_isolation(self): + ''' Create a service with 2 pods running nginx + Create a third busybox pod and validate that webservice is + load-balanced using NodePort service using NodePort service from with in the cluster + outside of the cluster with namespace isolation + + ''' + app = 'http_nodeport_test' + labels = {'app': app} + namespace1_name = get_random_name("ns1") + namespace2_name = get_random_name("ns2") + namespace1 = self.setup_namespace(name = namespace1_name, isolation = True) + namespace2 = self.setup_namespace(name = namespace2_name, isolation = True) + assert namespace1.verify_on_setup() + assert namespace2.verify_on_setup() + service = self.setup_http_service(namespace=namespace1.name, + labels=labels, type="NodePort") + pod1 = self.setup_nginx_pod(namespace=namespace1.name, labels=labels) + pod2 = self.setup_nginx_pod(namespace=namespace1.name, labels=labels) + pod3 = self.setup_busybox_pod(namespace=namespace1.name) + pod4 = self.setup_busybox_pod(namespace=namespace2.name) + assert self.verify_nginx_pod(pod1) + assert self.verify_nginx_pod(pod2) + assert pod3.verify_on_setup() + # validate Service Nodeport functionality with load-balancing on the service + for node_ip in self.inputs.k8s_slave_ips: + assert self.validate_nginx_lb([pod1, pod2], node_ip, + test_pod=pod3, + nodePort=service.nodePort) + #access the Nodeport from outside the cluster + assert self.validate_nginx_lb([pod1, pod2], node_ip, + nodePort=service.nodePort) + assert self.validate_nginx_lb([pod1, pod2], node_ip, + test_pod=pod4, + nodePort=service.nodePort, + expectation=False) + # end test_service_type_nodeport_with_namespace_isolation + + @preposttest_wrapper + def test_service_type_nodeport_with_namespace_isolation_with_userdefined_nodeport(self): + ''' Create a service with 2 pods running nginx + Create a third busybox pod and validate that webservice is + load-balanced NodePort service from with in the cluster + outside of the cluster with user defied noport + + ''' + app = 'http_nodeport_test' + labels = {'app': app} + user_node_port = 31111 + namespace1_name = get_random_name("ns1") + namespace2_name = get_random_name("ns2") + namespace1 = self.setup_namespace(name = namespace1_name, isolation = True) + namespace2 = self.setup_namespace(name = namespace2_name, isolation = True) + assert namespace1.verify_on_setup() + assert namespace2.verify_on_setup() + service = self.setup_http_service(namespace=namespace1.name, + labels=labels, type="NodePort", nodePort = user_node_port) + pod1 = self.setup_nginx_pod(namespace=namespace1.name, labels=labels) + pod2 = self.setup_nginx_pod(namespace=namespace1.name, labels=labels) + pod3 = self.setup_busybox_pod(namespace=namespace1.name) + pod4 = self.setup_busybox_pod(namespace=namespace2.name) + assert self.verify_nginx_pod(pod1) + assert self.verify_nginx_pod(pod2) + assert pod3.verify_on_setup() + assert (service.nodePort == user_node_port) , "service should be having the node port which is user provided" + # validate Service Nodeport functionality with load-balancing on the service + for node_ip in self.inputs.k8s_slave_ips: + assert self.validate_nginx_lb([pod1, pod2], node_ip, + test_pod=pod3, + nodePort=user_node_port) + #access the Nodeport from outside the cluster + assert self.validate_nginx_lb([pod1, pod2], node_ip, + nodePort=user_node_port) + assert self.validate_nginx_lb([pod1, pod2], node_ip, + test_pod=pod4, + nodePort=user_node_port, + expectation=False) + # end test_service_type_nodeport_with_namespace_isolation + + + @skip_because(mx_gw = False) @preposttest_wrapper def test_service_with_type_loadbalancer(self): @@ -82,8 +196,8 @@ def test_service_with_type_loadbalancer(self): assert self.validate_nginx_lb([pod1, pod2], service.cluster_ip, test_pod=pod3) - # When Isolation enabled we need to change SG to allow traffic - # from outside. For that we need to disiable service isolation + # When Isolation enabled we need to change SG to allow traffic + # from outside. For that we need to disiable service isolation if self.setup_namespace_isolation: namespace.disable_service_isolation() @@ -97,9 +211,9 @@ def test_service_access_from_different_ns(self): ''' Create a service in one namespace with 2 pods Create a third busybox pod in different namespace Validate busybox pod can access the service in - default mode. + default mode. When isolation is enabled service should not be - accessible from other namespace + accessible from other namespace ''' expectation = True app = 'http_test' @@ -187,6 +301,7 @@ def test_service_scale_up_down(self): # end test_service_scale_up_down + @test.attr(type=['k8s_sanity']) @preposttest_wrapper def test_kube_dns_lookup(self): namespace = self.setup_namespace() diff --git a/serial_scripts/k8s_scripts/test_service.py b/serial_scripts/k8s_scripts/test_service.py index 89a6a3009..790656009 100644 --- a/serial_scripts/k8s_scripts/test_service.py +++ b/serial_scripts/k8s_scripts/test_service.py @@ -7,6 +7,7 @@ from tcutils.util import get_lock import test from tcutils.util import skip_because +from tcutils.util import get_random_name class TestService(BaseK8sTest): @@ -30,7 +31,7 @@ def test_service_with_kube_manager_restart(self): Create a third busybox pod and validate that webservice is load-balanced Validate that webservice is load-balanced from outside network - Restart kube-manager and verify service + Restart kube-manager and verify service Please make sure BGP multipath and per packer load balancing is enabled on the MX ''' @@ -66,7 +67,7 @@ def test_service_with_kube_manager_restart(self): assert self.validate_nginx_lb([pod1, pod2], service.external_ips[0]) self.restart_kube_manager() - + # Now validate load-balancing on the service assert self.validate_nginx_lb([pod1, pod2], service.cluster_ip, test_pod=pod3) @@ -74,3 +75,130 @@ def test_service_with_kube_manager_restart(self): # Now validate ingress from public network assert self.validate_nginx_lb([pod1, pod2], service.external_ips[0]) + + def common_checks_for_nodeport_service(self, svclist, pdslist) : + """commo routine to validate the nodeport service across all the nodes + with user definned nodeport ad also the auto assigned nodeport + 1.access the mnodeport from outside the cluster + 2.access the odeport from with in the cluster + a. with in the same namespace + b. from the different namespace + """ + for node_ip in self.inputs.k8s_slave_ips: + import pdb;pdb.set_trace() + assert self.validate_nginx_lb([pdslist[0], pdslist[1]], node_ip, + test_pod=pdslist[2], + nodePort=svclist[0].nodePort) + assert self.validate_nginx_lb([pdslist[3], pdslist[4]], node_ip, + test_pod=pdslist[5], + nodePort=svclist[1].nodePort) + + #access the Nodeport from outside the cluster + assert self.validate_nginx_lb([pdslist[0], pdslist[1]], node_ip, + nodePort=svclist[0].nodePort) + assert self.validate_nginx_lb([pdslist[3], pdslist[4]], node_ip, + nodePort=svclist[1].nodePort) + + #access the Nodeport from outside of the anmespaces + assert self.validate_nginx_lb([pdslist[0], pdslist[1]], node_ip, + test_pod=pdslist[5], + nodePort=svclist[0].nodePort, + expectation=False) + assert self.validate_nginx_lb([pdslist[3], pdslist[4]], node_ip, + test_pod=pdslist[2], + nodePort=svclist[1].nodePort, + expectation=False) + #End of common_checks_for_nodeport_service + + def common_setup_for_nodeport(self, userport=None, tag1="http_nodeport_test", + tag2=None, isolation=True): + + labels = {'app': tag1} + if tag2 is None : + tag2 = tag1 + labels1 = {'app': tag2} + namespace1_name = get_random_name("ns1") + namespace2_name = get_random_name("ns2") + namespace1 = self.setup_namespace(name=namespace1_name, isolation=isolation) + namespace2 = self.setup_namespace(name=namespace2_name, isolation=isolation) + assert namespace1.verify_on_setup() + assert namespace2.verify_on_setup() + service_in_ns1 = self.setup_http_service(namespace=namespace1.name, + labels=labels, type="NodePort", nodePort=userport) + service_in_ns2 = self.setup_http_service(namespace=namespace2.name, + labels=labels1, type="NodePort") + pod1 = self.setup_nginx_pod(namespace=namespace1.name, labels=labels) + pod2 = self.setup_nginx_pod(namespace=namespace1.name, labels=labels) + pod3 = self.setup_busybox_pod(namespace=namespace1.name) + pod4 = self.setup_nginx_pod(namespace=namespace2.name, labels=labels1) + pod5 = self.setup_nginx_pod(namespace=namespace2.name, labels=labels1) + pod6 = self.setup_busybox_pod(namespace=namespace2.name) + assert self.verify_nginx_pod(pod1) + assert self.verify_nginx_pod(pod2) + assert self.verify_nginx_pod(pod4) + assert self.verify_nginx_pod(pod5) + assert pod3.verify_on_setup() + assert pod6.verify_on_setup() + namespace_list=[namespace1,namespace2] + service_list=[service_in_ns1,service_in_ns2] + pod_list=[pod1,pod2,pod3,pod4,pod5,pod6] + return (service_list,pod_list) + + @preposttest_wrapper + def test_kubelet_restart_on_slaves_with_multiple_nodeport_servoces_with_namespace_isolation(self): + app1 = 'http_nodeport_test1' + user_node_port = 31111 + (svc,pds) = self.common_setup_for_nodeport(user_node_port, tag2=app1) + assert (svc[0].nodePort == user_node_port),"nodeport is not refclecting" + self.common_checks_for_nodeport_service(svc,pds) + self.inputs.restart_service(service_name = "kubelet", + host_ips = self.inputs.k8s_slave_ips) + time.sleep(30) # Wait timer for all kubernetes pods to stablise. + self.common_checks_for_nodeport_service(svc,pds) + + @preposttest_wrapper + def test_docker_restart_on_slaves_with_multiple_nodeports_with_namespace_isolation(self): + user_node_port = 30001 + (svc,pds) = self.common_setup_for_nodeport(user_node_port) + assert (svc[0].nodePort == user_node_port),"nodeport is not refclecting" + self.common_checks_for_nodeport_service(svc, pds) + self.inputs.restart_service(service_name = "docker", + host_ips = self.inputs.k8s_slave_ips) + time.sleep(60) # Wait timer for all contrail service to come up. + self.common_checks_for_nodeport_service(svc, pds) + + @preposttest_wrapper + def test_vrouter_restart_on_slaves_with_multiple_nodeports_with_namespace_isolation(self): + (svc,pds) = self.common_setup_for_nodeport() + self.common_checks_for_nodeport_service(svc, pds) + # Restart Vrouter agent + self.restart_vrouter_agent() + self.common_checks_for_nodeport_service(svc, pds) + + @test.attr(type=['k8s_sanity']) + @preposttest_wrapper + def test_kube_manager_restart_with_multiple_nodeport_services_with_namespace_isolation(self): + (svc,pds) = self.common_setup_for_nodeport() + self.common_checks_for_nodeport_service(svc, pds) + # Restart Kube manager + self.restart_kube_manager() + self.common_checks_for_nodeport_service(svc, pds) + + @preposttest_wrapper + def test_reboot_slaves_with_multiple_nodeport_servoces_with_namespace_isolation(self): + (svc,pds) = self.common_setup_for_nodeport() + self.common_checks_for_nodeport_service(svc, pds) + for node in self.inputs.k8s_slave_ips: + self.inputs.reboot(node) + self.common_checks_for_nodeport_service(svc, pds) + + @preposttest_wrapper + def test_docker_restart_on_master_with_multiple_nodeport_servoces_with_namespace_isolation(self): + user_node_port = 30002 + (svc,pds) = self.common_setup_for_nodeport(user_node_port) + assert (svc[0].nodePort == user_node_port),"nodeport is not refclecting" + self.common_checks_for_nodeport_service(svc, pds) + self.inputs.restart_service(service_name = "docker", + host_ips = [self.inputs.k8s_master_ip]) + time.sleep(60) # Wait timer for all contrail service to come up. + self.common_checks_for_nodeport_service(svc,pds)