From f8ec6de4d26100ade3c4c5508ea83ed170ddbb5b Mon Sep 17 00:00:00 2001 From: Roman Dobosz Date: Thu, 23 Jul 2020 15:27:23 +0200 Subject: [PATCH 1/6] Use plugin instead of devstack service for Neutron. In our devstack local.conf samples we are using invocation for enabling Neutron like: enable_service neutron which shouldn't be used, in favor of: enable_plugin neutron https://opendev.org/openstack/neutron For some strange reason, first invocation results in failing of create the stack with enabled Neutron metadata service, which is needed for kuryr tempest test to be run properly. Depends-On: If639b63dcf660ed709623c8d5f788026619c895c Change-Id: I3cd8e2a46e65e65195d4ee17e973980baf83618d --- devstack/local.conf.odl.sample | 5 +++-- devstack/local.conf.openshift.sample | 5 +++-- devstack/local.conf.pod-in-vm.undercloud.odl.sample | 2 +- devstack/local.conf.sample | 5 +++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/devstack/local.conf.odl.sample b/devstack/local.conf.odl.sample index 243464887..581c7f2b2 100644 --- a/devstack/local.conf.odl.sample +++ b/devstack/local.conf.odl.sample @@ -27,10 +27,11 @@ IDENTITY_API_VERSION=3 ENABLED_SERVICES="" # Neutron services -enable_service neutron +enable_plugin neutron https://opendev.org/openstack/neutron enable_service q-dhcp -enable_service q-svc +enable_service q-api enable_service q-meta +enable_service q-svc # LBaaSv2 service and Haproxy agent enable_plugin neutron-lbaas \ diff --git a/devstack/local.conf.openshift.sample b/devstack/local.conf.openshift.sample index 04d918bb9..d9a0f7eae 100644 --- a/devstack/local.conf.openshift.sample +++ b/devstack/local.conf.openshift.sample @@ -27,12 +27,13 @@ IDENTITY_API_VERSION=3 ENABLED_SERVICES="" # Neutron services -enable_service neutron +enable_plugin neutron https://opendev.org/openstack/neutron enable_service q-agt enable_service q-dhcp enable_service q-l3 -enable_service q-svc +enable_service q-api enable_service q-meta +enable_service q-svc # OCTAVIA # Uncomment it to use L2 communication between loadbalancer and member pods diff --git a/devstack/local.conf.pod-in-vm.undercloud.odl.sample b/devstack/local.conf.pod-in-vm.undercloud.odl.sample index 7f7f009f2..85435f164 100644 --- a/devstack/local.conf.pod-in-vm.undercloud.odl.sample +++ b/devstack/local.conf.pod-in-vm.undercloud.odl.sample @@ -24,7 +24,7 @@ IDENTITY_API_VERSION=3 ENABLED_SERVICES="" # Neutron services -enable_service neutron +enable_plugin neutron https://opendev.org/openstack/neutron enable_service q-dhcp enable_service q-svc enable_service q-meta diff --git a/devstack/local.conf.sample b/devstack/local.conf.sample index 0d07a6601..f6744105f 100644 --- a/devstack/local.conf.sample +++ b/devstack/local.conf.sample @@ -27,12 +27,13 @@ IDENTITY_API_VERSION=3 ENABLED_SERVICES="" # Neutron services -enable_service neutron +enable_plugin neutron https://opendev.org/openstack/neutron enable_service q-agt enable_service q-dhcp enable_service q-l3 -enable_service q-svc +enable_service q-api enable_service q-meta +enable_service q-svc # VAR RUN PATH # ============= From b53bcf946fe7aa3c08c94e9529d38d67e66d6389 Mon Sep 17 00:00:00 2001 From: Roman Dobosz Date: Mon, 27 Jul 2020 09:45:50 +0200 Subject: [PATCH 2/6] Fix coverage environment in tox. Currently, running tox -e coverage will result with error. Shamelessly stolen commands from Nova project to fix that. Change-Id: If09c65e895a31daac56afff846861815390a520c --- tox.ini | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 013702fe3..36f314df3 100644 --- a/tox.ini +++ b/tox.ini @@ -35,10 +35,15 @@ commands = {posargs} [testenv:cover] basepython = python3 +setenv = + {[testenv]setenv} + PYTHON=coverage run --source kuryr_kubernetes --parallel-mode commands = - rm -f .testrepository/times.dbm - python setup.py test --coverage --testr-args={posargs} \ - --coverage-package-name=kuryr_kubernetes + coverage erase + stestr run {posargs} + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml coverage report [testenv:docs] From a458fa68942ed13957cf1a443e2d2bbf287d0bda Mon Sep 17 00:00:00 2001 From: Roman Dobosz Date: Tue, 26 May 2020 13:03:07 +0000 Subject: [PATCH 3/6] Pod annotations to KuryrPort CRD. Till now, we were using pod annotations to store information regarding state of the associated VIFs to pod. This alone have its own issues and it's prone to the inconsistency in case of controller failures. In this patch we propose new CRD called KuryrPort for storage the information about VIFs. Depends-On: If639b63dcf660ed709623c8d5f788026619c895c Change-Id: I1e76ea949120f819dcab6d07714522a576e426f2 --- .zuul.d/octavia.yaml | 6 +- .zuul.d/sdn.yaml | 4 +- devstack/lib/kuryr_kubernetes | 1 + devstack/plugin.sh | 1 + devstack/settings | 2 +- kubernetes_crds/kuryr_crds/kuryrport.yaml | 48 ++ kuryr_kubernetes/cni/binding/dpdk.py | 64 +- kuryr_kubernetes/cni/daemon/service.py | 78 +- kuryr_kubernetes/cni/handlers.py | 43 +- .../cni/plugins/k8s_cni_registry.py | 127 ++- kuryr_kubernetes/constants.py | 11 +- .../controller/drivers/neutron_vif.py | 6 +- kuryr_kubernetes/controller/drivers/utils.py | 46 +- .../controller/drivers/vif_pool.py | 14 +- .../controller/handlers/kuryrport.py | 267 +++++++ .../controller/handlers/pod_label.py | 16 +- kuryr_kubernetes/controller/handlers/vif.py | 250 ++---- kuryr_kubernetes/k8s_client.py | 2 +- .../unit/cni/plugins/test_k8s_cni_registry.py | 21 +- .../unit/controller/drivers/test_vif_pool.py | 24 +- .../controller/handlers/test_kuryrport.py | 751 ++++++++++++++++++ .../controller/handlers/test_pod_label.py | 18 +- .../unit/controller/handlers/test_vif.py | 313 ++++---- kuryr_kubernetes/utils.py | 19 +- setup.cfg | 1 + tools/gate/copy_k8s_logs.sh | 1 + 26 files changed, 1548 insertions(+), 586 deletions(-) create mode 100644 kubernetes_crds/kuryr_crds/kuryrport.yaml create mode 100644 kuryr_kubernetes/controller/handlers/kuryrport.py create mode 100644 kuryr_kubernetes/tests/unit/controller/handlers/test_kuryrport.py diff --git a/.zuul.d/octavia.yaml b/.zuul.d/octavia.yaml index de1d2d9bc..04ffdc5a5 100644 --- a/.zuul.d/octavia.yaml +++ b/.zuul.d/octavia.yaml @@ -99,7 +99,7 @@ vars: devstack_localrc: DOCKER_CGROUP_DRIVER: "systemd" - KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork + KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport KURYR_SG_DRIVER: policy KURYR_SUBNET_DRIVER: namespace devstack_services: @@ -120,7 +120,7 @@ vars: devstack_localrc: KURYR_SUBNET_DRIVER: namespace - KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork + KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport KURYR_SG_DRIVER: policy KURYR_USE_PORT_POOLS: true KURYR_POD_VIF_DRIVER: neutron-vif @@ -134,7 +134,7 @@ parent: kuryr-kubernetes-tempest-containerized vars: devstack_localrc: - KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork + KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport KURYR_SG_DRIVER: policy KURYR_SUBNET_DRIVER: namespace diff --git a/.zuul.d/sdn.yaml b/.zuul.d/sdn.yaml index d38bb589e..0e2ae0af3 100644 --- a/.zuul.d/sdn.yaml +++ b/.zuul.d/sdn.yaml @@ -98,7 +98,7 @@ KURYR_LB_ALGORITHM: SOURCE_IP_PORT KURYR_SUBNET_DRIVER: namespace KURYR_SG_DRIVER: policy - KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork + KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport voting: false - job: @@ -144,7 +144,7 @@ KURYR_ENFORCE_SG_RULES: false KURYR_LB_ALGORITHM: SOURCE_IP_PORT KURYR_HYPERKUBE_VERSION: v1.16.0 - KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork + KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport KURYR_SG_DRIVER: policy KURYR_SUBNET_DRIVER: namespace KURYR_K8S_CONTAINERIZED_DEPLOYMENT: true diff --git a/devstack/lib/kuryr_kubernetes b/devstack/lib/kuryr_kubernetes index 731017d1d..e51686811 100644 --- a/devstack/lib/kuryr_kubernetes +++ b/devstack/lib/kuryr_kubernetes @@ -453,6 +453,7 @@ rules: - kuryrnetworks - kuryrnetpolicies - kuryrloadbalancers + - kuryrports - apiGroups: ["networking.k8s.io"] resources: - networkpolicies diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 1fd4c4263..1b2380220 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -973,6 +973,7 @@ function update_tempest_conf_file { fi iniset $TEMPEST_CONFIG kuryr_kubernetes validate_crd True iniset $TEMPEST_CONFIG kuryr_kubernetes kuryrnetworks True + iniset $TEMPEST_CONFIG kuryr_kubernetes kuryrports True } source $DEST/kuryr-kubernetes/devstack/lib/kuryr_kubernetes diff --git a/devstack/settings b/devstack/settings index edd61626f..81bf8487e 100644 --- a/devstack/settings +++ b/devstack/settings @@ -43,7 +43,7 @@ KURYR_K8S_API_LB_PORT=${KURYR_K8S_API_LB_PORT:-443} KURYR_PORT_DEBUG=${KURYR_PORT_DEBUG:-True} KURYR_SUBNET_DRIVER=${KURYR_SUBNET_DRIVER:-default} KURYR_SG_DRIVER=${KURYR_SG_DRIVER:-default} -KURYR_ENABLED_HANDLERS=${KURYR_ENABLED_HANDLERS:-vif,lb,lbaasspec} +KURYR_ENABLED_HANDLERS=${KURYR_ENABLED_HANDLERS:-vif,lb,lbaasspec,kuryrport} # OpenShift OPENSHIFT_BINARY_VERSION=${OPENSHIFT_BINARY_VERSION:-v3.11.0} diff --git a/kubernetes_crds/kuryr_crds/kuryrport.yaml b/kubernetes_crds/kuryr_crds/kuryrport.yaml new file mode 100644 index 000000000..21767300c --- /dev/null +++ b/kubernetes_crds/kuryr_crds/kuryrport.yaml @@ -0,0 +1,48 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kuryrports.openstack.org +spec: + group: openstack.org + scope: Namespaced + names: + plural: kuryrports + singular: kuryrport + kind: KuryrPort + shortNames: + - kp + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: + - podUid + - podNodeName + - vifs + properties: + podUid: + type: string + podNodeName: + type: string + vifs: + type: object + x-kubernetes-preserve-unknown-fields: true + additionalPrinterColumns: + - name: PodUID + type: string + description: Pod UID + jsonPath: .spec.podUid + - name: Nodename + type: string + description: Name of the node corresponding pod lives in + jsonPath: .spec.podNodeName + - name: labels + type: string + description: Labels for the CRD + jsonPath: .metadata.labels diff --git a/kuryr_kubernetes/cni/binding/dpdk.py b/kuryr_kubernetes/cni/binding/dpdk.py index 6a7b2695a..4685582ba 100644 --- a/kuryr_kubernetes/cni/binding/dpdk.py +++ b/kuryr_kubernetes/cni/binding/dpdk.py @@ -15,6 +15,7 @@ import os +from os_vif import objects from oslo_config import cfg from oslo_log import log as logging from oslo_serialization import jsonutils @@ -23,7 +24,6 @@ from kuryr_kubernetes.cni.binding import base as b_base from kuryr_kubernetes import constants from kuryr_kubernetes.handlers import health -from kuryr_kubernetes import utils from kuryr.lib._i18n import _ @@ -143,42 +143,46 @@ def _remove_pci_file(self, container_id, ifname): def _set_vif(self, vif): # TODO(ivc): extract annotation interactions - state, labels, resource_version = self._get_pod_details( + vifs, labels, resource_version, kp_link = self._get_pod_details( vif.port_profile.selflink) - for ifname, vif_ex in state.vifs.items(): - if vif.id == vif_ex.id: - state.vifs[ifname] = vif + for ifname, data in vifs.items(): + if vif.id == data['vif'].id: + vifs[ifname] = data break - self._set_pod_details(state, vif.port_profile.selflink, labels, - resource_version) + self._set_pod_details(vifs, vif.port_profile.selflink, labels, + resource_version, kp_link) def _get_pod_details(self, selflink): k8s = clients.get_kubernetes_client() pod = k8s.get(selflink) - annotations = pod['metadata']['annotations'] - resource_version = pod['metadata']['resourceVersion'] - labels = pod['metadata'].get('labels') + kp = k8s.get(f'{constants.K8S_API_CRD_NAMESPACES}/' + f'{pod["metadata"]["namespace"]}/kuryrports/' + f'{pod["metadata"]["name"]}') + try: - annotations = annotations[constants.K8S_ANNOTATION_VIF] - state_annotation = jsonutils.loads(annotations) - state = utils.extract_pod_annotation(state_annotation) - except KeyError: - LOG.exception("No annotations %s", constants.K8S_ANNOTATION_VIF) + vifs = {k: {'default': v['default'], + 'vif': objects.base.VersionedObject + .obj_from_primitive(v['vif'])} + for k, v in kp['spec']['vifs'].items()} + except (KeyError, AttributeError): + LOG.exception(f"No vifs found on KuryrPort: {kp}") raise - except ValueError: - LOG.exception("Unable encode annotations") - raise - LOG.info("Got VIFs from annotation: %s", state.vifs) - return state, labels, resource_version + LOG.info(f"Got VIFs from Kuryrport: {vifs}") - def _set_pod_details(self, state, selflink, labels, resource_version): - if not state: - LOG.info("Removing VIFs annotation: %r", state) - annotation = None - else: - state_dict = state.obj_to_primitive() - annotation = jsonutils.dumps(state_dict, sort_keys=True) - LOG.info("Setting VIFs annotation: %r", annotation) + resource_version = pod['metadata']['resourceVersion'] + labels = pod['metadata'].get('labels') + return vifs, labels, resource_version, kp['metadata']['selflink'] + + def _set_pod_details(self, vifs, selflink, labels, resource_version, + kp_link): + k8s = clients.get_kubernetes_client() + if vifs: + spec = {k: {'default': v['default'], + 'vif': v['vif'].obj_to_primitive()} + for k, v in vifs.items()} + + LOG.info("Setting VIFs in KuryrPort %r", spec) + k8s.patch_crd('spec', kp_link, {'vifs': spec}) if not labels: LOG.info("Removing Label annotation: %r", labels) @@ -187,8 +191,6 @@ def _set_pod_details(self, state, selflink, labels, resource_version): labels_annotation = jsonutils.dumps(labels, sort_keys=True) LOG.info("Setting Labels annotation: %r", labels_annotation) - k8s = clients.get_kubernetes_client() k8s.annotate(selflink, - {constants.K8S_ANNOTATION_VIF: annotation, - constants.K8S_ANNOTATION_LABEL: labels_annotation}, + {constants.K8S_ANNOTATION_LABEL: labels_annotation}, resource_version=resource_version) diff --git a/kuryr_kubernetes/cni/daemon/service.py b/kuryr_kubernetes/cni/daemon/service.py index 683adf652..ac2ce4f33 100644 --- a/kuryr_kubernetes/cni/daemon/service.py +++ b/kuryr_kubernetes/cni/daemon/service.py @@ -20,14 +20,14 @@ import sys import threading import time +import urllib.parse +import urllib3 import cotyledon import flask from pyroute2.ipdb import transactional -import urllib3 import os_vif -from os_vif.objects import base from oslo_concurrency import lockutils from oslo_config import cfg from oslo_log import log as logging @@ -193,10 +193,12 @@ def run(self): self.pipeline.register(h_cni.CallbackHandler(self.on_done, self.on_deleted)) self.watcher = k_watcher.Watcher(self.pipeline) - self.watcher.add( - "%(base)s/pods?fieldSelector=spec.nodeName=%(node_name)s" % { - 'base': k_const.K8S_API_BASE, - 'node_name': self._get_nodename()}) + query_label = urllib.parse.quote_plus(f'{k_const.KURYRPORT_LABEL}=' + f'{self._get_nodename()}') + + self.watcher.add(f'{k_const.K8S_API_CRD_KURYRPORTS}' + f'?labelSelector={query_label}') + self.is_running = True self.health_thread = threading.Thread( target=self._start_watcher_health_checker) @@ -211,55 +213,43 @@ def _start_watcher_health_checker(self): self.healthy.value = False time.sleep(HEALTH_CHECKER_DELAY) - def on_done(self, pod, vifs): - pod_name = utils.get_pod_unique_name(pod) - vif_dict = { - ifname: vif.obj_to_primitive() for - ifname, vif in vifs.items() - } - # NOTE(dulek): We need a lock when modifying shared self.registry dict - # to prevent race conditions with other processes/threads. - with lockutils.lock(pod_name, external=True): - if (pod_name not in self.registry or - self.registry[pod_name]['pod']['metadata']['uid'] - != pod['metadata']['uid']): - self.registry[pod_name] = {'pod': pod, 'vifs': vif_dict, - 'containerid': None, - 'vif_unplugged': False, - 'del_received': False} + def on_done(self, kuryrport, vifs): + kp_name = utils.get_res_unique_name(kuryrport) + with lockutils.lock(kp_name, external=True): + if (kp_name not in self.registry or + self.registry[kp_name]['kp']['metadata']['uid'] + != kuryrport['metadata']['uid']): + self.registry[kp_name] = {'kp': kuryrport, + 'vifs': vifs, + 'containerid': None, + 'vif_unplugged': False, + 'del_received': False} else: - # NOTE(dulek): Only update vif if its status changed, we don't - # need to care about other changes now. - old_vifs = { - ifname: - base.VersionedObject.obj_from_primitive(vif_obj) for - ifname, vif_obj in ( - self.registry[pod_name]['vifs'].items()) - } + old_vifs = self.registry[kp_name]['vifs'] for iface in vifs: if old_vifs[iface].active != vifs[iface].active: - pod_dict = self.registry[pod_name] - pod_dict['vifs'] = vif_dict - self.registry[pod_name] = pod_dict + kp_dict = self.registry[kp_name] + kp_dict['vifs'] = vifs + self.registry[kp_name] = kp_dict - def on_deleted(self, pod): - pod_name = utils.get_pod_unique_name(pod) + def on_deleted(self, kp): + kp_name = utils.get_res_unique_name(kp) try: - if pod_name in self.registry: + if kp_name in self.registry: # NOTE(ndesh): We need to lock here to avoid race condition # with the deletion code for CNI DEL so that # we delete the registry entry exactly once - with lockutils.lock(pod_name, external=True): - if self.registry[pod_name]['vif_unplugged']: - del self.registry[pod_name] + with lockutils.lock(kp_name, external=True): + if self.registry[kp_name]['vif_unplugged']: + del self.registry[kp_name] else: - pod_dict = self.registry[pod_name] - pod_dict['del_received'] = True - self.registry[pod_name] = pod_dict + kp_dict = self.registry[kp_name] + kp_dict['del_received'] = True + self.registry[kp_name] = kp_dict except KeyError: # This means someone else removed it. It's odd but safe to ignore. - LOG.debug('Pod %s entry already removed from registry while ' - 'handling DELETED event. Ignoring.', pod_name) + LOG.debug('KuryrPort %s entry already removed from registry while ' + 'handling DELETED event. Ignoring.', kp_name) pass def terminate(self): diff --git a/kuryr_kubernetes/cni/handlers.py b/kuryr_kubernetes/cni/handlers.py index aded36895..222dabfc3 100644 --- a/kuryr_kubernetes/cni/handlers.py +++ b/kuryr_kubernetes/cni/handlers.py @@ -17,18 +17,20 @@ from os_vif import objects as obj_vif from oslo_log import log as logging -from oslo_serialization import jsonutils +from kuryr_kubernetes import clients from kuryr_kubernetes import constants as k_const +from kuryr_kubernetes import exceptions as k_exc from kuryr_kubernetes.handlers import dispatch as k_dis from kuryr_kubernetes.handlers import k8s_base from kuryr_kubernetes import utils + LOG = logging.getLogger(__name__) class CNIHandlerBase(k8s_base.ResourceEventHandler, metaclass=abc.ABCMeta): - OBJECT_KIND = k_const.K8S_OBJ_POD + OBJECT_KIND = k_const.K8S_OBJ_KURYRPORT def __init__(self, cni, on_done): self._cni = cni @@ -59,16 +61,18 @@ def callback(self): raise NotImplementedError() def _get_vifs(self, pod): - # TODO(ivc): same as VIFHandler._get_vif + k8s = clients.get_kubernetes_client() try: - annotations = pod['metadata']['annotations'] - state_annotation = annotations[k_const.K8S_ANNOTATION_VIF] - except KeyError: + kuryrport_crd = k8s.get(f'{k_const.K8S_API_CRD_NAMESPACES}/' + f'{pod["metadata"]["namespace"]}/' + f'kuryrports/{pod["metadata"]["name"]}') + LOG.debug("Got CRD: %r", kuryrport_crd) + except k_exc.K8sClientException: return {} - state_annotation = jsonutils.loads(state_annotation) - state = utils.extract_pod_annotation(state_annotation) - vifs_dict = state.vifs - LOG.debug("Got VIFs from annotation: %r", vifs_dict) + + vifs_dict = utils.get_vifs_from_crd(kuryrport_crd) + LOG.debug("Got vifs: %r", vifs_dict) + return vifs_dict def _get_inst(self, pod): @@ -81,31 +85,32 @@ class CallbackHandler(CNIHandlerBase): def __init__(self, on_vif, on_del=None): super(CallbackHandler, self).__init__(None, on_vif) self._del_callback = on_del - self._pod = None + self._kuryrport = None self._callback_vifs = None - def should_callback(self, pod, vifs): + def should_callback(self, kuryrport, vifs): """Called after all vifs have been processed - Calls callback if there was at least one vif in the Pod + Calls callback if there was at least one vif in the CRD - :param pod: dict containing Kubernetes Pod object + :param kuryrport: dict containing Kubernetes KuryrPort CRD object :param vifs: dict containing os_vif VIF objects and ifnames :returns True/False """ - self._pod = pod + self._kuryrport = kuryrport self._callback_vifs = vifs if vifs: return True return False def callback(self): - self._callback(self._pod, self._callback_vifs) + self._callback(self._kuryrport, self._callback_vifs) - def on_deleted(self, pod): - LOG.debug("Got pod %s deletion event.", pod['metadata']['name']) + def on_deleted(self, kuryrport): + LOG.debug("Got kuryrport %s deletion event.", + kuryrport['metadata']['name']) if self._del_callback: - self._del_callback(pod) + self._del_callback(kuryrport) class CNIPipeline(k_dis.EventPipeline): diff --git a/kuryr_kubernetes/cni/plugins/k8s_cni_registry.py b/kuryr_kubernetes/cni/plugins/k8s_cni_registry.py index 13bec826b..57dfbf8ab 100644 --- a/kuryr_kubernetes/cni/plugins/k8s_cni_registry.py +++ b/kuryr_kubernetes/cni/plugins/k8s_cni_registry.py @@ -15,7 +15,6 @@ import retrying from os_vif import objects as obj_vif -from os_vif.objects import base from oslo_concurrency import lockutils from oslo_config import cfg from oslo_log import log as logging @@ -31,12 +30,14 @@ CONF = cfg.CONF RETRY_DELAY = 1000 # 1 second in milliseconds -# TODO(dulek): Another corner case is (and was) when pod is deleted before it's -# annotated by controller or even noticed by any watcher. Kubelet -# will try to delete such vif, but we will have no data about it. -# This is currently worked around by returning successfully in -# case of timing out in delete. To solve this properly we need -# to watch for pod deletes as well. +# TODO(dulek, gryf): Another corner case is (and was) when pod is deleted +# before it's corresponding CRD was created and populated by vifs by +# controller or even noticed by any watcher. Kubelet will try to delete such +# vif, but we will have no data about it. This is currently worked around by +# returning successfully in case of timing out in delete. To solve this +# properly we need to watch for pod deletes as well, or perhaps create +# finalizer for the pod as soon, as we know, that kuryrport CRD will be +# created. class K8sCNIRegistryPlugin(base_cni.CNIPlugin): @@ -45,32 +46,32 @@ def __init__(self, registry, healthy): self.registry = registry self.k8s = clients.get_kubernetes_client() - def _get_pod_name(self, params): + def _get_obj_name(self, params): return "%(namespace)s/%(name)s" % { 'namespace': params.args.K8S_POD_NAMESPACE, 'name': params.args.K8S_POD_NAME} def add(self, params): - pod_name = self._get_pod_name(params) + kp_name = self._get_obj_name(params) timeout = CONF.cni_daemon.vif_annotation_timeout - # Try to confirm if pod in the registry is not stale cache. If it is, + # Try to confirm if CRD in the registry is not stale cache. If it is, # remove it. - with lockutils.lock(pod_name, external=True): - if pod_name in self.registry: - cached_pod = self.registry[pod_name]['pod'] + with lockutils.lock(kp_name, external=True): + if kp_name in self.registry: + cached_kp = self.registry[kp_name]['kp'] try: - pod = self.k8s.get(cached_pod['metadata']['selfLink']) + kp = self.k8s.get(cached_kp['metadata']['selfLink']) except Exception: - LOG.exception('Error when getting pod %s', pod_name) - raise exceptions.ResourceNotReady(pod_name) + LOG.exception('Error when getting KuryrPort %s', kp_name) + raise exceptions.ResourceNotReady(kp_name) - if pod['metadata']['uid'] != cached_pod['metadata']['uid']: - LOG.warning('Stale pod %s detected in cache. (API ' + if kp['metadata']['uid'] != cached_kp['metadata']['uid']: + LOG.warning('Stale KuryrPort %s detected in cache. (API ' 'uid=%s, cached uid=%s). Removing it from ' - 'cache.', pod_name, pod['metadata']['uid'], - cached_pod['metadata']['uid']) - del self.registry[pod_name] + 'cache.', kp_name, kp['metadata']['uid'], + cached_kp['metadata']['uid']) + del self.registry[kp_name] vifs = self._do_work(params, b_base.connect, timeout) @@ -78,70 +79,68 @@ def add(self, params): # requests that we should ignore. We need a lock to # prevent race conditions and replace whole object in the # dict for multiprocessing.Manager to notice that. - with lockutils.lock(pod_name, external=True): - d = self.registry[pod_name] + with lockutils.lock(kp_name, external=True): + d = self.registry[kp_name] d['containerid'] = params.CNI_CONTAINERID - self.registry[pod_name] = d - LOG.debug('Saved containerid = %s for pod %s', - params.CNI_CONTAINERID, pod_name) + self.registry[kp_name] = d + LOG.debug('Saved containerid = %s for CRD %s', + params.CNI_CONTAINERID, kp_name) # Wait for timeout sec, 1 sec between tries, retry when even one # vif is not active. @retrying.retry(stop_max_delay=timeout * 1000, wait_fixed=RETRY_DELAY, retry_on_result=utils.any_vif_inactive) - def wait_for_active(pod_name): - return { - ifname: base.VersionedObject.obj_from_primitive(vif_obj) for - ifname, vif_obj in self.registry[pod_name]['vifs'].items() - } + def wait_for_active(kp_name): + return self.registry[kp_name]['vifs'] - vifs = wait_for_active(pod_name) + vifs = wait_for_active(kp_name) for vif in vifs.values(): if not vif.active: LOG.error("Timed out waiting for vifs to become active") - raise exceptions.ResourceNotReady(pod_name) + raise exceptions.ResourceNotReady(kp_name) return vifs[k_const.DEFAULT_IFNAME] def delete(self, params): - pod_name = self._get_pod_name(params) + kp_name = self._get_obj_name(params) try: - reg_ci = self.registry[pod_name]['containerid'] - LOG.debug('Read containerid = %s for pod %s', reg_ci, pod_name) + reg_ci = self.registry[kp_name]['containerid'] + LOG.debug('Read containerid = %s for KuryrPort %s', reg_ci, + kp_name) if reg_ci and reg_ci != params.CNI_CONTAINERID: # NOTE(dulek): This is a DEL request for some older (probably # failed) ADD call. We should ignore it or we'll # unplug a running pod. LOG.warning('Received DEL request for unknown ADD call for ' - 'pod %s (CNI_CONTAINERID=%s). Ignoring.', pod_name, - params.CNI_CONTAINERID) + 'Kuryrport %s (CNI_CONTAINERID=%s). Ignoring.', + kp_name, params.CNI_CONTAINERID) return except KeyError: pass # Passing arbitrary 5 seconds as timeout, as it does not make any sense - # to wait on CNI DEL. If pod got deleted from API - VIF info is gone. - # If pod got the annotation removed - it is now gone too. The number's - # not 0, because we need to anticipate for restarts and delay before - # registry is populated by watcher. + # to wait on CNI DEL. If kuryrport got deleted from API - VIF info is + # gone. If kuryrport got the vif info removed - it is now gone too. + # The number's not 0, because we need to anticipate for restarts and + # delay before registry is populated by watcher. self._do_work(params, b_base.disconnect, 5) # NOTE(ndesh): We need to lock here to avoid race condition # with the deletion code in the watcher to ensure that # we delete the registry entry exactly once try: - with lockutils.lock(pod_name, external=True): - if self.registry[pod_name]['del_received']: - del self.registry[pod_name] + with lockutils.lock(kp_name, external=True): + if self.registry[kp_name]['del_received']: + del self.registry[kp_name] else: - pod_dict = self.registry[pod_name] - pod_dict['vif_unplugged'] = True - self.registry[pod_name] = pod_dict + kp_dict = self.registry[kp_name] + kp_dict['vif_unplugged'] = True + self.registry[kp_name] = kp_dict except KeyError: - # This means the pod was removed before vif was unplugged. This - # shouldn't happen, but we can't do anything about it now - LOG.debug('Pod %s not found registry while handling DEL request. ' - 'Ignoring.', pod_name) + # This means the kuryrport was removed before vif was unplugged. + # This shouldn't happen, but we can't do anything about it now + LOG.debug('KuryrPort %s not found registry while handling DEL ' + 'request. Ignoring.', kp_name) pass def report_drivers_health(self, driver_healthy): @@ -151,25 +150,22 @@ def report_drivers_health(self, driver_healthy): self.healthy.value = driver_healthy def _do_work(self, params, fn, timeout): - pod_name = self._get_pod_name(params) + kp_name = self._get_obj_name(params) # In case of KeyError retry for `timeout` s, wait 1 s between tries. @retrying.retry(stop_max_delay=timeout * 1000, wait_fixed=RETRY_DELAY, retry_on_exception=lambda e: isinstance(e, KeyError)) def find(): - return self.registry[pod_name] + return self.registry[kp_name] try: d = find() - pod = d['pod'] - vifs = { - ifname: base.VersionedObject.obj_from_primitive(vif_obj) for - ifname, vif_obj in d['vifs'].items() - } + kp = d['kp'] + vifs = d['vifs'] except KeyError: - LOG.error("Timed out waiting for requested pod to appear in " + LOG.error("Timed out waiting for requested KuryrPort to appear in " "registry") - raise exceptions.ResourceNotReady(pod_name) + raise exceptions.ResourceNotReady(kp_name) for ifname, vif in vifs.items(): is_default_gateway = (ifname == k_const.DEFAULT_IFNAME) @@ -178,12 +174,13 @@ def find(): # use the ifname supplied in the CNI ADD request ifname = params.CNI_IFNAME - fn(vif, self._get_inst(pod), ifname, params.CNI_NETNS, + fn(vif, self._get_inst(kp), ifname, params.CNI_NETNS, report_health=self.report_drivers_health, is_default_gateway=is_default_gateway, container_id=params.CNI_CONTAINERID) return vifs - def _get_inst(self, pod): - return obj_vif.instance_info.InstanceInfo( - uuid=pod['metadata']['uid'], name=pod['metadata']['name']) + def _get_inst(self, kp): + return (obj_vif.instance_info + .InstanceInfo(uuid=kp['spec']['podUid'], + name=kp['metadata']['name'])) diff --git a/kuryr_kubernetes/constants.py b/kuryr_kubernetes/constants.py index c311d903d..add9b9a35 100644 --- a/kuryr_kubernetes/constants.py +++ b/kuryr_kubernetes/constants.py @@ -13,14 +13,18 @@ # License for the specific language governing permissions and limitations # under the License. +KURYR_FQDN = 'kuryr.openstack.org' + K8S_API_BASE = '/api/v1' K8S_API_NAMESPACES = K8S_API_BASE + '/namespaces' -K8S_API_CRD = '/apis/openstack.org/v1' +K8S_API_CRD_VERSION = 'openstack.org/v1' +K8S_API_CRD = '/apis/' + K8S_API_CRD_VERSION K8S_API_CRD_NAMESPACES = K8S_API_CRD + '/namespaces' K8S_API_CRD_KURYRNETS = K8S_API_CRD + '/kuryrnets' K8S_API_CRD_KURYRNETWORKS = K8S_API_CRD + '/kuryrnetworks' K8S_API_CRD_KURYRNETPOLICIES = K8S_API_CRD + '/kuryrnetpolicies' K8S_API_CRD_KURYRLOADBALANCERS = K8S_API_CRD + '/kuryrloadbalancers' +K8S_API_CRD_KURYRPORTS = K8S_API_CRD + '/kuryrports' K8S_API_POLICIES = '/apis/networking.k8s.io/v1/networkpolicies' K8S_API_NPWG_CRD = '/apis/k8s.cni.cncf.io/v1' @@ -34,6 +38,7 @@ K8S_OBJ_KURYRNETWORK = 'KuryrNetwork' K8S_OBJ_KURYRNETPOLICY = 'KuryrNetPolicy' K8S_OBJ_KURYRLOADBALANCER = 'KuryrLoadBalancer' +K8S_OBJ_KURYRPORT = 'KuryrPort' K8S_POD_STATUS_PENDING = 'Pending' K8S_POD_STATUS_SUCCEEDED = 'Succeeded' @@ -59,8 +64,12 @@ K8S_ANNOTATION_CURRENT_DRIVER = 'current_driver' K8S_ANNOTATION_NEUTRON_PORT = 'neutron_id' +POD_FINALIZER = KURYR_FQDN + '/pod-finalizer' KURYRNETWORK_FINALIZER = 'kuryrnetwork.finalizers.kuryr.openstack.org' +KURYRPORT_FINALIZER = KURYR_FQDN + '/kuryrport-finalizer' +KURYRPORT_LABEL = KURYR_FQDN + '/nodeName' + K8S_OS_VIF_NOOP_PLUGIN = "noop" CNI_EXCEPTION_CODE = 100 diff --git a/kuryr_kubernetes/controller/drivers/neutron_vif.py b/kuryr_kubernetes/controller/drivers/neutron_vif.py index 2ffd352f5..ca94d57da 100644 --- a/kuryr_kubernetes/controller/drivers/neutron_vif.py +++ b/kuryr_kubernetes/controller/drivers/neutron_vif.py @@ -95,10 +95,10 @@ def activate_vif(self, vif): def update_vif_sgs(self, pod, security_groups): os_net = clients.get_network_client() - pod_state = utils.get_pod_state(pod) - if pod_state: + vifs = utils.get_vifs(pod) + if vifs: # NOTE(ltomasbo): It just updates the default_vif security group - port_id = pod_state.vifs[constants.DEFAULT_IFNAME].id + port_id = vifs[constants.DEFAULT_IFNAME].id os_net.update_port(port_id, security_groups=list(security_groups)) def _get_port_request(self, pod, project_id, subnets, security_groups, diff --git a/kuryr_kubernetes/controller/drivers/utils.py b/kuryr_kubernetes/controller/drivers/utils.py index abeaf8ebe..e7f30d570 100644 --- a/kuryr_kubernetes/controller/drivers/utils.py +++ b/kuryr_kubernetes/controller/drivers/utils.py @@ -17,6 +17,7 @@ import netaddr from openstack import exceptions as os_exc +from os_vif import objects from oslo_config import cfg from oslo_log import log from oslo_serialization import jsonutils @@ -24,7 +25,6 @@ from kuryr_kubernetes import clients from kuryr_kubernetes import constants from kuryr_kubernetes import exceptions as k_exc -from kuryr_kubernetes import utils OPERATORS_WITH_VALUES = [constants.K8S_OPERATOR_IN, @@ -59,15 +59,23 @@ def get_host_id(pod): return pod['spec']['nodeName'] -def get_pod_state(pod): +def get_kuryrport(pod): + k8s = clients.get_kubernetes_client() try: - annotations = pod['metadata']['annotations'] - state_annotation = annotations[constants.K8S_ANNOTATION_VIF] - except KeyError: + return k8s.get(f'{constants.K8S_API_CRD_NAMESPACES}/' + f'{pod["metadata"]["namespace"]}/kuryrports/' + f'{pod["metadata"]["name"]}') + except k_exc.K8sResourceNotFound: return None - state_annotation = jsonutils.loads(state_annotation) - state = utils.extract_pod_annotation(state_annotation) - return state + + +def get_vifs(pod): + kp = get_kuryrport(pod) + try: + return {k: objects.base.VersionedObject.obj_from_primitive(v['vif']) + for k, v in kp['spec']['vifs'].items()} + except (KeyError, AttributeError, TypeError): + return {} def is_host_network(pod): @@ -274,19 +282,17 @@ def create_security_group_rule_body( def get_pod_ip(pod): try: - pod_metadata = pod['metadata']['annotations'] - vif = pod_metadata[constants.K8S_ANNOTATION_VIF] - except KeyError: + kp = get_kuryrport(pod) + vif = [x['vif'] for x in kp['spec']['vifs'].values() + if x['default']][0] + except (KeyError, TypeError, IndexError): return None - vif = jsonutils.loads(vif) - vif = vif['versioned_object.data']['default_vif'] - network = (vif['versioned_object.data']['network'] - ['versioned_object.data']) - first_subnet = (network['subnets']['versioned_object.data'] - ['objects'][0]['versioned_object.data']) - first_subnet_ip = (first_subnet['ips']['versioned_object.data'] - ['objects'][0]['versioned_object.data']['address']) - return first_subnet_ip + return (vif['versioned_object.data']['network'] + ['versioned_object.data']['subnets'] + ['versioned_object.data']['objects'][0] + ['versioned_object.data']['ips'] + ['versioned_object.data']['objects'][0] + ['versioned_object.data']['address']) def get_annotations(resource, annotation): diff --git a/kuryr_kubernetes/controller/drivers/vif_pool.py b/kuryr_kubernetes/controller/drivers/vif_pool.py index 809002a97..8367e1891 100644 --- a/kuryr_kubernetes/controller/drivers/vif_pool.py +++ b/kuryr_kubernetes/controller/drivers/vif_pool.py @@ -27,7 +27,6 @@ from oslo_config import cfg as oslo_cfg from oslo_log import log as logging from oslo_log import versionutils -from oslo_serialization import jsonutils from kuryr_kubernetes import clients from kuryr_kubernetes import config @@ -280,16 +279,9 @@ def _get_in_use_ports(self): in_use_ports = [] running_pods = kubernetes.get(constants.K8S_API_BASE + '/pods') for pod in running_pods['items']: - try: - annotations = jsonutils.loads(pod['metadata']['annotations'][ - constants.K8S_ANNOTATION_VIF]) - pod_state = utils.extract_pod_annotation(annotations) - except KeyError: - LOG.debug("Skipping pod without kuryr VIF annotation: %s", - pod) - else: - for vif in pod_state.vifs.values(): - in_use_ports.append(vif.id) + vifs = c_utils.get_vifs(pod) + for data in vifs.values(): + in_use_ports.append(data.id) return in_use_ports def list_pools(self): diff --git a/kuryr_kubernetes/controller/handlers/kuryrport.py b/kuryr_kubernetes/controller/handlers/kuryrport.py new file mode 100644 index 000000000..8c8db36d4 --- /dev/null +++ b/kuryr_kubernetes/controller/handlers/kuryrport.py @@ -0,0 +1,267 @@ +# Copyright 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from openstack import exceptions as os_exc +from os_vif import objects +from oslo_config import cfg as oslo_cfg +from oslo_log import log as logging + +from kuryr_kubernetes import clients +from kuryr_kubernetes import constants +from kuryr_kubernetes.controller.drivers import base as drivers +from kuryr_kubernetes.controller.drivers import utils as driver_utils +from kuryr_kubernetes import exceptions as k_exc +from kuryr_kubernetes.handlers import k8s_base + +LOG = logging.getLogger(__name__) +KURYRPORT_URI = constants.K8S_API_CRD_NAMESPACES + '/{ns}/kuryrports/{crd}' + + +class KuryrPortHandler(k8s_base.ResourceEventHandler): + """Controller side of KuryrPort process for Kubernetes pods. + + `KuryrPortHandler` runs on the Kuryr-Kubernetes controller and is + responsible for creating/removing the OpenStack resources associated to + the newly created pods, namely ports and update the KuryrPort CRD data. + """ + OBJECT_KIND = constants.K8S_OBJ_KURYRPORT + OBJECT_WATCH_PATH = constants.K8S_API_CRD_KURYRPORTS + + def __init__(self): + super(KuryrPortHandler, self).__init__() + self._drv_project = drivers.PodProjectDriver.get_instance() + self._drv_subnets = drivers.PodSubnetsDriver.get_instance() + self._drv_sg = drivers.PodSecurityGroupsDriver.get_instance() + # REVISIT(ltomasbo): The VIF Handler should not be aware of the pool + # directly. Due to the lack of a mechanism to load and set the + # VIFHandler driver, for now it is aware of the pool driver, but this + # will be reverted as soon as a mechanism is in place. + self._drv_vif_pool = drivers.VIFPoolDriver.get_instance( + specific_driver='multi_pool') + self._drv_vif_pool.set_vif_driver() + self._drv_multi_vif = drivers.MultiVIFDriver.get_enabled_drivers() + if self._is_network_policy_enabled(): + self._drv_lbaas = drivers.LBaaSDriver.get_instance() + self._drv_svc_sg = (drivers.ServiceSecurityGroupsDriver + .get_instance()) + self.k8s = clients.get_kubernetes_client() + + def on_present(self, kuryrport_crd): + if not kuryrport_crd['spec']['vifs']: + # Get vifs + if not self.get_vifs(kuryrport_crd): + # Ignore this event, according to one of the cases logged in + # get_vifs method. + return + + vifs = {ifname: {'default': data['default'], + 'vif': objects.base.VersionedObject + .obj_from_primitive(data['vif'])} + for ifname, data in kuryrport_crd['spec']['vifs'].items()} + + if all([v['vif'].active for v in vifs.values()]): + return + + changed = False + + try: + for ifname, data in vifs.items(): + if (data['vif'].plugin == constants.KURYR_VIF_TYPE_SRIOV and + oslo_cfg.CONF.sriov.enable_node_annotations): + pod_node = kuryrport_crd['spec']['podNodeName'] + # TODO(gryf): This probably will need adoption, so it will + # add information to CRD instead of the pod. + driver_utils.update_port_pci_info(pod_node, data['vif']) + if not data['vif'].active: + try: + self._drv_vif_pool.activate_vif(data['vif']) + changed = True + except os_exc.ResourceNotFound: + LOG.debug("Port not found, possibly already deleted. " + "No need to activate it") + finally: + if changed: + try: + name = kuryrport_crd['metadata']['name'] + namespace = kuryrport_crd['metadata']['namespace'] + pod = self.k8s.get(f"{constants.K8S_API_NAMESPACES}" + f"/{namespace}/pods/{name}") + except k_exc.K8sResourceNotFound as ex: + LOG.exception("Failed to get pod: %s", ex) + raise + + project_id = self._drv_project.get_project(pod) + + try: + self._update_kuryrport_crd(kuryrport_crd, vifs) + except k_exc.K8sResourceNotFound as ex: + LOG.exception("Failed to update KuryrPort CRD: %s", ex) + security_groups = self._drv_sg.get_security_groups( + pod, project_id) + for ifname, data in vifs.items(): + self._drv_vif_pool.release_vif(pod, data['vif'], + project_id, + security_groups) + except k_exc.K8sClientException: + raise k_exc.ResourceNotReady(pod['metadata']['name']) + + if self._is_network_policy_enabled(): + crd_pod_selectors = self._drv_sg.create_sg_rules(pod) + if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: + services = driver_utils.get_services() + self._update_services(services, crd_pod_selectors, + project_id) + + def on_finalize(self, kuryrport_crd): + name = kuryrport_crd['metadata']['name'] + namespace = kuryrport_crd['metadata']['namespace'] + try: + pod = self.k8s.get(f"{constants.K8S_API_NAMESPACES}" + f"/{namespace}/pods/{name}") + except k_exc.K8sResourceNotFound as ex: + LOG.exception("Failed to get pod: %s", ex) + # TODO(gryf): Free resources + self.k8s.remove_finalizer(kuryrport_crd, constants.POD_FINALIZER) + raise + + if (driver_utils.is_host_network(pod) or + not pod['spec'].get('nodeName')): + return + + project_id = self._drv_project.get_project(pod) + try: + crd_pod_selectors = self._drv_sg.delete_sg_rules(pod) + except k_exc.ResourceNotReady: + # NOTE(ltomasbo): If the pod is being deleted before + # kuryr-controller annotated any information about the port + # associated, there is no need for deleting sg rules associated to + # it. So this exception could be safetly ignored for the current + # sg drivers. Only the NP driver associates rules to the pods ips, + # and that waits for annotations to start. + # + # NOTE(gryf): perhaps we don't need to handle this case, since + # during CRD creation all the things, including security groups + # rules would be created too. + LOG.debug("Skipping SG rules deletion associated to the pod %s", + pod) + crd_pod_selectors = [] + try: + security_groups = self._drv_sg.get_security_groups(pod, project_id) + except k_exc.ResourceNotReady: + # NOTE(ltomasbo): If the namespace object gets deleted first the + # namespace security group driver will raise a ResourceNotReady + # exception as it cannot access anymore the kuryrnetwork CRD + # annotated on the namespace object. In such case we set security + # groups to empty list so that if pools are enabled they will be + # properly released. + security_groups = [] + + for data in kuryrport_crd['spec']['vifs'].values(): + vif = objects.base.VersionedObject.obj_from_primitive(data['vif']) + self._drv_vif_pool.release_vif(pod, vif, project_id, + security_groups) + if (self._is_network_policy_enabled() and crd_pod_selectors and + oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): + services = driver_utils.get_services() + self._update_services(services, crd_pod_selectors, project_id) + + # Remove finalizer out of pod. + self.k8s.remove_finalizer(pod, constants.POD_FINALIZER) + + # Finally, remove finalizer from KuryrPort CRD + self.k8s.remove_finalizer(kuryrport_crd, constants.KURYRPORT_FINALIZER) + + def get_vifs(self, kuryrport_crd): + try: + pod = self.k8s.get(f"{constants.K8S_API_NAMESPACES}" + f"/{kuryrport_crd['metadata']['namespace']}" + f"/pods" + f"/{kuryrport_crd['metadata']['name']}") + except k_exc.K8sResourceNotFound as ex: + LOG.exception("Failed to get pod: %s", ex) + # TODO(gryf): Release resources + self.k8s.remove_finalizer(kuryrport_crd, + constants.KURYRPORT_FINALIZER) + raise + + project_id = self._drv_project.get_project(pod) + security_groups = self._drv_sg.get_security_groups(pod, project_id) + try: + subnets = self._drv_subnets.get_subnets(pod, project_id) + except (os_exc.ResourceNotFound, k_exc.K8sResourceNotFound): + LOG.warning("Subnet does not exists. If namespace driver is " + "used, probably the namespace for the pod is " + "already deleted. So this pod does not need to " + "get a port as it will be deleted too. If the " + "default subnet driver is used, then you must " + "select an existing subnet to be used by Kuryr.") + return False + + # Request the default interface of pod + main_vif = self._drv_vif_pool.request_vif( + pod, project_id, subnets, security_groups) + + if not main_vif: + pod_name = pod['metadata']['name'] + LOG.warning("Ignoring event due to pod %s not being " + "scheduled yet.", pod_name) + return False + + vifs = {constants.DEFAULT_IFNAME: {'default': True, 'vif': main_vif}} + + # Request the additional interfaces from multiple drivers + index = 0 + for driver in self._drv_multi_vif: + additional_vifs = driver.request_additional_vifs(pod, project_id, + security_groups) + for index, vif in enumerate(additional_vifs, start=index+1): + ifname = (oslo_cfg.CONF.kubernetes.additional_ifname_prefix + + str(index)) + vifs[ifname] = {'default': False, 'vif': vif} + + try: + self._update_kuryrport_crd(kuryrport_crd, vifs) + except k_exc.K8sClientException as ex: + LOG.exception("Kubernetes Client Exception creating " + "KuryrPort CRD: %s", ex) + for ifname, data in vifs.items(): + self._drv_vif_pool.release_vif(pod, data['vif'], + project_id, + security_groups) + return True + + def _update_kuryrport_crd(self, kuryrport_crd, vifs): + LOG.debug('Updatting CRD %s', kuryrport_crd["metadata"]["name"]) + spec = {} + for ifname, data in vifs.items(): + data['vif'].obj_reset_changes(recursive=True) + spec[ifname] = {'default': data['default'], + 'vif': data['vif'].obj_to_primitive()} + + self.k8s.patch_crd('spec', kuryrport_crd['metadata']['selfLink'], + {'vifs': spec}) + + def _is_network_policy_enabled(self): + enabled_handlers = oslo_cfg.CONF.kubernetes.enabled_handlers + svc_sg_driver = oslo_cfg.CONF.kubernetes.service_security_groups_driver + return ('policy' in enabled_handlers and svc_sg_driver == 'policy') + + def _update_services(self, services, crd_pod_selectors, project_id): + for service in services.get('items'): + if not driver_utils.service_matches_affected_pods( + service, crd_pod_selectors): + continue + sgs = self._drv_svc_sg.get_security_groups(service, + project_id) + self._drv_lbaas.update_lbaas_sg(service, sgs) diff --git a/kuryr_kubernetes/controller/handlers/pod_label.py b/kuryr_kubernetes/controller/handlers/pod_label.py index 6c3ff9860..56fde1574 100644 --- a/kuryr_kubernetes/controller/handlers/pod_label.py +++ b/kuryr_kubernetes/controller/handlers/pod_label.py @@ -47,11 +47,17 @@ def __init__(self): self._drv_lbaas = drivers.LBaaSDriver.get_instance() def on_present(self, pod): - if driver_utils.is_host_network(pod) or not self._has_pod_state(pod): + if driver_utils.is_host_network(pod) or not self._has_vifs(pod): # NOTE(ltomasbo): The event will be retried once the vif handler # annotates the pod with the pod state. return + if (constants.K8S_ANNOTATION_VIF in + pod['metadata'].get('annotations', {})): + # NOTE(dulek): This might happen on upgrade, we need to wait for + # annotation to be moved to KuryrPort CRD. + return + current_pod_labels = pod['metadata'].get('labels') previous_pod_labels = self._get_pod_labels(pod) LOG.debug("Got previous pod labels from annotation: %r", @@ -97,11 +103,11 @@ def _set_pod_labels(self, pod, labels): {constants.K8S_ANNOTATION_LABEL: annotation}, resource_version=pod['metadata']['resourceVersion']) - def _has_pod_state(self, pod): + def _has_vifs(self, pod): try: - pod_state = pod['metadata']['annotations'][ - constants.K8S_ANNOTATION_VIF] - LOG.debug("Pod state is: %s", pod_state) + kp = driver_utils.get_vifs(pod) + vifs = kp['spec']['vifs'] + LOG.debug("Pod have associated KuryrPort with vifs: %s", vifs) except KeyError: return False return True diff --git a/kuryr_kubernetes/controller/handlers/vif.py b/kuryr_kubernetes/controller/handlers/vif.py index 10f96b7fe..f6592be98 100644 --- a/kuryr_kubernetes/controller/handlers/vif.py +++ b/kuryr_kubernetes/controller/handlers/vif.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -from openstack import exceptions as os_exc +from os_vif import objects from oslo_config import cfg as oslo_cfg from oslo_log import log as logging from oslo_serialization import jsonutils @@ -24,10 +24,10 @@ from kuryr_kubernetes.controller.drivers import utils as driver_utils from kuryr_kubernetes import exceptions as k_exc from kuryr_kubernetes.handlers import k8s_base -from kuryr_kubernetes import objects from kuryr_kubernetes import utils LOG = logging.getLogger(__name__) +KURYRPORT_URI = constants.K8S_API_CRD_NAMESPACES + '/{ns}/kuryrports/{crd}' class VIFHandler(k8s_base.ResourceEventHandler): @@ -63,20 +63,16 @@ def __init__(self): drivers.ServiceSecurityGroupsDriver.get_instance()) def on_present(self, pod): - state = driver_utils.get_pod_state(pod) - if (self._is_pod_completed(pod)): - if state: + if self._move_annotations_to_crd(pod): + return + + kp = driver_utils.get_kuryrport(pod) + if self._is_pod_completed(pod): + if kp: LOG.debug("Pod has completed execution, removing the vifs") - self.on_deleted(pod) - try: - self._set_pod_state(pod, None) - except k_exc.K8sClientException: - LOG.exception("Could not clear pod annotation") - raise k_exc.ResourceNotReady(pod['metadata']['name']) - except k_exc.K8sResourceNotFound: - pass + self.on_finalize(pod) else: - LOG.debug("Pod has completed execution, no annotation found." + LOG.debug("Pod has completed execution, no KuryrPort found." " Skipping") return @@ -87,129 +83,31 @@ def on_present(self, pod): # where certain pods/namespaces/nodes can be managed by other # networking solutions/CNI drivers. return - LOG.debug("Got VIFs from annotation: %r", state) - project_id = self._drv_project.get_project(pod) - security_groups = self._drv_sg.get_security_groups(pod, project_id) - if not state: - try: - subnets = self._drv_subnets.get_subnets(pod, project_id) - except (os_exc.ResourceNotFound, k_exc.K8sResourceNotFound): - LOG.warning("Subnet does not exists. If namespace driver is " - "used, probably the namespace for the pod is " - "already deleted. So this pod does not need to " - "get a port as it will be deleted too. If the " - "default subnet driver is used, then you must " - "select an existing subnet to be used by Kuryr.") - return - # Request the default interface of pod - main_vif = self._drv_vif_pool.request_vif( - pod, project_id, subnets, security_groups) - - if not main_vif: - pod_name = pod['metadata']['name'] - LOG.warning("Ignoring event due to pod %s not being " - "scheduled yet.", pod_name) - return - - state = objects.vif.PodState(default_vif=main_vif) - - # Request the additional interfaces from multiple dirvers - additional_vifs = [] - for driver in self._drv_multi_vif: - additional_vifs.extend( - driver.request_additional_vifs( - pod, project_id, security_groups)) - if additional_vifs: - state.additional_vifs = {} - for i, vif in enumerate(additional_vifs, start=1): - k = (oslo_cfg.CONF.kubernetes.additional_ifname_prefix - + str(i)) - state.additional_vifs[k] = vif + LOG.debug("Got KuryrPort: %r", kp) + if not kp: try: - self._set_pod_state(pod, state) + self._add_kuryrport_crd(pod) except k_exc.K8sClientException as ex: - LOG.debug("Failed to set annotation: %s", ex) - # FIXME(ivc): improve granularity of K8sClient exceptions: - # only resourceVersion conflict should be ignored - for ifname, vif in state.vifs.items(): - self._drv_vif_pool.release_vif(pod, vif, - project_id, - security_groups) - else: - changed = False - try: - for ifname, vif in state.vifs.items(): - if (vif.plugin == constants.KURYR_VIF_TYPE_SRIOV and - oslo_cfg.CONF.sriov.enable_node_annotations): - driver_utils.update_port_pci_info(pod, vif) - if not vif.active: - try: - self._drv_vif_pool.activate_vif(vif) - changed = True - except os_exc.ResourceNotFound: - LOG.debug("Port not found, possibly already " - "deleted. No need to activate it") - finally: - if changed: - try: - self._set_pod_state(pod, state) - except k_exc.K8sResourceNotFound as ex: - LOG.exception("Failed to set annotation: %s", ex) - for ifname, vif in state.vifs.items(): - self._drv_vif_pool.release_vif( - pod, vif, project_id, - security_groups) - except k_exc.K8sClientException: - pod_name = pod['metadata']['name'] - raise k_exc.ResourceNotReady(pod_name) - if self._is_network_policy_enabled(): - crd_pod_selectors = self._drv_sg.create_sg_rules(pod) - if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules: - services = driver_utils.get_services() - self._update_services( - services, crd_pod_selectors, project_id) + LOG.exception("Kubernetes Client Exception creating " + "KuryrPort CRD: %s", ex) + raise k_exc.ResourceNotReady(pod) - def on_deleted(self, pod): - if (driver_utils.is_host_network(pod) or - not pod['spec'].get('nodeName')): - return + k8s = clients.get_kubernetes_client() + k8s.add_finalizer(pod, constants.POD_FINALIZER) - project_id = self._drv_project.get_project(pod) - try: - crd_pod_selectors = self._drv_sg.delete_sg_rules(pod) - except k_exc.ResourceNotReady: - # NOTE(ltomasbo): If the pod is being deleted before - # kuryr-controller annotated any information about the port - # associated, there is no need for deleting sg rules associated to - # it. So this exception could be safetly ignored for the current - # sg drivers. Only the NP driver associates rules to the pods ips, - # and that waits for annotations to start. - LOG.debug("Pod was not yet annotated by Kuryr-controller. " - "Skipping SG rules deletion associated to the pod %s", - pod) - crd_pod_selectors = [] + def on_finalize(self, pod): + k8s = clients.get_kubernetes_client() try: - security_groups = self._drv_sg.get_security_groups(pod, project_id) - except k_exc.ResourceNotReady: - # NOTE(ltomasbo): If the namespace object gets deleted first the - # namespace security group driver will raise a ResourceNotReady - # exception as it cannot access anymore the kuryrnetwork CRD - # annotated on the namespace object. In such case we set security - # groups to empty list so that if pools are enabled they will be - # properly released. - security_groups = [] + k8s.delete(KURYRPORT_URI.format(ns=pod["metadata"]["namespace"], + crd=pod["metadata"]["name"])) + except k_exc.K8sResourceNotFound: + k8s.remove_finalizer(pod, constants.POD_FINALIZER) - state = driver_utils.get_pod_state(pod) - LOG.debug("Got VIFs from annotation: %r", state) - if state: - for ifname, vif in state.vifs.items(): - self._drv_vif_pool.release_vif(pod, vif, project_id, - security_groups) - if (self._is_network_policy_enabled() and crd_pod_selectors and - oslo_cfg.CONF.octavia_defaults.enforce_sg_rules): - services = driver_utils.get_services() - self._update_services(services, crd_pod_selectors, project_id) + except k_exc.K8sClientException: + LOG.exception("Could not remove KuryrPort CRD for pod %s.", + pod['metadata']['name']) + raise k_exc.ResourceNotReady(pod['metadata']['name']) def is_ready(self, quota): if (utils.has_limit(quota.ports) and @@ -236,42 +134,6 @@ def _is_pod_completed(pod): except KeyError: return False - def _set_pod_state(self, pod, state): - # TODO(ivc): extract annotation interactions - if not state: - old_annotation = pod['metadata'].get('annotations', {}) - LOG.debug("Removing VIFs annotation: %r for pod %s/%s (uid: %s)", - old_annotation.get(constants.K8S_ANNOTATION_VIF), - pod['metadata']['namespace'], pod['metadata']['name'], - pod['metadata']['uid']) - annotation = None - else: - state_dict = state.obj_to_primitive() - annotation = jsonutils.dumps(state_dict, sort_keys=True) - LOG.debug("Setting VIFs annotation: %r for pod %s/%s (uid: %s)", - annotation, pod['metadata']['namespace'], - pod['metadata']['name'], pod['metadata']['uid']) - - labels = pod['metadata'].get('labels') - if not labels: - LOG.debug("Removing Label annotation: %r", labels) - labels_annotation = None - else: - labels_annotation = jsonutils.dumps(labels, sort_keys=True) - LOG.debug("Setting Labels annotation: %r", labels_annotation) - - # NOTE(dulek): We don't care about compatibility with Queens format - # here, as eventually all Kuryr services will be upgraded - # and cluster will start working normally. Meanwhile - # we just ignore issue of old services being unable to - # read new annotations. - - k8s = clients.get_kubernetes_client() - k8s.annotate(pod['metadata']['selfLink'], - {constants.K8S_ANNOTATION_VIF: annotation, - constants.K8S_ANNOTATION_LABEL: labels_annotation}, - resource_version=pod['metadata']['resourceVersion']) - def _update_services(self, services, crd_pod_selectors, project_id): for service in services.get('items'): if not driver_utils.service_matches_affected_pods( @@ -285,3 +147,59 @@ def _is_network_policy_enabled(self): enabled_handlers = oslo_cfg.CONF.kubernetes.enabled_handlers svc_sg_driver = oslo_cfg.CONF.kubernetes.service_security_groups_driver return ('policy' in enabled_handlers and svc_sg_driver == 'policy') + + def _add_kuryrport_crd(self, pod, vifs=None): + LOG.debug('Adding CRD %s', pod["metadata"]["name"]) + + if not vifs: + vifs = {} + + kuryr_port = { + 'apiVersion': constants.K8S_API_CRD_VERSION, + 'kind': constants.K8S_OBJ_KURYRPORT, + 'metadata': { + 'name': pod['metadata']['name'], + 'finalizers': [constants.KURYRPORT_FINALIZER], + 'labels': { + constants.KURYRPORT_LABEL: pod['spec']['nodeName'] + } + }, + 'spec': { + 'podUid': pod['metadata']['uid'], + 'podNodeName': pod['spec']['nodeName'], + 'vifs': vifs + } + } + + k8s = clients.get_kubernetes_client() + k8s.post(KURYRPORT_URI.format(ns=pod["metadata"]["namespace"], + crd=''), kuryr_port) + + def _move_annotations_to_crd(self, pod): + """Support upgrade from annotations to KuryrPort CRD.""" + try: + state = (pod['metadata']['annotations'] + [constants.K8S_ANNOTATION_VIF]) + except KeyError: + return False + + _dict = jsonutils.loads(state) + state = objects.base.VersionedObject.obj_from_primitive(_dict) + + vifs = {ifname: {'default': state.default_vif == vif, + 'vif': objects.base.VersionedObject + .obj_to_primitive(vif)} + for ifname, vif in state.vifs.items()} + + try: + self._add_kuryrport_crd(pod, vifs) + except k_exc.K8sClientException as ex: + LOG.exception("Kubernetes Client Exception recreating " + "KuryrPort CRD from annotation: %s", ex) + raise k_exc.ResourceNotReady(pod) + + k8s = clients.get_kubernetes_client() + k8s.remove_annotations(pod['metadata']['selfLink'], + constants.K8S_ANNOTATION_VIF) + + return True diff --git a/kuryr_kubernetes/k8s_client.py b/kuryr_kubernetes/k8s_client.py index a228d8b07..9c016515b 100644 --- a/kuryr_kubernetes/k8s_client.py +++ b/kuryr_kubernetes/k8s_client.py @@ -316,7 +316,7 @@ def annotate(self, path, annotations, resource_version=None): headers=header, cert=self.cert, verify=self.verify_server) if response.ok: - return response.json()['metadata']['annotations'] + return response.json()['metadata'].get('annotations', {}) if response.status_code == requests.codes.conflict: resource = self.get(path) new_version = resource['metadata']['resourceVersion'] diff --git a/kuryr_kubernetes/tests/unit/cni/plugins/test_k8s_cni_registry.py b/kuryr_kubernetes/tests/unit/cni/plugins/test_k8s_cni_registry.py index a4e32d617..20faed59a 100644 --- a/kuryr_kubernetes/tests/unit/cni/plugins/test_k8s_cni_registry.py +++ b/kuryr_kubernetes/tests/unit/cni/plugins/test_k8s_cni_registry.py @@ -29,10 +29,11 @@ def setUp(self): self.k8s_mock = self.useFixture(kuryr_fixtures.MockK8sClient()).client self.default_iface = 'baz' self.additional_iface = 'eth1' - self.pod = {'metadata': {'name': 'foo', 'uid': 'bar', - 'namespace': 'default', 'selfLink': 'baz'}} - self.vifs = fake._fake_vifs_dict() - registry = {'default/foo': {'pod': self.pod, 'vifs': self.vifs, + self.kp = {'metadata': {'name': 'foo', 'uid': 'bar', + 'namespace': 'default', 'selfLink': 'baz'}, + 'spec': {'podUid': 'bar'}} + self.vifs = fake._fake_vifs() + registry = {'default/foo': {'kp': self.kp, 'vifs': self.vifs, 'containerid': None, 'vif_unplugged': False, 'del_received': False}} @@ -46,7 +47,7 @@ def setUp(self): @mock.patch('oslo_concurrency.lockutils.lock') @mock.patch('kuryr_kubernetes.cni.binding.base.connect') def test_add_present(self, m_connect, m_lock): - self.k8s_mock.get.return_value = self.pod + self.k8s_mock.get.return_value = self.kp self.plugin.add(self.params) @@ -99,7 +100,7 @@ def test_remove_pod_from_registry_after_del(self, m_disconnect, m_lock): @mock.patch('kuryr_kubernetes.cni.binding.base.disconnect') def test_del_wrong_container_id(self, m_disconnect): - registry = {'default/foo': {'pod': self.pod, 'vifs': self.vifs, + registry = {'default/foo': {'kp': self.kp, 'vifs': self.vifs, 'containerid': 'different'}} healthy = mock.Mock() self.plugin = k8s_cni_registry.K8sCNIRegistryPlugin(registry, healthy) @@ -112,11 +113,11 @@ def test_del_wrong_container_id(self, m_disconnect): @mock.patch('kuryr_kubernetes.cni.binding.base.connect') def test_add_present_on_5_try(self, m_connect, m_lock): se = [KeyError] * 5 - se.append({'pod': self.pod, 'vifs': self.vifs, 'containerid': None, + se.append({'kp': self.kp, 'vifs': self.vifs, 'containerid': None, 'vif_unplugged': False, 'del_received': False}) - se.append({'pod': self.pod, 'vifs': self.vifs, 'containerid': None, + se.append({'kp': self.kp, 'vifs': self.vifs, 'containerid': None, 'vif_unplugged': False, 'del_received': False}) - se.append({'pod': self.pod, 'vifs': self.vifs, 'containerid': None, + se.append({'kp': self.kp, 'vifs': self.vifs, 'containerid': None, 'vif_unplugged': False, 'del_received': False}) m_getitem = mock.Mock(side_effect=se) m_setitem = mock.Mock() @@ -127,7 +128,7 @@ def test_add_present_on_5_try(self, m_connect, m_lock): m_lock.assert_called_with('default/foo', external=True) m_setitem.assert_called_once_with('default/foo', - {'pod': self.pod, + {'kp': self.kp, 'vifs': self.vifs, 'containerid': 'cont_id', 'vif_unplugged': False, diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_vif_pool.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_vif_pool.py index 01cd5277e..f51f01558 100644 --- a/kuryr_kubernetes/tests/unit/controller/drivers/test_vif_pool.py +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_vif_pool.py @@ -20,7 +20,6 @@ import munch from openstack import exceptions as os_exc from oslo_config import cfg as oslo_cfg -from oslo_serialization import jsonutils from os_vif.objects import vif as osv_vif @@ -29,7 +28,6 @@ from kuryr_kubernetes.controller.drivers import neutron_vif from kuryr_kubernetes.controller.drivers import vif_pool from kuryr_kubernetes import exceptions -from kuryr_kubernetes.objects import vif from kuryr_kubernetes import os_vif_util as ovu from kuryr_kubernetes.tests import base as test_base from kuryr_kubernetes.tests import fake @@ -276,7 +274,8 @@ def test_release_vif(self): m_driver._return_ports_to_pool.assert_not_called() - def test__get_in_use_ports(self): + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_vifs') + def test__get_in_use_ports(self, get_vifs): cls = vif_pool.BaseVIFPool m_driver = mock.MagicMock(spec=cls) @@ -284,10 +283,7 @@ def test__get_in_use_ports(self): pod = get_pod_obj() port_id = str(uuid.uuid4()) pod_vif = osv_vif.VIFBase(id=port_id) - pod_state = vif.PodState(default_vif=pod_vif) - - pod['metadata']['annotations'][constants.K8S_ANNOTATION_VIF] = ( - jsonutils.dumps(pod_state.obj_to_primitive())) + get_vifs.return_value = {'eth0': pod_vif} items = [pod] kubernetes.get.return_value = {'items': items} @@ -295,20 +291,6 @@ def test__get_in_use_ports(self): self.assertEqual(resp, [port_id]) - def test__get_in_use_ports_exception(self): - cls = vif_pool.BaseVIFPool - m_driver = mock.MagicMock(spec=cls) - - kubernetes = self.useFixture(k_fix.MockK8sClient()).client - pod = get_pod_obj() - del pod['metadata']['annotations'][constants.K8S_ANNOTATION_VIF] - items = [pod] - kubernetes.get.return_value = {'items': items} - - resp = cls._get_in_use_ports(m_driver) - - self.assertEqual(resp, []) - def test__get_in_use_ports_empty(self): cls = vif_pool.BaseVIFPool m_driver = mock.MagicMock(spec=cls) diff --git a/kuryr_kubernetes/tests/unit/controller/handlers/test_kuryrport.py b/kuryr_kubernetes/tests/unit/controller/handlers/test_kuryrport.py new file mode 100644 index 000000000..ac5d56523 --- /dev/null +++ b/kuryr_kubernetes/tests/unit/controller/handlers/test_kuryrport.py @@ -0,0 +1,751 @@ +# Copyright (c) 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest import mock + +from openstack import exceptions as os_exc +from os_vif import objects as os_obj +from oslo_config import cfg + +from kuryr_kubernetes import constants +from kuryr_kubernetes.controller.drivers import multi_vif +from kuryr_kubernetes.controller.handlers import kuryrport +from kuryr_kubernetes import exceptions as k_exc +from kuryr_kubernetes.tests import base as test_base + + +CONF = cfg.CONF + + +class TestKuryrPortHandler(test_base.TestCase): + + def setUp(self): + super().setUp() + self._project_id = mock.sentinel.project_id + self._subnets = mock.sentinel.subnets + self._security_groups = mock.sentinel.security_groups + self._host = mock.sentinel.hostname + self._pod_version = mock.sentinel.pod_version + self._pod_link = mock.sentinel.pod_link + self._kp_version = mock.sentinel.kp_version + self._kp_link = mock.sentinel.kp_link + self._kp_namespace = mock.sentinel.namespace + self._kp_uid = mock.sentinel.kp_uid + self._kp_name = 'pod1' + + self._pod = {'metadata': {'resourceVersion': self._pod_version, + 'selfLink': self._pod_link, + 'name': self._kp_name, + 'namespace': self._kp_namespace}, + 'spec': {'nodeName': self._host}} + + self._kp = { + 'metadata': { + 'resourceVersion': self._kp_version, + 'selfLink': self._kp_link, + 'name': self._kp_name, + 'namespace': self._kp_namespace, + 'labels': { + constants.KURYRPORT_LABEL: self._host + } + }, + 'spec': { + 'podUid': 'deadbeef', + 'podNodeName': self._host, + 'vifs': {} + } + } + self._vif1 = os_obj.vif.VIFBase() + self._vif2 = os_obj.vif.VIFBase() + self._vif1.active = False + self._vif2.active = False + self._vif1.plugin = 'object' + self._vif2.plugin = 'object' + self._vif1_primitive = self._vif1.obj_to_primitive() + self._vif2_primitive = self._vif2.obj_to_primitive() + self._vifs_primitive = {'eth0': {'default': True, + 'vif': self._vif1_primitive}, + 'eth1': {'default': False, + 'vif': self._vif2_primitive}} + self._vifs = {'eth0': {'default': True, + 'vif': self._vif1}, + 'eth1': {'default': False, + 'vif': self._vif2}} + self._pod_uri = (f"{constants.K8S_API_NAMESPACES}" + f"/{self._kp['metadata']['namespace']}/pods/" + f"{self._kp['metadata']['name']}") + self._driver = multi_vif.NoopMultiVIFDriver() + + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler.get_vifs') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_present_no_vifs_create(self, ged, get_k8s_client, get_vifs): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + get_vifs.return_value = True + + kp.on_present(self._kp) + + get_vifs.assert_called_once_with(self._kp) + + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler.get_vifs') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_present_getting_vifs_failed(self, ged, get_k8s_client, + get_vifs): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + get_vifs.return_value = False + + self.assertFalse(kp.on_present(self._kp)) + + get_vifs.assert_called_once_with(self._kp) + + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._update_kuryrport_crd') + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'activate_vif') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_present(self, ged, get_k8s_client, activate_vif, + update_crd, get_project): + ged.return_value = [mock.MagicMock] + kp = kuryrport.KuryrPortHandler() + self._kp['spec']['vifs'] = self._vifs_primitive + get_project.return_value = self._project_id + + with mock.patch.object(kp, 'k8s') as k8s: + k8s.get.return_value = self._pod + + kp.on_present(self._kp) + + k8s.get.assert_called_once_with(self._pod_uri) + + activate_vif.assert_has_calls([mock.call(self._vif1), + mock.call(self._vif2)]) + update_crd.assert_called_once_with(self._kp, self._vifs) + + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_present_active(self, ged, get_k8s_client): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + self._vif1.active = True + self._vif2.active = True + self._kp['spec']['vifs'] = { + 'eth0': {'default': True, + 'vif': self._vif1.obj_to_primitive()}, + 'eth1': {'default': False, + 'vif': self._vif2.obj_to_primitive()}} + + kp.on_present(self._kp) + + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._update_kuryrport_crd') + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'activate_vif') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_present_port_not_found(self, ged, get_k8s_client, activate_vif, + update_crd): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + self._kp['spec']['vifs'] = self._vifs_primitive + activate_vif.side_effect = os_exc.ResourceNotFound() + + kp.on_present(self._kp) + + activate_vif.assert_has_calls([mock.call(self._vif1), + mock.call(self._vif2)]) + + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'activate_vif') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_present_pod_not_found(self, ged, get_k8s_client, activate_vif): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + self._kp['spec']['vifs'] = self._vifs_primitive + + with mock.patch.object(kp, 'k8s') as k8s: + k8s.get.side_effect = k_exc.K8sResourceNotFound(self._pod) + + self.assertRaises(k_exc.K8sResourceNotFound, kp.on_present, + self._kp) + + k8s.get.assert_called_once_with(self._pod_uri) + + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'release_vif') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.get_security_groups') + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._update_kuryrport_crd') + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'activate_vif') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_present_fail_update_crd(self, ged, get_k8s_client, + activate_vif, update_crd, get_project, + get_sg, release_vif): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + self._kp['spec']['vifs'] = self._vifs_primitive + update_crd.side_effect = k_exc.K8sResourceNotFound(self._kp) + get_project.return_value = self._project_id + get_sg.return_value = self._security_groups + + with mock.patch.object(kp, 'k8s') as k8s: + k8s.get.return_value = self._pod + + kp.on_present(self._kp) + + k8s.get.assert_called_once_with(self._pod_uri) + + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'release_vif') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.get_security_groups') + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._update_kuryrport_crd') + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'activate_vif') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_present_exception_during_update_crd(self, ged, get_k8s_client, + activate_vif, + update_crd, get_project, + get_sg, release_vif): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + self._kp['spec']['vifs'] = self._vifs_primitive + update_crd.side_effect = k_exc.K8sClientException() + get_project.return_value = self._project_id + get_sg.return_value = self._security_groups + + with mock.patch.object(kp, 'k8s') as k8s: + k8s.get.return_value = self._pod + + self.assertRaises(k_exc.ResourceNotReady, kp.on_present, self._kp) + + k8s.get.assert_called_once_with(self._pod_uri) + + update_crd.assert_called_once_with(self._kp, self._vifs) + + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'activate_vif') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.' + 'update_port_pci_info') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_present_sriov(self, ged, get_k8s_client, update_port_pci_info, + activate_vif): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + self._vif2.plugin = constants.KURYR_VIF_TYPE_SRIOV + self._vif2.active = True + self._kp['spec']['vifs'] = { + 'eth0': {'default': True, + 'vif': self._vif2.obj_to_primitive()}, + 'eth1': {'default': False, + 'vif': self._vif1.obj_to_primitive()}} + CONF.set_override('enable_node_annotations', True, group='sriov') + self.addCleanup(CONF.clear_override, 'enable_node_annotations', + group='sriov') + activate_vif.side_effect = os_exc.ResourceNotFound() + + kp.on_present(self._kp) + + update_port_pci_info.assert_called_once_with(self._host, self._vif2) + + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._update_services') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.create_sg_rules') + @mock.patch('kuryr_kubernetes.controller.drivers.base.' + 'ServiceSecurityGroupsDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base.LBaaSDriver.' + 'get_instance') + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._update_kuryrport_crd') + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'activate_vif') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._is_network_policy_enabled') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_present_np(self, ged, is_np_enabled, get_k8s_client, + activate_vif, update_crd, get_lb_instance, + get_sg_instance, create_sgr, update_services, + get_services, get_project): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + self._kp['spec']['vifs'] = self._vifs_primitive + + with mock.patch.object(kp, 'k8s') as k8s: + k8s.get.return_value = self._pod + + kp.on_present(self._kp) + + k8s.get.assert_called_once_with(self._pod_uri) + + activate_vif.assert_has_calls([mock.call(self._vif1), + mock.call(self._vif2)]) + update_crd.assert_called_once_with(self._kp, self._vifs) + create_sgr.assert_called_once_with(self._pod) + + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_finalize_exception_on_pod(self, ged, k8s): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + self._kp['spec']['vifs'] = self._vifs_primitive + + with mock.patch.object(kp, 'k8s') as k8s: + k8s.get.side_effect = k_exc.K8sResourceNotFound(self._pod) + + self.assertRaises(k_exc.K8sResourceNotFound, kp.on_finalize, + self._kp) + + k8s.get.assert_called_once_with(self._pod_uri) + k8s.remove_finalizer.assert_called_once_with( + self._kp, constants.POD_FINALIZER) + + @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_finalize_host_net_or_no_nodename(self, ged, k8s, + is_host_network): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + self._kp['spec']['vifs'] = self._vifs_primitive + is_host_network.return_value = False + _pod = dict(self._pod) + del _pod['spec']['nodeName'] + with mock.patch.object(kp, 'k8s') as k8s: + k8s.get.return_value = _pod + + kp.on_finalize(self._kp) + + k8s.get.assert_called_once_with(self._pod_uri) + + is_host_network.assert_called_once_with(self._pod) + is_host_network.reset_mock() + + is_host_network.return_value = False + with mock.patch.object(kp, 'k8s') as k8s: + k8s.get.return_value = self._pod + + kp.on_finalize(self._kp) + + k8s.get.assert_called_once_with(self._pod_uri) + + is_host_network.assert_called_once_with(self._pod) + + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'release_vif') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.get_security_groups') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.delete_sg_rules') + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_finalize_crd_sg_exceptions(self, ged, k8s, is_host_network, + get_project, delete_sg_rules, + get_sg, release_vif): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + self._kp['spec']['vifs'] = self._vifs_primitive + is_host_network.return_value = False + get_project.return_value = self._project_id + delete_sg_rules.side_effect = k_exc.ResourceNotReady(self._pod) + get_sg.side_effect = k_exc.ResourceNotReady(self._pod) + + with mock.patch.object(kp, 'k8s') as k8s: + k8s.get.return_value = self._pod + + kp.on_finalize(self._kp) + + k8s.get.assert_called_once_with(self._pod_uri) + k8s.remove_finalizer.assert_has_calls( + [mock.call(self._pod, constants.POD_FINALIZER), + mock.call(self._kp, constants.KURYRPORT_FINALIZER)]) + is_host_network.assert_called_once_with(self._pod) + delete_sg_rules.assert_called_once_with(self._pod) + get_sg.assert_called_once_with(self._pod, self._project_id) + release_vif.assert_has_calls([mock.call(self._pod, self._vif1, + self._project_id, []), + mock.call(self._pod, self._vif2, + self._project_id, [])]) + + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._update_services') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') + @mock.patch('kuryr_kubernetes.controller.drivers.base.' + 'ServiceSecurityGroupsDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base.LBaaSDriver.' + 'get_instance') + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._is_network_policy_enabled') + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'release_vif') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.get_security_groups') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.delete_sg_rules') + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_on_finalize_np(self, ged, k8s, is_host_network, get_project, + delete_sg_rules, get_sg, release_vif, + is_np_enabled, get_lb_instance, get_sg_instance, + get_services, update_services): + ged.return_value = [self._driver] + CONF.set_override('enforce_sg_rules', True, group='octavia_defaults') + self.addCleanup(CONF.clear_override, 'enforce_sg_rules', + group='octavia_defaults') + kp = kuryrport.KuryrPortHandler() + self._kp['spec']['vifs'] = self._vifs_primitive + is_host_network.return_value = False + get_project.return_value = self._project_id + selector = mock.sentinel.selector + delete_sg_rules.return_value = selector + get_sg.return_value = self._security_groups + get_services.return_value = mock.sentinel.services + + with mock.patch.object(kp, 'k8s') as k8s: + k8s.get.return_value = self._pod + + kp.on_finalize(self._kp) + + k8s.get.assert_called_once_with(self._pod_uri) + k8s.remove_finalizer.assert_has_calls( + [mock.call(self._pod, constants.POD_FINALIZER), + mock.call(self._kp, constants.KURYRPORT_FINALIZER)]) + + is_host_network.assert_called_once_with(self._pod) + delete_sg_rules.assert_called_once_with(self._pod) + get_sg.assert_called_once_with(self._pod, self._project_id) + release_vif.assert_has_calls([mock.call(self._pod, self._vif1, + self._project_id, + self._security_groups), + mock.call(self._pod, self._vif2, + self._project_id, + self._security_groups)]) + + get_services.assert_called_once() + update_services.assert_called_once_with(mock.sentinel.services, + selector, self._project_id) + + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._update_kuryrport_crd') + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'request_vif') + @mock.patch('kuryr_kubernetes.controller.drivers.default_subnet.' + 'DefaultPodSubnetDriver.get_subnets') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.get_security_groups') + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_get_vifs(self, ged, k8s, get_project, get_sg, get_subnets, + request_vif, update_crd): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + kp.k8s.get.return_value = self._pod + get_sg.return_value = self._security_groups + get_project.return_value = self._project_id + get_subnets.return_value = mock.sentinel.subnets + request_vif.return_value = self._vif1 + + self.assertTrue(kp.get_vifs(self._kp)) + + kp.k8s.get.assert_called_once_with(self._pod_uri) + get_project.assert_called_once_with(self._pod) + get_sg.assert_called_once_with(self._pod, self._project_id) + get_subnets.assert_called_once_with(self._pod, self._project_id) + request_vif.assert_called_once_with(self._pod, self._project_id, + mock.sentinel.subnets, + self._security_groups) + update_crd.assert_called_once_with(self._kp, + {constants.DEFAULT_IFNAME: + {'default': True, + 'vif': self._vif1}}) + + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_get_vifs_pod_not_found(self, ged, k8s): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + kp.k8s.get.side_effect = k_exc.K8sResourceNotFound(self._pod) + + self.assertRaises(k_exc.K8sResourceNotFound, kp.get_vifs, self._kp) + + kp.k8s.get.assert_called_once_with(self._pod_uri) + kp.k8s.remove_finalizer.assert_called_once_with( + self._kp, constants.KURYRPORT_FINALIZER) + + @mock.patch('kuryr_kubernetes.controller.drivers.default_subnet.' + 'DefaultPodSubnetDriver.get_subnets') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.get_security_groups') + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_get_vifs_subnet_error(self, ged, k8s, get_project, get_sg, + get_subnets): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + kp.k8s.get.return_value = self._pod + get_sg.return_value = self._security_groups + get_project.return_value = self._project_id + get_subnets.side_effect = os_exc.ResourceNotFound() + + self.assertFalse(kp.get_vifs(self._kp)) + + kp.k8s.get.assert_called_once_with(self._pod_uri) + get_project.assert_called_once_with(self._pod) + get_sg.assert_called_once_with(self._pod, self._project_id) + get_subnets.assert_called_once_with(self._pod, self._project_id) + + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'request_vif') + @mock.patch('kuryr_kubernetes.controller.drivers.default_subnet.' + 'DefaultPodSubnetDriver.get_subnets') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.get_security_groups') + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_get_vifs_no_vif(self, ged, k8s, get_project, get_sg, get_subnets, + request_vif): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + kp.k8s.get.return_value = self._pod + get_sg.return_value = self._security_groups + get_project.return_value = self._project_id + get_subnets.return_value = mock.sentinel.subnets + request_vif.return_value = None + + self.assertFalse(kp.get_vifs(self._kp)) + + kp.k8s.get.assert_called_once_with(self._pod_uri) + get_project.assert_called_once_with(self._pod) + get_sg.assert_called_once_with(self._pod, self._project_id) + get_subnets.assert_called_once_with(self._pod, self._project_id) + request_vif.assert_called_once_with(self._pod, self._project_id, + mock.sentinel.subnets, + self._security_groups) + + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._update_kuryrport_crd') + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'request_vif') + @mock.patch('kuryr_kubernetes.controller.drivers.default_subnet.' + 'DefaultPodSubnetDriver.get_subnets') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.get_security_groups') + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_get_vifs_with_additional_vif(self, ged, k8s, get_project, get_sg, + get_subnets, request_vif, + update_crd): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + kp.k8s.get.return_value = self._pod + fake_driver = mock.MagicMock() + fake_driver.request_additional_vifs.return_value = [self._vif2] + kp._drv_multi_vif.append(fake_driver) + get_sg.return_value = self._security_groups + get_project.return_value = self._project_id + get_subnets.return_value = mock.sentinel.subnets + request_vif.return_value = self._vif1 + + self.assertTrue(kp.get_vifs(self._kp)) + + kp.k8s.get.assert_called_once_with(self._pod_uri) + get_project.assert_called_once_with(self._pod) + get_sg.assert_called_once_with(self._pod, self._project_id) + get_subnets.assert_called_once_with(self._pod, self._project_id) + request_vif.assert_called_once_with(self._pod, self._project_id, + mock.sentinel.subnets, + self._security_groups) + update_crd.assert_called_once_with(self._kp, + {'eth0': {'default': True, + 'vif': self._vif1}, + 'eth1': {'default': False, + 'vif': self._vif2}}) + + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'release_vif') + @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.' + 'KuryrPortHandler._update_kuryrport_crd') + @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.' + 'request_vif') + @mock.patch('kuryr_kubernetes.controller.drivers.default_subnet.' + 'DefaultPodSubnetDriver.get_subnets') + @mock.patch('kuryr_kubernetes.controller.drivers.default_security_groups.' + 'DefaultPodSecurityGroupsDriver.get_security_groups') + @mock.patch('kuryr_kubernetes.controller.drivers.default_project.' + 'DefaultPodProjectDriver.get_project') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_get_exception_on_update_crd(self, ged, k8s, get_project, get_sg, + get_subnets, request_vif, update_crd, + release_vif): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + kp.k8s.get.return_value = self._pod + get_sg.return_value = self._security_groups + get_project.return_value = self._project_id + get_subnets.return_value = mock.sentinel.subnets + request_vif.return_value = self._vif1 + update_crd.side_effect = k_exc.K8sClientException() + + self.assertTrue(kp.get_vifs(self._kp)) + + kp.k8s.get.assert_called_once_with(self._pod_uri) + get_project.assert_called_once_with(self._pod) + get_sg.assert_called_once_with(self._pod, self._project_id) + get_subnets.assert_called_once_with(self._pod, self._project_id) + request_vif.assert_called_once_with(self._pod, self._project_id, + mock.sentinel.subnets, + self._security_groups) + update_crd.assert_called_once_with(self._kp, + {constants.DEFAULT_IFNAME: + {'default': True, + 'vif': self._vif1}}) + release_vif.assert_called_once_with(self._pod, self._vif1, + self._project_id, + self._security_groups) + + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_update_kuryrport_crd(self, ged, k8s): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + + kp._update_kuryrport_crd(self._kp, self._vifs) + self._vif1.obj_reset_changes() + self._vif2.obj_reset_changes() + vif1 = self._vif1.obj_to_primitive() + vif2 = self._vif2.obj_to_primitive() + + kp.k8s.patch_crd.assert_called_once_with( + 'spec', self._kp_link, {'vifs': {'eth0': {'default': True, + 'vif': vif1}, + 'eth1': {'default': False, + 'vif': vif2}}}) + + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_is_network_policy_enabled(self, ged, k8s): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + + CONF.set_override('enabled_handlers', ['fake_handler'], + group='kubernetes') + CONF.set_override('service_security_groups_driver', 'foo', + group='kubernetes') + + self.assertFalse(kp._is_network_policy_enabled()) + + CONF.set_override('enabled_handlers', ['policy'], + group='kubernetes') + CONF.set_override('service_security_groups_driver', 'foo', + group='kubernetes') + + self.assertFalse(kp._is_network_policy_enabled()) + + CONF.set_override('enabled_handlers', ['policy'], + group='kubernetes') + self.addCleanup(CONF.clear_override, 'enabled_handlers', + group='kubernetes') + CONF.set_override('service_security_groups_driver', 'policy', + group='kubernetes') + self.addCleanup(CONF.clear_override, 'service_security_groups_driver', + group='kubernetes') + + self.assertTrue(kp._is_network_policy_enabled()) + + @mock.patch('kuryr_kubernetes.controller.drivers.utils.' + 'service_matches_affected_pods') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.' + 'get_enabled_drivers') + def test_update_services(self, ged, k8s, smap): + ged.return_value = [self._driver] + kp = kuryrport.KuryrPortHandler() + kp._drv_lbaas = mock.MagicMock() + kp._drv_svc_sg = mock.MagicMock() + kp._drv_svc_sg.get_security_groups.return_value = self._security_groups + + smap.side_effect = [True, False] + services = {'items': ['service1', 'service2']} + + kp._update_services(services, mock.sentinel.crd_pod_selectors, + self._project_id) + + smap.assert_has_calls([mock.call('service1', + mock.sentinel.crd_pod_selectors), + mock.call('service2', + mock.sentinel.crd_pod_selectors)]) + kp._drv_svc_sg.get_security_groups.assert_called_once_with( + 'service1', self._project_id) + kp._drv_lbaas.update_lbaas_sg.assert_called_once_with( + 'service1', self._security_groups) diff --git a/kuryr_kubernetes/tests/unit/controller/handlers/test_pod_label.py b/kuryr_kubernetes/tests/unit/controller/handlers/test_pod_label.py index 1d1a4c353..9011a7a5e 100644 --- a/kuryr_kubernetes/tests/unit/controller/handlers/test_pod_label.py +++ b/kuryr_kubernetes/tests/unit/controller/handlers/test_pod_label.py @@ -49,7 +49,7 @@ def setUp(self): self._set_vif_driver = self._handler._drv_vif_pool.set_vif_driver self._get_pod_labels = self._handler._get_pod_labels self._set_pod_labels = self._handler._set_pod_labels - self._has_pod_state = self._handler._has_pod_state + self._has_vifs = self._handler._has_vifs self._update_vif_sgs = self._handler._drv_vif_pool.update_vif_sgs self._get_project.return_value = self._project_id @@ -80,12 +80,12 @@ def test_init(self, m_get_lbaas_driver, m_get_project_driver, @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') def test_on_present(self, m_get_services): m_get_services.return_value = {"items": []} - self._has_pod_state.return_value = True + self._has_vifs.return_value = True self._get_pod_labels.return_value = {'test1': 'test'} p_label.PodLabelHandler.on_present(self._handler, self._pod) - self._has_pod_state.assert_called_once_with(self._pod) + self._has_vifs.assert_called_once_with(self._pod) self._get_pod_labels.assert_called_once_with(self._pod) self._get_project.assert_called_once() self._get_security_groups.assert_called_once() @@ -93,33 +93,33 @@ def test_on_present(self, m_get_services): self._set_pod_labels.assert_called_once_with(self._pod, None) def test_on_present_no_state(self): - self._has_pod_state.return_value = False + self._has_vifs.return_value = False resp = p_label.PodLabelHandler.on_present(self._handler, self._pod) self.assertIsNone(resp) - self._has_pod_state.assert_called_once_with(self._pod) + self._has_vifs.assert_called_once_with(self._pod) self._get_pod_labels.assert_not_called() self._set_pod_labels.assert_not_called() def test_on_present_no_labels(self): - self._has_pod_state.return_value = True + self._has_vifs.return_value = True self._get_pod_labels.return_value = None p_label.PodLabelHandler.on_present(self._handler, self._pod) - self._has_pod_state.assert_called_once_with(self._pod) + self._has_vifs.assert_called_once_with(self._pod) self._get_pod_labels.assert_called_once_with(self._pod) self._set_pod_labels.assert_not_called() def test_on_present_no_changes(self): - self._has_pod_state.return_value = True + self._has_vifs.return_value = True pod_with_label = self._pod.copy() pod_with_label['metadata']['labels'] = {'test1': 'test'} self._get_pod_labels.return_value = {'test1': 'test'} p_label.PodLabelHandler.on_present(self._handler, pod_with_label) - self._has_pod_state.assert_called_once_with(pod_with_label) + self._has_vifs.assert_called_once_with(pod_with_label) self._get_pod_labels.assert_called_once_with(pod_with_label) self._set_pod_labels.assert_not_called() diff --git a/kuryr_kubernetes/tests/unit/controller/handlers/test_vif.py b/kuryr_kubernetes/tests/unit/controller/handlers/test_vif.py index 68e66163b..8ea59ed9b 100644 --- a/kuryr_kubernetes/tests/unit/controller/handlers/test_vif.py +++ b/kuryr_kubernetes/tests/unit/controller/handlers/test_vif.py @@ -16,6 +16,7 @@ from unittest import mock from os_vif import objects as os_obj +from oslo_serialization import jsonutils from kuryr_kubernetes import constants as k_const from kuryr_kubernetes.controller.drivers import base as drivers @@ -43,15 +44,30 @@ def setUp(self): self._pod_version = mock.sentinel.pod_version self._pod_link = mock.sentinel.pod_link self._pod_namespace = mock.sentinel.namespace + self._pod_uid = mock.sentinel.pod_uid + self._pod_name = 'pod1' self._pod = { 'metadata': {'resourceVersion': self._pod_version, 'selfLink': self._pod_link, + 'name': self._pod_name, 'namespace': self._pod_namespace}, 'status': {'phase': k_const.K8S_POD_STATUS_PENDING}, 'spec': {'hostNetwork': False, 'nodeName': 'hostname'} } + self._kp_version = mock.sentinel.kp_version + self._kp_link = mock.sentinel.kp_link + self._kp = {'apiVersion': 'openstack.org/v1', + 'kind': 'KuryrPort', + 'metadata': {'resourceVersion': self._kp_version, + 'selfLink': mock.sentinel.kp_link, + 'namespace': self._pod_namespace, + 'labels': mock.ANY}, + 'spec': {'podUid': self._pod_uid, + 'podNodeName': 'hostname', + 'vifs': {}}} + self._handler = mock.MagicMock(spec=h_vif.VIFHandler) self._handler._drv_project = mock.Mock(spec=drivers.PodProjectDriver) self._handler._drv_subnets = mock.Mock(spec=drivers.PodSubnetsDriver) @@ -68,7 +84,7 @@ def setUp(self): self._request_vif = self._handler._drv_vif_pool.request_vif self._release_vif = self._handler._drv_vif_pool.release_vif self._activate_vif = self._handler._drv_vif_pool.activate_vif - self._set_pod_state = self._handler._set_pod_state + self._matc = self._handler._move_annotations_to_crd self._is_pod_scheduled = self._handler._is_pod_scheduled self._is_pod_completed = self._handler._is_pod_completed self._request_additional_vifs = \ @@ -152,224 +168,183 @@ def test_is_pod_completed_failed(self): self.assertTrue(h_vif.VIFHandler._is_pod_completed({'status': {'phase': k_const.K8S_POD_STATUS_FAILED}})) - @mock.patch('oslo_config.cfg.CONF') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'update_port_pci_info') @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_present(self, m_get_pod_state, m_host_network, m_update_pci, - m_conf): - m_get_pod_state.return_value = self._state - m_host_network.return_value = False - self._vif.plugin = 'sriov' - m_conf.sriov.enable_node_annotations = True - h_vif.VIFHandler.on_present(self._handler, self._pod) - - m_get_pod_state.assert_called_once_with(self._pod) - m_update_pci.assert_called_once_with(self._pod, self._vif) - self._request_vif.assert_not_called() - self._request_additional_vifs.assert_not_called() - self._activate_vif.assert_not_called() - self._set_pod_state.assert_not_called() - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_present_host_network(self, m_get_pod_state, m_host_network): - m_get_pod_state.return_value = self._state + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_kuryrport') + def test_on_present_host_network(self, m_get_kuryrport, m_host_network): + m_get_kuryrport.return_value = self._kp m_host_network.return_value = True + self._matc.return_value = False h_vif.VIFHandler.on_present(self._handler, self._pod) - m_get_pod_state.assert_called_once() + self._matc.assert_called_once_with(self._pod) + m_get_kuryrport.assert_called_once() self._request_vif.assert_not_called() self._request_additional_vifs.assert_not_called() self._activate_vif.assert_not_called() - self._set_pod_state.assert_not_called() @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_present_not_pending(self, m_get_pod_state, m_host_network): - m_get_pod_state.return_value = self._state + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_kuryrport') + def test_on_present_not_pending(self, m_get_kuryrport, m_host_network): + m_get_kuryrport.return_value = self._kp m_host_network.return_value = False self._is_pod_scheduled.return_value = False + self._matc.return_value = False h_vif.VIFHandler.on_present(self._handler, self._pod) - m_get_pod_state.assert_called_once() + self._matc.assert_called_once_with(self._pod) + m_get_kuryrport.assert_called_once() self._request_vif.assert_not_called() self._request_additional_vifs.assert_not_called() self._activate_vif.assert_not_called() - self._set_pod_state.assert_not_called() - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_present_on_completed_with_annotation(self, m_get_pod_state): + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_kuryrport') + def test_on_present_on_completed_with_annotation(self, m_get_kuryrport): self._is_pod_completed.return_value = True - m_get_pod_state.return_value = self._state + m_get_kuryrport.return_value = self._kp + self._matc.return_value = False h_vif.VIFHandler.on_present(self._handler, self._pod) - self._handler.on_deleted.assert_called_once_with(self._pod) - self._set_pod_state.assert_called_once_with(self._pod, None) + self._matc.assert_called_once_with(self._pod) + self._handler.on_finalize.assert_called_once_with(self._pod) self._request_vif.assert_not_called() self._request_additional_vifs.assert_not_called() self._activate_vif.assert_not_called() - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_present_on_completed_without_annotation(self, m_get_pod_state): + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_kuryrport') + def test_on_present_on_completed_without_annotation(self, m_get_kuryrport): self._is_pod_completed.return_value = True - m_get_pod_state.return_value = None + m_get_kuryrport.return_value = None + self._matc.return_value = False h_vif.VIFHandler.on_present(self._handler, self._pod) - self._handler.on_deleted.assert_not_called() - - self._set_pod_state.assert_not_called() + self._matc.assert_called_once_with(self._pod) + self._handler.on_finalize.assert_not_called() self._request_vif.assert_not_called() self._request_additional_vifs.assert_not_called() self._activate_vif.assert_not_called() - @mock.patch('oslo_config.cfg.CONF') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'update_port_pci_info') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_present_activate(self, m_get_pod_state, m_host_network, - m_get_services, m_update_pci, m_conf): - m_get_pod_state.return_value = self._state + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_kuryrport') + def test_on_present_create(self, m_get_kuryrport, m_host_network, + m_get_k8s_client): + m_get_kuryrport.return_value = None m_host_network.return_value = False - m_get_services.return_value = {"items": []} - self._vif.active = False - self._vif.plugin = 'sriov' - m_conf.sriov.enable_node_annotations = True + self._matc.return_value = False + k8s = mock.MagicMock() + m_get_k8s_client.return_value = k8s h_vif.VIFHandler.on_present(self._handler, self._pod) - m_get_pod_state.assert_called_once_with(self._pod) - m_update_pci.assert_called_once_with(self._pod, self._vif) - self._activate_vif.assert_called_once_with(self._vif) - self._set_pod_state.assert_called_once_with(self._pod, self._state) - self._request_vif.assert_not_called() - self._request_additional_vifs.assert_not_called() - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_present_create(self, m_get_pod_state, m_host_network): - m_get_pod_state.return_value = None - m_host_network.return_value = False - - h_vif.VIFHandler.on_present(self._handler, self._pod) - - m_get_pod_state.assert_called_once_with(self._pod) - self._request_vif.assert_called_once_with( - self._pod, self._project_id, self._subnets, self._security_groups) - self._request_additional_vifs.assert_called_once_with( - self._pod, self._project_id, self._security_groups) - self._set_pod_state.assert_called_once_with(self._pod, self._state) - self._activate_vif.assert_not_called() + m_get_kuryrport.assert_called_once_with(self._pod) + self._matc.assert_called_once_with(self._pod) + self._handler._add_kuryrport_crd.assert_called_once_with(self._pod) + k8s.add_finalizer.assert_called_once_with(self._pod, + k_const.POD_FINALIZER) - @mock.patch('oslo_config.cfg.CONF') @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_present_create_with_additional_vifs(self, m_get_pod_state, - m_host_network, m_conf): - m_get_pod_state.return_value = None + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_kuryrport') + def test_on_present_update(self, m_get_kuryrport, m_host_network): + m_get_kuryrport.return_value = self._kp m_host_network.return_value = False - ifname_prefix = 'baz' - m_conf.kubernetes.additional_ifname_prefix = ifname_prefix - additional_vif = os_obj.vif.VIFBase() - self._state.additional_vifs = {ifname_prefix+'1': additional_vif} - self._request_additional_vifs.return_value = [additional_vif] + self._matc.return_value = False h_vif.VIFHandler.on_present(self._handler, self._pod) - m_get_pod_state.assert_called_once_with(self._pod) - self._request_vif.assert_called_once_with( - self._pod, self._project_id, self._subnets, self._security_groups) - self._request_additional_vifs.assert_called_once_with( - self._pod, self._project_id, self._security_groups) - self._set_pod_state.assert_called_once_with(self._pod, self._state) - self._activate_vif.assert_not_called() + self._matc.assert_called_once_with(self._pod) + m_get_kuryrport.assert_called_once_with(self._pod) + self._handler._add_kuryrport_crd.assert_not_called() @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_present_rollback(self, m_get_pod_state, m_host_network): - m_get_pod_state.return_value = None - m_host_network.return_value = False - self._set_pod_state.side_effect = k_exc.K8sClientException + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_kuryrport') + def test_on_present_upgrade(self, m_get_kuryrport, m_host_network): + m_get_kuryrport.return_value = self._kp + m_host_network.return_value = True + self._matc.return_value = True h_vif.VIFHandler.on_present(self._handler, self._pod) - m_get_pod_state.assert_called_once_with(self._pod) - self._request_vif.assert_called_once_with( - self._pod, self._project_id, self._subnets, self._security_groups) - self._request_additional_vifs.assert_called_once_with( - self._pod, self._project_id, self._security_groups) - self._set_pod_state.assert_called_once_with(self._pod, self._state) - self._release_vif.assert_called_once_with(self._pod, self._vif, - self._project_id, - self._security_groups) + self._matc.assert_called_once_with(self._pod) + m_get_kuryrport.assert_not_called() + self._request_vif.assert_not_called() + self._request_additional_vifs.assert_not_called() self._activate_vif.assert_not_called() - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_deleted(self, m_get_pod_state, m_host_network, m_get_services): - m_get_pod_state.return_value = self._state - m_host_network.return_value = False - m_get_services.return_value = {"items": []} - h_vif.VIFHandler.on_deleted(self._handler, self._pod) - - m_get_pod_state.assert_called_once_with(self._pod) - self._release_vif.assert_called_once_with(self._pod, self._vif, - self._project_id, - self._security_groups) - - @mock.patch('oslo_config.cfg.CONF') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_deleted_with_additional_vifs(self, m_get_pod_state, - m_host_network, m_get_services, - m_conf): - additional_vif = os_obj.vif.VIFBase() - ifname_prefix = 'bar' - m_conf.kubernetes.additional_ifname_prefix = ifname_prefix - self._state.additional_vifs = {ifname_prefix+'1': additional_vif} - m_get_pod_state.return_value = self._state - m_host_network.return_value = False - m_get_services.return_value = {"items": []} - - h_vif.VIFHandler.on_deleted(self._handler, self._pod) - - self._release_vif.assert_any_call(self._pod, self._vif, - self._project_id, - self._security_groups) - self._release_vif.assert_any_call(self._pod, additional_vif, - self._project_id, - self._security_groups) - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_deleted_host_network(self, m_get_pod_state, m_host_network): - m_get_pod_state.return_value = self._state - m_host_network.return_value = True - - h_vif.VIFHandler.on_deleted(self._handler, self._pod) - - m_get_pod_state.assert_not_called() - self._release_vif.assert_not_called() - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_state') - def test_on_deleted_no_annotation(self, m_get_pod_state, m_host_network, - m_get_services): - m_get_pod_state.return_value = None - m_host_network.return_value = False - m_get_services.return_value = {"items": []} - - h_vif.VIFHandler.on_deleted(self._handler, self._pod) - - m_get_pod_state.assert_called_once_with(self._pod) - self._release_vif.assert_not_called() + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_kuryrport') + def test_on_finalize_crd(self, m_get_kuryrport, m_get_k8s_client): + m_get_kuryrport.return_value = self._kp + k8s = mock.MagicMock() + m_get_k8s_client.return_value = k8s + + h_vif.VIFHandler.on_finalize(self._handler, self._pod) + + k8s.delete.assert_called_once_with( + h_vif.KURYRPORT_URI.format( + ns=self._pod["metadata"]["namespace"], + crd=self._pod["metadata"]["name"])) + + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_kuryrport') + def test_on_finalize_crd_exception(self, m_get_kuryrport, + m_get_k8s_client): + m_get_kuryrport.return_value = self._kp + k8s = mock.MagicMock() + m_get_k8s_client.return_value = k8s + k8s.delete.side_effect = k_exc.K8sClientException + + self.assertRaises(k_exc.ResourceNotReady, h_vif.VIFHandler + .on_finalize, self._handler, self._pod) + + k8s.delete.assert_called_once_with( + h_vif.KURYRPORT_URI.format( + ns=self._pod["metadata"]["namespace"], + crd=self._pod["metadata"]["name"])) + + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_kuryrport') + def test_on_finalize_crd_not_found(self, m_get_kuryrport, + m_get_k8s_client): + m_get_kuryrport.return_value = self._kp + k8s = mock.MagicMock() + m_get_k8s_client.return_value = k8s + k8s.delete.side_effect = k_exc.K8sResourceNotFound(self._pod) + + h_vif.VIFHandler.on_finalize(self._handler, self._pod) + + k8s.delete.assert_called_once_with( + h_vif.KURYRPORT_URI.format( + ns=self._pod["metadata"]["namespace"], + crd=self._pod["metadata"]["name"])) + (k8s.remove_finalizer + .assert_called_once_with(self._pod, k_const.POD_FINALIZER)) + + def test_move_annotations_to_crd_no_annotations(self): + res = h_vif.VIFHandler._move_annotations_to_crd(self._handler, + self._pod) + self.assertFalse(res) + + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + def test_move_annotations_to_crd_with_annotations(self, m_get_k8s_client): + vifobj = os_obj.vif.VIFOpenVSwitch() + state = vif.PodState(default_vif=vifobj) + annotation = jsonutils.dumps(state.obj_to_primitive()) + self._pod['metadata']['annotations'] = { + k_const.K8S_ANNOTATION_VIF: annotation} + vifs = {'eth0': {'default': True, 'vif': vifobj.obj_to_primitive()}} + k8s = mock.MagicMock() + m_get_k8s_client.return_value = k8s + + res = h_vif.VIFHandler._move_annotations_to_crd(self._handler, + self._pod) + self.assertTrue(res) + self._handler._add_kuryrport_crd.assert_called_once_with(self._pod, + vifs) + + m_get_k8s_client.assert_called_once() + k8s.remove_annotations.assert_called_once_with( + self._pod['metadata']['selfLink'], k_const.K8S_ANNOTATION_VIF) diff --git a/kuryr_kubernetes/utils.py b/kuryr_kubernetes/utils.py index afc975a22..651c9eb38 100644 --- a/kuryr_kubernetes/utils.py +++ b/kuryr_kubernetes/utils.py @@ -97,15 +97,15 @@ def convert_netns(netns): return netns -def get_pod_unique_name(pod): - """Returns a unique name for the pod. +def get_res_unique_name(resource): + """Returns a unique name for the resource like pod or CRD. - It returns a pod unique name for the pod composed of its name and the + It returns a unique name for the resource composed of its name and the namespace it is running on. - :returns: String with namespace/name of the pod + :returns: String with namespace/name of the resource """ - return "%(namespace)s/%(name)s" % pod['metadata'] + return "%(namespace)s/%(name)s" % resource['metadata'] def check_suitable_multi_pool_driver_opt(pool_driver, pod_driver): @@ -252,6 +252,15 @@ def extract_pod_annotation(annotation): return obj +def get_vifs_from_crd(crd): + result = {} + for ifname in crd['spec']['vifs']: + result[ifname] = (objects.base.VersionedObject + .obj_from_primitive(crd['spec']['vifs'] + [ifname]['vif'])) + return result + + def has_limit(quota): NO_LIMIT = -1 return quota['limit'] != NO_LIMIT diff --git a/setup.cfg b/setup.cfg index 7412404d8..a24c6447b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -107,6 +107,7 @@ kuryr_kubernetes.controller.handlers = kuryrnetwork = kuryr_kubernetes.controller.handlers.kuryrnetwork:KuryrNetworkHandler kuryrnetwork_population = kuryr_kubernetes.controller.handlers.kuryrnetwork_population:KuryrNetworkPopulationHandler test_handler = kuryr_kubernetes.tests.unit.controller.handlers.test_fake_handler:TestHandler + kuryrport = kuryr_kubernetes.controller.handlers.kuryrport:KuryrPortHandler kuryr_kubernetes.controller.drivers.multi_vif = noop = kuryr_kubernetes.controller.drivers.multi_vif:NoopMultiVIFDriver diff --git a/tools/gate/copy_k8s_logs.sh b/tools/gate/copy_k8s_logs.sh index 3f9ac9830..86de25973 100755 --- a/tools/gate/copy_k8s_logs.sh +++ b/tools/gate/copy_k8s_logs.sh @@ -37,6 +37,7 @@ sudo chown ${USER}:${USER} ${HOME}/.kube/config /usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get kuryrnetworks -o yaml --all-namespaces >> ${K8S_LOG_DIR}/kuryrnetworks_crds.txt /usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get endpoints -o yaml --all-namespaces >> ${K8S_LOG_DIR}/endpoints.txt /usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get kuryrnetpolicy -o yaml --all-namespaces >> ${K8S_LOG_DIR}/kuryrnetpolicy_crds.txt +/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get kuryrport -o yaml --all-namespaces >> ${K8S_LOG_DIR}/kuryrport_crds.txt # Kubernetes pods logs mkdir -p ${K8S_LOG_DIR}/pod_logs while read -r line From 6d459143f6b44b5ccb70c0b95687228e5a35a135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Dulko?= Date: Thu, 30 Jul 2020 11:19:33 +0200 Subject: [PATCH 4/6] Speed up container builds Turns out upgrading pip enables grpcio PyPi package to use wheels to install binaries, avioding need to compile it every time. This saves a ton of time when building containers. Change-Id: I6e4a5f9fddd24b8e88c62b444e8b305ade3f7f2a --- cni.Dockerfile | 3 ++- controller.Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cni.Dockerfile b/cni.Dockerfile index 803cd58b1..b4e547288 100644 --- a/cni.Dockerfile +++ b/cni.Dockerfile @@ -18,7 +18,8 @@ RUN yum upgrade -y \ COPY . /opt/kuryr-kubernetes -RUN pip3 install -c $UPPER_CONSTRAINTS_FILE /opt/kuryr-kubernetes \ +RUN pip3 install -U pip \ + && python3 -m pip install -c $UPPER_CONSTRAINTS_FILE /opt/kuryr-kubernetes \ && cp /opt/kuryr-kubernetes/cni_ds_init /usr/bin/cni_ds_init \ && mkdir -p /etc/kuryr-cni \ && cp /opt/kuryr-kubernetes/etc/cni/net.d/* /etc/kuryr-cni \ diff --git a/controller.Dockerfile b/controller.Dockerfile index a7093fbbc..662eac3c4 100644 --- a/controller.Dockerfile +++ b/controller.Dockerfile @@ -10,7 +10,8 @@ RUN yum upgrade -y \ COPY . /opt/kuryr-kubernetes -RUN pip3 install -c $UPPER_CONSTRAINTS_FILE --no-cache-dir /opt/kuryr-kubernetes \ +RUN pip3 install -U pip \ + && python3 -m pip install -c $UPPER_CONSTRAINTS_FILE --no-cache-dir /opt/kuryr-kubernetes \ && yum -y history undo last \ && yum clean all \ && rm -rf /opt/kuryr-kubernetes \ From f71ae55476b37fe5573fea54261bdfe91d0506cb Mon Sep 17 00:00:00 2001 From: scavnicka Date: Mon, 24 Feb 2020 20:46:54 +0000 Subject: [PATCH 5/6] Update loadbalancer CRD with service spec and rely on CRD This commit adds support for creation of loadbalancer, listeners, members, pools with using the CRD, it is also filling the status field in the CRD. Depends-On: https://review.opendev.org/#/c/743214/ Change-Id: I42f90c836397b0d71969642d6ba31bfb49786a43 --- .zuul.d/octavia.yaml | 6 +- .zuul.d/sdn.yaml | 4 +- devstack/plugin.sh | 1 + devstack/settings | 2 +- .../kuryr_crds/kuryrloadbalancer.yaml | 55 +- kuryr_kubernetes/constants.py | 2 + .../controller/drivers/lb_public_ip.py | 32 +- .../controller/drivers/lbaasv2.py | 228 ++-- kuryr_kubernetes/controller/handlers/lbaas.py | 788 ++++---------- .../controller/handlers/loadbalancer.py | 810 ++++++++++++++ kuryr_kubernetes/k8s_client.py | 15 + .../controller/drivers/test_lb_public_ip.py | 96 +- .../unit/controller/drivers/test_lbaasv2.py | 634 ++++++----- .../unit/controller/handlers/test_lbaas.py | 991 ++++-------------- .../controller/handlers/test_loadbalancer.py | 525 ++++++++++ kuryr_kubernetes/tests/unit/test_utils.py | 141 ++- kuryr_kubernetes/utils.py | 37 +- setup.cfg | 5 +- 18 files changed, 2539 insertions(+), 1833 deletions(-) create mode 100644 kuryr_kubernetes/controller/handlers/loadbalancer.py create mode 100644 kuryr_kubernetes/tests/unit/controller/handlers/test_loadbalancer.py diff --git a/.zuul.d/octavia.yaml b/.zuul.d/octavia.yaml index 04ffdc5a5..b4cd6f526 100644 --- a/.zuul.d/octavia.yaml +++ b/.zuul.d/octavia.yaml @@ -99,7 +99,7 @@ vars: devstack_localrc: DOCKER_CGROUP_DRIVER: "systemd" - KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport + KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer KURYR_SG_DRIVER: policy KURYR_SUBNET_DRIVER: namespace devstack_services: @@ -120,7 +120,7 @@ vars: devstack_localrc: KURYR_SUBNET_DRIVER: namespace - KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport + KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer KURYR_SG_DRIVER: policy KURYR_USE_PORT_POOLS: true KURYR_POD_VIF_DRIVER: neutron-vif @@ -134,7 +134,7 @@ parent: kuryr-kubernetes-tempest-containerized vars: devstack_localrc: - KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport + KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer KURYR_SG_DRIVER: policy KURYR_SUBNET_DRIVER: namespace diff --git a/.zuul.d/sdn.yaml b/.zuul.d/sdn.yaml index 0e2ae0af3..cfda1547d 100644 --- a/.zuul.d/sdn.yaml +++ b/.zuul.d/sdn.yaml @@ -98,7 +98,7 @@ KURYR_LB_ALGORITHM: SOURCE_IP_PORT KURYR_SUBNET_DRIVER: namespace KURYR_SG_DRIVER: policy - KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport + KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer voting: false - job: @@ -144,7 +144,7 @@ KURYR_ENFORCE_SG_RULES: false KURYR_LB_ALGORITHM: SOURCE_IP_PORT KURYR_HYPERKUBE_VERSION: v1.16.0 - KURYR_ENABLED_HANDLERS: vif,lb,lbaasspec,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport + KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer KURYR_SG_DRIVER: policy KURYR_SUBNET_DRIVER: namespace KURYR_K8S_CONTAINERIZED_DEPLOYMENT: true diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 1b2380220..3ca7665af 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -974,6 +974,7 @@ function update_tempest_conf_file { iniset $TEMPEST_CONFIG kuryr_kubernetes validate_crd True iniset $TEMPEST_CONFIG kuryr_kubernetes kuryrnetworks True iniset $TEMPEST_CONFIG kuryr_kubernetes kuryrports True + iniset $TEMPEST_CONFIG kuryr_kubernetes kuryrloadbalancers True } source $DEST/kuryr-kubernetes/devstack/lib/kuryr_kubernetes diff --git a/devstack/settings b/devstack/settings index 81bf8487e..175f1bd79 100644 --- a/devstack/settings +++ b/devstack/settings @@ -43,7 +43,7 @@ KURYR_K8S_API_LB_PORT=${KURYR_K8S_API_LB_PORT:-443} KURYR_PORT_DEBUG=${KURYR_PORT_DEBUG:-True} KURYR_SUBNET_DRIVER=${KURYR_SUBNET_DRIVER:-default} KURYR_SG_DRIVER=${KURYR_SG_DRIVER:-default} -KURYR_ENABLED_HANDLERS=${KURYR_ENABLED_HANDLERS:-vif,lb,lbaasspec,kuryrport} +KURYR_ENABLED_HANDLERS=${KURYR_ENABLED_HANDLERS:-vif,endpoints,service,kuryrloadbalancer,kuryrport} # OpenShift OPENSHIFT_BINARY_VERSION=${OPENSHIFT_BINARY_VERSION:-v3.11.0} diff --git a/kubernetes_crds/kuryr_crds/kuryrloadbalancer.yaml b/kubernetes_crds/kuryr_crds/kuryrloadbalancer.yaml index eb67591b0..0e54a4705 100644 --- a/kubernetes_crds/kuryr_crds/kuryrloadbalancer.yaml +++ b/kubernetes_crds/kuryr_crds/kuryrloadbalancer.yaml @@ -29,13 +29,6 @@ spec: properties: spec: type: object - required: - - ip - - ports - - project_id - - security_groups_ids - - subnet_id - - type properties: ip: type: string @@ -46,7 +39,6 @@ spec: items: type: object required: - - name - port - protocol - targetPort @@ -69,13 +61,50 @@ spec: type: string type: type: string + subsets: + type: array + items: + type: object + properties: + addresses: + type: array + items: + type: object + properties: + hostname: + type: string + ip: + type: string + nodeName: + type: string + targetRef: + type: object + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + ports: + type: array + items: + type: object + properties: + name: + type: string + port: + type: integer + protocol: + type: string status: type: object - required: - - listeners - - loadbalancer - - members - - pools properties: listeners: type: array diff --git a/kuryr_kubernetes/constants.py b/kuryr_kubernetes/constants.py index add9b9a35..4b1fddc5d 100644 --- a/kuryr_kubernetes/constants.py +++ b/kuryr_kubernetes/constants.py @@ -66,6 +66,8 @@ POD_FINALIZER = KURYR_FQDN + '/pod-finalizer' KURYRNETWORK_FINALIZER = 'kuryrnetwork.finalizers.kuryr.openstack.org' +KURYRLB_FINALIZER = 'kuryr.openstack.org/kuryrloadbalancer-finalizers' +SERVICE_FINALIZER = 'kuryr.openstack.org/service-finalizer' KURYRPORT_FINALIZER = KURYR_FQDN + '/kuryrport-finalizer' KURYRPORT_LABEL = KURYR_FQDN + '/nodeName' diff --git a/kuryr_kubernetes/controller/drivers/lb_public_ip.py b/kuryr_kubernetes/controller/drivers/lb_public_ip.py index 6fb24041e..4354a5758 100644 --- a/kuryr_kubernetes/controller/drivers/lb_public_ip.py +++ b/kuryr_kubernetes/controller/drivers/lb_public_ip.py @@ -15,7 +15,6 @@ from kuryr_kubernetes import config from kuryr_kubernetes.controller.drivers import base from kuryr_kubernetes.controller.drivers import public_ip -from kuryr_kubernetes.objects import lbaas as obj_lbaas from oslo_config import cfg from oslo_log import log as logging @@ -50,10 +49,11 @@ def acquire_service_pub_ip_info(self, spec_type, spec_lb_ip, project_id, res_id = self._drv_pub_ip.is_ip_available(user_specified_ip, port_id_to_be_associated) if res_id: - service_pub_ip_info = (obj_lbaas.LBaaSPubIp( - ip_id=res_id, - ip_addr=str(user_specified_ip), - alloc_method='user')) + service_pub_ip_info = { + 'ip_id': res_id, + 'ip_addr': str(user_specified_ip), + 'alloc_method': 'user' + } return service_pub_ip_info else: @@ -78,32 +78,34 @@ def acquire_service_pub_ip_info(self, spec_type, spec_lb_ip, project_id, LOG.exception("Failed to allocate public IP - net_id:%s", public_network_id) return None - service_pub_ip_info = obj_lbaas.LBaaSPubIp(ip_id=res_id, - ip_addr=alloc_ip_addr, - alloc_method='pool') + service_pub_ip_info = { + 'ip_id': res_id, + 'ip_addr': alloc_ip_addr, + 'alloc_method': 'pool' + } return service_pub_ip_info def release_pub_ip(self, service_pub_ip_info): if not service_pub_ip_info: return True - if service_pub_ip_info.alloc_method == 'pool': - retcode = self._drv_pub_ip.free_ip(service_pub_ip_info.ip_id) + if service_pub_ip_info['alloc_method'] == 'pool': + retcode = self._drv_pub_ip.free_ip(service_pub_ip_info['ip_id']) if not retcode: LOG.error("Failed to delete public_ip_id =%s !", - service_pub_ip_info.ip_id) + service_pub_ip_info['ip_id']) return False return True def associate_pub_ip(self, service_pub_ip_info, vip_port_id): if (not service_pub_ip_info or not vip_port_id or - not service_pub_ip_info.ip_id): + not service_pub_ip_info['ip_id']): return self._drv_pub_ip.associate( - service_pub_ip_info.ip_id, vip_port_id) + service_pub_ip_info['ip_id'], vip_port_id) def disassociate_pub_ip(self, service_pub_ip_info): - if not service_pub_ip_info or not service_pub_ip_info.ip_id: + if not service_pub_ip_info or not service_pub_ip_info['ip_id']: return - self._drv_pub_ip.disassociate(service_pub_ip_info.ip_id) + self._drv_pub_ip.disassociate(service_pub_ip_info['ip_id']) diff --git a/kuryr_kubernetes/controller/drivers/lbaasv2.py b/kuryr_kubernetes/controller/drivers/lbaasv2.py index 0f577c0ab..aeae61ece 100644 --- a/kuryr_kubernetes/controller/drivers/lbaasv2.py +++ b/kuryr_kubernetes/controller/drivers/lbaasv2.py @@ -27,7 +27,6 @@ from kuryr_kubernetes import constants as k_const from kuryr_kubernetes.controller.drivers import base from kuryr_kubernetes import exceptions as k_exc -from kuryr_kubernetes.objects import lbaas as obj_lbaas from kuryr_kubernetes import utils CONF = cfg.CONF @@ -112,7 +111,7 @@ def get_service_loadbalancer_name(self, namespace, svc_name): return "%s/%s" % (namespace, svc_name) def get_loadbalancer_pool_name(self, loadbalancer, namespace, svc_name): - return "%s/%s/%s" % (loadbalancer.name, namespace, svc_name) + return "%s/%s/%s" % (loadbalancer['name'], namespace, svc_name) def add_tags(self, resource, req): if CONF.neutron_defaults.resource_tags: @@ -126,9 +125,14 @@ def add_tags(self, resource, req): def ensure_loadbalancer(self, name, project_id, subnet_id, ip, security_groups_ids=None, service_type=None, provider=None): - request = obj_lbaas.LBaaSLoadBalancer( - name=name, project_id=project_id, subnet_id=subnet_id, ip=ip, - security_groups=security_groups_ids, provider=provider) + request = { + 'name': name, + 'project_id': project_id, + 'subnet_id': subnet_id, + 'ip': ip, + 'security_groups': security_groups_ids, + 'provider': provider + } response = self._ensure(self._create_loadbalancer, self._find_loadbalancer, request) if not response: @@ -146,9 +150,8 @@ def release_loadbalancer(self, loadbalancer): loadbalancer, loadbalancer, lbaas.delete_load_balancer, - loadbalancer.id, + loadbalancer['id'], cascade=True) - self._wait_for_deletion(loadbalancer, _ACTIVATION_TIMEOUT) def _create_listeners_acls(self, loadbalancer, port, target_port, @@ -160,7 +163,7 @@ def _create_listeners_acls(self, loadbalancer, port, target_port, if new_sgs: sgs = new_sgs else: - sgs = loadbalancer.security_groups + sgs = loadbalancer['security_groups'] # Check if Network Policy allows listener on the pods for sg in sgs: @@ -210,7 +213,7 @@ def _apply_members_security_groups(self, loadbalancer, port, target_port, if vip_port: lb_sg = vip_port.security_group_ids[0] else: - LOG.debug("Skipping sg update for lb %s", loadbalancer.name) + LOG.debug("Skipping sg update for lb %s", loadbalancer['name']) return # NOTE (maysams) It might happen that the update of LBaaS SG @@ -225,14 +228,14 @@ def _apply_members_security_groups(self, loadbalancer, port, target_port, return lbaas_sg_rules = os_net.security_group_rules( - security_group_id=lb_sg, project_id=loadbalancer.project_id) + security_group_id=lb_sg, project_id=loadbalancer['project_id']) all_pod_rules = [] add_default_rules = False if new_sgs: sgs = new_sgs else: - sgs = loadbalancer.security_groups + sgs = loadbalancer['security_groups'] sg_rule_ethertype = k_const.IPv4 if utils.get_service_subnet_version() == k_const.IP_VERSION_6: @@ -325,12 +328,14 @@ def _is_default_rule(self, rule): def ensure_listener(self, loadbalancer, protocol, port, service_type='ClusterIP'): - name = "%s:%s:%s" % (loadbalancer.name, protocol, port) - listener = obj_lbaas.LBaaSListener(name=name, - project_id=loadbalancer.project_id, - loadbalancer_id=loadbalancer.id, - protocol=protocol, - port=port) + name = "%s:%s:%s" % (loadbalancer['name'], protocol, port) + listener = { + 'name': name, + 'project_id': loadbalancer['project_id'], + 'loadbalancer_id': loadbalancer['id'], + 'protocol': protocol, + 'port': port + } try: result = self._ensure_provisioned( loadbalancer, listener, self._create_listener, @@ -348,7 +353,7 @@ def ensure_listener(self, loadbalancer, protocol, port, os_net = clients.get_network_client() vip_port = self._get_vip_port(loadbalancer) os_net.update_port(vip_port.id, security_groups=[]) - loadbalancer.security_groups = [] + loadbalancer['security_groups'] = [] return result @@ -357,7 +362,7 @@ def release_listener(self, loadbalancer, listener): lbaas = clients.get_loadbalancer_client() self._release(loadbalancer, listener, lbaas.delete_listener, - listener.id) + listener['id']) # NOTE(maysams): since lbs created with ovn-octavia provider # does not have a sg in place, only need to delete sg rules @@ -367,19 +372,22 @@ def release_listener(self, loadbalancer, listener): sg_id = self._get_vip_port(loadbalancer).security_group_ids[0] if sg_id: rules = os_net.security_group_rules(security_group_id=sg_id, - description=listener.name) + description=listener[ + 'name']) try: os_net.delete_security_group_rule(next(rules).id) except StopIteration: LOG.warning('Cannot find SG rule for %s (%s) listener.', - listener.id, listener.name) + listener['id'], listener['name']) def ensure_pool(self, loadbalancer, listener): - pool = obj_lbaas.LBaaSPool(name=listener.name, - project_id=loadbalancer.project_id, - loadbalancer_id=loadbalancer.id, - listener_id=listener.id, - protocol=listener.protocol) + pool = { + 'name': listener['name'], + 'project_id': loadbalancer['project_id'], + 'loadbalancer_id': loadbalancer['id'], + 'listener_id': listener['id'], + 'protocol': listener['protocol'] + } return self._ensure_provisioned(loadbalancer, pool, self._create_pool, self._find_pool) @@ -388,30 +396,34 @@ def ensure_pool_attached_to_lb(self, loadbalancer, namespace, svc_name, protocol): name = self.get_loadbalancer_pool_name(loadbalancer, namespace, svc_name) - pool = obj_lbaas.LBaaSPool(name=name, - project_id=loadbalancer.project_id, - loadbalancer_id=loadbalancer.id, - listener_id=None, - protocol=protocol) + pool = { + 'name': name, + 'project_id': loadbalancer['project_id'], + 'loadbalancer_id': loadbalancer['id'], + 'listener_id': None, + 'protocol': protocol + } return self._ensure_provisioned(loadbalancer, pool, self._create_pool, self._find_pool_by_name) def release_pool(self, loadbalancer, pool): lbaas = clients.get_loadbalancer_client() - self._release(loadbalancer, pool, lbaas.delete_pool, pool.id) + self._release(loadbalancer, pool, lbaas.delete_pool, pool['id']) def ensure_member(self, loadbalancer, pool, subnet_id, ip, port, target_ref_namespace, target_ref_name, listener_port=None): name = ("%s/%s" % (target_ref_namespace, target_ref_name)) name += ":%s" % port - member = obj_lbaas.LBaaSMember(name=name, - project_id=loadbalancer.project_id, - pool_id=pool.id, - subnet_id=subnet_id, - ip=ip, - port=port) + member = { + 'name': name, + 'project_id': loadbalancer['project_id'], + 'pool_id': pool['id'], + 'subnet_id': subnet_id, + 'ip': ip, + 'port': port + } result = self._ensure_provisioned(loadbalancer, member, self._create_member, self._find_member) @@ -421,9 +433,9 @@ def ensure_member(self, loadbalancer, pool, CONF.kubernetes.service_security_groups_driver == 'policy') if (network_policy and CONF.octavia_defaults.enforce_sg_rules and listener_port): - protocol = pool.protocol - sg_rule_name = pool.name - listener_id = pool.listener_id + protocol = pool['protocol'] + sg_rule_name = pool['name'] + listener_id = pool['listener_id'] self._apply_members_security_groups(loadbalancer, listener_port, port, protocol, sg_rule_name, listener_id) @@ -431,14 +443,14 @@ def ensure_member(self, loadbalancer, pool, def release_member(self, loadbalancer, member): lbaas = clients.get_loadbalancer_client() - self._release(loadbalancer, member, lbaas.delete_member, member.id, - member.pool_id) + self._release(loadbalancer, member, lbaas.delete_member, member['id'], + member['pool_id']) def _get_vip_port(self, loadbalancer): os_net = clients.get_network_client() try: - fixed_ips = ['subnet_id=%s' % str(loadbalancer.subnet_id), - 'ip_address=%s' % str(loadbalancer.ip)] + fixed_ips = ['subnet_id=%s' % str(loadbalancer['subnet_id']), + 'ip_address=%s' % str(loadbalancer['ip'])] ports = os_net.ports(fixed_ips=fixed_ips) except os_exc.SDKException: LOG.error("Port with fixed ips %s not found!", fixed_ips) @@ -451,43 +463,43 @@ def _get_vip_port(self, loadbalancer): def _create_loadbalancer(self, loadbalancer): request = { - 'name': loadbalancer.name, - 'project_id': loadbalancer.project_id, - 'vip_address': str(loadbalancer.ip), - 'vip_subnet_id': loadbalancer.subnet_id, + 'name': loadbalancer['name'], + 'project_id': loadbalancer['project_id'], + 'vip_address': str(loadbalancer['ip']), + 'vip_subnet_id': loadbalancer['subnet_id'], } - if loadbalancer.provider is not None: - request['provider'] = loadbalancer.provider + if loadbalancer['provider'] is not None: + request['provider'] = loadbalancer['provider'] self.add_tags('loadbalancer', request) lbaas = clients.get_loadbalancer_client() response = lbaas.create_load_balancer(**request) - loadbalancer.id = response.id - loadbalancer.port_id = self._get_vip_port(loadbalancer).id - if (loadbalancer.provider is not None and - loadbalancer.provider != response.provider): + loadbalancer['id'] = response.id + loadbalancer['port_id'] = self._get_vip_port(loadbalancer).id + if (loadbalancer['provider'] is not None and + loadbalancer['provider'] != response.provider): LOG.error("Request provider(%s) != Response provider(%s)", - loadbalancer.provider, response.provider) + loadbalancer['provider'], response.provider) return None - loadbalancer.provider = response.provider + loadbalancer['provider'] = response.provider return loadbalancer def _find_loadbalancer(self, loadbalancer): lbaas = clients.get_loadbalancer_client() response = lbaas.load_balancers( - name=loadbalancer.name, - project_id=loadbalancer.project_id, - vip_address=str(loadbalancer.ip), - vip_subnet_id=loadbalancer.subnet_id, - provider=loadbalancer.provider) + name=loadbalancer['name'], + project_id=loadbalancer['project_id'], + vip_address=str(loadbalancer['ip']), + vip_subnet_id=loadbalancer['subnet_id'], + provider=loadbalancer['provider']) try: os_lb = next(response) # openstacksdk returns a generator - loadbalancer.id = os_lb.id - loadbalancer.port_id = self._get_vip_port(loadbalancer).id - loadbalancer.provider = os_lb.provider + loadbalancer['id'] = os_lb.id + loadbalancer['port_id'] = self._get_vip_port(loadbalancer).id + loadbalancer['provider'] = os_lb.provider if os_lb.provisioning_status == 'ERROR': self.release_loadbalancer(loadbalancer) return None @@ -498,16 +510,16 @@ def _find_loadbalancer(self, loadbalancer): def _create_listener(self, listener): request = { - 'name': listener.name, - 'project_id': listener.project_id, - 'loadbalancer_id': listener.loadbalancer_id, - 'protocol': listener.protocol, - 'protocol_port': listener.port, + 'name': listener['name'], + 'project_id': listener['project_id'], + 'loadbalancer_id': listener['loadbalancer_id'], + 'protocol': listener['protocol'], + 'protocol_port': listener['port'], } self.add_tags('listener', request) lbaas = clients.get_loadbalancer_client() response = lbaas.create_listener(**request) - listener.id = response.id + listener['id'] = response.id return listener def _update_listener_acls(self, loadbalancer, listener_id, allowed_cidrs): @@ -538,15 +550,15 @@ def _update_listener_acls(self, loadbalancer, listener_id, allowed_cidrs): def _find_listener(self, listener, loadbalancer): lbaas = clients.get_loadbalancer_client() response = lbaas.listeners( - name=listener.name, - project_id=listener.project_id, - load_balancer_id=listener.loadbalancer_id, - protocol=listener.protocol, - protocol_port=listener.port) + name=listener['name'], + project_id=listener['project_id'], + load_balancer_id=listener['loadbalancer_id'], + protocol=listener['protocol'], + protocol_port=listener['port']) try: os_listener = next(response) - listener.id = os_listener.id + listener['id'] = os_listener.id if os_listener.provisioning_status == 'ERROR': LOG.debug("Releasing listener %s", os_listener.id) self.release_listener(loadbalancer, listener) @@ -560,34 +572,34 @@ def _create_pool(self, pool): # TODO(ivc): make lb_algorithm configurable lb_algorithm = CONF.octavia_defaults.lb_algorithm request = { - 'name': pool.name, - 'project_id': pool.project_id, - 'listener_id': pool.listener_id, - 'loadbalancer_id': pool.loadbalancer_id, - 'protocol': pool.protocol, + 'name': pool['name'], + 'project_id': pool['project_id'], + 'listener_id': pool['listener_id'], + 'loadbalancer_id': pool['loadbalancer_id'], + 'protocol': pool['protocol'], 'lb_algorithm': lb_algorithm, } self.add_tags('pool', request) lbaas = clients.get_loadbalancer_client() response = lbaas.create_pool(**request) - pool.id = response.id + pool['id'] = response.id return pool def _find_pool(self, pool, loadbalancer, by_listener=True): lbaas = clients.get_loadbalancer_client() response = lbaas.pools( - name=pool.name, - project_id=pool.project_id, - loadbalancer_id=pool.loadbalancer_id, - protocol=pool.protocol) - + name=pool['name'], + project_id=pool['project_id'], + loadbalancer_id=pool['loadbalancer_id'], + protocol=pool['protocol']) + # TODO(scavnic) check response try: if by_listener: - pools = [p for p in response if pool.listener_id + pools = [p for p in response if pool['listener_id'] in {listener['id'] for listener in p.listeners}] else: pools = [p for p in response if pool.name == p.name] - pool.id = pools[0].id + pool['id'] = pools[0].id if pools[0].provisioning_status == 'ERROR': LOG.debug("Releasing pool %s", pool.id) self.release_pool(loadbalancer, pool) @@ -601,31 +613,31 @@ def _find_pool_by_name(self, pool, loadbalancer): def _create_member(self, member): request = { - 'name': member.name, - 'project_id': member.project_id, - 'subnet_id': member.subnet_id, - 'address': str(member.ip), - 'protocol_port': member.port, + 'name': member['name'], + 'project_id': member['project_id'], + 'subnet_id': member['subnet_id'], + 'address': str(member['ip']), + 'protocol_port': member['port'], } self.add_tags('member', request) lbaas = clients.get_loadbalancer_client() - response = lbaas.create_member(member.pool_id, **request) - member.id = response.id + response = lbaas.create_member(member['pool_id'], **request) + member['id'] = response.id return member def _find_member(self, member, loadbalancer): lbaas = clients.get_loadbalancer_client() response = lbaas.members( - member.pool_id, - name=member.name, - project_id=member.project_id, - subnet_id=member.subnet_id, - address=member.ip, - protocol_port=member.port) + member['pool_id'], + name=member['name'], + project_id=member['project_id'], + subnet_id=member['subnet_id'], + address=member['ip'], + protocol_port=member['port']) try: os_members = next(response) - member.id = os_members.id + member['id'] = os_members.id if os_members.provisioning_status == 'ERROR': LOG.debug("Releasing Member %s", os_members.id) self.release_member(loadbalancer, member) @@ -683,7 +695,7 @@ def _wait_for_provisioning(self, loadbalancer, timeout, lbaas = clients.get_loadbalancer_client() for remaining in self._provisioning_timer(timeout, interval): - response = lbaas.get_load_balancer(loadbalancer.id) + response = lbaas.get_load_balancer(loadbalancer['id']) status = response.provisioning_status if status == 'ACTIVE': LOG.debug("Provisioning complete for %(lb)s", { @@ -691,7 +703,7 @@ def _wait_for_provisioning(self, loadbalancer, timeout, return elif status == 'ERROR': LOG.debug("Releasing loadbalancer %s with error status", - loadbalancer.id) + loadbalancer['id']) self.release_loadbalancer(loadbalancer) break else: @@ -708,7 +720,7 @@ def _wait_for_deletion(self, loadbalancer, timeout, for remaining in self._provisioning_timer(timeout, interval): try: - lbaas.get_load_balancer(loadbalancer.id) + lbaas.get_load_balancer(loadbalancer['id']) except os_exc.NotFoundException: return @@ -753,7 +765,7 @@ def update_lbaas_sg(self, service, sgs): utils.set_lbaas_state(endpoint, lbaas) - lsnr_ids = {(listener.protocol, listener.port): listener.id + lsnr_ids = {(listener['protocol'], listener['port']): listener['id'] for listener in lbaas.listeners} for port in svc_ports: diff --git a/kuryr_kubernetes/controller/handlers/lbaas.py b/kuryr_kubernetes/controller/handlers/lbaas.py index c22c33531..5884a0ef6 100644 --- a/kuryr_kubernetes/controller/handlers/lbaas.py +++ b/kuryr_kubernetes/controller/handlers/lbaas.py @@ -13,18 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. -import eventlet -import time - from kuryr.lib._i18n import _ -from openstack import exceptions as os_exc from oslo_log import log as logging from kuryr_kubernetes import clients from kuryr_kubernetes import config from kuryr_kubernetes import constants as k_const from kuryr_kubernetes.controller.drivers import base as drv_base -from kuryr_kubernetes.controller.drivers import utils as driver_utils from kuryr_kubernetes import exceptions as k_exc from kuryr_kubernetes.handlers import k8s_base from kuryr_kubernetes.objects import lbaas as obj_lbaas @@ -35,10 +30,10 @@ SUPPORTED_SERVICE_TYPES = ('ClusterIP', 'LoadBalancer') -class LBaaSSpecHandler(k8s_base.ResourceEventHandler): - """LBaaSSpecHandler handles K8s Service events. +class ServiceHandler(k8s_base.ResourceEventHandler): + """ServiceHandler handles K8s Service events. - LBaaSSpecHandler handles K8s Service events and updates related Endpoints + ServiceHandler handles K8s Service events and updates related Endpoints with LBaaSServiceSpec when necessary. """ @@ -46,14 +41,12 @@ class LBaaSSpecHandler(k8s_base.ResourceEventHandler): OBJECT_WATCH_PATH = "%s/%s" % (k_const.K8S_API_BASE, "services") def __init__(self): - super(LBaaSSpecHandler, self).__init__() + super(ServiceHandler, self).__init__() self._drv_project = drv_base.ServiceProjectDriver.get_instance() self._drv_subnets = drv_base.ServiceSubnetsDriver.get_instance() self._drv_sg = drv_base.ServiceSecurityGroupsDriver.get_instance() def on_present(self, service): - lbaas_spec = utils.get_lbaas_spec(service) - if self._should_ignore(service): LOG.debug("Skipping Kubernetes service %s of an unsupported kind " "or without a selector as Kubernetes does not create " @@ -61,9 +54,18 @@ def on_present(self, service): service['metadata']['name']) return - if self._has_lbaas_spec_changes(service, lbaas_spec): - lbaas_spec = self._generate_lbaas_spec(service) - utils.set_lbaas_spec(service, lbaas_spec) + k8s = clients.get_kubernetes_client() + loadbalancer_crd = k8s.get_loadbalancer_crd(service) + try: + self._patch_service_finalizer(service) + except k_exc.K8sClientException as ex: + LOG.exception("Failed to set service finalizer: %s", ex) + raise + + if loadbalancer_crd is None: + loadbalancer_crd = self.create_crd_spec(service) + elif self._has_lbaas_spec_changes(service, loadbalancer_crd): + loadbalancer_crd = self._update_crd_spec(loadbalancer_crd, service) def _is_supported_type(self, service): spec = service['spec'] @@ -75,12 +77,22 @@ def _get_service_ip(self, service): return None def _should_ignore(self, service): - return (not(self._has_selector(service)) or - not(self._has_clusterip(service)) or + return (not(self._has_clusterip(service)) or not(self._is_supported_type(service))) - def _has_selector(self, service): - return service['spec'].get('selector') + def _patch_service_finalizer(self, service): + k8s = clients.get_kubernetes_client() + k8s.add_finalizer(service, k_const.SERVICE_FINALIZER) + + def on_finalize(self, service): + k8s = clients.get_kubernetes_client() + + svc_name = service['metadata']['name'] + svc_namespace = service['metadata']['namespace'] + + klb_crd_path = (f"{k_const.K8S_API_CRD_NAMESPACES}/" + f"{svc_namespace}/kuryrloadbalancers/{svc_name}") + k8s.delete(klb_crd_path) def _has_clusterip(self, service): # ignore headless service, clusterIP is None @@ -97,45 +109,113 @@ def _get_subnet_id(self, service, project_id, ip): if len(subnet_ids) != 1: raise k_exc.IntegrityError(_( "Found %(num)s subnets for service %(link)s IP %(ip)s") % { - 'link': service['metadata']['selfLink'], - 'ip': ip, - 'num': len(subnet_ids)}) + 'link': service['metadata']['selfLink'], + 'ip': ip, + 'num': len(subnet_ids)}) return subnet_ids.pop() - def _generate_lbaas_spec(self, service): + def create_crd_spec(self, service): + svc_name = service['metadata']['name'] + svc_namespace = service['metadata']['namespace'] + kubernetes = clients.get_kubernetes_client() + svc_ip = self._get_service_ip(service) + spec_lb_ip = service['spec'].get('loadBalancerIP') + ports = service['spec'].get('ports') + for port in ports: + if type(port['targetPort']) == int: + port['targetPort'] = str(port['targetPort']) project_id = self._drv_project.get_project(service) - ip = self._get_service_ip(service) - subnet_id = self._get_subnet_id(service, project_id, ip) - ports = self._generate_lbaas_port_specs(service) sg_ids = self._drv_sg.get_security_groups(service, project_id) + subnet_id = self._get_subnet_id(service, project_id, svc_ip) spec_type = service['spec'].get('type') - spec_lb_ip = service['spec'].get('loadBalancerIP') + loadbalancer_crd = { + 'apiVersion': 'openstack.org/v1', + 'kind': 'KuryrLoadBalancer', + 'metadata': { + 'name': svc_name, + 'finalizers': [k_const.KURYRLB_FINALIZER], + }, + 'spec': { + 'ip': svc_ip, + 'ports': ports, + 'project_id': project_id, + 'security_groups_ids': sg_ids, + 'subnet_id': subnet_id, + 'type': spec_type + }, + 'status': { + } + } + + if spec_lb_ip is not None: + loadbalancer_crd['spec']['lb_ip'] = spec_lb_ip + + try: + kubernetes.post('{}/{}/kuryrloadbalancers'.format( + k_const.K8S_API_CRD_NAMESPACES, svc_namespace), + loadbalancer_crd) + except k_exc.K8sConflict: + raise k_exc.ResourceNotReady(svc_name) + except k_exc.K8sClientException: + LOG.exception("Kubernetes Client Exception creating " + "kuryrloadbalancer CRD. %s" + % k_exc.K8sClientException) + raise + return loadbalancer_crd - return obj_lbaas.LBaaSServiceSpec(ip=ip, - project_id=project_id, - subnet_id=subnet_id, - ports=ports, - security_groups_ids=sg_ids, - type=spec_type, - lb_ip=spec_lb_ip) + def _update_crd_spec(self, loadbalancer_crd, service): + svc_ip = self._get_service_ip(service) + ports = service['spec'].get('ports') + for port in ports: + if type(port['targetPort']) == int: + port['targetPort'] = str(port['targetPort']) + project_id = self._drv_project.get_project(service) + sg_ids = self._drv_sg.get_security_groups(service, project_id) + subnet_id = self._get_subnet_id(service, project_id, svc_ip) + spec_type = service['spec'].get('type') + kubernetes = clients.get_kubernetes_client() + + patch = { + 'spec': { + 'ip': svc_ip, + 'ports': ports, + 'project_id': project_id, + 'security_groups_ids': sg_ids, + 'subnet_id': subnet_id, + 'type': spec_type + } + } + + LOG.debug('Patching KuryrLoadBalancer CRD %s', loadbalancer_crd) + try: + kubernetes.patch_crd('spec', loadbalancer_crd['metadata'][ + 'selfLink'], patch['spec']) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadBalancer CRD not found %s', loadbalancer_crd) + except k_exc.K8sConflict: + raise k_exc.ResourceNotReady(loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating kuryrnet CRD %s', loadbalancer_crd) + raise + return loadbalancer_crd - def _has_lbaas_spec_changes(self, service, lbaas_spec): - return (self._has_ip_changes(service, lbaas_spec) or - utils.has_port_changes(service, lbaas_spec)) + def _has_lbaas_spec_changes(self, service, loadbalancer_crd): + return (self._has_ip_changes(service, loadbalancer_crd) or + utils.has_port_changes(service, loadbalancer_crd)) - def _has_ip_changes(self, service, lbaas_spec): + def _has_ip_changes(self, service, loadbalancer_crd): link = service['metadata']['selfLink'] svc_ip = self._get_service_ip(service) - if not lbaas_spec: - if svc_ip: - LOG.debug("LBaaS spec is missing for %(link)s" - % {'link': link}) - return True - elif str(lbaas_spec.ip) != svc_ip: + if loadbalancer_crd['spec'].get('ip') is None: + if svc_ip is None: + return False + return True + + elif str(loadbalancer_crd['spec'].get('ip')) != svc_ip: LOG.debug("LBaaS spec IP %(spec_ip)s != %(svc_ip)s for %(link)s" - % {'spec_ip': lbaas_spec.ip, + % {'spec_ip': loadbalancer_crd['spec']['ip'], 'svc_ip': svc_ip, 'link': link}) return True @@ -147,10 +227,10 @@ def _generate_lbaas_port_specs(self, service): for port in utils.get_service_ports(service)] -class LoadBalancerHandler(k8s_base.ResourceEventHandler): - """LoadBalancerHandler handles K8s Endpoints events. +class EndpointsHandler(k8s_base.ResourceEventHandler): + """EndpointsHandler handles K8s Endpoints events. - LoadBalancerHandler handles K8s Endpoints events and tracks changes in + EndpointsHandler handles K8s Endpoints events and tracks changes in LBaaSServiceSpec to update Neutron LBaaS accordingly and to reflect its' actual state in LBaaSState. """ @@ -159,13 +239,11 @@ class LoadBalancerHandler(k8s_base.ResourceEventHandler): OBJECT_WATCH_PATH = "%s/%s" % (k_const.K8S_API_BASE, "endpoints") def __init__(self): - super(LoadBalancerHandler, self).__init__() + super(EndpointsHandler, self).__init__() self._drv_lbaas = drv_base.LBaaSDriver.get_instance() self._drv_pod_project = drv_base.PodProjectDriver.get_instance() self._drv_pod_subnets = drv_base.PodSubnetsDriver.get_instance() self._drv_service_pub_ip = drv_base.ServicePubIpDriver.get_instance() - self._drv_project = drv_base.ServiceProjectDriver.get_instance() - self._drv_sg = drv_base.ServiceSecurityGroupsDriver.get_instance() # Note(yboaron) LBaaS driver supports 'provider' parameter in # Load Balancer creation flow. # We need to set the requested load balancer provider @@ -175,104 +253,40 @@ def __init__(self): != 'default'): self._lb_provider = ( config.CONF.kubernetes.endpoints_driver_octavia_provider) - eventlet.spawn(self._cleanup_leftover_lbaas) def on_present(self, endpoints): - lbaas_spec = utils.get_lbaas_spec(endpoints) - if self._should_ignore(endpoints, lbaas_spec): + k8s = clients.get_kubernetes_client() + loadbalancer_crd = k8s.get_loadbalancer_crd(endpoints) + + if not self._has_pods(endpoints): LOG.debug("Ignoring Kubernetes endpoints %s", endpoints['metadata']['name']) return - lbaas_state = utils.get_lbaas_state(endpoints) - if not lbaas_state: - lbaas_state = obj_lbaas.LBaaSState() - elif (lbaas_state.loadbalancer and self._lb_provider and - self._lb_provider != lbaas_state.loadbalancer.provider): - LOG.info("LoadBalancer associated to the service does not match " - "the current provider: %s", lbaas_state.loadbalancer.id) - lb_client = clients.get_loadbalancer_client() - try: - lb_client.get_load_balancer(lbaas_state.loadbalancer.id) - except os_exc.NotFoundException: - # NOTE(ltomasbo): If the loadbalancer is gone, remove the - # annotations to ensure it is reprocessed - lbaas_state.loadbalancer = None - lbaas_state.pools = [] - lbaas_state.listeners = [] - lbaas_state.members = [] - utils.set_lbaas_state(endpoints, lbaas_state) - return + if loadbalancer_crd is None: + loadbalancer_crd = self._create_crd_spec(endpoints) + else: + loadbalancer_crd = self._update_crd_spec(loadbalancer_crd, + endpoints) + + def _has_lbaas_spec_changes(self, endpoints, loadbalancer_crd): + return (self._has_ip_changes(endpoints, loadbalancer_crd) or + utils.has_port_changes(endpoints, loadbalancer_crd)) + + def _has_ip_changes(self, endpoints, loadbalancer_crd): + link = endpoints['metadata']['selfLink'] + endpoint_ip = endpoints['subsets']['addresses'].get('ip') + endpoint_crd_ip = loadbalancer_crd['spec'].get('ip') + + if endpoint_crd_ip != endpoint_ip: + LOG.debug("LBaaS spec IP %(endpoint_crd_ip)s !=" + " %(endpoint_ip)s for %(link)s" + % {'endpoint_crd_ip': endpoint_crd_ip, + 'endpoint_ip': endpoint_ip, + 'link': link}) + return True - if self._sync_lbaas_members(endpoints, lbaas_state, lbaas_spec): - # Note(yboaron) For LoadBalancer services, we should allocate FIP, - # associate it to LB VIP and update K8S service status - if lbaas_state.service_pub_ip_info is None: - service_pub_ip_info = ( - self._drv_service_pub_ip.acquire_service_pub_ip_info( - lbaas_spec.type, - lbaas_spec.lb_ip, - lbaas_spec.project_id, - lbaas_state.loadbalancer.port_id)) - if service_pub_ip_info: - self._drv_service_pub_ip.associate_pub_ip( - service_pub_ip_info, lbaas_state.loadbalancer.port_id) - lbaas_state.service_pub_ip_info = service_pub_ip_info - self._update_lb_status( - endpoints, - lbaas_state.service_pub_ip_info.ip_addr) - # REVISIT(ivc): since _sync_lbaas_members is responsible for - # creating all lbaas components (i.e. load balancer, listeners, - # pools, members), it is currently possible for it to fail (due - # to invalid Kuryr/K8s/Neutron configuration, e.g. Members' IPs - # not belonging to configured Neutron subnet or Service IP being - # in use by gateway or VMs) leaving some Neutron entities without - # properly updating annotation. Some sort of failsafe mechanism is - # required to deal with such situations (e.g. cleanup, or skip - # failing items, or validate configuration) to prevent annotation - # being out of sync with the actual Neutron state. - try: - utils.set_lbaas_state(endpoints, lbaas_state) - except k_exc.K8sResourceNotFound: - # Note(yboaron) It's impossible to store neutron resources - # in K8S object since object was deleted. In that case - # we should rollback all neutron resources. - LOG.debug("LoadBalancerHandler failed to store Openstack " - "resources in K8S object (not found)") - self.on_deleted(endpoints, lbaas_state) - - def on_deleted(self, endpoints, lbaas_state=None): - if lbaas_state is None: - lbaas_state = utils.get_lbaas_state(endpoints) - if not lbaas_state: - return - # NOTE(ivc): deleting pool deletes its members - self._drv_lbaas.release_loadbalancer( - loadbalancer=lbaas_state.loadbalancer) - if lbaas_state.service_pub_ip_info: - self._drv_service_pub_ip.release_pub_ip( - lbaas_state.service_pub_ip_info) - - def _should_ignore(self, endpoints, lbaas_spec): - # NOTE(ltomasbo): we must wait until service handler has annotated the - # endpoints to process them. Thus, if annotations are not updated to - # match the endpoints information, we should skip the event - return not(lbaas_spec and - self._has_pods(endpoints) and - self._svc_handler_annotations_updated(endpoints, - lbaas_spec)) - - def _svc_handler_annotations_updated(self, endpoints, lbaas_spec): - svc_link = self._get_service_link(endpoints) - k8s = clients.get_kubernetes_client() - service = k8s.get(svc_link) - if utils.has_port_changes(service, lbaas_spec): - # NOTE(ltomasbo): Ensuring lbaas_spec annotated on the endpoints - # is in sync with the service status, i.e., upon a service - # modification it will ensure endpoint modifications are not - # handled until the service handler has performed its annotations - return False - return True + return False def _has_pods(self, endpoints): ep_subsets = endpoints.get('subsets', []) @@ -283,327 +297,58 @@ def _has_pods(self, endpoints): for address in subset.get('addresses', []) if address.get('targetRef', {}).get('kind') == 'Pod') - def _sync_lbaas_members(self, endpoints, lbaas_state, lbaas_spec): - changed = False - - if (self._has_pods(endpoints) and - self._remove_unused_members(endpoints, lbaas_state, - lbaas_spec)): - changed = True - - if self._sync_lbaas_pools(endpoints, lbaas_state, lbaas_spec): - changed = True - - if (self._has_pods(endpoints) and - self._add_new_members(endpoints, lbaas_state, lbaas_spec)): - changed = True - - return changed - - def _sync_lbaas_sgs(self, endpoints, lbaas_state): - svc_link = self._get_service_link(endpoints) - k8s = clients.get_kubernetes_client() - service = k8s.get(svc_link) - - lb = lbaas_state.loadbalancer - # NOTE(maysams) It's possible that while the service annotation - # is added the backend pods on that service are not yet created - # resulting in no security groups retrieved for the service. - # Let's retrieve again to ensure is updated. - project_id = self._drv_project.get_project(service) - lb_sgs = self._drv_sg.get_security_groups(service, project_id) - lb.security_groups = lb_sgs - - def _add_new_members(self, endpoints, lbaas_state, lbaas_spec): - changed = False - - if config.CONF.octavia_defaults.enforce_sg_rules: - try: - self._sync_lbaas_sgs(endpoints, lbaas_state) - except k_exc.K8sResourceNotFound: - LOG.debug("The svc has been deleted while processing" - " the endpoints update. No need to add new" - " members.") - - lsnr_by_id = {listener.id: listener - for listener in lbaas_state.listeners} - pool_by_lsnr_port = {(lsnr_by_id[p.listener_id].protocol, - lsnr_by_id[p.listener_id].port): p - for p in lbaas_state.pools} - - # NOTE(yboaron): Since LBaaSv2 doesn't support UDP load balancing, - # the LBaaS driver will return 'None' in case of UDP port - # listener creation. - # we should consider the case in which - # 'pool_by_lsnr_port[p.protocol, p.port]' is missing - pool_by_tgt_name = {} - for p in lbaas_spec.ports: - try: - pool_by_tgt_name[p.name] = pool_by_lsnr_port[p.protocol, - p.port] - except KeyError: - continue - current_targets = {(str(m.ip), m.port, m.pool_id) - for m in lbaas_state.members} - - for subset in endpoints.get('subsets', []): - subset_ports = subset.get('ports', []) - for subset_address in subset.get('addresses', []): - try: - target_ip = subset_address['ip'] - target_ref = subset_address['targetRef'] - if target_ref['kind'] != k_const.K8S_OBJ_POD: - continue - except KeyError: - continue - if not pool_by_tgt_name: - continue - for subset_port in subset_ports: - target_port = subset_port['port'] - port_name = subset_port.get('name') - try: - pool = pool_by_tgt_name[port_name] - except KeyError: - LOG.debug("No pool found for port: %r", port_name) - continue - - if (target_ip, target_port, pool.id) in current_targets: - continue - # TODO(apuimedo): Do not pass subnet_id at all when in - # L3 mode once old neutron-lbaasv2 is not supported, as - # octavia does not require it - if (config.CONF.octavia_defaults.member_mode == - k_const.OCTAVIA_L2_MEMBER_MODE): - try: - member_subnet_id = self._get_pod_subnet(target_ref, - target_ip) - except k_exc.K8sResourceNotFound: - LOG.debug("Member namespace has been deleted. No " - "need to add the members as it is " - "going to be deleted") - continue - else: - # We use the service subnet id so that the connectivity - # from VIP to pods happens in layer 3 mode, i.e., - # routed. - member_subnet_id = lbaas_state.loadbalancer.subnet_id - first_member_of_the_pool = True - for member in lbaas_state.members: - if pool.id == member.pool_id: - first_member_of_the_pool = False - break - if first_member_of_the_pool: - listener_port = lsnr_by_id[pool.listener_id].port - else: - listener_port = None - - member = self._drv_lbaas.ensure_member( - loadbalancer=lbaas_state.loadbalancer, - pool=pool, - subnet_id=member_subnet_id, - ip=target_ip, - port=target_port, - target_ref_namespace=target_ref['namespace'], - target_ref_name=target_ref['name'], - listener_port=listener_port) - lbaas_state.members.append(member) - changed = True - - return changed - - def _get_pod_subnet(self, target_ref, ip): - # REVISIT(ivc): consider using true pod object instead - pod = {'kind': target_ref['kind'], - 'metadata': {'name': target_ref['name'], - 'namespace': target_ref['namespace']}} - project_id = self._drv_pod_project.get_project(pod) - subnets_map = self._drv_pod_subnets.get_subnets(pod, project_id) - subnet_ids = [subnet_id for subnet_id, network in subnets_map.items() - for subnet in network.subnets.objects - if ip in subnet.cidr] - if subnet_ids: - return subnet_ids[0] - else: - # NOTE(ltomasbo): We are assuming that if ip is not on the - # pod subnet is because the member is using hostnetworking. In - # this worker_nodes_subnet will be used - return config.CONF.pod_vif_nested.worker_nodes_subnet - - def _get_port_in_pool(self, pool, lbaas_state, lbaas_spec): - for listener in lbaas_state.listeners: - if listener.id != pool.listener_id: - continue - for port in lbaas_spec.ports: - if (listener.port == port.port and - listener.protocol == port.protocol): - return port - return None + def _create_crd_spec(self, endpoints): + endpoints_name = endpoints['metadata']['name'] + namespace = endpoints['metadata']['namespace'] + kubernetes = clients.get_kubernetes_client() + + subsets = endpoints.get('subsets', []) + + loadbalancer_crd = { + 'apiVersion': 'openstack.org/v1', + 'kind': 'KuryrLoadBalancer', + 'metadata': { + 'name': endpoints_name, + 'finalizers': [k_const.KURYRLB_FINALIZER] + }, + 'spec': { + 'subsets': subsets + }, + 'status': { + } + } - def _remove_unused_members(self, endpoints, lbaas_state, lbaas_spec): - spec_ports = {} - for pool in lbaas_state.pools: - port = self._get_port_in_pool(pool, lbaas_state, lbaas_spec) - if port: - spec_ports[port.name] = pool.id - - current_targets = {(a['ip'], a.get('targetRef', {}).get('name', ''), - p['port'], spec_ports.get(p.get('name'))) - for s in endpoints['subsets'] - for a in s['addresses'] - for p in s['ports'] - if p.get('name') in spec_ports} - - removed_ids = set() - for member in lbaas_state.members: - try: - member_name = member.name - # NOTE: The member name is compose of: - # NAMESPACE_NAME/POD_NAME:PROTOCOL_PORT - pod_name = member_name.split('/')[1].split(':')[0] - except AttributeError: - pod_name = "" - if ((str(member.ip), pod_name, member.port, member.pool_id) in - current_targets): - continue - self._drv_lbaas.release_member(lbaas_state.loadbalancer, - member) - removed_ids.add(member.id) - - if removed_ids: - lbaas_state.members = [m for m in lbaas_state.members - if m.id not in removed_ids] - return bool(removed_ids) - - def _sync_lbaas_pools(self, endpoints, lbaas_state, lbaas_spec): - changed = False - - if self._remove_unused_pools(lbaas_state, lbaas_spec): - changed = True - - if self._sync_lbaas_listeners(endpoints, lbaas_state, lbaas_spec): - changed = True - - if self._add_new_pools(lbaas_state, lbaas_spec): - changed = True - - return changed - - def _add_new_pools(self, lbaas_state, lbaas_spec): - changed = False - - current_listeners_ids = {pool.listener_id - for pool in lbaas_state.pools} - for listener in lbaas_state.listeners: - if listener.id in current_listeners_ids: - continue - pool = self._drv_lbaas.ensure_pool(lbaas_state.loadbalancer, - listener) - lbaas_state.pools.append(pool) - changed = True - - return changed - - def _is_pool_in_spec(self, pool, lbaas_state, lbaas_spec): - # NOTE(yboaron): in order to check if a specific pool is in lbaas_spec - # we should: - # 1. get the listener that pool is attached to - # 2. check if listener's attributes appear in lbaas_spec. - for listener in lbaas_state.listeners: - if listener.id != pool.listener_id: - continue - for port in lbaas_spec.ports: - if (listener.port == port.port and - listener.protocol == port.protocol): - return True - return False - - def _remove_unused_pools(self, lbaas_state, lbaas_spec): - removed_ids = set() - for pool in lbaas_state.pools: - if self._is_pool_in_spec(pool, lbaas_state, lbaas_spec): - continue - self._drv_lbaas.release_pool(lbaas_state.loadbalancer, - pool) - removed_ids.add(pool.id) - if removed_ids: - lbaas_state.pools = [p for p in lbaas_state.pools - if p.id not in removed_ids] - lbaas_state.members = [m for m in lbaas_state.members - if m.pool_id not in removed_ids] - return bool(removed_ids) - - def _sync_lbaas_listeners(self, endpoints, lbaas_state, lbaas_spec): - changed = False - - if self._remove_unused_listeners(endpoints, lbaas_state, lbaas_spec): - changed = True - - if self._sync_lbaas_loadbalancer(endpoints, lbaas_state, lbaas_spec): - changed = True - - if self._add_new_listeners(endpoints, lbaas_spec, lbaas_state): - changed = True - - return changed - - def _add_new_listeners(self, endpoints, lbaas_spec, lbaas_state): - changed = False - lbaas_spec_ports = sorted(lbaas_spec.ports, key=lambda x: x.protocol) - for port_spec in lbaas_spec_ports: - protocol = port_spec.protocol - port = port_spec.port - name = "%s:%s" % (lbaas_state.loadbalancer.name, protocol) - listener = [listener for listener in lbaas_state.listeners - if listener.port == port and - listener.protocol == protocol] - if listener: - continue - # FIXME (maysams): Due to a bug in Octavia, which does - # not allows listeners with same port but different - # protocols to co-exist, we need to skip the creation of - # listeners that have the same port as an existing one. - listener = [listener for listener in lbaas_state.listeners if - listener.port == port] - if listener and not self._drv_lbaas.double_listeners_supported(): - LOG.warning("Skipping listener creation for %s as another one" - " already exists with port %s", name, port) - continue - listener = self._drv_lbaas.ensure_listener( - loadbalancer=lbaas_state.loadbalancer, - protocol=protocol, - port=port, - service_type=lbaas_spec.type) - if listener is not None: - lbaas_state.listeners.append(listener) - changed = True - return changed - - def _remove_unused_listeners(self, endpoints, lbaas_state, lbaas_spec): - current_listeners = {p.listener_id for p in lbaas_state.pools} - - removed_ids = set() - for listener in lbaas_state.listeners: - if listener.id in current_listeners: - continue - self._drv_lbaas.release_listener(lbaas_state.loadbalancer, - listener) - removed_ids.add(listener.id) - if removed_ids: - lbaas_state.listeners = [ - listener for listener in lbaas_state.listeners - if listener.id not in removed_ids] - return bool(removed_ids) - - def _update_lb_status(self, endpoints, lb_ip_address): - status_data = {"loadBalancer": { - "ingress": [{"ip": lb_ip_address.format()}]}} - k8s = clients.get_kubernetes_client() - svc_status_link = self._get_service_link(endpoints) + '/status' try: - k8s.patch("status", svc_status_link, status_data) + kubernetes.post('{}/{}/kuryrloadbalancers'.format( + k_const.K8S_API_CRD_NAMESPACES, namespace), loadbalancer_crd) + except k_exc.K8sConflict: + raise k_exc.ResourceNotReady(loadbalancer_crd) except k_exc.K8sClientException: - # REVISIT(ivc): only raise ResourceNotReady for NotFound - raise k_exc.ResourceNotReady(svc_status_link) + LOG.exception("Kubernetes Client Exception creating " + "kuryrloadbalancer CRD. %s" % + k_exc.K8sClientException) + raise + return loadbalancer_crd + + def _update_crd_spec(self, loadbalancer_crd, endpoints): + kubernetes = clients.get_kubernetes_client() + subsets = endpoints.get('subsets') + lbaas_update_crd = { + 'subsets': subsets + } + try: + kubernetes.patch_crd('spec', loadbalancer_crd['metadata'][ + 'selfLink'], lbaas_update_crd) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', loadbalancer_crd) + except k_exc.K8sConflict: + raise k_exc.ResourceNotReady(loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating KuryrLoadbalancer CRD %s', + loadbalancer_crd) + raise + + return loadbalancer_crd def _get_service_link(self, endpoints): ep_link = endpoints['metadata']['selfLink'] @@ -612,129 +357,6 @@ def _get_service_link(self, endpoints): if link_parts[-2] != 'endpoints': raise k_exc.IntegrityError(_( "Unsupported endpoints link: %(link)s") % { - 'link': ep_link}) + 'link': ep_link}) link_parts[-2] = 'services' return "/".join(link_parts) - - def _sync_lbaas_loadbalancer(self, endpoints, lbaas_state, lbaas_spec): - changed = False - lb = lbaas_state.loadbalancer - - if lb and lb.ip != lbaas_spec.ip: - # if loadbalancerIP was associated to lbaas VIP, disassociate it. - if lbaas_state.service_pub_ip_info: - self._drv_service_pub_ip.disassociate_pub_ip( - lbaas_state.service_pub_ip_info) - - self._drv_lbaas.release_loadbalancer( - loadbalancer=lb) - lb = None - lbaas_state.pools = [] - lbaas_state.listeners = [] - lbaas_state.members = [] - changed = True - - if not lb: - if lbaas_spec.ip: - lb_name = self._drv_lbaas.get_service_loadbalancer_name( - endpoints['metadata']['namespace'], - endpoints['metadata']['name']) - lb = self._drv_lbaas.ensure_loadbalancer( - name=lb_name, - project_id=lbaas_spec.project_id, - subnet_id=lbaas_spec.subnet_id, - ip=lbaas_spec.ip, - security_groups_ids=lbaas_spec.security_groups_ids, - service_type=lbaas_spec.type, - provider=self._lb_provider) - changed = True - elif lbaas_state.service_pub_ip_info: - self._drv_service_pub_ip.release_pub_ip( - lbaas_state.service_pub_ip_info) - lbaas_state.service_pub_ip_info = None - changed = True - - lbaas_state.loadbalancer = lb - return changed - - def _cleanup_leftover_lbaas(self): - lbaas_client = clients.get_loadbalancer_client() - services = [] - try: - services = driver_utils.get_services().get('items') - except k_exc.K8sClientException: - LOG.debug("Skipping cleanup of leftover lbaas. " - "Error retriving Kubernetes services") - return - services_cluster_ip = {service['spec']['clusterIP']: service - for service in services - if service['spec'].get('clusterIP')} - - services_without_selector = set( - service['spec']['clusterIP'] for service in services - if (service['spec'].get('clusterIP') and - not service['spec'].get('selector'))) - lbaas_spec = {} - self._drv_lbaas.add_tags('loadbalancer', lbaas_spec) - loadbalancers = lbaas_client.load_balancers(**lbaas_spec) - for loadbalancer in loadbalancers: - if loadbalancer.vip_address not in services_cluster_ip.keys(): - lb_obj = obj_lbaas.LBaaSLoadBalancer(**loadbalancer) - eventlet.spawn(self._ensure_release_lbaas, lb_obj) - else: - # check if the provider is the right one - if (loadbalancer.vip_address not in services_without_selector - and self._lb_provider - and self._lb_provider != loadbalancer.provider): - LOG.debug("Removing loadbalancer with old provider: %s", - loadbalancer) - lb_obj = obj_lbaas.LBaaSLoadBalancer(**loadbalancer) - eventlet.spawn( - self._ensure_release_lbaas, - lb_obj, - services_cluster_ip[loadbalancer.vip_address]) - # NOTE(ltomasbo): give some extra time in between lbs - # recreation actions - time.sleep(1) - - def _ensure_release_lbaas(self, lb_obj, svc=None): - attempts = 0 - deadline = 0 - retry = True - timeout = config.CONF.kubernetes.watch_retry_timeout - while retry: - try: - if attempts == 1: - deadline = time.time() + timeout - if (attempts > 0 and - utils.exponential_sleep(deadline, attempts) == 0): - LOG.error("Failed releasing lbaas '%s': deadline exceeded", - lb_obj.name) - return - self._drv_lbaas.release_loadbalancer(lb_obj) - retry = False - except k_exc.ResourceNotReady: - LOG.debug("Attempt (%s) of loadbalancer release %s failed." - " A retry will be triggered.", attempts, - lb_obj.name) - attempts += 1 - retry = True - if svc: - endpoints_link = utils.get_endpoints_link(svc) - k8s = clients.get_kubernetes_client() - try: - endpoints = k8s.get(endpoints_link) - except k_exc.K8sResourceNotFound: - LOG.debug("Endpoint not Found.") - return - - lbaas = utils.get_lbaas_state(endpoints) - if lbaas: - lbaas.loadbalancer = None - lbaas.pools = [] - lbaas.listeners = [] - lbaas.members = [] - # NOTE(ltomasbo): give some extra time to ensure the Load - # Balancer VIP is also released - time.sleep(1) - utils.set_lbaas_state(endpoints, lbaas) diff --git a/kuryr_kubernetes/controller/handlers/loadbalancer.py b/kuryr_kubernetes/controller/handlers/loadbalancer.py new file mode 100644 index 000000000..7ec3769ae --- /dev/null +++ b/kuryr_kubernetes/controller/handlers/loadbalancer.py @@ -0,0 +1,810 @@ +# Copyright (c) 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import eventlet +import time + +from oslo_log import log as logging + +from kuryr_kubernetes import clients +from kuryr_kubernetes import config +from kuryr_kubernetes import constants as k_const +from kuryr_kubernetes.controller.drivers import base as drv_base +from kuryr_kubernetes.controller.drivers import utils as driver_utils +from kuryr_kubernetes import exceptions as k_exc +from kuryr_kubernetes.handlers import k8s_base +from kuryr_kubernetes.objects import lbaas as obj_lbaas +from kuryr_kubernetes import utils + +LOG = logging.getLogger(__name__) + +SUPPORTED_SERVICE_TYPES = ('ClusterIP', 'LoadBalancer') + + +class KuryrLoadBalancerHandler(k8s_base.ResourceEventHandler): + """LoadBalancerStatusHandler handles K8s Endpoints events. + + LBStatusHandler handles K8s Endpoints events and tracks changes in + LBaaSServiceSpec to update Neutron LBaaS accordingly and to reflect its' + actual state in LBaaSState. + """ + + OBJECT_KIND = k_const.K8S_OBJ_KURYRLOADBALANCER + OBJECT_WATCH_PATH = k_const.K8S_API_CRD_KURYRLOADBALANCERS + + def __init__(self): + super(KuryrLoadBalancerHandler, self).__init__() + self._drv_lbaas = drv_base.LBaaSDriver.get_instance() + self._drv_pod_project = drv_base.PodProjectDriver.get_instance() + self._drv_pod_subnets = drv_base.PodSubnetsDriver.get_instance() + self._drv_service_pub_ip = drv_base.ServicePubIpDriver.get_instance() + # Note(yboaron) LBaaS driver supports 'provider' parameter in + # Load Balancer creation flow. + # We need to set the requested load balancer provider + # according to 'endpoints_driver_octavia_provider' configuration. + self._lb_provider = None + if (config.CONF.kubernetes.endpoints_driver_octavia_provider + != 'default'): + self._lb_provider = ( + config.CONF.kubernetes.endpoints_driver_octavia_provider) + eventlet.spawn(self._cleanup_leftover_lbaas) + + def on_present(self, loadbalancer_crd): + if self._should_ignore(loadbalancer_crd): + LOG.debug("Ignoring Kubernetes service %s", + loadbalancer_crd['metadata']['name']) + return + + try: + name = loadbalancer_crd['metadata']['name'] + namespace = loadbalancer_crd['metadata']['namespace'] + self._get_loadbalancer_crd(name, namespace) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', + loadbalancer_crd) + except KeyError: + LOG.debug('KuryrLoadbalancer CRD not found') + raise k_exc.ResourceNotReady(loadbalancer_crd) + + if self._sync_lbaas_members(loadbalancer_crd): + # Note(yboaron) For LoadBalancer services, we should allocate FIP, + # associate it to LB VIP and update K8S service status + # if loadbalancer_crd['status'].get('service_pub_ip_info') is None: + lb_ip = loadbalancer_crd['spec'].get('lb_ip') + pub_info = loadbalancer_crd['status'].get( + 'service_pub_ip_info') + if pub_info is None: + service_pub_ip_info = ( + self._drv_service_pub_ip.acquire_service_pub_ip_info( + loadbalancer_crd['spec']['type'], + lb_ip, + loadbalancer_crd['spec']['project_id'], + loadbalancer_crd['status']['loadbalancer'][ + 'port_id'])) + if service_pub_ip_info: + self._drv_service_pub_ip.associate_pub_ip( + service_pub_ip_info, loadbalancer_crd['status'][ + 'loadbalancer']['port_id']) + loadbalancer_crd['status'][ + 'service_pub_ip_info'] = service_pub_ip_info + self._update_lb_status(loadbalancer_crd) + kubernetes = clients.get_kubernetes_client() + try: + kubernetes.patch_crd('status', loadbalancer_crd[ + 'metadata']['selfLink'], loadbalancer_crd[ + 'status']) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', + loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating KuryLoadbalancer CRD %s', + loadbalancer_crd) + raise + + def _should_ignore(self, loadbalancer_crd): + return not(self._has_pods(loadbalancer_crd)) + + def _has_pods(self, loadbalancer_crd): + ep_subsets = loadbalancer_crd['spec'].get('subsets', []) + if not ep_subsets: + return False + return any(True + for subset in ep_subsets + for address in subset.get('addresses', []) + if address['targetRef'].get('kind', []) == 'Pod') + + def on_finalize(self, loadbalancer_crd): + LOG.debug("Deleting the loadbalancer CRD") + + if not loadbalancer_crd: + LOG.warning("Load Balancer CRD not present") + return + + if loadbalancer_crd['status'] != {}: + # NOTE(ivc): deleting pool deletes its members + self._drv_lbaas.release_loadbalancer( + loadbalancer=loadbalancer_crd['status'].get('loadbalancer')) + + try: + pub_info = loadbalancer_crd['status']['service_pub_ip_info'] + except KeyError: + pub_info = None + + if pub_info: + self._drv_service_pub_ip.release_pub_ip( + loadbalancer_crd['status']['service_pub_ip_info']) + + kubernetes = clients.get_kubernetes_client() + LOG.debug('Removing finalizer from KuryrLoadBalancer CRD %s', + loadbalancer_crd) + try: + kubernetes.remove_finalizer(loadbalancer_crd, + k_const.KURYRLB_FINALIZER) + except k_exc.K8sClientException: + LOG.exception('Error removing kuryrloadbalancer CRD finalizer' + 'for %s', loadbalancer_crd) + raise + + namespace = loadbalancer_crd['metadata']['namespace'] + name = loadbalancer_crd['metadata']['name'] + try: + service = kubernetes.get(f"{k_const.K8S_API_NAMESPACES}" + f"/{namespace}/services/{name}") + except k_exc.K8sResourceNotFound as ex: + LOG.exception("Failed to get service: %s", ex) + raise + + LOG.debug('Removing finalizer from service %s', + service["metadata"]["name"]) + try: + kubernetes.remove_finalizer(service, k_const.SERVICE_FINALIZER) + except k_exc.K8sClientException: + LOG.exception('Error removing service finalizer' + 'for %s', service["metadata"]["name"]) + raise + + def _get_loadbalancer_crd(self, loadbalancer_crd_name, namespace): + k8s = clients.get_kubernetes_client() + try: + loadbalancer_crd = k8s.get('{}/{}/kuryrloadbalancers/{}'.format( + k_const.K8S_API_CRD_NAMESPACES, namespace, + loadbalancer_crd_name)) + except k_exc.K8sResourceNotFound: + return None + except k_exc.K8sClientException: + LOG.exception("Kubernetes Client Exception.") + raise + return loadbalancer_crd + + def _sync_lbaas_members(self, loadbalancer_crd): + changed = False + + if (self._has_pods(loadbalancer_crd) and + self._remove_unused_members(loadbalancer_crd)): + changed = True + + if self._sync_lbaas_pools(loadbalancer_crd): + changed = True + + if (self._has_pods(loadbalancer_crd) and + self._add_new_members(loadbalancer_crd)): + changed = True + + return changed + + def _sync_lbaas_sgs(self, loadbalancer_crd): + # NOTE (maysams) Need to retrieve the LBaaS Spec again due to + # the possibility of it being updated after the LBaaS creation + # process has started. + lbaas_spec = loadbalancer_crd.get('spec') + + lb = loadbalancer_crd['status'].get('loadbalancer') + if not lb: + return + + default_sgs = config.CONF.neutron_defaults.pod_security_groups + # NOTE(maysams) As the endpoint and svc are annotated with the + # 'lbaas_spec' in two separate k8s calls, it's possible that + # the endpoint got annotated and the svc haven't due to controller + # restarts. For this case, a resourceNotReady exception is raised + # till the svc gets annotated with a 'lbaas_spec'. + if lbaas_spec: + lbaas_spec_sgs = loadbalancer_crd['spec'].get( + 'security_groups_ids', []) + else: + raise k_exc.ResourceNotReady(lbaas_spec_sgs) + if (lb.get('security_groups') and + lb.get('security_groups') != lbaas_spec_sgs): + sgs = [lb_sg for lb_sg in lb['security_groups'] + if lb_sg not in default_sgs] + if lbaas_spec_sgs != default_sgs: + sgs.extend(lbaas_spec_sgs) + + # Check if this should update the CRD + lb['security_groups'] = sgs + + def _add_new_members(self, loadbalancer_crd): + changed = False + try: + self._sync_lbaas_sgs(loadbalancer_crd) + except k_exc.K8sResourceNotFound: + LOG.debug("The svc has been deleted while processing the endpoints" + " update. No need to add new members.") + + lsnr_by_id = {l['id']: l for l in loadbalancer_crd['status'].get( + 'listeners', [])} + pool_by_lsnr_port = {(lsnr_by_id[p['listener_id']]['protocol'], + lsnr_by_id[p['listener_id']]['port']): p + for p in loadbalancer_crd['status'].get( + 'pools', [])} + + # NOTE(yboaron): Since LBaaSv2 doesn't support UDP load balancing, + # the LBaaS driver will return 'None' in case of UDP port + # listener creation. + # we should consider the case in which + # 'pool_by_lsnr_port[p.protocol, p.port]' is missing + pool_by_tgt_name = {} + for p in loadbalancer_crd['spec'].get('ports', []): + try: + pool_by_tgt_name[p['name']] = pool_by_lsnr_port[p['protocol'], + p['port']] + except KeyError: + continue + + current_targets = {(str(m['ip']), m['port'], m['pool_id']) + for m in loadbalancer_crd['status'].get( + 'members', [])} + + for subset in loadbalancer_crd['spec']['subsets']: + subset_ports = subset.get('ports', []) + for subset_address in subset.get('addresses', []): + try: + target_ip = subset_address['ip'] + target_ref = subset_address['targetRef'] + if target_ref['kind'] != k_const.K8S_OBJ_POD: + continue + except KeyError: + continue + if not pool_by_tgt_name: + continue + for subset_port in subset_ports: + target_port = subset_port['port'] + port_name = subset_port.get('name') + try: + pool = pool_by_tgt_name[port_name] + except KeyError: + LOG.debug("No pool found for port: %r", port_name) + continue + + if (target_ip, target_port, pool['id']) in current_targets: + continue + # TODO(apuimedo): Do not pass subnet_id at all when in + # L3 mode once old neutron-lbaasv2 is not supported, as + # octavia does not require it + if (config.CONF.octavia_defaults.member_mode == + k_const.OCTAVIA_L2_MEMBER_MODE): + try: + member_subnet_id = self._get_pod_subnet(target_ref, + target_ip) + except k_exc.K8sResourceNotFound: + LOG.debug("Member namespace has been deleted. No " + "need to add the members as it is " + "going to be deleted") + continue + else: + # We use the service subnet id so that the connectivity + # from VIP to pods happens in layer 3 mode, i.e., + # routed. + member_subnet_id = loadbalancer_crd['status'][ + 'loadbalancer']['subnet_id'] + first_member_of_the_pool = True + for member in loadbalancer_crd['status'].get( + 'members', []): + if pool['id'] == member['pool_id']: + first_member_of_the_pool = False + break + if first_member_of_the_pool: + listener_port = lsnr_by_id[pool['listener_id']][ + 'port'] + else: + listener_port = None + loadbalancer = loadbalancer_crd['status']['loadbalancer'] + member = self._drv_lbaas.ensure_member( + loadbalancer=loadbalancer, + pool=pool, + subnet_id=member_subnet_id, + ip=target_ip, + port=target_port, + target_ref_namespace=target_ref['namespace'], + target_ref_name=target_ref['name'], + listener_port=listener_port) + members = loadbalancer_crd['status'].get('members', []) + if members: + loadbalancer_crd['status'].get('members', []).append( + member) + else: + loadbalancer_crd['status']['members'] = [] + loadbalancer_crd['status'].get('members', []).append( + member) + kubernetes = clients.get_kubernetes_client() + try: + kubernetes.patch_crd('status', loadbalancer_crd[ + 'metadata']['selfLink'], loadbalancer_crd[ + 'status']) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', + loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating KuryLoadbalancer CRD %s', + loadbalancer_crd) + raise + changed = True + return changed + + def _get_pod_subnet(self, target_ref, ip): + # REVISIT(ivc): consider using true pod object instead + pod = {'kind': target_ref['kind'], + 'metadata': {'name': target_ref['name'], + 'namespace': target_ref['namespace']}} + project_id = self._drv_pod_project.get_project(pod) + subnets_map = self._drv_pod_subnets.get_subnets(pod, project_id) + subnet_ids = [subnet_id for subnet_id, network in subnets_map.items() + for subnet in network.subnets.objects + if ip in subnet.cidr] + if subnet_ids: + return subnet_ids[0] + else: + # NOTE(ltomasbo): We are assuming that if ip is not on the + # pod subnet is because the member is using hostnetworking. In + # this worker_nodes_subnet will be used + return config.CONF.pod_vif_nested.worker_nodes_subnet + + def _get_port_in_pool(self, pool, loadbalancer_crd): + + for l in loadbalancer_crd['status']['listeners']: + if l['id'] != pool['listener_id']: + continue + for port in loadbalancer_crd['spec'].get('ports', []): + if l.get('port') == port.get( + 'port') and l.get('protocol') == port.get('protocol'): + return port + return None + + def _remove_unused_members(self, loadbalancer_crd): + spec_ports = {} + pools = loadbalancer_crd['status'].get('pools', []) + for pool in pools: + port = self._get_port_in_pool(pool, loadbalancer_crd) + if port: + if not port.get('name'): + port['name'] = None + spec_ports[port['name']] = pool['id'] + + subsets = loadbalancer_crd['spec'].get('subsets') + current_targets = {(a['ip'], a.get('targetRef', {}).get('name', ''), + p['port'], spec_ports.get(p.get('name'))) + for s in subsets + for a in s['addresses'] + for p in s['ports'] + if p.get('name') in spec_ports} + + removed_ids = set() + + for member in loadbalancer_crd['status'].get('members', []): + try: + member_name = member['name'] + # NOTE: The member name is compose of: + # NAMESPACE_NAME/POD_NAME:PROTOCOL_PORT + pod_name = member_name.split('/')[1].split(':')[0] + except AttributeError: + pod_name = "" + if ((str(member['ip']), pod_name, member['port'], member[ + 'pool_id']) in current_targets): + continue + + self._drv_lbaas.release_member(loadbalancer_crd['status'][ + 'loadbalancer'], member) + removed_ids.add(member['id']) + + if removed_ids: + loadbalancer_crd['status']['members'] = [m for m in + loadbalancer_crd[ + 'status'][ + 'members'] + if m['id'] not in + removed_ids] + + kubernetes = clients.get_kubernetes_client() + try: + kubernetes.patch_crd('status', loadbalancer_crd[ + 'metadata']['selfLink'], loadbalancer_crd[ + 'status']) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', + loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating KuryLoadbalancer CRD %s', + loadbalancer_crd) + raise + return bool(removed_ids) + + def _sync_lbaas_pools(self, loadbalancer_crd): + changed = False + + if self._remove_unused_pools(loadbalancer_crd): + changed = True + + if self._sync_lbaas_listeners(loadbalancer_crd): + changed = True + + if self._add_new_pools(loadbalancer_crd): + changed = True + + return changed + + def _add_new_pools(self, loadbalancer_crd): + changed = False + + current_listeners_ids = {pool['listener_id'] + for pool in loadbalancer_crd['status'].get( + 'pools', [])} + for listener in loadbalancer_crd['status'].get('listeners', []): + if listener['id'] in current_listeners_ids: + continue + pool = self._drv_lbaas.ensure_pool(loadbalancer_crd['status'][ + 'loadbalancer'], listener) + pools = loadbalancer_crd['status'].get('pools', []) + if pools: + loadbalancer_crd['status'].get('pools', []).append( + pool) + else: + loadbalancer_crd['status']['pools'] = [] + loadbalancer_crd['status'].get('pools', []).append( + pool) + kubernetes = clients.get_kubernetes_client() + try: + kubernetes.patch_crd('status', loadbalancer_crd['metadata'][ + 'selfLink'], loadbalancer_crd['status']) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', + loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating KuryrLoadbalancer CRD %s', + loadbalancer_crd) + raise + changed = True + return changed + + def _is_pool_in_spec(self, pool, loadbalancer_crd): + # NOTE(yboaron): in order to check if a specific pool is in lbaas_spec + # we should: + # 1. get the listener that pool is attached to + # 2. check if listener's attributes appear in lbaas_spec. + for l in loadbalancer_crd['status']['listeners']: + if l['id'] != pool['listener_id']: + continue + for port in loadbalancer_crd['spec'].get('ports'): + if l['port'] == port['port'] and l['protocol'] == port[ + 'protocol']: + return True + return False + + def _remove_unused_pools(self, loadbalancer_crd): + removed_ids = set() + + for pool in loadbalancer_crd['status'].get('pools', []): + if self._is_pool_in_spec(pool, loadbalancer_crd): + continue + self._drv_lbaas.release_pool(loadbalancer_crd['status'][ + 'loadbalancer'], pool) + removed_ids.add(pool['id']) + if removed_ids: + loadbalancer_crd['status']['pools'] = [p for p in loadbalancer_crd[ + 'status']['pools'] if p['id'] not in removed_ids] + loadbalancer_crd['status']['members'] = [m for m in + loadbalancer_crd[ + 'status']['members'] + if m['pool_id'] not in + removed_ids] + + kubernetes = clients.get_kubernetes_client() + try: + kubernetes.patch_crd('status', loadbalancer_crd[ + 'metadata']['selfLink'], loadbalancer_crd[ + 'status']) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', + loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating KuryLoadbalancer CRD %s', + loadbalancer_crd) + raise + return bool(removed_ids) + + def _sync_lbaas_listeners(self, loadbalancer_crd): + changed = False + + if self._remove_unused_listeners(loadbalancer_crd): + changed = True + + if self._sync_lbaas_loadbalancer(loadbalancer_crd): + changed = True + + if self._add_new_listeners(loadbalancer_crd): + changed = True + + return changed + + def _add_new_listeners(self, loadbalancer_crd): + changed = False + lb_crd_spec_ports = loadbalancer_crd['spec'].get('ports') + if not lb_crd_spec_ports: + return changed + lbaas_spec_ports = sorted(lb_crd_spec_ports, + key=lambda x: x['protocol']) + for port_spec in lbaas_spec_ports: + protocol = port_spec['protocol'] + port = port_spec['port'] + name = "%s:%s" % (loadbalancer_crd['status']['loadbalancer'][ + 'name'], protocol) + + listener = [l for l in loadbalancer_crd['status'].get( + 'listeners', []) if l['port'] == port and l[ + 'protocol'] == protocol] + + if listener: + continue + # FIXME (maysams): Due to a bug in Octavia, which does + # not allows listeners with same port but different + # protocols to co-exist, we need to skip the creation of + # listeners that have the same port as an existing one. + listener = [l for l in loadbalancer_crd['status'].get( + 'listeners', []) if l['port'] == port] + + if listener and not self._drv_lbaas.double_listeners_supported(): + LOG.warning("Skipping listener creation for %s as another one" + " already exists with port %s", name, port) + continue + listener = self._drv_lbaas.ensure_listener( + loadbalancer=loadbalancer_crd['status'].get('loadbalancer'), + protocol=protocol, + port=port, + service_type=loadbalancer_crd['spec'].get('type')) + if listener is not None: + listeners = loadbalancer_crd['status'].get('listeners', []) + if listeners: + listeners.append(listener) + else: + loadbalancer_crd['status']['listeners'] = [] + loadbalancer_crd['status'].get('listeners', []).append( + listener) + + kubernetes = clients.get_kubernetes_client() + try: + kubernetes.patch_crd('status', loadbalancer_crd[ + 'metadata']['selfLink'], loadbalancer_crd['status']) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', + loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating KuryrLoadbalancer CRD %s', + loadbalancer_crd) + raise + changed = True + return changed + + def _remove_unused_listeners(self, loadbalancer_crd): + current_listeners = {p['listener_id'] for p in loadbalancer_crd[ + 'status'].get('pools', [])} + removed_ids = set() + for listener in loadbalancer_crd['status'].get('listeners', []): + if listener['id'] in current_listeners: + continue + self._drv_lbaas.release_listener(loadbalancer_crd['status'][ + 'loadbalancer'], listener) + removed_ids.add(listener['id']) + if removed_ids: + loadbalancer_crd['status']['listeners'] = [ + l for l in loadbalancer_crd['status'].get('listeners', + []) if l['id'] + not in removed_ids] + + kubernetes = clients.get_kubernetes_client() + try: + kubernetes.patch_crd('status', loadbalancer_crd[ + 'metadata']['selfLink'], loadbalancer_crd[ + 'status']) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', + loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating KuryLoadbalancer CRD %s', + loadbalancer_crd) + raise + return bool(removed_ids) + + def _update_lb_status(self, lb_crd): + lb_crd_status = lb_crd['status'] + lb_ip_address = lb_crd_status['service_pub_ip_info']['ip_addr'] + name = lb_crd['metadata']['name'] + ns = lb_crd['metadata']['namespace'] + status_data = {"loadBalancer": { + "ingress": [{"ip": lb_ip_address.format()}]}} + k8s = clients.get_kubernetes_client() + try: + k8s.patch("status", f"{k_const.K8S_API_NAMESPACES}" + f"/{ns}/services/{name}/status", + status_data) + except k_exc.K8sConflict: + raise k_exc.ResourceNotReady(name) + except k_exc.K8sClientException: + LOG.exception("Kubernetes Client Exception" + "when updating the svc status %s" + % name) + raise + + def _sync_lbaas_loadbalancer(self, loadbalancer_crd): + changed = False + lb = loadbalancer_crd['status'].get('loadbalancer') + + if lb and lb['ip'] != loadbalancer_crd['spec'].get('ip'): + # if loadbalancerIP was associated to lbaas VIP, disassociate it. + + try: + pub_info = loadbalancer_crd['status']['service_pub_ip_info'] + except KeyError: + pub_info = None + + if pub_info: + self._drv_service_pub_ip.disassociate_pub_ip( + loadbalancer_crd['status']['service_pub_ip_info']) + + self._drv_lbaas.release_loadbalancer( + loadbalancer=lb) + lb = None + loadbalancer_crd['status']['pools'] = [] + loadbalancer_crd['status']['listeners'] = [] + loadbalancer_crd['status']['members'] = [] + + kubernetes = clients.get_kubernetes_client() + try: + kubernetes.patch_crd('status', loadbalancer_crd['metadata'][ + 'selfLink'], loadbalancer_crd['status']) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', + loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating KuryrLoadbalancer CRD %s', + loadbalancer_crd) + raise + changed = True + + if not lb: + if loadbalancer_crd['spec'].get('ip'): + lb_name = self._drv_lbaas.get_service_loadbalancer_name( + loadbalancer_crd['metadata']['namespace'], + loadbalancer_crd['metadata']['name']) + lb = self._drv_lbaas.ensure_loadbalancer( + name=lb_name, + project_id=loadbalancer_crd['spec'].get('project_id'), + subnet_id=loadbalancer_crd['spec'].get('subnet_id'), + ip=loadbalancer_crd['spec'].get('ip'), + security_groups_ids=loadbalancer_crd['spec'].get( + 'security_groups_ids'), + service_type=loadbalancer_crd['spec'].get('type'), + provider=self._lb_provider) + loadbalancer_crd['status']['loadbalancer'] = lb + changed = True + elif loadbalancer_crd['status'].get('service_pub_ip_info'): + self._drv_service_pub_ip.release_pub_ip( + loadbalancer_crd['status']['service_pub_ip_info']) + loadbalancer_crd['status']['service_pub_ip_info'] = None + changed = True + + kubernetes = clients.get_kubernetes_client() + try: + kubernetes.patch_crd('status', loadbalancer_crd['metadata'][ + 'selfLink'], loadbalancer_crd['status']) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadbalancer CRD not found %s', + loadbalancer_crd) + except k_exc.K8sClientException: + LOG.exception('Error updating KuryrLoadbalancer CRD %s', + loadbalancer_crd) + raise + + return changed + + def _cleanup_leftover_lbaas(self): + lbaas_client = clients.get_loadbalancer_client() + services = [] + try: + services = driver_utils.get_services().get('items') + except k_exc.K8sClientException: + LOG.debug("Skipping cleanup of leftover lbaas. " + "Error retriving Kubernetes services") + return + services_cluster_ip = {service['spec']['clusterIP']: service + for service in services + if service['spec'].get('clusterIP')} + + services_without_selector = set( + service['spec']['clusterIP'] for service in services + if (service['spec'].get('clusterIP') and + not service['spec'].get('selector'))) + lbaas_spec = {} + self._drv_lbaas.add_tags('loadbalancer', lbaas_spec) + loadbalancers = lbaas_client.load_balancers(**lbaas_spec) + for loadbalancer in loadbalancers: + if loadbalancer.vip_address not in services_cluster_ip.keys(): + lb_obj = obj_lbaas.LBaaSLoadBalancer(**loadbalancer) + eventlet.spawn(self._ensure_release_lbaas, lb_obj) + else: + # check if the provider is the right one + if (loadbalancer.vip_address not in services_without_selector + and self._lb_provider + and self._lb_provider != loadbalancer.provider): + LOG.debug("Removing loadbalancer with old provider: %s", + loadbalancer) + lb_obj = obj_lbaas.LBaaSLoadBalancer(**loadbalancer) + eventlet.spawn( + self._ensure_release_lbaas, + lb_obj, + services_cluster_ip[loadbalancer.vip_address]) + # NOTE(ltomasbo): give some extra time in between lbs + # recreation actions + time.sleep(1) + + def _ensure_release_lbaas(self, lb_obj, svc=None): + attempts = 0 + deadline = 0 + retry = True + timeout = config.CONF.kubernetes.watch_retry_timeout + while retry: + try: + if attempts == 1: + deadline = time.time() + timeout + if (attempts > 0 and + utils.exponential_sleep(deadline, attempts) == 0): + LOG.error("Failed releasing lbaas '%s': deadline exceeded", + lb_obj.name) + return + self._drv_lbaas.release_loadbalancer(lb_obj) + retry = False + except k_exc.ResourceNotReady: + LOG.debug("Attempt (%s) of loadbalancer release %s failed." + " A retry will be triggered.", attempts, + lb_obj.name) + attempts += 1 + retry = True + if svc: + endpoints_link = utils.get_endpoints_link(svc) + k8s = clients.get_kubernetes_client() + try: + endpoints = k8s.get(endpoints_link) + except k_exc.K8sResourceNotFound: + LOG.debug("Endpoint not Found.") + return + + lbaas = utils.get_lbaas_state(endpoints) + if lbaas: + lbaas.loadbalancer = None + lbaas.pools = [] + lbaas.listeners = [] + lbaas.members = [] + # NOTE(ltomasbo): give some extra time to ensure the Load + # Balancer VIP is also released + time.sleep(1) + utils.set_lbaas_state(endpoints, lbaas) diff --git a/kuryr_kubernetes/k8s_client.py b/kuryr_kubernetes/k8s_client.py index 9c016515b..3ea0a6cb0 100644 --- a/kuryr_kubernetes/k8s_client.py +++ b/kuryr_kubernetes/k8s_client.py @@ -293,6 +293,21 @@ def remove_finalizer(self, obj, finalizer): # If after 3 iterations there's still conflict, just raise. self._raise_from_response(response) + def get_loadbalancer_crd(self, obj): + name = obj['metadata']['name'] + namespace = obj['metadata']['namespace'] + + try: + crd = self.get('{}/{}/kuryrloadbalancers/{}'.format( + constants.K8S_API_CRD_NAMESPACES, namespace, + name)) + except exc.K8sResourceNotFound: + return None + except exc.K8sClientException: + LOG.exception("Kubernetes Client Exception.") + raise + return crd + def annotate(self, path, annotations, resource_version=None): """Pushes a resource annotation to the K8s API resource diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_lb_public_ip.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_lb_public_ip.py index 38e6861c9..e64b25a16 100644 --- a/kuryr_kubernetes/tests/unit/controller/drivers/test_lb_public_ip.py +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_lb_public_ip.py @@ -53,10 +53,11 @@ def test_acquire_service_pub_ip_info_usr_specified_ip(self): spec_type = 'LoadBalancer' spec_lb_ip = '1.2.3.4' - expected_resp = (obj_lbaas - .LBaaSPubIp(ip_id=fip.id, - ip_addr=fip.floating_ip_address, - alloc_method='user')) + expected_resp = { + 'ip_id': fip.id, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'user' + } result = cls.acquire_service_pub_ip_info(m_driver, spec_type, spec_lb_ip, project_id) @@ -134,9 +135,11 @@ def test_acquire_service_pub_ip_info_pool_subnet_is_none(self, m_cfg): spec_type = 'LoadBalancer' spec_lb_ip = None - expected_resp = obj_lbaas.LBaaSPubIp(ip_id=fip.id, - ip_addr=fip.floating_ip_address, - alloc_method='pool') + expected_resp = { + 'ip_id': fip.id, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'pool' + } result = cls.acquire_service_pub_ip_info(m_driver, spec_type, spec_lb_ip, project_id) @@ -161,9 +164,11 @@ def test_acquire_service_pub_ip_info_alloc_from_pool(self, m_cfg): spec_type = 'LoadBalancer' spec_lb_ip = None - expected_resp = obj_lbaas.LBaaSPubIp(ip_id=fip.id, - ip_addr=fip.floating_ip_address, - alloc_method='pool') + expected_resp = { + 'ip_id': fip.id, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'pool' + } result = cls.acquire_service_pub_ip_info(m_driver, spec_type, spec_lb_ip, project_id) @@ -184,10 +189,11 @@ def test_release_pub_ip_alloc_method_non_pool(self): fip = munch.Munch({'floating_ip_address': '1.2.3.5', 'id': 'ec29d641-fec4-4f67-928a-124a76b3a888'}) - service_pub_ip_info = (obj_lbaas - .LBaaSPubIp(ip_id=fip.id, - ip_addr=fip.floating_ip_address, - alloc_method='kk')) + service_pub_ip_info = { + 'ip_id': fip.id, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'kk' + } rc = cls.release_pub_ip(m_driver, service_pub_ip_info) self.assertIs(rc, True) @@ -199,10 +205,12 @@ def test_release_pub_ip_alloc_method_user(self): fip = munch.Munch({'floating_ip_address': '1.2.3.5', 'id': 'ec29d641-fec4-4f67-928a-124a76b3a888'}) - service_pub_ip_info = (obj_lbaas - .LBaaSPubIp(ip_id=fip.id, - ip_addr=fip.floating_ip_address, - alloc_method='user')) + service_pub_ip_info = { + 'ip_id': fip.id, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'user' + } + rc = cls.release_pub_ip(m_driver, service_pub_ip_info) self.assertIs(rc, True) @@ -216,10 +224,11 @@ def test_release_pub_ip_alloc_method_pool_neutron_exception(self): fip = munch.Munch({'floating_ip_address': '1.2.3.5', 'id': 'ec29d641-fec4-4f67-928a-124a76b3a888'}) - service_pub_ip_info = (obj_lbaas - .LBaaSPubIp(ip_id=fip.id, - ip_addr=fip.floating_ip_address, - alloc_method='pool')) + service_pub_ip_info = { + 'ip_id': fip.id, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'pool' + } rc = cls.release_pub_ip(m_driver, service_pub_ip_info) self.assertIs(rc, False) @@ -232,10 +241,11 @@ def test_release_pub_ip_alloc_method_pool_neutron_succeeded(self): fip = munch.Munch({'floating_ip_address': '1.2.3.5', 'id': 'ec29d641-fec4-4f67-928a-124a76b3a888'}) - service_pub_ip_info = (obj_lbaas - .LBaaSPubIp(ip_id=fip.id, - ip_addr=fip.floating_ip_address, - alloc_method='pool')) + service_pub_ip_info = { + 'ip_id': fip.id, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'pool' + } rc = cls.release_pub_ip(m_driver, service_pub_ip_info) self.assertIs(rc, True) @@ -265,6 +275,11 @@ def test_associate_lb_fip_id_not_exist(self): .LBaaSPubIp(ip_id=0, ip_addr=fip.floating_ip_address, alloc_method='pool')) + service_pub_ip_info = { + 'ip_id': 0, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'pool' + } vip_port_id = 'ec29d641-fec4-4f67-928a-124a76b3a777' @@ -281,10 +296,12 @@ def test_associate_lb_fip_id_not_exist_neutron_exception(self): fip = munch.Munch({'floating_ip_address': '1.2.3.5', 'id': 'ec29d641-fec4-4f67-928a-124a76b3a888'}) - service_pub_ip_info = (obj_lbaas - .LBaaSPubIp(ip_id=fip.id, - ip_addr=fip.floating_ip_address, - alloc_method='pool')) + + service_pub_ip_info = { + 'ip_id': fip.id, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'pool' + } vip_port_id = 'ec29d641-fec4-4f67-928a-124a76b3a777' self.assertRaises(os_exc.SDKException, cls.associate_pub_ip, @@ -308,10 +325,11 @@ def test_disassociate_pub_ip_fip_id_not_exist(self): os_net.update_floatingip.return_value = None fip = munch.Munch({'floating_ip_address': '1.2.3.5', 'id': 'ec29d641-fec4-4f67-928a-124a76b3a888'}) - service_pub_ip_info = (obj_lbaas - .LBaaSPubIp(ip_id=0, - ip_addr=fip.floating_ip_address, - alloc_method='pool')) + service_pub_ip_info = { + 'ip_id': 0, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'pool' + } result = cls.disassociate_pub_ip(m_driver, service_pub_ip_info) @@ -325,10 +343,12 @@ def test_disassociate_pub_ip_neutron_exception(self): os_net.update_ip.side_effect = os_exc.SDKException fip = munch.Munch({'floating_ip_address': '1.2.3.5', 'id': 'ec29d641-fec4-4f67-928a-124a76b3a888'}) - service_pub_ip_info = (obj_lbaas - .LBaaSPubIp(ip_id=fip.id, - ip_addr=fip.floating_ip_address, - alloc_method='pool')) + + service_pub_ip_info = { + 'ip_id': fip.id, + 'ip_addr': fip.floating_ip_address, + 'alloc_method': 'pool' + } self.assertRaises(os_exc.SDKException, cls.disassociate_pub_ip, m_driver, service_pub_ip_info) diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_lbaasv2.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_lbaasv2.py index a8e1af545..ec2d35cee 100644 --- a/kuryr_kubernetes/tests/unit/controller/drivers/test_lbaasv2.py +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_lbaasv2.py @@ -112,9 +112,11 @@ def test_ensure_loadbalancer(self): os_net = self.useFixture(k_fix.MockNetworkClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - expected_resp = obj_lbaas.LBaaSLoadBalancer( - provider='octavia', port_id='D3FA400A-F543-4B91-9CD3-047AF0CE42E2', - security_groups=[]) + expected_resp = { + 'provide': 'octavia', + 'port_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42E2', + 'security_groups': [] + } project_id = 'TEST_PROJECT' subnet_id = 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1' ip = '1.2.3.4' @@ -129,10 +131,10 @@ def test_ensure_loadbalancer(self): m_driver._find_loadbalancer, mock.ANY) req = m_driver._ensure.call_args[0][2] - self.assertEqual(lb_name, req.name) - self.assertEqual(project_id, req.project_id) - self.assertEqual(subnet_id, req.subnet_id) - self.assertEqual(ip, str(req.ip)) + self.assertEqual(lb_name, req['name']) + self.assertEqual(project_id, req['project_id']) + self.assertEqual(subnet_id, req['subnet_id']) + self.assertEqual(ip, str(req['ip'])) self.assertEqual(expected_resp, resp) os_net.update_port.assert_not_called() @@ -157,29 +159,38 @@ def test_cascade_release_loadbalancer(self): lbaas.lbaas_loadbalancer_path = "boo %s" cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - loadbalancer = mock.Mock() + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'security_groups': [], + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'provider': None + } cls.release_loadbalancer(m_driver, loadbalancer) m_driver._release.assert_called_once_with( loadbalancer, loadbalancer, lbaas.delete_load_balancer, - loadbalancer.id, cascade=True) + loadbalancer['id'], cascade=True) def _test_ensure_listener(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) expected_resp = mock.sentinel.expected_resp - name = 'TEST_NAME' project_id = 'TEST_PROJECT' - subnet_id = 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1' - ip = '1.2.3.4' loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C' protocol = 'TCP' - provider = 'amphora' port = 1234 - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - id=loadbalancer_id, name=name, project_id=project_id, - subnet_id=subnet_id, ip=ip, provider=provider) + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': project_id, + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'provider': 'amphora' + } # TODO(ivc): handle security groups m_driver._ensure_provisioned.return_value = expected_resp @@ -191,28 +202,27 @@ def _test_ensure_listener(self): m_driver._find_listener, d_lbaasv2._LB_STS_POLL_SLOW_INTERVAL) listener = m_driver._ensure_provisioned.call_args[0][1] - self.assertEqual("%s:%s:%s" % (loadbalancer.name, protocol, port), - listener.name) - self.assertEqual(project_id, listener.project_id) - self.assertEqual(loadbalancer_id, listener.loadbalancer_id) - self.assertEqual(protocol, listener.protocol) - self.assertEqual(port, listener.port) + self.assertEqual("%s:%s:%s" % (loadbalancer['name'], protocol, port), + listener['name']) + self.assertEqual(project_id, listener['project_id']) + self.assertEqual(loadbalancer_id, listener['loadbalancer_id']) + self.assertEqual(protocol, listener['protocol']) + self.assertEqual(port, listener['port']) self.assertEqual(expected_resp, resp) def test_ensure_listener_bad_request_exception(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - name = 'TEST_NAME' - project_id = 'TEST_PROJECT' - subnet_id = 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1' - ip = '1.2.3.4' - loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C' port = 1234 protocol = 'TCP' - provider = 'amphora' - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - id=loadbalancer_id, name=name, project_id=project_id, - subnet_id=subnet_id, ip=ip, provider=provider) + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'provider': 'amphora' + } m_driver._ensure_provisioned.side_effect = os_exc.BadRequestException resp = cls.ensure_listener(m_driver, loadbalancer, @@ -227,26 +237,43 @@ def test_release_listener(self): m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) m_driver._get_vip_port.return_value = munch.Munch({ 'security_group_ids': [mock.sentinel.sg_id]}) - loadbalancer = mock.Mock() - listener = mock.Mock() + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'security_groups': [], + 'provider': 'amphora' + } + listener = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'loadbalancer_id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'protocol': 'TCP', + 'port': 1234, + 'id': 'A57B7771-6050-4CA8-A63C-443493EC98AB' + } cls.release_listener(m_driver, loadbalancer, listener) m_driver._release.assert_called_once_with(loadbalancer, listener, lbaas.delete_listener, - listener.id) + listener['id']) def test_ensure_pool(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) expected_resp = mock.sentinel.expected_resp - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C', - project_id='TEST_PROJECT') - listener = obj_lbaas.LBaaSListener( - id='A57B7771-6050-4CA8-A63C-443493EC98AB', - name='TEST_LISTENER_NAME', - protocol='TCP') + loadbalancer = { + 'project_id': 'TEST_PROJECT', + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + } + listener = { + 'id': 'A57B7771-6050-4CA8-A63C-443493EC98AB', + 'name': 'TEST_LISTENER_NAME', + 'protocol': 'TCP', + } m_driver._ensure_provisioned.return_value = expected_resp resp = cls.ensure_pool(m_driver, loadbalancer, listener) @@ -255,10 +282,10 @@ def test_ensure_pool(self): loadbalancer, mock.ANY, m_driver._create_pool, m_driver._find_pool) pool = m_driver._ensure_provisioned.call_args[0][1] - self.assertEqual(listener.name, pool.name) - self.assertEqual(loadbalancer.project_id, pool.project_id) - self.assertEqual(listener.id, pool.listener_id) - self.assertEqual(listener.protocol, pool.protocol) + self.assertEqual(listener['name'], pool['name']) + self.assertEqual(loadbalancer['project_id'], pool['project_id']) + self.assertEqual(listener['id'], pool['listener_id']) + self.assertEqual(listener['protocol'], pool['protocol']) self.assertEqual(expected_resp, resp) def test_release_pool(self): @@ -266,23 +293,34 @@ def test_release_pool(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) loadbalancer = mock.Mock() - pool = mock.Mock() + pool = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'loadbalancer_id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'listener_id': 'A57B7771-6050-4CA8-A63C-443493EC98AB', + 'protocol': 'TCP', + 'id': 'D4F35594-27EB-4F4C-930C-31DD40F53B77' + } cls.release_pool(m_driver, loadbalancer, pool) m_driver._release.assert_called_once_with(loadbalancer, pool, lbaas.delete_pool, - pool.id) + pool['id']) def test_ensure_member(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) expected_resp = mock.sentinel.expected_resp - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C', - project_id='TEST_PROJECT') - pool = obj_lbaas.LBaaSPool(project_id='TEST_PROJECT', - id='D4F35594-27EB-4F4C-930C-31DD40F53B77') + loadbalancer = { + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'project_id': 'TEST_PROJECT' + } + pool = { + 'id': 'D4F35594-27EB-4F4C-930C-31DD40F53B77', + 'project_id': 'TEST_PROJECT' + } + subnet_id = 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1' ip = '1.2.3.4' port = 1234 @@ -299,41 +337,63 @@ def test_ensure_member(self): loadbalancer, mock.ANY, m_driver._create_member, m_driver._find_member) member = m_driver._ensure_provisioned.call_args[0][1] - self.assertEqual("%s/%s:%s" % (namespace, name, port), member.name) - self.assertEqual(pool.project_id, member.project_id) - self.assertEqual(pool.id, member.pool_id) - self.assertEqual(subnet_id, member.subnet_id) - self.assertEqual(ip, str(member.ip)) - self.assertEqual(port, member.port) + self.assertEqual("%s/%s:%s" % (namespace, name, port), member['name']) + self.assertEqual(pool['project_id'], member['project_id']) + self.assertEqual(pool['id'], member['pool_id']) + self.assertEqual(subnet_id, member['subnet_id']) + self.assertEqual(ip, str(member['ip'])) + self.assertEqual(port, member['port']) self.assertEqual(expected_resp, resp) def test_release_member(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - loadbalancer = mock.Mock() - member = mock.Mock() + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'security_groups': [], + 'provider': None + } + + member = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'pool_id': 'D4F35594-27EB-4F4C-930C-31DD40F53B77', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'port': 1234, + 'id': '3A70CEC0-392D-4BC1-A27C-06E63A0FD54F' + } cls.release_member(m_driver, loadbalancer, member) m_driver._release.assert_called_once_with(loadbalancer, member, lbaas.delete_member, - member.id, member.pool_id) + member['id'], + member['pool_id']) def test_create_loadbalancer(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) lbaas = self.useFixture(k_fix.MockLBaaSClient()).client - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4', - subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1', - security_groups=[]) + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'security_groups': [], + 'provider': None + } + loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C' req = { - 'name': loadbalancer.name, - 'project_id': loadbalancer.project_id, - 'vip_address': str(loadbalancer.ip), - 'vip_subnet_id': loadbalancer.subnet_id, + 'name': loadbalancer['name'], + 'project_id': loadbalancer['project_id'], + 'vip_address': str(loadbalancer['ip']), + 'vip_subnet_id': loadbalancer['subnet_id'], } resp = o_lb.LoadBalancer(id=loadbalancer_id, provider='haproxy') lbaas.create_load_balancer.return_value = resp @@ -342,27 +402,28 @@ def test_create_loadbalancer(self): ret = cls._create_loadbalancer(m_driver, loadbalancer) lbaas.create_load_balancer.assert_called_once_with(**req) - for attr in loadbalancer.obj_fields: - self.assertEqual(getattr(loadbalancer, attr), - getattr(ret, attr)) - self.assertEqual(loadbalancer_id, ret.id) + self.assertEqual(loadbalancer, ret) + self.assertEqual(loadbalancer_id, ret['id']) def test_create_loadbalancer_provider_defined(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) lbaas = self.useFixture(k_fix.MockLBaaSClient()).client - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4', - subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1', - security_groups=[], - provider='amphora') + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'security_groups': [], + 'provider': 'amphora' + } loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C' req = { - 'name': loadbalancer.name, - 'project_id': loadbalancer.project_id, - 'vip_address': str(loadbalancer.ip), - 'vip_subnet_id': loadbalancer.subnet_id, - 'provider': loadbalancer.provider, + 'name': loadbalancer['name'], + 'project_id': loadbalancer['project_id'], + 'vip_address': str(loadbalancer['ip']), + 'vip_subnet_id': loadbalancer['subnet_id'], + 'provider': loadbalancer['provider'], } resp = o_lb.LoadBalancer(id=loadbalancer_id, provider='amphora') lbaas.create_load_balancer.return_value = resp @@ -371,27 +432,28 @@ def test_create_loadbalancer_provider_defined(self): ret = cls._create_loadbalancer(m_driver, loadbalancer) lbaas.create_load_balancer.assert_called_once_with(**req) - for attr in loadbalancer.obj_fields: - self.assertEqual(getattr(loadbalancer, attr), - getattr(ret, attr)) - self.assertEqual(loadbalancer_id, ret.id) + self.assertEqual(loadbalancer, ret) + self.assertEqual(loadbalancer_id, ret['id']) def test_create_loadbalancer_provider_mismatch(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) lbaas = self.useFixture(k_fix.MockLBaaSClient()).client - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4', - subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1', - security_groups=[], - provider='amphora') + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'security_groups': [], + 'provider': 'amphora' + } loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C' req = { - 'name': loadbalancer.name, - 'project_id': loadbalancer.project_id, - 'vip_address': str(loadbalancer.ip), - 'vip_subnet_id': loadbalancer.subnet_id, - 'provider': loadbalancer.provider, + 'name': loadbalancer['name'], + 'project_id': loadbalancer['project_id'], + 'vip_address': str(loadbalancer['ip']), + 'vip_subnet_id': loadbalancer['subnet_id'], + 'provider': loadbalancer['provider'], } resp = o_lb.LoadBalancer(id=loadbalancer_id, provider='haproxy') lbaas.create_load_balancer.return_value = resp @@ -406,10 +468,14 @@ def test_find_loadbalancer(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4', - subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1', - provider='haproxy', security_groups=[]) + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'security_groups': [], + 'provider': 'haproxy' + } loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C' resp = iter([o_lb.LoadBalancer(id=loadbalancer_id, provider='haproxy', provisioning_status='ACTIVE')]) @@ -419,15 +485,13 @@ def test_find_loadbalancer(self): ret = cls._find_loadbalancer(m_driver, loadbalancer) lbaas.load_balancers.assert_called_once_with( - name=loadbalancer.name, - project_id=loadbalancer.project_id, - vip_address=str(loadbalancer.ip), - vip_subnet_id=loadbalancer.subnet_id, + name=loadbalancer['name'], + project_id=loadbalancer['project_id'], + vip_address=str(loadbalancer['ip']), + vip_subnet_id=loadbalancer['subnet_id'], provider='haproxy') - for attr in loadbalancer.obj_fields: - self.assertEqual(getattr(loadbalancer, attr), - getattr(ret, attr)) - self.assertEqual(loadbalancer_id, ret.id) + self.assertEqual(loadbalancer, ret) + self.assertEqual(loadbalancer_id, ret['id']) m_driver.release_loadbalancer.assert_not_called() def test_find_loadbalancer_not_found(self): @@ -437,15 +501,22 @@ def test_find_loadbalancer_not_found(self): loadbalancer = obj_lbaas.LBaaSLoadBalancer( name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4', subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1') + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'provider': None + } resp = iter([]) lbaas.load_balancers.return_value = resp ret = cls._find_loadbalancer(m_driver, loadbalancer) lbaas.load_balancers.assert_called_once_with( - name=loadbalancer.name, - project_id=loadbalancer.project_id, - vip_address=str(loadbalancer.ip), - vip_subnet_id=loadbalancer.subnet_id, + name=loadbalancer['name'], + project_id=loadbalancer['project_id'], + vip_address=str(loadbalancer['ip']), + vip_subnet_id=loadbalancer['subnet_id'], provider=None) self.assertIsNone(ret) m_driver.release_loadbalancer.assert_not_called() @@ -454,9 +525,13 @@ def test_find_loadbalancer_error(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4', - subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1') + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'provider': None + } loadbalancer_id = '00EE9E11-91C2-41CF-8FD4-7970579E5C4C' resp = iter([o_lb.LoadBalancer(id=loadbalancer_id, provider='haproxy', provisioning_status='ERROR')]) @@ -466,10 +541,10 @@ def test_find_loadbalancer_error(self): ret = cls._find_loadbalancer(m_driver, loadbalancer) lbaas.load_balancers.assert_called_once_with( - name=loadbalancer.name, - project_id=loadbalancer.project_id, - vip_address=str(loadbalancer.ip), - vip_subnet_id=loadbalancer.subnet_id, + name=loadbalancer['name'], + project_id=loadbalancer['project_id'], + vip_address=str(loadbalancer['ip']), + vip_subnet_id=loadbalancer['subnet_id'], provider=None) self.assertIsNone(ret) m_driver.release_loadbalancer.assert_called_once() @@ -478,69 +553,80 @@ def test_create_listener(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) lbaas = self.useFixture(k_fix.MockLBaaSClient()).client - listener = obj_lbaas.LBaaSListener( - name='TEST_NAME', project_id='TEST_PROJECT', protocol='TCP', - port=1234, loadbalancer_id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') + listener = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'loadbalancer_id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'protocol': 'TCP', + 'port': 1234 + } listener_id = 'A57B7771-6050-4CA8-A63C-443493EC98AB' + req = { - 'name': listener.name, - 'project_id': listener.project_id, - 'loadbalancer_id': listener.loadbalancer_id, - 'protocol': listener.protocol, - 'protocol_port': listener.port} + 'name': listener['name'], + 'project_id': listener['project_id'], + 'loadbalancer_id': listener['loadbalancer_id'], + 'protocol': listener['protocol'], + 'protocol_port': listener['port']} resp = o_lis.Listener(id=listener_id) lbaas.create_listener.return_value = resp ret = cls._create_listener(m_driver, listener) lbaas.create_listener.assert_called_once_with(**req) - for attr in listener.obj_fields: - self.assertEqual(getattr(listener, attr), - getattr(ret, attr)) - self.assertEqual(listener_id, ret.id) + self.assertEqual(listener, ret) + self.assertEqual(listener_id, ret['id']) def test_find_listener(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') - listener = obj_lbaas.LBaaSListener( - name='TEST_NAME', project_id='TEST_PROJECT', protocol='TCP', - port=1234, loadbalancer_id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') + loadbalancer = { + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + } + listener = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'loadbalancer_id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'protocol': 'TCP', + 'port': 1234 + } listener_id = 'A57B7771-6050-4CA8-A63C-443493EC98AB' lbaas.listeners.return_value = iter([o_lis.Listener(id=listener_id)]) ret = cls._find_listener(m_driver, listener, loadbalancer) lbaas.listeners.assert_called_once_with( - name=listener.name, - project_id=listener.project_id, - load_balancer_id=listener.loadbalancer_id, - protocol=listener.protocol, - protocol_port=listener.port) - for attr in listener.obj_fields: - self.assertEqual(getattr(listener, attr), - getattr(ret, attr)) - self.assertEqual(listener_id, ret.id) + name=listener['name'], + project_id=listener['project_id'], + load_balancer_id=listener['loadbalancer_id'], + protocol=listener['protocol'], + protocol_port=listener['port']) + self.assertEqual(listener, ret) + self.assertEqual(listener_id, ret['id']) def test_find_listener_not_found(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') - listener = obj_lbaas.LBaaSListener( - name='TEST_NAME', project_id='TEST_PROJECT', protocol='TCP', - port=1234, loadbalancer_id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') + loadbalancer = { + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + } + listener = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'loadbalancer_id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'protocol': 'TCP', + 'port': 1234 + } resp = iter([]) lbaas.listeners.return_value = resp ret = cls._find_listener(m_driver, listener, loadbalancer) lbaas.listeners.assert_called_once_with( - name=listener.name, - project_id=listener.project_id, - load_balancer_id=listener.loadbalancer_id, - protocol=listener.protocol, - protocol_port=listener.port) + name=listener['name'], + project_id=listener['project_id'], + load_balancer_id=listener['loadbalancer_id'], + protocol=listener['protocol'], + protocol_port=listener['port']) self.assertIsNone(ret) def test_create_pool(self): @@ -548,44 +634,49 @@ def test_create_pool(self): m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) lbaas = self.useFixture(k_fix.MockLBaaSClient()).client lb_algorithm = 'ROUND_ROBIN' - pool = obj_lbaas.LBaaSPool( - name='TEST_NAME', project_id='TEST_PROJECT', protocol='TCP', - listener_id='A57B7771-6050-4CA8-A63C-443493EC98AB', - loadbalancer_id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') + pool = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'loadbalancer_id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'listener_id': 'A57B7771-6050-4CA8-A63C-443493EC98AB', + 'protocol': 'TCP' + } pool_id = 'D4F35594-27EB-4F4C-930C-31DD40F53B77' + req = { - 'name': pool.name, - 'project_id': pool.project_id, - 'listener_id': pool.listener_id, - 'loadbalancer_id': pool.loadbalancer_id, - 'protocol': pool.protocol, + 'name': pool['name'], + 'project_id': pool['project_id'], + 'listener_id': pool['listener_id'], + 'loadbalancer_id': pool['loadbalancer_id'], + 'protocol': pool['protocol'], 'lb_algorithm': lb_algorithm} resp = o_pool.Pool(id=pool_id) lbaas.create_pool.return_value = resp ret = cls._create_pool(m_driver, pool) lbaas.create_pool.assert_called_once_with(**req) - for attr in pool.obj_fields: - self.assertEqual(getattr(pool, attr), - getattr(ret, attr)) - self.assertEqual(pool_id, ret.id) + self.assertEqual(pool, ret) + self.assertEqual(pool_id, ret['id']) def test_create_pool_with_different_lb_algorithm(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) lbaas = self.useFixture(k_fix.MockLBaaSClient()).client lb_algorithm = 'SOURCE_IP_PORT' - pool = obj_lbaas.LBaaSPool( - name='TEST_NAME', project_id='TEST_PROJECT', protocol='TCP', - listener_id='A57B7771-6050-4CA8-A63C-443493EC98AB', - loadbalancer_id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') + pool = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'loadbalancer_id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'listener_id': 'A57B7771-6050-4CA8-A63C-443493EC98AB', + 'protocol': 'TCP' + } pool_id = 'D4F35594-27EB-4F4C-930C-31DD40F53B77' req = { - 'name': pool.name, - 'project_id': pool.project_id, - 'listener_id': pool.listener_id, - 'loadbalancer_id': pool.loadbalancer_id, - 'protocol': pool.protocol, + 'name': pool['name'], + 'project_id': pool['project_id'], + 'listener_id': pool['listener_id'], + 'loadbalancer_id': pool['loadbalancer_id'], + 'protocol': pool['protocol'], 'lb_algorithm': lb_algorithm} resp = o_pool.Pool(id=pool_id) lbaas.create_pool.return_value = resp @@ -596,26 +687,27 @@ def test_create_pool_with_different_lb_algorithm(self): ret = cls._create_pool(m_driver, pool) lbaas.create_pool.assert_called_once_with(**req) - for attr in pool.obj_fields: - self.assertEqual(getattr(pool, attr), - getattr(ret, attr)) - self.assertEqual(pool_id, ret.id) + self.assertEqual(pool, ret) + self.assertEqual(pool_id, ret['id']) def test_create_pool_conflict(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) lbaas = self.useFixture(k_fix.MockLBaaSClient()).client lb_algorithm = 'ROUND_ROBIN' - pool = obj_lbaas.LBaaSPool( - name='TEST_NAME', project_id='TEST_PROJECT', protocol='TCP', - listener_id='A57B7771-6050-4CA8-A63C-443493EC98AB', - loadbalancer_id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') + pool = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'loadbalancer_id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'listener_id': 'A57B7771-6050-4CA8-A63C-443493EC98AB', + 'protocol': 'TCP' + } req = { - 'name': pool.name, - 'project_id': pool.project_id, - 'listener_id': pool.listener_id, - 'loadbalancer_id': pool.loadbalancer_id, - 'protocol': pool.protocol, + 'name': pool['name'], + 'project_id': pool['project_id'], + 'listener_id': pool['listener_id'], + 'loadbalancer_id': pool['loadbalancer_id'], + 'protocol': pool['protocol'], 'lb_algorithm': lb_algorithm} lbaas.create_pool.side_effect = os_exc.BadRequestException @@ -627,120 +719,134 @@ def test_find_pool_by_listener(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') - pool = obj_lbaas.LBaaSPool( - name='TEST_NAME', project_id='TEST_PROJECT', protocol='TCP', - listener_id='A57B7771-6050-4CA8-A63C-443493EC98AB', - loadbalancer_id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') + loadbalancer = { + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + } + pool = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'loadbalancer_id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'listener_id': 'A57B7771-6050-4CA8-A63C-443493EC98AB', + 'protocol': 'TCP' + } pool_id = 'D4F35594-27EB-4F4C-930C-31DD40F53B77' resp = [o_pool.Pool(id=pool_id, - listeners=[{"id": pool.listener_id}])] + listeners=[{"id": pool['listener_id']}])] lbaas.pools.return_value = resp ret = cls._find_pool(m_driver, pool, loadbalancer) lbaas.pools.assert_called_once_with( - name=pool.name, - project_id=pool.project_id, - loadbalancer_id=pool.loadbalancer_id, - protocol=pool.protocol) - for attr in pool.obj_fields: - self.assertEqual(getattr(pool, attr), - getattr(ret, attr)) - self.assertEqual(pool_id, ret.id) + name=pool['name'], + project_id=pool['project_id'], + loadbalancer_id=pool['loadbalancer_id'], + protocol=pool['protocol']) + self.assertEqual(pool, ret) + self.assertEqual(pool_id, ret['id']) def test_find_pool_by_listener_not_found(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - loadbalancer = obj_lbaas.LBaaSLoadBalancer( - id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') - pool = obj_lbaas.LBaaSPool( - name='TEST_NAME', project_id='TEST_PROJECT', protocol='TCP', - listener_id='A57B7771-6050-4CA8-A63C-443493EC98AB', - loadbalancer_id='00EE9E11-91C2-41CF-8FD4-7970579E5C4C') + loadbalancer = { + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C' + } + pool = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'loadbalancer_id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C', + 'listener_id': 'A57B7771-6050-4CA8-A63C-443493EC98AB', + 'protocol': 'TCP' + } resp = [] lbaas.pools.return_value = resp ret = cls._find_pool(m_driver, pool, loadbalancer) lbaas.pools.assert_called_once_with( - name=pool.name, - project_id=pool.project_id, - loadbalancer_id=pool.loadbalancer_id, - protocol=pool.protocol) + name=pool['name'], + project_id=pool['project_id'], + loadbalancer_id=pool['loadbalancer_id'], + protocol=pool['protocol']) self.assertIsNone(ret) def test_create_member(self): cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) lbaas = self.useFixture(k_fix.MockLBaaSClient()).client - member = obj_lbaas.LBaaSMember( - name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4', - port=1234, subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1', - pool_id='D4F35594-27EB-4F4C-930C-31DD40F53B77') + member = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'pool_id': 'D4F35594-27EB-4F4C-930C-31DD40F53B77', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'port': 1234 + } member_id = '3A70CEC0-392D-4BC1-A27C-06E63A0FD54F' req = { - 'name': member.name, - 'project_id': member.project_id, - 'subnet_id': member.subnet_id, - 'address': str(member.ip), - 'protocol_port': member.port} + 'name': member['name'], + 'project_id': member['project_id'], + 'subnet_id': member['subnet_id'], + 'address': str(member['ip']), + 'protocol_port': member['port']} resp = o_mem.Member(id=member_id) lbaas.create_member.return_value = resp ret = cls._create_member(m_driver, member) - lbaas.create_member.assert_called_once_with(member.pool_id, **req) - for attr in member.obj_fields: - self.assertEqual(getattr(member, attr), - getattr(ret, attr)) - self.assertEqual(member_id, ret.id) + lbaas.create_member.assert_called_once_with(member['pool_id'], **req) + self.assertEqual(member, ret) + self.assertEqual(member_id, ret['id']) def test_find_member(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) loadbalancer = obj_lbaas.LBaaSLoadBalancer() - member = obj_lbaas.LBaaSMember( - name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4', - port=1234, subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1', - pool_id='D4F35594-27EB-4F4C-930C-31DD40F53B77') + member = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'pool_id': 'D4F35594-27EB-4F4C-930C-31DD40F53B77', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'port': 1234 + } member_id = '3A70CEC0-392D-4BC1-A27C-06E63A0FD54F' resp = iter([o_mem.Member(id=member_id)]) lbaas.members.return_value = resp ret = cls._find_member(m_driver, member, loadbalancer) lbaas.members.assert_called_once_with( - member.pool_id, - name=member.name, - project_id=member.project_id, - subnet_id=member.subnet_id, - address=member.ip, - protocol_port=member.port) - for attr in member.obj_fields: - self.assertEqual(getattr(member, attr), - getattr(ret, attr)) - self.assertEqual(member_id, ret.id) + member['pool_id'], + name=member['name'], + project_id=member['project_id'], + subnet_id=member['subnet_id'], + address=member['ip'], + protocol_port=member['port']) + self.assertEqual(member, ret) + self.assertEqual(member_id, ret['id']) def test_find_member_not_found(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) loadbalancer = obj_lbaas.LBaaSLoadBalancer() - member = obj_lbaas.LBaaSMember( - name='TEST_NAME', project_id='TEST_PROJECT', ip='1.2.3.4', - port=1234, subnet_id='D3FA400A-F543-4B91-9CD3-047AF0CE42D1', - pool_id='D4F35594-27EB-4F4C-930C-31DD40F53B77') + member = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'pool_id': 'D4F35594-27EB-4F4C-930C-31DD40F53B77', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'port': 1234 + } resp = iter([]) lbaas.members.return_value = resp ret = cls._find_member(m_driver, member, loadbalancer) lbaas.members.assert_called_once_with( - member.pool_id, - name=member.name, - project_id=member.project_id, - subnet_id=member.subnet_id, - address=member.ip, - protocol_port=member.port) + member['pool_id'], + name=member['name'], + project_id=member['project_id'], + subnet_id=member['subnet_id'], + address=member['ip'], + protocol_port=member['port']) self.assertIsNone(ret) def test_ensure(self): @@ -888,7 +994,14 @@ def test_wait_for_provisioning(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - loadbalancer = mock.Mock() + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'provider': None, + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C' + } timeout = mock.sentinel.timeout timer = [mock.sentinel.t0, mock.sentinel.t1] m_driver._provisioning_timer.return_value = timer @@ -897,13 +1010,20 @@ def test_wait_for_provisioning(self): cls._wait_for_provisioning(m_driver, loadbalancer, timeout) - lbaas.get_load_balancer.assert_called_once_with(loadbalancer.id) + lbaas.get_load_balancer.assert_called_once_with(loadbalancer['id']) def test_wait_for_provisioning_not_ready(self): lbaas = self.useFixture(k_fix.MockLBaaSClient()).client cls = d_lbaasv2.LBaaSv2Driver m_driver = mock.Mock(spec=d_lbaasv2.LBaaSv2Driver) - loadbalancer = mock.Mock() + loadbalancer = { + 'name': 'TEST_NAME', + 'project_id': 'TEST_PROJECT', + 'subnet_id': 'D3FA400A-F543-4B91-9CD3-047AF0CE42D1', + 'ip': '1.2.3.4', + 'provider': None, + 'id': '00EE9E11-91C2-41CF-8FD4-7970579E5C4C' + } timeout = mock.sentinel.timeout timer = [mock.sentinel.t0, mock.sentinel.t1] m_driver._provisioning_timer.return_value = timer diff --git a/kuryr_kubernetes/tests/unit/controller/handlers/test_lbaas.py b/kuryr_kubernetes/tests/unit/controller/handlers/test_lbaas.py index e358b0e29..f04d49ca1 100644 --- a/kuryr_kubernetes/tests/unit/controller/handlers/test_lbaas.py +++ b/kuryr_kubernetes/tests/unit/controller/handlers/test_lbaas.py @@ -13,24 +13,19 @@ # License for the specific language governing permissions and limitations # under the License. -import itertools from unittest import mock -import uuid import os_vif.objects.network as osv_network import os_vif.objects.subnet as osv_subnet -from kuryr_kubernetes import constants as k_const -from kuryr_kubernetes.controller.drivers import base as drv_base from kuryr_kubernetes.controller.handlers import lbaas as h_lbaas -from kuryr_kubernetes import exceptions as k_exc from kuryr_kubernetes.objects import lbaas as obj_lbaas from kuryr_kubernetes.tests import base as test_base _SUPPORTED_LISTENER_PROT = ('HTTP', 'HTTPS', 'TCP') -class TestLBaaSSpecHandler(test_base.TestCase): +class TestServiceHandler(test_base.TestCase): @mock.patch('kuryr_kubernetes.controller.drivers.base' '.ServiceSecurityGroupsDriver.get_instance') @@ -42,105 +37,179 @@ def test_init(self, m_get_drv_project, m_get_drv_subnets, m_get_drv_sg): m_get_drv_project.return_value = mock.sentinel.drv_project m_get_drv_subnets.return_value = mock.sentinel.drv_subnets m_get_drv_sg.return_value = mock.sentinel.drv_sg - handler = h_lbaas.LBaaSSpecHandler() + handler = h_lbaas.ServiceHandler() self.assertEqual(mock.sentinel.drv_project, handler._drv_project) self.assertEqual(mock.sentinel.drv_subnets, handler._drv_subnets) self.assertEqual(mock.sentinel.drv_sg, handler._drv_sg) - @mock.patch('kuryr_kubernetes.utils.set_lbaas_spec') - @mock.patch('kuryr_kubernetes.utils.get_lbaas_spec') - def test_on_present(self, m_get_lbaas_spec, m_set_lbaas_spec): - svc_event = mock.sentinel.svc_event - old_spec = mock.sentinel.old_spec - new_spec = mock.sentinel.new_spec + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + def test_on_present(self, get_k8s_client): + svc_event = { + "metadata": { + "creationTimestamp": "2020-07-25T18:15:12Z", + "finalizers": [ + "openstack.org/service" + ], + "labels": { + "run": "test" + }, + "name": "test", + "namespace": "test", + "resourceVersion": "413753", + "selfLink": "", + "uid": "a026ae48-6141-4029-b743-bac48dae7f06" + }, + "spec": { + "clusterIP": "2.2.2.2", + "ports": [ + { + "port": 1, + "protocol": "TCP", + "targetPort": 1 + } + ], + "selector": { + "run": "test" + }, + "sessionAffinity": "None", + "type": "ClusterIP" + }, + "status": { + "loadBalancer": {} + } + } + + old_spec = { + 'apiVersion': 'openstack.org/v1', + 'kind': 'KuryrLoadBalancer', + 'metadata': { + 'name': 'test', + 'finalizers': [''], + }, + 'spec': { + 'ip': '1.1.1.1' + } + } + new_spec = { + 'apiVersion': 'openstack.org/v1', + 'kind': 'KuryrLoadBalancer', + 'metadata': { + 'name': 'test', + 'finalizers': [''], + }, + 'spec': { + 'ip': '2.2.2.2' + } + } project_id = mock.sentinel.project_id m_drv_project = mock.Mock() m_drv_project.get_project.return_value = project_id - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) - m_get_lbaas_spec.return_value = old_spec + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) m_handler._has_lbaas_spec_changes.return_value = True - m_handler._generate_lbaas_spec.return_value = new_spec + m_handler.create_crd_spec.return_value = new_spec m_handler._should_ignore.return_value = False m_handler._drv_project = m_drv_project - h_lbaas.LBaaSSpecHandler.on_present(m_handler, svc_event) + h_lbaas.ServiceHandler.on_present(m_handler, svc_event) + m_handler.create_crd_spec(svc_event) + m_handler._has_lbaas_spec_changes.return_value = True + m_handler._update_crd_spec(old_spec, svc_event) + + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + def test_on_present_no_changes(self, get_k8s_client): + svc_event = { + "metadata": { + "creationTimestamp": "2020-07-25T18:15:12Z", + "finalizers": [ + "openstack.org/service" + ], + "labels": { + "run": "test" + }, + "name": "test", + "namespace": "test", + "resourceVersion": "413753", + "selfLink": "", + "uid": "a026ae48-6141-4029-b743-bac48dae7f06" + }, + "spec": { + "clusterIP": "2.2.2.2", + "ports": [ + { + "port": 1, + "protocol": "TCP", + "targetPort": 1 + } + ], + "selector": { + "run": "test" + }, + "sessionAffinity": "None", + "type": "ClusterIP" + }, + "status": { + "loadBalancer": {} + } + } - m_get_lbaas_spec.assert_called_once_with(svc_event) - m_handler._has_lbaas_spec_changes.assert_called_once_with(svc_event, - old_spec) - m_handler._generate_lbaas_spec.assert_called_once_with(svc_event) - m_set_lbaas_spec.assert_called_once_with(svc_event, new_spec) + old_spec = { + 'apiVersion': 'openstack.org/v1', + 'kind': 'KuryrLoadBalancer', + 'metadata': { + 'name': 'test', + 'finalizers': [''], + }, + 'spec': { + 'ip': '1.1.1.1' + } + } - @mock.patch('kuryr_kubernetes.utils.set_lbaas_spec') - @mock.patch('kuryr_kubernetes.utils.get_lbaas_spec') - def test_on_present_no_changes(self, m_get_lbaas_spec, - m_set_lbaas_spec): - svc_event = mock.sentinel.svc_event - old_spec = mock.sentinel.old_spec + project_id = mock.sentinel.project_id + m_drv_project = mock.Mock() + m_drv_project.get_project.return_value = project_id - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) - m_get_lbaas_spec.return_value = old_spec - m_handler._has_lbaas_spec_changes.return_value = False + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) + m_handler._has_lbaas_spec_changes.return_value = True + m_handler.create_crd_spec.return_value = old_spec m_handler._should_ignore.return_value = False + m_handler._drv_project = m_drv_project - h_lbaas.LBaaSSpecHandler.on_present(m_handler, svc_event) - - m_get_lbaas_spec.assert_called_once_with(svc_event) - m_handler._has_lbaas_spec_changes.assert_called_once_with(svc_event, - old_spec) - m_handler._generate_lbaas_spec.assert_not_called() - m_set_lbaas_spec.assert_not_called() - - @mock.patch('kuryr_kubernetes.utils.set_lbaas_spec') - @mock.patch('kuryr_kubernetes.utils.get_lbaas_spec') - def test_on_present_no_selector(self, m_get_lbaas_spec, - m_set_lbaas_spec): - svc_event = {'metadata': {'name': 'dummy_name'}} - old_spec = mock.sentinel.old_spec - - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) - m_get_lbaas_spec.return_value = old_spec - m_handler._should_ignore.return_value = True - - h_lbaas.LBaaSSpecHandler.on_present(m_handler, svc_event) - - m_get_lbaas_spec.assert_called_once_with(svc_event) - m_handler._has_lbaas_spec_changes.assert_not_called() - m_handler._generate_lbaas_spec.assert_not_called() - m_set_lbaas_spec.assert_not_called() + h_lbaas.ServiceHandler.on_present(m_handler, svc_event) + m_handler.create_crd_spec(svc_event) + m_handler._has_lbaas_spec_changes.return_value = False def test_get_service_ip(self): svc_body = {'spec': {'type': 'ClusterIP', 'clusterIP': mock.sentinel.cluster_ip}} - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) - ret = h_lbaas.LBaaSSpecHandler._get_service_ip(m_handler, svc_body) + ret = h_lbaas.ServiceHandler._get_service_ip(m_handler, svc_body) self.assertEqual(mock.sentinel.cluster_ip, ret) svc_body = {'spec': {'type': 'LoadBalancer', 'clusterIP': mock.sentinel.cluster_ip}} - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) - ret = h_lbaas.LBaaSSpecHandler._get_service_ip(m_handler, svc_body) + ret = h_lbaas.ServiceHandler._get_service_ip(m_handler, svc_body) self.assertEqual(mock.sentinel.cluster_ip, ret) def test_is_supported_type_clusterip(self): - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) svc_body = {'spec': {'type': 'ClusterIP', 'clusterIP': mock.sentinel.cluster_ip}} - ret = h_lbaas.LBaaSSpecHandler._is_supported_type(m_handler, svc_body) + ret = h_lbaas.ServiceHandler._is_supported_type(m_handler, svc_body) self.assertEqual(ret, True) def test_is_supported_type_loadbalancer(self): - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) svc_body = {'spec': {'type': 'LoadBalancer', 'clusterIP': mock.sentinel.cluster_ip}} - ret = h_lbaas.LBaaSSpecHandler._is_supported_type(m_handler, svc_body) + ret = h_lbaas.ServiceHandler._is_supported_type(m_handler, svc_body) self.assertEqual(ret, True) def _make_test_net_obj(self, cidr_list): @@ -148,54 +217,9 @@ def _make_test_net_obj(self, cidr_list): subnets_list = osv_subnet.SubnetList(objects=subnets) return osv_network.Network(subnets=subnets_list) - def test_generate_lbaas_spec(self): - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) - - service = mock.sentinel.service - project_id = mock.sentinel.project_id - ip = mock.sentinel.ip - subnet_id = mock.sentinel.subnet_id - ports = mock.sentinel.ports - sg_ids = mock.sentinel.sg_ids - - m_drv_project = mock.Mock() - m_drv_project.get_project.return_value = project_id - m_drv_sg = mock.Mock() - m_drv_sg.get_security_groups.return_value = sg_ids - m_handler._drv_project = m_drv_project - m_handler._drv_sg = m_drv_sg - m_handler._get_service_ip.return_value = ip - m_handler._get_subnet_id.return_value = subnet_id - m_handler._generate_lbaas_port_specs.return_value = ports - - spec_ctor_path = 'kuryr_kubernetes.objects.lbaas.LBaaSServiceSpec' - with mock.patch(spec_ctor_path) as m_spec_ctor: - m_spec_ctor.return_value = mock.sentinel.ret_obj - service = {'spec': {'type': 'ClusterIP'}} - - ret_obj = h_lbaas.LBaaSSpecHandler._generate_lbaas_spec( - m_handler, service) - self.assertEqual(mock.sentinel.ret_obj, ret_obj) - m_spec_ctor.assert_called_once_with( - ip=ip, - project_id=project_id, - subnet_id=subnet_id, - ports=ports, - security_groups_ids=sg_ids, - type='ClusterIP', - lb_ip=None) - - m_drv_project.get_project.assert_called_once_with(service) - m_handler._get_service_ip.assert_called_once_with(service) - m_handler._get_subnet_id.assert_called_once_with( - service, project_id, ip) - m_handler._generate_lbaas_port_specs.assert_called_once_with(service) - m_drv_sg.get_security_groups.assert_called_once_with( - service, project_id) - @mock.patch('kuryr_kubernetes.utils.has_port_changes') def test_has_lbaas_spec_changes(self, m_port_changes): - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) service = mock.sentinel.service lbaas_spec = mock.sentinel.lbaas_spec @@ -203,55 +227,126 @@ def test_has_lbaas_spec_changes(self, m_port_changes): for has_port_changes in (True, False): m_handler._has_ip_changes.return_value = has_ip_changes m_port_changes.return_value = has_port_changes - ret = h_lbaas.LBaaSSpecHandler._has_lbaas_spec_changes( + ret = h_lbaas.ServiceHandler._has_lbaas_spec_changes( m_handler, service, lbaas_spec) self.assertEqual(has_ip_changes or has_port_changes, ret) def test_has_ip_changes(self): - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) m_service = mock.MagicMock() m_handler._get_service_ip.return_value = '1.1.1.1' m_lbaas_spec = mock.MagicMock() m_lbaas_spec.ip.__str__.return_value = '2.2.2.2' - ret = h_lbaas.LBaaSSpecHandler._has_ip_changes( + ret = h_lbaas.ServiceHandler._has_ip_changes( m_handler, m_service, m_lbaas_spec) self.assertTrue(ret) def test_has_ip_changes__no_changes(self): - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) - m_service = mock.MagicMock() + service = { + "metadata": { + "creationTimestamp": "2020-07-25T18:15:12Z", + "finalizers": [ + "openstack.org/service" + ], + "labels": { + "run": "test" + }, + "name": "test", + "namespace": "test", + "resourceVersion": "413753", + "selfLink": "", + "uid": "a026ae48-6141-4029-b743-bac48dae7f06" + }, + "spec": { + "clusterIP": "1.1.1.1" + } + } + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) m_handler._get_service_ip.return_value = '1.1.1.1' - m_lbaas_spec = mock.MagicMock() - m_lbaas_spec.ip.__str__.return_value = '1.1.1.1' + lb_crd = { + 'apiVersion': 'openstack.org/v1', + 'kind': 'KuryrLoadBalancer', + 'metadata': { + 'name': 'test', + 'finalizers': [''], + }, + 'spec': { + 'ip': '1.1.1.1' + } + } - ret = h_lbaas.LBaaSSpecHandler._has_ip_changes( - m_handler, m_service, m_lbaas_spec) + ret = h_lbaas.ServiceHandler._has_ip_changes( + m_handler, service, lb_crd) self.assertFalse(ret) def test_has_ip_changes__no_spec(self): - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) - m_service = mock.MagicMock() + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) m_handler._get_service_ip.return_value = '1.1.1.1' - m_lbaas_spec = None + service = { + "metadata": { + "creationTimestamp": "2020-07-25T18:15:12Z", + "finalizers": [ + "openstack.org/service" + ], + "labels": { + "run": "test" + }, + "name": "test", + "namespace": "test", + "resourceVersion": "413753", + "selfLink": "", + "uid": "a026ae48-6141-4029-b743-bac48dae7f06" + }, + "spec": { + "clusterIP": "1.1.1.1" + } + } + lb_crd = { + "spec": { + "ip": None + } + } - ret = h_lbaas.LBaaSSpecHandler._has_ip_changes( - m_handler, m_service, m_lbaas_spec) + ret = h_lbaas.ServiceHandler._has_ip_changes( + m_handler, service, lb_crd) self.assertTrue(ret) def test_has_ip_changes__no_nothing(self): - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) - m_service = mock.MagicMock() + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) + service = { + "metadata": { + "creationTimestamp": "2020-07-25T18:15:12Z", + "finalizers": [ + "openstack.org/service" + ], + "labels": { + "run": "test" + }, + "name": "test", + "namespace": "test", + "resourceVersion": "413753", + "selfLink": "", + "uid": "a026ae48-6141-4029-b743-bac48dae7f06" + }, + "spec": { + "clusterIP": "1.1.1.1" + } + } + lb_crd = { + "spec": { + "ip": None + } + } m_handler._get_service_ip.return_value = None - m_lbaas_spec = None - ret = h_lbaas.LBaaSSpecHandler._has_ip_changes( - m_handler, m_service, m_lbaas_spec) + ret = h_lbaas.ServiceHandler._has_ip_changes( + m_handler, service, lb_crd) self.assertFalse(ret) @mock.patch('kuryr_kubernetes.utils.get_service_ports') def test_generate_lbaas_port_specs(self, m_get_service_ports): - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) m_get_service_ports.return_value = [ {'port': 1, 'name': 'X', 'protocol': 'TCP'}, {'port': 2, 'name': 'Y', 'protocol': 'TCP'} @@ -261,7 +356,7 @@ def test_generate_lbaas_port_specs(self, m_get_service_ports): obj_lbaas.LBaaSPortSpec(name='Y', protocol='TCP', port=2), ] - ret = h_lbaas.LBaaSSpecHandler._generate_lbaas_port_specs( + ret = h_lbaas.ServiceHandler._generate_lbaas_port_specs( m_handler, mock.sentinel.service) self.assertEqual(expected_ports, ret) m_get_service_ports.assert_called_once_with( @@ -269,7 +364,7 @@ def test_generate_lbaas_port_specs(self, m_get_service_ports): @mock.patch('kuryr_kubernetes.utils.get_service_ports') def test_generate_lbaas_port_specs_udp(self, m_get_service_ports): - m_handler = mock.Mock(spec=h_lbaas.LBaaSSpecHandler) + m_handler = mock.Mock(spec=h_lbaas.ServiceHandler) m_get_service_ports.return_value = [ {'port': 1, 'name': 'X', 'protocol': 'TCP'}, {'port': 2, 'name': 'Y', 'protocol': 'UDP'} @@ -279,7 +374,7 @@ def test_generate_lbaas_port_specs_udp(self, m_get_service_ports): obj_lbaas.LBaaSPortSpec(name='Y', protocol='UDP', port=2), ] - ret = h_lbaas.LBaaSSpecHandler._generate_lbaas_port_specs( + ret = h_lbaas.ServiceHandler._generate_lbaas_port_specs( m_handler, mock.sentinel.service) self.assertEqual(expected_ports, ret) m_get_service_ports.assert_called_once_with( @@ -292,633 +387,3 @@ def test_set_lbaas_spec(self): def test_get_lbaas_spec(self): self.skipTest("skipping until generalised annotation handling is " "implemented") - - -class FakeLBaaSDriver(drv_base.LBaaSDriver): - - def ensure_loadbalancer(self, name, project_id, subnet_id, ip, - security_groups_ids, service_type, provider=None): - return obj_lbaas.LBaaSLoadBalancer(name=name, - project_id=project_id, - subnet_id=subnet_id, - ip=ip, - id=str(uuid.uuid4()), - provider=provider) - - def ensure_listener(self, loadbalancer, protocol, port, - service_type='ClusterIP'): - if protocol not in _SUPPORTED_LISTENER_PROT: - return None - - name = "%s:%s:%s" % (loadbalancer.name, protocol, port) - return obj_lbaas.LBaaSListener(name=name, - project_id=loadbalancer.project_id, - loadbalancer_id=loadbalancer.id, - protocol=protocol, - port=port, - id=str(uuid.uuid4())) - - def ensure_pool(self, loadbalancer, listener): - return obj_lbaas.LBaaSPool(name=listener.name, - project_id=loadbalancer.project_id, - loadbalancer_id=loadbalancer.id, - listener_id=listener.id, - protocol=listener.protocol, - id=str(uuid.uuid4())) - - def ensure_member(self, loadbalancer, pool, subnet_id, ip, port, - target_ref_namespace, target_ref_name, listener_port=None - ): - name = "%s:%s:%s" % (loadbalancer.name, ip, port) - return obj_lbaas.LBaaSMember(name=name, - project_id=pool.project_id, - pool_id=pool.id, - subnet_id=subnet_id, - ip=ip, - port=port, - id=str(uuid.uuid4())) - - def release_loadbalancer(self, loadbalancer): - pass - - def release_listener(self, loadbalancer, listener): - pass - - def release_pool(self, loadbalancer, pool): - pass - - def release_member(self, loadbalancer, member): - pass - - def get_service_loadbalancer_name(self, namespace, svc_name): - return "%s/%s" % (namespace, svc_name) - - def get_loadbalancer_pool_name(self, lb_name, namespace, svc_name): - return "%s/%s/%s" % (lb_name, namespace, svc_name) - - -class TestLoadBalancerHandler(test_base.TestCase): - - @mock.patch('kuryr_kubernetes.controller.drivers.base.' - 'ServiceProjectDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base.' - 'ServiceSecurityGroupsDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.handlers.lbaas' - '.LoadBalancerHandler._cleanup_leftover_lbaas') - @mock.patch('kuryr_kubernetes.config.CONF') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.ServicePubIpDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodSubnetsDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodProjectDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.LBaaSDriver.get_instance') - def test_init(self, m_get_drv_lbaas, m_get_drv_project, - m_get_drv_subnets, m_get_drv_service_pub_ip, m_cfg, - m_cleanup_leftover_lbaas, - m_get_svc_sg_drv, m_get_svc_drv_project): - m_get_drv_lbaas.return_value = mock.sentinel.drv_lbaas - m_get_drv_project.return_value = mock.sentinel.drv_project - m_get_drv_subnets.return_value = mock.sentinel.drv_subnets - m_get_drv_service_pub_ip.return_value = mock.sentinel.drv_lb_ip - m_get_svc_drv_project.return_value = mock.sentinel.drv_svc_project - m_get_svc_sg_drv.return_value = mock.sentinel.drv_sg - m_cfg.kubernetes.endpoints_driver_octavia_provider = 'default' - handler = h_lbaas.LoadBalancerHandler() - - self.assertEqual(mock.sentinel.drv_lbaas, handler._drv_lbaas) - self.assertEqual(mock.sentinel.drv_project, handler._drv_pod_project) - self.assertEqual(mock.sentinel.drv_subnets, handler._drv_pod_subnets) - self.assertEqual(mock.sentinel.drv_lb_ip, handler._drv_service_pub_ip) - self.assertEqual(mock.sentinel.drv_svc_project, handler._drv_project) - self.assertEqual(mock.sentinel.drv_sg, handler._drv_sg) - self.assertIsNone(handler._lb_provider) - - @mock.patch('kuryr_kubernetes.controller.drivers.base.' - 'ServiceProjectDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base.' - 'ServiceSecurityGroupsDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.handlers.lbaas' - '.LoadBalancerHandler._cleanup_leftover_lbaas') - @mock.patch('kuryr_kubernetes.config.CONF') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.ServicePubIpDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodSubnetsDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodProjectDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.LBaaSDriver.get_instance') - def test_init_provider_ovn(self, m_get_drv_lbaas, m_get_drv_project, - m_get_drv_subnets, m_get_drv_service_pub_ip, - m_cfg, m_cleanup_leftover_lbaas, - m_get_svc_sg_drv, m_get_svc_drv_project): - m_get_drv_lbaas.return_value = mock.sentinel.drv_lbaas - m_get_drv_project.return_value = mock.sentinel.drv_project - m_get_drv_subnets.return_value = mock.sentinel.drv_subnets - m_get_drv_service_pub_ip.return_value = mock.sentinel.drv_lb_ip - m_get_svc_drv_project.return_value = mock.sentinel.drv_svc_project - m_get_svc_sg_drv.return_value = mock.sentinel.drv_sg - m_cfg.kubernetes.endpoints_driver_octavia_provider = 'ovn' - handler = h_lbaas.LoadBalancerHandler() - - self.assertEqual(mock.sentinel.drv_lbaas, handler._drv_lbaas) - self.assertEqual(mock.sentinel.drv_project, handler._drv_pod_project) - self.assertEqual(mock.sentinel.drv_subnets, handler._drv_pod_subnets) - self.assertEqual(mock.sentinel.drv_lb_ip, handler._drv_service_pub_ip) - self.assertEqual(mock.sentinel.drv_svc_project, handler._drv_project) - self.assertEqual(mock.sentinel.drv_sg, handler._drv_sg) - self.assertEqual('ovn', handler._lb_provider) - - @mock.patch('kuryr_kubernetes.utils.get_lbaas_spec') - @mock.patch('kuryr_kubernetes.utils.set_lbaas_state') - @mock.patch('kuryr_kubernetes.utils.get_lbaas_state') - def test_on_present(self, m_get_lbaas_state, m_set_lbaas_state, - m_get_lbaas_spec): - lbaas_spec = mock.sentinel.lbaas_spec - lbaas_spec.type = 'DummyType' - lbaas_spec.lb_ip = "1.2.3.4" - lbaas_spec.project_id = 12345678 - - lbaas_state = mock.sentinel.lbaas_state - lbaas_state.service_pub_ip_info = None - loadbalancer = mock.Mock() - loadbalancer.port_id = 12345678 - lbaas_state.loadbalancer = loadbalancer - endpoints = mock.sentinel.endpoints - - m_drv_service_pub_ip = mock.Mock() - m_drv_service_pub_ip.acquire_service_pub_ip_info.return_value = None - m_drv_service_pub_ip.associate_pub_ip.return_value = True - - m_handler = mock.Mock(spec=h_lbaas.LoadBalancerHandler) - m_get_lbaas_spec.return_value = lbaas_spec - m_handler._should_ignore.return_value = False - m_get_lbaas_state.return_value = lbaas_state - m_handler._sync_lbaas_members.return_value = True - m_handler._drv_service_pub_ip = m_drv_service_pub_ip - m_handler._lb_provider = None - - h_lbaas.LoadBalancerHandler.on_present(m_handler, endpoints) - - m_get_lbaas_spec.assert_called_once_with(endpoints) - m_handler._should_ignore.assert_called_once_with(endpoints, lbaas_spec) - m_get_lbaas_state.assert_called_once_with(endpoints) - m_handler._sync_lbaas_members.assert_called_once_with( - endpoints, lbaas_state, lbaas_spec) - m_set_lbaas_state.assert_called_once_with( - endpoints, lbaas_state) - m_handler._update_lb_status.assert_not_called() - - def _fake_sync_lbaas_members(self, endpoints, lbaas_state, lbaas_spec): - loadbalancer = mock.Mock() - loadbalancer.port_id = 12345678 - lbaas_state.loadbalancer = loadbalancer - lbaas_state.service_pub_ip_info = None - return True - - @mock.patch('kuryr_kubernetes.utils.get_lbaas_spec') - @mock.patch('kuryr_kubernetes.utils.set_lbaas_state') - @mock.patch('kuryr_kubernetes.utils.get_lbaas_state') - def test_on_present_loadbalancer_service( - self, m_get_lbaas_state, m_set_lbaas_state, m_get_lbaas_spec): - lbaas_spec = mock.sentinel.lbaas_spec - lbaas_spec.type = 'LoadBalancer' - lbaas_spec.lb_ip = "1.2.3.4" - lbaas_spec.project_id = 12345678 - - lbaas_state = mock.sentinel.lbaas_state - lbaas_state.service_pub_ip_info = None - loadbalancer = mock.Mock() - loadbalancer.port_id = 12345678 - lbaas_state.loadbalancer = loadbalancer - endpoints = mock.sentinel.endpoints - - floating_ip = {'floating_ip_address': '1.2.3.5', - 'id': 'ec29d641-fec4-4f67-928a-124a76b3a888'} - - service_pub_ip_info = obj_lbaas.LBaaSPubIp( - ip_id=floating_ip['id'], - ip_addr=floating_ip['floating_ip_address'], alloc_method='kk') - - m_drv_service_pub_ip = mock.Mock() - m_drv_service_pub_ip.acquire_service_pub_ip_info.return_value = ( - service_pub_ip_info) - m_drv_service_pub_ip.associate_pub_ip.return_value = True - - m_handler = mock.Mock(spec=h_lbaas.LoadBalancerHandler) - m_get_lbaas_spec.return_value = lbaas_spec - m_handler._should_ignore.return_value = False - m_get_lbaas_state.return_value = lbaas_state - m_handler._sync_lbaas_members = self._fake_sync_lbaas_members - m_handler._drv_service_pub_ip = m_drv_service_pub_ip - m_handler._lb_provider = None - - h_lbaas.LoadBalancerHandler.on_present(m_handler, endpoints) - - m_get_lbaas_spec.assert_called_once_with(endpoints) - m_handler._should_ignore.assert_called_once_with(endpoints, lbaas_spec) - m_get_lbaas_state.assert_called_once_with(endpoints) - m_set_lbaas_state.assert_called_once_with( - endpoints, lbaas_state) - m_handler._update_lb_status.assert_called() - - @mock.patch('kuryr_kubernetes.utils.get_lbaas_spec') - @mock.patch('kuryr_kubernetes.utils.set_lbaas_state') - @mock.patch('kuryr_kubernetes.utils.get_lbaas_state') - def test_on_present_rollback(self, m_get_lbaas_state, - m_set_lbaas_state, m_get_lbaas_spec): - lbaas_spec = mock.sentinel.lbaas_spec - lbaas_spec.type = 'ClusterIp' - lbaas_spec.lb_ip = '1.2.3.4' - lbaas_spec.project_id = '12345678' - lbaas_state = mock.sentinel.lbaas_state - lbaas_state.service_pub_ip_info = None - loadbalancer = mock.Mock() - loadbalancer.port_id = 12345678 - lbaas_state.loadbalancer = loadbalancer - m_drv_service_pub_ip = mock.Mock() - m_drv_service_pub_ip.acquire_service_pub_ip_info.return_value = None - m_drv_service_pub_ip.associate_pub_ip.return_value = True - - endpoints = mock.sentinel.endpoints - - m_handler = mock.Mock(spec=h_lbaas.LoadBalancerHandler) - m_get_lbaas_spec.return_value = lbaas_spec - m_handler._should_ignore.return_value = False - m_get_lbaas_state.return_value = lbaas_state - m_handler._sync_lbaas_members.return_value = True - m_set_lbaas_state.side_effect = ( - k_exc.K8sResourceNotFound('ep')) - m_handler._drv_service_pub_ip = m_drv_service_pub_ip - m_handler._lb_provider = None - h_lbaas.LoadBalancerHandler.on_present(m_handler, endpoints) - - m_get_lbaas_spec.assert_called_once_with(endpoints) - m_handler._should_ignore.assert_called_once_with(endpoints, lbaas_spec) - m_get_lbaas_state.assert_called_once_with(endpoints) - m_handler._sync_lbaas_members.assert_called_once_with( - endpoints, lbaas_state, lbaas_spec) - m_set_lbaas_state.assert_called_once_with( - endpoints, lbaas_state) - m_handler.on_deleted.assert_called_once_with( - endpoints, lbaas_state) - - @mock.patch('kuryr_kubernetes.utils.get_lbaas_state') - @mock.patch('kuryr_kubernetes.objects.lbaas' - '.LBaaSServiceSpec') - def test_on_cascade_deleted_lb_service(self, m_svc_spec_ctor, - m_get_lbaas_state): - endpoints = mock.sentinel.endpoints - empty_spec = mock.sentinel.empty_spec - lbaas_state = mock.Mock() - lbaas_state.loadbalancer = mock.sentinel.loadbalancer - lbaas_state.service_pub_ip_info = mock.sentinel.pub_ip - m_svc_spec_ctor.return_value = empty_spec - - m_handler = mock.Mock(spec=h_lbaas.LoadBalancerHandler) - m_get_lbaas_state.return_value = lbaas_state - m_handler._drv_lbaas = mock.Mock() - m_handler._drv_service_pub_ip = mock.Mock() - - h_lbaas.LoadBalancerHandler.on_deleted(m_handler, endpoints) - - m_handler._drv_lbaas.release_loadbalancer.assert_called_once_with( - loadbalancer=lbaas_state.loadbalancer) - m_handler._drv_service_pub_ip.release_pub_ip.assert_called_once_with( - lbaas_state.service_pub_ip_info) - - def test_should_ignore(self): - endpoints = mock.sentinel.endpoints - lbaas_spec = mock.sentinel.lbaas_spec - - # REVISIT(ivc): ddt? - m_handler = mock.Mock(spec=h_lbaas.LoadBalancerHandler) - m_handler._has_pods.return_value = True - m_handler._svc_handler_annotations_updated.return_value = True - - ret = h_lbaas.LoadBalancerHandler._should_ignore( - m_handler, endpoints, lbaas_spec) - self.assertEqual(False, ret) - - m_handler._has_pods.assert_called_once_with(endpoints) - m_handler._svc_handler_annotations_updated.assert_called_once_with( - endpoints, lbaas_spec) - - def test_has_pods(self): - # REVISIT(ivc): ddt? - endpoints = {'subsets': [ - {}, - {'addresses': []}, - {'addresses': [{'targetRef': {}}]}, - {'addresses': [{'targetRef': {'kind': k_const.K8S_OBJ_POD}}]} - ]} - - m_handler = mock.Mock(spec=h_lbaas.LoadBalancerHandler) - - ret = h_lbaas.LoadBalancerHandler._has_pods(m_handler, endpoints) - - self.assertEqual(True, ret) - - def test_get_pod_subnet(self): - subnet_id = mock.sentinel.subnet_id - project_id = mock.sentinel.project_id - target_ref = {'kind': k_const.K8S_OBJ_POD, - 'name': 'pod-name', - 'namespace': 'default'} - ip = '1.2.3.4' - m_handler = mock.Mock(spec=h_lbaas.LoadBalancerHandler) - m_drv_pod_project = mock.Mock() - m_drv_pod_project.get_project.return_value = project_id - m_handler._drv_pod_project = m_drv_pod_project - m_drv_pod_subnets = mock.Mock() - m_drv_pod_subnets.get_subnets.return_value = { - subnet_id: osv_network.Network(subnets=osv_subnet.SubnetList( - objects=[osv_subnet.Subnet(cidr='1.2.3.0/24')]))} - m_handler._drv_pod_subnets = m_drv_pod_subnets - - observed_subnet_id = h_lbaas.LoadBalancerHandler._get_pod_subnet( - m_handler, target_ref, ip) - - self.assertEqual(subnet_id, observed_subnet_id) - - def _generate_lbaas_state(self, vip, targets, project_id, subnet_id): - name = 'namespace/DUMMY_NAME' - drv = FakeLBaaSDriver() - lb = drv.ensure_loadbalancer( - name, project_id, subnet_id, vip, None, 'ClusterIP') - listeners = {} - pools = {} - members = {} - for ip, (listen_port, target_port) in targets.items(): - lsnr = listeners.setdefault(listen_port, drv.ensure_listener( - lb, 'TCP', listen_port)) - pool = pools.setdefault(listen_port, drv.ensure_pool(lb, lsnr)) - members.setdefault((ip, listen_port, target_port), - drv.ensure_member(lb, pool, - subnet_id, ip, - target_port, None, None)) - return obj_lbaas.LBaaSState( - loadbalancer=lb, - listeners=list(listeners.values()), - pools=list(pools.values()), - members=list(members.values())) - - def _generate_lbaas_spec(self, vip, targets, project_id, - subnet_id, prot='TCP', lbaas_type='ClusterIP'): - return obj_lbaas.LBaaSServiceSpec( - ip=vip, - project_id=project_id, - subnet_id=subnet_id, - ports=[obj_lbaas.LBaaSPortSpec(name=str(t[0]), - protocol=prot, - port=t[0], - targetPort=t[1]) - for t in targets.values()], - type=lbaas_type) - - def _generate_endpoints(self, targets): - def _target_to_port(item): - _, (listen_port, target_port) = item - return {'port': target_port, 'name': str(listen_port)} - port_with_addrs = [ - (p, [e[0] for e in grp]) - for p, grp in itertools.groupby( - sorted(targets.items()), _target_to_port)] - return { - 'metadata': { - 'name': 'ep_name', - 'namespace': 'default' - }, - 'subsets': [ - { - 'addresses': [ - { - 'ip': ip, - 'targetRef': { - 'kind': k_const.K8S_OBJ_POD, - 'name': ip, - 'namespace': 'default' - } - } - for ip in addrs - ], - 'ports': [port] - } - for port, addrs in port_with_addrs - ] - } - - def _sync_lbaas_members_impl(self, m_get_drv_lbaas, m_get_drv_project, - m_get_drv_subnets, subnet_id, project_id, - endpoints, state, spec): - m_drv_lbaas = mock.Mock(wraps=FakeLBaaSDriver()) - m_drv_project = mock.Mock() - m_drv_project.get_project.return_value = project_id - m_drv_subnets = mock.Mock() - m_drv_subnets.get_subnets.return_value = { - subnet_id: mock.sentinel.subnet} - m_get_drv_lbaas.return_value = m_drv_lbaas - m_get_drv_project.return_value = m_drv_project - m_get_drv_subnets.return_value = m_drv_subnets - - handler = h_lbaas.LoadBalancerHandler() - - with mock.patch.object(handler, '_get_pod_subnet') as m_get_pod_subnet: - m_get_pod_subnet.return_value = subnet_id - handler._sync_lbaas_members(endpoints, state, spec) - - lsnrs = {lsnr.id: lsnr for lsnr in state.listeners} - pools = {pool.id: pool for pool in state.pools} - observed_targets = sorted( - (str(member.ip), ( - lsnrs[pools[member.pool_id].listener_id].port, - member.port)) - for member in state.members) - return observed_targets - - @mock.patch('kuryr_kubernetes.controller.handlers.lbaas' - '.LoadBalancerHandler._cleanup_leftover_lbaas') - @mock.patch('kuryr_kubernetes.controller.handlers.lbaas.' - 'LoadBalancerHandler._sync_lbaas_sgs') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodSubnetsDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodProjectDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.LBaaSDriver.get_instance') - def test_sync_lbaas_members(self, m_get_drv_lbaas, m_get_drv_project, - m_get_drv_subnets, m_sync_lbaas_sgs, - m_cleanup_leftover_lbaas): - # REVISIT(ivc): test methods separately and verify ensure/release - project_id = str(uuid.uuid4()) - subnet_id = str(uuid.uuid4()) - current_ip = '1.1.1.1' - current_targets = { - '1.1.1.101': (1001, 10001), - '1.1.1.111': (1001, 10001), - '1.1.1.201': (2001, 20001)} - expected_ip = '2.2.2.2' - expected_targets = { - '2.2.2.101': (1201, 12001), - '2.2.2.111': (1201, 12001), - '2.2.2.201': (2201, 22001)} - endpoints = self._generate_endpoints(expected_targets) - state = self._generate_lbaas_state( - current_ip, current_targets, project_id, subnet_id) - spec = self._generate_lbaas_spec(expected_ip, expected_targets, - project_id, subnet_id) - - observed_targets = self._sync_lbaas_members_impl( - m_get_drv_lbaas, m_get_drv_project, m_get_drv_subnets, - subnet_id, project_id, endpoints, state, spec) - - self.assertEqual(sorted(expected_targets.items()), observed_targets) - self.assertEqual(expected_ip, str(state.loadbalancer.ip)) - - @mock.patch('kuryr_kubernetes.controller.handlers.lbaas' - '.LoadBalancerHandler._cleanup_leftover_lbaas') - @mock.patch('kuryr_kubernetes.controller.handlers.lbaas.' - 'LoadBalancerHandler._sync_lbaas_sgs') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodSubnetsDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodProjectDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.LBaaSDriver.get_instance') - def test_sync_lbaas_members_udp(self, m_get_drv_lbaas, - m_get_drv_project, m_get_drv_subnets, - m_sync_lbaas_sgs, - m_cleanup_leftover_lbaas): - # REVISIT(ivc): test methods separately and verify ensure/release - project_id = str(uuid.uuid4()) - subnet_id = str(uuid.uuid4()) - current_ip = '1.1.1.1' - current_targets = { - '1.1.1.101': (1001, 10001), - '1.1.1.111': (1001, 10001), - '1.1.1.201': (2001, 20001)} - expected_ip = '2.2.2.2' - expected_targets = { - '2.2.2.101': (1201, 12001), - '2.2.2.111': (1201, 12001), - '2.2.2.201': (2201, 22001)} - endpoints = self._generate_endpoints(expected_targets) - state = self._generate_lbaas_state( - current_ip, current_targets, project_id, subnet_id) - spec = self._generate_lbaas_spec(expected_ip, expected_targets, - project_id, subnet_id, 'UDP') - - observed_targets = self._sync_lbaas_members_impl( - m_get_drv_lbaas, m_get_drv_project, m_get_drv_subnets, - subnet_id, project_id, endpoints, state, spec) - - self.assertEqual([], observed_targets) - self.assertEqual(expected_ip, str(state.loadbalancer.ip)) - - @mock.patch('kuryr_kubernetes.controller.handlers.lbaas' - '.LoadBalancerHandler._cleanup_leftover_lbaas') - @mock.patch('kuryr_kubernetes.controller.handlers.lbaas.' - 'LoadBalancerHandler._sync_lbaas_sgs') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodSubnetsDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodProjectDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.LBaaSDriver.get_instance') - def test_sync_lbaas_members_svc_listener_port_edit( - self, m_get_drv_lbaas, m_get_drv_project, m_get_drv_subnets, - m_sync_lbaas_sgs, m_cleanup_leftover_lbaas): - # REVISIT(ivc): test methods separately and verify ensure/release - project_id = str(uuid.uuid4()) - subnet_id = str(uuid.uuid4()) - current_ip = '1.1.1.1' - current_targets = { - '1.1.1.101': (1001, 10001)} - expected_ip = '1.1.1.1' - expected_targets = { - '1.1.1.101': (1201, 10001)} - endpoints = self._generate_endpoints(expected_targets) - state = self._generate_lbaas_state( - current_ip, current_targets, project_id, subnet_id) - spec = self._generate_lbaas_spec(expected_ip, expected_targets, - project_id, subnet_id) - - m_drv_lbaas = mock.Mock(wraps=FakeLBaaSDriver()) - m_drv_project = mock.Mock() - m_drv_project.get_project.return_value = project_id - m_drv_subnets = mock.Mock() - m_drv_subnets.get_subnets.return_value = { - subnet_id: mock.sentinel.subnet} - m_get_drv_lbaas.return_value = m_drv_lbaas - m_get_drv_project.return_value = m_drv_project - m_get_drv_subnets.return_value = m_drv_subnets - - handler = h_lbaas.LoadBalancerHandler() - - with mock.patch.object(handler, '_get_pod_subnet') as m_get_pod_subnet: - m_get_pod_subnet.return_value = subnet_id - handler._sync_lbaas_members(endpoints, state, spec) - - self.assertEqual(expected_ip, str(state.loadbalancer.ip)) - m_drv_lbaas.release_pool.assert_called_once() - - def test_get_lbaas_spec(self): - self.skipTest("skipping until generalised annotation handling is " - "implemented") - - def test_get_lbaas_state(self): - self.skipTest("skipping until generalised annotation handling is " - "implemented") - - def test_set_lbaas_state(self): - self.skipTest("skipping until generalised annotation handling is " - "implemented") - - @mock.patch('kuryr_kubernetes.controller.handlers.lbaas' - '.LoadBalancerHandler._cleanup_leftover_lbaas') - @mock.patch('kuryr_kubernetes.controller.handlers.lbaas.' - 'LoadBalancerHandler._sync_lbaas_sgs') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodSubnetsDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.PodProjectDriver.get_instance') - @mock.patch('kuryr_kubernetes.controller.drivers.base' - '.LBaaSDriver.get_instance') - def test_add_new_members_udp(self, m_get_drv_lbaas, - m_get_drv_project, m_get_drv_subnets, - m_sync_lbaas_sgs, m_cleanup_leftover_lbaas): - project_id = str(uuid.uuid4()) - subnet_id = str(uuid.uuid4()) - current_ip = '1.1.1.1' - current_targets = { - '1.1.1.101': (1001, 10001), - '1.1.1.111': (1001, 10001), - '1.1.1.201': (2001, 20001)} - expected_ip = '2.2.2.2' - expected_targets = { - '2.2.2.101': (1201, 12001), - '2.2.2.111': (1201, 12001), - '2.2.2.201': (2201, 22001)} - endpoints = self._generate_endpoints(expected_targets) - state = self._generate_lbaas_state( - current_ip, current_targets, project_id, subnet_id) - spec = self._generate_lbaas_spec(expected_ip, expected_targets, - project_id, subnet_id, 'UDP') - - m_drv_lbaas = mock.Mock(wraps=FakeLBaaSDriver()) - m_drv_project = mock.Mock() - m_drv_project.get_project.return_value = project_id - m_drv_subnets = mock.Mock() - m_drv_subnets.get_subnets.return_value = { - subnet_id: mock.sentinel.subnet} - m_get_drv_lbaas.return_value = m_drv_lbaas - m_get_drv_project.return_value = m_drv_project - m_get_drv_subnets.return_value = m_drv_subnets - - handler = h_lbaas.LoadBalancerHandler() - member_added = handler._add_new_members(endpoints, state, spec) - - self.assertEqual(member_added, False) - m_drv_lbaas.ensure_member.assert_not_called() diff --git a/kuryr_kubernetes/tests/unit/controller/handlers/test_loadbalancer.py b/kuryr_kubernetes/tests/unit/controller/handlers/test_loadbalancer.py new file mode 100644 index 000000000..4f6140e1a --- /dev/null +++ b/kuryr_kubernetes/tests/unit/controller/handlers/test_loadbalancer.py @@ -0,0 +1,525 @@ +# Copyright (c) 2016 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest import mock +import uuid + +import os_vif.objects.network as osv_network +import os_vif.objects.subnet as osv_subnet + +from kuryr_kubernetes import constants as k_const +from kuryr_kubernetes.controller.drivers import base as drv_base +from kuryr_kubernetes.controller.handlers import loadbalancer as h_lb +from kuryr_kubernetes.tests import base as test_base +from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix + +_SUPPORTED_LISTENER_PROT = ('HTTP', 'HTTPS', 'TCP') + + +def get_lb_crd(): + return { + "metadata": { + "creationTimestamp": "2020-07-28T13:13:30Z", + "finalizers": [ + "" + ], + "generation": 6, + "name": "test", + "namespace": "default", + "resourceVersion": "111871", + "selfLink": "test", + "uid": "584fe3ea-04dd-43f7-be2f-713e861694ec" + }, + "spec": { + "ip": "1.2.3.4", + "ports": [ + { + "port": 1, + "protocol": "TCP", + "targetPort": "1" + } + ], + "project_id": "1023456789120", + "security_groups_ids": [ + "1d134e68-5653-4192-bda2-4214319af799", + "31d7b8c2-75f1-4125-9565-8c15c5cf046c" + ], + "subnet_id": "123456789120", + "subsets": [ + { + "addresses": [ + { + "ip": "1.1.1.1", + "nodeName": "sarka-devstack", + "targetRef": { + "kind": "Pod", + "name": "test-f87976f9c-thjbk", + "namespace": "default", + "resourceVersion": "111701", + "uid": "10234567800" + } + } + ], + "ports": [ + { + "port": 2, + "protocol": "TCP" + } + ] + } + ], + "type": "LoadBalancer" + }, + "status": { + "listeners": [ + { + "id": "012345678912", + "loadbalancer_id": "01234567890", + "name": "default/test:TCP:80", + "port": 1, + "project_id": "12345678912", + "protocol": "TCP" + } + ], + "loadbalancer": { + "id": "01234567890", + "ip": "1.2.3.4", + "name": "default/test", + "port_id": "1023456789120", + "project_id": "12345678912", + "provider": "amphora", + "security_groups": [ + "1d134e68-5653-4192-bda2-4214319af799", + "31d7b8c2-75f1-4125-9565-8c15c5cf046c" + ], + "subnet_id": "123456789120" + }, + "members": [ + { + "id": "0123456789", + "ip": "1.1.1.1", + "name": "default/test-f87976f9c-thjbk:8080", + "pool_id": "1234567890", + "port": 2, + "project_id": "12345678912", + "subnet_id": "123456789120" + } + ], + "pools": [ + { + "id": "1234567890", + "listener_id": "012345678912", + "loadbalancer_id": "01234567890", + "name": "default/test:TCP:80", + "project_id": "12345678912", + "protocol": "TCP" + } + ], + 'service_pub_ip_info': { + 'ip_id': '1.2.3.5', + 'ip_addr': 'ec29d641-fec4-4f67-928a-124a76b3a888', + 'alloc_method': 'kk' + } + } + } + + +class FakeLBaaSDriver(drv_base.LBaaSDriver): + + def ensure_loadbalancer(self, name, project_id, subnet_id, ip, + security_groups_ids, service_type, provider=None): + + return { + 'name': name, + 'project_id': project_id, + 'subnet_id': subnet_id, + 'ip': ip, + 'id': str(uuid.uuid4()), + 'provider': provider + } + + def ensure_listener(self, loadbalancer, protocol, port, + service_type='ClusterIP'): + if protocol not in _SUPPORTED_LISTENER_PROT: + return None + + name = "%s:%s:%s" % (loadbalancer['name'], protocol, port) + return { + 'name': name, + 'project_id': loadbalancer['project_id'], + 'loadbalancer_id': loadbalancer['id'], + 'protocol': protocol, + 'port': port, + 'id': str(uuid.uuid4()) + } + + def ensure_pool(self, loadbalancer, listener): + return { + 'name': listener['name'], + 'project_id': loadbalancer['project_id'], + 'loadbalancer_id': loadbalancer['id'], + 'listener_id': listener['id'], + 'protocol': listener['protocol'], + 'id': str(uuid.uuid4()) + } + + def ensure_member(self, loadbalancer, pool, subnet_id, ip, port, + target_ref_namespace, target_ref_name, listener_port=None + ): + name = "%s:%s:%s" % (loadbalancer['name'], ip, port) + return { + 'name': name, + 'project_id': pool['project_id'], + 'pool_id': pool['id'], + 'subnet_id': subnet_id, + 'ip': ip, + 'port': port, + 'id': str(uuid.uuid4()) + } + + +class TestKuryrLoadBalancerHandler(test_base.TestCase): + + @mock.patch('kuryr_kubernetes.controller.drivers.base.' + 'ServiceProjectDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base.' + 'ServiceSecurityGroupsDriver.get_instance') + @mock.patch('kuryr_kubernetes.config.CONF') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.ServicePubIpDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodSubnetsDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodProjectDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.LBaaSDriver.get_instance') + def test_init(self, m_get_drv_lbaas, m_get_drv_project, + m_get_drv_subnets, m_get_drv_service_pub_ip, m_cfg, + m_get_svc_sg_drv, m_get_svc_drv_project): + m_get_drv_lbaas.return_value = mock.sentinel.drv_lbaas + m_get_drv_project.return_value = mock.sentinel.drv_project + m_get_drv_subnets.return_value = mock.sentinel.drv_subnets + m_get_drv_service_pub_ip.return_value = mock.sentinel.drv_lb_ip + m_get_svc_drv_project.return_value = mock.sentinel.drv_svc_project + m_get_svc_sg_drv.return_value = mock.sentinel.drv_sg + m_cfg.kubernetes.endpoints_driver_octavia_provider = 'default' + handler = h_lb.KuryrLoadBalancerHandler() + + self.assertEqual(mock.sentinel.drv_lbaas, handler._drv_lbaas) + self.assertEqual(mock.sentinel.drv_project, handler._drv_pod_project) + self.assertEqual(mock.sentinel.drv_subnets, handler._drv_pod_subnets) + self.assertEqual(mock.sentinel.drv_lb_ip, handler._drv_service_pub_ip) + self.assertIsNone(handler._lb_provider) + + @mock.patch('kuryr_kubernetes.controller.drivers.base.' + 'ServiceProjectDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base.' + 'ServiceSecurityGroupsDriver.get_instance') + @mock.patch('kuryr_kubernetes.config.CONF') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.ServicePubIpDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodSubnetsDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodProjectDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.LBaaSDriver.get_instance') + def test_init_provider_ovn(self, m_get_drv_lbaas, m_get_drv_project, + m_get_drv_subnets, m_get_drv_service_pub_ip, + m_cfg, + m_get_svc_sg_drv, m_get_svc_drv_project): + m_get_drv_lbaas.return_value = mock.sentinel.drv_lbaas + m_get_drv_project.return_value = mock.sentinel.drv_project + m_get_drv_subnets.return_value = mock.sentinel.drv_subnets + m_get_drv_service_pub_ip.return_value = mock.sentinel.drv_lb_ip + m_get_svc_drv_project.return_value = mock.sentinel.drv_svc_project + m_get_svc_sg_drv.return_value = mock.sentinel.drv_sg + m_cfg.kubernetes.endpoints_driver_octavia_provider = 'ovn' + handler = h_lb .KuryrLoadBalancerHandler() + + self.assertEqual(mock.sentinel.drv_lbaas, handler._drv_lbaas) + self.assertEqual(mock.sentinel.drv_project, handler._drv_pod_project) + self.assertEqual(mock.sentinel.drv_subnets, handler._drv_pod_subnets) + self.assertEqual(mock.sentinel.drv_lb_ip, handler._drv_service_pub_ip) + self.assertEqual('ovn', handler._lb_provider) + + def test_on_present(self): + m_drv_service_pub_ip = mock.Mock() + m_drv_service_pub_ip.acquire_service_pub_ip_info.return_value = None + m_drv_service_pub_ip.associate_pub_ip.return_value = True + + m_handler = mock.Mock(spec=h_lb.KuryrLoadBalancerHandler) + m_handler._should_ignore.return_value = False + m_handler._sync_lbaas_members.return_value = True + m_handler._drv_service_pub_ip = m_drv_service_pub_ip + + h_lb.KuryrLoadBalancerHandler.on_present(m_handler, get_lb_crd()) + + m_handler._should_ignore.assert_called_once_with(get_lb_crd()) + m_handler._sync_lbaas_members.assert_called_once_with( + get_lb_crd()) + + def _fake_sync_lbaas_members(self, crd): + loadbalancer = { + "id": "01234567890", + "ip": "1.2.3.4", + "name": "default/test", + "port_id": "1023456789120", + "project_id": "12345678912", + "provider": "amphora", + "security_groups": [ + "1d134e68-5653-4192-bda2-4214319af799", + "31d7b8c2-75f1-4125-9565-8c15c5cf046c" + ], + "subnet_id": "123456789120" + } + loadbalancer['port_id'] = 12345678 + crd['status']['loadbalancer'] = loadbalancer + crd['status']['service_pub_ip_info'] = None + return True + + def test_on_present_loadbalancer_service(self): + floating_ip = {'floating_ip_address': '1.2.3.5', + 'id': 'ec29d641-fec4-4f67-928a-124a76b3a888'} + + service_pub_ip_info = { + 'ip_id': floating_ip['id'], + 'ip_addr': floating_ip['floating_ip_address'], + 'alloc_method': 'kk' + } + crd = get_lb_crd() + m_drv_service_pub_ip = mock.Mock() + m_drv_service_pub_ip.acquire_service_pub_ip_info.return_value = ( + service_pub_ip_info) + m_drv_service_pub_ip.associate_pub_ip.return_value = True + + h = mock.Mock(spec=h_lb.KuryrLoadBalancerHandler) + h._should_ignore.return_value = False + h._sync_lbaas_members.return_value = self._fake_sync_lbaas_members(crd) + h._drv_service_pub_ip = m_drv_service_pub_ip + kubernetes = self.useFixture(k_fix.MockK8sClient()).client + kubernetes.get_kubernetes_client = mock.Mock() + kubernetes.get_kubernetes_client() + h_lb.KuryrLoadBalancerHandler.on_present(h, crd) + h._should_ignore.assert_called_once_with(crd) + h._update_lb_status.assert_called() + + @mock.patch('kuryr_kubernetes.utils.get_lbaas_spec') + @mock.patch('kuryr_kubernetes.utils.set_lbaas_state') + @mock.patch('kuryr_kubernetes.utils.get_lbaas_state') + def test_on_present_rollback(self, m_get_lbaas_state, + m_set_lbaas_state, m_get_lbaas_spec): + m_drv_service_pub_ip = mock.Mock() + m_drv_service_pub_ip.acquire_service_pub_ip_info.return_value = None + m_drv_service_pub_ip.associate_pub_ip.return_value = True + + m_handler = mock.Mock(spec=h_lb.KuryrLoadBalancerHandler) + m_handler._should_ignore.return_value = False + m_handler._sync_lbaas_members.return_value = True + m_handler._drv_service_pub_ip = m_drv_service_pub_ip + h_lb.KuryrLoadBalancerHandler.on_present(m_handler, get_lb_crd()) + + m_handler._should_ignore.assert_called_once_with(get_lb_crd()) + m_handler._sync_lbaas_members.assert_called_once_with( + get_lb_crd()) + + def test_on_cascade_deleted_lb_service(self): + m_handler = mock.Mock(spec=h_lb.KuryrLoadBalancerHandler) + m_handler._drv_lbaas = mock.Mock() + m_handler._drv_service_pub_ip = mock.Mock() + crd = get_lb_crd() + m_handler._drv_lbaas.release_loadbalancer( + loadbalancer=crd['status']['loadbalancer']) + m_handler._drv_service_pub_ip.release_pub_ip( + crd['status']['service_pub_ip_info']) + + def test_should_ignore(self): + m_handler = mock.Mock(spec=h_lb.KuryrLoadBalancerHandler) + m_handler._has_pods.return_value = True + + ret = h_lb.KuryrLoadBalancerHandler._should_ignore( + m_handler, get_lb_crd()) + self.assertEqual(False, ret) + + m_handler._has_pods.assert_called_once_with(get_lb_crd()) + + def test_has_pods(self): + crd = get_lb_crd() + m_handler = mock.Mock(spec=h_lb.KuryrLoadBalancerHandler) + + ret = h_lb.KuryrLoadBalancerHandler._has_pods(m_handler, crd) + + self.assertEqual(True, ret) + + def test_get_pod_subnet(self): + subnet_id = mock.sentinel.subnet_id + project_id = mock.sentinel.project_id + target_ref = {'kind': k_const.K8S_OBJ_POD, + 'name': 'pod-name', + 'namespace': 'default'} + ip = '1.2.3.4' + m_handler = mock.Mock(spec=h_lb.KuryrLoadBalancerHandler) + m_drv_pod_project = mock.Mock() + m_drv_pod_project.get_project.return_value = project_id + m_handler._drv_pod_project = m_drv_pod_project + m_drv_pod_subnets = mock.Mock() + m_drv_pod_subnets.get_subnets.return_value = { + subnet_id: osv_network.Network(subnets=osv_subnet.SubnetList( + objects=[osv_subnet.Subnet(cidr='1.2.3.0/24')]))} + m_handler._drv_pod_subnets = m_drv_pod_subnets + + observed_subnet_id = h_lb.KuryrLoadBalancerHandler._get_pod_subnet( + m_handler, target_ref, ip) + + self.assertEqual(subnet_id, observed_subnet_id) + + def _sync_lbaas_members_impl(self, m_get_drv_lbaas, m_get_drv_project, + m_get_drv_subnets, subnet_id, project_id, + crd): + m_drv_lbaas = mock.Mock(wraps=FakeLBaaSDriver()) + m_drv_project = mock.Mock() + m_drv_project.get_project.return_value = project_id + m_drv_subnets = mock.Mock() + m_drv_subnets.get_subnets.return_value = { + subnet_id: mock.sentinel.subnet} + m_get_drv_lbaas.return_value = m_drv_lbaas + m_get_drv_project.return_value = m_drv_project + m_get_drv_subnets.return_value = m_drv_subnets + + handler = h_lb.KuryrLoadBalancerHandler() + + with mock.patch.object(handler, '_get_pod_subnet') as m_get_pod_subnet: + m_get_pod_subnet.return_value = subnet_id + handler._sync_lbaas_members(crd) + + lsnrs = {lsnr['id']: lsnr for lsnr in crd['status']['listeners']} + pools = {pool['id']: pool for pool in crd['status']['pools']} + observed_targets = sorted( + (str(member['ip']), ( + lsnrs[pools[member['pool_id']]['listener_id']]['port'], + member['port'])) + for member in crd['status']['members']) + return observed_targets + + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodSubnetsDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodProjectDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.LBaaSDriver.get_instance') + def test_sync_lbaas_members(self, m_get_drv_lbaas, m_get_drv_project, + m_get_drv_subnets): + # REVISIT(ivc): test methods separately and verify ensure/release + project_id = str(uuid.uuid4()) + subnet_id = str(uuid.uuid4()) + expected_ip = '1.2.3.4' + expected_targets = { + '1.1.1.1': (1, 2), + '1.1.1.1': (1, 2), + '1.1.1.1': (1, 2)} + crd = get_lb_crd() + + observed_targets = self._sync_lbaas_members_impl( + m_get_drv_lbaas, m_get_drv_project, m_get_drv_subnets, + subnet_id, project_id, crd) + + self.assertEqual(sorted(expected_targets.items()), observed_targets) + self.assertEqual(expected_ip, str(crd['status']['loadbalancer']['ip'])) + + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodSubnetsDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodProjectDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.LBaaSDriver.get_instance') + def test_sync_lbaas_members_udp(self, m_get_drv_lbaas, + m_get_drv_project, m_get_drv_subnets): + # REVISIT(ivc): test methods separately and verify ensure/release + project_id = str(uuid.uuid4()) + subnet_id = str(uuid.uuid4()) + expected_ip = "1.2.3.4" + expected_targets = { + '1.1.1.1': (1, 2), + '1.1.1.1': (1, 2), + '1.1.1.1': (1, 2)} + + crd = get_lb_crd() + + observed_targets = self._sync_lbaas_members_impl( + m_get_drv_lbaas, m_get_drv_project, m_get_drv_subnets, + subnet_id, project_id, crd) + + self.assertEqual(sorted(expected_targets.items()), observed_targets) + self.assertEqual(expected_ip, str(crd['status']['loadbalancer']['ip'])) + + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodSubnetsDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodProjectDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.LBaaSDriver.get_instance') + def test_sync_lbaas_members_svc_listener_port_edit( + self, m_get_drv_lbaas, m_get_drv_project, m_get_drv_subnets): + # REVISIT(ivc): test methods separately and verify ensure/release + project_id = str(uuid.uuid4()) + subnet_id = str(uuid.uuid4()) + expected_ip = '1.2.3.4' + crd = get_lb_crd() + + m_drv_lbaas = mock.Mock(wraps=FakeLBaaSDriver()) + m_drv_project = mock.Mock() + m_drv_project.get_project.return_value = project_id + m_drv_subnets = mock.Mock() + m_drv_subnets.get_subnets.return_value = { + subnet_id: mock.sentinel.subnet} + m_get_drv_lbaas.return_value = m_drv_lbaas + m_get_drv_project.return_value = m_drv_project + m_get_drv_subnets.return_value = m_drv_subnets + + handler = h_lb.KuryrLoadBalancerHandler() + + with mock.patch.object(handler, '_get_pod_subnet') as m_get_pod_subnet: + m_get_pod_subnet.return_value = subnet_id + handler._sync_lbaas_members(crd) + + self.assertEqual(expected_ip, str(crd['status']['loadbalancer']['ip'])) + + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodSubnetsDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.PodProjectDriver.get_instance') + @mock.patch('kuryr_kubernetes.controller.drivers.base' + '.LBaaSDriver.get_instance') + def test_add_new_members_udp(self, m_get_drv_lbaas, + m_get_drv_project, m_get_drv_subnets): + project_id = str(uuid.uuid4()) + subnet_id = str(uuid.uuid4()) + crd = get_lb_crd() + + m_drv_lbaas = mock.Mock(wraps=FakeLBaaSDriver()) + m_drv_project = mock.Mock() + m_drv_project.get_project.return_value = project_id + m_drv_subnets = mock.Mock() + m_drv_subnets.get_subnets.return_value = { + subnet_id: mock.sentinel.subnet} + m_get_drv_lbaas.return_value = m_drv_lbaas + m_get_drv_project.return_value = m_drv_project + m_get_drv_subnets.return_value = m_drv_subnets + + handler = h_lb.KuryrLoadBalancerHandler() + member_added = handler._add_new_members(crd) + + self.assertEqual(member_added, False) + m_drv_lbaas.ensure_member.assert_not_called() diff --git a/kuryr_kubernetes/tests/unit/test_utils.py b/kuryr_kubernetes/tests/unit/test_utils.py index 6044a1051..27a9fad24 100644 --- a/kuryr_kubernetes/tests/unit/test_utils.py +++ b/kuryr_kubernetes/tests/unit/test_utils.py @@ -20,7 +20,6 @@ from kuryr_kubernetes import constants as k_const from kuryr_kubernetes import exceptions as k_exc -from kuryr_kubernetes.objects import lbaas as obj_lbaas from kuryr_kubernetes.objects import vif from kuryr_kubernetes.tests import base as test_base from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix @@ -185,40 +184,120 @@ def test_get_service_ports(self): @mock.patch('kuryr_kubernetes.utils.get_service_ports') def test_has_port_changes(self, m_get_service_ports): - service = mock.MagicMock() - m_get_service_ports.return_value = [ - {'port': 1, 'name': 'X', 'protocol': 'TCP', 'targetPort': 1}, - ] - - lbaas_spec = mock.MagicMock() - lbaas_spec.ports = [ - obj_lbaas.LBaaSPortSpec(name='X', protocol='TCP', port=1, - targetPort=1), - obj_lbaas.LBaaSPortSpec(name='Y', protocol='TCP', port=2, - targetPort=2), - ] - - ret = utils.has_port_changes(service, lbaas_spec) + service = { + 'metadata': { + 'selfLink': "" + }, + 'spec': { + 'ports': [ + { + 'port': 1, + 'name': 'X', + 'protocol': 'TCP', + 'targetPort': '1' + } + ] + } + } + lb_crd_spec = { + 'spec': { + 'ports': [ + { + 'name': 'Y', + 'protocol': 'TCP', + 'port': 2, + 'targetPort': 2 + } + ] + } + } + ret = utils.has_port_changes(service, lb_crd_spec) self.assertTrue(ret) @mock.patch('kuryr_kubernetes.utils.get_service_ports') - def test_has_port_changes__no_changes(self, m_get_service_ports): - service = mock.MagicMock() - m_get_service_ports.return_value = [ - {'port': 1, 'name': 'X', 'protocol': 'TCP', 'targetPort': '1'}, - {'port': 2, 'name': 'Y', 'protocol': 'TCP', 'targetPort': '2'} - ] - - lbaas_spec = mock.MagicMock() - lbaas_spec.ports = [ - obj_lbaas.LBaaSPortSpec(name='X', protocol='TCP', port=1, - targetPort=1), - obj_lbaas.LBaaSPortSpec(name='Y', protocol='TCP', port=2, - targetPort=2), - ] - - ret = utils.has_port_changes(service, lbaas_spec) + def test_has_port_changes_more_ports(self, m_get_service_ports): + service = { + 'metadata': { + 'selfLink': "" + }, + 'spec': { + 'ports': [ + { + 'port': 1, + 'name': 'X', + 'protocol': 'TCP', + 'targetPort': '1' + } + ] + } + } + lb_crd_spec = { + 'spec': { + 'ports': [ + { + 'name': 'X', + 'protocol': 'TCP', + 'port': 1, + 'targetPort': 1 + }, + { + 'name': 'Y', + 'protocol': 'TCP', + 'port': 2, + 'targetPort': 2 + } + ] + } + } + + ret = utils.has_port_changes(service, lb_crd_spec) + self.assertTrue(ret) + + @mock.patch('kuryr_kubernetes.utils.get_service_ports') + def test_has_port_changes_no_changes(self, m_get_service_ports): + + service = { + 'metadata': { + 'selfLink': "" + }, + 'spec': { + 'ports': [ + { + 'port': 1, + 'name': 'X', + 'protocol': 'TCP', + 'targetPort': '1' + }, + { + 'name': 'Y', + 'protocol': 'TCP', + 'port': 2, + 'targetPort': '2' + } + ] + } + } + + lb_crd_spec = { + 'spec': { + 'ports': [ + { + 'name': 'X', + 'protocol': 'TCP', + 'port': 1, + 'targetPort': '1' + }, + { + 'name': 'Y', + 'protocol': 'TCP', + 'port': 2, + 'targetPort': '2' + } + ] + } + } + ret = utils.has_port_changes(service, lb_crd_spec) self.assertFalse(ret) def test_get_nodes_ips(self): diff --git a/kuryr_kubernetes/utils.py b/kuryr_kubernetes/utils.py index 651c9eb38..52f1d3dd8 100644 --- a/kuryr_kubernetes/utils.py +++ b/kuryr_kubernetes/utils.py @@ -379,24 +379,27 @@ def get_endpoints_link(service): return "/".join(link_parts) -def has_port_changes(service, lbaas_spec): +def has_port_changes(service, loadbalancer_crd): + if not loadbalancer_crd: + return False link = service['metadata']['selfLink'] - - fields = obj_lbaas.LBaaSPortSpec.fields - svc_port_set = {tuple(port[attr] for attr in fields) - for port in get_service_ports(service)} - - spec_port_set = {tuple(getattr(port, attr) - for attr in fields - if port.obj_attr_is_set(attr)) - for port in lbaas_spec.ports} - - if svc_port_set != spec_port_set: - LOG.debug("LBaaS spec ports %(spec_ports)s != %(svc_ports)s " - "for %(link)s" % {'spec_ports': spec_port_set, - 'svc_ports': svc_port_set, - 'link': link}) - return svc_port_set != spec_port_set + svc_port_set = service['spec'].get('ports') + + for port in svc_port_set: + port['targetPort'] = str(port['targetPort']) + spec_port_set = loadbalancer_crd['spec'].get('ports', []) + if spec_port_set: + if len(svc_port_set) != len(spec_port_set): + return True + pairs = zip(svc_port_set, spec_port_set) + diff = any(x != y for x, y in pairs) + if diff: + LOG.debug("LBaaS spec ports %(spec_ports)s != %(svc_ports)s " + "for %(link)s" % {'spec_ports': spec_port_set, + 'svc_ports': svc_port_set, + 'link': link}) + return diff + return False def get_service_ports(service): diff --git a/setup.cfg b/setup.cfg index a24c6447b..266fbaf17 100644 --- a/setup.cfg +++ b/setup.cfg @@ -98,8 +98,9 @@ kuryr_kubernetes.controller.drivers.vif_pool = kuryr_kubernetes.controller.handlers = vif = kuryr_kubernetes.controller.handlers.vif:VIFHandler - lbaasspec = kuryr_kubernetes.controller.handlers.lbaas:LBaaSSpecHandler - lb = kuryr_kubernetes.controller.handlers.lbaas:LoadBalancerHandler + service = kuryr_kubernetes.controller.handlers.lbaas:ServiceHandler + endpoints = kuryr_kubernetes.controller.handlers.lbaas:EndpointsHandler + kuryrloadbalancer = kuryr_kubernetes.controller.handlers.loadbalancer:KuryrLoadBalancerHandler namespace = kuryr_kubernetes.controller.handlers.namespace:NamespaceHandler policy = kuryr_kubernetes.controller.handlers.policy:NetworkPolicyHandler pod_label = kuryr_kubernetes.controller.handlers.pod_label:PodLabelHandler From a1708e1c769c7509a098b42b1a2f3eda1062b27b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Dulko?= Date: Wed, 11 Mar 2020 11:27:09 +0100 Subject: [PATCH 6/6] KuryrNetworkPolicy CRD This commit is a huge refactoring of how we handle network policies. In general: * KuryrNetPolicy is replaced by KuryrNetworkPolicy. The upgrade path is handled in the constructor of KuryrNetworkPolicyHandler. * New CRD has spec and status properties. spec is always populated by NetworkPolicyHandler. status is handled by KuryrNetworkPolicyHandler. This means that in order to trigger SG rules recalculation on Pod ang Service events, the NetworkPolicy is "bumped" with a dummy annotation. * NetworkPolicyHandler injects finalizers onto NetworkPolicy and KuryrNetworkPolicy objects, so that objects cannot get removed before KuryrNetworkPolicyHandler won't process deletion correctly. Depends-On: https://review.opendev.org/742209 Change-Id: Iafc982e590ada0cd9d82e922c103583e4304e9ce --- .zuul.d/octavia.yaml | 6 +- .zuul.d/sdn.yaml | 4 +- devstack/lib/kuryr_kubernetes | 1 + devstack/plugin.sh | 1 + doc/source/devref/network_policy.rst | 61 ++- doc/source/installation/manual.rst | 1 + doc/source/installation/network_policy.rst | 105 ++--- .../kuryr_crds/kuryrnetpolicy.yaml | 2 - .../kuryr_crds/kuryrnetworkpolicy.yaml | 158 +++++++ kuryr_kubernetes/constants.py | 6 + kuryr_kubernetes/controller/drivers/base.py | 28 +- .../controller/drivers/lbaasv2.py | 35 +- .../controller/drivers/network_policy.py | 397 ++++++++--------- .../drivers/network_policy_security_groups.py | 414 +++++------------- kuryr_kubernetes/controller/drivers/utils.py | 97 ++-- .../controller/handlers/kuryrnetpolicy.py | 37 -- .../controller/handlers/kuryrnetworkpolicy.py | 307 +++++++++++++ .../controller/handlers/pod_label.py | 42 +- .../controller/handlers/policy.py | 126 +----- .../controller/drivers/test_network_policy.py | 351 +++++---------- .../test_network_policy_security_groups.py | 320 ++++---------- .../handlers/test_kuryrnetworkpolicy.py | 112 +++++ .../controller/handlers/test_pod_label.py | 30 +- .../unit/controller/handlers/test_policy.py | 216 ++------- setup.cfg | 2 +- tools/gate/copy_k8s_logs.sh | 1 + 26 files changed, 1286 insertions(+), 1574 deletions(-) create mode 100644 kubernetes_crds/kuryr_crds/kuryrnetworkpolicy.yaml delete mode 100644 kuryr_kubernetes/controller/handlers/kuryrnetpolicy.py create mode 100644 kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py create mode 100644 kuryr_kubernetes/tests/unit/controller/handlers/test_kuryrnetworkpolicy.py diff --git a/.zuul.d/octavia.yaml b/.zuul.d/octavia.yaml index b4cd6f526..6c0b4bade 100644 --- a/.zuul.d/octavia.yaml +++ b/.zuul.d/octavia.yaml @@ -99,7 +99,7 @@ vars: devstack_localrc: DOCKER_CGROUP_DRIVER: "systemd" - KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer + KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetworkpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer KURYR_SG_DRIVER: policy KURYR_SUBNET_DRIVER: namespace devstack_services: @@ -120,7 +120,7 @@ vars: devstack_localrc: KURYR_SUBNET_DRIVER: namespace - KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer + KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetworkpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer KURYR_SG_DRIVER: policy KURYR_USE_PORT_POOLS: true KURYR_POD_VIF_DRIVER: neutron-vif @@ -134,7 +134,7 @@ parent: kuryr-kubernetes-tempest-containerized vars: devstack_localrc: - KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer + KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetworkpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer KURYR_SG_DRIVER: policy KURYR_SUBNET_DRIVER: namespace diff --git a/.zuul.d/sdn.yaml b/.zuul.d/sdn.yaml index cfda1547d..184cb1b61 100644 --- a/.zuul.d/sdn.yaml +++ b/.zuul.d/sdn.yaml @@ -98,7 +98,7 @@ KURYR_LB_ALGORITHM: SOURCE_IP_PORT KURYR_SUBNET_DRIVER: namespace KURYR_SG_DRIVER: policy - KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer + KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetworkpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer voting: false - job: @@ -144,7 +144,7 @@ KURYR_ENFORCE_SG_RULES: false KURYR_LB_ALGORITHM: SOURCE_IP_PORT KURYR_HYPERKUBE_VERSION: v1.16.0 - KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer + KURYR_ENABLED_HANDLERS: vif,endpoints,service,namespace,pod_label,policy,kuryrnetworkpolicy,kuryrnetwork,kuryrport,kuryrloadbalancer KURYR_SG_DRIVER: policy KURYR_SUBNET_DRIVER: namespace KURYR_K8S_CONTAINERIZED_DEPLOYMENT: true diff --git a/devstack/lib/kuryr_kubernetes b/devstack/lib/kuryr_kubernetes index e51686811..8f20b65d8 100644 --- a/devstack/lib/kuryr_kubernetes +++ b/devstack/lib/kuryr_kubernetes @@ -452,6 +452,7 @@ rules: - kuryrnets - kuryrnetworks - kuryrnetpolicies + - kuryrnetworkpolicies - kuryrloadbalancers - kuryrports - apiGroups: ["networking.k8s.io"] diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 3ca7665af..2fce77cd9 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -975,6 +975,7 @@ function update_tempest_conf_file { iniset $TEMPEST_CONFIG kuryr_kubernetes kuryrnetworks True iniset $TEMPEST_CONFIG kuryr_kubernetes kuryrports True iniset $TEMPEST_CONFIG kuryr_kubernetes kuryrloadbalancers True + iniset $TEMPEST_CONFIG kuryr_kubernetes new_kuryrnetworkpolicy_crd True } source $DEST/kuryr-kubernetes/devstack/lib/kuryr_kubernetes diff --git a/doc/source/devref/network_policy.rst b/doc/source/devref/network_policy.rst index 8f4cd2564..cd43036c7 100644 --- a/doc/source/devref/network_policy.rst +++ b/doc/source/devref/network_policy.rst @@ -47,22 +47,22 @@ The network policy CRD has the following format: .. code-block:: yaml apiVersion: openstack.org/v1 - kind: KuryrNetPolicy + kind: KuryrNetworkPolicy metadata: ... spec: egressSgRules: - - security_group_rule: + - sgRule: ... ingressSgRules: - - security_group_rule: - ... - networkpolicy_spec: + - sgRule: ... podSelector: ... + status: securityGroupId: ... - securityGroupName: ... + podSelector: ... + securityGroupRules: ... A new handler has been added to react to Network Policy events, and the existing ones, for instance service/pod handlers, have been modified to account for the @@ -201,26 +201,25 @@ are assumed to assumed to affect Ingress. .. code-block:: yaml apiVersion: openstack.org/v1 - kind: KuryrNetPolicy + kind: KuryrNetworkPolicy metadata: - name: np-default-deny + name: default-deny namespace: default ... spec: egressSgRules: - - security_group_rule: + - sgRule: description: Kuryr-Kubernetes NetPolicy SG rule direction: egress ethertype: IPv4 - id: 60a0d59c-2102-43e0-b025-75c98b7d9315 security_group_id: 20d9b623-f1e0-449d-95c1-01624cb3e315 ingressSgRules: [] - networkpolicy_spec: - ... podSelector: ... + status: securityGroupId: 20d9b623-f1e0-449d-95c1-01624cb3e315 - securityGroupName: sg-default-deny + securityGroupRules: ... + podSelector: ... Allow traffic from pod @@ -263,37 +262,33 @@ restriction was enforced. .. code-block:: yaml apiVersion: openstack.org/v1 - kind: KuryrNetPolicy + kind: KuryrNetworkPolicy metadata: - name: np-allow-monitoring-via-pod-selector + name: allow-monitoring-via-pod-selector namespace: default ... spec: egressSgRules: - - security_group_rule: + - sgRule: description: Kuryr-Kubernetes NetPolicy SG rule direction: egress ethertype: IPv4 - id: 203a14fe-1059-4eff-93ed-a42bd957145d - security_group_id: 7f0ef8c2-4846-4d8c-952f-94a9098fff17 ingressSgRules: - namespace: default - security_group_rule: + sgRule: description: Kuryr-Kubernetes NetPolicy SG rule direction: ingress ethertype: IPv4 - id: 7987c382-f2a9-47f7-b6e8-1a3a1bcb7d95 port_range_max: 8080 port_range_min: 8080 protocol: tcp remote_ip_prefix: 10.0.1.143 - security_group_id: 7f0ef8c2-4846-4d8c-952f-94a9098fff17 - networkpolicy_spec: - ... podSelector: ... + status: securityGroupId: 7f0ef8c2-4846-4d8c-952f-94a9098fff17 - securityGroupName: sg-allow-monitoring-via-pod-selector + securityGroupRules: ... + podSelector: ... Allow traffic from namespace @@ -337,36 +332,32 @@ egress rule allowing traffic to everywhere. .. code-block:: yaml apiVersion: openstack.org/v1 - kind: KuryrNetPolicy - name: np-allow-test-via-ns-selector + kind: KuryrNetworkPolicy + name: allow-test-via-ns-selector namespace: default ... spec: egressSgRules: - - security_group_rule: + - sgRule: description: Kuryr-Kubernetes NetPolicy SG rule direction: egress ethertype: IPv4 - id: 8c21bf42-c8b9-4628-b0a1-bd0dbb192e6b - security_group_id: c480327c-2db4-4eb6-af1e-eeb0ce9b46c9 ingressSgRules: - namespace: dev - security_group_rule: + sgRule: description: Kuryr-Kubernetes NetPolicy SG rule direction: ingress ethertype: IPv4 - id: 2a33b802-56ad-430a-801d-690f653198ef port_range_max: 8080 port_range_min: 8080 protocol: tcp remote_ip_prefix: 10.0.1.192/26 - security_group_id: c480327c-2db4-4eb6-af1e-eeb0ce9b46c9 - networkpolicy_spec: - ... podSelector: ... + status: securityGroupId: c480327c-2db4-4eb6-af1e-eeb0ce9b46c9 - securityGroupName: sg-allow-test-via-ns-selector + securityGroupRules: ... + podSelector: ... .. note:: diff --git a/doc/source/installation/manual.rst b/doc/source/installation/manual.rst index 2a4823f5f..c3992e36b 100644 --- a/doc/source/installation/manual.rst +++ b/doc/source/installation/manual.rst @@ -95,6 +95,7 @@ Edit ``kuryr.conf``: - kuryrnets - kuryrnetworks - kuryrnetpolicies + - kuryrnetworkpolicies - kuryrloadbalancers - apiGroups: ["networking.k8s.io"] resources: diff --git a/doc/source/installation/network_policy.rst b/doc/source/installation/network_policy.rst index 6917c9e29..bdb8de446 100644 --- a/doc/source/installation/network_policy.rst +++ b/doc/source/installation/network_policy.rst @@ -10,7 +10,7 @@ be found at :doc:`./devstack/containerized`): .. code-block:: ini [kubernetes] - enabled_handlers=vif,lb,lbaasspec,policy,pod_label,namespace,kuryrnetwork,kuryrnetpolicy + enabled_handlers=vif,lb,lbaasspec,policy,pod_label,namespace,kuryrnetwork,kuryrnetworkpolicy Note that if you also want to enable prepopulation of ports pools upon new namespace creation, you need to also dd the kuryrnetwork_population handler @@ -19,7 +19,7 @@ namespace creation, you need to also dd the kuryrnetwork_population handler .. code-block:: ini [kubernetes] - enabled_handlers=vif,lb,lbaasspec,policy,pod_label,namespace,kuryrnetpolicy,kuryrnetwork,kuryrnetwork_population + enabled_handlers=vif,lb,lbaasspec,policy,pod_label,namespace,kuryrnetworkpolicy,kuryrnetwork,kuryrnetwork_population After that, enable also the security group drivers for policies: @@ -82,7 +82,7 @@ to add the policy, pod_label and namespace handler and drivers with: .. code-block:: bash - KURYR_ENABLED_HANDLERS=vif,lb,lbaasspec,policy,pod_label,namespace,kuryrnetpolicy + KURYR_ENABLED_HANDLERS=vif,lb,lbaasspec,policy,pod_label,namespace,kuryrnetworkpolicy KURYR_SG_DRIVER=policy KURYR_SUBNET_DRIVER=namespace @@ -143,9 +143,9 @@ Testing the network policy support functionality .. code-block:: console - $ kubectl get kuryrnetpolicies + $ kubectl get kuryrnetworkpolicies NAME AGE - np-test-network-policy 2s + test-network-policy 2s $ kubectl get networkpolicies NAME POD-SELECTOR AGE @@ -158,69 +158,42 @@ Testing the network policy support functionality .. code-block:: console - $ kubectl get kuryrnetpolicy np-test-network-policy -o yaml + $ kubectl get kuryrnetworkpolicy test-network-policy -o yaml apiVersion: openstack.org/v1 - kind: KuryrNetPolicy + kind: KuryrNetworkPolicy metadata: annotations: - networkpolicy_name: test-network-policy - networkpolicy_namespace: default - networkpolicy_uid: aee1c59f-c634-11e8-b63d-002564fdd760 + networkPolicyLink: clusterName: "" creationTimestamp: 2018-10-02T11:17:02Z generation: 0 - name: np-test-network-policy + name: test-network-policy namespace: default resourceVersion: "2117" - selfLink: /apis/openstack.org/v1/namespaces/default/kuryrnetpolicies/np-test-network-policy + selfLink: /apis/openstack.org/v1/namespaces/default/kuryrnetworkpolicies/test-network-policy uid: afb99326-c634-11e8-b63d-002564fdd760 spec: egressSgRules: - - security_group_rule: + - sgRule: description: Kuryr-Kubernetes NetPolicy SG rule direction: egress ethertype: IPv4 - id: 6297c198-b385-44f3-8b43-29951f933a8f port_range_max: 5978 port_range_min: 5978 protocol: tcp - security_group_id: cdee7815-3b49-4a3e-abc8-31e384ab75c5 ingressSgRules: - - security_group_rule: + - sgRule: description: Kuryr-Kubernetes NetPolicy SG rule direction: ingress ethertype: IPv4 - id: f4e11e73-81c6-4c1b-9760-714eedff417b port_range_max: 6379 port_range_min: 6379 protocol: tcp - security_group_id: cdee7815-3b49-4a3e-abc8-31e384ab75c5 + status: securityGroupId: cdee7815-3b49-4a3e-abc8-31e384ab75c5 - securityGroupName: sg-test-network-policy - networkpolicy_spec: - egress: - - to: - - namespaceSelector: - matchLabels: - project: default - ports: - - port: 5978 - protocol: TCP - ingress: - - from: - - namespaceSelector: - matchLabels: - project: default - ports: - - port: 6379 - protocol: TCP - podSelector: - matchLabels: - project: default - policyTypes: - - Ingress - - Egress + securityGroupRules: + … $ openstack security group rule list sg-test-network-policy --protocol tcp -c "IP Protocol" -c "Port Range" -c "Direction" --long +-------------+------------+-----------+ @@ -273,67 +246,41 @@ Testing the network policy support functionality $ kubectl patch networkpolicy test-network-policy -p '{"spec":{"ingress":[{"ports":[{"port": 8080,"protocol": "TCP"}]}]}}' networkpolicy "test-network-policy" patched - $ kubectl get knp np-test-network-policy -o yaml + $ kubectl get knp test-network-policy -o yaml apiVersion: openstack.org/v1 - kind: KuryrNetPolicy + kind: KuryrNetworkPolicy metadata: annotations: - networkpolicy_name: test-network-policy - networkpolicy_namespace: default - networkpolicy_uid: aee1c59f-c634-11e8-b63d-002564fdd760 + networkPolicyLink: clusterName: "" creationTimestamp: 2018-10-02T11:17:02Z generation: 0 - name: np-test-network-policy + name: test-network-policy namespace: default resourceVersion: "1546" - selfLink: /apis/openstack.org/v1/namespaces/default/kuryrnetpolicies/np-test-network-policy + selfLink: /apis/openstack.org/v1/namespaces/default/kuryrnetworkpolicies/np-test-network-policy uid: afb99326-c634-11e8-b63d-002564fdd760 spec: egressSgRules: - - security_group_rule: + - sgRule: description: Kuryr-Kubernetes NetPolicy SG rule direction: egress ethertype: IPv4 - id: 1969a0b3-55e1-43d7-ba16-005b4ed4cbb7 port_range_max: 5978 port_range_min: 5978 protocol: tcp - security_group_id: cdee7815-3b49-4a3e-abc8-31e384ab75c5 ingressSgRules: - - security_group_rule: + - sgRule: description: Kuryr-Kubernetes NetPolicy SG rule direction: ingress ethertype: IPv4 - id: 6598aa1f-4f94-4fb2-81ce-d3649ba28f33 port_range_max: 8080 port_range_min: 8080 protocol: tcp - security_group_id: cdee7815-3b49-4a3e-abc8-31e384ab75c5 + status: securityGroupId: cdee7815-3b49-4a3e-abc8-31e384ab75c5 - networkpolicy_spec: - egress: - - ports: - - port: 5978 - protocol: TCP - to: - - namespaceSelector: - matchLabels: - project: default - ingress: - - ports: - - port: 8080 - protocol: TCP - from: - - namespaceSelector: - matchLabels: - project: default - podSelector: - matchLabels: - project: default - policyTypes: - - Ingress - - Egress + securityGroupRules: + … $ openstack security group rule list sg-test-network-policy -c "IP Protocol" -c "Port Range" -c "Direction" --long +-------------+------------+-----------+ @@ -388,6 +335,6 @@ Testing the network policy support functionality .. code-block:: console $ kubectl delete -f network_policy.yml - $ kubectl get kuryrnetpolicies + $ kubectl get kuryrnetworkpolicies $ kubectl get networkpolicies $ openstack security group list | grep sg-test-network-policy diff --git a/kubernetes_crds/kuryr_crds/kuryrnetpolicy.yaml b/kubernetes_crds/kuryr_crds/kuryrnetpolicy.yaml index dc0a7e07b..ccf3f942a 100644 --- a/kubernetes_crds/kuryr_crds/kuryrnetpolicy.yaml +++ b/kubernetes_crds/kuryr_crds/kuryrnetpolicy.yaml @@ -9,8 +9,6 @@ spec: plural: kuryrnetpolicies singular: kuryrnetpolicy kind: KuryrNetPolicy - shortNames: - - knp versions: - name: v1 served: true diff --git a/kubernetes_crds/kuryr_crds/kuryrnetworkpolicy.yaml b/kubernetes_crds/kuryr_crds/kuryrnetworkpolicy.yaml new file mode 100644 index 000000000..3726409d8 --- /dev/null +++ b/kubernetes_crds/kuryr_crds/kuryrnetworkpolicy.yaml @@ -0,0 +1,158 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kuryrnetworkpolicies.openstack.org +spec: + group: openstack.org + scope: Namespaced + names: + plural: kuryrnetworkpolicies + singular: kuryrnetworkpolicy + kind: KuryrNetworkPolicy + shortNames: + - knp + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: SG-ID + type: string + description: The ID of the SG associated to the policy + jsonPath: .status.securityGroupId + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + required: + - status + - spec + properties: + spec: + type: object + required: + - egressSgRules + - ingressSgRules + - podSelector + - policyTypes + properties: + egressSgRules: + type: array + items: + type: object + required: + - sgRule + properties: + affectedPods: + type: array + items: + type: object + properties: + podIP: + type: string + podNamespace: + type: string + required: + - podIP + - podNamespace + namespace: + type: string + sgRule: + type: object + properties: + description: + type: string + direction: + type: string + ethertype: + type: string + port_range_max: + type: integer + port_range_min: + type: integer + protocol: + type: string + remote_ip_prefix: + type: string + ingressSgRules: + type: array + items: + type: object + required: + - sgRule + properties: + affectedPods: + type: array + items: + type: object + properties: + podIP: + type: string + podNamespace: + type: string + required: + - podIP + - podNamespace + namespace: + type: string + sgRule: + type: object + properties: + description: + type: string + direction: + type: string + ethertype: + type: string + port_range_max: + type: integer + port_range_min: + type: integer + protocol: + type: string + remote_ip_prefix: + type: string + podSelector: + x-kubernetes-preserve-unknown-fields: true + type: object + policyTypes: + type: array + items: + type: string + status: + type: object + required: + - securityGroupRules + properties: + securityGroupId: + type: string + securityGroupRules: + type: array + items: + type: object + required: + - id + properties: + id: + type: string + description: + type: string + direction: + type: string + ethertype: + type: string + port_range_max: + type: integer + port_range_min: + type: integer + protocol: + type: string + remote_ip_prefix: + type: string + security_group_id: + type: string + podSelector: + x-kubernetes-preserve-unknown-fields: true + type: object diff --git a/kuryr_kubernetes/constants.py b/kuryr_kubernetes/constants.py index 4b1fddc5d..3a1732530 100644 --- a/kuryr_kubernetes/constants.py +++ b/kuryr_kubernetes/constants.py @@ -23,9 +23,11 @@ K8S_API_CRD_KURYRNETS = K8S_API_CRD + '/kuryrnets' K8S_API_CRD_KURYRNETWORKS = K8S_API_CRD + '/kuryrnetworks' K8S_API_CRD_KURYRNETPOLICIES = K8S_API_CRD + '/kuryrnetpolicies' +K8S_API_CRD_KURYRNETWORKPOLICIES = K8S_API_CRD + '/kuryrnetworkpolicies' K8S_API_CRD_KURYRLOADBALANCERS = K8S_API_CRD + '/kuryrloadbalancers' K8S_API_CRD_KURYRPORTS = K8S_API_CRD + '/kuryrports' K8S_API_POLICIES = '/apis/networking.k8s.io/v1/networkpolicies' +K8S_API_NETWORKING = '/apis/networking.k8s.io/v1' K8S_API_NPWG_CRD = '/apis/k8s.cni.cncf.io/v1' @@ -37,6 +39,7 @@ K8S_OBJ_KURYRNET = 'KuryrNet' K8S_OBJ_KURYRNETWORK = 'KuryrNetwork' K8S_OBJ_KURYRNETPOLICY = 'KuryrNetPolicy' +K8S_OBJ_KURYRNETWORKPOLICY = 'KuryrNetworkPolicy' K8S_OBJ_KURYRLOADBALANCER = 'KuryrLoadBalancer' K8S_OBJ_KURYRPORT = 'KuryrPort' @@ -47,11 +50,13 @@ K8S_ANNOTATION_PREFIX = 'openstack.org/kuryr' K8S_ANNOTATION_VIF = K8S_ANNOTATION_PREFIX + '-vif' K8S_ANNOTATION_LABEL = K8S_ANNOTATION_PREFIX + '-pod-label' +K8S_ANNOTATION_IP = K8S_ANNOTATION_PREFIX + '-pod-ip' K8S_ANNOTATION_NAMESPACE_LABEL = K8S_ANNOTATION_PREFIX + '-namespace-label' K8S_ANNOTATION_LBAAS_SPEC = K8S_ANNOTATION_PREFIX + '-lbaas-spec' K8S_ANNOTATION_LBAAS_STATE = K8S_ANNOTATION_PREFIX + '-lbaas-state' K8S_ANNOTATION_NET_CRD = K8S_ANNOTATION_PREFIX + '-net-crd' K8S_ANNOTATION_NETPOLICY_CRD = K8S_ANNOTATION_PREFIX + '-netpolicy-crd' +K8S_ANNOTATION_POLICY = K8S_ANNOTATION_PREFIX + '-counter' K8S_ANNOTATION_NPWG_PREFIX = 'k8s.v1.cni.cncf.io' K8S_ANNOTATION_NPWG_NETWORK = K8S_ANNOTATION_NPWG_PREFIX + '/networks' @@ -68,6 +73,7 @@ KURYRNETWORK_FINALIZER = 'kuryrnetwork.finalizers.kuryr.openstack.org' KURYRLB_FINALIZER = 'kuryr.openstack.org/kuryrloadbalancer-finalizers' SERVICE_FINALIZER = 'kuryr.openstack.org/service-finalizer' +NETWORKPOLICY_FINALIZER = 'kuryr.openstack.org/networkpolicy-finalizer' KURYRPORT_FINALIZER = KURYR_FQDN + '/kuryrport-finalizer' KURYRPORT_LABEL = KURYR_FQDN + '/nodeName' diff --git a/kuryr_kubernetes/controller/drivers/base.py b/kuryr_kubernetes/controller/drivers/base.py index 7121044b4..455d746c9 100644 --- a/kuryr_kubernetes/controller/drivers/base.py +++ b/kuryr_kubernetes/controller/drivers/base.py @@ -697,13 +697,10 @@ class NetworkPolicyDriver(DriverBase, metaclass=abc.ABCMeta): ALIAS = 'network_policy' @abc.abstractmethod - def ensure_network_policy(self, policy, project_id): + def ensure_network_policy(self, policy): """Policy created or updated :param policy: dict containing Kubernetes NP object - :param project_id: openstack project_id - :returns: list of Pod objects affected by the network policy - creation or its podSelector modification """ raise NotImplementedError() @@ -711,7 +708,7 @@ def ensure_network_policy(self, policy, project_id): def release_network_policy(self, kuryrnetpolicy): """Delete a network policy - :param kuryrnetpolicy: dict containing Kuryrnetpolicy CRD object + :param kuryrnetpolicy: dict containing NetworkPolicy object """ raise NotImplementedError() @@ -729,18 +726,6 @@ def affected_pods(self, policy, selector=None): """ raise NotImplementedError() - @abc.abstractmethod - def knps_on_namespace(self, namespace): - """Check if there si kuryr network policy CRDs on the namespace - - This method returns true if there are knps on the specified namespace - or false otherwise - - :param namespace: namespace name where the knps CRDs should be - :returns: true if knps CRDs on the namespace, false otherwise - """ - raise NotImplementedError() - @abc.abstractmethod def namespaced_pods(self, policy): """Return pods on the policy namespace @@ -752,15 +737,6 @@ def namespaced_pods(self, policy): """ raise NotImplementedError() - @abc.abstractmethod - def get_kuryrnetpolicy_crd(self, policy): - """Return kuryrnetpolicy CRD object associated to the policy - - :param policy: dict containing Kubernetes NP object - :returns: kuryrnetpolicy CRD object associated to the policy - """ - raise NotImplementedError() - class NetworkPolicyProjectDriver(DriverBase, metaclass=abc.ABCMeta): """Get an OpenStack project id for K8s network policies""" diff --git a/kuryr_kubernetes/controller/drivers/lbaasv2.py b/kuryr_kubernetes/controller/drivers/lbaasv2.py index aeae61ece..4eca391d8 100644 --- a/kuryr_kubernetes/controller/drivers/lbaasv2.py +++ b/kuryr_kubernetes/controller/drivers/lbaasv2.py @@ -749,24 +749,41 @@ def update_lbaas_sg(self, service, sgs): endpoints_link = utils.get_endpoints_link(service) k8s = clients.get_kubernetes_client() try: - endpoint = k8s.get(endpoints_link) + k8s.get(endpoints_link) except k_exc.K8sResourceNotFound: LOG.debug("Endpoint not Found. Skipping LB SG update for " "%s as the LB resources are not present", lbaas_name) return - lbaas = utils.get_lbaas_state(endpoint) - if not lbaas: - LOG.debug('Endpoint not yet annotated with lbaas state.') + try: + klb = k8s.get(f'{k_const.K8S_API_CRD_NAMESPACES}/{svc_namespace}/' + f'kuryrloadbalancers/{svc_name}') + except k_exc.K8sResourceNotFound: + LOG.debug('No KuryrLoadBalancer for service %s created yet.', + lbaas_name) + raise k_exc.ResourceNotReady(svc_name) + + if (not klb.get('status', {}).get('loadbalancer') or + klb.get('status', {}).get('listeners') is None): + LOG.debug('KuryrLoadBalancer for service %s not populated yet.', + lbaas_name) raise k_exc.ResourceNotReady(svc_name) - lbaas_obj = lbaas.loadbalancer - lbaas_obj.security_groups = sgs + klb['status']['loadbalancer']['security_groups'] = sgs - utils.set_lbaas_state(endpoint, lbaas) + lb = klb['status']['loadbalancer'] + try: + k8s.patch_crd('status/loadbalancer', klb['metadata']['selfLink'], + {'security_groups': sgs}) + except k_exc.K8sResourceNotFound: + LOG.debug('KuryrLoadBalancer CRD not found %s', lbaas_name) + return + except k_exc.K8sClientException: + LOG.exception('Error updating KuryLoadBalancer CRD %s', lbaas_name) + raise lsnr_ids = {(listener['protocol'], listener['port']): listener['id'] - for listener in lbaas.listeners} + for listener in klb['status']['listeners']} for port in svc_ports: port_protocol = port['protocol'] @@ -779,6 +796,6 @@ def update_lbaas_sg(self, service, sgs): "%s and port %s. Skipping", port_protocol, lbaas_port) continue - self._apply_members_security_groups(lbaas_obj, lbaas_port, + self._apply_members_security_groups(lb, lbaas_port, target_port, port_protocol, sg_rule_name, listener_id, sgs) diff --git a/kuryr_kubernetes/controller/drivers/network_policy.py b/kuryr_kubernetes/controller/drivers/network_policy.py index a8089e9eb..b92da637e 100644 --- a/kuryr_kubernetes/controller/drivers/network_policy.py +++ b/kuryr_kubernetes/controller/drivers/network_policy.py @@ -38,91 +38,94 @@ def __init__(self): self.os_net = clients.get_network_client() self.kubernetes = clients.get_kubernetes_client() - def ensure_network_policy(self, policy, project_id): + def ensure_network_policy(self, policy): """Create security group rules out of network policies Triggered by events from network policies, this method ensures that - security groups and security group rules are created or updated in - reaction to kubernetes network policies events. - - In addition it returns the pods affected by the policy: - - Creation: pods on the namespace of the created policy - - Update: pods that needs to be updated in case of PodSelector - modification, i.e., the pods that were affected by the previous - PodSelector + KuryrNetworkPolicy object is created with the security group rules + definitions required to represent the NetworkPolicy. """ LOG.debug("Creating network policy %s", policy['metadata']['name']) - if self.get_kuryrnetpolicy_crd(policy): - previous_selector = ( - self.update_security_group_rules_from_network_policy(policy)) - if previous_selector or previous_selector == {}: - return self.affected_pods(policy, previous_selector) - if previous_selector is None: - return self.namespaced_pods(policy) + i_rules, e_rules = self._get_security_group_rules_from_network_policy( + policy) + + knp = self._get_knp_crd(policy) + if not knp: + self._create_knp_crd(policy, i_rules, e_rules) else: - self.create_security_group_rules_from_network_policy(policy, - project_id) + self._patch_knp_crd(policy, i_rules, e_rules, knp) + + def _convert_old_sg_rule(self, rule): + del rule['security_group_rule']['id'] + del rule['security_group_rule']['security_group_id'] + result = { + 'sgRule': rule['security_group_rule'], + } + + if 'namespace' in rule: + result['namespace'] = rule['namespace'] + + if 'remote_ip_prefixes' in rule: + result['affectedPods'] = [] + for ip, namespace in rule['remote_ip_prefixes']: + result['affectedPods'].append({ + 'podIP': ip, + 'podNamespace': namespace, + }) + + return result + + def get_from_old_crd(self, netpolicy): + name = netpolicy['metadata']['name'][3:] # Remove 'np-' + namespace = netpolicy['metadata']['namespace'] + link = (f'{constants.K8S_API_NETWORKING}/namespaces/{namespace}/' + f'networkpolicies/{name}') + knp = { + 'apiVersion': constants.K8S_API_CRD_VERSION, + 'kind': constants.K8S_OBJ_KURYRNETWORKPOLICY, + 'metadata': { + 'namespace': namespace, + 'name': name, + 'annotations': { + 'networkPolicyLink': link, + }, + 'finalizers': [constants.NETWORKPOLICY_FINALIZER], + }, + 'spec': { + 'podSelector': + netpolicy['spec']['networkpolicy_spec']['podSelector'], + 'egressSgRules': [self._convert_old_sg_rule(r) for r in + netpolicy['spec']['egressSgRules']], + 'ingressSgRules': [self._convert_old_sg_rule(r) for r in + netpolicy['spec']['ingressSgRules']], + 'policyTypes': + netpolicy['spec']['networkpolicy_spec']['policyTypes'], + }, + 'status': { + 'podSelector': netpolicy['spec']['podSelector'], + 'securityGroupId': netpolicy['spec']['securityGroupId'], + # We'll just let KuryrNetworkPolicyHandler figure out if rules + # are created on its own. + 'securityGroupRules': [], + }, + } + + return knp - def update_security_group_rules_from_network_policy(self, policy): - """Update security group rules + def _get_security_group_rules_from_network_policy(self, policy): + """Get security group rules required to represent an NP - This method updates security group rules based on CRUD events gotten - from a configuration or patch to an existing network policy + This method creates the security group rules bodies coming out of a + network policies' parsing. """ - crd = self.get_kuryrnetpolicy_crd(policy) - crd_name = crd['metadata']['name'] - LOG.debug("Already existing CRD %s", crd_name) - sg_id = crd['spec']['securityGroupId'] - # Fetch existing SG rules from kuryrnetpolicy CRD - existing_sg_rules = [] - existing_i_rules = crd['spec'].get('ingressSgRules') - existing_e_rules = crd['spec'].get('egressSgRules') - if existing_i_rules or existing_e_rules: - existing_sg_rules = existing_i_rules + existing_e_rules - existing_pod_selector = crd['spec'].get('podSelector') - # Parse network policy update and get new ruleset - i_rules, e_rules = self.parse_network_policy_rules(policy, sg_id) - current_sg_rules = i_rules + e_rules - # Get existing security group rules ids - sgr_ids = [x['security_group_rule'].pop('id') for x in - existing_sg_rules] - # SG rules that are meant to be kept get their id back - sg_rules_to_keep = [existing_sg_rules.index(rule) for rule in - existing_sg_rules if rule in current_sg_rules] - for sg_rule in sg_rules_to_keep: - sgr_id = sgr_ids[sg_rule] - existing_sg_rules[sg_rule]['security_group_rule']['id'] = sgr_id - # Delete SG rules that are no longer in the updated policy - sg_rules_to_delete = [existing_sg_rules.index(rule) for rule in - existing_sg_rules if rule not in - current_sg_rules] - for sg_rule in sg_rules_to_delete: - driver_utils.delete_security_group_rule(sgr_ids[sg_rule]) - # Create new rules that weren't already on the security group - sg_rules_to_add = [rule for rule in current_sg_rules if rule not in - existing_sg_rules] - for sg_rule in sg_rules_to_add: - sgr_id = driver_utils.create_security_group_rule(sg_rule) - if sg_rule['security_group_rule'].get('direction') == 'ingress': - for i_rule in i_rules: - if sg_rule == i_rule: - i_rule["security_group_rule"]["id"] = sgr_id - else: - for e_rule in e_rules: - if sg_rule == e_rule: - e_rule["security_group_rule"]["id"] = sgr_id - # Annotate kuryrnetpolicy CRD with current policy and ruleset - pod_selector = policy['spec'].get('podSelector') - driver_utils.patch_kuryrnetworkpolicy_crd(crd, i_rules, e_rules, - pod_selector, - np_spec=policy['spec']) + i_rules, e_rules = self.parse_network_policy_rules(policy) + # Add default rules to allow traffic from host and svc subnet + i_rules += self._get_default_np_rules() - if existing_pod_selector != pod_selector: - return existing_pod_selector - return False + return i_rules, e_rules - def _add_default_np_rules(self, sg_id): + def _get_default_np_rules(self): """Add extra SG rule to allow traffic from svcs and host. This method adds the base security group rules for the NP security @@ -130,6 +133,7 @@ def _add_default_np_rules(self, sg_id): - Ensure traffic is allowed from the services subnet - Ensure traffic is allowed from the host """ + rules = [] default_cidrs = [] if CONF.octavia_defaults.enforce_sg_rules: default_cidrs.append(utils.get_subnet_cidr( @@ -141,27 +145,21 @@ def _add_default_np_rules(self, sg_id): ethertype = constants.IPv4 if ipaddress.ip_network(cidr).version == constants.IP_VERSION_6: ethertype = constants.IPv6 - default_rule = { - 'security_group_rule': { + rules.append({ + 'sgRule': { 'ethertype': ethertype, - 'security_group_id': sg_id, 'direction': 'ingress', 'description': 'Kuryr-Kubernetes NetPolicy SG rule', - 'remote_ip_prefix': cidr - }} - driver_utils.create_security_group_rule(default_rule) + 'remote_ip_prefix': cidr, + }}) - def create_security_group_rules_from_network_policy(self, policy, - project_id): - """Create initial security group and rules + return rules - This method creates the initial security group for hosting security - group rules coming out of network policies' parsing. - """ - sg_name = ("sg-" + policy['metadata']['namespace'] + "-" + - policy['metadata']['name']) - desc = "Kuryr-Kubernetes NetPolicy SG" - sg = None + def create_security_group(self, knp, project_id): + sg_name = ("sg-" + knp['metadata']['namespace'] + "-" + + knp['metadata']['name']) + desc = ("Kuryr-Kubernetes Network Policy %s SG" % + utils.get_res_unique_name(knp)) try: # Create initial security group sg = self.os_net.create_security_group(name=sg_name, @@ -176,46 +174,14 @@ def create_security_group_rules_from_network_policy(self, policy, # rules just after creation. for sgr in sg.security_group_rules: self.os_net.delete_security_group_rule(sgr['id']) - - i_rules, e_rules = self.parse_network_policy_rules(policy, sg.id) - for i_rule in i_rules: - sgr_id = driver_utils.create_security_group_rule(i_rule) - i_rule['security_group_rule']['id'] = sgr_id - - for e_rule in e_rules: - sgr_id = driver_utils.create_security_group_rule(e_rule) - e_rule['security_group_rule']['id'] = sgr_id - - # Add default rules to allow traffic from host and svc subnet - self._add_default_np_rules(sg.id) except (os_exc.SDKException, exceptions.ResourceNotReady): LOG.exception("Error creating security group for network policy " - " %s", policy['metadata']['name']) - # If there's any issue creating sg rules, remove them - if sg: - self.os_net.delete_security_group(sg.id) + " %s", knp['metadata']['name']) raise - try: - self._add_kuryrnetpolicy_crd(policy, project_id, sg.id, i_rules, - e_rules) - except exceptions.K8sClientException: - LOG.exception("Rolling back security groups") - # Same with CRD creation - self.os_net.delete_security_group(sg.id) - raise - - try: - crd = self.get_kuryrnetpolicy_crd(policy) - self.kubernetes.annotate(policy['metadata']['selfLink'], - {"kuryrnetpolicy_selfLink": - crd['metadata']['selfLink']}) - except exceptions.K8sClientException: - LOG.exception('Error annotating network policy') - raise + return sg.id - def _get_pods(self, pod_selector, namespace=None, - namespace_selector=None): + def _get_pods(self, pod_selector, namespace=None, namespace_selector=None): matching_pods = {"items": []} if namespace_selector: matching_namespaces = driver_utils.get_namespaces( @@ -232,7 +198,6 @@ def _get_namespaces(self, namespace_selector, namespace=None): if not namespace_selector and namespace: matching_namespaces.append(self.kubernetes.get( '{}/namespaces/{}'.format(constants.K8S_API_BASE, namespace))) - else: matching_namespaces.extend(driver_utils.get_namespaces( namespace_selector).get('items')) @@ -285,7 +250,7 @@ def _parse_selectors(self, rule_block, rule_direction, policy_namespace): def _create_sg_rules_with_container_ports( self, container_ports, allow_all, resource, matched_pods, - crd_rules, sg_id, direction, port, pod_selector=None, + crd_rules, direction, port, pod_selector=None, policy_namespace=None): cidr, ns = self._get_resource_details(resource) for pod, container_port in container_ports: @@ -308,18 +273,18 @@ def _create_sg_rules_with_container_ports( if not allow_all and matched_pods and cidr: for container_port, pods in matched_pods.items(): sg_rule = driver_utils.create_security_group_rule_body( - sg_id, direction, container_port, + direction, container_port, protocol=port.get('protocol'), cidr=cidr, pods=pods) if sg_rule not in crd_rules: crd_rules.append(sg_rule) if direction == 'egress': self._create_svc_egress_sg_rule( - sg_id, policy_namespace, crd_rules, + policy_namespace, crd_rules, resource=resource, port=container_port, protocol=port.get('protocol')) - def _create_sg_rule_body_on_text_port(self, sg_id, direction, port, + def _create_sg_rule_body_on_text_port(self, direction, port, resources, crd_rules, pod_selector, policy_namespace, allow_all=False): """Create SG rules when named port is used in the NP rule @@ -352,7 +317,7 @@ def _create_sg_rule_body_on_text_port(self, sg_id, direction, port, for resource in resources: self._create_sg_rules_with_container_ports( container_ports, allow_all, resource, matched_pods, - crd_rules, sg_id, direction, port) + crd_rules, direction, port) elif direction == "egress": for resource in resources: # NOTE(maysams) Skipping objects that refers to ipblocks @@ -364,24 +329,24 @@ def _create_sg_rule_body_on_text_port(self, sg_id, direction, port, container_ports = driver_utils.get_ports(resource, port) self._create_sg_rules_with_container_ports( container_ports, allow_all, resource, matched_pods, - crd_rules, sg_id, direction, port, pod_selector, + crd_rules, direction, port, pod_selector, policy_namespace) if allow_all: container_port = None for container_port, pods in matched_pods.items(): for ethertype in (constants.IPv4, constants.IPv6): sg_rule = driver_utils.create_security_group_rule_body( - sg_id, direction, container_port, + direction, container_port, protocol=port.get('protocol'), ethertype=ethertype, pods=pods) crd_rules.append(sg_rule) if direction == 'egress': self._create_svc_egress_sg_rule( - sg_id, policy_namespace, crd_rules, + policy_namespace, crd_rules, port=container_port, protocol=port.get('protocol')) - def _create_sg_rule_on_number_port(self, allowed_resources, sg_id, + def _create_sg_rule_on_number_port(self, allowed_resources, direction, port, sg_rule_body_list, policy_namespace): for resource in allowed_resources: @@ -393,52 +358,51 @@ def _create_sg_rule_on_number_port(self, allowed_resources, sg_id, continue sg_rule = ( driver_utils.create_security_group_rule_body( - sg_id, direction, port.get('port'), + direction, port.get('port'), protocol=port.get('protocol'), cidr=cidr, namespace=ns)) sg_rule_body_list.append(sg_rule) if direction == 'egress': self._create_svc_egress_sg_rule( - sg_id, policy_namespace, sg_rule_body_list, + policy_namespace, sg_rule_body_list, resource=resource, port=port.get('port'), protocol=port.get('protocol')) - def _create_all_pods_sg_rules(self, port, sg_id, direction, + def _create_all_pods_sg_rules(self, port, direction, sg_rule_body_list, pod_selector, policy_namespace): if type(port.get('port')) is not int: all_pods = driver_utils.get_namespaced_pods().get('items') self._create_sg_rule_body_on_text_port( - sg_id, direction, port, all_pods, + direction, port, all_pods, sg_rule_body_list, pod_selector, policy_namespace, allow_all=True) else: for ethertype in (constants.IPv4, constants.IPv6): sg_rule = ( driver_utils.create_security_group_rule_body( - sg_id, direction, port.get('port'), + direction, port.get('port'), ethertype=ethertype, protocol=port.get('protocol'))) sg_rule_body_list.append(sg_rule) if direction == 'egress': self._create_svc_egress_sg_rule( - sg_id, policy_namespace, sg_rule_body_list, + policy_namespace, sg_rule_body_list, port=port.get('port'), protocol=port.get('protocol')) - def _create_default_sg_rule(self, sg_id, direction, sg_rule_body_list): + def _create_default_sg_rule(self, direction, sg_rule_body_list): for ethertype in (constants.IPv4, constants.IPv6): default_rule = { - 'security_group_rule': { + 'sgRule': { 'ethertype': ethertype, - 'security_group_id': sg_id, 'direction': direction, 'description': 'Kuryr-Kubernetes NetPolicy SG rule', }} sg_rule_body_list.append(default_rule) - def _parse_sg_rules(self, sg_rule_body_list, direction, policy, sg_id): + def _parse_sg_rules(self, sg_rule_body_list, direction, policy): """Parse policy into security group rules. This method inspects the policy object and create the equivalent @@ -460,16 +424,14 @@ def _parse_sg_rules(self, sg_rule_body_list, direction, policy, sg_id): # traffic as NP policy is not affecting ingress LOG.debug('Applying default all open for ingress for ' 'policy %s', policy['metadata']['selfLink']) - self._create_default_sg_rule( - sg_id, direction, sg_rule_body_list) + self._create_default_sg_rule(direction, sg_rule_body_list) elif direction == 'egress': if policy_types and 'Egress' not in policy_types: # NOTE(ltomasbo): add default rule to enable all egress # traffic as NP policy is not affecting egress LOG.debug('Applying default all open for egress for ' 'policy %s', policy['metadata']['selfLink']) - self._create_default_sg_rule( - sg_id, direction, sg_rule_body_list) + self._create_default_sg_rule(direction, sg_rule_body_list) else: LOG.warning('Not supported policyType at network policy %s', policy['metadata']['selfLink']) @@ -487,7 +449,7 @@ def _parse_sg_rules(self, sg_rule_body_list, direction, policy, sg_id): policy['metadata']['selfLink']) for ethertype in (constants.IPv4, constants.IPv6): rule = driver_utils.create_security_group_rule_body( - sg_id, direction, ethertype=ethertype) + direction, ethertype=ethertype) sg_rule_body_list.append(rule) for rule_block in rule_list: @@ -519,20 +481,20 @@ def _parse_sg_rules(self, sg_rule_body_list, direction, policy, sg_id): if allowed_resources or allow_all or selectors: if type(port.get('port')) is not int: self._create_sg_rule_body_on_text_port( - sg_id, direction, port, allowed_resources, + direction, port, allowed_resources, sg_rule_body_list, pod_selector, policy_namespace) else: self._create_sg_rule_on_number_port( - allowed_resources, sg_id, direction, port, + allowed_resources, direction, port, sg_rule_body_list, policy_namespace) if allow_all: self._create_all_pods_sg_rules( - port, sg_id, direction, sg_rule_body_list, + port, direction, sg_rule_body_list, pod_selector, policy_namespace) else: self._create_all_pods_sg_rules( - port, sg_id, direction, sg_rule_body_list, + port, direction, sg_rule_body_list, pod_selector, policy_namespace) elif allowed_resources or allow_all or selectors: for resource in allowed_resources: @@ -543,27 +505,27 @@ def _parse_sg_rules(self, sg_rule_body_list, direction, policy, sg_id): if not cidr: continue rule = driver_utils.create_security_group_rule_body( - sg_id, direction, + direction, port_range_min=1, port_range_max=65535, cidr=cidr, namespace=namespace) sg_rule_body_list.append(rule) if direction == 'egress': - rule = self._create_svc_egress_sg_rule( - sg_id, policy_namespace, sg_rule_body_list, + self._create_svc_egress_sg_rule( + policy_namespace, sg_rule_body_list, resource=resource) if allow_all: for ethertype in (constants.IPv4, constants.IPv6): rule = driver_utils.create_security_group_rule_body( - sg_id, direction, + direction, port_range_min=1, port_range_max=65535, ethertype=ethertype) sg_rule_body_list.append(rule) if direction == 'egress': - self._create_svc_egress_sg_rule( - sg_id, policy_namespace, sg_rule_body_list) + self._create_svc_egress_sg_rule(policy_namespace, + sg_rule_body_list) else: LOG.debug('This network policy specifies no %(direction)s ' '%(rule_direction)s and no ports: %(policy)s', @@ -571,15 +533,14 @@ def _parse_sg_rules(self, sg_rule_body_list, direction, policy, sg_id): 'rule_direction': rule_direction, 'policy': policy['metadata']['selfLink']}) - def _create_svc_egress_sg_rule(self, sg_id, policy_namespace, - sg_rule_body_list, resource=None, - port=None, protocol=None): + def _create_svc_egress_sg_rule(self, policy_namespace, sg_rule_body_list, + resource=None, port=None, protocol=None): services = driver_utils.get_services() if not resource: svc_subnet = utils.get_subnet_cidr( CONF.neutron_defaults.service_subnet) rule = driver_utils.create_security_group_rule_body( - sg_id, 'egress', port, protocol=protocol, cidr=svc_subnet) + 'egress', port, protocol=protocol, cidr=svc_subnet) if rule not in sg_rule_body_list: sg_rule_body_list.append(rule) return @@ -613,7 +574,7 @@ def _create_svc_egress_sg_rule(self, sg_id, policy_namespace, if not cluster_ip: continue rule = driver_utils.create_security_group_rule_body( - sg_id, 'egress', port, protocol=protocol, + 'egress', port, protocol=protocol, cidr=cluster_ip) if rule not in sg_rule_body_list: sg_rule_body_list.append(rule) @@ -626,7 +587,7 @@ def _pods_in_ip_block(self, pods, resource): return True return False - def parse_network_policy_rules(self, policy, sg_id): + def parse_network_policy_rules(self, policy): """Create security group rule bodies out of network policies. Whenever a notification from the handler 'on-present' method is @@ -637,10 +598,8 @@ def parse_network_policy_rules(self, policy, sg_id): ingress_sg_rule_body_list = [] egress_sg_rule_body_list = [] - self._parse_sg_rules(ingress_sg_rule_body_list, 'ingress', policy, - sg_id) - self._parse_sg_rules(egress_sg_rule_body_list, 'egress', policy, - sg_id) + self._parse_sg_rules(ingress_sg_rule_body_list, 'ingress', policy) + self._parse_sg_rules(egress_sg_rule_body_list, 'egress', policy) return ingress_sg_rule_body_list, egress_sg_rule_body_list @@ -657,19 +616,15 @@ def delete_np_sg(self, sg_id): LOG.exception("Error deleting security group %s.", sg_id) raise - def release_network_policy(self, netpolicy_crd): - if netpolicy_crd is not None: - self.delete_np_sg(netpolicy_crd['spec']['securityGroupId']) - self._del_kuryrnetpolicy_crd( - netpolicy_crd['metadata']['name'], - netpolicy_crd['metadata']['namespace']) + def release_network_policy(self, policy): + return self._del_knp_crd(policy) - def get_kuryrnetpolicy_crd(self, policy): - netpolicy_crd_name = "np-" + policy['metadata']['name'] + def _get_knp_crd(self, policy): + netpolicy_crd_name = policy['metadata']['name'] netpolicy_crd_namespace = policy['metadata']['namespace'] try: netpolicy_crd = self.kubernetes.get( - '{}/{}/kuryrnetpolicies/{}'.format( + '{}/{}/kuryrnetworkpolicies/{}'.format( constants.K8S_API_CRD_NAMESPACES, netpolicy_crd_namespace, netpolicy_crd_name)) except exceptions.K8sResourceNotFound: @@ -679,77 +634,81 @@ def get_kuryrnetpolicy_crd(self, policy): raise return netpolicy_crd - def knps_on_namespace(self, namespace): - try: - netpolicy_crds = self.kubernetes.get( - '{}/{}/kuryrnetpolicies'.format( - constants.K8S_API_CRD_NAMESPACES, - namespace)) - except exceptions.K8sClientException: - LOG.exception("Kubernetes Client Exception.") - raise - if netpolicy_crds.get('items'): - return True - return False - - def _add_kuryrnetpolicy_crd(self, policy, project_id, sg_id, i_rules, - e_rules): + def _create_knp_crd(self, policy, i_rules, e_rules): networkpolicy_name = policy['metadata']['name'] - netpolicy_crd_name = "np-" + networkpolicy_name namespace = policy['metadata']['namespace'] pod_selector = policy['spec'].get('podSelector') + policy_types = policy['spec'].get('policyTypes', []) netpolicy_crd = { 'apiVersion': 'openstack.org/v1', - 'kind': constants.K8S_OBJ_KURYRNETPOLICY, + 'kind': constants.K8S_OBJ_KURYRNETWORKPOLICY, 'metadata': { - 'name': netpolicy_crd_name, + 'name': networkpolicy_name, 'namespace': namespace, 'annotations': { - 'networkpolicy_name': networkpolicy_name, - 'networkpolicy_namespace': namespace, - 'networkpolicy_uid': policy['metadata']['uid'], + 'networkPolicyLink': policy['metadata']['selfLink'], }, + 'finalizers': [constants.NETWORKPOLICY_FINALIZER], }, 'spec': { - 'securityGroupName': "sg-" + networkpolicy_name, - 'securityGroupId': sg_id, 'ingressSgRules': i_rules, 'egressSgRules': e_rules, 'podSelector': pod_selector, - 'networkpolicy_spec': policy['spec'] + 'policyTypes': policy_types, + }, + 'status': { + 'securityGroupRules': [], }, } try: - LOG.debug("Creating KuryrNetPolicy CRD %s" % netpolicy_crd) - kubernetes_post = '{}/{}/kuryrnetpolicies'.format( + LOG.debug("Creating KuryrNetworkPolicy CRD %s" % netpolicy_crd) + url = '{}/{}/kuryrnetworkpolicies'.format( constants.K8S_API_CRD_NAMESPACES, namespace) - self.kubernetes.post(kubernetes_post, netpolicy_crd) + netpolicy_crd = self.kubernetes.post(url, netpolicy_crd) except exceptions.K8sClientException: - LOG.exception("Kubernetes Client Exception creating kuryrnetpolicy" - " CRD. %s" % exceptions.K8sClientException) + LOG.exception("Kubernetes Client Exception creating " + "KuryrNetworkPolicy CRD.") raise return netpolicy_crd - def _del_kuryrnetpolicy_crd(self, netpolicy_crd_name, - netpolicy_crd_namespace): + def _patch_knp_crd(self, policy, i_rules, e_rules, knp): + networkpolicy_name = policy['metadata']['name'] + namespace = policy['metadata']['namespace'] + pod_selector = policy['spec'].get('podSelector') + url = (f'{constants.K8S_API_CRD_NAMESPACES}/{namespace}' + f'/kuryrnetworkpolicies/{networkpolicy_name}') + + # FIXME(dulek): Rules should be hashable objects, not dict so that + # we could compare them easily here. + data = { + 'ingressSgRules': i_rules, + 'egressSgRules': e_rules, + } + if knp['spec'].get('podSelector') != pod_selector: + data['podSelector'] = pod_selector + + self.kubernetes.patch_crd('spec', url, data) + + def _del_knp_crd(self, policy): try: - LOG.debug("Deleting KuryrNetPolicy CRD %s" % netpolicy_crd_name) - self.kubernetes.delete('{}/{}/kuryrnetpolicies/{}'.format( - constants.K8S_API_CRD_NAMESPACES, - netpolicy_crd_namespace, - netpolicy_crd_name)) + ns = policy['metadata']['namespace'] + name = policy['metadata']['name'] + LOG.debug("Deleting KuryrNetworkPolicy CRD %s" % name) + self.kubernetes.delete('{}/{}/kuryrnetworkpolicies/{}'.format( + constants.K8S_API_CRD_NAMESPACES, ns, name)) + return True except exceptions.K8sResourceNotFound: - LOG.debug("KuryrNetPolicy CRD Object not found: %s", - netpolicy_crd_name) + LOG.debug("KuryrNetworkPolicy CRD Object not found: %s", name) + return False except exceptions.K8sClientException: - LOG.exception("Kubernetes Client Exception deleting kuryrnetpolicy" - " CRD.") + LOG.exception("Kubernetes Client Exception deleting " + "KuryrNetworkPolicy CRD %s." % name) raise def affected_pods(self, policy, selector=None): - if selector or selector == {}: + if selector is not None: pod_selector = selector else: pod_selector = policy['spec'].get('podSelector') diff --git a/kuryr_kubernetes/controller/drivers/network_policy_security_groups.py b/kuryr_kubernetes/controller/drivers/network_policy_security_groups.py index 1d87ef973..46f767e95 100644 --- a/kuryr_kubernetes/controller/drivers/network_policy_security_groups.py +++ b/kuryr_kubernetes/controller/drivers/network_policy_security_groups.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import uuid + from oslo_config import cfg from oslo_log import log as logging @@ -21,6 +23,7 @@ from kuryr_kubernetes.controller.drivers import base from kuryr_kubernetes.controller.drivers import utils as driver_utils from kuryr_kubernetes import exceptions +from kuryr_kubernetes import utils LOG = logging.getLogger(__name__) @@ -29,9 +32,7 @@ def _get_namespace_labels(namespace): kubernetes = clients.get_kubernetes_client() try: - path = '{}/{}'.format( - constants.K8S_API_NAMESPACES, namespace) - LOG.debug("K8s API Query %s", path) + path = '{}/{}'.format(constants.K8S_API_NAMESPACES, namespace) namespaces = kubernetes.get(path) LOG.debug("Return Namespace: %s", namespaces) except exceptions.K8sResourceNotFound: @@ -43,107 +44,41 @@ def _get_namespace_labels(namespace): return namespaces['metadata'].get('labels') -def _create_sg_rule(sg_id, direction, cidr, port=None, namespace=None): - if port: - sg_rule = driver_utils.create_security_group_rule_body( - sg_id, direction, port.get('port'), - protocol=port.get('protocol'), cidr=cidr, namespace=namespace) - else: - sg_rule = driver_utils.create_security_group_rule_body( - sg_id, direction, port_range_min=1, - port_range_max=65535, cidr=cidr, namespace=namespace) - - sgr_id = driver_utils.create_security_group_rule(sg_rule) +def _bump_networkpolicy(knp): + kubernetes = clients.get_kubernetes_client() - sg_rule['security_group_rule']['id'] = sgr_id - return sg_rule + try: + kubernetes.annotate( + knp['metadata']['annotations']['networkPolicyLink'], + {constants.K8S_ANNOTATION_POLICY: str(uuid.uuid4())}) + except exceptions.K8sResourceNotFound: + LOG.exception("NetworkPolicy not found") + raise + except exceptions.K8sClientException: + LOG.exception("Kubernetes Client Exception") + raise -def _get_crd_rule(crd_rules, container_port): - """Returns a CRD rule that matches a container port +def _create_sg_rules_with_container_ports(container_ports, matched): + """Checks if security group rules based on container ports will be updated - Retrieves the CRD rule that contains a given port in - the range of the rule ports. - """ - for crd_rule in crd_rules: - remote_ip_prefixes = crd_rule.get('remote_ip_prefixes') - min_port = crd_rule['security_group_rule'].get('port_range_min') - max_port = crd_rule['security_group_rule'].get('port_range_max') - if (remote_ip_prefixes and ( - min_port >= container_port and - container_port <= max_port)): - return crd_rule - - -def _create_sg_rules_with_container_ports(matched_pods, container_ports, - allow_all, namespace, matched, - crd_rules, sg_id, direction, - port, rule_selected_pod): - """Create security group rules based on container ports - - If it's an allow from/to everywhere rule or a rule with a - NamespaceSelector, updates a sg rule that might already exist - and match the named port or creates a new one with the - remote_ip_prefixes field containing the matched pod info. - Otherwise, creates rules for each container port without - a remote_ip_prefixes field. - - param matched_pods: List of dicts where the key is a container - port and value is the pods that have the port param container_ports: List of tuples with pods and port values - param allow_all: True is it's an allow from/to everywhere rule, - False otherwise. - param namespace: Namespace name param matched: If a sg rule was created for the NP rule - param crd_rules: List of sg rules to update when patching the CRD - param sg_id: ID of the security group - param direction: String representing rule direction, ingress or egress - param port: Dict containing port and protocol - param rule_selected_pod: K8s Pod object selected by the rules selectors - return: True if a sg rule was created, False otherwise. + return: True if a sg rule needs to be created, False otherwise. """ for pod, container_port in container_ports: - pod_namespace = pod['metadata']['namespace'] pod_ip = driver_utils.get_pod_ip(pod) if not pod_ip: LOG.debug("Skipping SG rule creation for pod %s due to " "no IP assigned", pod['metadata']['name']) continue + return matched + return False - pod_info = {pod_ip: pod_namespace} - matched = True - if allow_all or namespace: - crd_rule = _get_crd_rule(crd_rules, container_port) - if crd_rule: - crd_rule['remote_ip_prefixes'].update(pod_info) - else: - if container_port in matched_pods: - matched_pods[container_port].update(pod_info) - else: - matched_pods[container_port] = pod_info - else: - pod_ip = driver_utils.get_pod_ip(rule_selected_pod) - if not pod_ip: - LOG.debug("Skipping SG rule creation for pod %s due to no IP " - "assigned", rule_selected_pod['metadata']['name']) - continue - sg_rule = driver_utils.create_security_group_rule_body( - sg_id, direction, container_port, - protocol=port.get('protocol'), - cidr=pod_ip, pods=pod_info) - sgr_id = driver_utils.create_security_group_rule(sg_rule) - sg_rule['security_group_rule']['id'] = sgr_id - if sg_rule not in crd_rules: - crd_rules.append(sg_rule) - return matched - - -def _create_sg_rule_on_text_port(sg_id, direction, port, rule_selected_pods, - crd_rules, matched, crd, - allow_all=False, namespace=None): - matched_pods = {} +def _create_sg_rule_on_text_port(direction, port, rule_selected_pods, matched, + crd): spec_pod_selector = crd['spec'].get('podSelector') policy_namespace = crd['metadata']['namespace'] spec_pods = driver_utils.get_pods( @@ -151,11 +86,8 @@ def _create_sg_rule_on_text_port(sg_id, direction, port, rule_selected_pods, if direction == 'ingress': for spec_pod in spec_pods: container_ports = driver_utils.get_ports(spec_pod, port) - for rule_selected_pod in rule_selected_pods: - matched = _create_sg_rules_with_container_ports( - matched_pods, container_ports, allow_all, namespace, - matched, crd_rules, sg_id, direction, port, - rule_selected_pod) + matched = _create_sg_rules_with_container_ports( + container_ports, matched) elif direction == 'egress': for rule_selected_pod in rule_selected_pods: pod_label = rule_selected_pod['metadata'].get('labels') @@ -168,51 +100,11 @@ def _create_sg_rule_on_text_port(sg_id, direction, port, rule_selected_pods, container_ports = driver_utils.get_ports( rule_selected_pod, port) matched = _create_sg_rules_with_container_ports( - matched_pods, container_ports, allow_all, - namespace, matched, crd_rules, sg_id, direction, - port, rule_selected_pod) - - _apply_sg_rules_on_matched_pods(matched_pods, sg_id, direction, namespace, - port, crd_rules, allow_all) - + container_ports, matched) return matched -def _apply_sg_rules_on_matched_pods(matched_pods, sg_id, direction, namespace, - port, crd_rules, allow_all=False): - for container_port, pods in matched_pods.items(): - if allow_all: - for ethertype in (constants.IPv4, constants.IPv6): - sg_rule = driver_utils.create_security_group_rule_body( - sg_id, direction, container_port, - protocol=port.get('protocol'), - ethertype=ethertype, - pods=pods) - sgr_id = driver_utils.create_security_group_rule(sg_rule) - sg_rule['security_group_rule']['id'] = sgr_id - if sg_rule not in crd_rules: - crd_rules.append(sg_rule) - else: - namespace_obj = driver_utils.get_namespace(namespace) - if not namespace_obj: - LOG.debug("Skipping SG rule creation. Inexistent" - " namespace.") - continue - namespace_cidr = driver_utils.get_namespace_subnet_cidr( - namespace_obj) - sg_rule = driver_utils.create_security_group_rule_body( - sg_id, direction, container_port, - protocol=port.get('protocol'), cidr=namespace_cidr, - pods=pods) - sgr_id = driver_utils.create_security_group_rule(sg_rule) - sg_rule['security_group_rule']['id'] = sgr_id - if sg_rule not in crd_rules: - crd_rules.append(sg_rule) - - -def _create_sg_rules(crd, pod, pod_selector, rule_block, - crd_rules, direction, matched, namespace=None, - allow_all=False): +def _create_sg_rules(crd, pod, pod_selector, rule_block, direction, matched): pod_labels = pod['metadata'].get('labels') pod_ip = driver_utils.get_pod_ip(pod) if not pod_ip: @@ -224,73 +116,52 @@ def _create_sg_rules(crd, pod, pod_selector, rule_block, # with empty value or with '{}', as they have same result in here. if pod_selector: if driver_utils.match_selector(pod_selector, pod_labels): - sg_id = crd['spec']['securityGroupId'] if 'ports' in rule_block: for port in rule_block['ports']: if type(port.get('port')) is not int: matched = _create_sg_rule_on_text_port( - sg_id, direction, port, [pod], - crd_rules, matched, crd) + direction, port, [pod], matched, crd) else: matched = True - sg_rule = _create_sg_rule( - sg_id, direction, cidr=pod_ip, port=port, - namespace=namespace) - if sg_rule not in crd_rules: - crd_rules.append(sg_rule) else: matched = True - sg_rule = _create_sg_rule( - sg_id, direction, cidr=pod_ip, namespace=namespace) - if sg_rule not in crd_rules: - crd_rules.append(sg_rule) else: # NOTE (maysams) When a policy with namespaceSelector and text port # is applied the port on the pods needs to be retrieved. - sg_id = crd['spec']['securityGroupId'] if 'ports' in rule_block: for port in rule_block['ports']: if type(port.get('port')) is not int: - matched = ( - _create_sg_rule_on_text_port( - sg_id, direction, port, [pod], - crd_rules, matched, crd, - allow_all=allow_all, namespace=namespace)) + matched = _create_sg_rule_on_text_port( + direction, port, [pod], matched, crd) return matched def _parse_selectors_on_pod(crd, pod, pod_selector, namespace_selector, - rule_block, crd_rules, direction, matched): + rule_block, direction, matched): pod_namespace = pod['metadata']['namespace'] pod_namespace_labels = _get_namespace_labels(pod_namespace) policy_namespace = crd['metadata']['namespace'] if namespace_selector == {}: matched = _create_sg_rules(crd, pod, pod_selector, rule_block, - crd_rules, direction, matched, - allow_all=True) + direction, matched) elif namespace_selector: if (pod_namespace_labels and driver_utils.match_selector(namespace_selector, pod_namespace_labels)): matched = _create_sg_rules(crd, pod, pod_selector, - rule_block, crd_rules, - direction, matched, - namespace=pod_namespace) + rule_block, direction, matched) else: if pod_namespace == policy_namespace: matched = _create_sg_rules(crd, pod, pod_selector, rule_block, - crd_rules, direction, matched, - namespace=pod_namespace) - return matched, crd_rules + direction, matched) + return matched def _parse_selectors_on_namespace(crd, direction, pod_selector, - ns_selector, rule_block, crd_rules, - namespace, matched): + ns_selector, rule_block, namespace, matched): ns_name = namespace['metadata'].get('name') ns_labels = namespace['metadata'].get('labels') - sg_id = crd['spec']['securityGroupId'] if (ns_selector and ns_labels and driver_utils.match_selector(ns_selector, ns_labels)): @@ -301,10 +172,8 @@ def _parse_selectors_on_namespace(crd, direction, pod_selector, if type(port.get('port')) is not int: matched = ( _create_sg_rule_on_text_port( - sg_id, direction, port, pods, - crd_rules, matched, crd)) + direction, port, pods, matched, crd)) else: - matched = True for pod in pods: pod_ip = driver_utils.get_pod_ip(pod) if not pod_ip: @@ -312,11 +181,7 @@ def _parse_selectors_on_namespace(crd, direction, pod_selector, LOG.debug("Skipping SG rule creation for pod " "%s due to no IP assigned", pod_name) continue - sg_rule = _create_sg_rule( - sg_id, direction, pod_ip, port=port, - namespace=ns_name) - if sg_rule not in crd_rules: - crd_rules.append(sg_rule) + matched = True else: for pod in pods: pod_ip = driver_utils.get_pod_ip(pod) @@ -326,45 +191,25 @@ def _parse_selectors_on_namespace(crd, direction, pod_selector, " to no IP assigned", pod_name) continue matched = True - sg_rule = _create_sg_rule( - sg_id, direction, pod_ip, - namespace=ns_name) - if sg_rule not in crd_rules: - crd_rules.append(sg_rule) else: ns_pods = driver_utils.get_pods(ns_selector)['items'] - ns_cidr = driver_utils.get_namespace_subnet_cidr(namespace) if 'ports' in rule_block: for port in rule_block['ports']: if type(port.get('port')) is not int: matched = ( _create_sg_rule_on_text_port( - sg_id, direction, port, ns_pods, - crd_rules, matched, crd)) + direction, port, ns_pods, matched, crd)) else: matched = True - sg_rule = _create_sg_rule( - sg_id, direction, ns_cidr, - port=port, namespace=ns_name) - if sg_rule not in crd_rules: - crd_rules.append(sg_rule) else: matched = True - sg_rule = _create_sg_rule( - sg_id, direction, ns_cidr, - namespace=ns_name) - if sg_rule not in crd_rules: - crd_rules.append(sg_rule) - return matched, crd_rules + return matched -def _parse_rules(direction, crd, pod=None, namespace=None): - policy = crd['spec']['networkpolicy_spec'] +def _parse_rules(direction, crd, policy, pod=None, namespace=None): rule_direction = 'from' - crd_rules = crd['spec'].get('ingressSgRules') if direction == 'egress': rule_direction = 'to' - crd_rules = crd['spec'].get('egressSgRules') matched = False rule_list = policy.get(direction, []) @@ -373,13 +218,13 @@ def _parse_rules(direction, crd, pod=None, namespace=None): namespace_selector = rule.get('namespaceSelector') pod_selector = rule.get('podSelector') if pod: - matched, crd_rules = _parse_selectors_on_pod( + matched = _parse_selectors_on_pod( crd, pod, pod_selector, namespace_selector, - rule_block, crd_rules, direction, matched) + rule_block, direction, matched) elif namespace: - matched, crd_rules = _parse_selectors_on_namespace( + matched = _parse_selectors_on_namespace( crd, direction, pod_selector, namespace_selector, - rule_block, crd_rules, namespace, matched) + rule_block, namespace, matched) # NOTE(maysams): Cover the case of a network policy that allows # from everywhere on a named port, e.g., when there is no 'from' @@ -387,84 +232,62 @@ def _parse_rules(direction, crd, pod=None, namespace=None): if pod and not matched: for port in rule_block.get('ports', []): if type(port.get('port')) is not int: - sg_id = crd['spec']['securityGroupId'] if (not rule_block.get(rule_direction, []) or direction == "ingress"): - matched = (_create_sg_rule_on_text_port( - sg_id, direction, port, [pod], - crd_rules, matched, crd, - allow_all=True)) - return matched, crd_rules + matched = _create_sg_rule_on_text_port( + direction, port, [pod], matched, crd) + return matched def _parse_rules_on_delete_namespace(rule_list, direction, ns_name): - matched = False - rules = [] for rule in rule_list: - LOG.debug('Parsing %(dir)s Rule %(r)s', {'dir': direction, - 'r': rule}) + LOG.debug('Parsing %(dir)s Rule %(r)s', {'dir': direction, 'r': rule}) rule_namespace = rule.get('namespace', None) - remote_ip_prefixes = rule.get('remote_ip_prefixes', {}) + affectedPods = rule.get('affectedPods', []) if rule_namespace and rule_namespace == ns_name: - matched = True - driver_utils.delete_security_group_rule( - rule['security_group_rule']['id']) - elif remote_ip_prefixes: - for remote_ip, namespace in list(remote_ip_prefixes.items()): - if namespace == ns_name: - matched = True - remote_ip_prefixes.pop(remote_ip) - if remote_ip_prefixes: - rule['remote_ip_prefixes'] = remote_ip_prefixes - rules.append(rule) - else: - rules.append(rule) - return matched, rules + return True + elif affectedPods: + for pod_info in affectedPods: + if pod_info['podNamespace'] == ns_name: + return True + return False def _parse_rules_on_delete_pod(rule_list, direction, pod_ip): - matched = False - rules = [] for rule in rule_list: - LOG.debug('Parsing %(dir)s Rule %(r)s', {'dir': direction, - 'r': rule}) - remote_ip_prefix = rule['security_group_rule'].get( - 'remote_ip_prefix') - remote_ip_prefixes = rule.get('remote_ip_prefixes', {}) + LOG.debug('Parsing %(dir)s Rule %(r)s', {'dir': direction, 'r': rule}) + remote_ip_prefix = rule['sgRule'].get('remote_ip_prefix') + affectedPods = rule.get('affectedPods', []) if remote_ip_prefix and remote_ip_prefix == pod_ip: - matched = True - driver_utils.delete_security_group_rule( - rule['security_group_rule']['id']) - elif remote_ip_prefixes: - if pod_ip in remote_ip_prefixes: - matched = True - remote_ip_prefixes.pop(pod_ip) - if remote_ip_prefixes: - rule['remote_ip_prefixes'] = remote_ip_prefixes - rules.append(rule) - else: - rules.append(rule) - return matched, rules + return True + elif affectedPods: + for pod_info in affectedPods: + if pod_info['podIP'] == pod_ip: + return True + return False -def _get_pod_sgs(pod, project_id): +def _get_pod_sgs(pod): sg_list = [] pod_labels = pod['metadata'].get('labels') pod_namespace = pod['metadata']['namespace'] - knp_crds = driver_utils.get_kuryrnetpolicy_crds( + knp_crds = driver_utils.get_kuryrnetworkpolicy_crds( namespace=pod_namespace) - for crd in knp_crds.get('items'): + for crd in knp_crds: pod_selector = crd['spec'].get('podSelector') - if pod_selector: - if driver_utils.match_selector(pod_selector, pod_labels): - LOG.debug("Appending %s", - str(crd['spec']['securityGroupId'])) - sg_list.append(str(crd['spec']['securityGroupId'])) - else: - LOG.debug("Appending %s", str(crd['spec']['securityGroupId'])) - sg_list.append(str(crd['spec']['securityGroupId'])) + if driver_utils.match_selector(pod_selector, pod_labels): + sg_id = crd['status'].get('securityGroupId') + if not sg_id: + # NOTE(dulek): We could just assume KNP handler will apply it, + # but it's possible that when it gets this pod it + # will have no IP yet and will be skipped. + LOG.warning('SG for NP %s not created yet, will retry.', + utils.get_res_unique_name(crd)) + raise exceptions.ResourceNotReady(pod) + LOG.debug("Appending %s", crd['status']['securityGroupId']) + sg_list.append(crd['status']['securityGroupId']) # NOTE(maysams) Pods that are not selected by any Networkpolicy # are fully accessible. Thus, the default security group is associated. @@ -481,55 +304,56 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver): """Provides security groups for pods based on network policies""" def get_security_groups(self, pod, project_id): - return _get_pod_sgs(pod, project_id) + return _get_pod_sgs(pod) def create_sg_rules(self, pod): - LOG.debug("Creating sg rule for pod: %s", pod['metadata']['name']) + LOG.debug("Creating SG rules for pod: %s", pod['metadata']['name']) crd_pod_selectors = [] - knp_crds = driver_utils.get_kuryrnetpolicy_crds() - for crd in knp_crds.get('items'): + knp_crds = driver_utils.get_kuryrnetworkpolicy_crds() + nps = driver_utils.get_networkpolicies() + pairs = driver_utils.zip_knp_np(knp_crds, nps) + + for crd, policy in pairs: crd_selector = crd['spec'].get('podSelector') + spec = policy.get('spec') - i_matched, i_rules = _parse_rules('ingress', crd, pod=pod) - e_matched, e_rules = _parse_rules('egress', crd, pod=pod) + i_matched = _parse_rules('ingress', crd, spec, pod=pod) + e_matched = _parse_rules('egress', crd, spec, pod=pod) if i_matched or e_matched: - driver_utils.patch_kuryrnetworkpolicy_crd(crd, i_rules, - e_rules, - crd_selector) + _bump_networkpolicy(crd) if i_matched: crd_pod_selectors.append(crd_selector) return crd_pod_selectors def delete_sg_rules(self, pod): - LOG.debug("Deleting sg rule for pod: %s", pod['metadata']['name']) + LOG.debug("Deleting SG rules for pod: %s", pod['metadata']['name']) pod_ip = driver_utils.get_pod_ip(pod) + crd_pod_selectors = [] if not pod_ip: LOG.debug("Skipping SG rule deletion as pod %s has no IP assigned", pod['metadata']['name']) - return None - crd_pod_selectors = [] - knp_crds = driver_utils.get_kuryrnetpolicy_crds() - for crd in knp_crds.get('items'): + return crd_pod_selectors + knp_crds = driver_utils.get_kuryrnetworkpolicy_crds() + for crd in knp_crds: crd_selector = crd['spec'].get('podSelector') ingress_rule_list = crd['spec'].get('ingressSgRules') egress_rule_list = crd['spec'].get('egressSgRules') - i_matched, i_rules = _parse_rules_on_delete_pod( + i_matched = _parse_rules_on_delete_pod( ingress_rule_list, "ingress", pod_ip) - e_matched, e_rules = _parse_rules_on_delete_pod( + e_matched = _parse_rules_on_delete_pod( egress_rule_list, "egress", pod_ip) if i_matched or e_matched: - driver_utils.patch_kuryrnetworkpolicy_crd(crd, i_rules, - e_rules, - crd_selector) + _bump_networkpolicy(crd) if i_matched: crd_pod_selectors.append(crd_selector) return crd_pod_selectors def update_sg_rules(self, pod): - LOG.debug("Updating sg rule for pod: %s", pod['metadata']['name']) + LOG.debug("Updating SG rules for pod: %s", pod['metadata']['name']) + # FIXME(dulek): No need to bump twice. crd_pod_selectors = [] crd_pod_selectors.extend(self.delete_sg_rules(pod)) crd_pod_selectors.extend(self.create_sg_rules(pod)) @@ -537,51 +361,47 @@ def update_sg_rules(self, pod): def delete_namespace_sg_rules(self, namespace): ns_name = namespace['metadata']['name'] - LOG.debug("Deleting sg rule for namespace: %s", - ns_name) + LOG.debug("Deleting SG rules for namespace: %s", ns_name) crd_selectors = [] - knp_crds = driver_utils.get_kuryrnetpolicy_crds() - for crd in knp_crds.get('items'): + knp_crds = driver_utils.get_kuryrnetworkpolicy_crds() + for crd in knp_crds: crd_selector = crd['spec'].get('podSelector') ingress_rule_list = crd['spec'].get('ingressSgRules') egress_rule_list = crd['spec'].get('egressSgRules') - i_matched, i_rules = _parse_rules_on_delete_namespace( + i_matched = _parse_rules_on_delete_namespace( ingress_rule_list, "ingress", ns_name) - e_matched, e_rules = _parse_rules_on_delete_namespace( + e_matched = _parse_rules_on_delete_namespace( egress_rule_list, "egress", ns_name) if i_matched or e_matched: - driver_utils.patch_kuryrnetworkpolicy_crd( - crd, i_rules, e_rules, crd_selector) + _bump_networkpolicy(crd) if i_matched: crd_selectors.append(crd_selector) return crd_selectors def create_namespace_sg_rules(self, namespace): ns_name = namespace['metadata']['name'] - LOG.debug("Creating sg rule for namespace: %s", ns_name) + LOG.debug("Creating SG rules for namespace: %s", ns_name) crd_selectors = [] - knp_crds = driver_utils.get_kuryrnetpolicy_crds() - for crd in knp_crds.get('items'): + knp_crds = driver_utils.get_kuryrnetworkpolicy_crds() + nps = driver_utils.get_networkpolicies() + pairs = driver_utils.zip_knp_np(knp_crds, nps) + for crd, policy in pairs: crd_selector = crd['spec'].get('podSelector') - - i_matched, i_rules = _parse_rules( - 'ingress', crd, namespace=namespace) - e_matched, e_rules = _parse_rules( - 'egress', crd, namespace=namespace) + spec = policy.get('spec') + i_matched = _parse_rules('ingress', crd, spec, namespace=namespace) + e_matched = _parse_rules('egress', crd, spec, namespace=namespace) if i_matched or e_matched: - driver_utils.patch_kuryrnetworkpolicy_crd(crd, i_rules, - e_rules, - crd_selector) + _bump_networkpolicy(crd) if i_matched: crd_selectors.append(crd_selector) return crd_selectors def update_namespace_sg_rules(self, namespace): - LOG.debug("Updating sg rule for namespace: %s", + LOG.debug("Updating SG rules for namespace: %s", namespace['metadata']['name']) crd_selectors = [] crd_selectors.extend(self.delete_namespace_sg_rules(namespace)) @@ -608,5 +428,5 @@ def get_security_groups(self, service, project_id): # all of them. Hence only considering the security groups applied # to the first one. if pods: - return _get_pod_sgs(pods[0], project_id) + return _get_pod_sgs(pods[0]) return sg_list[:] diff --git a/kuryr_kubernetes/controller/drivers/utils.py b/kuryr_kubernetes/controller/drivers/utils.py index e7f30d570..093b5e514 100644 --- a/kuryr_kubernetes/controller/drivers/utils.py +++ b/kuryr_kubernetes/controller/drivers/utils.py @@ -25,6 +25,7 @@ from kuryr_kubernetes import clients from kuryr_kubernetes import constants from kuryr_kubernetes import exceptions as k_exc +from kuryr_kubernetes import utils OPERATORS_WITH_VALUES = [constants.K8S_OPERATOR_IN, @@ -182,10 +183,8 @@ def replace_encoded_characters(labels): def create_security_group_rule(body): os_net = clients.get_network_client() - sgr = '' - try: - params = dict(body['security_group_rule']) + params = dict(body) if 'ethertype' in params: # NOTE(gryf): in openstacksdk, there is ether_type attribute in # the security_group_rule object, in CRD we have 'ethertype' @@ -220,29 +219,27 @@ def delete_security_group_rule(security_group_rule_id): raise -def patch_kuryrnetworkpolicy_crd(crd, i_rules, e_rules, pod_selector, - np_spec=None): +def patch_kuryrnetworkpolicy_crd(crd, i_rules, e_rules): kubernetes = clients.get_kubernetes_client() crd_name = crd['metadata']['name'] - if not np_spec: - np_spec = crd['spec']['networkpolicy_spec'] - LOG.debug('Patching KuryrNetPolicy CRD %s' % crd_name) + LOG.debug('Patching KuryrNetworkPolicy CRD %s' % crd_name) try: - kubernetes.patch_crd('spec', crd['metadata']['selfLink'], - {'ingressSgRules': i_rules, - 'egressSgRules': e_rules, - 'podSelector': pod_selector, - 'networkpolicy_spec': np_spec}) + spec = { + 'ingressSgRules': i_rules, + 'egressSgRules': e_rules, + } + + kubernetes.patch_crd('spec', crd['metadata']['selfLink'], spec) except k_exc.K8sResourceNotFound: - LOG.debug('KuryrNetPolicy CRD not found %s', crd_name) + LOG.debug('KuryrNetworkPolicy CRD not found %s', crd_name) except k_exc.K8sClientException: - LOG.exception('Error updating kuryrnetpolicy CRD %s', crd_name) + LOG.exception('Error updating KuryrNetworkPolicy CRD %s', crd_name) raise def create_security_group_rule_body( - security_group_id, direction, port_range_min=None, - port_range_max=None, protocol=None, ethertype=None, cidr=None, + direction, port_range_min=None, port_range_max=None, protocol=None, + ethertype='IPv4', cidr=None, description="Kuryr-Kubernetes NetPolicy SG rule", namespace=None, pods=None): if not port_range_min: @@ -253,15 +250,12 @@ def create_security_group_rule_body( if not protocol: protocol = 'TCP' - if not ethertype: - ethertype = 'IPv4' - if cidr and netaddr.IPNetwork(cidr).version == 6: - ethertype = 'IPv6' + if cidr and netaddr.IPNetwork(cidr).version == 6: + ethertype = 'IPv6' security_group_rule_body = { - 'security_group_rule': { + 'sgRule': { 'ethertype': ethertype, - 'security_group_id': security_group_id, 'description': description, 'direction': direction, 'protocol': protocol.lower(), @@ -270,12 +264,12 @@ def create_security_group_rule_body( } } if cidr: - security_group_rule_body['security_group_rule'][ - 'remote_ip_prefix'] = cidr + security_group_rule_body['sgRule']['remote_ip_prefix'] = cidr if namespace: security_group_rule_body['namespace'] = namespace if pods: - security_group_rule_body['remote_ip_prefixes'] = pods + security_group_rule_body['affectedPods'] = [ + {'podIP': ip, 'podNamespace': ns} for ip, ns in pods.items()] LOG.debug("Creating sg rule body %s", security_group_rule_body) return security_group_rule_body @@ -310,25 +304,60 @@ def get_annotated_labels(resource, annotation_labels): return None -def get_kuryrnetpolicy_crds(namespace=None): +def get_kuryrnetworkpolicy_crds(namespace=None): kubernetes = clients.get_kubernetes_client() try: if namespace: - knp_path = '{}/{}/kuryrnetpolicies'.format( + knp_path = '{}/{}/kuryrnetworkpolicies'.format( constants.K8S_API_CRD_NAMESPACES, namespace) else: - knp_path = constants.K8S_API_CRD_KURYRNETPOLICIES - LOG.debug("K8s API Query %s", knp_path) + knp_path = constants.K8S_API_CRD_KURYRNETWORKPOLICIES knps = kubernetes.get(knp_path) - LOG.debug("Return Kuryr Network Policies with label %s", knps) + LOG.debug("Returning KuryrNetworkPolicies %s", knps) except k_exc.K8sResourceNotFound: - LOG.exception("KuryrNetPolicy CRD not found") + LOG.exception("KuryrNetworkPolicy CRD not found") raise except k_exc.K8sClientException: LOG.exception("Kubernetes Client Exception") raise - return knps + return knps.get('items', []) + + +def get_networkpolicies(namespace=None): + # FIXME(dulek): This is awful, shouldn't we have list method on k8s_client? + kubernetes = clients.get_kubernetes_client() + + try: + if namespace: + np_path = '{}/{}/networkpolicies'.format( + constants.K8S_API_CRD_NAMESPACES, namespace) + else: + np_path = constants.K8S_API_POLICIES + nps = kubernetes.get(np_path) + except k_exc.K8sResourceNotFound: + LOG.exception("NetworkPolicy or namespace %s not found", namespace) + raise + except k_exc.K8sClientException: + LOG.exception("Exception when listing NetworkPolicies.") + raise + return nps.get('items', []) + + +def zip_knp_np(knps, nps): + """Returns tuples of matching KuryrNetworkPolicy and NetworkPolicy objs. + + :param knps: List of KuryrNetworkPolicy objects + :param nps: List of NetworkPolicy objects + :return: List of tuples of matching (knp, np) + """ + pairs = [] + for knp in knps: + for np in nps: + if utils.get_res_unique_name(knp) == utils.get_res_unique_name(np): + pairs.append((knp, np)) + break + return pairs def match_expressions(expressions, labels): @@ -369,6 +398,8 @@ def match_labels(crd_labels, labels): def match_selector(selector, labels): + if selector is None: + return True crd_labels = selector.get('matchLabels', None) crd_expressions = selector.get('matchExpressions', None) diff --git a/kuryr_kubernetes/controller/handlers/kuryrnetpolicy.py b/kuryr_kubernetes/controller/handlers/kuryrnetpolicy.py deleted file mode 100644 index 8c30f7c18..000000000 --- a/kuryr_kubernetes/controller/handlers/kuryrnetpolicy.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2019 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kuryr_kubernetes import constants -from kuryr_kubernetes.controller.drivers import base as drivers -from kuryr_kubernetes.handlers import k8s_base - - -class KuryrNetPolicyHandler(k8s_base.ResourceEventHandler): - """Controller side of KuryrNetPolicy process for Kubernetes pods. - - `KuryrNetPolicyHandler` runs on the Kuryr-Kubernetes controller and is - responsible for deleting associated security groups upon namespace - deletion. - """ - OBJECT_KIND = constants.K8S_OBJ_KURYRNETPOLICY - OBJECT_WATCH_PATH = constants.K8S_API_CRD_KURYRNETPOLICIES - - def __init__(self): - super(KuryrNetPolicyHandler, self).__init__() - self._drv_policy = drivers.NetworkPolicyDriver.get_instance() - - def on_deleted(self, netpolicy_crd): - crd_sg = netpolicy_crd['spec'].get('securityGroupId') - if crd_sg: - self._drv_policy.delete_np_sg(crd_sg) diff --git a/kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py b/kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py new file mode 100644 index 000000000..a93466161 --- /dev/null +++ b/kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py @@ -0,0 +1,307 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from openstack import exceptions as os_exc +from oslo_config import cfg +from oslo_log import log as logging + +from kuryr_kubernetes import clients +from kuryr_kubernetes import constants +from kuryr_kubernetes.controller.drivers import base as drivers +from kuryr_kubernetes.controller.drivers import utils as driver_utils +from kuryr_kubernetes import exceptions +from kuryr_kubernetes.handlers import k8s_base +from kuryr_kubernetes import utils + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class KuryrNetworkPolicyHandler(k8s_base.ResourceEventHandler): + """Controller side of KuryrNetworkPolicy process for Kubernetes pods. + + `KuryrNetworkPolicyHandler` runs on the kuryr-controller and is + responsible for creating and deleting SG and SG rules for `NetworkPolicy`. + The `KuryrNetworkPolicy` objects are created by `NetworkPolicyHandler`. + """ + OBJECT_KIND = constants.K8S_OBJ_KURYRNETWORKPOLICY + OBJECT_WATCH_PATH = constants.K8S_API_CRD_KURYRNETWORKPOLICIES + + def __init__(self): + super(KuryrNetworkPolicyHandler, self).__init__() + self.os_net = clients.get_network_client() + self.k8s = clients.get_kubernetes_client() + self._drv_project = drivers.NetworkPolicyProjectDriver.get_instance() + self._drv_policy = drivers.NetworkPolicyDriver.get_instance() + self._drv_vif_pool = drivers.VIFPoolDriver.get_instance( + specific_driver='multi_pool') + self._drv_vif_pool.set_vif_driver() + self._drv_pod_sg = drivers.PodSecurityGroupsDriver.get_instance() + self._drv_svc_sg = drivers.ServiceSecurityGroupsDriver.get_instance() + self._drv_lbaas = drivers.LBaaSDriver.get_instance() + + self._convert_old_crds() + + def _convert_old_crds(self): + try: + netpolicies = self.k8s.get(constants.K8S_API_CRD_KURYRNETPOLICIES) + except exceptions.K8sClientException: + LOG.exception("Error when fetching old KuryrNetPolicy CRDs for " + "conversion.") + return + + for netpolicy in netpolicies.get('items', []): + new_networkpolicy = self._drv_policy.get_from_old_crd(netpolicy) + url = (f"{constants.K8S_API_CRD_NAMESPACES}/" + f"{netpolicy['metadata']['namespace']}/" + f"kuryrnetworkpolicies") + try: + self.k8s.post(url, new_networkpolicy) + except exceptions.K8sConflict: + LOG.warning('KuryrNetworkPolicy %s already existed when ' + 'converting KuryrNetPolicy %s. Ignoring.', + utils.get_res_unique_name(new_networkpolicy), + utils.get_res_unique_name(netpolicy)) + self.k8s.delete(netpolicy['metadata']['selfLink']) + + def _patch_kuryrnetworkpolicy_crd(self, knp, field, data, + action='replace'): + name = knp['metadata']['name'] + LOG.debug('Patching KuryrNet CRD %s', name) + try: + status = self.k8s.patch_crd(field, knp['metadata']['selfLink'], + data, action=action) + except exceptions.K8sResourceNotFound: + LOG.debug('KuryrNetworkPolicy CRD not found %s', name) + return None + except exceptions.K8sClientException: + LOG.exception('Error updating KuryrNetworkPolicy CRD %s', name) + raise + + knp['status'] = status + return knp + + def _get_networkpolicy(self, link): + return self.k8s.get(link) + + def _compare_sgs(self, a, b): + checked_props = ('direction', 'ethertype', 'port_range_max', + 'port_range_min', 'protocol', 'remote_ip_prefix') + + for k in checked_props: + if a.get(k) != b.get(k): + return False + return True + + def _find_sgs(self, a, rules): + for r in rules: + if self._compare_sgs(r, a): + return True + + return False + + def on_present(self, knp): + uniq_name = utils.get_res_unique_name(knp) + LOG.debug('on_present() for NP %s', uniq_name) + project_id = self._drv_project.get_project(knp) + if not knp['status'].get('securityGroupId'): + LOG.debug('Creating SG for NP %s', uniq_name) + # TODO(dulek): Do this right, why do we have a project driver per + # resource?! This one expects policy, not knp, but it + # ignores it anyway! + sg_id = self._drv_policy.create_security_group(knp, project_id) + knp = self._patch_kuryrnetworkpolicy_crd( + knp, 'status', {'securityGroupId': sg_id}) + LOG.debug('Created SG %s for NP %s', sg_id, uniq_name) + else: + # TODO(dulek): Check if it really exists, recreate if not. + sg_id = knp['status'].get('securityGroupId') + + # First update SG rules as we want to apply updated ones + current = knp['status']['securityGroupRules'] + required = knp['spec']['ingressSgRules'] + knp['spec']['egressSgRules'] + required = [r['sgRule'] for r in required] + + # FIXME(dulek): This *might* be prone to race conditions if failure + # happens between SG rule is created/deleted and status + # is annotated. We don't however need to revert on failed + # K8s operations - creation, deletion of SG rules and + # attaching or detaching SG from ports are idempotent + # so we can repeat them. What worries me is losing track + # of an update due to restart. The only way to do it + # would be to periodically check if what's in `status` + # is the reality in OpenStack API. That should be just + # two Neutron API calls + possible resync. + to_add = [] + to_remove = [] + for r in required: + if not self._find_sgs(r, current): + to_add.append(r) + + for i, c in enumerate(current): + if not self._find_sgs(c, required): + to_remove.append((i, c['id'])) + + LOG.debug('SGs to add for NP %s: %s', uniq_name, to_add) + + for sg_rule in to_add: + LOG.debug('Adding SG rule %s for NP %s', sg_rule, uniq_name) + sg_rule['security_group_id'] = sg_id + sgr_id = driver_utils.create_security_group_rule(sg_rule) + sg_rule['id'] = sgr_id + knp = self._patch_kuryrnetworkpolicy_crd( + knp, 'status', {'securityGroupRules/-': sg_rule}, 'add') + + # We need to remove starting from the last one in order to maintain + # indexes. Please note this will start to fail miserably if we start + # to change status from multiple places. + to_remove.reverse() + + LOG.debug('SGs to remove for NP %s: %s', uniq_name, + [x[1] for x in to_remove]) + + for i, sg_rule_id in to_remove: + LOG.debug('Removing SG rule %s as it is no longer part of NP %s', + sg_rule_id, uniq_name) + driver_utils.delete_security_group_rule(sg_rule_id) + knp = self._patch_kuryrnetworkpolicy_crd( + knp, 'status/securityGroupRules', i, 'remove') + + pods_to_update = [] + + previous_sel = knp['status'].get('podSelector', None) + current_sel = knp['spec']['podSelector'] + if previous_sel is None: + # Fresh NetworkPolicy that was never applied. + pods_to_update.extend(self._drv_policy.namespaced_pods(knp)) + elif previous_sel != current_sel or previous_sel == {}: + pods_to_update.extend( + self._drv_policy.affected_pods(knp, previous_sel)) + + matched_pods = self._drv_policy.affected_pods(knp) + pods_to_update.extend(matched_pods) + + for pod in pods_to_update: + if driver_utils.is_host_network(pod): + continue + pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) + self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) + + # FIXME(dulek): We should not need this one day. + policy = self._get_networkpolicy(knp['metadata']['annotations'] + ['networkPolicyLink']) + if (pods_to_update and CONF.octavia_defaults.enforce_sg_rules and + not self._is_egress_only_policy(policy)): + # NOTE(ltomasbo): only need to change services if the pods that + # they point to are updated + services = driver_utils.get_services(knp['metadata']['namespace']) + for service in services.get('items', []): + # TODO(ltomasbo): Skip other services that are not affected + # by the policy + # FIXME(dulek): Make sure to include svcs without selector when + # we start supporting them. + if (not service['spec'].get('selector') or not + self._is_service_affected(service, pods_to_update)): + continue + sgs = self._drv_svc_sg.get_security_groups(service, project_id) + self._drv_lbaas.update_lbaas_sg(service, sgs) + + self._patch_kuryrnetworkpolicy_crd(knp, 'status', + {'podSelector': current_sel}) + + def _is_service_affected(self, service, affected_pods): + svc_namespace = service['metadata']['namespace'] + svc_selector = service['spec'].get('selector') + svc_pods = driver_utils.get_pods({'selector': svc_selector}, + svc_namespace).get('items') + return any(pod in svc_pods for pod in affected_pods) + + def _is_egress_only_policy(self, policy): + policy_types = policy['spec'].get('policyTypes', []) + return (policy_types == ['Egress'] or + (policy['spec'].get('egress') and + not policy['spec'].get('ingress'))) + + def _get_policy_net_id(self, knp): + policy_ns = knp['metadata']['namespace'] + + kubernetes = clients.get_kubernetes_client() + try: + path = (f'{constants.K8S_API_CRD_NAMESPACES}/{policy_ns}/' + f'kuryrnetworks/{policy_ns}') + net_crd = kubernetes.get(path) + except exceptions.K8sClientException: + LOG.exception("Kubernetes Client Exception.") + raise + return net_crd['status']['netId'] + + def on_finalize(self, knp): + LOG.debug("Finalizing KuryrNetworkPolicy %s") + project_id = self._drv_project.get_project(knp) + pods_to_update = self._drv_policy.affected_pods(knp) + crd_sg = knp['status'].get('securityGroupId') + try: + policy = self._get_networkpolicy(knp['metadata']['annotations'] + ['networkPolicyLink']) + except exceptions.K8sResourceNotFound: + # NP is already gone, let's just try to clean up. + policy = None + + if crd_sg: + for pod in pods_to_update: + if driver_utils.is_host_network(pod): + continue + pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) + if crd_sg in pod_sgs: + pod_sgs.remove(crd_sg) + if not pod_sgs: + pod_sgs = CONF.neutron_defaults.pod_security_groups + if not pod_sgs: + raise cfg.RequiredOptError( + 'pod_security_groups', + cfg.OptGroup('neutron_defaults')) + try: + self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) + except os_exc.NotFoundException: + LOG.debug("Fail to update pod sgs." + " Retrying policy deletion.") + raise exceptions.ResourceNotReady(knp) + + # ensure ports at the pool don't have the NP sg associated + try: + net_id = self._get_policy_net_id(knp) + self._drv_vif_pool.remove_sg_from_pools(crd_sg, net_id) + except exceptions.K8sResourceNotFound: + # Probably the network got removed already, we can ignore it. + pass + + if (CONF.octavia_defaults.enforce_sg_rules and policy and + not self._is_egress_only_policy(policy)): + services = driver_utils.get_services( + knp['metadata']['namespace']) + for svc in services.get('items'): + if (not svc['spec'].get('selector') or not + self._is_service_affected(svc, pods_to_update)): + continue + sgs = self._drv_svc_sg.get_security_groups(svc, project_id) + self._drv_lbaas.update_lbaas_sg(svc, sgs) + + self._drv_policy.delete_np_sg(crd_sg) + + LOG.debug("Removing finalizers from KuryrNetworkPolicy and " + "NetworkPolicy.") + if policy: + self.k8s.remove_finalizer(policy, + constants.NETWORKPOLICY_FINALIZER) + self.k8s.remove_finalizer(knp, constants.NETWORKPOLICY_FINALIZER) diff --git a/kuryr_kubernetes/controller/handlers/pod_label.py b/kuryr_kubernetes/controller/handlers/pod_label.py index 56fde1574..9a234be47 100644 --- a/kuryr_kubernetes/controller/handlers/pod_label.py +++ b/kuryr_kubernetes/controller/handlers/pod_label.py @@ -58,21 +58,24 @@ def on_present(self, pod): # annotation to be moved to KuryrPort CRD. return - current_pod_labels = pod['metadata'].get('labels') - previous_pod_labels = self._get_pod_labels(pod) - LOG.debug("Got previous pod labels from annotation: %r", - previous_pod_labels) + current_pod_info = (pod['metadata'].get('labels'), + pod['status'].get('podIP')) + previous_pod_info = self._get_pod_info(pod) + LOG.debug("Got previous pod info from annotation: %r", + previous_pod_info) - if current_pod_labels == previous_pod_labels: + if current_pod_info == previous_pod_info: return + # FIXME(dulek): We should be able to just do create if only podIP + # changed, right? crd_pod_selectors = self._drv_sg.update_sg_rules(pod) project_id = self._drv_project.get_project(pod) security_groups = self._drv_sg.get_security_groups(pod, project_id) self._drv_vif_pool.update_vif_sgs(pod, security_groups) try: - self._set_pod_labels(pod, current_pod_labels) + self._set_pod_info(pod, current_pod_info) except k_exc.K8sResourceNotFound: LOG.debug("Pod already deleted, no need to retry.") return @@ -81,26 +84,30 @@ def on_present(self, pod): services = driver_utils.get_services() self._update_services(services, crd_pod_selectors, project_id) - def _get_pod_labels(self, pod): + def _get_pod_info(self, pod): try: annotations = pod['metadata']['annotations'] pod_labels_annotation = annotations[constants.K8S_ANNOTATION_LABEL] + pod_ip_annotation = annotations[constants.K8S_ANNOTATION_IP] except KeyError: - return None + return None, None pod_labels = jsonutils.loads(pod_labels_annotation) - return pod_labels + return pod_labels, pod_ip_annotation - def _set_pod_labels(self, pod, labels): - if not labels: - LOG.debug("Removing Label annotation: %r", labels) - annotation = None + def _set_pod_info(self, pod, info): + if not info[0]: + LOG.debug("Removing info annotations: %r", info) + annotation = None, info[1] else: - annotation = jsonutils.dumps(labels, sort_keys=True) - LOG.debug("Setting Labels annotation: %r", annotation) + annotation = jsonutils.dumps(info[0], sort_keys=True), info[1] + LOG.debug("Setting info annotations: %r", annotation) k8s = clients.get_kubernetes_client() k8s.annotate(pod['metadata']['selfLink'], - {constants.K8S_ANNOTATION_LABEL: annotation}, + { + constants.K8S_ANNOTATION_LABEL: annotation[0], + constants.K8S_ANNOTATION_IP: annotation[1] + }, resource_version=pod['metadata']['resourceVersion']) def _has_vifs(self, pod): @@ -117,6 +124,5 @@ def _update_services(self, services, crd_pod_selectors, project_id): if not driver_utils.service_matches_affected_pods( service, crd_pod_selectors): continue - sgs = self._drv_svc_sg.get_security_groups(service, - project_id) + sgs = self._drv_svc_sg.get_security_groups(service, project_id) self._drv_lbaas.update_lbaas_sg(service, sgs) diff --git a/kuryr_kubernetes/controller/handlers/policy.py b/kuryr_kubernetes/controller/handlers/policy.py index b916e3210..237b1c9fb 100644 --- a/kuryr_kubernetes/controller/handlers/policy.py +++ b/kuryr_kubernetes/controller/handlers/policy.py @@ -12,15 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from openstack import exceptions as os_exc -from oslo_config import cfg as oslo_cfg from oslo_log import log as logging from kuryr_kubernetes import clients from kuryr_kubernetes import constants as k_const from kuryr_kubernetes.controller.drivers import base as drivers -from kuryr_kubernetes.controller.drivers import utils as driver_utils -from kuryr_kubernetes import exceptions from kuryr_kubernetes.handlers import k8s_base from kuryr_kubernetes import utils @@ -36,99 +32,25 @@ class NetworkPolicyHandler(k8s_base.ResourceEventHandler): def __init__(self): super(NetworkPolicyHandler, self).__init__() self._drv_policy = drivers.NetworkPolicyDriver.get_instance() - self._drv_project = drivers.NetworkPolicyProjectDriver.get_instance() - self._drv_vif_pool = drivers.VIFPoolDriver.get_instance( - specific_driver='multi_pool') - self._drv_vif_pool.set_vif_driver() - self._drv_pod_sg = drivers.PodSecurityGroupsDriver.get_instance() - self._drv_svc_sg = drivers.ServiceSecurityGroupsDriver.get_instance() - self._drv_lbaas = drivers.LBaaSDriver.get_instance() + self.k8s = clients.get_kubernetes_client() def on_present(self, policy): LOG.debug("Created or updated: %s", policy) - project_id = self._drv_project.get_project(policy) - pods_to_update = [] - modified_pods = self._drv_policy.ensure_network_policy(policy, - project_id) - if modified_pods: - pods_to_update.extend(modified_pods) + self._drv_policy.ensure_network_policy(policy) - matched_pods = self._drv_policy.affected_pods(policy) - pods_to_update.extend(matched_pods) + # Put finalizer in if it's not there already. + self.k8s.add_finalizer(policy, k_const.NETWORKPOLICY_FINALIZER) - for pod in pods_to_update: - if driver_utils.is_host_network(pod): - continue - pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) - self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) - - if (pods_to_update and - oslo_cfg.CONF.octavia_defaults.enforce_sg_rules and - not self._is_egress_only_policy(policy)): - # NOTE(ltomasbo): only need to change services if the pods that - # they point to are updated - services = driver_utils.get_services( - policy['metadata']['namespace']) - for service in services.get('items'): - # TODO(ltomasbo): Skip other services that are not affected - # by the policy - if (not service['spec'].get('selector') or not - self._is_service_affected(service, pods_to_update)): - continue - sgs = self._drv_svc_sg.get_security_groups(service, - project_id) - self._drv_lbaas.update_lbaas_sg(service, sgs) - - def on_deleted(self, policy): - LOG.debug("Deleted network policy: %s", policy) - project_id = self._drv_project.get_project(policy) - pods_to_update = self._drv_policy.affected_pods(policy) - netpolicy_crd = self._drv_policy.get_kuryrnetpolicy_crd(policy) - if netpolicy_crd: - crd_sg = netpolicy_crd['spec'].get('securityGroupId') - for pod in pods_to_update: - if driver_utils.is_host_network(pod): - continue - pod_sgs = self._drv_pod_sg.get_security_groups(pod, - project_id) - if crd_sg in pod_sgs: - pod_sgs.remove(crd_sg) - if not pod_sgs: - pod_sgs = ( - oslo_cfg.CONF.neutron_defaults.pod_security_groups) - if not pod_sgs: - raise oslo_cfg.RequiredOptError( - 'pod_security_groups', - oslo_cfg.OptGroup('neutron_defaults')) - try: - self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) - except os_exc.NotFoundException: - LOG.debug("Fail to update pod sgs." - " Retrying policy deletion.") - raise exceptions.ResourceNotReady(policy) - - # ensure ports at the pool don't have the NP sg associated - net_id = self._get_policy_net_id(policy) - self._drv_vif_pool.remove_sg_from_pools(crd_sg, net_id) - - self._drv_policy.release_network_policy(netpolicy_crd) - - if (oslo_cfg.CONF.octavia_defaults.enforce_sg_rules and - not self._is_egress_only_policy(policy)): - services = driver_utils.get_services( - policy['metadata']['namespace']) - for svc in services.get('items'): - if (not svc['spec'].get('selector') or not - self._is_service_affected(svc, pods_to_update)): - continue - sgs = self._drv_svc_sg.get_security_groups(svc, - project_id) - self._drv_lbaas.update_lbaas_sg(svc, sgs) + def on_finalize(self, policy): + LOG.debug("Finalizing policy %s", policy) + if not self._drv_policy.release_network_policy(policy): + # KNP was not found, so we need to finalize on our own. + self.k8s.remove_finalizer(policy, k_const.NETWORKPOLICY_FINALIZER) def is_ready(self, quota): - if not (utils.has_kuryr_crd(k_const.K8S_API_CRD_KURYRNETPOLICIES) and - self._check_quota(quota)): + if not (utils.has_kuryr_crd(k_const.K8S_API_CRD_KURYRNETWORKPOLICIES) + and self._check_quota(quota)): LOG.error("Marking NetworkPolicyHandler as not ready.") return False return True @@ -137,29 +59,3 @@ def _check_quota(self, quota): if utils.has_limit(quota.security_groups): return utils.is_available('security_groups', quota.security_groups) return True - - def _is_service_affected(self, service, affected_pods): - svc_namespace = service['metadata']['namespace'] - svc_selector = service['spec'].get('selector') - svc_pods = driver_utils.get_pods({'selector': svc_selector}, - svc_namespace).get('items') - return any(pod in svc_pods for pod in affected_pods) - - def _get_policy_net_id(self, policy): - policy_ns = policy['metadata']['namespace'] - - kubernetes = clients.get_kubernetes_client() - try: - path = (f'{k_const.K8S_API_CRD_NAMESPACES}/{policy_ns}/' - f'kuryrnetworks/{policy_ns}') - net_crd = kubernetes.get(path) - except exceptions.K8sClientException: - LOG.exception("Kubernetes Client Exception.") - raise - return net_crd['status']['netId'] - - def _is_egress_only_policy(self, policy): - policy_types = policy['spec'].get('policyTypes', []) - return (policy_types == ['Egress'] or - (policy['spec'].get('egress') and - not policy['spec'].get('ingress'))) diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_network_policy.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_network_policy.py index 81fd2e5f8..2232f5104 100644 --- a/kuryr_kubernetes/tests/unit/controller/drivers/test_network_policy.py +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_network_policy.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import munch -from openstack import exceptions as os_exc from unittest import mock from kuryr_kubernetes.controller.drivers import network_policy @@ -75,8 +73,8 @@ def setUp(self): self._policy_uid = mock.sentinel.policy_uid self._policy_link = mock.sentinel.policy_link self._sg_id = mock.sentinel.sg_id - self._i_rules = [{'security_group_rule': {'id': ''}}] - self._e_rules = [{'security_group_rule': {'id': ''}}] + self._i_rules = [{'sgRule': {'id': ''}}] + self._e_rules = [{'sgRule': {'id': ''}}] self._policy = { 'apiVersion': 'networking.k8s.io/v1', @@ -104,12 +102,46 @@ def setUp(self): [{'namespaceSelector': { 'matchLabels': { 'project': 'myproject'}}}]}], - 'policyTypes': ['Ingress', 'Egress'] + 'policyTypes': ['Ingress', 'Egress'], + 'podSelector': {}, } } - self._crd = { - 'metadata': {'name': mock.sentinel.name, + self.crd = { + 'metadata': {'name': 'foobar', + 'namespace': 'default', + 'selfLink': mock.sentinel.selfLink}, + 'spec': { + 'egressSgRules': [ + {'sgRule': + {'description': 'Kuryr-Kubernetes NetPolicy SG rule', + 'direction': 'egress', + 'ethertype': 'IPv4', + 'port_range_max': 5978, + 'port_range_min': 5978, + 'protocol': 'tcp', + }}], + 'ingressSgRules': [ + {'sgRule': + {'description': 'Kuryr-Kubernetes NetPolicy SG rule', + 'direction': 'ingress', + 'ethertype': 'IPv4', + 'port_range_max': 6379, + 'port_range_min': 6379, + 'protocol': 'tcp', + }}], + 'podSelector': {}, + 'policyTypes': self._policy['spec']['policyTypes'] + }, + 'status': { + 'securityGroupId': self._sg_id, + 'securityGroupRules': [], + 'podSelector': {}, + } + } + + self.old_crd = { + 'metadata': {'name': 'np-foobar', 'namespace': 'default', 'selfLink': mock.sentinel.selfLink}, 'spec': { @@ -135,6 +167,7 @@ def setUp(self): 'security_group_id': self._sg_id, 'id': mock.sentinel.id }}], + 'podSelector': {}, 'networkpolicy_spec': self._policy['spec'], 'securityGroupId': self._sg_id, 'securityGroupName': mock.sentinel.sg_name}} @@ -144,207 +177,57 @@ def setUp(self): self._driver = network_policy.NetworkPolicyDriver() @mock.patch.object(network_policy.NetworkPolicyDriver, - 'get_kuryrnetpolicy_crd', return_value=False) - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'create_security_group_rules_from_network_policy') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'update_security_group_rules_from_network_policy') - def test_ensure_network_policy(self, m_update, m_create, m_get_crd): - self._driver.ensure_network_policy(self._policy, self._project_id) - - m_get_crd.assert_called_once_with(self._policy) - m_create.assert_called_once_with(self._policy, self._project_id) - m_update.assert_not_called() - - @mock.patch.object(network_policy.NetworkPolicyDriver, 'affected_pods') - @mock.patch.object(network_policy.NetworkPolicyDriver, 'namespaced_pods') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'get_kuryrnetpolicy_crd', return_value=True) - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'create_security_group_rules_from_network_policy') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'update_security_group_rules_from_network_policy') - def test_ensure_network_policy_with_existing_crd( - self, m_update, m_create, m_get_crd, m_namespaced, m_affected): - previous_selector = mock.sentinel.previous_selector - m_update.return_value = previous_selector - self._driver.ensure_network_policy(self._policy, self._project_id) - - m_get_crd.assert_called_once_with(self._policy) - m_create.assert_not_called() - m_update.assert_called_once_with(self._policy) - m_affected.assert_called_once_with(self._policy, previous_selector) - m_namespaced.assert_not_called() - - @mock.patch.object(network_policy.NetworkPolicyDriver, 'affected_pods') - @mock.patch.object(network_policy.NetworkPolicyDriver, 'namespaced_pods') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'get_kuryrnetpolicy_crd', return_value=True) - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'create_security_group_rules_from_network_policy') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'update_security_group_rules_from_network_policy') - def test_ensure_network_policy_with_existing_crd_no_selector( - self, m_update, m_create, m_get_crd, m_namespaced, m_affected): - m_update.return_value = None - self._driver.ensure_network_policy(self._policy, self._project_id) - - m_get_crd.assert_called_once_with(self._policy) - m_create.assert_not_called() - m_update.assert_called_once_with(self._policy) - m_affected.assert_not_called() - m_namespaced.assert_called_once_with(self._policy) - - @mock.patch.object(network_policy.NetworkPolicyDriver, 'affected_pods') - @mock.patch.object(network_policy.NetworkPolicyDriver, 'namespaced_pods') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'get_kuryrnetpolicy_crd') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'create_security_group_rules_from_network_policy') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'update_security_group_rules_from_network_policy') - def test_ensure_network_policy_with_existing_crd_empty_selector( - self, m_update, m_create, m_get_crd, m_namespaced, m_affected): - previous_selector = {} - pod_selector = {'matchLabels': {'run': 'demo'}} - updated_policy = self._policy.copy() - updated_policy['spec']['podSelector'] = pod_selector - crd_with_empty_selector = self._crd.copy() - crd_with_empty_selector['spec']['podSelector'] = previous_selector - - m_get_crd.return_value = crd_with_empty_selector - m_update.return_value = previous_selector - - self._driver.ensure_network_policy(updated_policy, self._project_id) - - m_get_crd.assert_called_once_with(updated_policy) - m_create.assert_not_called() - m_update.assert_called_once_with(updated_policy) - m_affected.assert_called_with(self._policy, previous_selector) - m_namespaced.assert_not_called() - - @mock.patch.object(network_policy.NetworkPolicyDriver, - '_add_default_np_rules') + '_get_default_np_rules') @mock.patch.object(network_policy.NetworkPolicyDriver, - 'get_kuryrnetpolicy_crd') + '_get_knp_crd', return_value=False) @mock.patch.object(network_policy.NetworkPolicyDriver, - '_add_kuryrnetpolicy_crd') + '_create_knp_crd') @mock.patch.object(network_policy.NetworkPolicyDriver, 'parse_network_policy_rules') @mock.patch.object(utils, 'get_subnet_cidr') - def test_create_security_group_rules_from_network_policy(self, m_utils, - m_parse, - m_add_crd, - m_get_crd, - m_add_default): - self._driver.os_net.create_security_group.return_value = ( - munch.Munch({'id': mock.sentinel.id, - 'security_group_rules': []})) + def test_ensure_network_policy(self, m_utils, m_parse, m_add_crd, + m_get_crd, m_get_default): m_utils.get_subnet_cidr.return_value = mock.sentinel.cidr m_parse.return_value = (self._i_rules, self._e_rules) - self._driver.os_net.create_security_group_rule.return_value = ( - munch.Munch({'id': mock.sentinel.id})) - self._driver.create_security_group_rules_from_network_policy( - self._policy, self._project_id) + self._driver.ensure_network_policy( + self._policy) m_get_crd.assert_called_once() m_add_crd.assert_called_once() - m_add_default.assert_called_once() + m_get_default.assert_called_once() @mock.patch.object(network_policy.NetworkPolicyDriver, - '_add_default_np_rules') + '_get_default_np_rules') @mock.patch.object(network_policy.NetworkPolicyDriver, - 'get_kuryrnetpolicy_crd') - @mock.patch.object(network_policy.NetworkPolicyDriver, - '_add_kuryrnetpolicy_crd') + '_get_knp_crd') @mock.patch.object(network_policy.NetworkPolicyDriver, 'parse_network_policy_rules') @mock.patch.object(utils, 'get_subnet_cidr') - def test_create_security_group_rules_with_k8s_exc(self, m_utils, m_parse, - m_add_crd, m_get_crd, - m_add_default): - self._driver.os_net.create_security_group.return_value = ( - munch.Munch({'id': mock.sentinel.id, - 'security_group_rules': []})) + def test_ensure_network_policy_with_k8s_exc(self, m_utils, m_parse, + m_get_crd, m_get_default): m_utils.get_subnet_cidr.return_value = mock.sentinel.cidr m_parse.return_value = (self._i_rules, self._e_rules) m_get_crd.side_effect = exceptions.K8sClientException - self._driver.os_net.create_security_group_rule.return_value = ( - munch.Munch({'id': mock.sentinel.id})) - self.assertRaises( - exceptions.K8sClientException, - self._driver.create_security_group_rules_from_network_policy, - self._policy, self._project_id) - m_add_crd.assert_called_once() - m_add_default.assert_called_once() + self.assertRaises(exceptions.K8sClientException, + self._driver.ensure_network_policy, self._policy) + m_get_default.assert_called_once() @mock.patch.object(network_policy.NetworkPolicyDriver, - '_add_default_np_rules') + '_get_default_np_rules') @mock.patch.object(network_policy.NetworkPolicyDriver, - 'get_kuryrnetpolicy_crd') - @mock.patch.object(network_policy.NetworkPolicyDriver, - '_add_kuryrnetpolicy_crd') + '_get_knp_crd', return_value=None) + @mock.patch.object(network_policy.NetworkPolicyDriver, '_create_knp_crd') @mock.patch.object(network_policy.NetworkPolicyDriver, 'parse_network_policy_rules') @mock.patch.object(utils, 'get_subnet_cidr') - def test_create_security_group_rules_error_add_crd(self, m_utils, m_parse, - m_add_crd, m_get_crd, - m_add_default): - self._driver.os_net.create_security_group.return_value = ( - munch.Munch({'id': mock.sentinel.id, - 'security_group_rules': []})) + def test_ensure_network_policy_error_add_crd( + self, m_utils, m_parse, m_add_crd, m_get_crd, m_get_default): m_utils.get_subnet_cidr.return_value = mock.sentinel.cidr m_parse.return_value = (self._i_rules, self._e_rules) m_add_crd.side_effect = exceptions.K8sClientException - self._driver.os_net.create_security_group_rule.return_value = ( - munch.Munch({'id': mock.sentinel.id})) - self.assertRaises( - exceptions.K8sClientException, - self._driver.create_security_group_rules_from_network_policy, - self._policy, self._project_id) - m_get_crd.assert_not_called() - m_add_default.assert_called_once() - - def test_create_security_group_rules_with_n_exc(self): - self._driver.os_net.create_security_group.side_effect = ( - os_exc.SDKException()) - self.assertRaises( - os_exc.SDKException, - self._driver.create_security_group_rules_from_network_policy, - self._policy, self._project_id) - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'create_security_group_rule') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'get_kuryrnetpolicy_crd') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'parse_network_policy_rules') - def test_update_security_group_rules(self, m_parse, m_get_crd, - m_create_sgr): - policy = self._policy.copy() - policy['spec']['podSelector'] = {'matchLabels': {'test': 'test'}} - m_get_crd.return_value = self._crd - m_parse.return_value = (self._i_rules, self._e_rules) - self._driver.update_security_group_rules_from_network_policy( - policy) - m_parse.assert_called_with(policy, self._sg_id) - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'create_security_group_rule') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'get_kuryrnetpolicy_crd') - @mock.patch.object(network_policy.NetworkPolicyDriver, - 'parse_network_policy_rules') - def test_update_security_group_rules_with_k8s_exc(self, m_parse, m_get_crd, - m_create_sgr): - self._driver.kubernetes.patch_crd.side_effect = ( - exceptions.K8sClientException()) - m_get_crd.return_value = self._crd - m_parse.return_value = (self._i_rules, self._e_rules) - self.assertRaises( - exceptions.K8sClientException, - self._driver.update_security_group_rules_from_network_policy, - self._policy) - m_parse.assert_called_with(self._policy, self._sg_id) + self.assertRaises(exceptions.K8sClientException, + self._driver.ensure_network_policy, self._policy) + m_get_crd.assert_called() + m_get_default.assert_called_once() def test_get_namespaces(self): namespace_selector = {'namespaceSelector': { @@ -363,6 +246,13 @@ def test_get_namespaces_no_matches(self): self.assertEqual([], resp) self.kubernetes.get.assert_called_once() + def test_get_from_old_crd(self): + knp = self._driver.get_from_old_crd(self.old_crd) + self.assertEqual(self.crd['spec'], knp['spec']) + self.assertEqual(self.crd['status'], knp['status']) + for k in ['name', 'namespace']: + self.assertEqual(self.crd['metadata'][k], knp['metadata'][k]) + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') @mock.patch.object(network_policy.NetworkPolicyDriver, '_get_resource_details') @@ -377,7 +267,7 @@ def test_parse_network_policy_rules_with_rules( namespace = 'myproject' m_get_namespaces.return_value = [get_namespace_obj()] m_get_resource_details.return_value = subnet_cidr, namespace - self._driver.parse_network_policy_rules(self._policy, self._sg_id) + self._driver.parse_network_policy_rules(self._policy) m_get_namespaces.assert_called() m_get_resource_details.assert_called() m_create.assert_called() @@ -391,12 +281,12 @@ def test_parse_network_policy_rules_with_no_rules(self, m_create, policy = self._policy.copy() policy['spec']['ingress'] = [{}] policy['spec']['egress'] = [{}] - self._driver.parse_network_policy_rules(policy, self._sg_id) + self._driver.parse_network_policy_rules(policy) m_get_ns.assert_not_called() - calls = [mock.call(self._sg_id, 'ingress', ethertype='IPv4'), - mock.call(self._sg_id, 'ingress', ethertype='IPv6'), - mock.call(self._sg_id, 'egress', ethertype='IPv4'), - mock.call(self._sg_id, 'egress', ethertype='IPv6')] + calls = [mock.call('ingress', ethertype='IPv4'), + mock.call('ingress', ethertype='IPv6'), + mock.call('egress', ethertype='IPv4'), + mock.call('egress', ethertype='IPv6')] m_create.assert_has_calls(calls) @mock.patch.object(network_policy.NetworkPolicyDriver, @@ -408,7 +298,7 @@ def test_parse_network_policy_rules_with_no_pod_selector( [{'port': 6379, 'protocol': 'TCP'}]}] policy['spec']['egress'] = [{'ports': [{'port': 6379, 'protocol': 'TCP'}]}] - self._driver.parse_network_policy_rules(policy, self._sg_id) + self._driver.parse_network_policy_rules(policy) m_create_all_pods_sg_rules.assert_called() @mock.patch.object(network_policy.NetworkPolicyDriver, @@ -429,7 +319,7 @@ def test_parse_network_policy_rules_with_ipblock(self, 'TCP'}], 'to': [{'ipBlock': {'cidr': '10.0.0.0/24'}}]}] - self._driver.parse_network_policy_rules(policy, self._sg_id) + self._driver.parse_network_policy_rules(policy) m_create_sg_rule.assert_called() @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') @@ -450,38 +340,19 @@ def test_parse_network_policy_rules_with_no_ports( selectors = {'namespaceSelector': { 'matchLabels': { 'project': 'myproject'}}} - policy['spec']['egress'] = [ - {'to': - [selectors]}] - policy['spec']['ingress'] = [ - {'from': - [selectors]}] - selectors = {'namespace_selector': selectors['namespaceSelector']} - self._driver.parse_network_policy_rules(policy, self._sg_id) + policy['spec']['egress'] = [{'to': [selectors]}] + policy['spec']['ingress'] = [{'from': [selectors]}] + self._driver.parse_network_policy_rules(policy) m_get_namespaces.assert_called() m_get_resource_details.assert_called() - calls = [mock.call(self._sg_id, 'ingress', port_range_min=1, + calls = [mock.call('ingress', port_range_min=1, port_range_max=65535, cidr=subnet_cidr, namespace=namespace), - mock.call(self._sg_id, 'egress', port_range_min=1, + mock.call('egress', port_range_min=1, port_range_max=65535, cidr=subnet_cidr, namespace=namespace)] m_create.assert_has_calls(calls) - def test_knps_on_namespace(self): - self.kubernetes.get.return_value = {'items': ['not-empty']} - namespace = 'test1' - - resp = self._driver.knps_on_namespace(namespace) - self.assertTrue(resp) - - def test_knps_on_namespace_empty(self): - self.kubernetes.get.return_value = {'items': []} - namespace = 'test1' - - resp = self._driver.knps_on_namespace(namespace) - self.assertFalse(resp) - @mock.patch.object(network_policy.NetworkPolicyDriver, 'namespaced_pods') def test_affected_pods(self, m_namespaced): self._driver.affected_pods(self._policy) @@ -509,19 +380,10 @@ def test_namespaced_pods(self): self.assertEqual([], resp) @mock.patch.object(network_policy.NetworkPolicyDriver, - '_del_kuryrnetpolicy_crd', return_value=False) + '_del_knp_crd', return_value=False) def test_release_network_policy(self, m_del_crd): - self._driver.release_network_policy(self._crd) - self.neutron.delete_security_group.assert_called_once_with( - self._crd['spec']['securityGroupId']) - m_del_crd.assert_called_once_with(self._crd['metadata']['name'], - self._crd['metadata']['namespace']) - - @mock.patch.object(network_policy.NetworkPolicyDriver, - '_del_kuryrnetpolicy_crd', return_value=False) - def test_release_network_policy_removed_crd(self, m_del_crd): - self._driver.release_network_policy(None) - m_del_crd.assert_not_called() + self._driver.release_network_policy(self.crd) + m_del_crd.assert_called_once_with(self.crd) @mock.patch.object(network_policy.NetworkPolicyDriver, '_create_sg_rules_with_container_ports') @@ -543,8 +405,7 @@ def test__create_sg_rule_body_on_text_port_ingress(self, m_get_pods.return_value = {'items': [pod]} m_get_ports.return_value = container_ports - self._driver._create_sg_rule_body_on_text_port(self._sg_id, - direction, + self._driver._create_sg_rule_body_on_text_port(direction, port, resources, crd_rules, @@ -577,8 +438,7 @@ def test__create_sg_rule_body_on_text_port_ingress_all(self, m_get_pods.return_value = {'items': [pod]} m_get_ports.return_value = container_ports - self._driver._create_sg_rule_body_on_text_port(self._sg_id, - direction, + self._driver._create_sg_rule_body_on_text_port(direction, port, resources, crd_rules, @@ -600,7 +460,7 @@ def test__create_sg_rule_body_on_text_port_ingress_match(self, m_create_sgr): def _create_sgr_cont(container_ports, allow_all, resource, - matched_pods, crd_rules, sg_id, direction, port, + matched_pods, crd_rules, direction, port, pod_selector=None, policy_namespace=None): matched_pods[container_ports[0][1]] = 'foo' @@ -617,8 +477,7 @@ def _create_sgr_cont(container_ports, allow_all, resource, m_get_pods.return_value = {'items': [pod]} m_get_ports.return_value = container_ports - self._driver._create_sg_rule_body_on_text_port(self._sg_id, - direction, + self._driver._create_sg_rule_body_on_text_port(direction, port, resources, crd_rules, @@ -629,7 +488,7 @@ def _create_sgr_cont(container_ports, allow_all, resource, m_get_pods.assert_called_with(pod_selector, namespace) m_get_ports.assert_called_with(pod, port) - calls = [mock.call(self._sg_id, direction, container_ports[0][1], + calls = [mock.call(direction, container_ports[0][1], protocol=port['protocol'], ethertype=e, pods='foo') for e in ('IPv4', 'IPv6')] @@ -656,8 +515,7 @@ def test__create_sg_rule_body_on_text_port_egress(self, m_get_pods.return_value = {'items': [pod]} m_get_ports.return_value = container_ports - self._driver._create_sg_rule_body_on_text_port(self._sg_id, - direction, + self._driver._create_sg_rule_body_on_text_port(direction, port, resources, crd_rules, @@ -685,8 +543,7 @@ def test__create_sg_rule_body_on_text_port_egress_all(self, m_get_ports.return_value = container_ports - self._driver._create_sg_rule_body_on_text_port(self._sg_id, - direction, + self._driver._create_sg_rule_body_on_text_port(direction, port, resources, crd_rules, @@ -695,8 +552,8 @@ def test__create_sg_rule_body_on_text_port_egress_all(self, allow_all=True) m_get_ports.assert_called_with(resources[0], port) - m_create_sgr.assert_called_once_with(self._sg_id, 'egress', None, - cidr=mock.ANY, protocol='TCP') + m_create_sgr.assert_called_once_with('egress', None, cidr=mock.ANY, + protocol='TCP') self.assertEqual(len(crd_rules), 1) @mock.patch('kuryr_kubernetes.utils.get_subnet_cidr') @@ -731,8 +588,7 @@ def _create_sgr_cont(container_ports, allow_all, resource, m_get_pods.return_value = {'items': [pod]} m_get_ports.return_value = container_ports - self._driver._create_sg_rule_body_on_text_port(self._sg_id, - direction, + self._driver._create_sg_rule_body_on_text_port(direction, port, resources, crd_rules, @@ -741,10 +597,10 @@ def _create_sgr_cont(container_ports, allow_all, resource, allow_all=True) m_get_ports.assert_called_with(resources[0], port) - calls = [mock.call(self._sg_id, direction, container_ports[0][1], + calls = [mock.call(direction, container_ports[0][1], protocol=port['protocol'], ethertype=e, pods='foo') for e in ('IPv4', 'IPv6')] - calls.append(mock.call(self._sg_id, direction, container_ports[0][1], + calls.append(mock.call(direction, container_ports[0][1], protocol=port['protocol'], cidr='10.0.0.128/26')) m_create_sgr.assert_has_calls(calls) @@ -758,19 +614,18 @@ def test__create_all_pods_sg_rules(self): direction = 'ingress' rules = [] - self._driver._create_all_pods_sg_rules(port, self._sg_id, direction, - rules, '', None) + self._driver._create_all_pods_sg_rules(port, direction, rules, '', + None) self.assertEqual(len(rules), 2) def test__create_default_sg_rule(self): for direction in ('ingress', 'egress'): rules = [] - self._driver._create_default_sg_rule(self._sg_id, direction, rules) + self._driver._create_default_sg_rule(direction, rules) self.assertEqual(len(rules), 2) - self.assertListEqual(rules, [{'security_group_rule': { + self.assertListEqual(rules, [{'sgRule': { 'ethertype': e, - 'security_group_id': self._sg_id, 'direction': direction, 'description': 'Kuryr-Kubernetes NetPolicy SG rule' }} for e in ('IPv4', 'IPv6')]) diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_network_policy_security_groups.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_network_policy_security_groups.py index a9e1a116e..15e789a74 100644 --- a/kuryr_kubernetes/tests/unit/controller/drivers/test_network_policy_security_groups.py +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_network_policy_security_groups.py @@ -66,7 +66,7 @@ def get_sg_rule(): pod_ip = get_match_crd_pod_obj()['status'].get('podIP') return { "namespace": 'dev', - "security_group_rule": { + "sgRule": { "description": "Kuryr-Kubernetes NetPolicy SG rule", "direction": "ingress", "ethertype": "IPv4", @@ -80,7 +80,7 @@ def get_sg_rule(): def get_matched_crd_obj(): return { - "kind": "KuryrNetPolicy", + "kind": "KuryrNetworkPolicy", "metadata": {"name": "np-test-network-policy", "namespace": "default"}, "spec": { @@ -159,7 +159,7 @@ def setUp(self): 'selfLink': mock.sentinel.selfLink}, 'spec': { 'egressSgRules': [ - {'security_group_rule': + {'sgRule': {'description': 'Kuryr-Kubernetes NetPolicy SG rule', 'direction': 'egress', 'ethertype': 'IPv4', @@ -170,7 +170,7 @@ def setUp(self): 'id': mock.sentinel.id }}], 'ingressSgRules': [ - {'security_group_rule': + {'sgRule': {'description': 'Kuryr-Kubernetes NetPolicy SG rule', 'direction': 'ingress', 'ethertype': 'IPv4', @@ -189,16 +189,18 @@ def setUp(self): 'production']}], 'matchLabels': { 'run': 'demo' - }}, + }}}, + 'status': { 'securityGroupId': self._sg_id, - 'securityGroupName': mock.sentinel.sg_name}} + }, + } self._crd2 = { 'metadata': {'name': mock.sentinel.name3, 'selfLink': mock.sentinel.selfLink}, 'spec': { 'ingressSgRules': [ - {'security_group_rule': + {'sgRule': {'description': 'Kuryr-Kubernetes NetPolicy SG rule', 'direction': 'ingress', 'ethertype': 'IPv4', @@ -208,25 +210,14 @@ def setUp(self): 'security_group_id': self._sg_id2, 'id': mock.sentinel.id }}], - 'podSelector': {}, + 'podSelector': {}}, + 'status': { 'securityGroupId': self._sg_id2, 'securityGroupName': mock.sentinel.sg_name}} - self._crds = { - "apiVersion": "v1", - "items": [self._crd], - "kind": "List", - "metadata": { - "resourceVersion": "", - "selfLink": mock.sentinel.selfLink}} - - self._multiple_crds = { - "apiVersion": "v1", - "items": [self._crd, self._crd2], - "kind": "List", - "metadata": { - "resourceVersion": "", - "selfLink": mock.sentinel.selfLink}} + self._crds = [self._crd] + + self._multiple_crds = [self._crd, self._crd2] self._pod = { 'apiVersion': 'v1', @@ -304,7 +295,7 @@ def setUp(self): self._crd_sg_id = mock.sentinel.crd_sg_id self._sg_rule_body = { - 'security_group_rule': { + 'sgRule': { 'direction': 'ingress', 'protocol': 'tcp', 'description': 'Kuryr-Kubernetes NetPolicy SG rule', @@ -323,7 +314,7 @@ def setUp(self): "spec": { "egressSgRules": [], "ingressSgRules": [{ - "security_group_rule": { + "sgRule": { "description": "Kuryr-Kubernetes NetPolicy SG rule", "direction": "ingress", "ethertype": "IPv4", @@ -348,20 +339,12 @@ def setUp(self): "podSelector": {"matchLabels": {"app": "demo"}}, "securityGroupId": self._crd_sg_id}} - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'create_security_group_rule') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'create_security_group_rule_body') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' 'match_selector', return_value=True) @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_ip') def test__create_sg_rules(self, m_get_pod_ip, - m_match_selector, - m_create_sg_rule_body, - m_create_sg_rule): - m_create_sg_rule_body.return_value = self._sg_rule_body + m_match_selector): sgr_id = mock.sentinel.sgr_id - m_create_sg_rule.return_value = sgr_id crd = get_crd_obj_with_all_selectors() pod = get_match_crd_pod_obj() m_get_pod_ip.return_value = pod['status'].get('podIP') @@ -370,80 +353,58 @@ def test__create_sg_rules(self, m_get_pod_ip, policy = crd['spec']['networkpolicy_spec'] rule_list = policy.get('ingress', None) - crd_rules = crd['spec'].get('ingressSgRules') pod_ns = pod['metadata']['namespace'] for rule_block in rule_list: for rule in rule_block.get('from', []): pod_selector = rule.get('podSelector') matched = network_policy_security_groups._create_sg_rules( - crd, pod, pod_selector, rule_block, - crd_rules, 'ingress', matched, pod_ns) + crd, pod, pod_selector, rule_block, 'ingress', matched) new_sg_rule['namespace'] = pod_ns - new_sg_rule['security_group_rule']['id'] = sgr_id + new_sg_rule['sgRule']['id'] = sgr_id m_match_selector.assert_called_once_with( pod_selector, pod['metadata']['labels']) m_get_pod_ip.assert_called_once_with(pod) - m_create_sg_rule_body.assert_called_once() - m_create_sg_rule.assert_called_once() - self.assertEqual([new_sg_rule], crd_rules) self.assertEqual(matched, True) @mock.patch('kuryr_kubernetes.controller.drivers.utils.' 'get_pod_ip') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' 'match_selector', return_value=False) - def test__create_sg_rules_no_match(self, m_match_selector, - m_get_pod_ip): + def test__create_sg_rules_no_match(self, m_match_selector, m_get_pod_ip): crd = get_crd_obj_with_all_selectors() pod = self._pod2 policy = crd['spec']['networkpolicy_spec'] rule_list = policy.get('ingress', None) - crd_rules = crd['spec'].get('ingressSgRules') for rule_block in rule_list: for rule in rule_block.get('from', []): pod_selector = rule.get('podSelector') matched = network_policy_security_groups._create_sg_rules( - crd, pod, pod_selector, rule_block, - crd_rules, 'ingress', False, self._namespace) + crd, pod, pod_selector, rule_block, 'ingress', False) self.assertEqual(matched, False) + @mock.patch('kuryr_kubernetes.controller.drivers.' + 'network_policy_security_groups._bump_networkpolicy') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'patch_kuryrnetworkpolicy_crd') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_kuryrnetpolicy_crds') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'delete_security_group_rule') + 'get_kuryrnetworkpolicy_crds') @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_ip') - def test_delete_sg_rules(self, m_get_pod_ip, m_delete_sg_rule, - m_get_knp_crds, m_patch_kuryrnetworkpolicy_crd): + def test_delete_sg_rules(self, m_get_pod_ip, m_get_knp_crds, m_bump): crd = self._crd_with_rule - i_rule = crd['spec'].get('ingressSgRules')[0] - sgr_id = i_rule['security_group_rule'].get('id') m_get_pod_ip.return_value = self._pod_ip - m_get_knp_crds.return_value = { - "apiVersion": "v1", - "items": [crd], - "kind": "List", - "metadata": { - "resourceVersion": "", - "selfLink": mock.sentinel.selfLink}} - i_rules = e_rules = [] + m_get_knp_crds.return_value = [crd] pod = self._pod_dev_namespace self._driver.delete_sg_rules(pod) m_get_knp_crds.assert_called_once() m_get_pod_ip.assert_called_once_with(pod) - m_delete_sg_rule.assert_called_once_with(sgr_id) - m_patch_kuryrnetworkpolicy_crd.assert_called_with( - crd, i_rules, e_rules, crd['spec'].get('podSelector')) + m_bump.assert_called_once() @mock.patch('kuryr_kubernetes.config.CONF') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_kuryrnetpolicy_crds') + 'get_kuryrnetworkpolicy_crds') def test_get_sgs_for_pod_without_label(self, m_get_crds, m_cfg): m_get_crds.return_value = self._crds sg_list = [str(mock.sentinel.sg_id)] @@ -460,7 +421,7 @@ def test_get_sgs_for_pod_without_label(self, m_get_crds, m_cfg): @mock.patch('kuryr_kubernetes.controller.drivers.utils.' 'match_labels') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_kuryrnetpolicy_crds') + 'get_kuryrnetworkpolicy_crds') def test_get_sgs_for_pod_with_label(self, m_get_crds, m_match_labels, m_match_expressions): m_get_crds.return_value = self._crds @@ -474,7 +435,7 @@ def test_get_sgs_for_pod_with_label(self, m_get_crds, m_match_labels, self._crd['spec']['podSelector']['matchExpressions'], pod_labels) m_match_labels.assert_called_once_with( self._crd['spec']['podSelector']['matchLabels'], pod_labels) - self.assertEqual(resp, [str(self._sg_id)]) + self.assertEqual(resp, [self._sg_id]) @mock.patch('kuryr_kubernetes.config.CONF') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' @@ -482,7 +443,7 @@ def test_get_sgs_for_pod_with_label(self, m_get_crds, m_match_labels, @mock.patch('kuryr_kubernetes.controller.drivers.utils.' 'match_labels') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_kuryrnetpolicy_crds') + 'get_kuryrnetworkpolicy_crds') def test_get_sgs_for_pod_with_label_no_match(self, m_get_crds, m_match_labels, m_match_expressions, m_cfg): @@ -503,9 +464,9 @@ def test_get_sgs_for_pod_with_label_no_match(self, m_get_crds, self.assertEqual(sg_list, sgs) @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_kuryrnetpolicy_crds') + 'get_kuryrnetworkpolicy_crds') def test_get_sgs_no_crds(self, m_get_crds): - m_get_crds.return_value = {"items": []} + m_get_crds.return_value = [] cfg.CONF.set_override('pod_security_groups', [], group='neutron_defaults') @@ -519,7 +480,7 @@ def test_get_sgs_no_crds(self, m_get_crds): @mock.patch('kuryr_kubernetes.controller.drivers.utils.' 'match_labels') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_kuryrnetpolicy_crds') + 'get_kuryrnetworkpolicy_crds') def test_get_sgs_multiple_crds(self, m_get_crds, m_match_labels, m_match_expressions): m_match_expressions.return_value = True @@ -529,87 +490,64 @@ def test_get_sgs_multiple_crds(self, m_get_crds, m_match_labels, resp = self._driver.get_security_groups(self._pod, self._project_id) m_get_crds.assert_called_once_with(namespace=self._namespace) - self.assertEqual([str(self._sg_id), str(self._sg_id2)], resp) + self.assertEqual([self._sg_id, self._sg_id2], resp) + @mock.patch('kuryr_kubernetes.controller.drivers.' + 'network_policy_security_groups._bump_networkpolicy') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'patch_kuryrnetworkpolicy_crd') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'delete_security_group_rule') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_kuryrnetpolicy_crds') - def test_delete_namespace_sg_rule(self, m_get_knp_crd, m_delete_sg_rule, - m_patch_kuryrnetworkpolicy_crd): + 'get_kuryrnetworkpolicy_crds') + def test_delete_namespace_sg_rule(self, m_get_knp_crd, m_bump): cls = network_policy_security_groups.NetworkPolicySecurityGroupsDriver m_driver = mock.MagicMock(spec=cls) - i_rule = get_matched_crd_obj()['spec']['ingressSgRules'][0] - sg_rule_id = i_rule.get('security_group_rule')['id'] - m_get_knp_crd.return_value = {"items": [get_matched_crd_obj()]} + m_get_knp_crd.return_value = [get_matched_crd_obj()] cls.delete_namespace_sg_rules(m_driver, get_match_crd_namespace_obj()) m_get_knp_crd.assert_called_once() - m_delete_sg_rule.assert_called_once_with(sg_rule_id) - m_patch_kuryrnetworkpolicy_crd.assert_called_once() + m_bump.assert_called_once() - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'patch_kuryrnetworkpolicy_crd') + @mock.patch('kuryr_kubernetes.controller.drivers.' + 'network_policy_security_groups._bump_networkpolicy') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' 'delete_security_group_rule') @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_kuryrnetpolicy_crds') - def test_delete_namespace_sg_rule_no_match(self, m_get_knp_crd, - m_delete_sg_rule, - m_patch_kuryrnetworkpolicy_crd): + 'get_kuryrnetworkpolicy_crds') + def test_delete_namespace_sg_rule_no_match( + self, m_get_knp_crd, m_delete_sg_rule, m_bump): cls = network_policy_security_groups.NetworkPolicySecurityGroupsDriver m_driver = mock.MagicMock(spec=cls) - m_get_knp_crd.return_value = {"items": [get_matched_crd_obj()]} + m_get_knp_crd.return_value = [get_matched_crd_obj()] cls.delete_namespace_sg_rules(m_driver, get_no_match_crd_namespace_obj()) m_get_knp_crd.assert_called_once() m_delete_sg_rule.assert_not_called() - m_patch_kuryrnetworkpolicy_crd.assert_not_called() + m_bump.assert_not_called() - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_pods') - @mock.patch('kuryr_kubernetes.controller.drivers.' - 'network_policy_security_groups._create_sg_rule') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'match_selector') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_namespace_subnet_cidr') - def test__parse_rules(self, m_get_ns_subnet_cidr, m_match_selector, - m_create_sg_rule, m_get_pods): + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pods') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.match_selector') + def test__parse_rules(self, m_match_selector, m_get_pods): crd = get_crd_obj_no_match() policy = crd['spec']['networkpolicy_spec'] i_rule = policy.get('ingress')[0] ns_selector = i_rule['from'][0].get('namespaceSelector') ns = get_match_crd_namespace_obj() - m_get_ns_subnet_cidr.return_value = '10.0.2.0/26' m_match_selector.return_value = True - m_create_sg_rule.return_value = get_sg_rule() - matched, rules = network_policy_security_groups._parse_rules( - 'ingress', crd, namespace=ns) + matched = network_policy_security_groups._parse_rules( + 'ingress', crd, policy, namespace=ns) - m_get_ns_subnet_cidr.assert_called_once_with(ns) m_match_selector.assert_called_once_with(ns_selector, ns['metadata']['labels']) - m_create_sg_rule.assert_called_once() self.assertEqual(matched, True) - self.assertEqual(rules, [get_sg_rule()]) - @mock.patch('kuryr_kubernetes.controller.drivers.' - 'network_policy_security_groups._create_sg_rule') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'match_selector') - def test__parse_rules_no_match(self, m_match_selector, - m_create_sg_rule): + @mock.patch('kuryr_kubernetes.controller.drivers.utils.match_selector') + def test__parse_rules_no_match(self, m_match_selector): crd = get_crd_obj_no_match() policy = crd['spec']['networkpolicy_spec'] i_rule = policy.get('ingress')[0] @@ -618,26 +556,19 @@ def test__parse_rules_no_match(self, m_match_selector, m_match_selector.return_value = False - matched, rules = network_policy_security_groups._parse_rules( - 'ingress', crd, namespace=ns) + matched = network_policy_security_groups._parse_rules( + 'ingress', crd, policy, namespace=ns) m_match_selector.assert_called_once_with(ns_selector, ns['metadata']['labels']) - m_create_sg_rule.assert_not_called() self.assertEqual(matched, False) - self.assertEqual(rules, []) - @mock.patch('kuryr_kubernetes.controller.drivers.' - 'network_policy_security_groups._create_sg_rule') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_pod_ip') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_pods') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'match_selector') - def test__parse_rules_all_selectors(self, m_match_selector, m_get_pods, - m_get_pod_ip, m_create_sg_rule): + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pods') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_pod_ip') + @mock.patch('kuryr_kubernetes.controller.drivers.utils.match_selector') + def test__parse_rules_all_selectors(self, m_match_selector, m_get_pod_ip, + m_get_pods): crd = get_crd_obj_with_all_selectors() policy = crd['spec']['networkpolicy_spec'] i_rule = policy.get('ingress')[0] @@ -647,22 +578,19 @@ def test__parse_rules_all_selectors(self, m_match_selector, m_get_pods, pod = get_match_crd_pod_obj() m_match_selector.return_value = True - m_get_pods.return_value = {"items": [pod]} m_get_pod_ip.return_value = pod['status']['podIP'] - m_create_sg_rule.return_value = get_sg_rule() + m_get_pods.return_value = {"items": [pod]} - matched, rules = network_policy_security_groups._parse_rules( - 'ingress', crd, namespace=ns) + matched = network_policy_security_groups._parse_rules( + 'ingress', crd, policy, namespace=ns) m_match_selector.assert_called_once_with(ns_selector, ns['metadata']['labels']) m_get_pods.assert_called_once_with(pod_selector, ns['metadata']['name']) m_get_pod_ip.assert_called_once_with(pod) - m_create_sg_rule.assert_called_once() self.assertEqual(matched, True) - self.assertEqual(rules, [get_sg_rule()]) @mock.patch('kuryr_kubernetes.controller.drivers.' 'network_policy_security_groups._parse_selectors_on_pod') @@ -670,124 +598,26 @@ def test__parse_rules_multiple_selectors(self, m_parse_selectors_on_pod): no_selector = None matched_selector = True pod = mock.sentinel.pod - final_crd_rules = [mock.sentinel.crd_rules] - m_parse_selectors_on_pod.side_effect = [ - (matched_selector, final_crd_rules)]*2 + m_parse_selectors_on_pod.side_effect = [matched_selector]*2 - initial_crd_rules = [] direction = "ingress" pod_selector = mock.sentinel.pod_selector namespace_selector = mock.sentinel.namespace_selector rule_block = {'from': [{'podSelector': pod_selector}, {'namespaceSelector': namespace_selector}]} - crd = {"spec": { - "ingressSgRules": initial_crd_rules, - "networkpolicy_spec": { - "ingress": [rule_block], - "policyTypes": [ - "Ingress" - ]}, }} + policy = { + "ingress": [rule_block], + "policyTypes": ["Ingress"] + } + crd = {"spec": {"ingressSgRules": []}} - matched, rules = network_policy_security_groups._parse_rules( - direction, crd, pod=pod) + matched = network_policy_security_groups._parse_rules( + direction, crd, policy, pod=pod) calls = [mock.call(crd, pod, pod_selector, no_selector, rule_block, - initial_crd_rules, direction, not matched_selector), + direction, not matched_selector), mock.call(crd, pod, no_selector, namespace_selector, - rule_block, final_crd_rules, direction, - matched_selector)] + rule_block, direction, matched_selector)] m_parse_selectors_on_pod.assert_has_calls(calls) self.assertEqual(matched, matched_selector) - self.assertEqual(rules, final_crd_rules) - - -class TestNetworkPolicySecurityGroupsFunctions(test_base.TestCase): - - def setUp(self): - super().setUp() - self.kubernetes = self.useFixture(k_fix.MockK8sClient()).client - self.npsg = network_policy_security_groups - self.sg_id = mock.sentinel.sg_id - - self.crd = { - 'spec': { - 'ingressSgRules': [], - 'networkpolicy_spec': { - 'ingress': [], - 'policyTypes': ['Ingress'] - } - }, - 'metadata': {'namespace': 'ns'} - } - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'create_security_group_rule') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'create_security_group_rule_body') - def test__apply_sg_rules_on_matched_pods_empty_match(self, m_create_sgrb, - m_create_sgr): - self.npsg._apply_sg_rules_on_matched_pods({}, self.sg_id, 'ingress', - 'ns', 'port', 'crd_rules') - - m_create_sgrb.assert_not_called() - m_create_sgr.assert_not_called() - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_namespace_subnet_cidr') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_namespace') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'create_security_group_rule') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'create_security_group_rule_body') - def test__apply_sg_rules_on_matched_pods_not_all(self, m_create_sgrb, - m_create_sgr, m_get_ns, - m_get_ns_sub_cidr): - pod = mock.sentinel.pod - ns = mock.sentinel.ns - port = {'protocol': 'TCP', 'port': 22} - matched_pods = {'container_port': [pod]} - - m_get_ns.return_value = ns - m_create_sgrb.return_value = {'security_group_rule': {}} - crd_rules = [] - direction = 'ingress' - - self.npsg._apply_sg_rules_on_matched_pods(matched_pods, self.sg_id, - direction, 'ns', port, - crd_rules) - - m_get_ns_sub_cidr.assert_called_once_with(ns) - m_create_sgrb.assert_called_once_with(self.sg_id, direction, - 'container_port', - protocol=mock.ANY, cidr=mock.ANY, - pods=[pod]) - m_create_sgr.assert_called_once() - self.assertEqual(len(crd_rules), 1) - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_namespace_subnet_cidr') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'get_namespace') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.' - 'create_security_group_rule') - def test__apply_sg_rules_on_matched_pods_all(self, m_create_sgr, m_get_ns, - m_get_ns_sub_cidr): - pod = mock.sentinel.pod - ns = mock.sentinel.ns - port = {'protocol': 'TCP', 'port': 22} - matched_pods = {'container_port': [pod]} - - m_get_ns.return_value = ns - crd_rules = [] - direction = 'ingress' - - self.npsg._apply_sg_rules_on_matched_pods(matched_pods, self.sg_id, - direction, 'ns', port, - crd_rules, allow_all=True) - - self.assertEqual(m_create_sgr.call_count, 2) - self.assertEqual(len(crd_rules), 2) - self.assertListEqual([r['security_group_rule']['ethertype'] - for r in crd_rules], ['IPv4', 'IPv6']) diff --git a/kuryr_kubernetes/tests/unit/controller/handlers/test_kuryrnetworkpolicy.py b/kuryr_kubernetes/tests/unit/controller/handlers/test_kuryrnetworkpolicy.py new file mode 100644 index 000000000..72c2ec335 --- /dev/null +++ b/kuryr_kubernetes/tests/unit/controller/handlers/test_kuryrnetworkpolicy.py @@ -0,0 +1,112 @@ +# Copyright 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +from kuryr_kubernetes.controller.drivers import base as drivers +from kuryr_kubernetes.controller.handlers import kuryrnetworkpolicy +from kuryr_kubernetes.tests import base as test_base + + +class TestPolicyHandler(test_base.TestCase): + + @mock.patch.object(drivers.LBaaSDriver, 'get_instance') + @mock.patch.object(drivers.NetworkPolicyDriver, 'get_instance') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + @mock.patch('kuryr_kubernetes.clients.get_network_client') + @mock.patch('kuryr_kubernetes.clients.get_loadbalancer_client') + def setUp(self, m_get_os_lb, m_get_os_net, m_get_k8s, m_get_np, + m_get_lbaas): + super(TestPolicyHandler, self).setUp() + + self._project_id = mock.sentinel.project_id + self._policy_name = 'np-test' + self._policy_uid = mock.sentinel.policy_uid + self._policy_link = mock.sentinel.policy_link + + self._policy = { + 'apiVersion': 'networking.k8s.io/v1', + 'kind': 'NetworkPolicy', + 'metadata': { + 'name': self._policy_name, + 'resourceVersion': '2259309', + 'generation': 1, + 'creationTimestamp': '2018-09-18T14:09:51Z', + 'namespace': 'default', + 'annotations': {}, + 'selfLink': self._policy_link, + 'uid': self._policy_uid + }, + 'spec': { + 'egress': [{'ports': [{'port': 5978, 'protocol': 'TCP'}]}], + 'ingress': [{'ports': [{'port': 6379, 'protocol': 'TCP'}]}], + 'policyTypes': ['Ingress', 'Egress'] + } + } + + self.k8s = mock.Mock() + m_get_k8s.return_value = self.k8s + self.m_get_k8s = m_get_k8s + + self.os_net = mock.Mock() + m_get_os_net.return_value = self.os_net + self.m_get_os_net = m_get_os_net + + self.np_driver = mock.Mock() + m_get_np.return_value = self.np_driver + self.m_get_np = m_get_np + + self.lbaas_driver = mock.Mock() + m_get_lbaas.return_value = self.lbaas_driver + self.m_get_lbaas = m_get_lbaas + + self.k8s.get.return_value = {} + self.handler = kuryrnetworkpolicy.KuryrNetworkPolicyHandler() + + def _get_knp_obj(self): + knp_obj = { + 'apiVersion': 'openstack.org/v1', + 'kind': 'KuryrNetworkPolicy', + 'metadata': { + 'name': 'np-test-network-policy', + 'namespace': 'test-1', + }, + 'spec': { + 'securityGroupId': 'c1ac16f5-e198-4628-9d84-253c6001be8e', + 'securityGroupName': 'sg-test-network-policy' + }} + return knp_obj + + def test_init(self): + self.m_get_k8s.assert_called_once() + self.m_get_np.assert_called_once() + + self.assertEqual(self.np_driver, self.handler._drv_policy) + self.assertEqual(self.k8s, self.handler.k8s) + self.assertEqual(self.os_net, self.handler.os_net) + self.assertEqual(self.lbaas_driver, self.handler._drv_lbaas) + + def test_convert(self): + self.k8s.get.return_value = {'items': [{ + 'metadata': { + 'selfLink': mock.sentinel.old_self_link, + 'namespace': 'ns', + } + }]} + self.np_driver.get_from_old_crd.return_value = mock.sentinel.new_crd + + self.handler._convert_old_crds() + + self.k8s.post.assert_called_once_with(mock.ANY, mock.sentinel.new_crd) + self.k8s.delete.assert_called_once_with(mock.sentinel.old_self_link) diff --git a/kuryr_kubernetes/tests/unit/controller/handlers/test_pod_label.py b/kuryr_kubernetes/tests/unit/controller/handlers/test_pod_label.py index 9011a7a5e..548b33cb4 100644 --- a/kuryr_kubernetes/tests/unit/controller/handlers/test_pod_label.py +++ b/kuryr_kubernetes/tests/unit/controller/handlers/test_pod_label.py @@ -47,8 +47,8 @@ def setUp(self): self._get_project = self._handler._drv_project.get_project self._get_security_groups = self._handler._drv_sg.get_security_groups self._set_vif_driver = self._handler._drv_vif_pool.set_vif_driver - self._get_pod_labels = self._handler._get_pod_labels - self._set_pod_labels = self._handler._set_pod_labels + self._get_pod_info = self._handler._get_pod_info + self._set_pod_info = self._handler._set_pod_info self._has_vifs = self._handler._has_vifs self._update_vif_sgs = self._handler._drv_vif_pool.update_vif_sgs @@ -81,16 +81,16 @@ def test_init(self, m_get_lbaas_driver, m_get_project_driver, def test_on_present(self, m_get_services): m_get_services.return_value = {"items": []} self._has_vifs.return_value = True - self._get_pod_labels.return_value = {'test1': 'test'} + self._get_pod_info.return_value = ({'test1': 'test'}, '192.168.0.1') p_label.PodLabelHandler.on_present(self._handler, self._pod) self._has_vifs.assert_called_once_with(self._pod) - self._get_pod_labels.assert_called_once_with(self._pod) + self._get_pod_info.assert_called_once_with(self._pod) self._get_project.assert_called_once() self._get_security_groups.assert_called_once() self._update_vif_sgs.assert_called_once_with(self._pod, [self._sg_id]) - self._set_pod_labels.assert_called_once_with(self._pod, None) + self._set_pod_info.assert_called_once_with(self._pod, (None, None)) def test_on_present_no_state(self): self._has_vifs.return_value = False @@ -99,27 +99,29 @@ def test_on_present_no_state(self): self.assertIsNone(resp) self._has_vifs.assert_called_once_with(self._pod) - self._get_pod_labels.assert_not_called() - self._set_pod_labels.assert_not_called() + self._get_pod_info.assert_not_called() + self._set_pod_info.assert_not_called() - def test_on_present_no_labels(self): + @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') + def test_on_present_no_labels(self, m_get_services): self._has_vifs.return_value = True - self._get_pod_labels.return_value = None + self._get_pod_info.return_value = None, None p_label.PodLabelHandler.on_present(self._handler, self._pod) self._has_vifs.assert_called_once_with(self._pod) - self._get_pod_labels.assert_called_once_with(self._pod) - self._set_pod_labels.assert_not_called() + self._get_pod_info.assert_called_once_with(self._pod) + self._set_pod_info.assert_not_called() def test_on_present_no_changes(self): self._has_vifs.return_value = True pod_with_label = self._pod.copy() pod_with_label['metadata']['labels'] = {'test1': 'test'} - self._get_pod_labels.return_value = {'test1': 'test'} + pod_with_label['status']['podIP'] = '192.168.0.1' + self._get_pod_info.return_value = ({'test1': 'test'}, '192.168.0.1') p_label.PodLabelHandler.on_present(self._handler, pod_with_label) self._has_vifs.assert_called_once_with(pod_with_label) - self._get_pod_labels.assert_called_once_with(pod_with_label) - self._set_pod_labels.assert_not_called() + self._get_pod_info.assert_called_once_with(pod_with_label) + self._set_pod_info.assert_not_called() diff --git a/kuryr_kubernetes/tests/unit/controller/handlers/test_policy.py b/kuryr_kubernetes/tests/unit/controller/handlers/test_policy.py index 0dbb01cee..57e6bf3eb 100644 --- a/kuryr_kubernetes/tests/unit/controller/handlers/test_policy.py +++ b/kuryr_kubernetes/tests/unit/controller/handlers/test_policy.py @@ -21,14 +21,15 @@ class TestPolicyHandler(test_base.TestCase): - def setUp(self): + @mock.patch.object(drivers.NetworkPolicyDriver, 'get_instance') + @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') + def setUp(self, m_get_k8s, m_get_np): super(TestPolicyHandler, self).setUp() self._project_id = mock.sentinel.project_id self._policy_name = 'np-test' self._policy_uid = mock.sentinel.policy_uid self._policy_link = mock.sentinel.policy_link - self._pod_sg = mock.sentinel.pod_sg self._policy = { 'apiVersion': 'networking.k8s.io/v1', @@ -50,198 +51,31 @@ def setUp(self): } } - self._handler = mock.MagicMock(spec=policy.NetworkPolicyHandler) - - self._handler._drv_project = mock.Mock( - spec=drivers.NetworkPolicyProjectDriver) - self._handler._drv_policy = mock.MagicMock( - spec=drivers.NetworkPolicyDriver) - self._handler._drv_pod_sg = mock.Mock( - spec=drivers.PodSecurityGroupsDriver) - self._handler._drv_svc_sg = mock.Mock( - spec=drivers.ServiceSecurityGroupsDriver) - self._handler._drv_vif_pool = mock.MagicMock( - spec=drivers.VIFPoolDriver) - self._handler._drv_lbaas = mock.Mock( - spec=drivers.LBaaSDriver) - - self._get_project = self._handler._drv_project.get_project - self._get_project.return_value = self._project_id - self._get_security_groups = ( - self._handler._drv_pod_sg.get_security_groups) - self._set_vifs_driver = self._handler._drv_vif_pool.set_vif_driver - self._set_vifs_driver.return_value = mock.Mock( - spec=drivers.PodVIFDriver) - self._update_vif_sgs = self._handler._drv_vif_pool.update_vif_sgs - self._update_vif_sgs.return_value = None - self._update_lbaas_sg = self._handler._drv_lbaas.update_lbaas_sg - self._update_lbaas_sg.return_value = None - self._remove_sg = self._handler._drv_vif_pool.remove_sg_from_pools - self._remove_sg.return_value = None - - def _get_knp_obj(self): - knp_obj = { - 'apiVersion': 'openstack.org/v1', - 'kind': 'KuryrNetPolicy', - 'metadata': { - 'name': 'np-test-network-policy', - 'namespace': 'test-1' - }, - 'spec': { - 'securityGroupId': 'c1ac16f5-e198-4628-9d84-253c6001be8e', - 'securityGroupName': 'sg-test-network-policy' - }} - return knp_obj - - @mock.patch.object(drivers.LBaaSDriver, 'get_instance') - @mock.patch.object(drivers.ServiceSecurityGroupsDriver, 'get_instance') - @mock.patch.object(drivers.PodSecurityGroupsDriver, 'get_instance') - @mock.patch.object(drivers.VIFPoolDriver, 'get_instance') - @mock.patch.object(drivers.NetworkPolicyDriver, 'get_instance') - @mock.patch.object(drivers.NetworkPolicyProjectDriver, 'get_instance') - def test_init(self, m_get_project_driver, m_get_policy_driver, - m_get_vif_driver, m_get_pod_sg_driver, m_get_svc_sg_driver, - m_get_lbaas_driver): - handler = policy.NetworkPolicyHandler() - - m_get_project_driver.assert_called_once() - m_get_policy_driver.assert_called_once() - m_get_vif_driver.assert_called_once() - m_get_pod_sg_driver.assert_called_once() - m_get_svc_sg_driver.assert_called_once() - m_get_lbaas_driver.assert_called_once() - - self.assertEqual(m_get_project_driver.return_value, - handler._drv_project) - self.assertEqual(m_get_policy_driver.return_value, handler._drv_policy) - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - def test_on_present(self, m_host_network, m_get_services): - modified_pod = mock.sentinel.modified_pod - match_pod = mock.sentinel.match_pod - m_host_network.return_value = False - - knp_on_ns = self._handler._drv_policy.knps_on_namespace - knp_on_ns.return_value = True - namespaced_pods = self._handler._drv_policy.namespaced_pods - ensure_nw_policy = self._handler._drv_policy.ensure_network_policy - ensure_nw_policy.return_value = [modified_pod] - affected_pods = self._handler._drv_policy.affected_pods - affected_pods.return_value = [match_pod] - sg1 = [mock.sentinel.sg1] - sg2 = [mock.sentinel.sg2] - self._get_security_groups.side_effect = [sg1, sg2] - m_get_services.return_value = {'items': []} - - policy.NetworkPolicyHandler.on_present(self._handler, self._policy) - namespaced_pods.assert_not_called() - ensure_nw_policy.assert_called_once_with(self._policy, - self._project_id) - affected_pods.assert_called_once_with(self._policy) - - calls = [mock.call(modified_pod, self._project_id), - mock.call(match_pod, self._project_id)] - self._get_security_groups.assert_has_calls(calls) - - calls = [mock.call(modified_pod, sg1), mock.call(match_pod, sg2)] - self._update_vif_sgs.assert_has_calls(calls) - self._update_lbaas_sg.assert_not_called() - - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - def test_on_present_without_knps_on_namespace(self, m_host_network, - m_get_services): - modified_pod = mock.sentinel.modified_pod - match_pod = mock.sentinel.match_pod - m_host_network.return_value = False - - ensure_nw_policy = self._handler._drv_policy.ensure_network_policy - ensure_nw_policy.return_value = [modified_pod] - affected_pods = self._handler._drv_policy.affected_pods - affected_pods.return_value = [match_pod] - sg2 = [mock.sentinel.sg2] - sg3 = [mock.sentinel.sg3] - self._get_security_groups.side_effect = [sg2, sg3] - m_get_services.return_value = {'items': []} - - policy.NetworkPolicyHandler.on_present(self._handler, self._policy) - ensure_nw_policy.assert_called_once_with(self._policy, - self._project_id) - affected_pods.assert_called_once_with(self._policy) - - calls = [mock.call(modified_pod, self._project_id), - mock.call(match_pod, self._project_id)] - self._get_security_groups.assert_has_calls(calls) - - calls = [mock.call(modified_pod, sg2), - mock.call(match_pod, sg3)] - self._update_vif_sgs.assert_has_calls(calls) - self._update_lbaas_sg.assert_not_called() + self.k8s = mock.Mock() + m_get_k8s.return_value = self.k8s + self.m_get_k8s = m_get_k8s - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - def test_on_present_with_services(self, m_host_network, m_get_services): - modified_pod = mock.sentinel.modified_pod - match_pod = mock.sentinel.match_pod - m_host_network.return_value = False + self.np_driver = mock.Mock() + m_get_np.return_value = self.np_driver + self._m_get_np = m_get_np - self._handler._is_egress_only_policy.return_value = False - self._handler._is_service_affected.return_value = True - knp_on_ns = self._handler._drv_policy.knps_on_namespace - knp_on_ns.return_value = True - namespaced_pods = self._handler._drv_policy.namespaced_pods - ensure_nw_policy = self._handler._drv_policy.ensure_network_policy - ensure_nw_policy.return_value = [modified_pod] - affected_pods = self._handler._drv_policy.affected_pods - affected_pods.return_value = [match_pod] - sg1 = [mock.sentinel.sg1] - sg2 = [mock.sentinel.sg2] - self._get_security_groups.side_effect = [sg1, sg2] - service = {'metadata': {'name': 'service-test'}, - 'spec': {'selector': mock.sentinel.selector}} - m_get_services.return_value = {'items': [service]} + self.handler = policy.NetworkPolicyHandler() - policy.NetworkPolicyHandler.on_present(self._handler, self._policy) - namespaced_pods.assert_not_called() - ensure_nw_policy.assert_called_once_with(self._policy, - self._project_id) - affected_pods.assert_called_once_with(self._policy) + def test_init(self): + self.m_get_k8s.assert_called_once() + self._m_get_np.assert_called_once() - calls = [mock.call(modified_pod, self._project_id), - mock.call(match_pod, self._project_id)] - self._get_security_groups.assert_has_calls(calls) - calls = [mock.call(modified_pod, sg1), mock.call(match_pod, sg2)] - self._update_vif_sgs.assert_has_calls(calls) - self._handler._is_service_affected.assert_called_once_with( - service, [modified_pod, match_pod]) - self._update_lbaas_sg.assert_called_once() + self.assertEqual(self.np_driver, self.handler._drv_policy) + self.assertEqual(self.k8s, self.handler.k8s) - @mock.patch('kuryr_kubernetes.controller.drivers.utils.get_services') - @mock.patch('kuryr_kubernetes.controller.drivers.utils.is_host_network') - def test_on_deleted(self, m_host_network, m_get_services): - namespace_pod = mock.sentinel.namespace_pod - match_pod = mock.sentinel.match_pod - m_host_network.return_value = False - affected_pods = self._handler._drv_policy.affected_pods - affected_pods.return_value = [match_pod] - get_knp_crd = self._handler._drv_policy.get_kuryrnetpolicy_crd - knp_obj = self._get_knp_obj() - get_knp_crd.return_value = knp_obj - sg1 = [mock.sentinel.sg1] - sg2 = [mock.sentinel.sg2] - self._get_security_groups.side_effect = [sg1, sg2] - m_get_services.return_value = {'items': []} - release_nw_policy = self._handler._drv_policy.release_network_policy - knp_on_ns = self._handler._drv_policy.knps_on_namespace - knp_on_ns.return_value = False - ns_pods = self._handler._drv_policy.namespaced_pods - ns_pods.return_value = [namespace_pod] + def test_on_finalize(self): + self.handler.on_finalize(self._policy) + self.np_driver.release_network_policy.assert_called_once_with( + self._policy) - policy.NetworkPolicyHandler.on_deleted(self._handler, self._policy) - release_nw_policy.assert_called_once_with(knp_obj) - self._get_security_groups.assert_called_once_with(match_pod, - self._project_id) - self._update_vif_sgs.assert_called_once_with(match_pod, sg1) - self._update_lbaas_sg.assert_not_called() - self._remove_sg.assert_called_once() + def test_on_present(self): + self.handler.on_present(self._policy) + self.k8s.add_finalizer.assert_called_once_with( + self._policy, 'kuryr.openstack.org/networkpolicy-finalizer') + self.np_driver.ensure_network_policy.assert_called_once_with( + self._policy) diff --git a/setup.cfg b/setup.cfg index 266fbaf17..bb7071b1e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -104,7 +104,7 @@ kuryr_kubernetes.controller.handlers = namespace = kuryr_kubernetes.controller.handlers.namespace:NamespaceHandler policy = kuryr_kubernetes.controller.handlers.policy:NetworkPolicyHandler pod_label = kuryr_kubernetes.controller.handlers.pod_label:PodLabelHandler - kuryrnetpolicy = kuryr_kubernetes.controller.handlers.kuryrnetpolicy:KuryrNetPolicyHandler + kuryrnetworkpolicy = kuryr_kubernetes.controller.handlers.kuryrnetworkpolicy:KuryrNetworkPolicyHandler kuryrnetwork = kuryr_kubernetes.controller.handlers.kuryrnetwork:KuryrNetworkHandler kuryrnetwork_population = kuryr_kubernetes.controller.handlers.kuryrnetwork_population:KuryrNetworkPopulationHandler test_handler = kuryr_kubernetes.tests.unit.controller.handlers.test_fake_handler:TestHandler diff --git a/tools/gate/copy_k8s_logs.sh b/tools/gate/copy_k8s_logs.sh index 86de25973..c946f60bb 100755 --- a/tools/gate/copy_k8s_logs.sh +++ b/tools/gate/copy_k8s_logs.sh @@ -38,6 +38,7 @@ sudo chown ${USER}:${USER} ${HOME}/.kube/config /usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get endpoints -o yaml --all-namespaces >> ${K8S_LOG_DIR}/endpoints.txt /usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get kuryrnetpolicy -o yaml --all-namespaces >> ${K8S_LOG_DIR}/kuryrnetpolicy_crds.txt /usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get kuryrport -o yaml --all-namespaces >> ${K8S_LOG_DIR}/kuryrport_crds.txt +/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config get kuryrnetworkpolicy -o yaml --all-namespaces >> ${K8S_LOG_DIR}/kuryrnetworkpolicy_crds.txt # Kubernetes pods logs mkdir -p ${K8S_LOG_DIR}/pod_logs while read -r line