Add partial specs support in ML2 for vlan provider networks

ML2 provider networks partial specs let admins choose some provider
network attributes and let neutron choose remaining attributes. This
change provides the implementation for VLAN provider networks.

In practice, for VLAN provider networks provider:physical_network
and provider:segmentation_id choices can be delegated to neutron,
in such case neutron will try to find a network in tenant network
pools which respects provided provider attributes.

DocImpact

Related to blueprint provider-network-partial-specs
Partial-Bug: #1330562

Change-Id: I2c52c71167edaa153b2e04681273e2f1be8d03aa

neutron/common/exceptions.py

@@ -179,6 +179,11 @@ class NoNetworkAvailable(ResourceExhausted):
                 "No tenant network is available for allocation.")
 
 
+class NoNetworkFoundInMaximumAllowedAttempts(ServiceUnavailable):
+    message = _("Unable to create the network. "
+                "No available network found in maximum allowed attempts.")
+
+
 class SubnetMismatchForPort(BadRequest):
     message = _("Subnet on port %(port_id)s does not match "
                 "the requested subnet %(subnet_id)s")

neutron/plugins/ml2/driver_api.py

@@ -88,7 +88,8 @@ def reserve_provider_segment(self, session, segment):
         """Reserve resource associated with a provider network segment.
 
         :param session: database session
-        :param segment: segment dictionary using keys defined above
+        :param segment: segment dictionary
+        :returns: segment dictionary
 
         Called inside transaction context on session to reserve the
         type-specific resource for a provider network segment. The

neutron/plugins/ml2/drivers/helpers.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2014 Thales Services SAS
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo.db import exception as db_exc
+
+from neutron.common import exceptions as exc
+from neutron.openstack.common import log
+from neutron.plugins.ml2 import driver_api as api
+
+
+# Number of retries to find a valid segment candidate and allocate it
+DB_MAX_RETRIES = 10
+
+
+LOG = log.getLogger(__name__)
+
+
+class TypeDriverHelper(api.TypeDriver):
+    """TypeDriver Helper for segment allocation.
+
+    Provide methods helping to perform segment allocation fully or partially
+    specified.
+    """
+
+    def __init__(self, model):
+        self.model = model
+        self.primary_keys = set(dict(model.__table__.columns))
+        self.primary_keys.remove("allocated")
+
+    def allocate_fully_specified_segment(self, session, **raw_segment):
+        """Allocate segment fully specified by raw_segment.
+
+        If segment exists, then try to allocate it and return db object
+        If segment does not exists, then try to create it and return db object
+        If allocation/creation failed, then return None
+        """
+
+        network_type = self.get_type()
+        try:
+            with session.begin(subtransactions=True):
+                alloc = (session.query(self.model).filter_by(**raw_segment).
+                         first())
+                if alloc:
+                    if alloc.allocated:
+                        # Segment already allocated
+                        return
+                    else:
+                        # Segment not allocated
+                        LOG.debug("%(type)s segment %(segment)s allocate "
+                                  "started ",
+                                  type=network_type, segment=raw_segment)
+                        count = (session.query(self.model).
+                                 filter_by(allocated=False, **raw_segment).
+                                 update({"allocated": True}))
+                        if count:
+                            LOG.debug("%(type)s segment %(segment)s allocate "
+                                      "done ",
+                                      type=network_type, segment=raw_segment)
+                            return alloc
+
+                        # Segment allocated or deleted since select
+                        LOG.debug("%(type)s segment %(segment)s allocate "
+                                  "failed: segment has been allocated or "
+                                  "deleted",
+                                  type=network_type, segment=raw_segment)
+
+                # Segment to create or already allocated
+                LOG.debug("%(type)s segment %(segment)s create started",
+                          type=network_type, segment=raw_segment)
+                alloc = self.model(allocated=True, **raw_segment)
+                alloc.save(session)
+                LOG.debug("%(type)s segment %(segment)s create done",
+                          type=network_type, segment=raw_segment)
+
+        except db_exc.DBDuplicateEntry:
+            # Segment already allocated (insert failure)
+            alloc = None
+            LOG.debug("%(type)s segment %(segment)s create failed",
+                      type=network_type, segment=raw_segment)
+
+        return alloc
+
+    def allocate_partially_specified_segment(self, session, **filters):
+        """Allocate model segment from pool partially specified by filters.
+
+        Return allocated db object or None.
+        """
+
+        network_type = self.get_type()
+        with session.begin(subtransactions=True):
+            select = (session.query(self.model).
+                      filter_by(allocated=False, **filters))
+
+            # Selected segment can be allocated before update by someone else,
+            # We retry until update success or DB_MAX_RETRIES retries
+            for attempt in range(1, DB_MAX_RETRIES + 1):
+                alloc = select.first()
+
+                if not alloc:
+                    # No resource available
+                    return
+
+                raw_segment = dict((k, alloc[k]) for k in self.primary_keys)
+                LOG.debug("%(type)s segment allocate from pool, attempt "
+                          "%(attempt)s started with %(segment)s ",
+                          type=network_type, attempt=attempt,
+                          segment=raw_segment)
+                count = (session.query(self.model).
+                         filter_by(allocated=False, **raw_segment).
+                         update({"allocated": True}))
+                if count:
+                    LOG.debug("%(type)s segment allocate from pool, attempt "
+                              "%(attempt)s success with %(segment)s ",
+                              type=network_type, attempt=attempt,
+                              segment=raw_segment)
+                    return alloc
+
+                # Segment allocated since select
+                LOG.debug("Allocate %(type)s segment from pool, "
+                          "attempt %(attempt)s failed with segment "
+                          "%(segment)s",
+                          type=network_type, attempt=attempt,
+                          segment=raw_segment)
+
+        LOG.warning(_("Allocate %(type)s segment from pool failed "
+                      "after %(number)s failed attempts"),
+                    {"type": network_type, "number": DB_MAX_RETRIES})
+        raise exc.NoNetworkFoundInMaximumAllowedAttempts

neutron/plugins/ml2/drivers/type_flat.py

@@ -14,6 +14,7 @@
 #    under the License.
 
 from oslo.config import cfg
+from oslo.db import exception as db_exc
 import sqlalchemy as sa
 
 from neutron.common import exceptions as exc
@@ -100,17 +101,14 @@ def reserve_provider_segment(self, session, segment):
         physical_network = segment[api.PHYSICAL_NETWORK]
         with session.begin(subtransactions=True):
             try:
-                alloc = (session.query(FlatAllocation).
-                         filter_by(physical_network=physical_network).
-                         with_lockmode('update').
-                         one())
-                raise exc.FlatNetworkInUse(
-                    physical_network=physical_network)
-            except sa.orm.exc.NoResultFound:
                 LOG.debug(_("Reserving flat network on physical "
                             "network %s"), physical_network)
                 alloc = FlatAllocation(physical_network=physical_network)
-                session.add(alloc)
+                alloc.save(session)
+            except db_exc.DBDuplicateEntry:
+                raise exc.FlatNetworkInUse(
+                    physical_network=physical_network)
+        return segment
 
     def allocate_tenant_segment(self, session):
         # Tenant flat networks are not supported.

neutron/plugins/ml2/drivers/type_gre.py

@@ -91,6 +91,7 @@ def reserve_provider_segment(self, session, segment):
                 alloc = GreAllocation(gre_id=segmentation_id)
                 alloc.allocated = True
                 session.add(alloc)
+        return segment
 
     def allocate_tenant_segment(self, session):
         with session.begin(subtransactions=True):

neutron/plugins/ml2/drivers/type_local.py

@@ -48,7 +48,7 @@ def validate_provider_segment(self, segment):
 
     def reserve_provider_segment(self, session, segment):
         # No resources to reserve
-        pass
+        return segment
 
     def allocate_tenant_segment(self, session):
         # No resources to allocate

neutron/plugins/ml2/drivers/type_vlan.py

@@ -28,6 +28,7 @@
 from neutron.plugins.common import constants as p_const
 from neutron.plugins.common import utils as plugin_utils
 from neutron.plugins.ml2 import driver_api as api
+from neutron.plugins.ml2.drivers import helpers
 
 LOG = log.getLogger(__name__)
 
@@ -67,7 +68,7 @@ class VlanAllocation(model_base.BASEV2):
     allocated = sa.Column(sa.Boolean, nullable=False)
 
 
-class VlanTypeDriver(api.TypeDriver):
+class VlanTypeDriver(helpers.TypeDriverHelper):
     """Manage state for VLAN networks with ML2.
 
     The VlanTypeDriver implements the 'vlan' network_type. VLAN
@@ -79,6 +80,7 @@ class VlanTypeDriver(api.TypeDriver):
     """
 
     def __init__(self):
+        super(VlanTypeDriver, self).__init__(VlanAllocation)
         self._parse_network_vlan_ranges()
 
     def _parse_network_vlan_ranges(self):
@@ -160,25 +162,27 @@ def initialize(self):
         self._sync_vlan_allocations()
         LOG.info(_("VlanTypeDriver initialization complete"))
 
+    def is_partial_segment(self, segment):
+        return segment.get(api.SEGMENTATION_ID) is None
+
     def validate_provider_segment(self, segment):
         physical_network = segment.get(api.PHYSICAL_NETWORK)
-        if not physical_network:
-            msg = _("physical_network required for VLAN provider network")
-            raise exc.InvalidInput(error_message=msg)
-        if physical_network not in self.network_vlan_ranges:
-            msg = (_("physical_network '%s' unknown for VLAN provider network")
-                   % physical_network)
-            raise exc.InvalidInput(error_message=msg)
-
         segmentation_id = segment.get(api.SEGMENTATION_ID)
-        if segmentation_id is None:
-            msg = _("segmentation_id required for VLAN provider network")
-            raise exc.InvalidInput(error_message=msg)
-        if not utils.is_valid_vlan_tag(segmentation_id):
-            msg = (_("segmentation_id out of range (%(min)s through "
-                     "%(max)s)") %
-                   {'min': q_const.MIN_VLAN_TAG,
-                    'max': q_const.MAX_VLAN_TAG})
+        if physical_network:
+            if physical_network not in self.network_vlan_ranges:
+                msg = (_("physical_network '%s' unknown "
+                         " for VLAN provider network") % physical_network)
+                raise exc.InvalidInput(error_message=msg)
+            if segmentation_id:
+                if not utils.is_valid_vlan_tag(segmentation_id):
+                    msg = (_("segmentation_id out of range (%(min)s through "
+                             "%(max)s)") %
+                           {'min': q_const.MIN_VLAN_TAG,
+                            'max': q_const.MAX_VLAN_TAG})
+                    raise exc.InvalidInput(error_message=msg)
+        elif segmentation_id:
+            msg = _("segmentation_id requires physical_network for VLAN "
+                    "provider network")
             raise exc.InvalidInput(error_message=msg)
 
         for key, value in segment.items():
@@ -189,48 +193,36 @@ def validate_provider_segment(self, segment):
                 raise exc.InvalidInput(error_message=msg)
 
     def reserve_provider_segment(self, session, segment):
-        physical_network = segment[api.PHYSICAL_NETWORK]
-        vlan_id = segment[api.SEGMENTATION_ID]
-        with session.begin(subtransactions=True):
-            try:
-                alloc = (session.query(VlanAllocation).
-                         filter_by(physical_network=physical_network,
-                                   vlan_id=vlan_id).
-                         with_lockmode('update').
-                         one())
-                if alloc.allocated:
-                    raise exc.VlanIdInUse(vlan_id=vlan_id,
-                                          physical_network=physical_network)
-                LOG.debug(_("Reserving specific vlan %(vlan_id)s on physical "
-                            "network %(physical_network)s from pool"),
-                          {'vlan_id': vlan_id,
-                           'physical_network': physical_network})
-                alloc.allocated = True
-            except sa.orm.exc.NoResultFound:
-                LOG.debug(_("Reserving specific vlan %(vlan_id)s on physical "
-                            "network %(physical_network)s outside pool"),
-                          {'vlan_id': vlan_id,
-                           'physical_network': physical_network})
-                alloc = VlanAllocation(physical_network=physical_network,
-                                       vlan_id=vlan_id,
-                                       allocated=True)
-                session.add(alloc)
+        filters = {}
+        physical_network = segment.get(api.PHYSICAL_NETWORK)
+        if physical_network is not None:
+            filters['physical_network'] = physical_network
+            vlan_id = segment.get(api.SEGMENTATION_ID)
+            if vlan_id is not None:
+                filters['vlan_id'] = vlan_id
+
+        if self.is_partial_segment(segment):
+            alloc = self.allocate_partially_specified_segment(
+                session, **filters)
+            if not alloc:
+                raise exc.NoNetworkAvailable
+        else:
+            alloc = self.allocate_fully_specified_segment(
+                session, **filters)
+            if not alloc:
+                raise exc.VlanIdInUse(**filters)
+
+        return {api.NETWORK_TYPE: p_const.TYPE_VLAN,
+                api.PHYSICAL_NETWORK: alloc.physical_network,
+                api.SEGMENTATION_ID: alloc.vlan_id}
 
     def allocate_tenant_segment(self, session):
-        with session.begin(subtransactions=True):
-            alloc = (session.query(VlanAllocation).
-                     filter_by(allocated=False).
-                     with_lockmode('update').
-                     first())
-            if alloc:
-                LOG.debug(_("Allocating vlan %(vlan_id)s on physical network "
-                            "%(physical_network)s from pool"),
-                          {'vlan_id': alloc.vlan_id,
-                           'physical_network': alloc.physical_network})
-                alloc.allocated = True
-                return {api.NETWORK_TYPE: p_const.TYPE_VLAN,
-                        api.PHYSICAL_NETWORK: alloc.physical_network,
-                        api.SEGMENTATION_ID: alloc.vlan_id}
+        alloc = self.allocate_partially_specified_segment(session)
+        if not alloc:
+            return
+        return {api.NETWORK_TYPE: p_const.TYPE_VLAN,
+                api.PHYSICAL_NETWORK: alloc.physical_network,
+                api.SEGMENTATION_ID: alloc.vlan_id}
 
     def release_segment(self, session, segment):
         physical_network = segment[api.PHYSICAL_NETWORK]

neutron/plugins/ml2/drivers/type_vxlan.py

@@ -99,6 +99,7 @@ def reserve_provider_segment(self, session, segment):
                 alloc = VxlanAllocation(vxlan_vni=segmentation_id)
                 alloc.allocated = True
                 session.add(alloc)
+        return segment
 
     def allocate_tenant_segment(self, session):
         with session.begin(subtransactions=True):

neutron/plugins/ml2/managers.py

@@ -85,7 +85,7 @@ def validate_provider_segment(self, segment):
     def reserve_provider_segment(self, session, segment):
         network_type = segment.get(api.NETWORK_TYPE)
         driver = self.drivers.get(network_type)
-        driver.obj.reserve_provider_segment(session, segment)
+        return driver.obj.reserve_provider_segment(session, segment)
 
     def allocate_tenant_segment(self, session):
         for network_type in self.tenant_network_types:

neutron/plugins/ml2/plugin.py

@@ -378,8 +378,8 @@ def create_network(self, context, network):
             # to TypeManager.
             if segments:
                 for segment in segments:
-                    self.type_manager.reserve_provider_segment(session,
-                                                               segment)
+                    segment = self.type_manager.reserve_provider_segment(
+                        session, segment)
                     db.add_network_segment(session, network_id, segment)
             else:
                 segment = self.type_manager.allocate_tenant_segment(session)