Add API extension for reporting IP availability usage statistics

Implements an API extension for reporting availibility of IP
addresses on Neutron networks/subnets based on the blueprint
proposed at https://review.openstack.org/#/c/180803/

This provides an easy way for operators to count the number of
used and total IP addresses on any or all networks and/or
subnets.

Co-Authored-By: David Bingham <dbingham@godaddy.com>
Co-Authored-By: Craig Jellick <craig.jellick@gmail.com>

APIImpact
DocImpact: As a new API, will need all new docs. See devref for details.

Implements: blueprint network-ip-usage-api
Closes-Bug: 1457986
Change-Id: I81406054d46b2c0e0ffcd56e898e329f943ba46f

doc/source/devref/index.rst

@@ -76,6 +76,7 @@ Neutron Internals
    instrumentation
    address_scopes
    openvswitch_firewall
+   network_ip_availability
 
 Testing
 -------

doc/source/devref/network_ip_availability.rst

@@ -0,0 +1,180 @@
+..
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+
+Network IP Availability Extension
+=================================
+
+This extension is an information-only API that allows a user or process
+to determine the amount of IPs that are consumed across networks and
+their subnets' allocation pools. Each network and embedded subnet returns
+with values for **used_ips** and **total_ips** making it easy
+to determine how much of your network's IP space is consumed.
+
+This API provides the ability for network administrators to periodically
+list usage (manual or automated) in order to preemptively add new network
+capacity when thresholds are exceeded.
+
+**Important Note:**
+
+This API tracks a network's "consumable" IPs. What's the distinction?
+After a network and its subnets are created, consumable IPs
+are:
+
+* Consumed in the subnet's allocations (derives used IPs)
+* Consumed from the subnet's allocation pools (derives total IPs)
+
+This API tracks consumable IPs so network administrators know when their
+subnet's IP pools (and and ultimately a network's) IPs are about to run out.
+This API does not account reserved IPs such as a subnet's gateway IP or other
+reserved or unused IPs of a subnet's cidr that are consumed as a result of
+the subnet creation itself.
+
+Enabling in Neutron
+-------------------
+
+To enable this plugin within neutron, append this pluging class to the
+comma-delimited plugin list to the end of the **service_plugins** configuration
+property within your neutron.conf file.
+
+Example::
+
+    service_plugins=router, network_ip_availability
+
+
+API Specification
+-----------------
+
+Availability for all networks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+GET /v2.0/network-ip-availabilities ::
+
+    Request to url: v2.0/network-ip-availabilities
+      headers: {'content-type': 'application/json', 'X-Auth-Token': 'SOME_AUTH_TOKEN'}
+
+Example response ::
+
+    Response:
+      HTTP/1.1 200 OK
+      Content-Type: application/json; charset=UTF-8
+
+.. code::
+
+    {
+        "network_ip_availabilities": [
+            {
+                "network_id": "f944c153-3f46-417b-a3c2-487cd9a456b9",
+                "network_name": "net1",
+                "subnet_ip_availability": [
+                    {
+                        "cidr": "10.0.0.0/24",
+                        "ip_version": 4,
+                        "subnet_id": "46b1406a-8373-454c-8eb8-500a09eb77fb",
+                        "subnet_name": "",
+                        "total_ips": 253,
+                        "used_ips": 3
+                    }
+                ],
+                "tenant_id": "test-tenant",
+                "total_ips": 253,
+                "used_ips": 3
+            },
+            {
+                "network_id": "47035bae-4f29-4fef-be2e-2941b72528a8",
+                "network_name": "net2",
+                "subnet_ip_availability": [],
+                "tenant_id": "test-tenant",
+                "total_ips": 0,
+                "used_ips": 0
+            },
+            {
+                "network_id": "2e3ea0cd-c757-44bf-bb30-42d038687e3f",
+                "network_name": "net3",
+                "subnet_ip_availability": [
+                    {
+                        "cidr": "40.0.0.0/24",
+                        "ip_version": 4,
+                        "subnet_id": "aab6b35c-16b5-489c-a5c7-fec778273495",
+                        "subnet_name": "",
+                        "total_ips": 253,
+                        "used_ips": 2
+                    }
+                ],
+                "tenant_id": "test-tenant",
+                "total_ips": 253,
+                "used_ips": 2
+            }
+        ]
+    }
+
+Availability by network ID
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+GET /v2.0/network-ip-availabilities/{network\_uuid} ::
+
+    Request to url: /v2.0/network-ip-availabilities/aba3b29b-c119-4b45-afbd-88e500acd970
+      headers: {'content-type': 'application/json', 'X-Auth-Token': 'SOME_AUTH_TOKEN'}
+
+Example response ::
+
+    Response:
+      HTTP/1.1 200 OK
+      Content-Type: application/json; charset=UTF-8
+
+.. code::
+
+    {
+        "network_ip_availability": {
+            "network_id": "f944c153-3f46-417b-a3c2-487cd9a456b9",
+            "network_name": "net1",
+            "subnet_ip_availability": [
+                {
+                    "cidr": "10.0.0.0/24",
+                    "ip_version": 4,
+                    "subnet_name": "",
+                    "subnet_id": "46b1406a-8373-454c-8eb8-500a09eb77fb",
+                    "total_ips": 253,
+                    "used_ips": 3
+                }
+            ],
+            "tenant_id": "test-tenant",
+            "total_ips": 253,
+            "used_ips": 3
+        }
+    }
+
+Supported Query Filters
+~~~~~~~~~~~~~~~~~~~~~~~
+This API currently supports the following query parameters:
+
+* **network_id**: Returns availability for the network matching the network ID.
+  Note: This query (?network_id={network_id_guid})is roughly equivalent to
+  *Availability by network ID* section except it returns the plural
+  response form as a list rather than as an item.
+* **network_name**: Returns availability for network matching
+  the provided name
+* **tenant_id**: Returns availability for all networks owned by the provided
+  tenant ID.
+* **ip_version**: Filters network subnets by those supporting the supplied
+  ip version. Values can be either 4 or 6.
+
+Query filters can be combined to further narrow results and what is returned
+will match all criteria. When a parameter is specified more
+than once, it will return results that match both. Examples: ::
+
+    # Fetch IPv4 availability for a specific tenant uuid
+    GET /v2.0/network-ip-availabilities?ip_version=4&tenant_id=example-tenant-uuid
+
+    # Fetch multiple networks by their ids
+    GET /v2.0/network-ip-availabilities?network_id=uuid_sample_1&network_id=uuid_sample_2

etc/policy.json

@@ -43,6 +43,8 @@
     "get_network:provider:physical_network": "rule:admin_only",
     "get_network:provider:segmentation_id": "rule:admin_only",
     "get_network:queue_id": "rule:admin_only",
+    "get_network_ip_availabilities": "rule:admin_only",
+    "get_network_ip_availability": "rule:admin_only",
     "create_network:shared": "rule:admin_only",
     "create_network:router:external": "rule:admin_only",
     "create_network:is_default": "rule:admin_only",

neutron/db/network_ip_availability_db.py

@@ -0,0 +1,182 @@
+# Copyright 2016 GoDaddy.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+#  implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import netaddr
+import six
+from sqlalchemy import func
+
+import neutron.db.models_v2 as mod
+import oslo_log.log as logging
+
+LOG = logging.getLogger(__name__)
+
+NETWORK_ID = 'network_id'
+NETWORK_NAME = 'network_name'
+SUBNET_ID = 'subnet_id'
+SUBNET_NAME = 'subnet_name'
+
+SUPPORTED_FILTERS = {
+    NETWORK_ID: mod.Network.id,
+    NETWORK_NAME: mod.Network.name,
+    'tenant_id': mod.Network.tenant_id,
+    'ip_version': mod.Subnet.ip_version,
+}
+SUPPORTED_FILTER_KEYS = six.viewkeys(SUPPORTED_FILTERS)
+
+
+class IpAvailabilityMixin(object):
+    """Mixin class to query for IP availability."""
+
+    # Columns common to all queries
+    common_columns = [
+        mod.Network.id.label(NETWORK_ID),
+        mod.Subnet.id.label(SUBNET_ID),
+        mod.Subnet.cidr,
+        mod.Subnet.ip_version
+    ]
+
+    # Columns for the network/subnet and used_ip counts
+    network_used_ips_columns = list(common_columns)
+    network_used_ips_columns.append(mod.Network.name.label(NETWORK_NAME))
+    network_used_ips_columns.append(mod.Network.tenant_id)
+    network_used_ips_columns.append(mod.Subnet.name.label(SUBNET_NAME))
+    # Aggregate query computed column
+    network_used_ips_computed_columns = [
+        func.count(mod.IPAllocation.subnet_id).label('used_ips')]
+
+    # Columns for total_ips query
+    total_ips_columns = list(common_columns)
+    total_ips_columns.append(mod.IPAllocationPool.first_ip)
+    total_ips_columns.append(mod.IPAllocationPool.last_ip)
+
+    @classmethod
+    def get_network_ip_availabilities(cls, context, filters=None):
+        """Get IP availability stats on a per subnet basis.
+
+        Returns a list of network summaries which internally contains a list
+        of subnet summaries. The used_ip and total_ip counts are returned at
+        both levels.
+        """
+
+        # Fetch total_ips by subnet
+        subnet_total_ips_dict = cls._generate_subnet_total_ips_dict(context,
+                                                                    filters)
+        # Query network/subnet data along with used IP counts
+        record_and_count_query = cls._build_network_used_ip_query(context,
+                                                                  filters)
+        # Assemble results
+        result_dict = {}
+        for row in record_and_count_query:
+            cls._add_result(row, result_dict,
+                            subnet_total_ips_dict.get(row.subnet_id, 0))
+
+        # Convert result back into the list it expects
+        net_ip_availabilities = list(six.viewvalues(result_dict))
+        return net_ip_availabilities
+
+    @classmethod
+    def _build_network_used_ip_query(cls, context, filters):
+        # Generate a query to gather network/subnet/used_ips.
+        # Ensure query is tolerant of missing child table data (outerjoins)
+        # Process these outerjoin columns assuming their values may be None
+        query = context.session.query()
+        query = query.add_columns(*cls.network_used_ips_columns)
+        query = query.add_columns(*cls.network_used_ips_computed_columns)
+        query = query.outerjoin(mod.Subnet,
+                                mod.Network.id == mod.Subnet.network_id)
+        query = query.outerjoin(mod.IPAllocation,
+                                mod.Subnet.id == mod.IPAllocation.subnet_id)
+        query = query.group_by(*cls.network_used_ips_columns)
+
+        return cls._adjust_query_for_filters(query, filters)
+
+    @classmethod
+    def _build_total_ips_query(cls, context, filters):
+        query = context.session.query()
+        query = query.add_columns(*cls.total_ips_columns)
+        query = query.outerjoin(mod.Subnet,
+                                mod.Network.id == mod.Subnet.network_id)
+        query = query.outerjoin(
+                mod.IPAllocationPool,
+                mod.Subnet.id == mod.IPAllocationPool.subnet_id)
+        return cls._adjust_query_for_filters(query, filters)
+
+    @classmethod
+    def _generate_subnet_total_ips_dict(cls, context, filters):
+        """Generates a dict whose key=subnet_id, value=total_ips in subnet"""
+
+        # Query to get total_ips counts
+        total_ips_query = cls._build_total_ips_query(context, filters)
+
+        subnet_totals_dict = {}
+        for row in total_ips_query:
+            # Skip networks without subnets
+            if not row.subnet_id:
+                continue
+
+            # Add IPAllocationPool data
+            if row.last_ip:
+                pool_total = netaddr.IPRange(
+                        netaddr.IPAddress(row.first_ip),
+                        netaddr.IPAddress(row.last_ip)).size
+                cur_total = subnet_totals_dict.get(row.subnet_id, 0)
+                subnet_totals_dict[row.subnet_id] = cur_total + pool_total
+            else:
+                subnet_totals_dict[row.subnet_id] = netaddr.IPNetwork(
+                        row.cidr, version=row.ip_version).size
+
+        return subnet_totals_dict
+
+    @classmethod
+    def _adjust_query_for_filters(cls, query, filters):
+        # The intersect of sets gets us applicable filter keys (others ignored)
+        common_keys = six.viewkeys(filters) & SUPPORTED_FILTER_KEYS
+        for key in common_keys:
+            filter_vals = filters[key]
+            if filter_vals:
+                query = query.filter(SUPPORTED_FILTERS[key].in_(filter_vals))
+        return query
+
+    @classmethod
+    def _add_result(cls, db_row, result_dict, subnet_total_ips):
+        # Find network in results. Create and add if missing
+        if db_row.network_id in result_dict:
+            network = result_dict[db_row.network_id]
+        else:
+            network = {NETWORK_ID: db_row.network_id,
+                       NETWORK_NAME: db_row.network_name,
+                       'tenant_id': db_row.tenant_id,
+                       'subnet_ip_availability': [],
+                       'used_ips': 0, 'total_ips': 0}
+            result_dict[db_row.network_id] = network
+
+        # Only add subnet data if outerjoin rows have it
+        if db_row.subnet_id:
+            cls._add_subnet_data_to_net(db_row, network, subnet_total_ips)
+
+    @classmethod
+    def _add_subnet_data_to_net(cls, db_row, network_dict, subnet_total_ips):
+        subnet = {
+            SUBNET_ID: db_row.subnet_id,
+            'ip_version': db_row.ip_version,
+            'cidr': db_row.cidr,
+            SUBNET_NAME: db_row.subnet_name,
+            'used_ips': db_row.used_ips if db_row.used_ips else 0,
+            'total_ips': subnet_total_ips
+        }
+        # Attach subnet result and rollup subnet sums into the parent
+        network_dict['subnet_ip_availability'].append(subnet)
+        network_dict['total_ips'] += subnet['total_ips']
+        network_dict['used_ips'] += subnet['used_ips']