Implement a Catalog SQL backend

This adds a catalog SQL backend.  Makes use of 3 tables: endpoint,
service and service_endpoint_association.  Services and endpoints are
joined via the association table.  New sqlalchemy models have been defined
in keystone/catalog/backends/sql.py and are imported during the initial
migration (v001).

Configuring the service catalog is possible with changes to
python-keystoneclient.  I will be proposing a merge for that and I'll
update this commit msg with a link to its review.  With those client
changes, admins can now create and delete endpoints that are associated
with existing services. Existing service commands on the client-side work
as expected against this new backend.  This driver's get_catalog method
properly translates existing services, endpoints and relatoins into something
consumable by keystone non-admin users / clients.

Update: Some cleanup as per bcwaldon's suggestions

Update: Match functionality of existing catalog backend by returning
        IDs instead of sql objects for list_services() and list_endpoints()

Update: pep8 fixes

Update (1/2): Remove legacy OS-KSADM stuff

Update (2/2): Remove ServiceEndpointAssociation table/model
              in favor of a FK, endpoint.service_id -> service.id

Resolves bug 928053

Change-Id: Icc11889920744c36255f06356744cb247d79f4aa

keystone/catalog/backends/sql.py

@@ -0,0 +1,167 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# Copyright 2012 OpenStack LLC
+# Copyright 2012 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy.exc
+import webob.exc
+
+from keystone import catalog
+from keystone import exception
+from keystone.common import sql
+from keystone.common.sql import migration
+
+
+class Service(sql.ModelBase, sql.DictBase):
+    __tablename__ = 'service'
+    id = sql.Column(sql.String(64), primary_key=True)
+    type = sql.Column(sql.String(255))
+    extra = sql.Column(sql.JsonBlob())
+
+    @classmethod
+    def from_dict(cls, service_dict):
+        extra = {}
+        for k, v in service_dict.copy().iteritems():
+            if k not in ['id', 'type']:
+                extra[k] = service_dict.pop(k)
+
+        service_dict['extra'] = extra
+        return cls(**service_dict)
+
+    def to_dict(self):
+        extra_copy = self.extra.copy()
+        extra_copy['id'] = self.id
+        extra_copy['type'] = self.type
+        return extra_copy
+
+
+class Endpoint(sql.ModelBase, sql.DictBase):
+    __tablename__ = 'endpoint'
+    id = sql.Column(sql.String(64), primary_key=True)
+    region = sql.Column('region', sql.String(255))
+    service_id = sql.Column(sql.String(64),
+                            sql.ForeignKey('service.id'),
+                            nullable=False)
+    extra = sql.Column(sql.JsonBlob())
+
+    @classmethod
+    def from_dict(cls, endpoint_dict):
+        extra = {}
+        for k, v in endpoint_dict.copy().iteritems():
+            if k not in ['id', 'region', 'service_id']:
+                extra[k] = endpoint_dict.pop(k)
+        endpoint_dict['extra'] = extra
+        return cls(**endpoint_dict)
+
+    def to_dict(self):
+        extra_copy = self.extra.copy()
+        extra_copy['id'] = self.id
+        extra_copy['region'] = self.region
+        extra_copy['service_id'] = self.service_id
+        return extra_copy
+
+
+class Catalog(sql.Base):
+    def db_sync(self):
+        migration.db_sync()
+
+    # Services
+    def list_services(self):
+        session = self.get_session()
+        services = session.query(Service)
+        return [s['id'] for s in list(services)]
+
+    def get_service(self, service_id):
+        session = self.get_session()
+        service_ref = session.query(Service).filter_by(id=service_id).first()
+        return service_ref.to_dict()
+
+    def delete_service(self, service_id):
+        session = self.get_session()
+        service_ref = session.query(Service).filter_by(id=service_id).first()
+        with session.begin():
+            session.delete(service_ref)
+            session.flush()
+
+    def create_service(self, context, service_ref):
+        session = self.get_session()
+        with session.begin():
+            service = Service.from_dict(service_ref)
+            session.add(service)
+            session.flush()
+        return service.to_dict()
+
+    def service_exists(self, service_id):
+        session = self.get_session()
+        if not session.query(Service).filter_by(id=service_id).first():
+            return False
+        return True
+
+    # Endpoints
+    def create_endpoint(self, context, endpoint_ref):
+        session = self.get_session()
+        new_endpoint = Endpoint.from_dict(endpoint_ref)
+        with session.begin():
+            session.add(new_endpoint)
+            session.flush()
+        return new_endpoint.to_dict()
+
+    def delete_endpoint(self, endpoint_id):
+        session = self.get_session()
+        endpoint_ref = session.query(Endpoint)
+        endpoint_ref = endpoint_ref.filter_by(id=endpoint_id).first()
+        if not endpoint_ref:
+            raise exception.NotFound('Endpoint not found')
+        with session.begin():
+            session.delete(endpoint_ref)
+            session.flush()
+
+    def get_endpoint(self, endpoint_id):
+        session = self.get_session()
+        endpoint_ref = session.query(Endpoint)
+        endpoint_ref = endpoint_ref.filter_by(id=endpoint_id).first()
+        return endpoint_ref.to_dict()
+
+    def list_endpoints(self):
+        session = self.get_session()
+        endpoints = session.query(Endpoint)
+        return [e['id'] for e in list(endpoints)]
+
+    def get_catalog(self, user_id, tenant_id, metadata=None):
+        d = {'tenant_id': tenant_id, 'user_id': user_id}
+        catalog = {}
+
+        endpoints = [self.get_endpoint(e)
+                     for e in self.list_endpoints()]
+        for ep in endpoints:
+            service = self.get_service(ep['service_id'])
+            srv_type = service['type']
+            srv_name = service['name']
+            region = ep['region']
+
+            if region not in catalog:
+                catalog[region] = {}
+
+            catalog[region][srv_type] = {}
+
+            internal_url = ep['internalurl'].replace('$(', '%(')
+            public_url = ep['publicurl'].replace('$(', '%(')
+            admin_url = ep['adminurl'].replace('$(', '%(')
+            catalog[region][srv_type]['name'] = srv_name
+            catalog[region][srv_type]['publicURL'] = public_url % d
+            catalog[region][srv_type]['adminURL'] = admin_url % d
+            catalog[region][srv_type]['internalURL'] = internal_url % d
+
+        return catalog

keystone/catalog/core.py

@@ -1,6 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2012 OpenStack LLC
+# Copyright 2012 Canonical Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -21,6 +22,9 @@
 import webob.exc
 
 from keystone import config
+from keystone import identity
+from keystone import policy
+from keystone import token
 from keystone.common import manager
 from keystone.common import wsgi
 
@@ -69,3 +73,38 @@ def create_service(self, context, OS_KSADM_service):
         new_service_ref = self.catalog_api.create_service(
                 context, service_id, service_ref)
         return {'OS-KSADM:service': new_service_ref}
+
+
+class EndpointController(wsgi.Application):
+    def __init__(self):
+        self.catalog_api = Manager()
+        self.identity_api = identity.Manager()
+        self.policy_api = policy.Manager()
+        self.token_api = token.Manager()
+        super(EndpointController, self).__init__()
+
+    def get_endpoints(self, context):
+        self.assert_admin(context)
+        endpoint_list = self.catalog_api.list_endpoints(context)
+        endpoint_refs = [self.catalog_api.get_endpoint(context, e)
+                         for e in endpoint_list]
+        return {'endpoints': endpoint_refs}
+
+    def create_endpoint(self, context, endpoint):
+        self.assert_admin(context)
+        endpoint_id = uuid.uuid4().hex
+        endpoint_ref = endpoint.copy()
+        endpoint_ref['id'] = endpoint_id
+
+        service_id = endpoint_ref['service_id']
+        if not self.catalog_api.service_exists(context, service_id):
+            msg = 'No service exists with id %s' % service_id
+            raise webob.exc.HTTPBadRequest(msg)
+
+        new_endpoint_ref = self.catalog_api.create_endpoint(
+                                context, endpoint_id, endpoint_ref)
+        return {'endpoint': new_endpoint_ref}
+
+    def delete_endpoint(self, context, endpoint_id):
+        self.assert_admin(context)
+        endpoint_ref = self.catalog_api.delete_endpoint(context, endpoint_id)

keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py

@@ -23,6 +23,7 @@
 import keystone.identity.backends.sql
 import keystone.token.backends.sql
 import keystone.contrib.ec2.backends.sql
+import keystone.catalog.backends.sql
 
 
 def upgrade(migrate_engine):

keystone/contrib/admin_crud/core.py

@@ -31,6 +31,7 @@ def add_routes(self, mapper):
         user_controller = identity.UserController()
         role_controller = identity.RoleController()
         service_controller = catalog.ServiceController()
+        endpoint_controller = catalog.EndpointController()
 
         # Tenant Operations
         mapper.connect('/tenants', controller=tenant_controller,
@@ -145,6 +146,20 @@ def add_routes(self, mapper):
                        action='get_service',
                        conditions=dict(method=['GET']))
 
+        # Endpoint Templates
+        mapper.connect('/endpoints',
+                       controller=endpoint_controller,
+                       action='get_endpoints',
+                       conditions=dict(method=['GET']))
+        mapper.connect('/endpoints',
+                       controller=endpoint_controller,
+                       action='create_endpoint',
+                       conditions=dict(method=['POST']))
+        mapper.connect('/endpoints/{endpoint_id}',
+                       controller=endpoint_controller,
+                       action='delete_endpoint',
+                       conditions=dict(method=['DELETE']))
+
         # Role Operations
         mapper.connect('/OS-KSADM/roles',
                        controller=role_controller,

keystone/identity/core.py

@@ -22,7 +22,6 @@
 
 import webob.exc
 
-from keystone import catalog
 from keystone import config
 from keystone import exception
 from keystone import policy
@@ -352,7 +351,6 @@ def _format_tenant_list(self, tenant_refs, **kwargs):
 
 class UserController(wsgi.Application):
     def __init__(self):
-        self.catalog_api = catalog.Manager()
         self.identity_api = Manager()
         self.policy_api = policy.Manager()
         self.token_api = token.Manager()
@@ -412,7 +410,6 @@ def update_user_tenant(self, context, user_id, user):
 
 class RoleController(wsgi.Application):
     def __init__(self):
-        self.catalog_api = catalog.Manager()
         self.identity_api = Manager()
         self.token_api = token.Manager()
         self.policy_api = policy.Manager()