Merge "feat: admin endpoint to query domains by provider_url"

poppy/manager/default/services.py

@@ -107,6 +107,13 @@ def get_certs_by_status(self, status):
 
         return services_project_ids
 
+    def get_domains_by_provider_url(self, provider_url):
+
+        domains = \
+            self.storage_controller.get_domains_by_provider_url(provider_url)
+
+        return domains
+
     def _append_defaults(self, service_json, operation='create'):
         # default origin rule
         for origin in service_json.get('origins', []):

poppy/storage/cassandra/migrations/010_provider_url_domain.cql

@@ -0,0 +1,9 @@
+CREATE TABLE provider_url_domain (
+    provider_url VARCHAR,
+    domain_name VARCHAR,
+    PRIMARY KEY (provider_url, domain_name));
+
+
+--//@UNDO
+
+DROP TABLE provider_url_domain;

poppy/storage/cassandra/services.py

@@ -278,6 +278,23 @@
         %(status)s)
 '''
 
+CQL_SET_PROVIDER_URL = '''
+    INSERT INTO provider_url_domain(provider_url,
+    domain_name)
+    VALUES (%(provider_url)s, %(domain_name)s)
+'''
+
+CQL_DELETE_PROVIDER_URL = '''
+    DELETE FROM provider_url_domain
+    WHERE provider_url = %(provider_url)s
+    AND domain_name = %(domain_name)s
+'''
+
+CQL_GET_BY_PROVIDER_URL = '''
+    SELECT domain_name FROM provider_url_domain
+    WHERE provider_url = %(provider_url)s
+'''
+
 CQL_GET_SERVICE_STATUS = '''
     SELECT project_id,
         service_id
@@ -489,6 +506,39 @@ def delete_services_by_status(self, project_id, service_id, status):
 
         self.session.execute(stmt, args)
 
+    def get_domains_by_provider_url(self, provider_url):
+
+        LOG.info("Getting domains by provider_url: {0}".format(provider_url))
+
+        get_domain_provider_url_args = {
+            'provider_url': provider_url,
+        }
+
+        stmt = query.SimpleStatement(
+            CQL_GET_BY_PROVIDER_URL,
+            consistency_level=self._driver.consistency_level)
+
+        resultset = self.session.execute(stmt, get_domain_provider_url_args)
+
+        return list(resultset)
+
+    def delete_provider_url(self, provider_url, domain_name):
+
+        LOG.info("Deleting provider_url: {0} and "
+                 "domain_name: {1} from provider_url_domain "
+                 "column family".format(provider_url, domain_name))
+
+        del_provider_url_args = {
+            'provider_url': provider_url,
+            'domain_name': domain_name
+        }
+
+        stmt = query.SimpleStatement(
+            CQL_DELETE_PROVIDER_URL,
+            consistency_level=self._driver.consistency_level)
+
+        self.session.execute(stmt, del_provider_url_args)
+
     def get_service_limit(self, project_id):
         """get_service_limit
 
@@ -577,12 +627,22 @@ def set_service_provider_details(self, project_id, service_id, status):
 
         """
 
-        LOG.info("Setting service"
-                 "status for"
+        LOG.info("Setting service "
+                 "status for "
                  "service_id : {0}, "
                  "project_id: {1} to be {2}".format(service_id,
                                                     project_id,
                                                     status))
+        status_args = {
+            'service_id': uuid.UUID(str(service_id)),
+            'project_id': project_id,
+            'status': status
+        }
+
+        stmt = query.SimpleStatement(
+            CQL_SET_SERVICE_STATUS,
+            consistency_level=self._driver.consistency_level)
+        self.session.execute(stmt, status_args)
 
         provider_details_dict = self.get_provider_details(
             project_id=project_id,
@@ -839,6 +899,7 @@ def update(self, project_id, service_id, service_obj):
             consistency_level=self._driver.consistency_level)
         self.session.execute(stmt, args)
 
+        self.set_service_provider_details(project_id, service_id, status)
         # claim new domains
         batch_claim = query.BatchStatement(
             consistency_level=self._driver.consistency_level)
@@ -875,17 +936,6 @@ def update(self, project_id, service_id, service_obj):
                 consistency_level=self._driver.consistency_level)
             self.session.execute(stmt, args)
 
-        status_args = {
-            'service_id': uuid.UUID(str(service_id)),
-            'project_id': project_id,
-            'status': status
-        }
-
-        stmt = query.SimpleStatement(
-            CQL_SET_SERVICE_STATUS,
-            consistency_level=self._driver.consistency_level)
-        self.session.execute(stmt, status_args)
-
     def update_state(self, project_id, service_id, state):
         """update_state
 
@@ -931,11 +981,24 @@ def delete(self, project_id, service_id):
             pds = result.get('provider_details', {}) or {}
             pds = {key: value for key, value in pds.items()}
             status = None
+            provider_urls_domain = []
+
             for provider in pds:
                 pds_provider_dict = json.loads(pds.get(provider, {}))
                 status = pds_provider_dict.get('status', '')
+                access_urls = pds_provider_dict.get('access_urls', [])
+                for access_url in access_urls:
+                    provider_url = access_url.get('provider_url', None)
+                    domain = access_url.get('domain', None)
+                    if provider_url and domain:
+                        provider_urls_domain.append((provider_url, domain))
+
             self.delete_services_by_status(project_id, service_id, status)
 
+            for provider_url_domain in provider_urls_domain:
+                provider_url, domain = provider_url_domain
+                self.delete_provider_url(provider_url, domain)
+
             if self._driver.archive_on_delete:
                 archive_args = {
                     'project_id': result.get('project_id'),
@@ -1082,12 +1145,19 @@ def update_provider_details(self, project_id, service_id,
 
         provider_detail_dict = {}
         status = None
+        domain_names_provider_urls = []
         for provider_name in sorted(provider_details.keys()):
             the_provider_detail_dict = collections.OrderedDict()
             the_provider_detail_dict["id"] = (
                 provider_details[provider_name].provider_service_id)
             the_provider_detail_dict["access_urls"] = (
                 provider_details[provider_name].access_urls)
+            for access_url in the_provider_detail_dict["access_urls"]:
+                domain_name = access_url.get("domain", None)
+                provider_url = access_url.get("provider_url", None)
+                if domain_name and provider_url:
+                    domain_names_provider_urls.append((domain_name,
+                                                       provider_url))
             the_provider_detail_dict["status"] = (
                 provider_details[provider_name].status)
             status = the_provider_detail_dict["status"]
@@ -1118,7 +1188,7 @@ def update_provider_details(self, project_id, service_id,
             consistency_level=self._driver.consistency_level)
         self.session.execute(stmt, args)
 
-        args = {
+        service_args = {
             'project_id': project_id,
             'service_id': uuid.UUID(str(service_id)),
             'status': status
@@ -1127,7 +1197,19 @@ def update_provider_details(self, project_id, service_id,
         stmt = query.SimpleStatement(
             CQL_SET_SERVICE_STATUS,
             consistency_level=self._driver.consistency_level)
-        self.session.execute(stmt, args)
+        self.session.execute(stmt, service_args)
+
+        if domain_names_provider_urls:
+            for domain_name, provider_url in domain_names_provider_urls:
+                provider_url_args = {
+                    'domain_name': domain_name,
+                    'provider_url': provider_url
+                }
+
+                stmt = query.SimpleStatement(
+                    CQL_SET_PROVIDER_URL,
+                    consistency_level=self._driver.consistency_level)
+                self.session.execute(stmt, provider_url_args)
 
     def update_cert_info(self, domain_name, cert_type, flavor_id,
                          cert_details):

poppy/transport/pecan/controllers/v1/admin.py

@@ -413,7 +413,6 @@ class AdminCertController(base.Controller, hooks.HookController):
     def __init__(self, driver):
         super(AdminCertController, self).__init__(driver)
 
-    @pecan.expose('json')
     @pecan.expose('json')
     @decorators.validate(
         request=rule.Rule(
@@ -443,7 +442,6 @@ def __init__(self, driver):
         self.__class__.action = OperatorServiceActionController(driver)
         self.__class__.status = ServiceStatusController(driver)
 
-    @pecan.expose('json')
     @pecan.expose('json')
     @decorators.validate(
         request=rule.Rule(
@@ -490,6 +488,25 @@ def get_one(self, domain_name):
         # convert a service model into a response service model
         return resp_service_model.Model(service_obj, self)
 
+    @pecan.expose('json')
+    @decorators.validate(
+        request=rule.Rule(
+            helpers.is_valid_provider_url(),
+            helpers.abort_with_message,
+            stoplight_helpers.pecan_getter)
+    )
+    def get(self):
+        services_controller = self._driver.manager.services_controller
+
+        call_args = getattr(pecan.request.context,
+                            "call_args")
+        provider_url = call_args.pop('provider_url')
+        domains = services_controller.get_domains_by_provider_url(
+            provider_url)
+
+        return pecan.Response(json_body=domains,
+                              status=200)
+
 
 class AdminController(base.Controller, hooks.HookController):
     def __init__(self, driver):

poppy/transport/validators/helpers.py

@@ -467,6 +467,30 @@ def is_valid_domain_by_name(domain_name):
                 u'Domain {0} is not valid'.format(domain_name))
 
 
+@decorators.validation_function
+def is_valid_provider_url(request):
+
+    provider_url = request.GET.get("provider_url", None)
+    if not provider_url:
+        raise exceptions.ValidationFailed('provider_url needs to be '
+                                          'provided as a query parameter')
+    provider_url_regex_1 = ('^([A-Za-z0-9-]){1,255}\.([A-Za-z0-9-]){1,255}\.'
+                            '([A-Za-z0-9-]){1,255}\.([A-Za-z0-9-]){1,255}\.'
+                            '([A-Za-z0-9-]){1,255}\.([A-Za-z0-9-]){1,255}$')
+    provider_url_regex_2 = ('^([A-Za-z0-9-]){1,255}\.([A-Za-z0-9-]){1,255}\.'
+                            '([A-Za-z0-9-]){1,255}\.([A-Za-z0-9-]){1,255}\.'
+                            '([A-Za-z0-9-]){1,255}$')
+    if not re.match(provider_url_regex_1, provider_url):
+        if not re.match(provider_url_regex_2, provider_url):
+            raise exceptions.ValidationFailed(
+                u'Provider url {0} is not valid'.format(provider_url))
+
+    # Update context so the decorated function can get all this parameters
+    request.context.call_args = {
+        'provider_url': provider_url,
+    }
+
+
 def is_valid_flavor_configuration(flavor, schema):
     if schema is not None:
         errors_list = list(

tests/api/utils/schema/admin.py

@@ -19,7 +19,6 @@
 from tests.api.utils.schema.services import service_id
 
 get_service_project_status = {
-    'type': 'array',
     'items': [
         {
             'type': 'object',

tests/functional/transport/pecan/controllers/test_get_domain_by_provider_url.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2015 Rackspace, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+import ddt
+from hypothesis import given
+from hypothesis import strategies
+import mock
+import six
+
+from poppy.manager.default.services import DefaultServicesController
+
+from tests.functional.transport.pecan import base
+
+
+@ddt.ddt
+class TestGetDomainsbyProviderurl(base.FunctionalTest):
+
+    def test_get_domains_provider_url_no_queryparam(self):
+        # no provider_url field
+        url = '/v1.0/admin/domains'
+        response = self.app.get(url,
+                                headers={'Content-Type':
+                                         'application/json',
+                                         'X-Project-ID':
+                                         str(uuid.uuid4())},
+                                expect_errors=True)
+        self.assertEqual(response.status_code, 400)
+
+    @given(strategies.text())
+    def test_get_domains_provider_url_invalid_queryparam(self,
+                                                         provider_url):
+        # invalid provider_url field
+        try:
+            # NOTE(TheSriram): Py3k Hack
+            if six.PY3 and type(provider_url) == str:
+                provider_url = provider_url.encode('utf-8')
+                url = '/v1.0/admin/domains?' \
+                      'provider_url={0}'.format(provider_url)
+
+            else:
+                url = '/v1.0/admin/domains?provider_url=%s' \
+                      % provider_url.decode('utf-8')
+        except (UnicodeDecodeError, UnicodeEncodeError):
+            pass
+        else:
+            response = self.app.get(url,
+                                    headers={'Content-Type':
+                                             'application/json',
+                                             'X-Project-ID':
+                                             str(uuid.uuid4())},
+                                    expect_errors=True)
+
+            self.assertEqual(response.status_code, 400)
+
+    @ddt.data('provider.com.extension.provideredge.net',
+              'secure.shard.domain.com.provideredge.net',
+              'www.domain.com.provideredge.net')
+    def test_get_domains_provider_url_valid_queryparam(self, provider_url):
+        # valid provider_url
+        with mock.patch.object(DefaultServicesController,
+                               'get_domains_by_provider_url'):
+            response = self.app.get('/v1.0/admin/domains'
+                                    '?provider_url={0}'.format(provider_url),
+                                    headers={'Content-Type':
+                                             'application/json',
+                                             'X-Project-ID':
+                                             str(uuid.uuid4())})
+
+            self.assertEqual(response.status_code, 200)

tests/unit/storage/cassandra/data_provider_details.json

@@ -1,9 +1,9 @@
 {
 	"provider_details":
 			{
-			   "MaxCDN": "{\"id\": 11942, \"access_urls\": [\"mypullzone.netdata.com\"], \"domains_certificate_status\":{\"mypullzone.com\": \"failed\"} }",
-			   "Mock": "{\"id\": 73242, \"access_urls\": [\"mycdn.mock.com\"], \"domains_certificate_status\":{\"mycdn.mock.com\": \"deployed\"} }",
-			   "CloudFront": "{\"id\": \"5ABC892\", \"access_urls\": [\"cf123.cloudcf.com\"]}",
-			   "Fastly": "{\"id\": 3488, \"access_urls\": [\"mockcf123.fastly.prod.com\"], \"domains_certificate_status\":{\"mockcf123.com\": \"create_in_progress\"}}"
+			   "MaxCDN": "{\"id\": 11942, \"access_urls\": [{\"provider_url\": \"maxcdn.provider.com\", \"domain\": \"xk.cd\"}], \"domains_certificate_status\":{\"mypullzone.com\": \"failed\"} }",
+			   "Mock": "{\"id\": 73242, \"access_urls\": [{\"provider_url\": \"mycdn.mock.com\", \"domain\": \"xk.cd\"}], \"domains_certificate_status\":{\"mycdn.mock.com\": \"deployed\"} }",
+			   "CloudFront": "{\"id\": \"5ABC892\", \"access_urls\": [{\"provider_url\": \"cloudfront.provider.com\", \"domain\": \"xk.cd\"}]}",
+			   "Fastly": "{\"id\": 3488, \"access_urls\": [{\"provider_url\": \"fastly.provider.com\", \"domain\": \"xk.cd\"}], \"domains_certificate_status\":{\"mockcf123.com\": \"create_in_progress\"}}"
 			}
 }