Merge "Delete old policies when upgrading to HTTPS"

poppy/cmd/delete_obsolete_http_policies.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2016 Rackspace, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+from oslo_config import cfg
+from oslo_log import log
+
+from poppy import bootstrap
+from poppy.common import cli
+
+LOG = log.getLogger(__name__)
+
+CLI_OPT = [
+    cfg.BoolOpt(
+        'run_as_daemon',
+        default=True,
+        required=False,
+        help='Run this script as a long running process.'
+    ),
+    cfg.IntOpt(
+        'sleep_interval',
+        default=60,
+        required=False,
+        help='Sleep interval between runs of http policy delete.'
+    ),
+]
+
+
+@cli.runnable
+def run():
+    # TODO(kgriffs): For now, we have to use the global config
+    # to pick up common options from openstack.common.log, since
+    # that module uses the global CONF instance exclusively.
+    conf = cfg.ConfigOpts()
+    conf.register_cli_opts(CLI_OPT)
+    log.register_options(conf)
+    conf(project='poppy', prog='poppy')
+    log.setup(conf, 'poppy')
+    server = bootstrap.Bootstrap(conf)
+
+    sleep_interval = conf.sleep_interval
+    while True:
+        (
+            run_list,
+            ignore_list
+        ) = server.manager.background_job_controller.delete_http_policy()
+
+        LOG.info(
+            "Policies, attempting to delete {0}, ignored {0}".format(
+                run_list, ignore_list))
+        if conf.run_as_daemon is False:
+            break
+        time.sleep(sleep_interval)

poppy/common/util.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import cgi
+import itertools
 import pprint
 
 from oslo_log import log
@@ -54,3 +55,14 @@ def help_escape(potentially_bad_string):
     if potentially_bad_string is None:
         LOG.warning('Should not happen: trying to escape a None object')
     return cgi.escape(potentially_bad_string or "")
+
+
+# remove duplicates
+# see http://bit.ly/1mX2Vcb for details
+def remove_duplicates(data):
+    """Remove duplicates from the data (normally a list).
+
+    The data must be sortable and have an equality operator
+    """
+    data = sorted(data)
+    return [k for k, _ in itertools.groupby(data)]

poppy/manager/default/background_job.py

@@ -17,11 +17,14 @@
 
 from oslo_log import log
 
+from poppy.common import util
 from poppy.manager import base
 from poppy.model import ssl_certificate
 from poppy.notification.mailgun import driver as n_driver
 from poppy.provider.akamai.background_jobs.check_cert_status_and_update \
     import check_cert_status_and_update_flow
+from poppy.provider.akamai.background_jobs.delete_policy \
+    import delete_obsolete_http_policy_flow
 from poppy.provider.akamai.background_jobs.update_property import \
     update_property_flow
 from poppy.provider.akamai import driver as a_driver
@@ -286,3 +289,59 @@ def put_san_mapping_list(self, san_mapping_list):
             deleted = tuple(x for x in orig if x not in res)
         # other provider's san_mapping_list implementation goes here
         return res, deleted
+
+    def delete_http_policy(self):
+        http_policies = []
+        run_list = []
+        ignore_list = []
+        if 'akamai' in self._driver.providers:
+            akamai_driver = self._driver.providers['akamai'].obj
+            http_policies += akamai_driver.http_policy_queue.traverse_queue(
+                consume=True
+            )
+            http_policies = [json.loads(x) for x in http_policies]
+            http_policies = util.remove_duplicates(http_policies)
+
+            for policy_dict in http_policies:
+                cert_for_domain = self.cert_storage.get_certs_by_domain(
+                    policy_dict['policy_name'],
+                    project_id=policy_dict['project_id'],
+                    cert_type='san'
+                )
+                if cert_for_domain == []:
+                    ignore_list.append(policy_dict)
+                    LOG.info(
+                        "No cert found for policy name. "
+                        "Policy {0} won't persist on the queue. ".format(
+                            policy_dict
+                        )
+                    )
+                    continue
+
+                if cert_for_domain.get_cert_status() != 'deployed':
+                    ignore_list.append(policy_dict)
+                    LOG.info(
+                        "Policy {0} is not ready for deletion. "
+                        "Certificate exists but hasn't deployed yet. "
+                        "Sending back to queue for later retry.".format(
+                            policy_dict
+                        )
+                    )
+                    akamai_driver.http_policy_queue.enqueue_http_policy(
+                        json.dumps(policy_dict)
+                    )
+                    continue
+
+                kwargs = {
+                    'configuration_number': policy_dict[
+                        'configuration_number'],
+                    'policy_name': policy_dict['policy_name']
+                }
+                self.distributed_task_controller.submit_task(
+                    delete_obsolete_http_policy_flow.
+                    delete_obsolete_http_policy,
+                    **kwargs
+                )
+                run_list.append(policy_dict)
+
+        return run_list, ignore_list

poppy/manager/default/ssl_certificate.py

@@ -13,13 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import itertools
 import json
 
 from oslo_context import context as context_utils
 from oslo_log import log
 
 from poppy.common import errors
+from poppy.common import util
 from poppy.distributed_task.taskflow.flow import create_ssl_certificate
 from poppy.distributed_task.taskflow.flow import delete_ssl_certificate
 from poppy.distributed_task.taskflow.flow import recreate_ssl_certificate
@@ -176,16 +176,7 @@ def rerun_san_retry_list(self):
                 res = akamai_driver.mod_san_queue.dequeue_mod_san_request()
                 retry_list.append(json.loads(res.decode('utf-8')))
 
-            # remove duplicates
-            # see http://bit.ly/1mX2Vcb for details
-            def remove_duplicates(data):
-                """Remove duplicates from the data (normally a list).
-
-                The data must be sortable and have an equality operator
-                """
-                data = sorted(data)
-                return [k for k, _ in itertools.groupby(data)]
-            retry_list = remove_duplicates(retry_list)
+            retry_list = util.remove_duplicates(retry_list)
 
             # double check in POST. This check should really be first done in
             # PUT

poppy/provider/akamai/background_jobs/delete_policy/delete_obsolete_http_policy_flow.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2016 Rackspace, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+from oslo_log import log
+from taskflow.patterns import linear_flow
+
+from poppy.provider.akamai.background_jobs.delete_policy import (
+    delete_obsolete_http_policy_tasks)
+
+LOG = log.getLogger(__name__)
+
+
+conf = cfg.CONF
+conf(project='poppy', prog='poppy', args=[])
+
+
+def delete_obsolete_http_policy():
+    flow = linear_flow.Flow('Deleting obsolete HTTP policy').add(
+        delete_obsolete_http_policy_tasks.DeleteObsoleteHTTPPolicy(),
+    )
+    return flow

poppy/provider/akamai/background_jobs/delete_policy/delete_obsolete_http_policy_tasks.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2016 Rackspace, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+from oslo_log import log
+from taskflow import task
+
+from poppy.distributed_task.utils import memoized_controllers
+
+LOG = log.getLogger(__name__)
+
+conf = cfg.CONF
+conf(project='poppy', prog='poppy', args=[])
+
+
+class DeleteObsoleteHTTPPolicy(task.Task):
+    """Delete old HTTP policy once a domain is upgraded to HTTPS SAN."""
+
+    def __init__(self):
+        super(DeleteObsoleteHTTPPolicy, self).__init__()
+        service_controller, self.providers = \
+            memoized_controllers.task_controllers('poppy', 'providers')
+        self.akamai_driver = self.providers['akamai'].obj
+
+    def execute(self, configuration_number, policy_name):
+        """Deletes old HTTP policy once a domain is upgraded to HTTPS+san.
+
+        :param configuration_number: akamai configuration number
+        :param policy_name: name of policy on akamai policy api
+        """
+
+        resp = self.akamai_driver.policy_api_client.delete(
+            self.akamai_driver.akamai_policy_api_base_url.format(
+                configuration_number=configuration_number,
+                policy_name=policy_name
+            )
+        )
+        LOG.info(
+            'akamai response code: {0}'.format(resp.status_code))
+        LOG.info('akamai response text: {0}'.format(resp.text))
+        if resp.status_code != 200:
+            raise RuntimeError(resp.text)
+        LOG.info(
+            'Delete old policy {0} complete'.format(policy_name))

poppy/provider/akamai/driver.py

@@ -27,6 +27,7 @@
 from poppy.provider.akamai import controllers
 from poppy.provider.akamai.domain_san_mapping_queue import zk_san_mapping_queue
 from poppy.provider.akamai import geo_zone_code_mapping
+from poppy.provider.akamai.http_policy_queue import http_policy_queue
 from poppy.provider.akamai.mod_san_queue import zookeeper_queue
 from poppy.provider import base
 import uuid
@@ -200,6 +201,9 @@ def __init__(self, conf):
         self.san_mapping_queue = zk_san_mapping_queue.ZookeeperSanMappingQueue(
             self._conf
         )
+        self.http_policy_queue = http_policy_queue.ZookeeperHttpPolicyQueue(
+            self._conf
+        )
 
         self.metrics_resolution = self.akamai_conf.metrics_resolution
 

poppy/provider/akamai/http_policy_queue/base.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2016 Rackspace, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class HttpPolicyQueue(object):
+    """Keep track of old HTTP policies for deletion.
+
+    The policy object on queue is used to kick off a task
+    that deletes obsolete akamai http policies at pre-defined intervals.
+    """
+
+    def __init__(self, conf):
+        self._conf = conf
+
+    def enqueue_http_policy(self, http_policy):
+        """Add new http policy element to the queue.
+
+        :param http_policy: a new element to add to the queue
+        :type http_policy: dict
+        """
+        raise NotImplementedError
+
+    def dequeue_http_policy(self, consume=True):
+        """Remove and return an item from the queue.
+
+        :param consume: if true the policy is removed from the list and
+            returned otherwise the policy is retrieved queue
+        """
+        raise NotImplementedError
+
+    def traverse_queue(self, consume=False):
+        """Traverse queue and return all items on the queue in a list"""
+        raise NotImplementedError
+
+    def put_queue_data(self, queue_data_list):
+        """Clear the queue and put new queue data list in the queue.
+
+        :param queue_data_list: new queue data to replace current queue data
+        :type queue_data_list: [dict()] -- list of dictionaries
+        """
+        raise NotImplementedError