Add retry and timeout to http template handler (#1145)

The python request library does not provide retries by
default. To make the http template handler more robust
we retry the download a few times. Also by default the
request library timeout is set to off which means it
can hang indefinitely if it get stuck downloading templates.
We add a timeout to make sure downloads don't get stuck.

docs/_source/docs/stack_group_config.rst

@@ -20,6 +20,7 @@ Sceptre. The available keys are listed below.
 -  `template_bucket_name`_ *(optional)*
 -  `template_key_prefix`_ *(optional)*
 -  `j2_environment`_ *(optional)*
+-  `http_template_handler`_ *(optional)*
 
 Sceptre will only check for and uses the above keys in StackGroup config files
 and are directly accessible from Stack(). Any other keys added by the user are
@@ -92,6 +93,18 @@ This will impact :ref:`Templating` of stacks by modifying the behavior of jinja.
       trim_blocks: True
       newline_sequence: \n
 
+http_template_handler
+~~~~~~~~~~~~~~~~~~~~~
+
+Options passed to the `http template handler`_.
+  * retries - The number of retry attempts (default is 5)
+  * timeout - The timeout for the session in seconds (default is 5)
+
+.. code-block:: yaml
+
+   http_template_handler:
+      retries: 10
+      timeout: 20
 
 require_version
 ~~~~~~~~~~~~~~~
@@ -312,3 +325,4 @@ Examples
 .. _region which supports CloudFormation: http://docs.aws.amazon.com/general/latest/gr/rande.html#cfn_region
 .. _PEP 440: https://www.python.org/dev/peps/pep-0440/#version-specifiers
 .. _AWS_CLI_Configure: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html
+.. _http template handler: template_handlers.html#http

docs/_source/docs/template_handlers.rst

@@ -67,8 +67,10 @@ Example:
 http
 ~~~~~~~~~~~~~
 
-Downloads a template from a url on the web.  This handler supports templates with .json, .yaml,
-.template, .j2 and .py extensions.
+Downloads a template from a url on the web.  By default, this handler will attempt to download
+templates with 5 retries and a download timeout of 5 seconds.  The default retry and timeout
+options can be overridden by setting the `http_template_handler key`_ in the stack group config
+file.
 
 Syntax:
 
@@ -87,6 +89,7 @@ Example:
      url: https://raw.githubusercontent.com/acme/infra-templates/v1/storage/bucket.yaml
 
 
+
 Custom Template Handlers
 ------------------------
 
@@ -216,3 +219,4 @@ This template handler can be used in a Stack config file with the following synt
 .. _jsonschema library: https://github.com/Julian/jsonschema
 .. _Custom Template Handlers: #custom-template-handlers
 .. _Boto3: https://aws.amazon.com/sdk-for-python/
+.. _http_template_handler key: stack_group_config.html#http-template-handler

sceptre/template_handlers/http.py

@@ -4,10 +4,19 @@
 import tempfile
 import sceptre.template_handlers.helper as helper
 
+from requests.adapters import HTTPAdapter
+from requests.exceptions import InvalidURL, ConnectTimeout
+from requests.packages.urllib3.util.retry import Retry
 from sceptre.exceptions import UnsupportedTemplateFileTypeError
 from sceptre.template_handlers import TemplateHandler
 from urllib.parse import urlparse
 
+HANDLER_OPTION_KEY = "http_template_handler"
+HANDLER_RETRIES_OPTION_PARAM = "retries"
+DEFAULT_RETRIES_OPTION = 5
+HANDLER_TIMEOUT_OPTION_PARAM = "timeout"
+DEFAULT_TIMEOUT_OPTION = 5
+
 
 class Http(TemplateHandler):
     """
@@ -41,8 +50,10 @@ def handle(self):
                 path.suffix, ",".join(self.supported_template_extensions)
             )
 
+        retries = self._get_handler_option(HANDLER_RETRIES_OPTION_PARAM, DEFAULT_RETRIES_OPTION)
+        timeout = self._get_handler_option(HANDLER_TIMEOUT_OPTION_PARAM, DEFAULT_TIMEOUT_OPTION)
         try:
-            template = self._get_template(url)
+            template = self._get_template(url, retries=retries, timeout=timeout)
             if path.suffix in self.jinja_template_extensions + self.python_template_extensions:
                 file = tempfile.NamedTemporaryFile(prefix=path.stem)
                 self.logger.debug("Template file saved to: %s", file.name)
@@ -66,18 +77,57 @@ def handle(self):
 
         return template
 
-    def _get_template(self, url):
+    def _get_template(self, url, retries, timeout):
         """
         Get template from the web
         :param url: The url to the template
         :type: str
-        :returns: The body of the CloudFormation template.
-        :rtype: str
+        :param retries: The number of retry attempts.
+        :rtype: int
+        :param timeout: The timeout for the session in seconds.
+        :rtype: int
         """
         self.logger.debug("Downloading file from: %s", url)
+        session = self._get_retry_session(retries=retries)
         try:
-            response = requests.get(url)
+            response = session.get(url, timeout=timeout)
             return response.content
-        except requests.exceptions.RequestException as e:
-            self.logger.fatal(e)
+        except (InvalidURL, ConnectTimeout) as e:
             raise e
+
+    def _get_retry_session(self,
+                           retries,
+                           backoff_factor=0.3,
+                           status_forcelist=(429, 500, 502, 503, 504),
+                           session=None):
+        """
+        Get a request session with retries.  Retry options are explained in the request libraries
+        https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#module-urllib3.util.retry
+        """
+        session = session or requests.Session()
+        retry = Retry(
+            total=retries,
+            read=retries,
+            connect=retries,
+            backoff_factor=backoff_factor,
+            status_forcelist=status_forcelist,
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+        return session
+
+    def _get_handler_option(self, name, default):
+        """
+        Get the template handler options
+        :param url: The option name
+        :type: str
+        :param default: The default value if option is not set.
+        :rtype: int
+        """
+        if HANDLER_OPTION_KEY in self.stack_group_config:
+            option = self.stack_group_config.get(HANDLER_OPTION_KEY)
+            if name in option:
+                return option.get(name)
+
+        return default

tests/test_template_handlers/test_http.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import json
+
 import pytest
 
 from sceptre.exceptions import UnsupportedTemplateFileTypeError
@@ -69,3 +70,26 @@ def test_handler_python_template(self, mock_get_template, mock_call_sceptre_hand
         handler = Http("http_handler", {'url': 'https://raw.githubusercontent.com/acme/bucket.py'})
         handler.handle()
         assert mock_call_sceptre_handler.call_count == 1
+
+    @patch('sceptre.template_handlers.helper.call_sceptre_handler')
+    @patch('sceptre.template_handlers.http.Http._get_template')
+    def test_handler_override_handler_options(self, mock_get_template, mock_call_sceptre_handler):
+        mock_get_template_response = {
+            "Description": "test template",
+            "AWSTemplateFormatVersion": "2010-09-09",
+            "Resources": {
+                "touchNothing": {
+                    "Type": "AWS::CloudFormation::WaitConditionHandle"
+                }
+            }
+        }
+        mock_get_template.return_value = json.dumps(mock_get_template_response).encode('utf-8')
+        custom_handler_options = {"timeout": 10, "retries": 20}
+        handler = Http("http_handler",
+                       {'url': 'https://raw.githubusercontent.com/acme/bucket.py'},
+                       stack_group_config={"http_template_handler": custom_handler_options}
+                       )
+        handler.handle()
+        assert mock_get_template.call_count == 1
+        args, options = mock_get_template.call_args
+        assert options == {"timeout": 10, "retries": 20}