[Resolves #1124] http template handler (#1125)

Add a web (http) template handler to allow referencing templates
directly from the web. Supports standard template files (.yaml, .json,
.template), jinja and python templates.

docs/_source/docs/template_handlers.rst

@@ -63,7 +63,30 @@ Example:
 
    template:
      type: s3
-     path: infra-templates/s3/v1/bucket.yaml
+     path: infra-templates/v1/storage/bucket.yaml
+
+http
+~~~~~~~~~~~~~
+
+Downloads a template from a url on the web.  This handler supports templates with .json, .yaml,
+.template, .j2 and .py extensions.
+
+Syntax:
+
+.. code-block:: yaml
+
+   template:
+     type: http
+     url: <url>
+
+Example:
+
+.. code-block:: yaml
+
+   template:
+     type: http
+     url: https://raw.githubusercontent.com/acme/infra-templates/v1/storage/bucket.yaml
+
 
 Custom Template Handlers
 ------------------------

requirements/dev.txt

@@ -10,6 +10,7 @@ pytest>=6.2.0,<7.0.0
 pytest-cov>=2.11.1,<3.0.0
 pytest-sugar>=0.9.4,<1.0.0
 readme-renderer>=24.0,<25.0
+requests-mock>=1.9.3,<2.0
 setuptools>=40.6.2,<41.0.0
 Sphinx>=1.6.5,<5.0.0
 sphinx-click>=2.0.1,<4.0.0

sceptre/template_handlers/__init__.py

@@ -29,6 +29,12 @@ class TemplateHandler:
 
     __metaclass__ = abc.ABCMeta
 
+    standard_template_extensions = [".json", ".yaml", ".template"]
+    jinja_template_extensions = [".j2"]
+    python_template_extensions = [".py"]
+    supported_template_extensions = standard_template_extensions + \
+        jinja_template_extensions + python_template_extensions
+
     def __init__(self, name, arguments=None, sceptre_user_data=None, connection_manager=None):
         self.logger = logging.getLogger(__name__)
         self.name = name

sceptre/template_handlers/http.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+import pathlib
+import os
+import requests
+import tempfile
+import sceptre.template_handlers.helper as helper
+
+from sceptre.exceptions import UnsupportedTemplateFileTypeError
+from sceptre.template_handlers import TemplateHandler
+from urllib.parse import urlparse
+
+
+class Http(TemplateHandler):
+    """
+    Template handler that can resolve templates from the web.  Standard CFN templates
+    with extension (.json, .yaml, .template) are deployed directly from memory
+    while references to jinja (.j2) and python (.py) templates are downloaded,
+    transformed into CFN templates then deployed to AWS.
+    """
+    def __init__(self, *args, **kwargs):
+        super(Http, self).__init__(*args, **kwargs)
+
+    def schema(self):
+        return {
+            "type": "object",
+            "properties": {
+                "url": {"type": "string"}
+            },
+            "required": ["url"]
+        }
+
+    def handle(self):
+        """
+        handle template from web
+        """
+        url = self.arguments["url"]
+        path = pathlib.Path(urlparse(url).path)
+
+        if path.suffix not in self.supported_template_extensions:
+            raise UnsupportedTemplateFileTypeError(
+                "Template has file extension %s. Only %s are supported.",
+                path.suffix, ",".join(self.supported_template_extensions)
+            )
+
+        try:
+            template = self._get_template(url)
+            if path.suffix in self.jinja_template_extensions + self.python_template_extensions:
+                file = tempfile.NamedTemporaryFile(prefix=path.stem)
+                self.logger.debug("Template file saved to: %s", file.name)
+                with file as f:
+                    f.write(template)
+                    f.seek(0)
+                    f.read()
+                    if path.suffix in self.jinja_template_extensions:
+                        template = helper.render_jinja_template(
+                            os.path.dirname(f.name),
+                            os.path.basename(f.name),
+                            {"sceptre_user_data": self.sceptre_user_data}
+                        )
+                    elif path.suffix in self.python_template_extensions:
+                        template = helper.call_sceptre_handler(
+                            f.name,
+                            self.sceptre_user_data
+                        )
+
+        except Exception as e:
+            helper.print_template_traceback(path)
+            raise e
+
+        return template
+
+    def _get_template(self, url):
+        """
+        Get template from the web
+        :param url: The url to the template
+        :type: str
+        :returns: The body of the CloudFormation template.
+        :rtype: str
+        """
+        self.logger.debug("Downloading file from: %s", url)
+        try:
+            response = requests.get(url)
+            return response.content
+        except requests.exceptions.RequestException as e:
+            self.logger.fatal(e)
+            raise e

setup.py

@@ -74,7 +74,8 @@ def get_version(rel_path):
         ],
         "sceptre.template_handlers": [
             "file = sceptre.template_handlers.file:File",
-            "s3 = sceptre.template_handlers.s3:S3"
+            "s3 = sceptre.template_handlers.s3:S3",
+            "http = sceptre.template_handlers.http:Http"
         ]
     },
     data_files=[

tests/test_template_handlers/test_http.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+import json
+import pytest
+
+from sceptre.exceptions import UnsupportedTemplateFileTypeError
+from sceptre.template_handlers.http import Http
+from unittest.mock import patch
+
+
+class TestHttp(object):
+
+    def test_get_template(self, requests_mock):
+        url = "https://raw.githubusercontent.com/acme/bucket.yaml"
+        requests_mock.get(url, content=b"Stuff is working")
+        template_handler = Http(
+            name="vpc",
+            arguments={"url": url},
+        )
+        result = template_handler.handle()
+        assert result == b"Stuff is working"
+
+    def test_handler_unsupported_type(self):
+        handler = Http("http_handler", {'url': 'https://raw.githubusercontent.com/acme/bucket.unsupported'})
+        with pytest.raises(UnsupportedTemplateFileTypeError):
+            handler.handle()
+
+    @pytest.mark.parametrize("url", [
+        ("https://raw.githubusercontent.com/acme/bucket.json"),
+        ("https://raw.githubusercontent.com/acme/bucket.yaml"),
+        ("https://raw.githubusercontent.com/acme/bucket.template")
+    ])
+    @patch('sceptre.template_handlers.http.Http._get_template')
+    def test_handler_raw_template(self, mock_get_template, url):
+        mock_get_template.return_value = {}
+        handler = Http("http_handler", {'url': url})
+        handler.handle()
+        assert mock_get_template.call_count == 1
+
+    @patch('sceptre.template_handlers.helper.render_jinja_template')
+    @patch('sceptre.template_handlers.http.Http._get_template')
+    def test_handler_jinja_template(slef, mock_get_template, mock_render_jinja_template):
+        mock_get_template_response = {
+            "Description": "test template",
+            "AWSTemplateFormatVersion": "2010-09-09",
+            "Resources": {
+                "touchNothing": {
+                    "Type": "AWS::CloudFormation::WaitConditionHandle"
+                }
+            }
+        }
+        mock_get_template.return_value = json.dumps(mock_get_template_response).encode('utf-8')
+        handler = Http("http_handler", {'url': 'https://raw.githubusercontent.com/acme/bucket.j2'})
+        handler.handle()
+        assert mock_render_jinja_template.call_count == 1
+
+    @patch('sceptre.template_handlers.helper.call_sceptre_handler')
+    @patch('sceptre.template_handlers.http.Http._get_template')
+    def test_handler_python_template(self, mock_get_template, mock_call_sceptre_handler):
+        mock_get_template_response = {
+            "Description": "test template",
+            "AWSTemplateFormatVersion": "2010-09-09",
+            "Resources": {
+                "touchNothing": {
+                    "Type": "AWS::CloudFormation::WaitConditionHandle"
+                }
+            }
+        }
+        mock_get_template.return_value = json.dumps(mock_get_template_response).encode('utf-8')
+        handler = Http("http_handler", {'url': 'https://raw.githubusercontent.com/acme/bucket.py'})
+        handler.handle()
+        assert mock_call_sceptre_handler.call_count == 1