New feature: support for {include} syntax. Fixes getpelican#1902.

The new {include} syntax makes it possible to include frequently used text snippets into your content.
MrSenko · Mar 11, 2016 · 6847446 · 6847446
1 parent 945b838
commit 6847446
Show file tree

Hide file tree

Showing 9 changed files with 214 additions and 16 deletions.
diff --git a/docs/changelog.rst b/docs/changelog.rst
@@ -4,7 +4,7 @@ Release history
 Next release
 ============
 
-- Nothing yet
+* Add support for the ``{include}`` syntax
 
 3.6.3 (2015-08-14)
 ==================

diff --git a/docs/content.rst b/docs/content.rst
@@ -335,6 +335,45 @@ Linking to authors, categories, index and tags
 You can link to authors, categories, index and tags using the ``{author}name``,
 ``{category}foobar``, ``{index}`` and ``{tag}tagname`` syntax.
 
+Including common text into your content
+---------------------------------------
+
+From Pelican 3.6.4 you can include common text snippets into your content using
+the ``{include}file.ext`` syntax. You can specify semi-absolute paths starting
+from the ``PATH`` directory, e.g. ``{include}/pages/disclaimer.html`` or use
+relative paths, e.g. ``{include}notice.html``. Relativity is
+calculated based on the location of the file containing the ``{include}``.
+For example when you have the following content layout::
+
+    content
+    └── notice2.html
+    └── pages
+        ├── page1.html
+        └── notice1.html
+
+Then the includes may look like::
+
+    <html>
+        <head>
+            <title>PAGE 1</title>
+        </head>
+        <body>
+            This is the content of page 1
+
+            {include}../notice2.html
+        </body>
+    </html>
+
+
+``notice2.html`` looks like::
+
+    {include}pages/notice1.html
+    This is the second warning about relative paths
+
+When using ``{include}`` it is best to blacklist the included files using the
+``IGNORE_FILES`` setting. Otherwise Pelican will try to render them as regular
+content and will most likely fail!
+
 Deprecated internal link syntax
 -------------------------------
 

diff --git a/pelican/contents.py b/pelican/contents.py
@@ -16,7 +16,7 @@
 from pelican import signals
 from pelican.settings import DEFAULT_CONFIG
 from pelican.utils import (SafeDatetime, deprecated_attribute, memoized,
-                           path_to_url, posixize_path,
+                           path_to_url, pelican_open, posixize_path,
                            python_2_unicode_compatible, set_date_tzinfo,
                            slugify, strftime, truncate_html_words)
 
@@ -186,6 +186,30 @@ def get_url_setting(self, key):
         key = key if self.in_default_lang else 'lang_%s' % key
         return self._expand_settings(key)
 
+    def _path_replacer(self, path, relative_dir=None):
+        """
+        Update path depending on whether this is an absolute
+        or relative value.
+        """
+        if not relative_dir:
+            relative_dir = self.relative_dir
+
+        if path.startswith('/'):
+            path = path[1:]
+        else:
+            # relative to the source path of this content
+            path = self.get_relative_source_path(
+                os.path.join(relative_dir, path)
+            )
+
+        if path not in self._context['filenames']:
+            unquoted_path = path.replace('%20', ' ')
+
+            if unquoted_path in self._context['filenames']:
+                path = unquoted_path
+
+        return path
+
     def _update_content(self, content, siteurl):
         """Update the content attribute.
 
@@ -217,19 +241,7 @@ def replacer(m):
 
             # XXX Put this in a different location.
             if what in {'filename', 'attach'}:
-                if path.startswith('/'):
-                    path = path[1:]
-                else:
-                    # relative to the source path of this content
-                    path = self.get_relative_source_path(
-                        os.path.join(self.relative_dir, path)
-                    )
-
-                if path not in self._context['filenames']:
-                    unquoted_path = path.replace('%20', ' ')
-
-                    if unquoted_path in self._context['filenames']:
-                        path = unquoted_path
+                path = self._path_replacer(path)
 
                 linked_content = self._context['filenames'].get(path)
                 if linked_content:
@@ -276,12 +288,45 @@ def replacer(m):
     def get_siteurl(self):
         return self._context.get('localsiteurl', '')
 
+    def _update_includes(self, content, source_path=None):
+        """
+            Replace {include}some.file with the
+            contents of this file.
+        """
+        regex = r"""[{|]include[|}](?P<path>[\w./]+)"""
+        hrefs = re.compile(regex, re.X)
+
+        def replacer(m):
+            path = m.group('path')
+            path = self._path_replacer(path, source_path)
+            path = posixize_path(
+                    os.path.abspath(
+                        os.path.join(self.settings['PATH'], path)
+                    )
+                )
+
+            if not os.path.isfile(path):
+                logger.warning("Unable to find `%s`, skipping include.", path)
+                return ''.join(('{include}', m.group('path')))
+
+            with pelican_open(path) as text:
+                # if we recurse into another file to perform more includes
+                # self._path_replacer needs to know in which directory
+                # it operates otherwise it produces wrong paths
+                source_dir = posixize_path(os.path.dirname(path))
+
+                text = self._update_includes(text, source_dir)
+                return text
+
+        return hrefs.sub(replacer, content)
+
     @memoized
     def get_content(self, siteurl):
         if hasattr(self, '_get_content'):
             content = self._get_content()
         else:
             content = self._content
+        content = self._update_includes(content)
         return self._update_content(content, siteurl)
 
     @property

diff --git a/pelican/tests/content/include/include3.html b/pelican/tests/content/include/include3.html
@@ -0,0 +1,2 @@
+this file includes another in a different directory
+{include}../include1.html
diff --git a/pelican/tests/content/include/include4.html b/pelican/tests/content/include/include4.html
@@ -0,0 +1,2 @@
+this file includes another via absolute path
+{include}/include1.html
diff --git a/pelican/tests/content/include1.html b/pelican/tests/content/include1.html
@@ -0,0 +1 @@
+<span>this content has been included</span>
diff --git a/pelican/tests/content/include2.html b/pelican/tests/content/include2.html
@@ -0,0 +1,2 @@
+this file includes another
+{include}include1.html
diff --git a/pelican/tests/test_cache.py b/pelican/tests/test_cache.py
@@ -60,8 +60,11 @@ def test_article_object_caching(self):
         - article_with_comments.html
         - article_with_null_attributes.html
         - 2012-11-30_md_w_filename_meta#foo-bar.md
+
+        There are 4 more include*.html files which are HTML snippets
+        and also not valid.
         """
-        self.assertEqual(generator.readers.read_file.call_count, 4)
+        self.assertEqual(generator.readers.read_file.call_count, 8)
 
     @unittest.skipUnless(MagicMock, 'Needs Mock module')
     def test_article_reader_content_caching(self):

diff --git a/pelican/tests/test_contents.py b/pelican/tests/test_contents.py
@@ -22,6 +22,8 @@
 TEST_CONTENT = str(generate_lorem_ipsum(n=1))
 TEST_SUMMARY = generate_lorem_ipsum(n=1, html=False)
 
+CONTENT_PATH = os.path.join(os.path.dirname(__file__), 'content')
+
 
 class TestPage(LoggedTestCase):
 
@@ -418,6 +420,108 @@ def test_intrasite_link_markdown_spaces(self):
             '<a href="http://notmyidea.org/article-spaces.html">link</a>'
         )
 
+    def test_includes(self):
+        args = self.page_kwargs.copy()
+        args['settings'] = get_settings()
+        args['source_path'] = CONTENT_PATH
+        args['context']['filenames'] = {}
+        settings = get_settings()
+        settings['PATH'] = CONTENT_PATH
+        args['settings'] = settings
+
+        # one include via relative path
+        args['content'] = (
+            'There is a simple include here '
+            '{include}include1.html'
+            ' Included content is above'
+        )
+        content = Page(**args).get_content('http://notmyidea.org')
+        self.assertEqual(
+            content,
+            'There is a simple include here '
+            '<span>this content has been included</span>'
+            ' Included content is above'
+        )
+
+        # two nested includes via relative paths
+        args['content'] = (
+            'There is a simple include here '
+            '{include}include2.html'
+            ' Included content is above'
+        )
+        content = Page(**args).get_content('http://notmyidea.org')
+        self.assertEqual(
+            content,
+            'There is a simple include here '
+            'this file includes another\n'
+            '<span>this content has been included</span>'
+            ' Included content is above'
+        )
+
+        # include via full path
+        args['content'] = (
+            'There is a simple include here '
+            '{include}/include1.html'
+            ' Included content is above'
+        )
+        content = Page(**args).get_content('http://notmyidea.org')
+        self.assertEqual(
+            content,
+            'There is a simple include here '
+            '<span>this content has been included</span>'
+            ' Included content is above'
+        )
+
+        # 2nd include is in different directory
+        # include paths are relative to the caller directory
+        args['content'] = (
+            'There is a simple include here '
+            '{include}include/include3.html'
+            ' Included content is above'
+        )
+        content = Page(**args).get_content('http://notmyidea.org')
+        self.assertEqual(
+            content,
+            'There is a simple include here '
+            'this file includes another in a different directory\n'
+            '<span>this content has been included</span>'
+            ' Included content is above'
+        )
+
+        # 2nd include using absolute path in the included file
+        args['content'] = (
+            'There is a simple include here '
+            '{include}include/include4.html'
+            ' Included content is above'
+        )
+        content = Page(**args).get_content('http://notmyidea.org')
+        self.assertEqual(
+            content,
+            'There is a simple include here '
+            'this file includes another via absolute path\n'
+            '<span>this content has been included</span>'
+            ' Included content is above'
+        )
+
+        # include non-existing file => inclusion is skipped
+        args['content'] = (
+            'There is a simple include here '
+            '{include}missing.html'
+            ' Included content is above'
+        )
+        content = Page(**args).get_content('http://notmyidea.org')
+        self.assertEqual(
+            content,
+            'There is a simple include here '
+            '{include}missing.html'
+            ' Included content is above'
+        )
+        # we have a warning in this case
+        self.assertLogCountEqual(
+                count=1,
+                msg="Unable to find `.*`, skipping include\.",
+                level=logging.WARNING)
+
     def test_multiple_authors(self):
         """Test article with multiple authors."""
         args = self.page_kwargs.copy()