Merge pull request #2462 from getnikola/add-data-metadata

Add data metadata (Fix #2450)

CHANGES.txt

@@ -4,6 +4,7 @@ New in master
 Features
 --------
 
+* New ``data`` metadata that loads data from external files (Issue #2450)
 * Shortcode to escape to the template language (Issue #1227)
 * Added link to raw file in listings (Issue #1995)
 * New NO_DOCUTILS_TITLE_TRANSFORM (Issue #2382)

docs/manual.txt

@@ -363,6 +363,16 @@ to your configuration:
    enclosure
        Add an enclosure to this post when it's used in RSS. See `more information about enclosures <http://en.wikipedia.org/wiki/RSS_enclosure>`__
 
+   data
+       Path to an external data file, relative to ``conf.py``. Based on it's extension it will
+       be interpreted as JSON or YAML and be available as ``post.data``. The data **must**
+       decode into a python dictionary, and it's keys are available for templates as ``post.data('key')``.
+
+       Translated posts can have different values for this field, and the correct one will be
+       used.
+
+       This is specially useful used in combination with `shortcodes.`__
+
 .. note:: The Two-File Format
 
    Nikola originally used a separate ``.meta`` file. That will still work!
@@ -910,6 +920,10 @@ will create a shortcode called ``mycode`` you can use. Any options you pass to
 the shortcode will be available as variables for that template. Non-keyword
 options will be passed in a tuple varaible named ``_args``.
 
+The post in which the shortcode is being used is available as the ``post``
+variable, so you can access the title as ``post.title``, and data loaded
+via the ``data`` field in the metadata using ``post.data(key)``.
+
 If you use the shortcode as paired, then the contents between the paired tags
 will be available in the ``data`` variable. If you want to access the Nikola
 object, it will be available as ``site``. Use with care :-)
@@ -938,8 +952,9 @@ Finally, you can use a template shortcode without a file, by inserting the templ
     {{% /template %}}
     {{% /raw %}}
 
-In that case, the template engine used will be your theme's and the arguments you pass, as well as the global
-context from your ``conf.py``, are available to the template you are creating.
+In that case, the template engine used will be your theme's and the arguments you pass,
+as well as the global context from your ``conf.py``, are available to the template you
+are creating.
 
 Redirections
 ------------

nikola/nikola.py

@@ -1499,11 +1499,11 @@ def register_shortcode(self, name, f):
         self.shortcode_registry[name] = f
 
     # XXX in v8, get rid of with_dependencies
-    def apply_shortcodes(self, data, filename=None, lang=None, with_dependencies=False):
+    def apply_shortcodes(self, data, filename=None, lang=None, with_dependencies=False, extra_context={}):
         """Apply shortcodes from the registry on data."""
         if lang is None:
             lang = utils.LocaleBorg().current_lang
-        return shortcodes.apply_shortcodes(data, self.shortcode_registry, self, filename, lang=lang, with_dependencies=with_dependencies)
+        return shortcodes.apply_shortcodes(data, self.shortcode_registry, self, filename, lang=lang, with_dependencies=with_dependencies, extra_context=extra_context)
 
     def generic_rss_renderer(self, lang, title, link, description, timeline, output_path,
                              rss_teasers, rss_plain, feed_length=10, feed_url=None,

nikola/plugins/compile/html.py

@@ -44,16 +44,18 @@ class CompileHtml(PageCompiler):
     def compile_html(self, source, dest, is_two_file=True):
         """Compile source file into HTML and save as dest."""
         makedirs(os.path.dirname(dest))
+        try:
+            post = self.site.post_per_input_file[source]
+        except KeyError:
+            post = None
         with io.open(dest, "w+", encoding="utf8") as out_file:
             with io.open(source, "r", encoding="utf8") as in_file:
                 data = in_file.read()
             if not is_two_file:
                 _, data = self.split_metadata(data)
-            data, shortcode_deps = self.site.apply_shortcodes(source, with_dependencies=True)
+            data, shortcode_deps = self.site.apply_shortcodes(source, with_dependencies=True, extra_context=dict(post=post))
             out_file.write(data)
-        try:
-            post = self.site.post_per_input_file[source]
-        except KeyError:
+        if post is None:
             if shortcode_deps:
                 self.logger.error(
                     "Cannot save dependencies for post {0} due to unregistered source file name",

nikola/plugins/compile/ipynb.py

@@ -91,13 +91,15 @@ def compile_html_string(self, source, is_two_file=True):
     def compile_html(self, source, dest, is_two_file=True):
         """Compile source file into HTML and save as dest."""
         makedirs(os.path.dirname(dest))
-        with io.open(dest, "w+", encoding="utf8") as out_file:
-            output = self.compile_html_string(source, is_two_file)
-            output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True)
-            out_file.write(output)
         try:
             post = self.site.post_per_input_file[source]
         except KeyError:
+            post = None
+        with io.open(dest, "w+", encoding="utf8") as out_file:
+            output = self.compile_html_string(source, is_two_file)
+            output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post))
+            out_file.write(output)
+        if post is None:
             if shortcode_deps:
                 self.logger.error(
                     "Cannot save dependencies for post {0} due to unregistered source file name",

nikola/plugins/compile/markdown/__init__.py

@@ -69,17 +69,19 @@ def compile_html(self, source, dest, is_two_file=True):
             req_missing(['markdown'], 'build this site (compile Markdown)')
         makedirs(os.path.dirname(dest))
         self.extensions += self.site.config.get("MARKDOWN_EXTENSIONS")
+        try:
+            post = self.site.post_per_input_file[source]
+        except KeyError:
+            post = None
         with io.open(dest, "w+", encoding="utf8") as out_file:
             with io.open(source, "r", encoding="utf8") as in_file:
                 data = in_file.read()
             if not is_two_file:
                 _, data = self.split_metadata(data)
             output = markdown(data, self.extensions)
-            output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True)
+            output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post))
             out_file.write(output)
-        try:
-            post = self.site.post_per_input_file[source]
-        except KeyError:
+        if post is None:
             if shortcode_deps:
                 self.logger.error(
                     "Cannot save dependencies for post {0} due to unregistered source file name",

nikola/plugins/compile/pandoc.py

@@ -54,14 +54,16 @@ def compile_html(self, source, dest, is_two_file=True):
         """Compile source file into HTML and save as dest."""
         makedirs(os.path.dirname(dest))
         try:
+            try:
+                post = self.site.post_per_input_file[source]
+            except KeyError:
+                post = None
             subprocess.check_call(['pandoc', '-o', dest, source] + self.site.config['PANDOC_OPTIONS'])
             with open(dest, 'r', encoding='utf-8') as inf:
                 output, shortcode_deps = self.site.apply_shortcodes(inf.read(), with_dependencies=True)
             with open(dest, 'w', encoding='utf-8') as outf:
                 outf.write(output)
-            try:
-                post = self.site.post_per_input_file[source]
-            except KeyError:
+            if post is None:
                 if shortcode_deps:
                     self.logger.error(
                         "Cannot save dependencies for post {0} due to unregistered source file name",

nikola/plugins/compile/rest/__init__.py

@@ -95,14 +95,16 @@ def compile_html(self, source, dest, is_two_file=True):
         makedirs(os.path.dirname(dest))
         error_level = 100
         with io.open(dest, "w+", encoding="utf8") as out_file:
+            try:
+                post = self.site.post_per_input_file[source]
+            except KeyError:
+                post = None
             with io.open(source, "r", encoding="utf8") as in_file:
                 data = in_file.read()
                 output, error_level, deps = self.compile_html_string(data, source, is_two_file)
-                output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True)
+                output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post))
                 out_file.write(output)
-            try:
-                post = self.site.post_per_input_file[source]
-            except KeyError:
+            if post is None:
                 if deps.list:
                     self.logger.error(
                         "Cannot save dependencies for post {0} due to unregistered source file name",

nikola/post.py

@@ -161,6 +161,12 @@ def __init__(
             for lang in sorted(self.translated_to):
                 default_metadata.update(self.meta[lang])
 
+        # Load data field from metadata
+        self.data = Functionary(lambda: None, self.default_lang)
+        for lang in self.translations:
+            if self.meta[lang].get('data') is not None:
+                self.data[lang] = utils.load_data(self.meta[lang]['data'])
+
         if 'date' not in default_metadata and not use_in_feeds:
             # For stories we don't *really* need a date
             if self.config['__invariant__']:
@@ -475,6 +481,8 @@ def deps(self, lang):
             cand_3 = get_translation_candidate(self.config, self.metadata_path, lang)
             if os.path.exists(cand_3):
                 deps.append(cand_3)
+        if self.meta('data', lang):
+            deps.append(self.meta('data', lang))
         deps += self._get_dependencies(self._dependency_file_page[lang])
         deps += self._get_dependencies(self._dependency_file_page[None])
         return sorted(deps)

nikola/shortcodes.py

@@ -257,7 +257,7 @@ def _split_shortcodes(data):
 
 
 # FIXME: in v8, get rid of with_dependencies
-def apply_shortcodes(data, registry, site=None, filename=None, raise_exceptions=False, lang=None, with_dependencies=False):
+def apply_shortcodes(data, registry, site=None, filename=None, raise_exceptions=False, lang=None, with_dependencies=False, extra_context={}):
     """Apply Hugo-style shortcodes on data.
 
     {{% name parameters %}} will end up calling the registered "name" function with the given parameters.
@@ -312,6 +312,7 @@ def apply_shortcodes(data, registry, site=None, filename=None, raise_exceptions=
                 kw['site'] = site
                 kw['data'] = data_arg
                 kw['lang'] = lang
+                kw.update(extra_context)
                 if name in registry:
                     f = registry[name]
                     if getattr(f, 'nikola_shortcode_pass_filename', None):

nikola/utils.py

@@ -56,6 +56,10 @@
     from urllib.parse import urlparse, urlunparse  # NOQA
 import warnings
 import PyRSS2Gen as rss
+try:
+    import yaml
+except ImportError:
+    yaml = None
 from collections import defaultdict, Callable, OrderedDict
 from logbook.compat import redirect_logging
 from logbook.more import ExceptionHandler, ColorizedStderrHandler
@@ -81,7 +85,8 @@
            'adjust_name_for_index_path', 'adjust_name_for_index_link',
            'NikolaPygmentsHTML', 'create_redirect', 'TreeNode',
            'flatten_tree_structure', 'parse_escaped_hierarchical_category_name',
-           'join_hierarchical_category_path', 'clean_before_deployment', 'indent')
+           'join_hierarchical_category_path', 'clean_before_deployment', 'indent',
+           'load_data')
 
 # Are you looking for 'generic_rss_renderer'?
 # It's defined in nikola.nikola.Nikola (the site object).
@@ -1909,3 +1914,17 @@ def prefixed_lines():
         for line in text.splitlines(True):
             yield (prefix + line if predicate(line) else line)
     return ''.join(prefixed_lines())
+
+
+def load_data(path):
+    """Given path to a file, load data from it."""
+    ext = os.path.splitext(path)[-1]
+    if ext in {'.yml', '.yaml'}:
+        if yaml is None:
+            req_missing(['yaml'], 'use YAML data files')
+            return {}
+        with io.open(path, 'r', encoding='utf8') as inf:
+            return yaml.load(inf)
+    elif ext in {'.json', '.js'}:
+        with io.open(path, 'r', encoding='utf8') as inf:
+            return json.load(inf)