Skip to content
Permalink
Browse files

Merge pull request #2462 from getnikola/add-data-metadata

Add data metadata (Fix #2450)
  • Loading branch information
ralsina committed Aug 21, 2016
2 parents 39a1ae9 + ee78400 commit c12a6a1281a7b29b8c8098e170eedbc7a2bb1d4f
@@ -4,6 +4,7 @@ New in master
Features
--------

* New ``data`` metadata that loads data from external files (Issue #2450)
* Shortcode to escape to the template language (Issue #1227)
* Added link to raw file in listings (Issue #1995)
* New NO_DOCUTILS_TITLE_TRANSFORM (Issue #2382)
@@ -363,6 +363,16 @@ to your configuration:
enclosure
Add an enclosure to this post when it's used in RSS. See `more information about enclosures <http://en.wikipedia.org/wiki/RSS_enclosure>`__

data
Path to an external data file, relative to ``conf.py``. Based on it's extension it will
be interpreted as JSON or YAML and be available as ``post.data``. The data **must**
decode into a python dictionary, and it's keys are available for templates as ``post.data('key')``.

Translated posts can have different values for this field, and the correct one will be
used.

This is specially useful used in combination with `shortcodes.`__

.. note:: The Two-File Format

Nikola originally used a separate ``.meta`` file. That will still work!
@@ -910,6 +920,10 @@ will create a shortcode called ``mycode`` you can use. Any options you pass to
the shortcode will be available as variables for that template. Non-keyword
options will be passed in a tuple varaible named ``_args``.

The post in which the shortcode is being used is available as the ``post``
variable, so you can access the title as ``post.title``, and data loaded
via the ``data`` field in the metadata using ``post.data(key)``.

If you use the shortcode as paired, then the contents between the paired tags
will be available in the ``data`` variable. If you want to access the Nikola
object, it will be available as ``site``. Use with care :-)
@@ -938,8 +952,9 @@ Finally, you can use a template shortcode without a file, by inserting the templ
{{% /template %}}
{{% /raw %}}

In that case, the template engine used will be your theme's and the arguments you pass, as well as the global
context from your ``conf.py``, are available to the template you are creating.
In that case, the template engine used will be your theme's and the arguments you pass,
as well as the global context from your ``conf.py``, are available to the template you
are creating.

Redirections
------------
@@ -1499,11 +1499,11 @@ def register_shortcode(self, name, f):
self.shortcode_registry[name] = f

# XXX in v8, get rid of with_dependencies
def apply_shortcodes(self, data, filename=None, lang=None, with_dependencies=False):
def apply_shortcodes(self, data, filename=None, lang=None, with_dependencies=False, extra_context={}):
"""Apply shortcodes from the registry on data."""
if lang is None:
lang = utils.LocaleBorg().current_lang
return shortcodes.apply_shortcodes(data, self.shortcode_registry, self, filename, lang=lang, with_dependencies=with_dependencies)
return shortcodes.apply_shortcodes(data, self.shortcode_registry, self, filename, lang=lang, with_dependencies=with_dependencies, extra_context=extra_context)

def generic_rss_renderer(self, lang, title, link, description, timeline, output_path,
rss_teasers, rss_plain, feed_length=10, feed_url=None,
@@ -44,16 +44,18 @@ class CompileHtml(PageCompiler):
def compile_html(self, source, dest, is_two_file=True):
"""Compile source file into HTML and save as dest."""
makedirs(os.path.dirname(dest))
try:
post = self.site.post_per_input_file[source]
except KeyError:
post = None
with io.open(dest, "w+", encoding="utf8") as out_file:
with io.open(source, "r", encoding="utf8") as in_file:
data = in_file.read()
if not is_two_file:
_, data = self.split_metadata(data)
data, shortcode_deps = self.site.apply_shortcodes(source, with_dependencies=True)
data, shortcode_deps = self.site.apply_shortcodes(source, with_dependencies=True, extra_context=dict(post=post))
out_file.write(data)
try:
post = self.site.post_per_input_file[source]
except KeyError:
if post is None:
if shortcode_deps:
self.logger.error(
"Cannot save dependencies for post {0} due to unregistered source file name",
@@ -91,13 +91,15 @@ def compile_html_string(self, source, is_two_file=True):
def compile_html(self, source, dest, is_two_file=True):
"""Compile source file into HTML and save as dest."""
makedirs(os.path.dirname(dest))
with io.open(dest, "w+", encoding="utf8") as out_file:
output = self.compile_html_string(source, is_two_file)
output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True)
out_file.write(output)
try:
post = self.site.post_per_input_file[source]
except KeyError:
post = None
with io.open(dest, "w+", encoding="utf8") as out_file:
output = self.compile_html_string(source, is_two_file)
output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post))
out_file.write(output)
if post is None:
if shortcode_deps:
self.logger.error(
"Cannot save dependencies for post {0} due to unregistered source file name",
@@ -69,17 +69,19 @@ def compile_html(self, source, dest, is_two_file=True):
req_missing(['markdown'], 'build this site (compile Markdown)')
makedirs(os.path.dirname(dest))
self.extensions += self.site.config.get("MARKDOWN_EXTENSIONS")
try:
post = self.site.post_per_input_file[source]
except KeyError:
post = None
with io.open(dest, "w+", encoding="utf8") as out_file:
with io.open(source, "r", encoding="utf8") as in_file:
data = in_file.read()
if not is_two_file:
_, data = self.split_metadata(data)
output = markdown(data, self.extensions)
output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True)
output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post))
out_file.write(output)
try:
post = self.site.post_per_input_file[source]
except KeyError:
if post is None:
if shortcode_deps:
self.logger.error(
"Cannot save dependencies for post {0} due to unregistered source file name",
@@ -54,14 +54,16 @@ def compile_html(self, source, dest, is_two_file=True):
"""Compile source file into HTML and save as dest."""
makedirs(os.path.dirname(dest))
try:
try:
post = self.site.post_per_input_file[source]
except KeyError:
post = None
subprocess.check_call(['pandoc', '-o', dest, source] + self.site.config['PANDOC_OPTIONS'])
with open(dest, 'r', encoding='utf-8') as inf:
output, shortcode_deps = self.site.apply_shortcodes(inf.read(), with_dependencies=True)
with open(dest, 'w', encoding='utf-8') as outf:
outf.write(output)
try:
post = self.site.post_per_input_file[source]
except KeyError:
if post is None:
if shortcode_deps:
self.logger.error(
"Cannot save dependencies for post {0} due to unregistered source file name",
@@ -95,14 +95,16 @@ def compile_html(self, source, dest, is_two_file=True):
makedirs(os.path.dirname(dest))
error_level = 100
with io.open(dest, "w+", encoding="utf8") as out_file:
try:
post = self.site.post_per_input_file[source]
except KeyError:
post = None
with io.open(source, "r", encoding="utf8") as in_file:
data = in_file.read()
output, error_level, deps = self.compile_html_string(data, source, is_two_file)
output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True)
output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post))
out_file.write(output)
try:
post = self.site.post_per_input_file[source]
except KeyError:
if post is None:
if deps.list:
self.logger.error(
"Cannot save dependencies for post {0} due to unregistered source file name",
@@ -161,6 +161,12 @@ def __init__(
for lang in sorted(self.translated_to):
default_metadata.update(self.meta[lang])

# Load data field from metadata
self.data = Functionary(lambda: None, self.default_lang)
for lang in self.translations:
if self.meta[lang].get('data') is not None:
self.data[lang] = utils.load_data(self.meta[lang]['data'])

if 'date' not in default_metadata and not use_in_feeds:
# For stories we don't *really* need a date
if self.config['__invariant__']:
@@ -475,6 +481,8 @@ def deps(self, lang):
cand_3 = get_translation_candidate(self.config, self.metadata_path, lang)
if os.path.exists(cand_3):
deps.append(cand_3)
if self.meta('data', lang):
deps.append(self.meta('data', lang))
deps += self._get_dependencies(self._dependency_file_page[lang])
deps += self._get_dependencies(self._dependency_file_page[None])
return sorted(deps)
@@ -257,7 +257,7 @@ def _split_shortcodes(data):


# FIXME: in v8, get rid of with_dependencies
def apply_shortcodes(data, registry, site=None, filename=None, raise_exceptions=False, lang=None, with_dependencies=False):
def apply_shortcodes(data, registry, site=None, filename=None, raise_exceptions=False, lang=None, with_dependencies=False, extra_context={}):
"""Apply Hugo-style shortcodes on data.
{{% name parameters %}} will end up calling the registered "name" function with the given parameters.
@@ -312,6 +312,7 @@ def apply_shortcodes(data, registry, site=None, filename=None, raise_exceptions=
kw['site'] = site
kw['data'] = data_arg
kw['lang'] = lang
kw.update(extra_context)
if name in registry:
f = registry[name]
if getattr(f, 'nikola_shortcode_pass_filename', None):
@@ -56,6 +56,10 @@
from urllib.parse import urlparse, urlunparse # NOQA
import warnings
import PyRSS2Gen as rss
try:
import yaml
except ImportError:
yaml = None
from collections import defaultdict, Callable, OrderedDict
from logbook.compat import redirect_logging
from logbook.more import ExceptionHandler, ColorizedStderrHandler
@@ -81,7 +85,8 @@
'adjust_name_for_index_path', 'adjust_name_for_index_link',
'NikolaPygmentsHTML', 'create_redirect', 'TreeNode',
'flatten_tree_structure', 'parse_escaped_hierarchical_category_name',
'join_hierarchical_category_path', 'clean_before_deployment', 'indent')
'join_hierarchical_category_path', 'clean_before_deployment', 'indent',
'load_data')

# Are you looking for 'generic_rss_renderer'?
# It's defined in nikola.nikola.Nikola (the site object).
@@ -1909,3 +1914,17 @@ def prefixed_lines():
for line in text.splitlines(True):
yield (prefix + line if predicate(line) else line)
return ''.join(prefixed_lines())


def load_data(path):
"""Given path to a file, load data from it."""
ext = os.path.splitext(path)[-1]
if ext in {'.yml', '.yaml'}:
if yaml is None:
req_missing(['yaml'], 'use YAML data files')
return {}
with io.open(path, 'r', encoding='utf8') as inf:
return yaml.load(inf)
elif ext in {'.json', '.js'}:
with io.open(path, 'r', encoding='utf8') as inf:
return json.load(inf)

0 comments on commit c12a6a1

Please sign in to comment.
You can’t perform that action at this time.