Merge 3613d12 into 608d936

chrisjsewell · Feb 1, 2019 · 178c648 · 178c648
2 parents 608d936 + 3613d12
commit 178c648
Show file tree

Hide file tree

Showing 81 changed files with 1,222 additions and 1,341 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -14,6 +14,7 @@
     },
     "python.pythonPath": "/anaconda/envs/lsr/bin/python",
     "restructuredtext.confPath": "${workspaceFolder}/docs/source",
+    "restructuredtext.builtDocumentationPath": "${workspaceFolder}/docs/build/html",
     "restructuredtext.preview.scrollEditorWithPreview": false,
     "restructuredtext.preview.scrollPreviewWithEditor": false,
     "restructuredtext.updateOnTextChanged": "false",

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,2 +1,3 @@
 recursive-include ipypublish *.json
+recursive-include ipypublish *.j2
 recursive-include ipypublish/tests/test_files *
diff --git a/docs/get_intersphinx_inv.py b/docs/get_intersphinx_inv.py
@@ -21,8 +21,8 @@ def warn(self, msg):
     from sphinx.ext import intersphinx
     import warnings
     # uri = 'http://jinja.pocoo.org/docs/dev/objects.inv'
-    # uri = "http://nbconvert.readthedocs.io/en/latest/objects.inv"
-    uri = "http://nbformat.readthedocs.io/en/latest/objects.inv"
+    uri = "http://nbconvert.readthedocs.io/en/latest/objects.inv"
+    # uri = "http://nbformat.readthedocs.io/en/latest/objects.inv"
 
     # Read inventory into a dictionary
     inv = fetch_inventory(uri)

diff --git a/docs/source/api/ipypublish.scripts.create_template.rst b/docs/source/api/ipypublish.scripts.create_template.rst
diff --git a/docs/source/api/ipypublish.scripts.rst b/docs/source/api/ipypublish.scripts.rst
@@ -6,7 +6,6 @@ Submodules
 
 .. toctree::
 
-   ipypublish.scripts.create_template
    ipypublish.scripts.ipynb_latex_setup
    ipypublish.scripts.nbmerge
    ipypublish.scripts.pdfexport

diff --git a/docs/source/api/ipypublish.templates.create_template.rst b/docs/source/api/ipypublish.templates.create_template.rst
@@ -0,0 +1,7 @@
+ipypublish\.templates\.create\_template module
+==============================================
+
+.. automodule:: ipypublish.templates.create_template
+    :members:
+    :undoc-members:
+    :show-inheritance:
diff --git a/docs/source/api/ipypublish.templates.outline_schemas.convert_format_str.rst b/docs/source/api/ipypublish.templates.outline_schemas.convert_format_str.rst
@@ -0,0 +1,7 @@
+ipypublish\.templates\.outline\_schemas\.convert\_format\_str module
+====================================================================
+
+.. automodule:: ipypublish.templates.outline_schemas.convert_format_str
+    :members:
+    :undoc-members:
+    :show-inheritance:
diff --git a/docs/source/api/ipypublish.templates.outline_schemas.rst b/docs/source/api/ipypublish.templates.outline_schemas.rst
@@ -1,6 +1,13 @@
 ipypublish\.templates\.outline\_schemas package
 ===============================================
 
+Submodules
+----------
+
+.. toctree::
+
+   ipypublish.templates.outline_schemas.convert_format_str
+
 Module contents
 ---------------
 

diff --git a/docs/source/api/ipypublish.templates.rst b/docs/source/api/ipypublish.templates.rst
@@ -9,6 +9,13 @@ Subpackages
     ipypublish.templates.outline_schemas
     ipypublish.templates.segments
 
+Submodules
+----------
+
+.. toctree::
+
+   ipypublish.templates.create_template
+
 Module contents
 ---------------
 

diff --git a/docs/source/custom_export_config.rst b/docs/source/custom_export_config.rst
@@ -73,7 +73,7 @@ The configuration file is a JSON file, with a validation schema given in
         "template": {
             "outline": {
                 "module": "ipypublish.templates.outline_schemas",
-                "file": "latex_tplx_schema.json"
+                "file": "latex_outline.latex.j2"
             },
             "segments": [
                 {
@@ -125,24 +125,26 @@ Template Construction
 ~~~~~~~~~~~~~~~~~~~~~
 
 In line 22, we define how to construct the `jinja`_ template.
-The ``outline`` key defines the path to an outline schema,
+The ``outline`` key defines the path to an outline template,
 such as in :ref:`outline_schema`.
-This file achieves two things; to define an outline of the `jinja`_ template
-structural blocks,
-with placeholders to be replaced by :py:func:`str.format`, and to
-provide a schema for segment files which are used to replace
-one or more of the placeholders.
+
+This template file can be a full jinja template file, extending
+an existing nbconvert template, but may optionally contain 'placeholders'
+(of the form ``@ipubreplace{below}{key_name}``)
+that can be replaced by injecting zero or more segments into them.
+The first option states whether segment injections are appended above or below
+previous injections, and the second option defines the key for that segment.
 
 This approach allows independent aspects of the document to be stored
 separately then pieced together in the desired manner. For example,
-the segment defined in :ref:`segment_config` defines only parts of the document
-which define how the bibliography is constructed.
+the segment file in :ref:`segment_config` defines only parts of the document
+which control how the bibliography is constructed.
 This could be removed or replaced by a custom export configuration.
 Similarly, input and output prompts can be added/removed in html documents.
 
-Segments are applied in the order they are defined,
-and the outline schema defines whether they are appended
-above or below existing content. For example, these segments:
+Segments are applied in the order they are defined, and appended
+above or below existing content, as defined by the placeholder.
+For example, these segments:
 
 .. code-block:: JSON
 
@@ -158,15 +160,27 @@ above or below existing content. For example, these segments:
         }
     ]
 
+applied to this template outline:
+
+.. code-block:: html+jinja
+
+    {% block markdowncell scoped %}
+    @ipubreplace{above}{notebook_input_markdown_pre}
+    @ipubreplace{below}{notebook_input_markdown}
+    @ipubreplace{below}{notebook_input_markdown_post}
+    {% endblock markdowncell %}
+
 will result in a template containing:
 
-.. code-block:: html
+.. code-block:: html+jinja
 
-   <div class='outer'>
-   <div class='inner'>
-     test
-   </div>
-   </div>
+    {% block markdowncell scoped %}
+    <div class='outer'>
+    <div class='inner'>
+        test
+    </div>
+    </div>
+    {% endblock markdowncell %}
 
 
 Segment configuration files also have an optional ``overwrite`` key, which

diff --git a/docs/source/outline_schema.rst b/docs/source/outline_schema.rst
@@ -1,7 +1,7 @@
 .. _outline_schema:
 
-Template Outline Configuration Example
---------------------------------------
+Template Outline Example
+------------------------
 
-.. literalinclude:: ../../ipypublish/templates/outline_schemas/latex_tplx_schema.json
-  :language: JSON
+.. literalinclude:: ../../ipypublish/templates/outline_schemas/html_outline.html.j2
+  :language: html+jinja
diff --git a/ipypublish/__init__.py b/ipypublish/__init__.py
@@ -1 +1 @@
-__version__ = '0.7.1'
+__version__ = '0.8.0'
diff --git a/ipypublish/convert/config_manager.py b/ipypublish/convert/config_manager.py
@@ -1,72 +1,22 @@
 import os
-import inspect
 import glob
-import json
 import importlib
 import logging
 
 from six import string_types
 from jinja2 import DictLoader
 import jsonschema
 
-from ipypublish.utils import pathlib
+from ipypublish.utils import (pathlib, handle_error, get_module_path,
+                              read_file_from_directory, read_file_from_module)
 from ipypublish import export_plugins
-from ipypublish.scripts.create_template import create_template
+from ipypublish.templates.create_template import create_template
 
 _TEMPLATE_KEY = 'new_template'
 _EXPORT_SCHEMA_FILE = "export_config.schema.json"
 _EXPORT_SCHEMA = None
 
-
-def handle_error(msg, err_type, raise_msg=None, log_msg=None):
-    """handle an error, by logging it, then raising"""
-    if raise_msg is None:
-        raise_msg = msg
-    if log_msg is None:
-        log_msg = msg
-
-    logging.error(log_msg)
-    raise err_type(raise_msg)
-
-
-def read_json_from_directory(dir_path, file_name, jtype):
-    """load a json file situated in a directory"""
-    if isinstance(dir_path, string_types):
-        dir_path = pathlib.Path(dir_path)
-
-    file_path = dir_path.joinpath(file_name)
-
-    if not file_path.exists():
-        handle_error(
-            "the {} does not exist: {}".format(jtype, file_path),
-            IOError)
-
-    with file_path.open() as fobj:
-        try:
-            data = json.load(fobj)
-        except Exception as err:
-            handle_error("failed to read {} ({}): {}".format(
-                jtype, file_path, err), IOError)
-    return data
-
-
-def get_module_path(module):
-    """return a directory path to a module"""
-    return pathlib.Path(os.path.dirname(
-        os.path.abspath(inspect.getfile(module))))
-
-
-def read_json_from_module(module_path, file_name, jtype):
-    """load a json file situated in a python module"""
-    try:
-        outline_module = importlib.import_module(module_path)
-    except ModuleNotFoundError:
-        handle_error(
-            "module {} containing {} {} not found".format(
-                module_path, jtype, file_name), ModuleNotFoundError)
-
-    return read_json_from_directory(get_module_path(outline_module),
-                                    file_name, jtype)
+logger = logging.getLogger("configuration")
 
 
 def get_export_config_path(export_key, config_folder_paths=()):
@@ -99,24 +49,24 @@ def load_export_config(export_config_path):
     if isinstance(export_config_path, string_types):
         export_config_path = pathlib.Path(export_config_path)
 
-    data = read_json_from_directory(
+    data = read_file_from_directory(
         export_config_path.parent, export_config_path.name,
-        "export configuration")
+        "export configuration", logger, as_json=True)
 
     # validate against schema
     global _EXPORT_SCHEMA
     if _EXPORT_SCHEMA is None:
         # lazy load schema once
-        _EXPORT_SCHEMA = read_json_from_directory(
+        _EXPORT_SCHEMA = read_file_from_directory(
             os.path.dirname(os.path.realpath(__file__)), _EXPORT_SCHEMA_FILE,
-            "export configuration schema")
+            "export configuration schema", logger, as_json=True)
     try:
         jsonschema.validate(data, _EXPORT_SCHEMA)
     except jsonschema.ValidationError as err:
         handle_error(
             "validation of export config {} failed against {}: {}".format(
                 export_config_path, _EXPORT_SCHEMA_FILE, err.message),
-            jsonschema.ValidationError)
+            jsonschema.ValidationError, logger=logger)
 
     return data
 
@@ -144,13 +94,13 @@ def create_exporter_cls(class_str):
     except ModuleNotFoundError:
         handle_error(
             "module {} containing exporter class {} not found".format(
-                module_path, class_name), ModuleNotFoundError)
+                module_path, class_name), ModuleNotFoundError, logger=logger)
     if hasattr(export_module, class_name):
         export_class = getattr(export_module, class_name)
     else:
         handle_error(
             "module {} does not contain class {}".format(
-                module_path, class_name), ImportError)
+                module_path, class_name), ImportError, logger=logger)
 
     # if a template is used we need to override the default template
     class BespokeTemplateExporter(export_class):
@@ -176,27 +126,43 @@ def load_template(template_dict):
         return None
 
     if "directory" in template_dict["outline"]:
-        outline_schema = read_json_from_directory(
+        outline_template = read_file_from_directory(
             template_dict["outline"]["directory"],
-            template_dict["outline"]["file"], "template outline")
+            template_dict["outline"]["file"],
+            "template outline", logger, as_json=False)
+        outline_name = os.path.join(template_dict["outline"]["directory"],
+                                    template_dict["outline"]["file"])
     else:
-        outline_schema = read_json_from_module(
+        outline_template = read_file_from_module(
             template_dict["outline"]["module"],
-            template_dict["outline"]["file"], "template outline")
+            template_dict["outline"]["file"],
+            "template outline", logger, as_json=False)
+        outline_name = os.path.join(template_dict["outline"]["module"],
+                                    template_dict["outline"]["file"])
+
     segments = []
-    for segment in template_dict["segments"]:
+    for snum, segment in enumerate(template_dict.get("segments", [])):
+
+        if "file" not in segment:
+            handle_error(
+                "'file' expected in segment {}".format(snum),
+                KeyError, logger)
 
         if "directory" in segment:
-            seg_data = read_json_from_directory(
+            seg_data = read_file_from_directory(
                 segment["directory"],
-                segment["file"], "template segment")
-        else:
-            seg_data = read_json_from_module(
+                segment["file"], "template segment", logger, as_json=True)
+        elif "module" in segment:
+            seg_data = read_file_from_module(
                 segment["module"],
-                segment["file"], "template segment")
+                segment["file"], "template segment", logger, as_json=True)
+        else:
+            handle_error(
+                "'directory' or 'module' expected in segment {}".format(snum),
+                KeyError, logger)
 
         segments.append(seg_data)
 
-    template_str = create_template(outline_schema, segments)
+    template_str = create_template(outline_template, outline_name, segments)
 
     return str_to_jinja(template_str)
diff --git a/ipypublish/convert/export_config.schema.json b/ipypublish/convert/export_config.schema.json
@@ -79,8 +79,7 @@
             ],
             "additionalProperties": false,
             "required": [
-                "outline",
-                "segments"
+                "outline"
             ],
             "properties": {
                 "outline": {