From a181842ee0d9e5ca9faad282189be001d3806388 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 1 Feb 2019 06:12:32 +0000 Subject: [PATCH 1/5] extracted templates into separate files --- MANIFEST.in | 1 + ipypublish/convert/config_manager.py | 88 ++----- ipypublish/convert/main.py | 17 +- ipypublish/scripts/create_template.py | 97 +++++--- .../outline_schemas/convert_format_str.py | 49 ++++ .../outline_schemas/html_outline.html.j2 | 187 +++++++++++++++ .../outline_schemas/html_tpl_schema.json | 198 +--------------- .../outline_schemas/latex_outline.latex.j2 | 154 +++++++++++++ .../outline_schemas/latex_tplx_schema.json | 161 +------------ .../outline_schemas/python_outline.py.j2 | 16 ++ .../outline_schemas/python_tpl_schema.json | 37 +-- .../outline_schemas/sphinx_outline.rst.j2 | 214 ++++++++++++++++++ ipypublish/utils.py | 75 ++++++ 13 files changed, 805 insertions(+), 489 deletions(-) create mode 100644 ipypublish/templates/outline_schemas/convert_format_str.py create mode 100644 ipypublish/templates/outline_schemas/html_outline.html.j2 create mode 100644 ipypublish/templates/outline_schemas/latex_outline.latex.j2 create mode 100644 ipypublish/templates/outline_schemas/python_outline.py.j2 create mode 100644 ipypublish/templates/outline_schemas/sphinx_outline.rst.j2 diff --git a/MANIFEST.in b/MANIFEST.in index 4af877a..b4d5013 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ recursive-include ipypublish *.json +recursive-include ipypublish *.j2 recursive-include ipypublish/tests/test_files * \ No newline at end of file diff --git a/ipypublish/convert/config_manager.py b/ipypublish/convert/config_manager.py index 3da0e55..447df2f 100644 --- a/ipypublish/convert/config_manager.py +++ b/ipypublish/convert/config_manager.py @@ -1,7 +1,5 @@ import os -import inspect import glob -import json import importlib import logging @@ -9,7 +7,8 @@ from jinja2 import DictLoader import jsonschema -from ipypublish.utils import pathlib +from ipypublish.utils import (pathlib, handle_error, get_module_path, + read_file_from_directory, read_file_from_module) from ipypublish import export_plugins from ipypublish.scripts.create_template import create_template @@ -17,56 +16,7 @@ _EXPORT_SCHEMA_FILE = "export_config.schema.json" _EXPORT_SCHEMA = None - -def handle_error(msg, err_type, raise_msg=None, log_msg=None): - """handle an error, by logging it, then raising""" - if raise_msg is None: - raise_msg = msg - if log_msg is None: - log_msg = msg - - logging.error(log_msg) - raise err_type(raise_msg) - - -def read_json_from_directory(dir_path, file_name, jtype): - """load a json file situated in a directory""" - if isinstance(dir_path, string_types): - dir_path = pathlib.Path(dir_path) - - file_path = dir_path.joinpath(file_name) - - if not file_path.exists(): - handle_error( - "the {} does not exist: {}".format(jtype, file_path), - IOError) - - with file_path.open() as fobj: - try: - data = json.load(fobj) - except Exception as err: - handle_error("failed to read {} ({}): {}".format( - jtype, file_path, err), IOError) - return data - - -def get_module_path(module): - """return a directory path to a module""" - return pathlib.Path(os.path.dirname( - os.path.abspath(inspect.getfile(module)))) - - -def read_json_from_module(module_path, file_name, jtype): - """load a json file situated in a python module""" - try: - outline_module = importlib.import_module(module_path) - except ModuleNotFoundError: - handle_error( - "module {} containing {} {} not found".format( - module_path, jtype, file_name), ModuleNotFoundError) - - return read_json_from_directory(get_module_path(outline_module), - file_name, jtype) +logger = logging.getLogger(__name__) def get_export_config_path(export_key, config_folder_paths=()): @@ -99,24 +49,24 @@ def load_export_config(export_config_path): if isinstance(export_config_path, string_types): export_config_path = pathlib.Path(export_config_path) - data = read_json_from_directory( + data = read_file_from_directory( export_config_path.parent, export_config_path.name, - "export configuration") + "export configuration", logger, as_json=True) # validate against schema global _EXPORT_SCHEMA if _EXPORT_SCHEMA is None: # lazy load schema once - _EXPORT_SCHEMA = read_json_from_directory( + _EXPORT_SCHEMA = read_file_from_directory( os.path.dirname(os.path.realpath(__file__)), _EXPORT_SCHEMA_FILE, - "export configuration schema") + "export configuration schema", logger, as_json=True) try: jsonschema.validate(data, _EXPORT_SCHEMA) except jsonschema.ValidationError as err: handle_error( "validation of export config {} failed against {}: {}".format( export_config_path, _EXPORT_SCHEMA_FILE, err.message), - jsonschema.ValidationError) + jsonschema.ValidationError, logger=logger) return data @@ -144,13 +94,13 @@ def create_exporter_cls(class_str): except ModuleNotFoundError: handle_error( "module {} containing exporter class {} not found".format( - module_path, class_name), ModuleNotFoundError) + module_path, class_name), ModuleNotFoundError, logger=logger) if hasattr(export_module, class_name): export_class = getattr(export_module, class_name) else: handle_error( "module {} does not contain class {}".format( - module_path, class_name), ImportError) + module_path, class_name), ImportError, logger=logger) # if a template is used we need to override the default template class BespokeTemplateExporter(export_class): @@ -176,24 +126,26 @@ def load_template(template_dict): return None if "directory" in template_dict["outline"]: - outline_schema = read_json_from_directory( + outline_schema = read_file_from_directory( template_dict["outline"]["directory"], - template_dict["outline"]["file"], "template outline") + template_dict["outline"]["file"], + "template outline", logger, as_json=True) else: - outline_schema = read_json_from_module( + outline_schema = read_file_from_module( template_dict["outline"]["module"], - template_dict["outline"]["file"], "template outline") + template_dict["outline"]["file"], + "template outline", logger, as_json=True) segments = [] for segment in template_dict["segments"]: if "directory" in segment: - seg_data = read_json_from_directory( + seg_data = read_file_from_directory( segment["directory"], - segment["file"], "template segment") + segment["file"], "template segment", logger, as_json=True) else: - seg_data = read_json_from_module( + seg_data = read_file_from_module( segment["module"], - segment["file"], "template segment") + segment["file"], "template segment", logger, as_json=True) segments.append(seg_data) diff --git a/ipypublish/convert/main.py b/ipypublish/convert/main.py index 6ad336a..b592b2e 100755 --- a/ipypublish/convert/main.py +++ b/ipypublish/convert/main.py @@ -13,7 +13,7 @@ import ipypublish -from ipypublish.utils import pathlib +from ipypublish.utils import pathlib, handle_error from ipypublish.scripts.nbmerge import merge_notebooks from ipypublish.convert.config_manager import (get_export_config_path, load_export_config, @@ -21,16 +21,7 @@ create_exporter_cls) from ipypublish.scripts.pdfexport import export_pdf - -def handle_error(msg, err_type, raise_msg=None, log_msg=None): - """handle an error, by logging it, then raising""" - if raise_msg is None: - raise_msg = msg - if log_msg is None: - log_msg = msg - - logging.error(log_msg) - raise err_type(raise_msg) +logger = logging.getLogger(__name__) def publish(ipynb_path, @@ -124,7 +115,7 @@ def publish(ipynb_path, if export_config_path is None: handle_error( "could not find conversion configuration: {}".format(conversion), - IOError) + IOError, logger) # read conversion configuration and create logging.info('loading conversion configuration') @@ -168,7 +159,7 @@ def publish(ipynb_path, html_viewer=True, debug_mode=pdf_debug): handle_error('pdf export failed, try running with pdf_debug=True', - RuntimeError) + RuntimeError, logger) logging.info('process finished successfully') diff --git a/ipypublish/scripts/create_template.py b/ipypublish/scripts/create_template.py index c7878cf..4ad5857 100644 --- a/ipypublish/scripts/create_template.py +++ b/ipypublish/scripts/create_template.py @@ -9,20 +9,44 @@ """ from typing import List, Tuple, Union # noqa: F401 +import re import logging import jsonschema + # from ipypublish import __version__ +from ipypublish.utils import (read_file_from_module, + read_file_from_directory, + handle_error) + +logger = logging.getLogger(__name__) + + +def multireplace(string, replacements): + """ + Given a string and a replacement map, it returns the replaced string. + + From https://gist.github.com/bgusach/a967e0587d6e01e889fd1d776c5f3729 + :param str string: string to execute replacements on + :param dict replacements: replacement dictionary {value to find: value to replace} + :rtype: str -def handle_error(msg, err_type, raise_msg=None, log_msg=None): - """handle an error, by logging it, then raising""" - if raise_msg is None: - raise_msg = msg - if log_msg is None: - log_msg = msg + """ + if not replacements: + return string + + # Place longer ones first to keep shorter substrings from matching, + # where the longer ones should take place + # For instance given the replacements {'ab': 'AB', 'abc': 'ABC'} + # against the string 'hey abc', it should produce + # 'hey ABC' and not 'hey ABc' + substrs = sorted(replacements, key=len, reverse=True) + + # Create a big OR regex that matches any of the substrings to replace + regexp = re.compile('|'.join(map(re.escape, substrs))) - logging.error(log_msg) - raise err_type(raise_msg) + # For each match, look up the new string in the replacements + return regexp.sub(lambda match: replacements[match.group(0)], string) def create_template(outline_schema, segment_datas, outpath=None): @@ -46,9 +70,21 @@ def create_template(outline_schema, segment_datas, outpath=None): """ # TODO validate outline schema - # get outline info - outline_content = "\n".join(outline_schema.get("outline")) - placeholders = outline_schema["properties"]["segments"]["properties"] + + # get jinja template + if "directory" in outline_schema["outline"]: + outline_content = read_file_from_directory( + outline_schema["outline"]["directory"], + outline_schema["outline"]["file"], "template outline", logger) + else: + outline_content = read_file_from_module( + outline_schema["outline"]["module"], + outline_schema["outline"]["file"], "template outline", logger) + + # get the placeholders @ipubreplace{above|below}{name} + regex = re.compile("@ipubreplace{(.+)}{(.+)}", re.MULTILINE) + placeholder_tuple = regex.findall(outline_content) + placeholders = {name: append for append, name in placeholder_tuple} replacements = {key: "" for key in placeholders.keys()} docstrings = [ @@ -66,7 +102,7 @@ def create_template(outline_schema, segment_datas, outpath=None): handle_error( "validation of template segment {} failed: {}".format( seg_num, err.message), - jsonschema.ValidationError) + jsonschema.ValidationError, logger=logger) # get description of segment docstrings.append( @@ -76,30 +112,37 @@ def create_template(outline_schema, segment_datas, outpath=None): # find what key to overwrite overwrite = segment_data.get("overwrite", []) - logging.debug('overwrite keys: {}'.format(overwrite)) + logger.debug('overwrite keys: {}'.format(overwrite)) for key, val in segment_data.get("segments").items(): valstring = "\n".join(val) if key in overwrite: replacements[key] = valstring - elif 'append_before' in placeholders[key]["$ref"]: + elif placeholders[key] == "above": replacements[key] = valstring + '\n' + replacements[key] - elif 'append_after' in placeholders[key]["$ref"]: + elif placeholders[key] == "below": replacements[key] = replacements[key] + '\n' + valstring else: - # TODO this should be part of the schema + # TODO this should be part of the schema? handle_error(( - "properties/segments/properties/{0}/$ref ".format(key) + - "should contain either append_before or append_before"), - jsonschema.ValidationError) - - replacements["meta_docstring"] = "\n".join(docstrings).replace("'", '"') - # TODO add option to include ipypub version in output file - # not included by default, - # since it would require the test files to be updated with every version - replacements["ipypub_version"] = "" # str(__version__) - - outline = outline_content.format(**replacements) + "the placeholder @ipubreplace{{{0}}}{{{1}}} ".format( + key, placeholders[key]) + + "should specify above or below appending"), + jsonschema.ValidationError, logger=logger) + + if "meta_docstring" in placeholders: + docstring = "\n".join(docstrings).replace("'", '"') + replacements["meta_docstring"] = docstring + if "ipypub_version" in placeholders: + # TODO add option to include ipypub version in output file + # not included by default, + # since tests need to be changed to ignore version number + replacements["ipypub_version"] = "" # str(__version__) + + replace_dict = { + "@ipubreplace{{{0}}}{{{1}}}".format(append, name): replacements[name] + for append, name in placeholder_tuple} + outline = multireplace(outline_content, replace_dict) if outpath is not None: with open(outpath, 'w') as f: # TODO use pathlib diff --git a/ipypublish/templates/outline_schemas/convert_format_str.py b/ipypublish/templates/outline_schemas/convert_format_str.py new file mode 100644 index 0000000..6927ea6 --- /dev/null +++ b/ipypublish/templates/outline_schemas/convert_format_str.py @@ -0,0 +1,49 @@ +import string + + +class DefaultFormatter(string.Formatter): + + def get_value(self, key, args, kwargs): + # Handle a key not found + try: + val = super(DefaultFormatter, self).get_value( + key, args, kwargs) + # Python 3, 'super().get_field(field_name, args, kwargs)' works + except (IndexError, KeyError): + if str(key).endswith("_pre"): + val = "@ipubreplace{above}{" + str(key) + "}" + else: + val = "@ipubreplace{below}{" + str(key) + "}" + return val + + +def convert_format_str(template): + + temp_str = "\n".join(template) + fmt = DefaultFormatter() + outstr = fmt.format(temp_str) + + return outstr + + +if __name__ == "__main__": + + template = ["{{%- extends 'null.tpl' -%}}", + "{{% block header %}}", + "{{{{ nb.metadata | meta2yaml('#~~ ') }}}}", + "{{% endblock header %}}", + "{{% block codecell %}}", + "#%%", + "{{{{ super() }}}}", + "{{% endblock codecell %}}", + "{{% block in_prompt %}}{{% endblock in_prompt %}}", + "{{% block input %}}{{{{ cell.metadata | meta2yaml('#~~ ') }}}}", + "{{{{ cell.source | ipython2python }}}}", + "{{% endblock input %}}", + "{{% block markdowncell scoped %}}#%% [markdown]", + "{{{{ cell.metadata | meta2yaml('#~~ ') }}}}", + "{{{{ cell.source | comment_lines }}}}", + "{{% endblock markdowncell %}}" + ] + + print(convert_format_str(template)) diff --git a/ipypublish/templates/outline_schemas/html_outline.html.j2 b/ipypublish/templates/outline_schemas/html_outline.html.j2 new file mode 100644 index 0000000..e04c0f6 --- /dev/null +++ b/ipypublish/templates/outline_schemas/html_outline.html.j2 @@ -0,0 +1,187 @@ + + + +{%- extends 'display_priority.tpl' -%} + +@ipubreplace{below}{globals} + +%% HTML Setup +%% ==================== + +{%- block header %} + + + + +{%- block html_head -%} + @ipubreplace{below}{html_header} +{%- endblock html_head -%} + +{%- endblock header -%} + +{% block body %} + + @ipubreplace{below}{html_body_start} + {{ super() }} + @ipubreplace{below}{html_body_end} + +{%- endblock body %} + +{%- block footer %} + @ipubreplace{below}{html_footer} + +{%- endblock footer-%} + +%% Notebook Input +%% ============== + +{%- block any_cell scoped %} +@ipubreplace{below}{notebook_all} +{% endblock any_cell %} + +{% block input_group -%} +@ipubreplace{above}{notebook_input_code_pre} +{{ super() }} +@ipubreplace{below}{notebook_input_code_post} +{% endblock input_group %} + +{% block in_prompt -%} +@ipubreplace{below}{notebook_input_code_prompt} +{%- endblock in_prompt %} + +{% block input scoped %} +@ipubreplace{below}{notebook_input_code} +{% endblock input %} + +{% block rawcell scoped %} +@ipubreplace{above}{notebook_input_raw_pre} +@ipubreplace{below}{notebook_input_raw} +@ipubreplace{below}{notebook_input_raw_post} +{% endblock rawcell %} + +{% block markdowncell scoped %} +@ipubreplace{above}{notebook_input_markdown_pre} +@ipubreplace{below}{notebook_input_markdown} +@ipubreplace{below}{notebook_input_markdown_post} +{% endblock markdowncell %} + +{% block unknowncell scoped %} +@ipubreplace{above}{notebook_input_unknown_pre} +@ipubreplace{below}{notebook_input_unknown} +@ipubreplace{below}{notebook_input_unknown_post} +{% endblock unknowncell %} + +%% Notebook Outbook +%% ================ + +{% block output %} +@ipubreplace{above}{notebook_output_pre} +@ipubreplace{below}{notebook_output_prompt} +{{ super() }} +@ipubreplace{below}{notebook_output_post} +{% endblock output %} + +% Redirect execute_result to display data priority. +{%- block execute_result scoped %} + {%- set extra_class="output_execute_result" -%} + {% block data_priority scoped %} +@ipubreplace{below}{notebook_output} + {% endblock %} + {%- set extra_class="" -%} +{% endblock execute_result %} + +{% block error %} +@ipubreplace{above}{notebook_output_error_pre} +@ipubreplace{below}{notebook_output_error} +@ipubreplace{below}{notebook_output_error_post} +{% endblock error %} + +{% block traceback_line %} +@ipubreplace{above}{notebook_output_traceback_pre} +@ipubreplace{below}{notebook_output_traceback} +@ipubreplace{below}{notebook_output_traceback_post} +{% endblock traceback_line %} + +{% block data_text %} +@ipubreplace{above}{notebook_output_text_pre} +@ipubreplace{below}{notebook_output_text} +@ipubreplace{below}{notebook_output_text_post} +{% endblock data_text %} + +{% block data_latex %} +@ipubreplace{above}{notebook_output_latex_pre} +@ipubreplace{below}{notebook_output_latex} +@ipubreplace{below}{notebook_output_latex_post} +{% endblock data_latex %} + + +{% block stream_stdout %} +@ipubreplace{above}{notebook_output_stream_stdout_pre} +@ipubreplace{below}{notebook_output_stream_stdout} +@ipubreplace{below}{notebook_output_stream_stdout_post} +{% endblock stream_stdout %} + +{% block stream_stderr %} +@ipubreplace{above}{notebook_output_stream_stderr_pre} +@ipubreplace{below}{notebook_output_stream_stderr} +@ipubreplace{below}{notebook_output_stream_stderr_post} +{% endblock stream_stderr %} + +{%- block data_markdown -%} +@ipubreplace{above}{notebook_output_markdown_pre} +@ipubreplace{below}{notebook_output_markdown} +@ipubreplace{below}{notebook_output_markdown_post} +{% endblock data_markdown %} + +{%- block data_jpg -%} +@ipubreplace{above}{notebook_output_jpg_pre} +@ipubreplace{below}{notebook_output_jpg} +@ipubreplace{below}{notebook_output_jpg_post} +{%- endblock data_jpg -%} + +{%- block data_png -%} +@ipubreplace{above}{notebook_output_png_pre} +@ipubreplace{below}{notebook_output_png} +@ipubreplace{below}{notebook_output_png_post} +{%- endblock data_png -%} + +{%- block data_svg -%} +@ipubreplace{above}{notebook_output_svg_pre} +@ipubreplace{below}{notebook_output_svg} +@ipubreplace{below}{notebook_output_svg_post} +{%- endblock data_svg -%} + +{%- block data_pdf -%} +@ipubreplace{above}{notebook_output_pdf_pre} +@ipubreplace{below}{notebook_output_pdf} +@ipubreplace{below}{notebook_output_pdf_post} +{%- endblock -%} + +{% block data_html -%} +@ipubreplace{above}{notebook_output_html_pre} +@ipubreplace{below}{notebook_output_html} +@ipubreplace{below}{notebook_output_html_post} +{% endblock data_html%} + +{%- block data_javascript scoped %} +@ipubreplace{above}{notebook_output_javascript_pre} +@ipubreplace{below}{notebook_output_javascript} +@ipubreplace{below}{notebook_output_javascript_post} +{%- endblock -%} + +{%- block data_widget_state scoped %} +@ipubreplace{above}{notebook_output_widget_state_pre} +@ipubreplace{below}{notebook_output_widget_state} +@ipubreplace{below}{notebook_output_widget_state_post} +{%- endblock data_widget_state -%} + +{%- block data_widget_view scoped %} +@ipubreplace{above}{notebook_output_widget_view_pre} +@ipubreplace{below}{notebook_output_widget_view} +@ipubreplace{below}{notebook_output_widget_view_post} +{%- endblock data_widget_view -%} + +%% Jinja Macros +%% ================ + +@ipubreplace{below}{jinja_macros} diff --git a/ipypublish/templates/outline_schemas/html_tpl_schema.json b/ipypublish/templates/outline_schemas/html_tpl_schema.json index 8c7de65..c680fce 100644 --- a/ipypublish/templates/outline_schemas/html_tpl_schema.json +++ b/ipypublish/templates/outline_schemas/html_tpl_schema.json @@ -2,196 +2,10 @@ "$schema": "http://json-schema.org/draft-04/schema", "$id": "html-tpl", "description": "define segments to insert into a jinja template of id html-tpl", - "outline": [ - "", - "", - "", - "", - "{{%- extends 'display_priority.tpl' -%}}", - "", - "{globals}", - "", - "%% HTML Setup", - "%% ====================", - "", - "{{%- block header %}}", - "", - "", - "", - "", - "{{%- block html_head -%}}", - " {html_header}", - "{{%- endblock html_head -%}}", - "", - "{{%- endblock header -%}}", - "", - "{{% block body %}}", - " ", - " {html_body_start}", - " {{{{ super() }}}}", - " {html_body_end}", - " ", - "{{%- endblock body %}}", - "", - "{{%- block footer %}}", - " {html_footer}", - "", - "{{%- endblock footer-%}}", - "", - "%% Notebook Input", - "%% ==============", - "", - "{{%- block any_cell scoped %}}", - "{notebook_all}", - "{{% endblock any_cell %}}", - "", - "{{% block input_group -%}}", - "{notebook_input_code_pre}", - "{{{{ super() }}}}", - "{notebook_input_code_post}", - "{{% endblock input_group %}}", - "", - "{{% block in_prompt -%}}", - "{notebook_input_code_prompt}", - "{{%- endblock in_prompt %}}", - "", - "{{% block input scoped %}}", - "{notebook_input_code}", - "{{% endblock input %}}", - "", - "{{% block rawcell scoped %}}", - "{notebook_input_raw_pre}", - "{notebook_input_raw}", - "{notebook_input_raw_post}", - "{{% endblock rawcell %}}", - "", - "{{% block markdowncell scoped %}}", - "{notebook_input_markdown_pre}", - "{notebook_input_markdown}", - "{notebook_input_markdown_post}", - "{{% endblock markdowncell %}}", - "", - "{{% block unknowncell scoped %}}", - "{notebook_input_unknown_pre}", - "{notebook_input_unknown}", - "{notebook_input_unknown_post}", - "{{% endblock unknowncell %}}", - "", - "%% Notebook Outbook", - "%% ================", - "", - "{{% block output %}}", - "{notebook_output_pre}", - "{notebook_output_prompt}", - "{{{{ super() }}}}", - "{notebook_output_post}", - "{{% endblock output %}}", - "", - "% Redirect execute_result to display data priority.", - "{{%- block execute_result scoped %}}", - " {{%- set extra_class=\"output_execute_result\" -%}}", - " {{% block data_priority scoped %}}", - "{notebook_output}", - " {{% endblock %}}", - " {{%- set extra_class=\"\" -%}}", - "{{% endblock execute_result %}}", - "", - "{{% block error %}}", - "{notebook_output_error_pre}", - "{notebook_output_error}", - "{notebook_output_error_post}", - "{{% endblock error %}}", - "", - "{{% block traceback_line %}}", - "{notebook_output_traceback_pre}", - "{notebook_output_traceback}", - "{notebook_output_traceback_post}", - "{{% endblock traceback_line %}}", - "", - "{{% block data_text %}}", - "{notebook_output_text_pre}", - "{notebook_output_text}", - "{notebook_output_text_post}", - "{{% endblock data_text %}}", - "", - "{{% block data_latex %}}", - "{notebook_output_latex_pre}", - "{notebook_output_latex}", - "{notebook_output_latex_post}", - "{{% endblock data_latex %}}", - "", - "", - "{{% block stream_stdout %}}", - "{notebook_output_stream_stdout_pre}", - "{notebook_output_stream_stdout}", - "{notebook_output_stream_stdout_post}", - "{{% endblock stream_stdout %}}", - "", - "{{% block stream_stderr %}}", - "{notebook_output_stream_stderr_pre}", - "{notebook_output_stream_stderr}", - "{notebook_output_stream_stderr_post}", - "{{% endblock stream_stderr %}}", - "", - "{{%- block data_markdown -%}}", - "{notebook_output_markdown_pre}", - "{notebook_output_markdown}", - "{notebook_output_markdown_post}", - "{{% endblock data_markdown %}}", - "", - "{{%- block data_jpg -%}}", - "{notebook_output_jpg_pre}", - "{notebook_output_jpg}", - "{notebook_output_jpg_post}", - "{{%- endblock data_jpg -%}}", - "", - "{{%- block data_png -%}}", - "{notebook_output_png_pre}", - "{notebook_output_png}", - "{notebook_output_png_post}", - "{{%- endblock data_png -%}}", - "", - "{{%- block data_svg -%}}", - "{notebook_output_svg_pre}", - "{notebook_output_svg}", - "{notebook_output_svg_post}", - "{{%- endblock data_svg -%}}", - "", - "{{%- block data_pdf -%}}", - "{notebook_output_pdf_pre}", - "{notebook_output_pdf}", - "{notebook_output_pdf_post}", - "{{%- endblock -%}}", - "", - "{{% block data_html -%}}", - "{notebook_output_html_pre}", - "{notebook_output_html}", - "{notebook_output_html_post}", - "{{% endblock data_html%}}", - "", - "{{%- block data_javascript scoped %}}", - "{notebook_output_javascript_pre}", - "{notebook_output_javascript}", - "{notebook_output_javascript_post}", - "{{%- endblock -%}}", - "", - "{{%- block data_widget_state scoped %}}", - "{notebook_output_widget_state_pre}", - "{notebook_output_widget_state}", - "{notebook_output_widget_state_post}", - "{{%- endblock data_widget_state -%}}", - "", - "{{%- block data_widget_view scoped %}}", - "{notebook_output_widget_view_pre}", - "{notebook_output_widget_view}", - "{notebook_output_widget_view_post}", - "{{%- endblock data_widget_view -%}}", - "", - "%% Jinja Macros", - "%% ================", - "", - "{jinja_macros}" - ], + "outline": { + "module": "ipypublish.templates.outline_schemas", + "file": "html_outline.html.j2" + }, "type": "object", "additionalProperties": false, "definitions": { @@ -230,7 +44,9 @@ }, "template": { "description": "the template identifier", - "enum": ["html-tpl"] + "enum": [ + "html-tpl" + ] }, "segments": { "additionalProperties": false, diff --git a/ipypublish/templates/outline_schemas/latex_outline.latex.j2 b/ipypublish/templates/outline_schemas/latex_outline.latex.j2 new file mode 100644 index 0000000..56f682e --- /dev/null +++ b/ipypublish/templates/outline_schemas/latex_outline.latex.j2 @@ -0,0 +1,154 @@ +((*- extends 'display_priority.tplx' -*)) + +%% Latex Document Setup +%% ==================== + +((* block header *)) +% A latex document created by ipypublish @ipubreplace{below}{ipypub_version} +((( '@ipubreplace{below}{meta_docstring}' | comment_lines('% ') ))) +% + + ((*- block docclass *)) +%%%%%%%%%%%% DOCCLASS + @ipubreplace{below}{document_docclass} +%%%%%%%%%%%% + ((* endblock docclass -*)) + + ((*- block packages *)) +%%%%%%%%%%%% PACKAGES + @ipubreplace{below}{document_packages} +%%%%%%%%%%%% + ((* endblock packages *)) + + ((*- block definitions *)) + ((* block date *))((* endblock date *)) + ((* block author *))((* endblock author *)) + ((* block title *))((* endblock title *)) + +%%%%%%%%%%%% DEFINITIONS + @ipubreplace{below}{document_definitions} +%%%%%%%%%%%% + + ((* endblock definitions *)) + + ((*- block commands *)) + ((* block margins *)) +%%%%%%%%%%%% MARGINS + @ipubreplace{below}{document_margins} +%%%%%%%%%%%% + ((* endblock margins *)) +%%%%%%%%%%%% COMMANDS + @ipubreplace{below}{document_commands} +%%%%%%%%%%%% + ((* endblock commands *)) + +%%%%%%%%%%%% FINAL HEADER MATERIAL + @ipubreplace{below}{document_header_end} +%%%%%%%%%%%% + +((* endblock header *)) + +((* block body *)) + +\begin{document} + + ((* block predoc *)) + ((* block maketitle *)) + @ipubreplace{below}{document_title} + ((* endblock maketitle *)) + ((* block abstract *)) + @ipubreplace{below}{document_abstract} + ((* endblock abstract *)) + @ipubreplace{below}{document_predoc} + ((* endblock predoc *)) + + ((( super() ))) + + ((* block postdoc *)) + ((* block bibliography *)) + @ipubreplace{below}{document_bibliography} + ((* endblock bibliography *)) + @ipubreplace{below}{document_postdoc} + ((* endblock postdoc *)) + +\end{document} + +((* endblock body *)) + +%% Notebook Input +%% ============== + +((*- block any_cell scoped *)) + @ipubreplace{below}{notebook_input} +((* endblock any_cell *)) + +((* block input scoped *)) + @ipubreplace{below}{notebook_input_code} +((* endblock input *)) + +((* block rawcell scoped *)) + @ipubreplace{below}{notebook_input_raw} +((* endblock rawcell *)) + +((* block markdowncell scoped *)) + @ipubreplace{below}{notebook_input_markdown} +((* endblock markdowncell *)) + +((* block unknowncell scoped *)) + @ipubreplace{below}{notebook_input_unknown} +((* endblock unknowncell *)) + +%% Notebook Output +%% ================ + +% Redirect execute_result to display data priority. +((*- block execute_result scoped *)) + ((* block data_priority scoped *)) + @ipubreplace{below}{notebook_output} + ((* endblock *)) +((* endblock execute_result *)) + +((* block data_text *)) + @ipubreplace{below}{notebook_output_text} +((* endblock data_text *)) + +((* block error *)) + @ipubreplace{below}{notebook_output_error} +((* endblock error *)) + +((* block traceback_line *)) + @ipubreplace{below}{notebook_output_traceback} +((* endblock traceback_line *)) + +((* block stream *)) + @ipubreplace{below}{notebook_output_stream} +((* endblock stream *)) + +((* block data_latex -*)) + @ipubreplace{below}{notebook_output_latex} +((* endblock data_latex *)) + +((*- block data_markdown -*)) + @ipubreplace{below}{notebook_output_markdown} +((* endblock data_markdown *)) + +((*- block data_png -*)) + @ipubreplace{below}{notebook_output_png} +((*- endblock data_png -*)) + +((*- block data_jpg -*)) + @ipubreplace{below}{notebook_output_jpg} +((*- endblock data_jpg -*)) + +((*- block data_svg -*)) + @ipubreplace{below}{notebook_output_svg} +((*- endblock data_svg -*)) + +((*- block data_pdf -*)) + @ipubreplace{below}{notebook_output_pdf} +((*- endblock -*)) + +%% Jinja Macros +%% ================ + +@ipubreplace{below}{jinja_macros} diff --git a/ipypublish/templates/outline_schemas/latex_tplx_schema.json b/ipypublish/templates/outline_schemas/latex_tplx_schema.json index 83b34c1..4a143d6 100644 --- a/ipypublish/templates/outline_schemas/latex_tplx_schema.json +++ b/ipypublish/templates/outline_schemas/latex_tplx_schema.json @@ -2,163 +2,10 @@ "$schema": "http://json-schema.org/draft-04/schema", "$id": "latex-tplx", "description": "define segments to insert into the the template jinja template of id latex-tplx", - "outline": [ - "((*- extends 'display_priority.tplx' -*))", - "", - "%% Latex Document Setup", - "%% ====================", - "", - "((* block header *))", - "% A latex document created by ipypublish {ipypub_version}", - "((( '{meta_docstring}' | comment_lines('% ') )))", - "%", - "", - " ((*- block docclass *))", - "%%%%%%%%%%%% DOCCLASS", - " {document_docclass}", - "%%%%%%%%%%%%", - " ((* endblock docclass -*))", - "", - " ((*- block packages *))", - "%%%%%%%%%%%% PACKAGES", - " {document_packages}", - "%%%%%%%%%%%%", - " ((* endblock packages *))", - "", - " ((*- block definitions *))", - " ((* block date *))((* endblock date *))", - " ((* block author *))((* endblock author *))", - " ((* block title *))((* endblock title *))", - "", - "%%%%%%%%%%%% DEFINITIONS", - " {document_definitions}", - "%%%%%%%%%%%%", - "", - " ((* endblock definitions *))", - "", - " ((*- block commands *))", - " ((* block margins *))", - "%%%%%%%%%%%% MARGINS", - " {document_margins}", - "%%%%%%%%%%%%", - " ((* endblock margins *))", - "%%%%%%%%%%%% COMMANDS", - " {document_commands}", - "%%%%%%%%%%%%", - " ((* endblock commands *))", - "", - "%%%%%%%%%%%% FINAL HEADER MATERIAL", - " {document_header_end}", - "%%%%%%%%%%%%", - "", - "((* endblock header *))", - "", - "((* block body *))", - "", - "\\begin{{document}}", - "", - " ((* block predoc *))", - " ((* block maketitle *))", - " {document_title}", - " ((* endblock maketitle *))", - " ((* block abstract *))", - " {document_abstract}", - " ((* endblock abstract *))", - " {document_predoc}", - " ((* endblock predoc *))", - "", - " ((( super() )))", - "", - " ((* block postdoc *))", - " ((* block bibliography *))", - " {document_bibliography}", - " ((* endblock bibliography *))", - " {document_postdoc}", - " ((* endblock postdoc *))", - "", - "\\end{{document}}", - "", - "((* endblock body *))", - "", - "%% Notebook Input", - "%% ==============", - "", - "((*- block any_cell scoped *))", - " {notebook_input}", - "((* endblock any_cell *))", - "", - "((* block input scoped *))", - " {notebook_input_code}", - "((* endblock input *))", - "", - "((* block rawcell scoped *))", - " {notebook_input_raw}", - "((* endblock rawcell *))", - "", - "((* block markdowncell scoped *))", - " {notebook_input_markdown}", - "((* endblock markdowncell *))", - "", - "((* block unknowncell scoped *))", - " {notebook_input_unknown}", - "((* endblock unknowncell *))", - "", - "%% Notebook Output", - "%% ================", - "", - "% Redirect execute_result to display data priority.", - "((*- block execute_result scoped *))", - " ((* block data_priority scoped *))", - " {notebook_output}", - " ((* endblock *))", - "((* endblock execute_result *))", - "", - "((* block data_text *))", - " {notebook_output_text}", - "((* endblock data_text *))", - "", - "((* block error *))", - " {notebook_output_error}", - "((* endblock error *))", - "", - "((* block traceback_line *))", - " {notebook_output_traceback}", - "((* endblock traceback_line *))", - "", - "((* block stream *))", - " {notebook_output_stream}", - "((* endblock stream *))", - "", - "((* block data_latex -*))", - " {notebook_output_latex}", - "((* endblock data_latex *))", - "", - "((*- block data_markdown -*))", - " {notebook_output_markdown}", - "((* endblock data_markdown *))", - "", - "((*- block data_png -*))", - " {notebook_output_png}", - "((*- endblock data_png -*))", - "", - "((*- block data_jpg -*))", - " {notebook_output_jpg}", - "((*- endblock data_jpg -*))", - "", - "((*- block data_svg -*))", - " {notebook_output_svg}", - "((*- endblock data_svg -*))", - "", - "((*- block data_pdf -*))", - " {notebook_output_pdf}", - "((*- endblock -*))", - "", - "", - "%% Jinja Macros", - "%% ================", - "", - "{jinja_macros}" - ], + "outline": { + "module": "ipypublish.templates.outline_schemas", + "file": "latex_outline.latex.j2" + }, "type": "object", "additionalProperties": false, "definitions": { diff --git a/ipypublish/templates/outline_schemas/python_outline.py.j2 b/ipypublish/templates/outline_schemas/python_outline.py.j2 new file mode 100644 index 0000000..5efc4a5 --- /dev/null +++ b/ipypublish/templates/outline_schemas/python_outline.py.j2 @@ -0,0 +1,16 @@ +{%- extends 'null.tpl' -%} +{% block header %} +{{ nb.metadata | meta2yaml('#~~ ') }} +{% endblock header %} +{% block codecell %} +#%% +{{ super() }} +{% endblock codecell %} +{% block in_prompt %}{% endblock in_prompt %} +{% block input %}{{ cell.metadata | meta2yaml('#~~ ') }} +{{ cell.source | ipython2python }} +{% endblock input %} +{% block markdowncell scoped %}#%% [markdown] +{{ cell.metadata | meta2yaml('#~~ ') }} +{{ cell.source | comment_lines }} +{% endblock markdowncell %} diff --git a/ipypublish/templates/outline_schemas/python_tpl_schema.json b/ipypublish/templates/outline_schemas/python_tpl_schema.json index da01beb..fa47b76 100644 --- a/ipypublish/templates/outline_schemas/python_tpl_schema.json +++ b/ipypublish/templates/outline_schemas/python_tpl_schema.json @@ -2,41 +2,12 @@ "$schema": "http://json-schema.org/draft-04/schema", "$id": "python-tpl", "description": "define segments to insert into a jinja template of id html-tpl", - "outline": ["{{%- extends 'null.tpl' -%}}", - "{{% block header %}}", - "{{{{ nb.metadata | meta2yaml('#~~ ') }}}}", - "{{% endblock header %}}", - "{{% block codecell %}}", - "#%%", - "{{{{ super() }}}}", - "{{% endblock codecell %}}", - "{{% block in_prompt %}}{{% endblock in_prompt %}}", - "{{% block input %}}{{{{ cell.metadata | meta2yaml('#~~ ') }}}}", - "{{{{ cell.source | ipython2python }}}}", - "{{% endblock input %}}", - "{{% block markdowncell scoped %}}#%% [markdown]", - "{{{{ cell.metadata | meta2yaml('#~~ ') }}}}", - "{{{{ cell.source | comment_lines }}}}", - "{{% endblock markdowncell %}}" - ], + "outline": { + "module": "ipypublish.templates.outline_schemas", + "file": "python_outline.py.j2" + }, "type": "object", "additionalProperties": false, - "definitions": { - "append_before": { - "description": "a segment of the template, each line should be a separate list item", - "type": "array", - "items": { - "type": "string" - } - }, - "append_after": { - "description": "a segment of the template, each line should be a separate list item", - "type": "array", - "items": { - "type": "string" - } - } - }, "required": [ "description", "identifier", diff --git a/ipypublish/templates/outline_schemas/sphinx_outline.rst.j2 b/ipypublish/templates/outline_schemas/sphinx_outline.rst.j2 new file mode 100644 index 0000000..a0d3881 --- /dev/null +++ b/ipypublish/templates/outline_schemas/sphinx_outline.rst.j2 @@ -0,0 +1,214 @@ +{% extends 'rst.tpl' %} + +{% macro insert_empty_lines(text) %} +{%- set before, after = text | get_empty_lines %} +{%- if before %} + :empty-lines-before: {{ before }} +{%- endif %} +{%- if after %} + :empty-lines-after: {{ after }} +{%- endif %} +{%- endmacro %} + + +{% block any_cell %} +{%- if cell.metadata.nbsphinx != 'hidden' %} +{{ super() }} +{% endif %} +{%- endblock any_cell %} + + +{% block input -%} +.. nbinput:: {% if cell.metadata.magics_language -%} +{{ cell.metadata.magics_language }} +{%- elif nb.metadata.language_info -%} +{{ nb.metadata.language_info.pygments_lexer or nb.metadata.language_info.name }} +{%- else -%} +{{ resources.codecell_lexer }} +{%- endif -%} +{{ insert_empty_lines(cell.source) }} +{%- if cell.execution_count %} + :execution-count: {{ cell.execution_count }} +{%- endif %} +{%- if not cell.outputs %} + :no-output: +{%- endif %} +{%- if cell.source.strip() %} + +{{ cell.source.strip('\n') | indent }} +{%- endif %} +{% endblock input %} + + +{% macro insert_nboutput(datatype, output, cell) -%} +.. nboutput:: +{%- if datatype == 'text/plain' %}{# nothing #} +{%- elif datatype == 'ansi' %} ansi +{%- else %} rst +{%- endif %} +{%- if output.output_type == 'execute_result' and cell.execution_count %} + :execution-count: {{ cell.execution_count }} +{%- endif %} +{%- if output != cell.outputs[-1] %} + :more-to-come: +{%- endif %} +{%- if output.name == 'stderr' %} + :class: stderr +{%- endif %} +{%- if datatype == 'text/plain' -%} +{{ insert_empty_lines(output.data[datatype]) }} + +{{ output.data[datatype].strip('\n') | indent }} +{%- elif datatype in ['image/svg+xml', 'image/png', 'image/jpeg', 'application/pdf'] %} + + .. image:: {{ output.metadata.filenames[datatype] | posix_path }} +{%- if datatype in output.metadata %} +{%- set width = output.metadata[datatype].width %} +{%- if width %} + :width: {{ width }} +{%- endif %} +{%- set height = output.metadata[datatype].height %} +{%- if height %} + :height: {{ height }} +{% endif %} +{% endif %} +{%- elif datatype in ['text/markdown'] %} + +{{ output.data['text/markdown'] | markdown2rst | indent }} +{%- elif datatype in ['text/latex'] %} + + .. math:: + :nowrap: + +{{ output.data['text/latex'] | indent | indent }} +{%- elif datatype == 'text/html' %} + :class: rendered_html + + .. raw:: html + +{{ output.data['text/html'] | indent | indent }} +{%- elif datatype == 'application/javascript' %} + + .. raw:: html + +
+ +{%- elif datatype.startswith('application/vnd.jupyter') and datatype.endswith('+json') %} + + .. raw:: html + + +{%- elif datatype == 'ansi' %} + + .. rst-class:: highlight + + .. raw:: html + + 
+{{ output.data[datatype] | ansi2html | indent | indent }}
+
+ + .. raw:: latex + + % + { + \\kern-\\sphinxverbatimsmallskipamount\\kern-\\baselineskip + \\kern+\\FrameHeightAdjust\\kern-\\fboxrule + \\vspace{\\nbsphinxcodecellspacing} + \\sphinxsetup{VerbatimBorderColor={named}{nbsphinx-code-border}} + {%- if output.name == 'stderr' %} + \\sphinxsetup{VerbatimColor={named}{nbsphinx-stderr}} + {%- else %} + \\sphinxsetup{VerbatimColor={named}{white}} + {%- endif %} + \\fvset{hllines={, ,}}% + \\begin{sphinxVerbatim}[commandchars=\\\\\\{\\}] +{{ output.data[datatype] | escape_latex | ansi2latex | indent | indent }} + \\end{sphinxVerbatim} + } + % The following \\relax is needed to avoid problems with adjacent ANSI + % cells and some other stuff (e.g. bullet lists) following ANSI cells. + % See https://github.com/sphinx-doc/sphinx/issues/3594 + \\relax +{% else %} + + .. nbwarning:: Data type cannot be displayed: {{ datatype }} +{%- endif %} +{% endmacro %} + + +{% block nboutput -%} +{%- set html_datatype, latex_datatype = output | get_output_type %} +{%- if html_datatype == latex_datatype %} +{{ insert_nboutput(html_datatype, output, cell) }} +{% else %} +.. only:: html + +{{ insert_nboutput(html_datatype, output, cell) | indent }} +.. only:: latex + +{{ insert_nboutput(latex_datatype, output, cell) | indent }} +{% endif %} +{% endblock nboutput %} + + +{% block execute_result %}{{ self.nboutput() }}{% endblock execute_result %} +{% block display_data %}{{ self.nboutput() }}{% endblock display_data %} +{% block stream %}{{ self.nboutput() }}{% endblock stream %} +{% block error %}{{ self.nboutput() }}{% endblock error %} + + +{% block markdowncell %} +{%- if 'nbsphinx-toctree' in cell.metadata %} +{{ cell | extract_toctree }} +{%- else %} +{{ cell | save_attachments or super() | replace_attachments }} +{% endif %} +{% endblock markdowncell %} + + +{% block rawcell %} +{%- set raw_mimetype = cell.metadata.get('raw_mimetype', '').lower() %} +{%- if raw_mimetype == '' %} +.. raw:: html + +{{ cell.source | indent }} + +.. raw:: latex + +{{ cell.source | indent }} +{%- elif raw_mimetype == 'text/html' %} +.. raw:: html + +{{ cell.source | indent }} +{%- elif raw_mimetype == 'text/latex' %} +.. raw:: latex + +{{ cell.source | indent }} +{%- elif raw_mimetype == 'text/markdown' %} +.. +{# Empty comment to make sure the preceding directive (if any) is closed #} +{{ cell.source | markdown2rst }} +{%- elif raw_mimetype == 'text/restructuredtext' %} +.. +{# Empty comment to make sure the preceding directive (if any) is closed #} +{{ cell.source }} +{% endif %} +{% endblock rawcell %} + + +{% block footer %} + +{% if 'application/vnd.jupyter.widget-state+json' in nb.metadata.widgets %} + +.. raw:: html + + +{% endif %} +{{ super() }} +{% endblock footer %} \ No newline at end of file diff --git a/ipypublish/utils.py b/ipypublish/utils.py index 57b4c05..19f2d0a 100644 --- a/ipypublish/utils.py +++ b/ipypublish/utils.py @@ -1,4 +1,79 @@ +import os +import json +import inspect +import importlib + +from six import string_types + +# python 2/3 compatibility try: import pathlib except ImportError: import pathlib2 as pathlib + + +def handle_error(msg, err_type, logger, raise_msg=None, log_msg=None): + """handle an error, by logging it, then raising an exception""" + if raise_msg is None: + raise_msg = msg + if log_msg is None: + log_msg = msg + + logger.error(log_msg) + raise err_type(raise_msg) + + +def read_file_from_directory(dir_path, file_name, jtype, + logger, as_json=False): + """load a file situated in a directory + + if as_json=True load file contents to a dict + + """ + if isinstance(dir_path, string_types): + dir_path = pathlib.Path(dir_path) + + file_path = dir_path.joinpath(file_name) + + if not file_path.exists(): + handle_error( + "the {} does not exist: {}".format(jtype, file_path), + IOError, logger=logger) + + if as_json: + with file_path.open() as fobj: + try: + data = json.load(fobj) + except Exception as err: + handle_error("failed to read {} ({}): {}".format( + jtype, file_path, err), IOError, logger=logger) + else: + with file_path.open() as fobj: + data = fobj.read() + + return data + + +def get_module_path(module): + """return a directory path to a module""" + return pathlib.Path(os.path.dirname( + os.path.abspath(inspect.getfile(module)))) + + +def read_file_from_module(module_path, file_name, jtype, + logger, as_json=False): + """load a file situated in a python module + + if as_json=True load file contents to a dict + + """ + try: + outline_module = importlib.import_module(module_path) + except ModuleNotFoundError: + handle_error( + "module {} containing {} {} not found".format( + module_path, jtype, file_name), + ModuleNotFoundError, logger=logger) + + return read_file_from_directory(get_module_path(outline_module), + file_name, jtype, logger, as_json=as_json) From 2d79e40c438152ff8bff179ee2709dcc4b666977 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 1 Feb 2019 08:35:39 +0000 Subject: [PATCH 2/5] use template outline file instead of json schema also improve front end logging --- .vscode/settings.json | 1 + docs/source/custom_export_config.rst | 2 +- docs/source/outline_schema.rst | 2 +- ipypublish/convert/config_manager.py | 32 +- ipypublish/convert/export_config.schema.json | 3 +- ipypublish/convert/main.py | 34 +- .../export_plugins/html_ipypublish_all.json | 2 +- .../export_plugins/html_ipypublish_main.json | 2 +- .../html_ipypublish_nocode.json | 2 +- ipypublish/export_plugins/html_standard.json | 2 +- .../export_plugins/latex_ipypublish_all.json | 2 +- .../export_plugins/latex_ipypublish_main.json | 2 +- .../latex_ipypublish_nocode.json | 2 +- .../latex_standard_article.json | 2 +- .../export_plugins/python_with_meta.json | 5 +- .../export_plugins/slides_ipypublish_all.json | 2 +- .../slides_ipypublish_main.json | 2 +- .../slides_ipypublish_nocode.json | 2 +- .../export_plugins/slides_mkdown_all.json | 2 +- .../export_plugins/slides_mkdown_main.json | 2 +- .../export_plugins/slides_mkdown_nocode.json | 2 +- .../export_plugins/slides_standard.json | 2 +- ipypublish/frontend/nbpresent.py | 37 +- ipypublish/frontend/nbpublish.py | 28 +- ipypublish/port_api/plugin_to_json.py | 6 +- ipypublish/port_api/tpl_dct_to_json.py | 2 +- ipypublish/scripts/nbmerge.py | 20 +- ipypublish/scripts/pdfexport.py | 34 +- .../{scripts => templates}/create_template.py | 99 +- .../outline_schemas/html_outline.html.j2 | 9 +- .../outline_schemas/html_tpl_schema.json | 343 - .../outline_schemas/latex_tplx_schema.json | 170 - ...on_tpl_schema.json => segment.schema.json} | 36 +- .../segments/ipy-biblio_natbib.latex-tpl.json | 5 +- .../ipy-contents_framed_code.latex-tpl.json | 5 +- .../ipy-contents_output.latex-tpl.json | 5 +- .../segments/ipy-doc_article.latex-tpl.json | 5 +- .../segments/ipy-flexbox_css.html-tplx.json | 5 +- .../segments/ipy-front_pages.latex-tpl.json | 5 +- .../segments/ipy-latex_doc.html-tplx.json | 5 +- .../ipy-slides_ipypublish.html-tplx.json | 5 +- .../segments/ipy-slides_mkdown.html-tplx.json | 5 +- .../segments/ipy-toc_sidebar.html-tplx.json | 5 +- .../ipy-toggle_buttons.html-tplx.json | 5 +- .../segments/std-content.html-tplx.json | 5 +- .../std-content_tagging.html-tplx.json | 5 +- .../segments/std-document.html-tplx.json | 5 +- .../std-in_out_prompts.latex-tpl.json | 5 +- .../segments/std-inout_prompt.html-tplx.json | 5 +- .../segments/std-mathjax.html-tplx.json | 5 +- .../segments/std-slides.html-tplx.json | 5 +- .../std-standard_article.latex-tpl.json | 5 +- .../std-standard_contents.latex-tpl.json | 5 +- .../std-standard_definitions.latex-tpl.json | 5 +- .../std-standard_packages.latex-tpl.json | 5 +- .../segments/std-widgets.html-tplx.json | 5 +- .../tests/test_files/converted/ipynb1.html | 11940 ++++++++++++++++ .../tests/test_files/example_new_plugin.json | 2 +- .../html_ipypublish_main.html | 10 +- .../ipynb1_converted/latex_ipypublish_all.tex | 3 +- .../latex_ipypublish_main.tex | 3 +- .../latex_ipypublish_nocode.tex | 3 +- .../latex_standard_article.tex | 3 +- .../ipynb_with_external.tex | 3 +- ipypublish/tests/test_frontend.py | 44 +- ipypublish_plugins/example_new_plugin.json | 2 +- 66 files changed, 12246 insertions(+), 775 deletions(-) rename ipypublish/{scripts => templates}/create_template.py (64%) delete mode 100644 ipypublish/templates/outline_schemas/html_tpl_schema.json delete mode 100644 ipypublish/templates/outline_schemas/latex_tplx_schema.json rename ipypublish/templates/{outline_schemas/python_tpl_schema.json => segment.schema.json} (54%) create mode 100644 ipypublish/tests/test_files/converted/ipynb1.html diff --git a/.vscode/settings.json b/.vscode/settings.json index 1636e17..bb87823 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,6 +14,7 @@ }, "python.pythonPath": "/anaconda/envs/lsr/bin/python", "restructuredtext.confPath": "${workspaceFolder}/docs/source", + "restructuredtext.builtDocumentationPath": "${workspaceFolder}/docs/build/html", "restructuredtext.preview.scrollEditorWithPreview": false, "restructuredtext.preview.scrollPreviewWithEditor": false, "restructuredtext.updateOnTextChanged": "false", diff --git a/docs/source/custom_export_config.rst b/docs/source/custom_export_config.rst index 92e058b..46a91d6 100644 --- a/docs/source/custom_export_config.rst +++ b/docs/source/custom_export_config.rst @@ -73,7 +73,7 @@ The configuration file is a JSON file, with a validation schema given in "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "latex_tplx_schema.json" + "file": "latex_outline.latex.j2" }, "segments": [ { diff --git a/docs/source/outline_schema.rst b/docs/source/outline_schema.rst index dbc8645..b859801 100644 --- a/docs/source/outline_schema.rst +++ b/docs/source/outline_schema.rst @@ -3,5 +3,5 @@ Template Outline Configuration Example -------------------------------------- -.. literalinclude:: ../../ipypublish/templates/outline_schemas/latex_tplx_schema.json +.. literalinclude:: ../../ipypublish/templates/outline_schemas/latex_outline.latex.j2 :language: JSON diff --git a/ipypublish/convert/config_manager.py b/ipypublish/convert/config_manager.py index 447df2f..fd4d204 100644 --- a/ipypublish/convert/config_manager.py +++ b/ipypublish/convert/config_manager.py @@ -10,13 +10,13 @@ from ipypublish.utils import (pathlib, handle_error, get_module_path, read_file_from_directory, read_file_from_module) from ipypublish import export_plugins -from ipypublish.scripts.create_template import create_template +from ipypublish.templates.create_template import create_template _TEMPLATE_KEY = 'new_template' _EXPORT_SCHEMA_FILE = "export_config.schema.json" _EXPORT_SCHEMA = None -logger = logging.getLogger(__name__) +logger = logging.getLogger("configuration") def get_export_config_path(export_key, config_folder_paths=()): @@ -126,29 +126,43 @@ def load_template(template_dict): return None if "directory" in template_dict["outline"]: - outline_schema = read_file_from_directory( + outline_template = read_file_from_directory( template_dict["outline"]["directory"], template_dict["outline"]["file"], - "template outline", logger, as_json=True) + "template outline", logger, as_json=False) + outline_name = os.path.join(template_dict["outline"]["directory"], + template_dict["outline"]["file"]) else: - outline_schema = read_file_from_module( + outline_template = read_file_from_module( template_dict["outline"]["module"], template_dict["outline"]["file"], - "template outline", logger, as_json=True) + "template outline", logger, as_json=False) + outline_name = os.path.join(template_dict["outline"]["module"], + template_dict["outline"]["file"]) + segments = [] - for segment in template_dict["segments"]: + for snum, segment in enumerate(template_dict.get("segments", [])): + + if "file" not in segment: + handle_error( + "'file' expected in segment {}".format(snum), + KeyError, logger) if "directory" in segment: seg_data = read_file_from_directory( segment["directory"], segment["file"], "template segment", logger, as_json=True) - else: + elif "module" in segment: seg_data = read_file_from_module( segment["module"], segment["file"], "template segment", logger, as_json=True) + else: + handle_error( + "'directory' or 'module' expected in segment {}".format(snum), + KeyError, logger) segments.append(seg_data) - template_str = create_template(outline_schema, segments) + template_str = create_template(outline_template, outline_name, segments) return str_to_jinja(template_str) diff --git a/ipypublish/convert/export_config.schema.json b/ipypublish/convert/export_config.schema.json index 901c3f7..d01e92e 100644 --- a/ipypublish/convert/export_config.schema.json +++ b/ipypublish/convert/export_config.schema.json @@ -79,8 +79,7 @@ ], "additionalProperties": false, "required": [ - "outline", - "segments" + "outline" ], "properties": { "outline": { diff --git a/ipypublish/convert/main.py b/ipypublish/convert/main.py index b592b2e..2e2a821 100755 --- a/ipypublish/convert/main.py +++ b/ipypublish/convert/main.py @@ -21,7 +21,7 @@ create_exporter_cls) from ipypublish.scripts.pdfexport import export_pdf -logger = logging.getLogger(__name__) +logger = logging.getLogger("conversion") def publish(ipynb_path, @@ -85,20 +85,20 @@ def publish(ipynb_path, os.mkdir(outdir) # log start of conversion - logging.info('started ipypublish v{0} at {1}'.format( + logger.info('started ipypublish v{0} at {1}'.format( ipypublish.__version__, time.strftime("%c"))) - logging.info('logging to: {}'.format( + logger.info('logging to: {}'.format( os.path.join(outdir, ipynb_name + '.nbpub.log'))) - logging.info('running for ipynb(s) at: {0}'.format(ipynb_path)) - logging.info('with conversion configuration: {0}'.format(conversion)) + logger.info('running for ipynb(s) at: {0}'.format(ipynb_path)) + logger.info('with conversion configuration: {0}'.format(conversion)) # merge all notebooks (this handles checking ipynb_path exists) final_nb, meta_path = merge_notebooks(ipynb_path, ignore_prefix=ignore_prefix) - logging.debug('notebooks meta path: {}'.format(meta_path)) + logger.debug('notebooks meta path: {}'.format(meta_path)) # find conversion configuration - logging.info('finding conversion configuration: {}'.format(conversion)) + logger.info('finding conversion configuration: {}'.format(conversion)) export_config_path = None if isinstance(conversion, string_types): outformat_path = pathlib.Path(conversion) @@ -118,19 +118,19 @@ def publish(ipynb_path, IOError, logger) # read conversion configuration and create - logging.info('loading conversion configuration') + logger.info('loading conversion configuration') data = load_export_config(export_config_path) - logging.info('creating exporter') + logger.info('creating exporter') exporter_cls = create_exporter_cls(data["exporter"]["class"]) - logging.info('creating template') + logger.info('creating template') jinja_template = load_template(data["template"]) - logging.info('creating nbconvert configuration') + logger.info('creating nbconvert configuration') config = create_config(data["exporter"], {"${meta_path}": str(meta_path), "${files_path}": str(files_folder)}) # run nbconvert - logging.info('running nbconvert') + logger.info('running nbconvert') exporter, body, resources = export_notebook(final_nb, exporter_cls, config, jinja_template) @@ -141,7 +141,7 @@ def publish(ipynb_path, return outpath, exporter # write results - logging.info("writing results") + logger.info("writing results") main_file_name = ipynb_name + exporter.file_extension outpath, outfilespath = write_output(body, resources, outdir, main_file_name, @@ -151,7 +151,7 @@ def publish(ipynb_path, # create pdf if create_pdf and exporter.output_mimetype == 'text/latex': - logging.info('running pdf conversion') + logger.info('running pdf conversion') if not export_pdf(outpath, outdir=outdir, files_path=outfilespath, @@ -161,7 +161,7 @@ def publish(ipynb_path, handle_error('pdf export failed, try running with pdf_debug=True', RuntimeError, logger) - logging.info('process finished successfully') + logger.info('process finished successfully') return outpath, exporter @@ -246,13 +246,13 @@ def write_output(body, resources, outdir, main_file_name, output_external, outpath = os.path.join(outdir, main_file_name) outfilespath = os.path.join(outdir, files_folder) - logging.info('outputting converted file to: {}'.format(outpath)) + logger.info('outputting converted file to: {}'.format(outpath)) with io.open(outpath, "w", encoding='utf8') as fh: fh.write(body) # output external files if output_external: - logging.info('dumping external files to: {}'.format(outfilespath)) + logger.info('dumping external files to: {}'.format(outfilespath)) if os.path.exists(outfilespath): if clear_existing: diff --git a/ipypublish/export_plugins/html_ipypublish_all.json b/ipypublish/export_plugins/html_ipypublish_all.json index 6f0e177..cd7ea57 100644 --- a/ipypublish/export_plugins/html_ipypublish_all.json +++ b/ipypublish/export_plugins/html_ipypublish_all.json @@ -78,7 +78,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/html_ipypublish_main.json b/ipypublish/export_plugins/html_ipypublish_main.json index 545dd92..bf6faa2 100644 --- a/ipypublish/export_plugins/html_ipypublish_main.json +++ b/ipypublish/export_plugins/html_ipypublish_main.json @@ -48,7 +48,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/html_ipypublish_nocode.json b/ipypublish/export_plugins/html_ipypublish_nocode.json index 3f2f4ff..370a000 100644 --- a/ipypublish/export_plugins/html_ipypublish_nocode.json +++ b/ipypublish/export_plugins/html_ipypublish_nocode.json @@ -77,7 +77,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/html_standard.json b/ipypublish/export_plugins/html_standard.json index 96e4077..a17aab1 100644 --- a/ipypublish/export_plugins/html_standard.json +++ b/ipypublish/export_plugins/html_standard.json @@ -11,7 +11,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/latex_ipypublish_all.json b/ipypublish/export_plugins/latex_ipypublish_all.json index 844363f..77ce89d 100644 --- a/ipypublish/export_plugins/latex_ipypublish_all.json +++ b/ipypublish/export_plugins/latex_ipypublish_all.json @@ -70,7 +70,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "latex_tplx_schema.json" + "file": "latex_outline.latex.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/latex_ipypublish_main.json b/ipypublish/export_plugins/latex_ipypublish_main.json index 6163b95..9734ca8 100644 --- a/ipypublish/export_plugins/latex_ipypublish_main.json +++ b/ipypublish/export_plugins/latex_ipypublish_main.json @@ -39,7 +39,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "latex_tplx_schema.json" + "file": "latex_outline.latex.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/latex_ipypublish_nocode.json b/ipypublish/export_plugins/latex_ipypublish_nocode.json index 4c8b714..8196f70 100644 --- a/ipypublish/export_plugins/latex_ipypublish_nocode.json +++ b/ipypublish/export_plugins/latex_ipypublish_nocode.json @@ -72,7 +72,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "latex_tplx_schema.json" + "file": "latex_outline.latex.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/latex_standard_article.json b/ipypublish/export_plugins/latex_standard_article.json index 3e18836..a8e0feb 100644 --- a/ipypublish/export_plugins/latex_standard_article.json +++ b/ipypublish/export_plugins/latex_standard_article.json @@ -11,7 +11,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "latex_tplx_schema.json" + "file": "latex_outline.latex.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/python_with_meta.json b/ipypublish/export_plugins/python_with_meta.json index 4cca532..b01a84b 100644 --- a/ipypublish/export_plugins/python_with_meta.json +++ b/ipypublish/export_plugins/python_with_meta.json @@ -14,8 +14,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "python_tpl_schema.json" - }, - "segments": [] + "file": "python_outline.py.j2" + } } } \ No newline at end of file diff --git a/ipypublish/export_plugins/slides_ipypublish_all.json b/ipypublish/export_plugins/slides_ipypublish_all.json index 270e4bf..d8a1808 100644 --- a/ipypublish/export_plugins/slides_ipypublish_all.json +++ b/ipypublish/export_plugins/slides_ipypublish_all.json @@ -88,7 +88,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/slides_ipypublish_main.json b/ipypublish/export_plugins/slides_ipypublish_main.json index ffbac97..c3c7c5f 100644 --- a/ipypublish/export_plugins/slides_ipypublish_main.json +++ b/ipypublish/export_plugins/slides_ipypublish_main.json @@ -57,7 +57,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/slides_ipypublish_nocode.json b/ipypublish/export_plugins/slides_ipypublish_nocode.json index 7e61885..0f432be 100644 --- a/ipypublish/export_plugins/slides_ipypublish_nocode.json +++ b/ipypublish/export_plugins/slides_ipypublish_nocode.json @@ -88,7 +88,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/slides_mkdown_all.json b/ipypublish/export_plugins/slides_mkdown_all.json index 4a9b4d5..ba4e1c0 100644 --- a/ipypublish/export_plugins/slides_mkdown_all.json +++ b/ipypublish/export_plugins/slides_mkdown_all.json @@ -68,7 +68,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/slides_mkdown_main.json b/ipypublish/export_plugins/slides_mkdown_main.json index 3456ca0..d77a25a 100644 --- a/ipypublish/export_plugins/slides_mkdown_main.json +++ b/ipypublish/export_plugins/slides_mkdown_main.json @@ -38,7 +38,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/slides_mkdown_nocode.json b/ipypublish/export_plugins/slides_mkdown_nocode.json index e9d8472..e64d83d 100644 --- a/ipypublish/export_plugins/slides_mkdown_nocode.json +++ b/ipypublish/export_plugins/slides_mkdown_nocode.json @@ -68,7 +68,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/export_plugins/slides_standard.json b/ipypublish/export_plugins/slides_standard.json index dcdfc13..d31bff0 100644 --- a/ipypublish/export_plugins/slides_standard.json +++ b/ipypublish/export_plugins/slides_standard.json @@ -11,7 +11,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/frontend/nbpresent.py b/ipypublish/frontend/nbpresent.py index a104448..8054bc6 100644 --- a/ipypublish/frontend/nbpresent.py +++ b/ipypublish/frontend/nbpresent.py @@ -7,6 +7,8 @@ from ipypublish.convert.main import publish from ipypublish.scripts.reveal_serve import RevealServer +logger = logging.getLogger("nbpresent") + def nbpresent(inpath, outformat='slides_standard', @@ -46,6 +48,7 @@ def nbpresent(inpath, inpath_name, inpath_ext = os.path.splitext(os.path.basename(inpath)) + outpath = None if inpath_ext == '.ipynb': outdir = os.path.join( os.getcwd(), 'converted') if outpath is None else outpath @@ -55,17 +58,27 @@ def nbpresent(inpath, outdir, inpath_name + '.nbpub.log'), 'w') flogger.setLevel(getattr(logging, log_level.upper())) root.addHandler(flogger) - inpath, exporter = publish(inpath, - conversion=outformat, - outpath=outpath, dump_files=dump_files, - ignore_prefix=ignore_prefix, - clear_existing=clear_files, - create_pdf=False, dry_run=dry_run, - plugin_folder_paths=export_paths) - server = RevealServer() - if not dry_run: - server.serve(inpath) + try: + outpath, exporter = publish(inpath, + conversion=outformat, + outpath=outpath, dump_files=dump_files, + ignore_prefix=ignore_prefix, + clear_existing=clear_files, + create_pdf=False, dry_run=dry_run, + plugin_folder_paths=export_paths) + except Exception as err: + logger.error("Run Failed: {}".format(err)) + return 1 + else: + outpath = inpath + + if outpath: + server = RevealServer() + if not dry_run: + server.serve(inpath) + + return 0 def run(sys_args=None): @@ -75,6 +88,6 @@ def run(sys_args=None): filepath, options = parse_options(sys_args, "nbpresent") - nbpresent(filepath, **options) + outcode = nbpresent(filepath, **options) - return 0 + return outcode diff --git a/ipypublish/frontend/nbpublish.py b/ipypublish/frontend/nbpublish.py index ed288eb..07338ae 100644 --- a/ipypublish/frontend/nbpublish.py +++ b/ipypublish/frontend/nbpublish.py @@ -6,6 +6,8 @@ from ipypublish.frontend.shared import parse_options from ipypublish.convert.main import publish +logger = logging.getLogger("nbpublish") + def nbpublish(ipynb_path, outformat='latex_ipypublish_main', @@ -55,7 +57,7 @@ def nbpublish(ipynb_path, root.setLevel(logging.DEBUG) slogger = logging.StreamHandler(sys.stdout) slogger.setLevel(getattr(logging, log_level.upper())) - formatter = logging.Formatter('%(levelname)s:%(module)s:%(message)s') + formatter = logging.Formatter('%(levelname)s:%(name)s:%(message)s') slogger.setFormatter(formatter) slogger.propogate = False root.addHandler(slogger) @@ -69,13 +71,19 @@ def nbpublish(ipynb_path, root.addHandler(flogger) # run - return publish(ipynb_path, - conversion=outformat, - outpath=outpath, dump_files=dump_files, - ignore_prefix=ignore_prefix, clear_existing=clear_files, - create_pdf=create_pdf, pdf_in_temp=pdf_in_temp, - pdf_debug=pdf_debug, dry_run=dry_run, - plugin_folder_paths=export_paths) + try: + publish(ipynb_path, + conversion=outformat, + outpath=outpath, dump_files=dump_files, + ignore_prefix=ignore_prefix, clear_existing=clear_files, + create_pdf=create_pdf, pdf_in_temp=pdf_in_temp, + pdf_debug=pdf_debug, dry_run=dry_run, + plugin_folder_paths=export_paths) + except Exception as err: + logger.error("Run Failed: {}".format(err)) + return 1 + + return 0 def run(sys_args=None): @@ -85,6 +93,6 @@ def run(sys_args=None): filepath, options = parse_options(sys_args, "nbpublish") - nbpublish(filepath, **options) + outcode = nbpublish(filepath, **options) - return 0 + return outcode diff --git a/ipypublish/port_api/plugin_to_json.py b/ipypublish/port_api/plugin_to_json.py index dfe5c85..009de38 100644 --- a/ipypublish/port_api/plugin_to_json.py +++ b/ipypublish/port_api/plugin_to_json.py @@ -98,19 +98,19 @@ def convert_oformat(oformat): exporter = 'nbconvert.exporters.LatexExporter' outline = { "module": "ipypublish.templates.outline_schemas", - "file": "latex_tplx_schema.json" + "file": "latex_outline.latex.j2" } elif oformat == "HTML": exporter = 'nbconvert.exporters.HTMLExporter' outline = { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" } elif oformat == "Slides": exporter = 'nbconvert.exporters.SlidesExporter' outline = { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" } else: raise ValueError("expected oformat to be: " diff --git a/ipypublish/port_api/tpl_dct_to_json.py b/ipypublish/port_api/tpl_dct_to_json.py index 91b4b78..60c44a0 100644 --- a/ipypublish/port_api/tpl_dct_to_json.py +++ b/ipypublish/port_api/tpl_dct_to_json.py @@ -56,7 +56,7 @@ def assess_syntax(path): "identifier": os.path.splitext(os.path.basename(path))[0], "description": docstring, "segments": output, - "template": "latex-tplx" if dtype == "tplx_dict" else "html-tpl" + "$schema": "../segment.schema.json" } diff --git a/ipypublish/scripts/nbmerge.py b/ipypublish/scripts/nbmerge.py index 1806733..d0d4a93 100755 --- a/ipypublish/scripts/nbmerge.py +++ b/ipypublish/scripts/nbmerge.py @@ -16,7 +16,9 @@ import nbformat from six import string_types -from ipypublish.utils import pathlib +from ipypublish.utils import pathlib, handle_error + +logger = logging.getLogger("nbmerge") def alphanumeric_sort(l): @@ -60,14 +62,12 @@ def merge_notebooks(ipynb_path, ignore_prefix='_', if isinstance(ipynb_path, string_types): ipynb_path = pathlib.Path(ipynb_path) if not ipynb_path.exists(): - logging.error('the notebook path does not exist: {}'.format( - ipynb_path)) - raise IOError('the notebook path does not exist: {}'.format( - ipynb_path)) + handle_error('the notebook path does not exist: {}'.format( + ipynb_path), IOError, logger) final_nb = None if ipynb_path.is_dir(): - logging.info('Merging all notebooks in directory') + logger.info('Merging all notebooks in directory') for ipath in alphanumeric_sort(ipynb_path.glob('*.ipynb')): if os.path.basename(ipath.name).startswith(ignore_prefix): continue @@ -87,7 +87,7 @@ def merge_notebooks(ipynb_path, ignore_prefix='_', else: final_nb.cells.extend(nb.cells) else: - logging.info('Reading notebook') + logger.info('Reading notebook') with ipynb_path.open('r', encoding='utf-8') as f: if (sys.version_info.major == 3 and sys.version_info.minor < 6 @@ -110,9 +110,7 @@ def merge_notebooks(ipynb_path, ignore_prefix='_', return nbformat.writes(final_nb).encode('utf-8') if final_nb is None: - logging.error('no acceptable notebooks found for path: {}'.format( - ipynb_path.name)) - raise IOError('no acceptable notebooks found for path: {}'.format( - ipynb_path.name)) + handle_error('no acceptable notebooks found for path: {}'.format( + ipynb_path.name), IOError, logger) return final_nb, meta_path diff --git a/ipypublish/scripts/pdfexport.py b/ipypublish/scripts/pdfexport.py index 6164783..9fdf9d5 100755 --- a/ipypublish/scripts/pdfexport.py +++ b/ipypublish/scripts/pdfexport.py @@ -19,6 +19,10 @@ except ImportError: from distutils.spawn import find_executable as exe_exists +from ipypublish.utils import handle_error + +logger = logging.getLogger("pdfexport") + VIEW_PDF = r""" @@ -98,7 +102,7 @@ def export_pdf(texpath, outdir, files_path=None, if isinstance(texpath, string_types): texpath = pathlib.Path(texpath) if not texpath.exists() or not texpath.is_file(): - logging.error('the tex file path does not exist: {}'.format(texpath)) + logger.error('the tex file path does not exist: {}'.format(texpath)) raise IOError('the tex file path does not exist: {}'.format(texpath)) texname = os.path.splitext(texpath.name)[0] @@ -107,18 +111,16 @@ def export_pdf(texpath, outdir, files_path=None, if isinstance(files_path, string_types): files_path = pathlib.Path(files_path) if not files_path.exists() or not files_path.is_dir(): - logging.error( + logger.error( 'the external folder path does not exist: {}'.format(texpath)) raise IOError( 'the external folder path does not exist: {}'.format(texpath)) if not exe_exists('latexmk'): - logging.error( - 'requires the latexmk executable to run. ' - 'See http://mg.readthedocs.io/latexmk.html#installation') - raise RuntimeError( + handle_error( 'requires the latexmk executable to run. ' - 'See http://mg.readthedocs.io/latexmk.html#installation') + 'See http://mg.readthedocs.io/latexmk.html#installation', + RuntimeError, logger) if convert_in_temp: out_folder = tempfile.mkdtemp() @@ -134,7 +136,7 @@ def export_pdf(texpath, outdir, files_path=None, exitcode = run_conversion(texpath, outdir, files_path, debug_mode) if exitcode == 0: - logging.info('pdf conversion complete') + logger.info('pdf conversion complete') if html_viewer: view_pdf = VIEW_PDF.format( @@ -144,8 +146,10 @@ def export_pdf(texpath, outdir, files_path=None, f.write(view_pdf) return True else: - logging.error('pdf conversion failed: ' - 'Try running with pdf_debug=True') + handle_error( + 'pdf conversion failed: ' + 'Try running with pdf_debug=True', + RuntimeError, logger) return False @@ -156,17 +160,17 @@ def run_conversion(texpath, out_folder, files_folder=None, debug_mode=False): # make sure tex file in right place outpath = os.path.join(out_folder, texpath.name) if os.path.dirname(str(texpath)) != str(out_folder): - logging.debug('copying tex file to: {}'.format( + logger.debug('copying tex file to: {}'.format( os.path.join(str(out_folder), texpath.name))) shutil.copyfile(str(texpath), os.path.join( str(out_folder), texpath.name)) # make sure the external files folder is in right place if files_folder is not None: - logging.debug('external files folder set') + logger.debug('external files folder set') outfilespath = os.path.join(out_folder, str(files_folder.name)) if str(files_folder) != str(outfilespath): - logging.debug('copying external files to: {}'.format(outfilespath)) + logger.debug('copying external files to: {}'.format(outfilespath)) if os.path.exists(outfilespath): shutil.rmtree(outfilespath) shutil.copytree(str(files_folder), str(outfilespath)) @@ -175,12 +179,12 @@ def run_conversion(texpath, out_folder, files_folder=None, debug_mode=False): with change_dir(out_folder): latexmk = ['latexmk', '-xelatex', '-bibtex', '-pdf'] latexmk += [] if debug_mode else ["--interaction=batchmode"] - logging.info('running: ' + ' '.join(latexmk + [''])) + logger.info('running: ' + ' '.join(latexmk + [''])) latexmk += [outpath] def log_latexmk_output(pipe): for line in iter(pipe.readline, b''): - logging.info('latexmk: {}'.format( + logger.info('latexmk: {}'.format( line.decode("utf-8").strip())) process = Popen(latexmk, stdout=PIPE, stderr=STDOUT) diff --git a/ipypublish/scripts/create_template.py b/ipypublish/templates/create_template.py similarity index 64% rename from ipypublish/scripts/create_template.py rename to ipypublish/templates/create_template.py index 4ad5857..7683ba3 100644 --- a/ipypublish/scripts/create_template.py +++ b/ipypublish/templates/create_template.py @@ -10,15 +10,18 @@ """ from typing import List, Tuple, Union # noqa: F401 import re +import io +import os import logging import jsonschema # from ipypublish import __version__ -from ipypublish.utils import (read_file_from_module, - read_file_from_directory, - handle_error) +from ipypublish.utils import handle_error, read_file_from_directory -logger = logging.getLogger(__name__) +logger = logging.getLogger("template") + +_SEGMENT_SCHEMA_FILE = "segment.schema.json" +_SEGMENT_SCHEMA = None def multireplace(string, replacements): @@ -28,7 +31,7 @@ def multireplace(string, replacements): From https://gist.github.com/bgusach/a967e0587d6e01e889fd1d776c5f3729 :param str string: string to execute replacements on - :param dict replacements: replacement dictionary {value to find: value to replace} + :param dict replacements: replacement dictionary {find value: replacement} :rtype: str """ @@ -49,13 +52,22 @@ def multireplace(string, replacements): return regexp.sub(lambda match: replacements[match.group(0)], string) -def create_template(outline_schema, segment_datas, outpath=None): +def _output_to_file(content, outpath): + if outpath is not None: + with io.open(outpath, "w", encoding='utf8') as f: # TODO use pathlib + f.write(content) + return + + +def create_template(outline_template, outline_name, + segment_datas, + outpath=None): # type: (dict, Tuple[dict]) -> str """ build a latex jinja template from; - - a json file, defining a jinja template outline, - containing segment placeholders, and a schema for segments, - - and json segment files adhering to the schema + - a jinja(2) template outline, + which may contain segment placeholders, + - and json segment files adhering to the segment.schema.json schema if a segment contains the key "overwrite", then its value should be a list of keys, @@ -63,41 +75,51 @@ def create_template(outline_schema, segment_datas, outpath=None): Parameters ---------- - outline_schema: dict + outline_template: str segment_datas: tuple or dict outpath: None or str if not None, output to path """ - # TODO validate outline schema - - # get jinja template - if "directory" in outline_schema["outline"]: - outline_content = read_file_from_directory( - outline_schema["outline"]["directory"], - outline_schema["outline"]["file"], "template outline", logger) - else: - outline_content = read_file_from_module( - outline_schema["outline"]["module"], - outline_schema["outline"]["file"], "template outline", logger) - # get the placeholders @ipubreplace{above|below}{name} regex = re.compile("@ipubreplace{(.+)}{(.+)}", re.MULTILINE) - placeholder_tuple = regex.findall(outline_content) + placeholder_tuple = regex.findall(outline_template) + + if not placeholder_tuple: + if segment_datas: + handle_error( + "the segment data is provided, " + + "but the outline template contains no placeholders", + KeyError, logger) + + if outpath: + _output_to_file(outline_template, outpath) + return outline_template + placeholders = {name: append for append, name in placeholder_tuple} + # TODO validate that placeholders to not exist multiple times, + # with above and below replacements = {key: "" for key in placeholders.keys()} docstrings = [ - outline_schema.get("$id"), - outline_schema.get("description"), - "with segments:" + "outline: {}".format(outline_name), ] + if segment_datas: + docstrings.append("with segments:") + global _SEGMENT_SCHEMA + if _SEGMENT_SCHEMA is None: + # lazy segment schema once + _SEGMENT_SCHEMA = read_file_from_directory( + os.path.dirname(os.path.realpath(__file__)), + _SEGMENT_SCHEMA_FILE, + "segment configuration schema", logger, as_json=True) + for seg_num, segment_data in enumerate(segment_datas): - # validate segment against outline schema + # validate segment try: - jsonschema.validate(segment_data, outline_schema) + jsonschema.validate(segment_data, _SEGMENT_SCHEMA) except jsonschema.ValidationError as err: handle_error( "validation of template segment {} failed: {}".format( @@ -115,6 +137,13 @@ def create_template(outline_schema, segment_datas, outpath=None): logger.debug('overwrite keys: {}'.format(overwrite)) for key, val in segment_data.get("segments").items(): + + if key not in placeholders: + handle_error( + "the segment key '{}' ".format(key) + + "is not contained in the outline template", + KeyError, logger) + valstring = "\n".join(val) if key in overwrite: replacements[key] = valstring @@ -123,11 +152,10 @@ def create_template(outline_schema, segment_datas, outpath=None): elif placeholders[key] == "below": replacements[key] = replacements[key] + '\n' + valstring else: - # TODO this should be part of the schema? handle_error(( "the placeholder @ipubreplace{{{0}}}{{{1}}} ".format( key, placeholders[key]) + - "should specify above or below appending"), + "should specify 'above' or 'below' appending"), jsonschema.ValidationError, logger=logger) if "meta_docstring" in placeholders: @@ -140,13 +168,12 @@ def create_template(outline_schema, segment_datas, outpath=None): replacements["ipypub_version"] = "" # str(__version__) replace_dict = { - "@ipubreplace{{{0}}}{{{1}}}".format(append, name): replacements[name] + "@ipubreplace{{{0}}}{{{1}}}".format(append, name): replacements.get( + name, "") for append, name in placeholder_tuple} - outline = multireplace(outline_content, replace_dict) + outline = multireplace(outline_template, replace_dict) - if outpath is not None: - with open(outpath, 'w') as f: # TODO use pathlib - f.write(outline) - return + if outpath: + _output_to_file(outline, outpath) return outline diff --git a/ipypublish/templates/outline_schemas/html_outline.html.j2 b/ipypublish/templates/outline_schemas/html_outline.html.j2 index e04c0f6..082bee5 100644 --- a/ipypublish/templates/outline_schemas/html_outline.html.j2 +++ b/ipypublish/templates/outline_schemas/html_outline.html.j2 @@ -1,6 +1,3 @@ - - - {%- extends 'display_priority.tpl' -%} @ipubreplace{below}{globals} @@ -8,7 +5,11 @@ %% HTML Setup %% ==================== -{%- block header %} +{%- block header %} + diff --git a/ipypublish/templates/outline_schemas/html_tpl_schema.json b/ipypublish/templates/outline_schemas/html_tpl_schema.json deleted file mode 100644 index c680fce..0000000 --- a/ipypublish/templates/outline_schemas/html_tpl_schema.json +++ /dev/null @@ -1,343 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema", - "$id": "html-tpl", - "description": "define segments to insert into a jinja template of id html-tpl", - "outline": { - "module": "ipypublish.templates.outline_schemas", - "file": "html_outline.html.j2" - }, - "type": "object", - "additionalProperties": false, - "definitions": { - "append_before": { - "description": "a segment of the template, each line should be a separate list item", - "type": "array", - "items": { - "type": "string" - } - }, - "append_after": { - "description": "a segment of the template, each line should be a separate list item", - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "description", - "identifier", - "template", - "segments" - ], - "properties": { - "$schema": { - "type": "string" - }, - "identifier": { - "description": "the identification of the segments", - "type": "string" - }, - "description": { - "description": "the purpose of this group of segments", - "type": "string" - }, - "template": { - "description": "the template identifier", - "enum": [ - "html-tpl" - ] - }, - "segments": { - "additionalProperties": false, - "description": "the segments to insert", - "properties": { - "globals": { - "description": "inserted at the beginning of the template", - "$ref": "#/definitions/append_after" - }, - "html_header": { - "$ref": "#/definitions/append_after" - }, - "html_body_start": { - "$ref": "#/definitions/append_after" - }, - "html_body_end": { - "$ref": "#/definitions/append_after" - }, - "html_footer": { - "$ref": "#/definitions/append_after" - }, - "notebook_all": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_code_prompt": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_code": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_raw": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_markdown": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_unknown": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_code_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_input_raw_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_input_markdown_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_input_unknown_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_input_code_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_raw_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_markdown_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_unknown_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_prompt": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_text": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_error": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_traceback": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_stream_stderr": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_stream_stdout": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_latex": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_markdown": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_png": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_jpg": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_svg": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_pdf": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_html": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_javascript": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_widget_state": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_widget_view": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_text_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_error_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_traceback_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_stream_stderr_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_stream_stdout_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_latex_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_markdown_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_png_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_jpg_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_svg_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_pdf_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_html_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_javascript_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_widget_state_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_widget_view_pre": { - "$ref": "#/definitions/append_before" - }, - "notebook_output_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_text_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_error_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_traceback_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_stream_stderr_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_stream_stdout_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_latex_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_markdown_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_png_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_jpg_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_svg_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_pdf_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_html_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_javascript_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_widget_state_post": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_widget_view_post": { - "$ref": "#/definitions/append_after" - }, - "jinja_macros": { - "$ref": "#/definitions/append_after" - } - } - }, - "overwrite": { - "description": "a list of segment keys that may overwrite previously defined entries", - "type": "array", - "items": { - "enum": [ - "globals", - "html_header", - "html_body_start", - "html_body_end", - "html_footer", - "notebook_all", - "notebook_input_code_prompt", - "notebook_input_code", - "notebook_input_raw", - "notebook_input_markdown", - "notebook_input_unknown", - "notebook_input_code_pre", - "notebook_input_raw_pre", - "notebook_input_markdown_pre", - "notebook_input_unknown_pre", - "notebook_input_code_post", - "notebook_input_raw_post", - "notebook_input_markdown_post", - "notebook_input_unknown_post", - "notebook_output", - "notebook_output_prompt", - "notebook_output_text", - "notebook_output_error", - "notebook_output_traceback", - "notebook_output_stream_stderr", - "notebook_output_stream_stdout", - "notebook_output_latex", - "notebook_output_markdown", - "notebook_output_png", - "notebook_output_jpg", - "notebook_output_svg", - "notebook_output_pdf", - "notebook_output_html", - "notebook_output_javascript", - "notebook_output_widget_state", - "notebook_output_widget_view", - "notebook_output_pre", - "notebook_output_text_pre", - "notebook_output_error_pre", - "notebook_output_traceback_pre", - "notebook_output_stream_stderr_pre", - "notebook_output_stream_stdout_pre", - "notebook_output_latex_pre", - "notebook_output_markdown_pre", - "notebook_output_png_pre", - "notebook_output_jpg_pre", - "notebook_output_svg_pre", - "notebook_output_pdf_pre", - "notebook_output_html_pre", - "notebook_output_javascript_pre", - "notebook_output_widget_state_pre", - "notebook_output_widget_view_pre", - "notebook_output_post", - "notebook_output_text_post", - "notebook_output_error_post", - "notebook_output_traceback_post", - "notebook_output_stream_stderr_post", - "notebook_output_stream_stdout_post", - "notebook_output_latex_post", - "notebook_output_markdown_post", - "notebook_output_png_post", - "notebook_output_jpg_post", - "notebook_output_svg_post", - "notebook_output_pdf_post", - "notebook_output_html_post", - "notebook_output_javascript_post", - "notebook_output_widget_state_post", - "notebook_output_widget_view_post", - "jinja_macros" - ] - } - } - } -} \ No newline at end of file diff --git a/ipypublish/templates/outline_schemas/latex_tplx_schema.json b/ipypublish/templates/outline_schemas/latex_tplx_schema.json deleted file mode 100644 index 4a143d6..0000000 --- a/ipypublish/templates/outline_schemas/latex_tplx_schema.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema", - "$id": "latex-tplx", - "description": "define segments to insert into the the template jinja template of id latex-tplx", - "outline": { - "module": "ipypublish.templates.outline_schemas", - "file": "latex_outline.latex.j2" - }, - "type": "object", - "additionalProperties": false, - "definitions": { - "append_after": { - "description": "a segment of the template, each line should be a separate item", - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "description", - "identifier", - "template", - "segments" - ], - "properties": { - "$schema": { - "type": "string" - }, - "identifier": { - "description": "the identification of the segments", - "type": "string" - }, - "description": { - "description": "the purpose of this group of segments", - "type": "string" - }, - "template": { - "description": "the template identifier", - "enum": ["latex-tplx"] - }, - "segments": { - "additionalProperties": false, - "description": "the segments to insert", - "properties": { - "document_docclass": { - "$ref": "#/definitions/append_after" - }, - "document_packages": { - "$ref": "#/definitions/append_after" - }, - "document_definitions": { - "$ref": "#/definitions/append_after" - }, - "document_margins": { - "$ref": "#/definitions/append_after" - }, - "document_commands": { - "$ref": "#/definitions/append_after" - }, - "document_header_end": { - "$ref": "#/definitions/append_after" - }, - "document_title": { - "$ref": "#/definitions/append_after" - }, - "document_abstract": { - "$ref": "#/definitions/append_after" - }, - "document_predoc": { - "$ref": "#/definitions/append_after" - }, - "document_bibliography": { - "$ref": "#/definitions/append_after" - }, - "document_postdoc": { - "$ref": "#/definitions/append_after" - }, - "notebook_input": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_code": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_raw": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_markdown": { - "$ref": "#/definitions/append_after" - }, - "notebook_input_unknown": { - "$ref": "#/definitions/append_after" - }, - "notebook_output": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_text": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_error": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_traceback": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_stream": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_latex": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_markdown": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_png": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_jpg": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_svg": { - "$ref": "#/definitions/append_after" - }, - "notebook_output_pdf": { - "$ref": "#/definitions/append_after" - }, - "jinja_macros": { - "$ref": "#/definitions/append_after" - } - } - }, - "overwrite": { - "description": "a list of segment keys that may overwrite previously defined entries", - "type": "array", - "items": { - "enum": [ - "meta_docstring", - "document_docclass", - "document_packages", - "document_definitions", - "document_margins", - "document_commands", - "document_header_end", - "document_title", - "document_abstract", - "document_predoc", - "document_bibliography", - "document_postdoc", - "notebook_input", - "notebook_input_code", - "notebook_input_raw", - "notebook_input_markdown", - "notebook_input_unknown", - "notebook_output", - "notebook_output_text", - "notebook_output_error", - "notebook_output_traceback", - "notebook_output_stream", - "notebook_output_latex", - "notebook_output_markdown", - "notebook_output_png", - "notebook_output_jpg", - "notebook_output_svg", - "notebook_output_pdf", - "jinja_macros" - ] - } - } - } -} \ No newline at end of file diff --git a/ipypublish/templates/outline_schemas/python_tpl_schema.json b/ipypublish/templates/segment.schema.json similarity index 54% rename from ipypublish/templates/outline_schemas/python_tpl_schema.json rename to ipypublish/templates/segment.schema.json index fa47b76..1881b3b 100644 --- a/ipypublish/templates/outline_schemas/python_tpl_schema.json +++ b/ipypublish/templates/segment.schema.json @@ -1,17 +1,12 @@ { "$schema": "http://json-schema.org/draft-04/schema", - "$id": "python-tpl", - "description": "define segments to insert into a jinja template of id html-tpl", - "outline": { - "module": "ipypublish.templates.outline_schemas", - "file": "python_outline.py.j2" - }, + "$id": "segment", + "description": "define segments to insert into a jinja template", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "required": [ - "description", "identifier", - "template", + "description", "segments" ], "properties": { @@ -26,16 +21,25 @@ "description": "the purpose of this group of segments", "type": "string" }, - "template": { - "description": "the template identifier", - "enum": [ - "python-tpl" - ] - }, "segments": { "additionalProperties": false, "description": "the segments to insert", - "properties": {} + "patternProperties": { + ".+": { + "description": "a segment of the template, each line should be a separate list item", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "overwrite": { + "description": "a list of segment keys that may overwrite previously defined entries", + "type": "array", + "items": { + "type": "string" + } } } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-biblio_natbib.latex-tpl.json b/ipypublish/templates/segments/ipy-biblio_natbib.latex-tpl.json index 6852c5e..23750b4 100644 --- a/ipypublish/templates/segments/ipy-biblio_natbib.latex-tpl.json +++ b/ipypublish/templates/segments/ipy-biblio_natbib.latex-tpl.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/latex_tplx_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-biblio_natbib", "description": "with the main ipypublish bibliography", "segments": { @@ -30,6 +30,5 @@ "((*- endif *))", "((*- endif *))" ] - }, - "template": "latex-tplx" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-contents_framed_code.latex-tpl.json b/ipypublish/templates/segments/ipy-contents_framed_code.latex-tpl.json index 4239452..59aee0f 100644 --- a/ipypublish/templates/segments/ipy-contents_framed_code.latex-tpl.json +++ b/ipypublish/templates/segments/ipy-contents_framed_code.latex-tpl.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/latex_tplx_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-contents_framed_code", "description": "with the input code wrapped and framed", "segments": { @@ -165,6 +165,5 @@ "((*- endif *))", "((*- endmacro *))" ] - }, - "template": "latex-tplx" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-contents_output.latex-tpl.json b/ipypublish/templates/segments/ipy-contents_output.latex-tpl.json index 1e59c60..3f93f99 100644 --- a/ipypublish/templates/segments/ipy-contents_output.latex-tpl.json +++ b/ipypublish/templates/segments/ipy-contents_output.latex-tpl.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/latex_tplx_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-contents_output", "description": "with the main ipypublish content", "segments": { @@ -262,6 +262,5 @@ "((*- endmacro *))", "" ] - }, - "template": "latex-tplx" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-doc_article.latex-tpl.json b/ipypublish/templates/segments/ipy-doc_article.latex-tpl.json index e6a610a..0b5a333 100644 --- a/ipypublish/templates/segments/ipy-doc_article.latex-tpl.json +++ b/ipypublish/templates/segments/ipy-doc_article.latex-tpl.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/latex_tplx_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-doc_article", "description": "with the main ipypublish article setup", "segments": { @@ -146,6 +146,5 @@ "\\usepackage{cleveref}", "\\creflabelformat{equation}{#2#1#3}" ] - }, - "template": "latex-tplx" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-flexbox_css.html-tplx.json b/ipypublish/templates/segments/ipy-flexbox_css.html-tplx.json index 220c382..2d198d9 100644 --- a/ipypublish/templates/segments/ipy-flexbox_css.html-tplx.json +++ b/ipypublish/templates/segments/ipy-flexbox_css.html-tplx.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-flexbox_css", "description": " adds flex boxes (https://github.com/Munawwar/flex-helper) ", "segments": { @@ -88,6 +88,5 @@ "", "" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-front_pages.latex-tpl.json b/ipypublish/templates/segments/ipy-front_pages.latex-tpl.json index 87fcf4c..61c514f 100644 --- a/ipypublish/templates/segments/ipy-front_pages.latex-tpl.json +++ b/ipypublish/templates/segments/ipy-front_pages.latex-tpl.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/latex_tplx_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-front_pages", "description": "with the main ipypublish title and contents page setup", "segments": { @@ -148,6 +148,5 @@ " \\endgroup", "((*- endif *))" ] - }, - "template": "latex-tplx" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-latex_doc.html-tplx.json b/ipypublish/templates/segments/ipy-latex_doc.html-tplx.json index 1ae4c09..b1560ac 100644 --- a/ipypublish/templates/segments/ipy-latex_doc.html-tplx.json +++ b/ipypublish/templates/segments/ipy-latex_doc.html-tplx.json @@ -10,7 +10,7 @@ "notebook_output_stream_stderr", "notebook_output_stream_stdout" ], - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-latex_doc", "description": " caption and label elements according to ipub meta tags ", "segments": { @@ -332,6 +332,5 @@ "{%- endmacro %}", "" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-slides_ipypublish.html-tplx.json b/ipypublish/templates/segments/ipy-slides_ipypublish.html-tplx.json index 0dd4494..59b7b66 100644 --- a/ipypublish/templates/segments/ipy-slides_ipypublish.html-tplx.json +++ b/ipypublish/templates/segments/ipy-slides_ipypublish.html-tplx.json @@ -4,7 +4,7 @@ "html_body_start", "notebook_input_markdown" ], - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-slides_ipypublish", "description": "marks up html with slide tags, based on metadata", "segments": { @@ -111,6 +111,5 @@ "", "{{ cell.source | markdown2html | strip_files_prefix }}" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-slides_mkdown.html-tplx.json b/ipypublish/templates/segments/ipy-slides_mkdown.html-tplx.json index 1112d6b..4737b4e 100644 --- a/ipypublish/templates/segments/ipy-slides_mkdown.html-tplx.json +++ b/ipypublish/templates/segments/ipy-slides_mkdown.html-tplx.json @@ -3,7 +3,7 @@ "notebook_all", "notebook_input_markdown" ], - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-slides_mkdown", "description": "sets markdown main titles (with one #) as their own slides,\nremove code cells ", "segments": { @@ -41,6 +41,5 @@ "", "{{ cell.source | markdown2html | strip_files_prefix }}" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-toc_sidebar.html-tplx.json b/ipypublish/templates/segments/ipy-toc_sidebar.html-tplx.json index 4cc3d30..262057d 100644 --- a/ipypublish/templates/segments/ipy-toc_sidebar.html-tplx.json +++ b/ipypublish/templates/segments/ipy-toc_sidebar.html-tplx.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-toc_sidebar", "description": "number headers and add sidebar with hyperlinked table of contents", "segments": { @@ -52,6 +52,5 @@ "", "" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/ipy-toggle_buttons.html-tplx.json b/ipypublish/templates/segments/ipy-toggle_buttons.html-tplx.json index eaaa60b..4ef8514 100644 --- a/ipypublish/templates/segments/ipy-toggle_buttons.html-tplx.json +++ b/ipypublish/templates/segments/ipy-toggle_buttons.html-tplx.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "ipypublish-toggle_buttons", "description": "add buttons to toggle input code and output cells", "segments": { @@ -69,6 +69,5 @@ "", "" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-content.html-tplx.json b/ipypublish/templates/segments/std-content.html-tplx.json index 6ddbf9f..7d05917 100644 --- a/ipypublish/templates/segments/std-content.html-tplx.json +++ b/ipypublish/templates/segments/std-content.html-tplx.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-content", "description": "with standard nbconvert input/output content", "segments": { @@ -123,6 +123,5 @@ "", "{{ output.data[datatype] | json_dumps }}" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-content_tagging.html-tplx.json b/ipypublish/templates/segments/std-content_tagging.html-tplx.json index adf34b5..68e3862 100644 --- a/ipypublish/templates/segments/std-content_tagging.html-tplx.json +++ b/ipypublish/templates/segments/std-content_tagging.html-tplx.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-content_tagging", "description": "standard html tag wrapping", "segments": { @@ -210,6 +210,5 @@ "", "" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-document.html-tplx.json b/ipypublish/templates/segments/std-document.html-tplx.json index 61852bc..a9f6378 100644 --- a/ipypublish/templates/segments/std-document.html-tplx.json +++ b/ipypublish/templates/segments/std-document.html-tplx.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-document", "description": "with standard nbconvert css layout", "segments": { @@ -64,6 +64,5 @@ " ", " " ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-in_out_prompts.latex-tpl.json b/ipypublish/templates/segments/std-in_out_prompts.latex-tpl.json index 42cf39e..8632077 100644 --- a/ipypublish/templates/segments/std-in_out_prompts.latex-tpl.json +++ b/ipypublish/templates/segments/std-in_out_prompts.latex-tpl.json @@ -3,7 +3,7 @@ "notebook_output", "notebook_input_code" ], - "$schema": "../outline_schema/latex_tplx_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-in_out_prompts", "description": "with in/out prompts", "segments": { @@ -47,6 +47,5 @@ "((*- endmacro *))", "" ] - }, - "template": "latex-tplx" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-inout_prompt.html-tplx.json b/ipypublish/templates/segments/std-inout_prompt.html-tplx.json index 87ad154..a40a4ea 100644 --- a/ipypublish/templates/segments/std-inout_prompt.html-tplx.json +++ b/ipypublish/templates/segments/std-inout_prompt.html-tplx.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-inout_prompt", "description": "with standard nbconvert input/output prompts", "segments": { @@ -33,6 +33,5 @@ "{% endblock output_area_prompt %}", "{% endif %}" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-mathjax.html-tplx.json b/ipypublish/templates/segments/std-mathjax.html-tplx.json index efdfc68..b0b50f7 100644 --- a/ipypublish/templates/segments/std-mathjax.html-tplx.json +++ b/ipypublish/templates/segments/std-mathjax.html-tplx.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-mathjax", "description": "with mathjax loaded", "segments": { @@ -27,6 +27,5 @@ "", "" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-slides.html-tplx.json b/ipypublish/templates/segments/std-slides.html-tplx.json index 1fc7a8b..7589306 100644 --- a/ipypublish/templates/segments/std-slides.html-tplx.json +++ b/ipypublish/templates/segments/std-slides.html-tplx.json @@ -2,7 +2,7 @@ "overwrite": [ "notebook_all" ], - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-slides", "description": "marks up html with slide tags, based on metadata", "segments": { @@ -214,6 +214,5 @@ "", "{%- endif -%}" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-standard_article.latex-tpl.json b/ipypublish/templates/segments/std-standard_article.latex-tpl.json index 917d0a3..c3c06cf 100644 --- a/ipypublish/templates/segments/std-standard_article.latex-tpl.json +++ b/ipypublish/templates/segments/std-standard_article.latex-tpl.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/latex_tplx_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-standard_article", "description": "with standard nbconvert article setup", "segments": { @@ -63,6 +63,5 @@ "\\title{((( resources.metadata.name | ascii_only | escape_latex )))}", "\\maketitle" ] - }, - "template": "latex-tplx" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-standard_contents.latex-tpl.json b/ipypublish/templates/segments/std-standard_contents.latex-tpl.json index 913e578..5ed84ba 100644 --- a/ipypublish/templates/segments/std-standard_contents.latex-tpl.json +++ b/ipypublish/templates/segments/std-standard_contents.latex-tpl.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/latex_tplx_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-standard_contents", "description": "with standard nbconvert input/output content", "segments": { @@ -81,6 +81,5 @@ "((*- endmacro *))", "" ] - }, - "template": "latex-tplx" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-standard_definitions.latex-tpl.json b/ipypublish/templates/segments/std-standard_definitions.latex-tpl.json index b16ae93..a8360f5 100644 --- a/ipypublish/templates/segments/std-standard_definitions.latex-tpl.json +++ b/ipypublish/templates/segments/std-standard_definitions.latex-tpl.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/latex_tplx_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-standard_definitions", "description": "with standard nbconvert definitions", "segments": { @@ -78,6 +78,5 @@ "\\def\\lt{<}", "" ] - }, - "template": "latex-tplx" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-standard_packages.latex-tpl.json b/ipypublish/templates/segments/std-standard_packages.latex-tpl.json index 8f3932e..ea81360 100644 --- a/ipypublish/templates/segments/std-standard_packages.latex-tpl.json +++ b/ipypublish/templates/segments/std-standard_packages.latex-tpl.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/latex_tplx_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-standard_packages", "description": "with standard nbconvert packages", "segments": { @@ -38,6 +38,5 @@ " % normalem makes italics be italics, not underlines", "" ] - }, - "template": "latex-tplx" + } } \ No newline at end of file diff --git a/ipypublish/templates/segments/std-widgets.html-tplx.json b/ipypublish/templates/segments/std-widgets.html-tplx.json index c95650b..4e6f24d 100644 --- a/ipypublish/templates/segments/std-widgets.html-tplx.json +++ b/ipypublish/templates/segments/std-widgets.html-tplx.json @@ -1,5 +1,5 @@ { - "$schema": "../outline_schema/html_tpl_schema.json", + "$schema": "../segment.schema.json", "identifier": "standard-widgets", "description": "with jupyter wigets", "segments": { @@ -25,6 +25,5 @@ "{% endif %}", "" ] - }, - "template": "html-tpl" + } } \ No newline at end of file diff --git a/ipypublish/tests/test_files/converted/ipynb1.html b/ipypublish/tests/test_files/converted/ipynb1.html new file mode 100644 index 0000000..550c5e8 --- /dev/null +++ b/ipypublish/tests/test_files/converted/ipynb1.html @@ -0,0 +1,11940 @@ + + + + + +Notebook + + + + + + + + + + + + + + + + + + + + +
+ +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+
+
+ +

a title

some text

+ +
+
+
+
+ +
+
+
+
+
+
a=1
+print(a)
+
+ +
+
+
+
+
+ +
+ +
+
1
+
+
+
+ +
+ +
+
+ + + \ No newline at end of file diff --git a/ipypublish/tests/test_files/example_new_plugin.json b/ipypublish/tests/test_files/example_new_plugin.json index aa85ac7..ef37c7c 100644 --- a/ipypublish/tests/test_files/example_new_plugin.json +++ b/ipypublish/tests/test_files/example_new_plugin.json @@ -45,7 +45,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { diff --git a/ipypublish/tests/test_files/ipynb1_converted/html_ipypublish_main.html b/ipypublish/tests/test_files/ipynb1_converted/html_ipypublish_main.html index 3550437..2e1fdf0 100644 --- a/ipypublish/tests/test_files/ipynb1_converted/html_ipypublish_main.html +++ b/ipypublish/tests/test_files/ipynb1_converted/html_ipypublish_main.html @@ -1,6 +1,6 @@ - - +- ipypublish-latex_doc: caption and label elements according to ipub meta tags +--> + diff --git a/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_all.tex b/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_all.tex index 236fda5..a877dae 100644 --- a/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_all.tex +++ b/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_all.tex @@ -1,8 +1,7 @@ % A latex document created by ipypublish -% latex-tplx -% define segments to insert into the the template jinja template of id latex-tplx +% outline: ipypublish.templates.outline_schemas/latex_outline.latex.j2 % with segments: % - standard-standard_packages: with standard nbconvert packages % - standard-standard_definitions: with standard nbconvert definitions diff --git a/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_main.tex b/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_main.tex index 7efebf7..57fd6de 100644 --- a/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_main.tex +++ b/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_main.tex @@ -1,8 +1,7 @@ % A latex document created by ipypublish -% latex-tplx -% define segments to insert into the the template jinja template of id latex-tplx +% outline: ipypublish.templates.outline_schemas/latex_outline.latex.j2 % with segments: % - standard-standard_packages: with standard nbconvert packages % - standard-standard_definitions: with standard nbconvert definitions diff --git a/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_nocode.tex b/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_nocode.tex index 5fa8d08..a18bf69 100644 --- a/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_nocode.tex +++ b/ipypublish/tests/test_files/ipynb1_converted/latex_ipypublish_nocode.tex @@ -1,8 +1,7 @@ % A latex document created by ipypublish -% latex-tplx -% define segments to insert into the the template jinja template of id latex-tplx +% outline: ipypublish.templates.outline_schemas/latex_outline.latex.j2 % with segments: % - standard-standard_packages: with standard nbconvert packages % - standard-standard_definitions: with standard nbconvert definitions diff --git a/ipypublish/tests/test_files/ipynb1_converted/latex_standard_article.tex b/ipypublish/tests/test_files/ipynb1_converted/latex_standard_article.tex index 83b74dc..f48fd8f 100644 --- a/ipypublish/tests/test_files/ipynb1_converted/latex_standard_article.tex +++ b/ipypublish/tests/test_files/ipynb1_converted/latex_standard_article.tex @@ -1,8 +1,7 @@ % A latex document created by ipypublish -% latex-tplx -% define segments to insert into the the template jinja template of id latex-tplx +% outline: ipypublish.templates.outline_schemas/latex_outline.latex.j2 % with segments: % - standard-standard_packages: with standard nbconvert packages % - standard-standard_definitions: with standard nbconvert definitions diff --git a/ipypublish/tests/test_files/ipynb_with_external/ipynb_with_external.tex b/ipypublish/tests/test_files/ipynb_with_external/ipynb_with_external.tex index f3083cd..a33b6a5 100644 --- a/ipypublish/tests/test_files/ipynb_with_external/ipynb_with_external.tex +++ b/ipypublish/tests/test_files/ipynb_with_external/ipynb_with_external.tex @@ -1,8 +1,7 @@ % A latex document created by ipypublish -% latex-tplx -% define segments to insert into the the template jinja template of id latex-tplx +% outline: ipypublish.templates.outline_schemas/latex_outline.latex.j2 % with segments: % - standard-standard_packages: with standard nbconvert packages % - standard-standard_definitions: with standard nbconvert definitions diff --git a/ipypublish/tests/test_frontend.py b/ipypublish/tests/test_frontend.py index b830890..6897e4e 100644 --- a/ipypublish/tests/test_frontend.py +++ b/ipypublish/tests/test_frontend.py @@ -7,33 +7,36 @@ def test_nbpresent_dry_run(temp_folder, ipynb1): # type: (str, pathlib.Path) -> None - nbpresent.run([str(ipynb1), "--outpath", temp_folder, - "--dry-run", "--log-level", "debug"]) + assert 0 == nbpresent.run([str(ipynb1), "--outpath", temp_folder, + "--dry-run", "--log-level", "debug"]) def test_nbpublish_dry_run(temp_folder, ipynb1): # type: (str, pathlib.Path) -> None - nbpublish.run([str(ipynb1), "--outpath", temp_folder, - "--dry-run", "--log-level", "debug"]) + assert 0 == nbpublish.run([str(ipynb1), "--outpath", temp_folder, + "--dry-run", "--log-level", "debug"]) def test_nbpublish_dry_run_with_external_plugin( temp_folder, ipynb1, external_export_plugin): # type: (str, pathlib.Path) -> None - nbpublish.run([str(ipynb1), "--outformat", str(external_export_plugin), - "--outpath", temp_folder, - "--dry-run", "--log-level", "debug"]) + assert 0 == nbpublish.run([str(ipynb1), + "--outformat", str(external_export_plugin), + "--outpath", temp_folder, + "--dry-run", "--log-level", "debug"]) def test_nbpublish_dry_run_with_external_plugin_key( temp_folder, ipynb1, external_export_plugin): # type: (str, pathlib.Path, pathlib.Path) -> None - nbpublish.run([str(ipynb1), - "--export-paths", str(external_export_plugin.parent), - "--outformat", - os.path.splitext(str(external_export_plugin.name))[0], - "--outpath", temp_folder, - "--dry-run", "--log-level", "debug"]) + assert 0 == nbpublish.run([str(ipynb1), + "--export-paths", + str(external_export_plugin.parent), + "--outformat", + os.path.splitext( + str(external_export_plugin.name))[0], + "--outpath", temp_folder, + "--dry-run", "--log-level", "debug"]) def test_nbpresent_list_exports(): @@ -52,9 +55,9 @@ def test_nbpublish_list_exports(): def test_nbpublish_write(temp_folder, ipynb1): # type: (str, pathlib.Path) -> None - nbpublish.run([str(ipynb1), - "--outformat", "latex_ipypublish_main", - "--outpath", temp_folder]) + assert 0 == nbpublish.run([str(ipynb1), + "--outformat", "latex_ipypublish_main", + "--outpath", temp_folder]) assert os.path.exists(os.path.join(temp_folder, ipynb1.name.replace(".ipynb", ".tex"))) @@ -62,10 +65,9 @@ def test_nbpublish_write(temp_folder, ipynb1): @pytest.mark.requires_latexmk def test_nbpublish_to_pdf(temp_folder, ipynb1): # type: (str, pathlib.Path) -> None - nbpublish.run([str(ipynb1), - "--outformat", "latex_ipypublish_main", - "--outpath", temp_folder, - "--create-pdf"]) + assert 0 == nbpublish.run([str(ipynb1), + "--outformat", "latex_ipypublish_main", + "--outpath", temp_folder, + "--create-pdf"]) assert os.path.exists(os.path.join(temp_folder, ipynb1.name.replace(".ipynb", ".pdf"))) - diff --git a/ipypublish_plugins/example_new_plugin.json b/ipypublish_plugins/example_new_plugin.json index aa85ac7..ef37c7c 100644 --- a/ipypublish_plugins/example_new_plugin.json +++ b/ipypublish_plugins/example_new_plugin.json @@ -45,7 +45,7 @@ "template": { "outline": { "module": "ipypublish.templates.outline_schemas", - "file": "html_tpl_schema.json" + "file": "html_outline.html.j2" }, "segments": [ { From 965a43483a6b4ec8472fe50cf7a9734f4fff2d90 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 1 Feb 2019 08:37:03 +0000 Subject: [PATCH 3/5] remove uneeded file --- .../tests/test_files/converted/ipynb1.html | 11940 ---------------- 1 file changed, 11940 deletions(-) delete mode 100644 ipypublish/tests/test_files/converted/ipynb1.html diff --git a/ipypublish/tests/test_files/converted/ipynb1.html b/ipypublish/tests/test_files/converted/ipynb1.html deleted file mode 100644 index 550c5e8..0000000 --- a/ipypublish/tests/test_files/converted/ipynb1.html +++ /dev/null @@ -1,11940 +0,0 @@ - - - - - -Notebook - - - - - - - - - - - - - - - - - - - - -
- -
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- -
-
-
-
- -

a title

some text

- -
-
-
-
- -
-
-
-
-
-
a=1
-print(a)
-
- -
-
-
-
-
- -
- -
-
1
-
-
-
- -
- -
-
- - - \ No newline at end of file From daff32135d04cc61a6094b1578c4680734dee5a5 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 1 Feb 2019 09:08:48 +0000 Subject: [PATCH 4/5] update documentation --- docs/get_intersphinx_inv.py | 4 +- .../ipypublish.scripts.create_template.rst | 7 --- docs/source/api/ipypublish.scripts.rst | 1 - .../ipypublish.templates.create_template.rst | 7 +++ ...tes.outline_schemas.convert_format_str.rst | 7 +++ .../ipypublish.templates.outline_schemas.rst | 7 +++ docs/source/api/ipypublish.templates.rst | 7 +++ docs/source/custom_export_config.rst | 48 ++++++++++++------- docs/source/outline_schema.rst | 8 ++-- 9 files changed, 65 insertions(+), 31 deletions(-) delete mode 100644 docs/source/api/ipypublish.scripts.create_template.rst create mode 100644 docs/source/api/ipypublish.templates.create_template.rst create mode 100644 docs/source/api/ipypublish.templates.outline_schemas.convert_format_str.rst diff --git a/docs/get_intersphinx_inv.py b/docs/get_intersphinx_inv.py index d02d5ba..a262593 100644 --- a/docs/get_intersphinx_inv.py +++ b/docs/get_intersphinx_inv.py @@ -21,8 +21,8 @@ def warn(self, msg): from sphinx.ext import intersphinx import warnings # uri = 'http://jinja.pocoo.org/docs/dev/objects.inv' - # uri = "http://nbconvert.readthedocs.io/en/latest/objects.inv" - uri = "http://nbformat.readthedocs.io/en/latest/objects.inv" + uri = "http://nbconvert.readthedocs.io/en/latest/objects.inv" + # uri = "http://nbformat.readthedocs.io/en/latest/objects.inv" # Read inventory into a dictionary inv = fetch_inventory(uri) diff --git a/docs/source/api/ipypublish.scripts.create_template.rst b/docs/source/api/ipypublish.scripts.create_template.rst deleted file mode 100644 index 5c7348c..0000000 --- a/docs/source/api/ipypublish.scripts.create_template.rst +++ /dev/null @@ -1,7 +0,0 @@ -ipypublish\.scripts\.create\_template module -============================================ - -.. automodule:: ipypublish.scripts.create_template - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api/ipypublish.scripts.rst b/docs/source/api/ipypublish.scripts.rst index f9ef698..a898ee3 100644 --- a/docs/source/api/ipypublish.scripts.rst +++ b/docs/source/api/ipypublish.scripts.rst @@ -6,7 +6,6 @@ Submodules .. toctree:: - ipypublish.scripts.create_template ipypublish.scripts.ipynb_latex_setup ipypublish.scripts.nbmerge ipypublish.scripts.pdfexport diff --git a/docs/source/api/ipypublish.templates.create_template.rst b/docs/source/api/ipypublish.templates.create_template.rst new file mode 100644 index 0000000..dbd5c5d --- /dev/null +++ b/docs/source/api/ipypublish.templates.create_template.rst @@ -0,0 +1,7 @@ +ipypublish\.templates\.create\_template module +============================================== + +.. automodule:: ipypublish.templates.create_template + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/ipypublish.templates.outline_schemas.convert_format_str.rst b/docs/source/api/ipypublish.templates.outline_schemas.convert_format_str.rst new file mode 100644 index 0000000..18547b2 --- /dev/null +++ b/docs/source/api/ipypublish.templates.outline_schemas.convert_format_str.rst @@ -0,0 +1,7 @@ +ipypublish\.templates\.outline\_schemas\.convert\_format\_str module +==================================================================== + +.. automodule:: ipypublish.templates.outline_schemas.convert_format_str + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/ipypublish.templates.outline_schemas.rst b/docs/source/api/ipypublish.templates.outline_schemas.rst index 3b1fbc3..ba2049a 100644 --- a/docs/source/api/ipypublish.templates.outline_schemas.rst +++ b/docs/source/api/ipypublish.templates.outline_schemas.rst @@ -1,6 +1,13 @@ ipypublish\.templates\.outline\_schemas package =============================================== +Submodules +---------- + +.. toctree:: + + ipypublish.templates.outline_schemas.convert_format_str + Module contents --------------- diff --git a/docs/source/api/ipypublish.templates.rst b/docs/source/api/ipypublish.templates.rst index fb08e57..72c6f02 100644 --- a/docs/source/api/ipypublish.templates.rst +++ b/docs/source/api/ipypublish.templates.rst @@ -9,6 +9,13 @@ Subpackages ipypublish.templates.outline_schemas ipypublish.templates.segments +Submodules +---------- + +.. toctree:: + + ipypublish.templates.create_template + Module contents --------------- diff --git a/docs/source/custom_export_config.rst b/docs/source/custom_export_config.rst index 46a91d6..2ae8990 100644 --- a/docs/source/custom_export_config.rst +++ b/docs/source/custom_export_config.rst @@ -125,24 +125,26 @@ Template Construction ~~~~~~~~~~~~~~~~~~~~~ In line 22, we define how to construct the `jinja`_ template. -The ``outline`` key defines the path to an outline schema, +The ``outline`` key defines the path to an outline template, such as in :ref:`outline_schema`. -This file achieves two things; to define an outline of the `jinja`_ template -structural blocks, -with placeholders to be replaced by :py:func:`str.format`, and to -provide a schema for segment files which are used to replace -one or more of the placeholders. + +This template file can be a full jinja template file, extending +an existing nbconvert template, but may optionally contain 'placeholders' +(of the form ``@ipubreplace{below}{key_name}``) +that can be replaced by injecting zero or more segments into them. +The first option states whether segment injections are appended above or below +previous injections, and the second option defines the key for that segment. This approach allows independent aspects of the document to be stored separately then pieced together in the desired manner. For example, -the segment defined in :ref:`segment_config` defines only parts of the document -which define how the bibliography is constructed. +the segment file in :ref:`segment_config` defines only parts of the document +which control how the bibliography is constructed. This could be removed or replaced by a custom export configuration. Similarly, input and output prompts can be added/removed in html documents. -Segments are applied in the order they are defined, -and the outline schema defines whether they are appended -above or below existing content. For example, these segments: +Segments are applied in the order they are defined, and appended +above or below existing content, as defined by the placeholder. +For example, these segments: .. code-block:: JSON @@ -158,15 +160,27 @@ above or below existing content. For example, these segments: } ] +applied to this template outline: + +.. code-block:: html+jinja + + {% block markdowncell scoped %} + @ipubreplace{above}{notebook_input_markdown_pre} + @ipubreplace{below}{notebook_input_markdown} + @ipubreplace{below}{notebook_input_markdown_post} + {% endblock markdowncell %} + will result in a template containing: -.. code-block:: html +.. code-block:: html+jinja -
-
- test -
-
+ {% block markdowncell scoped %} +
+
+ test +
+
+ {% endblock markdowncell %} Segment configuration files also have an optional ``overwrite`` key, which diff --git a/docs/source/outline_schema.rst b/docs/source/outline_schema.rst index b859801..7bc859c 100644 --- a/docs/source/outline_schema.rst +++ b/docs/source/outline_schema.rst @@ -1,7 +1,7 @@ .. _outline_schema: -Template Outline Configuration Example --------------------------------------- +Template Outline Example +------------------------ -.. literalinclude:: ../../ipypublish/templates/outline_schemas/latex_outline.latex.j2 - :language: JSON +.. literalinclude:: ../../ipypublish/templates/outline_schemas/html_outline.html.j2 + :language: html+jinja From 3613d12202f7fff44e54bdccad1d33e3216bdd50 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 1 Feb 2019 09:24:43 +0000 Subject: [PATCH 5/5] version bump --- ipypublish/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipypublish/__init__.py b/ipypublish/__init__.py index f0788a8..32a90a3 100644 --- a/ipypublish/__init__.py +++ b/ipypublish/__init__.py @@ -1 +1 @@ -__version__ = '0.7.1' +__version__ = '0.8.0'