From 6c7ae12c9c1d235988cbac79ccaa6aa04ce10714 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Fri, 4 Nov 2016 12:27:42 -0700 Subject: [PATCH 01/16] adding link filtering --- nbconvert/exporters/templateexporter.py | 3 ++- nbconvert/filters/pandoc.py | 5 +++++ nbconvert/templates/latex/document_contents.tplx | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nbconvert/exporters/templateexporter.py b/nbconvert/exporters/templateexporter.py index 9e1c75472..56259bc82 100644 --- a/nbconvert/exporters/templateexporter.py +++ b/nbconvert/exporters/templateexporter.py @@ -17,7 +17,7 @@ from nbconvert import filters from .exporter import Exporter - +from ..filters.filter_links import wrapped_convert_link #Jinja2 extensions to load. JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols'] @@ -50,6 +50,7 @@ 'prevent_list_blocks': filters.prevent_list_blocks, 'get_metadata': filters.get_metadata, 'convert_pandoc': filters.convert_pandoc, + 'wrapped_convert_link': wrapped_convert_link, } diff --git a/nbconvert/filters/pandoc.py b/nbconvert/filters/pandoc.py index 966bcaa61..c36efcef2 100644 --- a/nbconvert/filters/pandoc.py +++ b/nbconvert/filters/pandoc.py @@ -1,4 +1,5 @@ from nbconvert.utils.pandoc import pandoc +import os.path def convert_pandoc(source, from_format, to_format, extra_args=None): @@ -23,4 +24,8 @@ def convert_pandoc(source, from_format, to_format, extra_args=None): out : string Output as returned by pandoc. """ +# mydir = os.path.dirname(os.path.abspath(__file__)) +# filter_links = os.path.join(mydir, 'filter_links.py') +# if extra_args is not None: +# extra_args.extend(['--filter',filter_links]) return pandoc(source, from_format, to_format, extra_args=extra_args) diff --git a/nbconvert/templates/latex/document_contents.tplx b/nbconvert/templates/latex/document_contents.tplx index e047e061f..c2a1e02b8 100644 --- a/nbconvert/templates/latex/document_contents.tplx +++ b/nbconvert/templates/latex/document_contents.tplx @@ -64,7 +64,7 @@ % Render markdown ((* block markdowncell scoped *)) - ((( cell.source | citation2latex | strip_files_prefix | convert_pandoc('markdown', 'latex') ))) + ((( cell.source | citation2latex | strip_files_prefix | convert_pandoc('markdown', 'json',extra_args=[]) | wrapped_convert_link | convert_pandoc('json','latex')))) ((* endblock markdowncell *)) % Don't display unknown types From 7c351b8d190e8492bb0124d698a66198e62af12f Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Fri, 4 Nov 2016 12:28:10 -0700 Subject: [PATCH 02/16] Include filter link function --- nbconvert/filters/filter_links.py | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 nbconvert/filters/filter_links.py diff --git a/nbconvert/filters/filter_links.py b/nbconvert/filters/filter_links.py new file mode 100644 index 000000000..7f4866c16 --- /dev/null +++ b/nbconvert/filters/filter_links.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""A pandoc filter used in converting notebooks to Latex. +Converts links between notebooks to Latex cross-references. +""" +import re + +from pandocfilters import applyJSONFilters, RawInline + + +def wrapped_convert_link(source): + return applyJSONFilters([convert_link], source) + +def convert_link(key, val, fmt, meta): + """ + """ + + if key == 'Link': + target = val[2][0] + # Links to other notebooks + m = re.match(r'(\d+\-.+)\.ipynb$', target) + if m: + return RawInline('tex', 'Section \\ref{sec:%s}' % m.group(1)) + + # Links to sections of this or other notebooks + m = re.match(r'(\d+\-.+\.ipynb)?#(.+)$', target) + if m: + # pandoc automatically makes labels for headings. + label = m.group(2).lower() + label = re.sub(r'[^\w-]+', '', label) # Strip HTML entities + return RawInline('tex', 'Section \\ref{%s}' % label) + + # Other elements will be returned unchanged. + From 1216e24de7af2efdcc12514a1cbb4715864db4d0 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Fri, 4 Nov 2016 12:29:16 -0700 Subject: [PATCH 03/16] Remove internal mechanism and put in pandocfilters --- nbconvert/filters/filter_links.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nbconvert/filters/filter_links.py b/nbconvert/filters/filter_links.py index 7f4866c16..86088870f 100644 --- a/nbconvert/filters/filter_links.py +++ b/nbconvert/filters/filter_links.py @@ -6,7 +6,6 @@ from pandocfilters import applyJSONFilters, RawInline - def wrapped_convert_link(source): return applyJSONFilters([convert_link], source) From a7ca351235e54dea8aa67a91a68d74428154ea43 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Fri, 4 Nov 2016 14:50:11 -0700 Subject: [PATCH 04/16] Move utility as backup in pandoc utils --- nbconvert/utils/pandoc.py | 22 ++++++++++++++++++++++ setup.py | 1 + 2 files changed, 23 insertions(+) diff --git a/nbconvert/utils/pandoc.py b/nbconvert/utils/pandoc.py index 2c66839dd..78fe175d2 100644 --- a/nbconvert/utils/pandoc.py +++ b/nbconvert/utils/pandoc.py @@ -109,6 +109,28 @@ def check_pandoc_version(): RuntimeWarning, stacklevel=2) return ok +try: + from pandocfilters import applyJSONFilters +except ImportError: + import json + from pandocfilters import walk + + def applyJSONFilters(actions, source, format=""): + + doc = json.loads(source) + + if 'meta' in doc: + meta = doc['meta'] + elif doc[0]: # old API + meta = doc[0]['unMeta'] + else: + meta = {} + altered = doc + for action in actions: + altered = walk(altered, action, format, meta) + + return json.dumps(altered) + #----------------------------------------------------------------------------- # Exception handling #----------------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 895c4b941..2809ab715 100644 --- a/setup.py +++ b/setup.py @@ -193,6 +193,7 @@ def run(self): 'nbformat', 'entrypoints', 'bleach', + 'pandocfilters', ] extras_require = setuptools_args['extras_require'] = { From 6326932dfb4eedfcc6c26a3ba9a97e8e534b10a3 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Fri, 4 Nov 2016 15:44:14 -0700 Subject: [PATCH 05/16] Use fallback if you cannot get applyJSONFilters from pandocfilters --- nbconvert/filters/filter_links.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nbconvert/filters/filter_links.py b/nbconvert/filters/filter_links.py index 86088870f..3e083dbf2 100644 --- a/nbconvert/filters/filter_links.py +++ b/nbconvert/filters/filter_links.py @@ -4,7 +4,8 @@ """ import re -from pandocfilters import applyJSONFilters, RawInline +from pandocfilters import RawInline +from ..utils.pandoc import applyJSONFilters def wrapped_convert_link(source): return applyJSONFilters([convert_link], source) From f0b0d3be1aa1114c8c330a7cecc6ed040562c67c Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Thu, 10 Nov 2016 10:02:00 -0800 Subject: [PATCH 06/16] Make dependency on new pandocfilters explicit --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2809ab715..ebdcfa05c 100644 --- a/setup.py +++ b/setup.py @@ -193,7 +193,7 @@ def run(self): 'nbformat', 'entrypoints', 'bleach', - 'pandocfilters', + 'pandocfilters>=1.4.1', ] extras_require = setuptools_args['extras_require'] = { From 18cbee4be40df9a6fb9b2093e713c25af6ebb3d0 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Thu, 10 Nov 2016 10:03:08 -0800 Subject: [PATCH 07/16] explicit dependency on applyJSONFilters --- nbconvert/utils/pandoc.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nbconvert/utils/pandoc.py b/nbconvert/utils/pandoc.py index 78fe175d2..0f2c38b8b 100644 --- a/nbconvert/utils/pandoc.py +++ b/nbconvert/utils/pandoc.py @@ -13,6 +13,7 @@ from ipython_genutils.py3compat import cast_bytes, which from .exceptions import ConversionException +from pandocfilters import applyJSONFilters _minimal_version = "1.12.1" @@ -109,12 +110,6 @@ def check_pandoc_version(): RuntimeWarning, stacklevel=2) return ok -try: - from pandocfilters import applyJSONFilters -except ImportError: - import json - from pandocfilters import walk - def applyJSONFilters(actions, source, format=""): doc = json.loads(source) From 2a45d711eb12422515182d93a9202ef572bac934 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Thu, 10 Nov 2016 10:43:21 -0800 Subject: [PATCH 08/16] remove commented lines --- nbconvert/filters/pandoc.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nbconvert/filters/pandoc.py b/nbconvert/filters/pandoc.py index c36efcef2..966bcaa61 100644 --- a/nbconvert/filters/pandoc.py +++ b/nbconvert/filters/pandoc.py @@ -1,5 +1,4 @@ from nbconvert.utils.pandoc import pandoc -import os.path def convert_pandoc(source, from_format, to_format, extra_args=None): @@ -24,8 +23,4 @@ def convert_pandoc(source, from_format, to_format, extra_args=None): out : string Output as returned by pandoc. """ -# mydir = os.path.dirname(os.path.abspath(__file__)) -# filter_links = os.path.join(mydir, 'filter_links.py') -# if extra_args is not None: -# extra_args.extend(['--filter',filter_links]) return pandoc(source, from_format, to_format, extra_args=extra_args) From 19b9c7ff101954f5950d5f1f8c2a5407770da614 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Thu, 10 Nov 2016 10:46:32 -0800 Subject: [PATCH 09/16] fix the regex --- nbconvert/filters/filter_links.py | 15 ++++----------- nbconvert/utils/pandoc.py | 1 - 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/nbconvert/filters/filter_links.py b/nbconvert/filters/filter_links.py index 3e083dbf2..734a20929 100644 --- a/nbconvert/filters/filter_links.py +++ b/nbconvert/filters/filter_links.py @@ -4,8 +4,7 @@ """ import re -from pandocfilters import RawInline -from ..utils.pandoc import applyJSONFilters +from pandocfilters import RawInline, applyJSONFilters def wrapped_convert_link(source): return applyJSONFilters([convert_link], source) @@ -16,18 +15,12 @@ def convert_link(key, val, fmt, meta): if key == 'Link': target = val[2][0] - # Links to other notebooks - m = re.match(r'(\d+\-.+)\.ipynb$', target) - if m: - return RawInline('tex', 'Section \\ref{sec:%s}' % m.group(1)) - - # Links to sections of this or other notebooks - m = re.match(r'(\d+\-.+\.ipynb)?#(.+)$', target) + m = re.match(r'#(.+)$', target) if m: # pandoc automatically makes labels for headings. - label = m.group(2).lower() + label = m.group(1).lower() label = re.sub(r'[^\w-]+', '', label) # Strip HTML entities - return RawInline('tex', 'Section \\ref{%s}' % label) + return RawInline('tex', r'Section \ref{%s}' % label) # Other elements will be returned unchanged. diff --git a/nbconvert/utils/pandoc.py b/nbconvert/utils/pandoc.py index 0f2c38b8b..7a9f527de 100644 --- a/nbconvert/utils/pandoc.py +++ b/nbconvert/utils/pandoc.py @@ -13,7 +13,6 @@ from ipython_genutils.py3compat import cast_bytes, which from .exceptions import ConversionException -from pandocfilters import applyJSONFilters _minimal_version = "1.12.1" From dfb6b8f0f52e561cdd781a7c053986c3469f692d Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Thu, 10 Nov 2016 14:31:26 -0800 Subject: [PATCH 10/16] remove applyJSONFilters --- nbconvert/utils/pandoc.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/nbconvert/utils/pandoc.py b/nbconvert/utils/pandoc.py index 7a9f527de..2c66839dd 100644 --- a/nbconvert/utils/pandoc.py +++ b/nbconvert/utils/pandoc.py @@ -109,22 +109,6 @@ def check_pandoc_version(): RuntimeWarning, stacklevel=2) return ok - def applyJSONFilters(actions, source, format=""): - - doc = json.loads(source) - - if 'meta' in doc: - meta = doc['meta'] - elif doc[0]: # old API - meta = doc[0]['unMeta'] - else: - meta = {} - altered = doc - for action in actions: - altered = walk(altered, action, format, meta) - - return json.dumps(altered) - #----------------------------------------------------------------------------- # Exception handling #----------------------------------------------------------------------------- From d07e11808211d03e332ca87e9f1faa78a71144a5 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Fri, 11 Nov 2016 13:39:16 -0800 Subject: [PATCH 11/16] remove wrapped_convert_link --- nbconvert/exporters/templateexporter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nbconvert/exporters/templateexporter.py b/nbconvert/exporters/templateexporter.py index 56259bc82..4b61141b6 100644 --- a/nbconvert/exporters/templateexporter.py +++ b/nbconvert/exporters/templateexporter.py @@ -50,7 +50,6 @@ 'prevent_list_blocks': filters.prevent_list_blocks, 'get_metadata': filters.get_metadata, 'convert_pandoc': filters.convert_pandoc, - 'wrapped_convert_link': wrapped_convert_link, } From 178ce3d31c92d06a4294634f12c8cc393db37216 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Fri, 11 Nov 2016 15:47:22 -0800 Subject: [PATCH 12/16] change naming scheme for latex link filtering --- nbconvert/exporters/latex.py | 6 +++++- nbconvert/filters/filter_links.py | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nbconvert/exporters/latex.py b/nbconvert/exporters/latex.py index 5f51135c5..10ea88e6f 100644 --- a/nbconvert/exporters/latex.py +++ b/nbconvert/exporters/latex.py @@ -10,7 +10,7 @@ from nbconvert.filters.highlight import Highlight2Latex from .templateexporter import TemplateExporter - +from ..filters.filter_links import resolve_references class LatexExporter(TemplateExporter): """ @@ -43,6 +43,10 @@ def _template_skeleton_path_default(self): template_extension = Unicode(".tplx").tag(config=True) output_mimetype = 'text/latex' + + def default_filters(self): + yield from super(TemplateExporter,self).default_filters() + yield ('resolve_references', resolve_references) @property def default_config(self): diff --git a/nbconvert/filters/filter_links.py b/nbconvert/filters/filter_links.py index 734a20929..b2508e8b5 100644 --- a/nbconvert/filters/filter_links.py +++ b/nbconvert/filters/filter_links.py @@ -6,10 +6,10 @@ from pandocfilters import RawInline, applyJSONFilters -def wrapped_convert_link(source): - return applyJSONFilters([convert_link], source) +def resolve_references(source): + return applyJSONFilters([resolve_one_reference], source) -def convert_link(key, val, fmt, meta): +def resolve_one_reference(key, val, fmt, meta): """ """ From e3155e5a44098f5c2a3a56b21d6bcc703953039a Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Fri, 11 Nov 2016 15:51:50 -0800 Subject: [PATCH 13/16] remove call to nonexistant wrapped_convert_link --- nbconvert/exporters/templateexporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbconvert/exporters/templateexporter.py b/nbconvert/exporters/templateexporter.py index 4b61141b6..9e1c75472 100644 --- a/nbconvert/exporters/templateexporter.py +++ b/nbconvert/exporters/templateexporter.py @@ -17,7 +17,7 @@ from nbconvert import filters from .exporter import Exporter -from ..filters.filter_links import wrapped_convert_link + #Jinja2 extensions to load. JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols'] From 6ee0ebb386ae1e8b1ebaf56f7d38bbbabbb70de4 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Fri, 11 Nov 2016 23:47:05 -0800 Subject: [PATCH 14/16] invoke the superclass appropriately --- nbconvert/exporters/latex.py | 6 +++--- nbconvert/templates/latex/document_contents.tplx | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nbconvert/exporters/latex.py b/nbconvert/exporters/latex.py index 10ea88e6f..959722e81 100644 --- a/nbconvert/exporters/latex.py +++ b/nbconvert/exporters/latex.py @@ -9,8 +9,8 @@ from traitlets.config import Config from nbconvert.filters.highlight import Highlight2Latex +from nbconvert.filters.filter_links import resolve_references from .templateexporter import TemplateExporter -from ..filters.filter_links import resolve_references class LatexExporter(TemplateExporter): """ @@ -43,9 +43,9 @@ def _template_skeleton_path_default(self): template_extension = Unicode(".tplx").tag(config=True) output_mimetype = 'text/latex' - + def default_filters(self): - yield from super(TemplateExporter,self).default_filters() + yield from super(LatexExporter,self).default_filters() yield ('resolve_references', resolve_references) @property diff --git a/nbconvert/templates/latex/document_contents.tplx b/nbconvert/templates/latex/document_contents.tplx index c2a1e02b8..c32c39c55 100644 --- a/nbconvert/templates/latex/document_contents.tplx +++ b/nbconvert/templates/latex/document_contents.tplx @@ -64,7 +64,7 @@ % Render markdown ((* block markdowncell scoped *)) - ((( cell.source | citation2latex | strip_files_prefix | convert_pandoc('markdown', 'json',extra_args=[]) | wrapped_convert_link | convert_pandoc('json','latex')))) + ((( cell.source | citation2latex | strip_files_prefix | convert_pandoc('markdown', 'json',extra_args=[]) | resolve_references | convert_pandoc('json','latex')))) ((* endblock markdowncell *)) % Don't display unknown types From fa79c87b1cf814d84be5fa1a8976bc83080d9771 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Fri, 11 Nov 2016 23:56:10 -0800 Subject: [PATCH 15/16] Add docstrings, fix comma spacing --- nbconvert/exporters/latex.py | 2 +- nbconvert/filters/filter_links.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/nbconvert/exporters/latex.py b/nbconvert/exporters/latex.py index 959722e81..d059e7f5b 100644 --- a/nbconvert/exporters/latex.py +++ b/nbconvert/exporters/latex.py @@ -45,7 +45,7 @@ def _template_skeleton_path_default(self): output_mimetype = 'text/latex' def default_filters(self): - yield from super(LatexExporter,self).default_filters() + yield from super(LatexExporter, self).default_filters() yield ('resolve_references', resolve_references) @property diff --git a/nbconvert/filters/filter_links.py b/nbconvert/filters/filter_links.py index b2508e8b5..30006745c 100644 --- a/nbconvert/filters/filter_links.py +++ b/nbconvert/filters/filter_links.py @@ -7,10 +7,22 @@ from pandocfilters import RawInline, applyJSONFilters def resolve_references(source): + """ + This applies the resolve_one_reference to the text passed in via the source argument. + + This expects content in the form of a string encoded JSON object as represented + internally in ``pandoc``. + """ return applyJSONFilters([resolve_one_reference], source) def resolve_one_reference(key, val, fmt, meta): """ + This takes a tuple of arguments that are compatible with ``pandocfilters.walk()`` that + allows identifying hyperlinks in the document and transforms them into valid LaTeX + \\ref{} calls so that linking to headers between cells is possible. + + See the documentation in ``pandocfilters.walk()`` for further information on the meaning + and specification of ``key``, ``val``, ``fmt``, and ``meta``. """ if key == 'Link': From 9f302893d198110b37e08190da5e09842c29dd03 Mon Sep 17 00:00:00 2001 From: michaelpacer Date: Sun, 13 Nov 2016 16:07:06 -0800 Subject: [PATCH 16/16] Use python2 compatible syntax --- nbconvert/exporters/latex.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nbconvert/exporters/latex.py b/nbconvert/exporters/latex.py index d059e7f5b..081ecb459 100644 --- a/nbconvert/exporters/latex.py +++ b/nbconvert/exporters/latex.py @@ -45,7 +45,8 @@ def _template_skeleton_path_default(self): output_mimetype = 'text/latex' def default_filters(self): - yield from super(LatexExporter, self).default_filters() + for x in super(LatexExporter, self).default_filters(): + yield x yield ('resolve_references', resolve_references) @property