Move python to jupyter CLI inside notebook file

This allows for testing directly on the python module responsible for
the rendering of the notebooks.

bin/sphx_glr_python_to_jupyter.py

@@ -12,27 +12,9 @@
 # License: 3-clause BSD
 
 from __future__ import division, absolute_import, print_function
-import argparse
-import os
 
-import sphinx_gallery.gen_rst as sg
-from sphinx_gallery.notebook import jupyter_notebook, save_notebook
-
-parser = argparse.ArgumentParser(
-    description='Sphinx-Gallery Notebook converter')
-parser.add_argument('python_src_file', nargs='+',
-                    help='Input Python file script to convert. '
-                    'Supports multiple files and shell wildcards'
-                    ' (e.g. *.py)')
-
-
-def main(args):
-    for src_file in args.python_src_file:
-        blocks = sg.split_code_and_text_blocks(src_file)
-        print('Converting {0}'.format(src_file))
-        example_nb = jupyter_notebook(blocks)
-        save_notebook(example_nb, src_file.replace('.py', '.ipynb'))
+from sphinx_gallery.notebook import python_to_jupyter_cli
 
 
 if __name__ == '__main__':
-    main(parser.parse_args())
+    python_to_jupyter_cli()

doc/reference.rst

@@ -12,5 +12,6 @@ Sphinx-Gallery API Reference
    gen_gallery
    backreferences
    gen_rst
+   py_source_parser
    docs_resolv
    downloads

sphinx_gallery/gen_rst.py

@@ -16,7 +16,6 @@
 # tricky errors come up with exec(code_blocks, ...) calls
 from __future__ import division, print_function, absolute_import
 from time import time
-import ast
 import codecs
 import hashlib
 import os
@@ -29,7 +28,6 @@
 
 
 # Try Python 2 first, otherwise load from Python 3
-from textwrap import dedent
 try:
     # textwrap indent only exists in python 3
     from textwrap import indent
@@ -82,6 +80,8 @@ def prefixed_lines():
 from . import glr_path_static
 from .backreferences import write_backreferences, _thumbnail_div
 from .downloads import CODE_DOWNLOAD
+from .py_source_parser import get_docstring_and_rest, split_code_and_text_blocks
+
 from .notebook import jupyter_notebook, text2string, save_notebook
 
 try:
@@ -158,80 +158,6 @@ def write(self, data):
     `Generated by Sphinx-Gallery <http://sphinx-gallery.readthedocs.io>`_\n"""
 
 
-def get_docstring_and_rest(filename):
-    """Separate `filename` content between docstring and the rest
-
-    Strongly inspired from ast.get_docstring.
-
-    Returns
-    -------
-    docstring: str
-        docstring of `filename`
-    rest: str
-        `filename` content without the docstring
-    """
-    # can't use codecs.open(filename, 'r', 'utf-8') here b/c ast doesn't
-    # seem to work with unicode strings in Python2.7
-    # "SyntaxError: encoding declaration in Unicode string"
-    with open(filename, 'rb') as f:
-        content = f.read()
-
-    node = ast.parse(content)
-    if not isinstance(node, ast.Module):
-        raise TypeError("This function only supports modules. "
-                        "You provided {0}".format(node.__class__.__name__))
-    if node.body and isinstance(node.body[0], ast.Expr) and \
-       isinstance(node.body[0].value, ast.Str):
-        docstring_node = node.body[0]
-        docstring = docstring_node.value.s
-        if hasattr(docstring, 'decode'):  # python2.7
-            docstring = docstring.decode('utf-8')
-        # This get the content of the file after the docstring last line
-        # Note: 'maxsplit' argument is not a keyword argument in python2
-        rest = content.decode('utf-8').split('\n', docstring_node.lineno)[-1]
-        return docstring, rest
-    else:
-        raise ValueError(('Could not find docstring in file "{0}". '
-                          'A docstring is required by sphinx-gallery')
-                         .format(filename))
-
-
-def split_code_and_text_blocks(source_file):
-    """Return list with source file separated into code and text blocks.
-
-    Returns
-    -------
-    blocks : list of (label, content)
-        List where each element is a tuple with the label ('text' or 'code'),
-        and content string of block.
-    """
-    docstring, rest_of_content = get_docstring_and_rest(source_file)
-    blocks = [('text', docstring)]
-
-    pattern = re.compile(
-        r'(?P<header_line>^#{20,}.*)\s(?P<text_content>(?:^#.*\s)*)',
-        flags=re.M)
-
-    pos_so_far = 0
-    for match in re.finditer(pattern, rest_of_content):
-        match_start_pos, match_end_pos = match.span()
-        code_block_content = rest_of_content[pos_so_far:match_start_pos]
-        text_content = match.group('text_content')
-        sub_pat = re.compile('^#', flags=re.M)
-        text_block_content = dedent(re.sub(sub_pat, '', text_content)).lstrip()
-        if code_block_content.strip():
-            blocks.append(('code', code_block_content))
-        if text_block_content.strip():
-            blocks.append(('text', text_block_content))
-        pos_so_far = match_end_pos
-
-    remaining_content = rest_of_content[pos_so_far:]
-    if remaining_content.strip():
-        blocks.append(('code', remaining_content))
-
-    return blocks
-
-
 def codestr2rst(codestr, lang='python'):
     """Return reStructuredText code block from code string"""
     code_directive = "\n.. code-block:: {0}\n\n".format(lang)

sphinx_gallery/notebook.py

@@ -12,9 +12,11 @@
 
 from __future__ import division, absolute_import, print_function
 from functools import partial
+import argparse
 import json
 import re
 import sys
+from .py_source_parser import split_code_and_text_blocks
 
 
 def text2string(content):
@@ -175,3 +177,26 @@ def save_notebook(work_notebook, write_file):
     """Saves the Jupyter work_notebook to write_file"""
     with open(write_file, 'w') as out_nb:
         json.dump(work_notebook, out_nb, indent=2)
+
+
+###############################################################################
+# Notebook shell utility
+
+def python_to_jupyter_cli(args=None, namespace=None):
+    """Exposes the jupyter notebook renderer to the command line
+
+    Takes the same arguments as ArgumentParser.parse_args
+    """
+    parser = argparse.ArgumentParser(
+        description='Sphinx-Gallery Notebook converter')
+    parser.add_argument('python_src_file', nargs='+',
+                        help='Input Python file script to convert. '
+                        'Supports multiple files and shell wildcards'
+                        ' (e.g. *.py)')
+    args = parser.parse_args(args, namespace)
+
+    for src_file in args.python_src_file:
+        blocks = split_code_and_text_blocks(src_file)
+        print('Converting {0}'.format(src_file))
+        example_nb = jupyter_notebook(blocks)
+        save_notebook(example_nb, src_file.replace('.py', '.ipynb'))

sphinx_gallery/py_source_parser.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+r"""
+Parser for python source files
+==============================
+"""
+# Created Sun Nov 27 14:03:07 2016
+# Author: Óscar Nájera
+
+from __future__ import division, absolute_import, print_function
+import ast
+import re
+from textwrap import dedent
+
+
+def get_docstring_and_rest(filename):
+    """Separate `filename` content between docstring and the rest
+
+    Strongly inspired from ast.get_docstring.
+
+    Returns
+    -------
+    docstring: str
+        docstring of `filename`
+    rest: str
+        `filename` content without the docstring
+    """
+    # can't use codecs.open(filename, 'r', 'utf-8') here b/c ast doesn't
+    # seem to work with unicode strings in Python2.7
+    # "SyntaxError: encoding declaration in Unicode string"
+    with open(filename, 'rb') as f:
+        content = f.read()
+
+    node = ast.parse(content)
+    if not isinstance(node, ast.Module):
+        raise TypeError("This function only supports modules. "
+                        "You provided {0}".format(node.__class__.__name__))
+    if node.body and isinstance(node.body[0], ast.Expr) and \
+       isinstance(node.body[0].value, ast.Str):
+        docstring_node = node.body[0]
+        docstring = docstring_node.value.s
+        if hasattr(docstring, 'decode'):  # python2.7
+            docstring = docstring.decode('utf-8')
+        # This get the content of the file after the docstring last line
+        # Note: 'maxsplit' argument is not a keyword argument in python2
+        rest = content.decode('utf-8').split('\n', docstring_node.lineno)[-1]
+        return docstring, rest
+    else:
+        raise ValueError(('Could not find docstring in file "{0}". '
+                          'A docstring is required by sphinx-gallery')
+                         .format(filename))
+
+
+def split_code_and_text_blocks(source_file):
+    """Return list with source file separated into code and text blocks.
+
+    Returns
+    -------
+    blocks : list of (label, content)
+        List where each element is a tuple with the label ('text' or 'code'),
+        and content string of block.
+    """
+    docstring, rest_of_content = get_docstring_and_rest(source_file)
+    blocks = [('text', docstring)]
+
+    pattern = re.compile(
+        r'(?P<header_line>^#{20,}.*)\s(?P<text_content>(?:^#.*\s)*)',
+        flags=re.M)
+
+    pos_so_far = 0
+    for match in re.finditer(pattern, rest_of_content):
+        match_start_pos, match_end_pos = match.span()
+        code_block_content = rest_of_content[pos_so_far:match_start_pos]
+        text_content = match.group('text_content')
+        sub_pat = re.compile('^#', flags=re.M)
+        text_block_content = dedent(re.sub(sub_pat, '', text_content)).lstrip()
+        if code_block_content.strip():
+            blocks.append(('code', code_block_content))
+        if text_block_content.strip():
+            blocks.append(('text', text_block_content))
+        pos_so_far = match_end_pos
+
+    remaining_content = rest_of_content[pos_so_far:]
+    if remaining_content.strip():
+        blocks.append(('code', remaining_content))
+
+    return blocks

sphinx_gallery/tests/test_notebook.py

@@ -8,9 +8,16 @@
 from __future__ import division, absolute_import, print_function
 import json
 import tempfile
+import os
+from unittest import TestCase
 from nose.tools import assert_equal
 import sphinx_gallery.gen_rst as sg
-from sphinx_gallery.notebook import rst2md, jupyter_notebook, save_notebook
+from sphinx_gallery.notebook import rst2md, jupyter_notebook, save_notebook, python_to_jupyter_cli
+try:
+    FileNotFoundError
+except NameError:
+    # Python2
+    FileNotFoundError = IOError
 
 
 def test_latex_conversion():
@@ -79,3 +86,28 @@ def test_jupyter_notebook():
         save_notebook(example_nb, nb_file.name)
         with open(nb_file.name, "r") as fname:
             assert_equal(json.load(fname), example_nb)
+
+###############################################################################
+# Notebook shell utility
+
+
+class CommandLineTest(TestCase):
+    """Test the Sphinx-Gallery python to Jupyter notebook converter CLI"""
+
+    def test_with_empty_args(self):
+        """ User passes no args, should fail with SystemExit """
+        with self.assertRaises(SystemExit):
+            python_to_jupyter_cli([])
+
+    def test_missing_file(self):
+        """ User passes non existing file, should fail with FileNotFoundError """
+        with self.assertRaises(FileNotFoundError):
+            python_to_jupyter_cli(['nofile.py'])
+
+
+def test_file_is_generated():
+    """User passes good python file. Check notebook file is created"""
+
+    python_to_jupyter_cli(['examples/plot_quantum.py'])
+    assert os.path.isfile('examples/plot_quantum.ipynb')
+    os.remove('examples/plot_quantum.ipynb')