Merge pull request #330 from maartenbreddels/feat_support_script_files

Feat: support script files

.travis.yml

@@ -18,13 +18,13 @@ before_install:
   - conda config --set always_yes yes --set changeps1 no
   - conda update -q conda
   - conda info -a
-  - conda create -q -n test-environment -c conda-forge python=$PYTHON_VERSION jupyter_server==0.1.0 jupyterlab_pygments==0.1.0 pytest==3.10.1 nbconvert=5.5 pytest-cov nodejs flake8 ipywidgets matplotlib
+  - conda create -q -n test-environment -c conda-forge python=$PYTHON_VERSION jupyter_server==0.1.0 jupyterlab_pygments==0.1.0 pytest==3.10.1 nbconvert=5.5 pytest-cov nodejs flake8 ipywidgets matplotlib xeus-cling
   - source activate test-environment
 install:
   - pip install ".[test]"
   - cd tests/test_template; pip install .; cd ../../;
 before_script:
   - flake8 voila tests setup.py
 script:
-  - VOILA_TEST_DEBUG=1 py.test tests/ --async-test-timeout=20
+  - VOILA_TEST_DEBUG=1 VOILA_TEST_XEUS_CLING=1 py.test tests/ --async-test-timeout=240
   - voila --help  # Making sure we can run `voila --help`

share/jupyter/voila/templates/default/templates/tree.html

@@ -80,7 +80,7 @@
   {% endif %}
 
   {% for content in contents.content %}
-    {% if content.type == 'notebook' %}
+    {% if content.type in ['notebook', 'file'] %}
       <li><a href="{{base_url}}voila/render/{{content.path}}"><i class="fa fa-book"></i>{{content.name}}</a></li>
     {% endif %}
     {% if content.type == 'directory' %}

tests/app/execute_cpp_test.py

@@ -0,0 +1,27 @@
+import os
+import pytest
+
+TEST_XEUS_CLING = os.environ.get('VOILA_TEST_XEUS_CLING', '') == '1'
+
+
+@pytest.fixture
+def cpp_file_url(base_url):
+    return base_url + "/voila/render/print.xcpp"
+
+
+@pytest.fixture
+def voila_args_extra():
+    return ['--VoilaConfiguration.extension_language_mapping={".xcpp": "C++11"}']
+
+
+@pytest.fixture
+def voila_args(notebook_directory, voila_args_extra):
+    return ['--VoilaTest.root_dir=%r' % notebook_directory] + voila_args_extra
+
+
+@pytest.mark.skipif(not TEST_XEUS_CLING, reason='opt in to avoid having to install xeus-cling')
+@pytest.mark.gen_test
+def test_non_existing_kernel(http_client, cpp_file_url):
+    response = yield http_client.fetch(cpp_file_url)
+    assert response.code == 200
+    assert 'Hello voila, from c++' in response.body.decode('utf-8')

tests/app/serve_directory_test.py

@@ -1,15 +1,40 @@
-# test serving a notebook
+# test serving a notebook or python/c++ notebook
+import os
 import pytest
 
+TEST_XEUS_CLING = os.environ.get('VOILA_TEST_XEUS_CLING', '') == '1'
+
 
 @pytest.fixture
 def voila_args(notebook_directory, voila_args_extra):
     return ['--VoilaTest.root_dir=%r' % notebook_directory, '--VoilaTest.log_level=DEBUG'] + voila_args_extra
 
 
 @pytest.mark.gen_test
-def test_hello_world(http_client, print_notebook_url):
+def test_print(http_client, print_notebook_url):
     print(print_notebook_url)
     response = yield http_client.fetch(print_notebook_url)
     assert response.code == 200
     assert 'Hi Voila' in response.body.decode('utf-8')
+
+
+@pytest.fixture
+def voila_args_extra():
+    return ['--VoilaConfiguration.extension_language_mapping={".py": "python"}']
+
+
+@pytest.mark.gen_test
+def test_print_py(http_client, print_notebook_url):
+    print(print_notebook_url)
+    response = yield http_client.fetch(print_notebook_url.replace('ipynb', 'py'))
+    assert response.code == 200
+    assert 'Hi Voila' in response.body.decode('utf-8')
+
+
+@pytest.mark.skipif(not TEST_XEUS_CLING, reason='opt in to avoid having to install xeus-cling')
+@pytest.mark.gen_test
+def test_print_julia_notebook(http_client, print_notebook_url):
+    print(print_notebook_url)
+    response = yield http_client.fetch(print_notebook_url.replace('.ipynb', '_cpp.ipynb'))
+    assert response.code == 200
+    assert 'Hi Voila, from c++' in response.body.decode('utf-8')

tests/app/tree_test.py

@@ -0,0 +1,22 @@
+# test tree rendering
+import pytest
+
+
+@pytest.fixture
+def voila_args(notebook_directory, voila_args_extra):
+    return ['--VoilaTest.root_dir=%r' % notebook_directory, '--VoilaTest.log_level=DEBUG'] + voila_args_extra
+
+
+@pytest.fixture
+def voila_args_extra():
+    return ['--VoilaConfiguration.extension_language_mapping={".xcpp": "C++11"}']
+
+
+@pytest.mark.gen_test
+def test_tree(http_client, base_url):
+    response = yield http_client.fetch(base_url)
+    assert response.code == 200
+    text = response.body.decode('utf-8')
+    assert 'print.ipynb' in text, 'tree handler should render ipynb files'
+    assert 'print.xcpp' in text, 'tree handler should render xcpp files (due to extension_language_mapping)'
+    assert 'print.py' not in text, 'tree handler should not render .py files (due to extension_language_mapping)'

tests/notebooks/print.py

@@ -0,0 +1,15 @@
+# ---
+# jupyter:
+#   jupytext:
+#     text_representation:
+#       extension: .py
+#       format_name: light
+#       format_version: '1.4'
+#       jupytext_version: 1.2.1
+#   kernelspec:
+#     display_name: Python 3
+#     language: python
+#     name: python
+# ---
+
+print('Hi Voila!')

tests/notebooks/print.xcpp

@@ -0,0 +1,23 @@
+// ---
+// jupyter:
+//   jupytext:
+//     text_representation:
+//       extension: .cpp
+//       format_name: light
+//       format_version: '1.4'
+//       jupytext_version: 1.2.1
+//   kernelspec:
+//     display_name: C++11
+//     language: C++11
+//     name: xcpp11
+// ---
+
+
+// this is not a valid .cpp file, since it does not have a main()
+// however, we can ask voila to execute this by using the xeus-cling kernel
+// or relying on jupytext
+
+#include <iostream>
+using namespace std;
+
+cout << "Hello voila, from c++";

tests/notebooks/print_cpp.ipynb

@@ -0,0 +1,39 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#include <iostream>\n",
+    "using namespace std;"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cout << \"Hi Voila, from c++\";"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "C++11",
+   "language": "C++11",
+   "name": "xcpp11"
+  },
+  "language_info": {
+   "codemirror_mode": "text/x-c++src",
+   "file_extension": ".cpp",
+   "mimetype": "text/x-c++src",
+   "name": "c++",
+   "version": "-std=c++11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tests/server/execute_cpp_test.py

@@ -0,0 +1,27 @@
+import os
+import pytest
+
+TEST_XEUS_CLING = os.environ.get('VOILA_TEST_XEUS_CLING', '') == '1'
+
+
+@pytest.fixture
+def cpp_file_url(base_url):
+    return base_url + "/voila/render/print.xcpp"
+
+
+@pytest.fixture
+def jupyter_server_args_extra():
+    return ['--VoilaConfiguration.extension_language_mapping={".xcpp": "C++11"}']
+
+
+@pytest.fixture
+def voila_args(notebook_directory, voila_args_extra):
+    return ['--VoilaTest.root_dir=%r' % notebook_directory] + voila_args_extra
+
+
+@pytest.mark.skipif(not TEST_XEUS_CLING, reason='opt in to avoid having to install xeus-cling')
+@pytest.mark.gen_test
+def test_non_existing_kernel(http_client, cpp_file_url):
+    response = yield http_client.fetch(cpp_file_url)
+    assert response.code == 200
+    assert 'Hello voila, from c++' in response.body.decode('utf-8')

tests/server/tree_test.py

@@ -0,0 +1,22 @@
+# test tree rendering
+import pytest
+
+
+@pytest.fixture
+def voila_args(notebook_directory, voila_args_extra):
+    return ['--VoilaTest.root_dir=%r' % notebook_directory, '--VoilaTest.log_level=DEBUG'] + voila_args_extra
+
+
+@pytest.fixture
+def jupyter_server_args_extra():
+    return ['--VoilaConfiguration.extension_language_mapping={".xcpp": "C++11"}']
+
+
+@pytest.mark.gen_test
+def test_tree(http_client, base_url):
+    response = yield http_client.fetch(base_url+'/voila/tree')
+    assert response.code == 200
+    text = response.body.decode('utf-8')
+    assert 'print.ipynb' in text, 'tree handler should render ipynb files'
+    assert 'print.xcpp' in text, 'tree handler should render xcpp files (due to extension_language_mapping)'
+    assert 'print.py' not in text, 'tree handler should not render .py files (due to extension_language_mapping)'

voila/app.py

@@ -453,6 +453,9 @@ def start(self):
             )
         )
 
+        tree_handler_conf = {
+            'voila_configuration': self.voila_configuration
+        }
         if self.notebook_path:
             handlers.append((
                 url_path_join(self.server_url, r'/(.*)'),
@@ -467,8 +470,8 @@ def start(self):
         else:
             self.log.debug('serving directory: %r', self.root_dir)
             handlers.extend([
-                (self.server_url, VoilaTreeHandler),
-                (url_path_join(self.server_url, r'/voila/tree' + path_regex), VoilaTreeHandler),
+                (self.server_url, VoilaTreeHandler, tree_handler_conf),
+                (url_path_join(self.server_url, r'/voila/tree' + path_regex), VoilaTreeHandler, tree_handler_conf),
                 (url_path_join(self.server_url, r'/voila/render/(.*)'), VoilaHandler,
                     {
                         'nbconvert_template_paths': self.nbconvert_template_paths,

voila/configuration.py

@@ -51,5 +51,21 @@ class VoilaConfiguration(traitlets.config.Configurable):
     Example:
     --VoilaConfiguration.file_whitelist="['.*']" # all files
     --VoilaConfiguration.file_blacklist="['private.*', '.*\.(ipynb)']" # except files in the private dir and notebook files
-    """,
+    """
+    ).tag(config=True)
+
+    language_kernel_mapping = Dict(
+        {},
+        help="""Mapping of language name to kernel name
+        Example mapping python to use xeus-python, and C++11 to use xeus-cling:
+        --VoilaConfiguration.extension_language_mapping='{"python": "xpython", "C++11": "xcpp11"}'
+        """,
+    ).tag(config=True)
+
+    extension_language_mapping = Dict(
+        {},
+        help='''Mapping of file extension to kernel language
+        Example mapping .py files to a python language kernel, and .cpp to a C++11 language kernel:
+        --VoilaConfiguration.extension_language_mapping='{".py": "python", ".cpp": "C++11"}'
+        ''',
     ).tag(config=True)

voila/handler.py

@@ -107,10 +107,15 @@ def load_notebook(self, path):
         model = self.contents_manager.get(path=path)
         if 'content' not in model:
             raise tornado.web.HTTPError(404, 'file not found')
+        __, extension = os.path.splitext(model.get('path', ''))
         if model.get('type') == 'notebook':
             notebook = model['content']
             notebook = yield self.fix_notebook(notebook)
             raise tornado.gen.Return(notebook)  # TODO py2: replace by return
+        elif extension in self.voila_configuration.extension_language_mapping:
+            language = self.voila_configuration.extension_language_mapping[extension]
+            notebook = yield self.create_notebook(model, language=language)
+            raise tornado.gen.Return(notebook)  # TODO py2: replace by return
         else:
             self.redirect_to_file(path)
             raise tornado.gen.Return(None)
@@ -140,17 +145,36 @@ def fix_notebook(self, notebook):
         notebook.metadata.kernelspec.language = all_kernel_specs[kernel_name]['spec']['language']
         raise tornado.gen.Return(notebook)  # TODO py2: replace by return
 
+    @tornado.gen.coroutine
+    def create_notebook(self, model, language):
+        all_kernel_specs = yield tornado.gen.maybe_future(self.kernel_spec_manager.get_all_specs())
+        kernel_name = yield self.find_kernel_name_for_language(language, kernel_specs=all_kernel_specs)
+        spec = all_kernel_specs[kernel_name]
+        notebook = nbformat.v4.new_notebook(
+            metadata={
+                'kernelspec': {
+                    'display_name': spec['spec']['display_name'],
+                    'language': spec['spec']['language'],
+                    'name': kernel_name
+                }
+            },
+            cells=[nbformat.v4.new_code_cell(model['content'])],
+        )
+        raise tornado.gen.Return(notebook)  # TODO py2: replace by return
+
     @tornado.gen.coroutine
     def find_kernel_name_for_language(self, kernel_language, kernel_specs=None):
         """Finds a best matching kernel name given a kernel language.
 
         If multiple kernels matches are found, we try to return the same kernel name each time.
         """
+        if kernel_language in self.voila_configuration.language_kernel_mapping:
+            raise tornado.gen.Return(self.voila_configuration.language_kernel_mapping[kernel_language])  # TODO py2: replace by return
         if kernel_specs is None:
             kernel_specs = yield tornado.gen.maybe_future(self.kernel_spec_manager.get_all_specs())
         matches = [
             name for name, kernel in kernel_specs.items()
-            if kernel["spec"]["language"].lower() == kernel_language
+            if kernel["spec"]["language"].lower() == kernel_language.lower()
         ]
         if matches:
             # Sort by display name to get the same kernel each time.

voila/server_extension.py

@@ -48,14 +48,17 @@ def load_jupyter_server_extension(server_app):
     host_pattern = '.*$'
     base_url = url_path_join(web_app.settings['base_url'])
 
+    tree_handler_conf = {
+        'voila_configuration': voila_configuration
+    }
     web_app.add_handlers(host_pattern, [
         (url_path_join(base_url, '/voila/render/(.*)'), VoilaHandler, {
             'config': server_app.config,
             'nbconvert_template_paths': nbconvert_template_paths,
             'voila_configuration': voila_configuration
         }),
-        (url_path_join(base_url, '/voila'), VoilaTreeHandler),
-        (url_path_join(base_url, '/voila/tree' + path_regex), VoilaTreeHandler),
+        (url_path_join(base_url, '/voila'), VoilaTreeHandler, tree_handler_conf),
+        (url_path_join(base_url, '/voila/tree' + path_regex), VoilaTreeHandler, tree_handler_conf),
         (url_path_join(base_url, '/voila/static/(.*)'), MultiStaticFileHandler, {'paths': static_paths}),
         (
             url_path_join(base_url, r'/voila/files/(.*)'),