Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Progressively render the template using jinja's generate method #133

Merged
merged 11 commits into from Aug 30, 2019
Merged

Progressively render the template using jinja's generate method #133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
f4d9599
test: make sure the test dont accidently pass
maartenbreddels May 14, 2019
9a2f290
feat(template): allow kernel starting and notebook execution control …
maartenbreddels Aug 7, 2019
adbd1ad
style: support theming for spinner
maartenbreddels May 22, 2019
cee7a72
py27 fix
maartenbreddels Aug 7, 2019
c04df34
feat: progressive cell rendering
maartenbreddels Aug 8, 2019
55316ee
add more docs
maartenbreddels Aug 8, 2019
4c29ed9
subdivide body block to avoid having to call body's super in other te…
maartenbreddels Aug 23, 2019
39dacd1
html fixes
maartenbreddels Aug 23, 2019
a80cb32
yield after writing out and flushing chunk
maartenbreddels Aug 23, 2019
65db8bf
remove console.log
maartenbreddels Aug 28, 2019
4fdf385
fix: filter_empty_code_cells extra argument
maartenbreddels Aug 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
18 changes: 12 additions & 6 deletions share/jupyter/voila/templates/default/nbconvert_templates/base.tpl
View file
Open in desktop
Expand Up @@ -19,12 +19,18 @@
crossorigin="anonymous">
</script>

<script id="jupyter-config-data" type="application/json">
{
"baseUrl": "{{resources.base_url}}",
"kernelId": "{{resources.kernel_id}}"
}
</script>
{% block notebook_execute %}
{%- set kernel_id = kernel_start() -%}
<script id="jupyter-config-data" type="application/json">
{
"baseUrl": "{{resources.base_url}}",
"kernelId": "{{kernel_id}}"
}
</script>
{# from this point on, nb.cells contains output of the executed cells #}
{% do notebook_execute(nb, kernel_id) %}
{%- endblock notebook_execute -%}

{%- endblock html_head_js -%}

{%- block html_head_css -%}
Expand Down
82 changes: 77 additions & 5 deletions share/jupyter/voila/templates/default/nbconvert_templates/voila.tpl
View file
Open in desktop
@@ -1,17 +1,22 @@
{%- extends 'base.tpl' -%}
{% from 'mathjax.tpl' import mathjax %}

{# this overrides the default behaviour of directly starting the kernel and executing the notebook #}
{% block notebook_execute %}
{% endblock notebook_execute %}


{%- block html_head_css -%}
<link rel="stylesheet" type="text/css" href="{{resources.base_url}}voila/static/index.css"></link>
<link rel="stylesheet" type="text/css" href="{{resources.base_url}}voila/static/index.css">

{% if resources.theme == 'dark' %}
<link rel="stylesheet" type="text/css" href="{{resources.base_url}}voila/static/theme-dark.css"></link>
<link rel="stylesheet" type="text/css" href="{{resources.base_url}}voila/static/theme-dark.css">
{% else %}
<link rel="stylesheet" type="text/css" href="{{resources.base_url}}voila/static/theme-light.css"></link>
<link rel="stylesheet" type="text/css" href="{{resources.base_url}}voila/static/theme-light.css">
{% endif %}

{% for css in resources.inlining.css %}
<style type="text/css">
<style>
{{ css }}
</style>
{% endfor %}
Expand All @@ -26,15 +31,82 @@ a.anchor-link {
</style>

{{ mathjax() }}

<!-- voila spinner -->
<style>
#loading {
display: flex;
align-items: center;
justify-content: center;
height: 75vh;
color: var(--jp-content-font-color1);
font-family: sans-serif;
}
</style>
{%- endblock html_head_css -%}

{%- block body -%}
{%- block body_header -%}
{% if resources.theme == 'dark' %}
<body class="jp-Notebook theme-dark" data-base-url="{{resources.base_url}}voila/">
{% else %}
<body class="jp-Notebook theme-light" data-base-url="{{resources.base_url}}voila/">
{% endif %}
{{ super() }}
<div id="loading">
<h2><i class="fa fa-spinner fa-spin" style="font-size:36px;"></i><span id="loading_text">Running {{nb_title}}...</span></h2>
</div>
<script>
var voila_process = function(cell_index, cell_count) {
var el = document.getElementById("loading_text")
el.innerHTML = `Executing ${cell_index} of ${cell_count}`
}
</script>

<div id="rendered_cells" style="display: none">
{%- endblock body_header -%}

{%- block body_loop -%}
{# from this point on, the kernel is started #}
{%- with kernel_id = kernel_start() -%}
<script id="jupyter-config-data" type="application/json">
{
"baseUrl": "{{resources.base_url}}",
"kernelId": "{{kernel_id}}"
}
</script>
{% set cell_count = nb.cells|length %}
{#
Voila is using Jinja's Template.generate method to not render the whole template in one go.
The current implementation of Jinja will however not yield template snippets if we call a blocks' super()
Therefore it is important to have the cell loop in the template.
The issue for Jinja is: https://github.com/pallets/jinja/issues/1044
#}
{%- for cell in cell_generator(nb, kernel_id) -%}
{% set cellloop = loop %}
{%- block any_cell scoped -%}
<script>
voila_process({{ cellloop.index }}, {{ cell_count }})
</script>
{{ super() }}
{%- endblock any_cell -%}
{%- endfor -%}
{% endwith %}
{%- endblock body_loop -%}

{%- block body_footer -%}
</div>

<script type="text/javascript">
(function() {
// remove the loading element
var el = document.getElementById("loading")
el.parentNode.removeChild(el)
// show the cell output
el = document.getElementById("rendered_cells")
el.style.display = 'unset'
})();
</script>
</body>
{%- endblock body_footer -%}
{%- endblock body -%}

2 changes: 1 addition & 1 deletion tests/notebooks/print.ipynb
View file
Open in desktop
Expand Up @@ -6,7 +6,7 @@
"metadata": {},
"outputs": [],
"source": [
"print('Hi Voila!')"
"print('Hi ' +'Voila!')"
]
}
],
Expand Down
35 changes: 35 additions & 0 deletions voila/exporter.py
View file
Open in desktop
Expand Up @@ -15,6 +15,8 @@

from nbconvert.filters.markdown_mistune import IPythonRenderer, MarkdownWithMath
from nbconvert.exporters.html import HTMLExporter
from nbconvert.exporters.templateexporter import TemplateExporter
from nbconvert.filters.highlight import Highlight2HTML


class VoilaMarkdownRenderer(IPythonRenderer):
Expand Down Expand Up @@ -73,3 +75,36 @@ def _default_preprocessors(self):
@traitlets.default('template_file')
def default_template_file(self):
return 'voila.tpl'

def generate_from_notebook_node(self, nb, resources=None, extra_context={}, **kw):
# this replaces from_notebook_node, but calls template.generate instead of template.render
langinfo = nb.metadata.get('language_info', {})
lexer = langinfo.get('pygments_lexer', langinfo.get('name', None))
highlight_code = self.filters.get('highlight_code', Highlight2HTML(pygments_lexer=lexer, parent=self))
self.register_filter('highlight_code', highlight_code)

# NOTE: we don't call HTML or TemplateExporter' from_notebook_node
nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw)
resources.setdefault('raw_mimetypes', self.raw_mimetypes)
resources['global_content_filter'] = {
'include_code': not self.exclude_code_cell,
'include_markdown': not self.exclude_markdown,
'include_raw': not self.exclude_raw,
'include_unknown': not self.exclude_unknown,
'include_input': not self.exclude_input,
'include_output': not self.exclude_output,
'include_input_prompt': not self.exclude_input_prompt,
'include_output_prompt': not self.exclude_output_prompt,
'no_prompt': self.exclude_input_prompt and self.exclude_output_prompt,
}

# Top level variables are passed to the template_exporter here.
for output in self.template.generate(nb=nb_copy, resources=resources, **extra_context):
yield output, resources

@property
def environment(self):
env = super(type(self), self).environment
if 'jinja2.ext.do' not in env.extensions:
env.add_extension('jinja2.ext.do')
return env
82 changes: 61 additions & 21 deletions voila/handler.py
View file
Open in desktop
Expand Up @@ -19,13 +19,24 @@
from .exporter import VoilaExporter


# Filter for empty cells.
def filter_empty_code_cells(cell, exporter):
return (
cell.cell_type != 'code' or # keep non-code cells
(cell.outputs and not exporter.exclude_output) # keep cell if output not excluded and not empty
or not exporter.exclude_input # keep cell if input not excluded
)


class VoilaHandler(JupyterHandler):

def initialize(self, **kwargs):
self.notebook_path = kwargs.pop('notebook_path', []) # should it be []
self.nbconvert_template_paths = kwargs.pop('nbconvert_template_paths', [])
self.traitlet_config = kwargs.pop('config', None)
self.voila_configuration = kwargs['voila_configuration']
# we want to avoid starting multiple kernels due to template mistakes
self.kernel_started = False

@tornado.web.authenticated
@tornado.gen.coroutine
Expand All @@ -50,19 +61,13 @@ def get(self, path=None):
else:
nbextensions = []

notebook = yield self.load_notebook(notebook_path)
if not notebook:
self.notebook = yield self.load_notebook(notebook_path)
if not self.notebook:
return

# Launch kernel and execute notebook
cwd = os.path.dirname(notebook_path)
kernel_id = yield tornado.gen.maybe_future(self.kernel_manager.start_kernel(kernel_name=notebook.metadata.kernelspec.name, path=cwd))
km = self.kernel_manager.get_kernel(kernel_id)
result = executenb(notebook, km=km, cwd=cwd, config=self.traitlet_config)
self.cwd = os.path.dirname(notebook_path)

# render notebook to html
resources = {
'kernel_id': kernel_id,
'base_url': self.base_url,
'nbextensions': nbextensions,
'theme': self.voila_configuration.theme
Expand All @@ -78,30 +83,65 @@ def get(self, path=None):
config=self.traitlet_config,
contents_manager=self.contents_manager # for the image inlining
)

if self.voila_configuration.strip_sources:
exporter.exclude_input = True
exporter.exclude_output_prompt = True
exporter.exclude_input_prompt = True

# Filtering out empty cells.
def filter_empty_code_cells(cell):
return (
cell.cell_type != 'code' or # keep non-code cells
(cell.outputs and not exporter.exclude_output) # keep cell if output not excluded and not empty
or not exporter.exclude_input # keep cell if input not excluded
)
result.cells = list(filter(filter_empty_code_cells, result.cells))

html, resources = exporter.from_notebook_node(result, resources=resources)
# These functions allow the start of a kernel and execution of the notebook after (parts of) the template
# has been rendered and send to the client to allow progressive rendering.
# Template should first call kernel_start, and then decide to use notebook_execute
# or cell_generator to implement progressive cell rendering
extra_context = {
# NOTE: we can remove the lambda is we use jinja's async feature, which will automatically await the future
'kernel_start': lambda: self._jinja_kernel_start().result(), # pass the result (not the future) to the template
'cell_generator': self._jinja_cell_generator,
'notebook_execute': self._jinja_notebook_execute,
}

# Compose reply
self.set_header('Content-Type', 'text/html')
self.write(html)
# render notebook in snippets, and flush them out to the browser can render progresssively
for html_snippet, resources in exporter.generate_from_notebook_node(self.notebook, resources=resources, extra_context=extra_context):
self.write(html_snippet)
self.flush() # we may not want to consider not flusing after each snippet, but add an explicit flush function to the jinja context
yield # give control back to tornado's IO loop, so it can handle static files or other requests
self.flush()

def redirect_to_file(self, path):
self.redirect(url_path_join(self.base_url, 'voila', 'files', path))

@tornado.gen.coroutine
def _jinja_kernel_start(self):
assert not self.kernel_started, "kernel was already started"
# Launch kernel
kernel_id = yield tornado.gen.maybe_future(self.kernel_manager.start_kernel(kernel_name=self.notebook.metadata.kernelspec.name, path=self.cwd))
self.kernel_started = True
raise tornado.gen.Return(kernel_id)

def _jinja_notebook_execute(self, nb, kernel_id):
km = self.kernel_manager.get_kernel(kernel_id)
result = executenb(nb, km=km, cwd=self.cwd, config=self.traitlet_config)
result.cells = list(filter(lambda cell: filter_empty_code_cells(cell, self.exporter), result.cells))
# we modify the notebook in place, since the nb variable cannot be reassigned it seems in jinja2
# e.g. if we do {% with nb = notebook_execute(nb, kernel_id) %}, the base template/blocks will not
# see the updated variable (it seems to be local to our block)
nb.cells = result.cells

def _jinja_cell_generator(self, nb, kernel_id):
"""Generator that will execute a single notebook cell at a time"""
km = self.kernel_manager.get_kernel(kernel_id)

all_cells = list(nb.cells) # copy the cells, since we will modify in place
for cell in all_cells:
# we execute one cell at a time
nb.cells = [cell] # reuse the same notebook
result = executenb(nb, km=km, cwd=self.cwd, config=self.traitlet_config)
cell = result.cells[0] # keep a reference to the executed cell
nb.cells = all_cells # restore notebook in case we access it from the template
# we don't filter empty cells, since we do not know how many empty code cells we will have
yield cell

@tornado.gen.coroutine
def load_notebook(self, path):
model = self.contents_manager.get(path=path)
Expand Down