diff --git a/execnb/_modidx.py b/execnb/_modidx.py index 0f060ab..11df90b 100644 --- a/execnb/_modidx.py +++ b/execnb/_modidx.py @@ -8,7 +8,7 @@ 'copyright': 'Put your copyright info here', 'custom_sidebar': 'False', 'description': 'A description of your project', - 'dev_requirements': 'matplotlib', + 'dev_requirements': 'matplotlib Pillow', 'doc_baseurl': '/execnb/', 'doc_host': 'https://fastai.github.io', 'doc_path': 'docs', diff --git a/execnb/shell.py b/execnb/shell.py index 32a8144..53b47b4 100644 --- a/execnb/shell.py +++ b/execnb/shell.py @@ -10,6 +10,7 @@ from IPython.core.interactiveshell import InteractiveShell from IPython.core.displayhook import DisplayHook from IPython.core.displaypub import DisplayPublisher +from base64 import b64encode from io import StringIO from .fastshell import FastInteractiveShell @@ -47,9 +48,18 @@ def publish(self, data, metadata=None, **kwargs): self.shell._add_out(data, meta # %% ../nbs/02_shell.ipynb 6 # These are the standard notebook formats for exception and stream data (e.g stdout) def _out_exc(ename, evalue, traceback): return dict(ename=str(ename), evalue=str(evalue), output_type='error', traceback=traceback) -def _out_stream(text): return dict(name='stdout', output_type='stream', text=text.splitlines(False)) - -# %% ../nbs/02_shell.ipynb 8 +def _out_stream(text): return dict(name='stdout', output_type='stream', text=text.splitlines(True)) + +# %% ../nbs/02_shell.ipynb 7 +def _format_mimedata(k, v): + "Format mime-type keyed data consistently with Jupyter" + if k.startswith('text/'): return v.splitlines(True) + if k.startswith('image/') and isinstance(v, bytes): + v = b64encode(v).decode() + return v+'\n' if not v.endswith('\n') else v + return v + +# %% ../nbs/02_shell.ipynb 9 class CaptureShell(FastInteractiveShell): "Execute the IPython/Jupyter source code" def __init__(self, @@ -82,11 +92,12 @@ def _showtraceback(self, etype, evalue, stb: str): self.out.append(_out_exc(etype, evalue, stb)) self.exc = (etype, evalue, '\n'.join(stb)) - def _add_out(self, data, meta, typ='execute_result', **kwargs): self.out.append(dict(data=data, metadata=meta, output_type=typ, **kwargs)) + def _add_out(self, data, meta, typ='execute_result', **kwargs): + fd = {k:_format_mimedata(k,v) for k,v in data.items()} + self.out.append(dict(data=fd, metadata=meta, output_type=typ, **kwargs)) def _add_exec(self, result, meta, typ='execute_result'): - fd = {k:v.splitlines(True) for k,v in result.items()} - self._add_out(fd, meta, execution_count=self.count) + self._add_out(result, meta, execution_count=self.count) self.count += 1 def _result(self, result): @@ -97,7 +108,7 @@ def _stream(self, std): text = std.getvalue() if text: self.out.append(_out_stream(text)) -# %% ../nbs/02_shell.ipynb 11 +# %% ../nbs/02_shell.ipynb 12 @patch def run(self:CaptureShell, code:str, # Python/IPython code to run @@ -115,7 +126,7 @@ def run(self:CaptureShell, self._stream(stdout) return [*self.out] -# %% ../nbs/02_shell.ipynb 22 +# %% ../nbs/02_shell.ipynb 23 @patch def cell(self:CaptureShell, cell, stdout=True, stderr=True): "Run `cell`, skipping if not code, and store outputs back in cell" @@ -127,7 +138,7 @@ def cell(self:CaptureShell, cell, stdout=True, stderr=True): for o in outs: if 'execution_count' in o: cell['execution_count'] = o['execution_count'] -# %% ../nbs/02_shell.ipynb 26 +# %% ../nbs/02_shell.ipynb 27 def _false(o): return False @patch @@ -147,7 +158,7 @@ def run_all(self:CaptureShell, postproc(cell) if self.exc and exc_stop: raise self.exc[1] from None -# %% ../nbs/02_shell.ipynb 40 +# %% ../nbs/02_shell.ipynb 41 @patch def execute(self:CaptureShell, src:str|Path, # Notebook path to read from @@ -168,7 +179,7 @@ def execute(self:CaptureShell, inject_code=inject_code, inject_idx=inject_idx) if dest: write_nb(nb, dest) -# %% ../nbs/02_shell.ipynb 43 +# %% ../nbs/02_shell.ipynb 44 @patch def prettytb(self:CaptureShell, fname:str|Path=None): # filename to print alongside the traceback @@ -180,7 +191,7 @@ def prettytb(self:CaptureShell, fname_str = f' in {fname}' if fname else '' return f"{type(self.exc[1]).__name__}{fname_str}:\n{_fence}\n{cell_str}\n" -# %% ../nbs/02_shell.ipynb 56 +# %% ../nbs/02_shell.ipynb 60 @call_parse def exec_nb( src:str, # Notebook path to read from diff --git a/nbs/02_shell.ipynb b/nbs/02_shell.ipynb index 7e7eb9f..2435f14 100644 --- a/nbs/02_shell.ipynb +++ b/nbs/02_shell.ipynb @@ -35,6 +35,7 @@ "from IPython.core.interactiveshell import InteractiveShell\n", "from IPython.core.displayhook import DisplayHook\n", "from IPython.core.displaypub import DisplayPublisher\n", + "from base64 import b64encode\n", "from io import StringIO\n", "\n", "from execnb.fastshell import FastInteractiveShell\n", @@ -92,7 +93,23 @@ "#|export\n", "# These are the standard notebook formats for exception and stream data (e.g stdout)\n", "def _out_exc(ename, evalue, traceback): return dict(ename=str(ename), evalue=str(evalue), output_type='error', traceback=traceback)\n", - "def _out_stream(text): return dict(name='stdout', output_type='stream', text=text.splitlines(False))" + "def _out_stream(text): return dict(name='stdout', output_type='stream', text=text.splitlines(True))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#|export\n", + "def _format_mimedata(k, v):\n", + " \"Format mime-type keyed data consistently with Jupyter\"\n", + " if k.startswith('text/'): return v.splitlines(True)\n", + " if k.startswith('image/') and isinstance(v, bytes):\n", + " v = b64encode(v).decode()\n", + " return v+'\\n' if not v.endswith('\\n') else v\n", + " return v" ] }, { @@ -141,11 +158,12 @@ " self.out.append(_out_exc(etype, evalue, stb))\n", " self.exc = (etype, evalue, '\\n'.join(stb))\n", "\n", - " def _add_out(self, data, meta, typ='execute_result', **kwargs): self.out.append(dict(data=data, metadata=meta, output_type=typ, **kwargs))\n", + " def _add_out(self, data, meta, typ='execute_result', **kwargs):\n", + " fd = {k:_format_mimedata(k,v) for k,v in data.items()}\n", + " self.out.append(dict(data=fd, metadata=meta, output_type=typ, **kwargs))\n", "\n", " def _add_exec(self, result, meta, typ='execute_result'):\n", - " fd = {k:v.splitlines(True) for k,v in result.items()}\n", - " self._add_out(fd, meta, execution_count=self.count)\n", + " self._add_out(result, meta, execution_count=self.count)\n", " self.count += 1\n", "\n", " def _result(self, result):\n", @@ -206,7 +224,7 @@ { "data": { "text/plain": [ - "[{'name': 'stdout', 'output_type': 'stream', 'text': ['1']}]" + "[{'name': 'stdout', 'output_type': 'stream', 'text': ['1\\n']}]" ] }, "execution_count": null, @@ -239,8 +257,8 @@ " 'execution_count': 1},\n", " {'name': 'stdout',\n", " 'output_type': 'stream',\n", - " 'text': ['CPU times: user 2 us, sys: 1 us, total: 3 us',\n", - " 'Wall time: 4.77 us']}]" + " 'text': ['CPU times: user 3 us, sys: 1 us, total: 4 us\\n',\n", + " 'Wall time: 7.87 us\\n']}]" ] }, "execution_count": null, @@ -428,7 +446,7 @@ " 'metadata': {},\n", " 'output_type': 'execute_result',\n", " 'execution_count': 2},\n", - " {'name': 'stdout', 'output_type': 'stream', 'text': ['1']}]" + " {'name': 'stdout', 'output_type': 'stream', 'text': ['1\\n']}]" ] }, "execution_count": null, @@ -654,7 +672,7 @@ " 'metadata': {},\n", " 'output_type': 'execute_result',\n", " 'execution_count': 10},\n", - " {'name': 'stdout', 'output_type': 'stream', 'text': ['1']}]" + " {'name': 'stdout', 'output_type': 'stream', 'text': ['1\\n']}]" ] }, "execution_count": null, @@ -717,7 +735,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[{'data': {'text/plain': ['2']}, 'execution_count': 2, 'metadata': {}, 'output_type': 'execute_result'}, {'name': 'stdout', 'output_type': 'stream', 'text': ['1']}]\n" + "[{'data': {'text/plain': ['2']}, 'execution_count': 2, 'metadata': {}, 'output_type': 'execute_result'}, {'name': 'stdout', 'output_type': 'stream', 'text': ['1\\n']}]\n" ] } ], @@ -829,6 +847,48 @@ "test_eq(res, [])" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#|hide\n", + "# Streams are split on and keep newlines\n", + "res = CaptureShell().run(r\"print('a\\nb'); print('c\\n\\n'); print('d')\")\n", + "test_eq(res[0]['text'], ['a\\n', 'b\\n', 'c\\n', '\\n', '\\n', 'd\\n'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#|hide\n", + "# Text mime data are split on and keep newlines\n", + "res = CaptureShell().run(r\"from IPython.display import Markdown; display(Markdown('a\\nb'))\")\n", + "test_eq(res[0]['data']['text/markdown'], ['a\\n', 'b'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#|hide\n", + "# Binary image mime data are base64-encoded and end in a single `\\n`\n", + "from PIL import Image\n", + "\n", + "def _pil2b64(im): return b64encode(im._repr_png_()).decode()+'\\n'\n", + "im = Image.new('RGB', (3,3), 'red')\n", + "imb64 = _pil2b64(im)\n", + "\n", + "res = CaptureShell().run(\"from PIL import Image; Image.new('RGB', (3,3), 'red')\")\n", + "test_eq(res[0]['data']['image/png'], imb64)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -888,7 +948,7 @@ " 'execution_count': None,\n", " 'id': 'ea528db5',\n", " 'metadata': {},\n", - " 'outputs': [{'name': 'stdout', 'output_type': 'stream', 'text': ['2']}],\n", + " 'outputs': [{'name': 'stdout', 'output_type': 'stream', 'text': ['2\\n']}],\n", " 'source': 'print(a)',\n", " 'idx_': 1}]" ] diff --git a/settings.ini b/settings.ini index 31df9d3..e4e7653 100644 --- a/settings.ini +++ b/settings.ini @@ -11,7 +11,7 @@ branch = master version = 0.0.6 min_python = 3.7 requirements = fastcore>=1.3.27 ipython -dev_requirements = matplotlib +dev_requirements = matplotlib Pillow console_scripts = exec_nb=execnb.shell:exec_nb audience = Developers language = English