update

Author: jph00

nbprocess/docs.py

@@ -1,7 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/06_docs.ipynb.
 
 # %% auto 0
-__all__ = ['rm_blank_proc', 'strip_ansi_proc', 'html_escape', 'insert_warning', 'write_md']
+__all__ = ['rm_blank_proc', 'html_escape', 'insert_warning', 'write_md']
 
 # %% ../nbs/06_docs.ipynb 3
 from .read import *
@@ -11,35 +11,25 @@
 
 from fastcore.script import *
 from fastcore.imports import *
+from fastcore.basics import *
 from fastcore.xtras import *
 
 import uuid
 import tempfile
 
-# %% ../nbs/06_docs.ipynb 5
+# %% ../nbs/06_docs.ipynb 6
 def rm_blank_proc(cell):
     "Remove empty cells"
     if(cell.source.strip()==''): cell.source = None
 
-# %% ../nbs/06_docs.ipynb 6
-_re_ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
-
-def strip_ansi_proc(cell):
-    "Strip Ansi Characters"
-    for o in cell.get('outputs', []):
-        if o.get('name') == 'stdout': o['text'] = _re_ansi_escape.sub('', o.text)
-
 # %% ../nbs/06_docs.ipynb 7
-def html_escape(cell):
+@outp_proc('data', 'text/html')
+def html_escape(cell, outp, item):
     "Place HTML in a codeblock and surround it with a <HTMLOutputBlock> component."
-    for o in cell.get('outputs', []):
-        html = nested_idx(o, 'data', 'text/html')
-        if html:
-            cell.metadata.html_output = True
-            html = ''.join(html).strip()
-            o['data']['text/html'] = f'```html\n{html}\n```'
+    cell.metadata.html_output = True
+    return f'```html\n{item}\n```'
 
-# %% ../nbs/06_docs.ipynb 8
+# %% ../nbs/06_docs.ipynb 10
 def _get_cell_id(id_length=36): return uuid.uuid4().hex[:id_length]
 
 def _get_md_cell(content="<!--- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT!-->"):
@@ -50,10 +40,11 @@ def insert_warning(nb):
     "Insert Autogenerated Warning Into Notebook after the first cell."
     nb.cells = nb.cells[:1] + [_get_md_cell()] + nb.cells[1:]
 
-# %% ../nbs/06_docs.ipynb 10
-def write_md(nb_path, procs=None, post_procs=None, tpl_file='ob.tpl'):
-    nbp = NBProcessor(nb_path, procs)
+# %% ../nbs/06_docs.ipynb 11
+def write_md(nb_path, procs=None, post_procs=None, outp_procs=None, pre_procs=None, tpl_file='ob.tpl'):
+    nbp = NBProcessor(nb_path, procs, outp_procs=outp_procs)
     nb = nbp.nb
+    for proc in L(pre_procs): proc(nb)
     nbp.process()
     for proc in L(post_procs): proc(nb)
 

nbprocess/export.py

@@ -1,7 +1,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_export.ipynb.
 
 # %% auto 0
-__all__ = ['extract_comments', 'NBProcessor', 'ExportModuleProc', 'rm_comments_proc', 'create_modules', 'nb_export', 'nbs_export']
+__all__ = ['extract_comments', 'NBProcessor', 'outp_proc', 'strip_ansi', 'ExportModuleProc', 'rm_comments_proc', 'create_modules',
+           'nb_export', 'nbs_export']
 
 # %% ../nbs/02_export.ipynb 3
 from .read import *
@@ -36,31 +37,51 @@ def _param_count(f):
 # %% ../nbs/02_export.ipynb 12
 class NBProcessor:
     "Process cells and nbdev comments in a notebook"
-    def __init__(self, path=None, procs=None, nb=None, debug=False):
+    def __init__(self, path=None, procs=None, outp_procs=None, nb=None, debug=False):
         self.nb = read_nb(path) if nb is None else nb
-        self.procs,self.debug = L(procs),debug
+        self.procs,self.outp_procs,self.debug = L(procs),L(outp_procs),debug
 
     def _process_cell(self, cell):
         self.cell = cell
         cell._comments = extract_comments(cell.source)
         for proc in self.procs:
             if callable(proc): proc(cell)
-            if cell.cell_type=='code':
-                for comment in cell._comments: self._process_comment(proc, comment)
+            if cell.cell_type=='code': cell._comments.map(self._process_comment,proc)
+        for proc in self.outp_procs: L(cell.get('outputs', [])).map(proc, cell)
 
     def _process_comment(self, proc, comment):
         cmd,*args = comment
         f = getattr(proc, f'_{cmd}_', None)
         if not f or _param_count(f)-1<len(args): return True
         if self.debug: print(cmd, args, f)
         return f(self, *args)
-
+        
     def process(self):
         "Process all cells with `process_cell`"
         for i in range_of(self.nb.cells): self._process_cell(self.nb.cells[i])
         self.nb.cells = [c for c in self.nb.cells if c.source is not None]
 
 # %% ../nbs/02_export.ipynb 16
+class outp_proc:
+    "Decorator for funcs to pass to `NBProcessor` for processing cell output"
+    def __init__(self, *idxs): self.idxs = idxs
+    def __call__(self, func):
+        def _inner(cell, outp):
+            item = nested_idx(outp, *self.idxs)
+            if not item: return
+            res = func(cell, outp, ''.join(item))
+            if res is not None: set_nested_idx(outp, res.splitlines(True), *self.idxs)
+        return _inner
+
+# %% ../nbs/02_export.ipynb 17
+_re_ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+
+@outp_proc('text')
+def strip_ansi(cell, outp, item):
+    "Output processor to strip Ansi Characters"
+    if outp.get('name') == 'stdout': return _re_ansi_escape.sub('', item)
+
+# %% ../nbs/02_export.ipynb 20
 class ExportModuleProc:
     "A processor which exports code to a module"
     def __init__(self): self.modules,self.in_all = defaultdict(L),defaultdict(L)
@@ -70,12 +91,12 @@ def _export_(self, nbp, exp_to=None):
         self._exporti_(nbp, exp_to)
         self.in_all[ifnone(exp_to, '#')].append(nbp.cell)
 
-# %% ../nbs/02_export.ipynb 19
+# %% ../nbs/02_export.ipynb 23
 def rm_comments_proc(cell):
     "A proc that removes comments from each NB cell source"
     cell.source = ''.join(cell.source.splitlines(True)[len(cell._comments):])
 
-# %% ../nbs/02_export.ipynb 20
+# %% ../nbs/02_export.ipynb 24
 def create_modules(path, dest, procs=None, debug=False, mod_maker=ModuleMaker):
     "Create module(s) from notebook"
     exp = ExportModuleProc()
@@ -87,12 +108,12 @@ def create_modules(path, dest, procs=None, debug=False, mod_maker=ModuleMaker):
         mm = mod_maker(dest=dest, name=name, nb_path=path, is_new=mod=='#')
         mm.make(cells, all_cells)
 
-# %% ../nbs/02_export.ipynb 27
+# %% ../nbs/02_export.ipynb 31
 def nb_export(nbname, lib_name=None):
     if lib_name is None: lib_name = get_config().lib_name
     create_modules(nbname, lib_name)
 
-# %% ../nbs/02_export.ipynb 28
+# %% ../nbs/02_export.ipynb 32
 @call_parse
 def nbs_export(
     path:str='.', # path or filename

nbprocess/read.py

@@ -20,8 +20,6 @@ def __init__(self, idx, cell):
         super().__init__(cell)
         self.idx_ = idx
         if 'source' in self: self.set_source(self.source)
-        for o in cell.get('outputs', []):
-            if 'text' in o: o['text'] = ''.join(o.text)
 
     def __repr__(self): return self.source
 

nbprocess/sync.py

@@ -17,7 +17,7 @@
 # %% ../nbs/03_sync.ipynb 5
 def nb2dict(d, k=None):
     "Convert parsed notebook to `dict`"
-    if k in ('source','text'): return d.splitlines(keepends=True)
+    if k in ('source',): return d.splitlines(keepends=True)
     if isinstance(d, (L,list)): return list(L(d).map(nb2dict))
     if not isinstance(d, dict): return d
     return dict(**{k:nb2dict(v,k) for k,v in d.items() if k[-1] != '_'})

nbs/00_read.ipynb

@@ -184,8 +184,6 @@
     "        super().__init__(cell)\n",
     "        self.idx_ = idx\n",
     "        if 'source' in self: self.set_source(self.source)\n",
-    "        for o in cell.get('outputs', []):\n",
-    "            if 'text' in o: o['text'] = ''.join(o.text)\n",
     "\n",
     "    def __repr__(self): return self.source\n",
     "\n",

nbs/02_export.ipynb

@@ -126,32 +126,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
     "#export\n",
     "class NBProcessor:\n",
     "    \"Process cells and nbdev comments in a notebook\"\n",
-    "    def __init__(self, path=None, procs=None, nb=None, debug=False):\n",
+    "    def __init__(self, path=None, procs=None, outp_procs=None, nb=None, debug=False):\n",
     "        self.nb = read_nb(path) if nb is None else nb\n",
-    "        self.procs,self.debug = L(procs),debug\n",
+    "        self.procs,self.outp_procs,self.debug = L(procs),L(outp_procs),debug\n",
     "\n",
     "    def _process_cell(self, cell):\n",
     "        self.cell = cell\n",
     "        cell._comments = extract_comments(cell.source)\n",
     "        for proc in self.procs:\n",
     "            if callable(proc): proc(cell)\n",
-    "            if cell.cell_type=='code':\n",
-    "                for comment in cell._comments: self._process_comment(proc, comment)\n",
+    "            if cell.cell_type=='code': cell._comments.map(self._process_comment,proc)\n",
+    "        for proc in self.outp_procs: L(cell.get('outputs', [])).map(proc, cell)\n",
     "\n",
     "    def _process_comment(self, proc, comment):\n",
     "        cmd,*args = comment\n",
     "        f = getattr(proc, f'_{cmd}_', None)\n",
     "        if not f or _param_count(f)-1<len(args): return True\n",
     "        if self.debug: print(cmd, args, f)\n",
     "        return f(self, *args)\n",
-    "\n",
+    "        \n",
     "    def process(self):\n",
     "        \"Process all cells with `process_cell`\"\n",
     "        for i in range_of(self.nb.cells): self._process_cell(self.nb.cells[i])\n",
@@ -160,7 +160,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -170,7 +170,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -187,6 +187,61 @@
     "proc.process()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Output processors -"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#export\n",
+    "class outp_proc:\n",
+    "    \"Decorator for funcs to pass to `NBProcessor` for processing cell output\"\n",
+    "    def __init__(self, *idxs): self.idxs = idxs\n",
+    "    def __call__(self, func):\n",
+    "        def _inner(cell, outp):\n",
+    "            item = nested_idx(outp, *self.idxs)\n",
+    "            if not item: return\n",
+    "            res = func(cell, outp, ''.join(item))\n",
+    "            if res is not None: set_nested_idx(outp, res.splitlines(True), *self.idxs)\n",
+    "        return _inner"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#export\n",
+    "_re_ansi_escape = re.compile(r'\\x1B(?:[@-Z\\\\-_]|\\[[0-?]*[ -/]*[@-~])')\n",
+    "\n",
+    "@outp_proc('text')\n",
+    "def strip_ansi(cell, outp, item):\n",
+    "    \"Output processor to strip Ansi Characters\"\n",
+    "    if outp.get('name') == 'stdout': return _re_ansi_escape.sub('', item)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "proc = NBProcessor(everything_fn, outp_procs=strip_ansi)\n",
+    "proc.process()\n",
+    "\n",
+    "def _valid(o): return not o or not _re_ansi_escape.search(o)\n",
+    "for cell in proc.nb.cells:\n",
+    "    assert all(_valid(nested_idx(o, 'text', 0)) for o in cell.get('outputs', []))        "
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -196,7 +251,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -222,7 +277,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -236,7 +291,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -248,7 +303,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -274,7 +329,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -295,7 +350,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -317,7 +372,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 23,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -329,7 +384,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 24,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -341,7 +396,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 25,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -374,7 +429,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 26,
    "metadata": {},
    "outputs": [],
    "source": [