nbs/01_read.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|hide\n",
    "#|default_exp read"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# read\n",
    "> Reading a notebook, and initial bootstrapping for notebook exporting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "from datetime import datetime\n",
    "from fastcore.imports import *\n",
    "from fastcore.foundation import *\n",
    "from fastcore.basics import *\n",
    "from fastcore.imports import *\n",
    "from fastcore.script import *\n",
    "from fastcore.xtras import *\n",
    "\n",
    "import ast,functools\n",
    "from execnb.nbio import read_nb,NbCell\n",
    "from pprint import pformat,pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|hide\n",
    "from fastcore.test import *\n",
    "import tempfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def mk_cell(text, cell_type='code'):\n",
    "    \"Create a `NbCell` containing `text`\"\n",
    "    assert cell_type in {'code', 'markdown', 'raw'}\n",
    "    return NbCell(0, dict(cell_type=cell_type, metadata={}, source=text, directives_={}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "def create_output(txt, mime):\n",
    "    \"Add a cell output containing `txt` of the `mime` text MIME sub-type\"\n",
    "    return [{\"data\": { f\"text/{mime}\": str(txt).splitlines(True) },\n",
    "             \"execution_count\": 1, \"metadata\": {}, \"output_type\": \"execute_result\"}]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "nbdev uses a `settings.ini` file in the root of the project to store all configuration details. This file is in `ConfigParser` format, and can be read and written conveniently using fastcore's `Config` class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "@call_parse\n",
    "def nbdev_create_config(\n",
    "    user:str, # Repo username\n",
    "    lib_name:str=None, # Name of library\n",
    "    description='TODO fill me in', # Description for PyPI\n",
    "    author='TODO fill me in', # Author for PyPI\n",
    "    author_email='todo@example.org', # Email for PyPI\n",
    "    path:str='.', # Path to create config file\n",
    "    cfg_name:str='settings.ini', # Name of config file to create\n",
    "    branch:str='master', # Repo branch\n",
    "    host:str='github', # Repo hostname\n",
    "    git_url:str=\"https://github.com/%(user)s/%(lib_name)s/tree/%(branch)s/\", # Repo URL\n",
    "    custom_sidebar:bool_arg=False, # Create custom sidebar?\n",
    "    nbs_path:str='.', # Name of folder containing notebooks\n",
    "    lib_path:str='%(lib_name)s', # Folder name of root module\n",
    "    doc_path:str='_docs', # Folder name containing docs\n",
    "    tst_flags:str='', # Test flags\n",
    "    version:str='0.0.1', # Version number\n",
    "    keywords='python', # Keywords for PyPI\n",
    "    license='apache2', # License for PyPI\n",
    "    copyright='', # Copyright for PyPI, defaults to author from current year\n",
    "    status='3', # Status for PyPI\n",
    "    min_python='3.6', # Minimum python version for PyPI\n",
    "    audience='Developers', # Audience for PyPI\n",
    "    language='English' # Language for PyPI\n",
    "):\n",
    "    \"Create a config file\"\n",
    "    if lib_name is None:\n",
    "        parent = Path.cwd().parent\n",
    "        lib_name = parent.parent.name if parent.name=='nbs' else parent.name\n",
    "    if not copyright: copyright = f'{datetime.now().year} ownwards, {author}'\n",
    "    g = locals()\n",
    "    config = {o:g[o] for o in 'host lib_name user branch nbs_path doc_path \\\n",
    "        description author author_email keywords license tst_flags version custom_sidebar \\\n",
    "        copyright status min_python audience language git_url lib_path'.split()}\n",
    "    save_config_file(Path(path)/cfg_name, config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a wrapper for `fastcore`'s `save_config_file` which sets some `nbdev` defaults. It is also installed as a CLI command."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "@functools.lru_cache(maxsize=None)\n",
    "def get_config(cfg_name='settings.ini', path=None):\n",
    "    \"`Config` for ini file found in `path` (defaults to `cwd`)\"\n",
    "    cfg_path = Path.cwd() if path is None else Path(path)\n",
    "    while cfg_path != cfg_path.parent and not (cfg_path/cfg_name).exists(): cfg_path = cfg_path.parent\n",
    "    return Config(cfg_path, cfg_name=cfg_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`get_config` searches for `settings.ini` in the current directory, and then in all parent directories, stopping when it is found."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nbdev_create_config('fastai', path='..', nbs_path='nbs', tst_flags='tst', cfg_name='test_settings.ini')\n",
    "cfg = get_config('test_settings.ini')\n",
    "test_eq(cfg.lib_name, 'nbdev')\n",
    "test_eq(cfg.git_url, \"https://github.com/fastai/nbdev/tree/master/\")\n",
    "cwd = Path.cwd()\n",
    "test_eq(cfg.config_path, cwd.parent.absolute())\n",
    "test_eq(cfg.path('lib_path'), cwd.parent/'nbdev')\n",
    "test_eq(cfg.path('nbs_path'), cwd)\n",
    "test_eq(cfg.path('doc_path'), cwd.parent/'_docs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "def config_key(c, default=None, path=True, missing_ok=False):\n",
    "    \"Look for key `c` in settings.ini and fail gracefully if not found and no default provided\"\n",
    "    try: cfg = get_config()\n",
    "    except FileNotFoundError:\n",
    "        if missing_ok and default is not None: return default\n",
    "        else: raise ValueError('settings.ini not found')\n",
    "    res = cfg.path(c) if path else cfg.get(c, default=default)\n",
    "    if res is None: raise ValueError(f'`{c}` not specified in settings.ini')\n",
    "    return res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Exporting a basic module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "_init = '__init__.py'\n",
    "\n",
    "def _has_py(fs): return any(1 for f in fs if f.endswith('.py'))\n",
    "\n",
    "def add_init(path):\n",
    "    \"Add `__init__.py` in all subdirs of `path` containing python files if it's not there already\"\n",
    "    # we add the lowest-level `__init__.py` files first, which ensures _has_py succeeds for parent modules\n",
    "    path = Path(path)\n",
    "    path.mkdir(exist_ok=True)\n",
    "    if not (path/_init).exists(): (path/_init).touch()\n",
    "    for r,ds,fs in os.walk(path, topdown=False):\n",
    "        r = Path(r)\n",
    "        subds = (os.listdir(r/d) for d in ds)\n",
    "        if _has_py(fs) or any(filter(_has_py, subds)) and not (r/_init).exists(): (r/_init).touch()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Python modules require a `__init.py__` file in all directories that are modules. We assume that all directories containing a python file (including in subdirectories of any depth) is a module, and therefore add a `__init__.py` to each."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tempfile.TemporaryDirectory() as d:\n",
    "    d = Path(d)\n",
    "    (d/'a/b').mkdir(parents=True)\n",
    "    (d/'a/b/f.py').touch()\n",
    "    (d/'a/c').mkdir()\n",
    "    add_init(d)\n",
    "    assert not (d/'a/c'/_init).exists(), \"Should not add init to dir without py file\"\n",
    "    for e in [d, d/'a', d/'a/b']: assert (e/_init).exists(),f\"Missing init in {e}\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "def write_cells(cells, hdr, file, offset=0):\n",
    "    \"Write `cells` to `file` along with header `hdr` starting at index `offset` (mainly for nbdev internal use)\"\n",
    "    for cell in cells:\n",
    "        if cell.source.strip(): file.write(f'\\n\\n{hdr} {cell.idx_+offset}\\n{cell.source}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "def basic_export_nb(fname, name, dest=None):\n",
    "    \"Basic exporter to bootstrap nbdev\"\n",
    "    if dest is None: dest = config_key('lib_path')\n",
    "    fname,dest = Path(fname),Path(dest)\n",
    "    nb = read_nb(fname)\n",
    "\n",
    "    # grab the source from all the cells that have an `export` comment\n",
    "    cells = L(cell for cell in nb.cells if re.match(r'#\\s*\\|export', cell.source))\n",
    "\n",
    "    # find all the exported functions, to create `__all__`:\n",
    "    trees = cells.map(NbCell.parsed_).concat()\n",
    "    funcs = trees.filter(risinstance((ast.FunctionDef,ast.ClassDef))).attrgot('name')\n",
    "    exp_funcs = [f for f in funcs if f[0]!='_']\n",
    "\n",
    "    # write out the file\n",
    "    with (dest/name).open('w') as f:\n",
    "        f.write(f\"# %% auto 0\\n__all__ = {exp_funcs}\")\n",
    "        write_cells(cells, f\"# %% {fname.relpath(dest)}\", f)\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a simple exporter with just enough functionality to correctly export this notebook, in order to bootstrap the creation of nbdev itself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|hide\n",
    "#| eval: false\n",
    "path = Path('../nbdev')\n",
    "(path/'read.py').unlink(missing_ok=True)\n",
    "\n",
    "add_init(path)\n",
    "basic_export_nb(\"01_read.ipynb\", 'read.py')\n",
    "\n",
    "g = exec_new('from nbdev import read')\n",
    "assert g['read'].add_init\n",
    "assert 'add_init' in g['read'].__all__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}