♻️ Re-design package and introduce large parts of the API (#90)

* notebook read write and dependencies

* add dev

* add meta api

* add integrity

* add repr to meta

* title check in nb

* fix_title

* restructure store

* lazily load infer_dependencies

* add initialization function

* add docstrings

* change to google style

* upd docs

* autosummary

* Clear cache

* remove meta from docs

* Add meta back to autosummary

* 🚧 add a regular import to test

* avoid loading meta on import

* without metadata class

* Suggestion on using autoclass instead of autosummary

* renamings in meta

* fix docs

* further edits

* add write to meta

Co-authored-by: Alex Wolf <f.alexander.wolf@gmail.com>

.github/workflows/build.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Cache
         uses: actions/cache@v3
         env:
-          cache-name: cache-0
+          cache-name: cache-1
         with:
           path: |
             .nox

docs/guides/example-after-init-without-title.ipynb

@@ -65,6 +65,16 @@
    "source": [
     "nbproject.meta"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e5d5dd20",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "meta.live.title"
+   ]
   }
  ],
  "metadata": {

docs/guides/example-after-init.ipynb

@@ -66,7 +66,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "meta.id"
+    "meta.store.id"
    ]
   },
   {
@@ -80,8 +80,8 @@
    },
    "outputs": [],
    "source": [
-    "assert meta.id == \"z14KWQKD4bwE\"\n",
-    "assert hasattr(meta, \"time_init\")"
+    "assert meta.store.id == \"z14KWQKD4bwE\"\n",
+    "assert hasattr(meta.store, \"time_init\")"
    ]
   },
   {
@@ -91,7 +91,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "meta.title"
+    "meta.live.title"
    ]
   }
  ],
@@ -114,7 +114,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.7"
+   "version": "3.8.13"
   },
   "nbproject": {
    "id": "z14KWQKD4bwE",

docs/index.md

@@ -27,11 +27,21 @@ If you want more configuration, call the class
 
 ```
 from nbproject import Header
-header = Header(*args, **kwargs, show=False)
-header.infer_dependencies()
-header.add_dependency("pytorch")
-header.check_for_integrity()
-header.show()
+header = Header(*args, **kwargs)
+```
+
+If you want to access the nbproject metadata
+
+```
+from nbproject import meta
+
+meta.store
+
+meta.live.dependency
+meta.live.title
+meta.live.integrity
+meta.live.time_run
+meta.live.time_passed
 ```
 
 For more functionality, check out the [guides](guides/index)! A comprehensive API documentation is to come.

nbproject/__init__.py

@@ -7,44 +7,67 @@
 Access `nbproject` metadata through the API::
 
    from nbproject import meta
+   meta.store
+   meta.live.dependency
+   meta.live.title
+   meta.live.integrity
+   meta.live.time_run
+   meta.live.time_passed
+
+You can access developer functions via `nbproject.dev`.
 
 Display with configurable arguments & update::
 
    from nbproject import Header
    header = Header(*args, **kwargs)
-   header.infer_dependencies()
 
-The API consists of a single class `Header`.
+The one-liner `from nbproject import header` offers a mere shortcut for
+initializing `Header` with default arguments.
 
-.. autosummary::
-   :toctree: .
+.. autoclass:: Header
+   :members:
+   :undoc-members:
 
-   Header
+For more detailed control, we offer an instance of `Meta` via `nbproject.meta`.
 
-The one-liner `from nbproject import header` offers a mere shortcut for
-initializing `Header` with default arguments!
-"""
-__version__ = "0.0.9"
+.. autoclass:: Meta
+   :members:
+   :undoc-members:
 
-import sys
-from types import ModuleType
+The `nbproject.meta.live` gives access to live metadata of the current notebook
+and is an instance of
 
-from ._header import Header  # noqa
+.. autoclass:: MetaLive
+   :members:
+   :undoc-members:
 
-_module = sys.modules[__name__]
+The `nbproject.meta.store` stores nbproject metadata from the current notebook file
+and is an instance of
 
+.. autoclass:: MetaStore
+   :members:
+   :undoc-members:
+"""
+__version__ = "0.0.9"
+
+from ._header import Header  # noqa
+from ._meta import Meta, MetaLive, MetaStore
 
-class LazyMeta(ModuleType):
-    _meta = None
+_meta = None
+# see this for context: https://stackoverflow.com/questions/880530
+def __getattr__(name):  # user experience is that of a property on a class!
+    global _meta
 
-    @property
-    def meta(self):
-        if self._meta is None:
+    if name == "meta":
+        if _meta is None:
             from ._meta import _load_meta
 
-            self._meta = _load_meta()
-
+            _meta = _load_meta()
         return _meta
 
+    if name == "dev":
+        from ._dev import init_dev
+
+        return init_dev
 
-_module.__class__ = LazyMeta
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")

nbproject/_dependency.py → nbproject/_dev/_dependency.py

@@ -7,6 +7,8 @@
 import packaging
 from importlib_metadata import PackageNotFoundError, packages_distributions, version
 
+from ._notebook import Notebook
+
 major, minor = sys.version_info[0], sys.version_info[1]
 if major == 3 and minor > 9:
     std_libs = sys.stdlib_module_names  # type: ignore
@@ -34,10 +36,20 @@ def cell_imports(cell_source: str):
                 yield name
 
 
-def notebook_deps(content: Union[dict, list], pin_versions: bool = False):
-    # parse the notebook content and infer all dependencies
-    if isinstance(content, dict) and "cells" in content:
-        cells = content["cells"]
+def infer_dependencies(content: Union[Notebook, list], pin_versions: bool = True):
+    """Parse the notebook content and infer all dependencies.
+
+    Args:
+        nb: A notebook or a list of cells to parse for dependencies.
+        pin_versions: If `True`, fixes versions from the current environment.
+
+    Examples:
+        >>> dependencies = nbproject.dev.infer_dependencies(nb)
+        >>> dependencies
+        {"scanpy": "1.8.7", "pandas": "1.4.3"}
+    """
+    if isinstance(content, Notebook):
+        cells = content.cells
     elif isinstance(content, list) and len(content) > 0 and "cell_type" in content[0]:
         cells = content
     else:
@@ -86,6 +98,7 @@ def notebook_deps(content: Union[dict, list], pin_versions: bool = False):
 def resolve_versions(
     notebooks_pkgs: List[dict], strategy: Literal["older", "newer"] = "newer"
 ):
+    """Harmonize packages' versions from lists of packages."""
     parse_version = packaging.version.parse
 
     if strategy == "newer":

nbproject/_dev/_initialize.py

@@ -0,0 +1,45 @@
+import secrets
+import string
+from datetime import datetime, timezone
+from typing import Mapping, Optional
+
+from pydantic import BaseModel, Extra
+
+from ._notebook import Notebook
+
+
+class MetaStore(BaseModel):
+    """The metadata stored in the notebook file."""
+
+    id: str
+    time_init: str
+    dependency: Optional[Mapping[str, str]] = None
+
+    class Config:  # noqa
+        extra = Extra.allow
+
+
+def nbproject_id():  # rename to nbproject_id also in metadata slot?
+    """An 8-byte ID encoded as a 12-character base62 string."""
+    # https://github.com/laminlabs/notes/blob/main/2022-04-04-human-friendly-ids.ipynb
+    base62 = string.digits + string.ascii_letters.swapcase()
+    id = "".join(secrets.choice(base62) for i in range(12))
+    return id
+
+
+def initialize_metadata(nb: Optional[Notebook]) -> MetaStore:
+    """Initialize nbproject metadata.
+
+    Args:
+        nb: If a notebook is provided, also infer dependencies from the notebook.
+    """
+    meta = MetaStore(
+        id=nbproject_id(), time_init=datetime.now(timezone.utc).isoformat()
+    )
+
+    if nb is not None:
+        from ._dependency import infer_dependencies
+
+        meta.dependency = infer_dependencies(nb, pin_versions=True)
+
+    return meta

nbproject/_dev/_integrity.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from ._notebook import Notebook
+
+
+def check_integrity(nb: Notebook, ignore_code: Optional[str] = None) -> bool:
+    """Get current integrity status of the passed notebook.
+
+    For `True` the code cells of the notebook must be executed consequently, i.e.
+    execution count for each code cell should increase by one.
+
+    Args:
+        nb: The notebook to check.
+        ignore_code: Ignore all cells which contain this code.
+    """
+    cells = nb.cells
+
+    integrity = True
+    prev = 0
+
+    for cell in cells:
+        if cell["cell_type"] != "code" or cell["source"] == []:
+            continue
+
+        if ignore_code is not None and ignore_code in "".join(cell["source"]):
+            continue
+
+        ccount = cell["execution_count"]
+        if ccount is None or ccount - prev != 1:
+            integrity = False
+            break
+
+        prev = ccount
+
+    return integrity

nbproject/_jupyter_communicate.py → nbproject/_dev/_jupyter_communicate.py

@@ -9,6 +9,7 @@
 
 
 def prepare_url(server: dict, query_str: str = ""):
+    """Prepare url to query the jupyter server."""
     token = server["token"]
     if token:
         query_str = f"{query_str}?token={token}"
@@ -18,6 +19,7 @@ def prepare_url(server: dict, query_str: str = ""):
 
 
 def query_server(server: dict):
+    """Query the jupyter server for sessions' info."""
     # based on https://github.com/msm1089/ipynbname
     try:
         url = prepare_url(server)
@@ -32,6 +34,7 @@ def query_server(server: dict):
 
 
 def running_servers():
+    """Return the info about running jupyter servers."""
     try:
         from notebook.notebookapp import list_running_servers
 
@@ -50,7 +53,12 @@ def running_servers():
 
 
 def notebook_path(return_env=False):
+    """Return the path to the current notebook.
 
+    Args:
+        return_env: If `True`, return the environment of execution:
+            `'lab'` for jupyter lab and `'notebook'` for jupyter notebook.
+    """
     if "NBPRJ_TEST_NBPATH" in os.environ:
         nb_path = os.environ["NBPRJ_TEST_NBPATH"]
         if return_env:

nbproject/_dev/_notebook.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+from typing import Union
+
+import orjson
+from pydantic import BaseModel
+
+
+class Notebook(BaseModel):
+    metadata: dict
+    nbformat: int
+    nbformat_minor: int
+    cells: list
+
+
+def read_notebook(filepath: Union[str, Path]) -> Notebook:
+    """Read a notebook from disk.
+
+    Args:
+        filepath: A path to the notebook to read.
+    """
+    with open(filepath, "rb") as f:
+        nb = orjson.loads(f.read())
+
+    return Notebook(**nb)
+
+
+def write_notebook(nb: Notebook, filepath: Union[str, Path]):
+    """Write the notebook to disk.
+
+    Args:
+        nb: Notebook to write.
+        filepath: Path where to write the notebook.
+    """
+    with open(filepath, "wb") as f:
+        f.write(orjson.dumps(nb.dict()))