From eacd22bf92d2540fdc4e4d325756cf300d652a9e Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Thu, 25 Jun 2020 09:01:17 -0700 Subject: [PATCH 01/25] Add new clean, convert, export commands --- nbautoexport/__init__.py | 9 +- nbautoexport/convert.py | 91 ++++++++++ nbautoexport/jupyter_config.py | 90 ++++++++++ nbautoexport/nbautoexport.py | 312 +++++++++++---------------------- nbautoexport/sentinel.py | 97 ++++++++++ nbautoexport/utils.py | 5 + requirements.txt | 1 + 7 files changed, 394 insertions(+), 211 deletions(-) create mode 100644 nbautoexport/convert.py create mode 100644 nbautoexport/jupyter_config.py create mode 100644 nbautoexport/sentinel.py create mode 100644 nbautoexport/utils.py diff --git a/nbautoexport/__init__.py b/nbautoexport/__init__.py index 79634b6..2c4c3e6 100644 --- a/nbautoexport/__init__.py +++ b/nbautoexport/__init__.py @@ -1,7 +1,4 @@ -from nbautoexport.nbautoexport import post_save -from nbautoexport._version import get_versions +from nbautoexport.convert import post_save +from nbautoexport.utils import __version__ -__all__ = [post_save] - -__version__ = get_versions()["version"] -del get_versions +__all__ = [post_save, __version__] diff --git a/nbautoexport/convert.py b/nbautoexport/convert.py new file mode 100644 index 0000000..13d397c --- /dev/null +++ b/nbautoexport/convert.py @@ -0,0 +1,91 @@ +import os +from pathlib import Path +import re +import shutil + +from nbconvert.nbconvertapp import NbConvertApp +from nbconvert.postprocessors.base import PostProcessorBase +from notebook.services.contents.filemanager import FileContentsManager + +from nbautoexport.sentinel import ( + find_unwanted_outputs, + NbAutoexportConfig, + SAVE_PROGRESS_INDICATOR_FILE, +) + + +class CopyToSubfolderPostProcessor(PostProcessorBase): + def __init__(self, subfolder=""): + self.subfolder = subfolder + super(CopyToSubfolderPostProcessor, self).__init__() + + def postprocess(self, input: str): + """ Save converted file to a separate directory. """ + input = Path(input) + + new_dir = input.parent / self.subfolder + new_dir.mkdir(new_dir, exist_ok=True) + new_path = new_dir / input.name + + with input.open("r") as f: + text = f.read() + + with new_path.open("w") as f: + f.write(re.sub(r"\n#\sIn\[(([0-9]+)|(\s))\]:\n{2}", "", text)) + + os.remove(input) + + +def post_save(model: dict, os_path: str, contents_manager: FileContentsManager): + """Post-save hook for converting notebooks to other formats using Jupyter nbconvert and saving + in a subfolder. + + The following arguments are standard for Jupyter post-save hooks. See [Jupyter Documentation]( + https://jupyter-notebook.readthedocs.io/en/stable/extending/savehooks.html). + + Args: + model (dict): the model representing the file. See [Jupyter documentation]( + https://jupyter-notebook.readthedocs.io/en/stable/extending/contents.html#data-model). + os_path (str): the filesystem path to the file just written + contents_manager (FileContentsManager): FileContentsManager instance that hook is bound to + """ + # only do this for notebooks + if model["type"] != "notebook": + return + + # only do this if we've added the special indicator file to the working directory + os_path = Path(os_path) + cwd = os_path.parent + save_progress_indicator = cwd / SAVE_PROGRESS_INDICATOR_FILE + should_convert = save_progress_indicator.exists() + + if should_convert: + config = NbAutoexportConfig.parse_file( + path=save_progress_indicator, content_type="application/json" + ) + export_notebook(os_path, config=config) + + if config.autoclean: + to_remove = find_unwanted_outputs(cwd, config) + for path in to_remove: + if path.is_dir(): + shutil.rmtree(path) + else: + os.remove(path) + + +def export_notebook(notebook_path: Path, config: NbAutoexportConfig): + converter = NbConvertApp() + + for export_format in config.export_formats: + if config.subfolder_type == "notebook": + subfolder = notebook_path.stem + + elif config.subfolder_type == "extension": + subfolder = export_format + + converter.postprocessor = CopyToSubfolderPostProcessor(subfolder=subfolder) + converter.export_format = export_format + converter.initialize() + converter.notebooks = [notebook_path] + converter.convert_notebooks() diff --git a/nbautoexport/jupyter_config.py b/nbautoexport/jupyter_config.py new file mode 100644 index 0000000..dc2852d --- /dev/null +++ b/nbautoexport/jupyter_config.py @@ -0,0 +1,90 @@ +from inspect import getsourcelines +from pathlib import Path +from pkg_resources import parse_version +import re +import textwrap + +from jupyter_core.paths import jupyter_config_dir +from traitlets.config.loader import Config + +from nbautoexport.utils import __version__, logger + + +def initialize_post_save_hook(c: Config): + # >>> nbautoexport initialize, version=[{version}] >>> + try: + from nbautoexport import post_save + + if callable(c.FileContentsManager.post_save_hook): + old_post_save = c.FileContentsManager.post_save_hook + + def _post_save(model, os_path, contents_manager): + old_post_save(model=model, os_path=os_path, contents_manager=contents_manager) + post_save(model=model, os_path=os_path, contents_manager=contents_manager) + + c.FileContentsManager.post_save_hook = _post_save + else: + c.FileContentsManager.post_save_hook = post_save + except Exception: + pass + # <<< nbautoexport initialize <<< + pass # need this line for above comment to be included in function source + + +post_save_hook_initialize_block = textwrap.dedent( + "".join(getsourcelines(initialize_post_save_hook)[0][1:-1]).format(version=__version__) +) + +block_regex = re.compile( + r"# >>> nbautoexport initialize.*# <<< nbautoexport initialize <<<\n?", + re.DOTALL, # dot matches newline +) +version_regex = re.compile(r"(?<=# >>> nbautoexport initialize, version=\[).*(?=\] >>>)") + + +def install_post_save_hook(): + """Splices the post save hook into the global Jupyter configuration file + """ + config_dir = jupyter_config_dir() + config_path = (Path(config_dir) / "jupyter_notebook_config.py").expanduser().resolve() + + if not config_path.exists(): + logger.debug(f"No existing Jupyter configuration detected at {config_path}. Creating...") + config_path.parent.mkdir(exist_ok=True, parents=True) + with config_path.open("w") as fp: + fp.write(post_save_hook_initialize_block) + logger.info("nbautoexport post-save hook installed.") + return + + # If config exists, check for existing nbautoexport initialize block and install as appropriate + logger.debug(f"Detected existing Jupyter configuration at {config_path}") + + with config_path.open("r") as fp: + config = fp.read() + + if block_regex.search(config): + logger.debug("Detected existing nbautoexport post-save hook.") + + version_match = version_regex.search(config) + if version_match: + existing_version = version_match.group() + logger.debug(f"Existing post-save hook is version {existing_version}") + else: + existing_version = "" + logger.debug("Existing post-save hook predates versioning.") + + if parse_version(existing_version) < parse_version(__version__): + logger.info(f"Updating nbautoexport post-save hook with version {__version__}...") + with config_path.open("w") as fp: + # Open as w replaces existing file. We're replacing entire config. + fp.write(block_regex.sub(post_save_hook_initialize_block, config)) + else: + logger.debug("No changes made.") + return + else: + logger.info("Installing post-save hook.") + with config_path.open("a") as fp: + # Open as a just appends. We append block at the end of existing file. + fp.write("\n" + post_save_hook_initialize_block) + + logger.info("nbautoexport post-save hook installed.") diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index cf07f41..df4719c 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -1,241 +1,141 @@ -from enum import Enum -from inspect import getsourcelines -import json import logging import os from pathlib import Path -from pkg_resources import parse_version -import re -import textwrap +import shutil from typing import List -from jupyter_core.paths import jupyter_config_dir -from nbconvert.nbconvertapp import NbConvertApp -from nbconvert.postprocessors.base import PostProcessorBase -from traitlets.config.loader import Config import typer -from nbautoexport._version import get_versions +from nbautoexport.jupyter_config import install_post_save_hook +from nbautoexport.convert import export_notebook +from nbautoexport.sentinel import ( + ExportFormat, + find_unwanted_outputs, + install_sentinel, + NbAutoexportConfig, + OrganizeBy, + SAVE_PROGRESS_INDICATOR_FILE, +) +from nbautoexport.utils import __version__ -logger = logging.getLogger(__name__) app = typer.Typer() -__version__ = get_versions()["version"] - - -class ExportFormat(str, Enum): - html = "html" - latex = "latex" - pdf = "pdf" - slides = "slides" - markdown = "markdown" - asciidoc = "asciidoc" - script = "script" - notebook = "notebook" - - -class OrganizeBy(str, Enum): - notebook = "notebook" - extension = "extension" - - -class CopyToSubfolderPostProcessor(PostProcessorBase): - def __init__(self, subfolder=None): - self.subfolder = subfolder - super(CopyToSubfolderPostProcessor, self).__init__() - - def postprocess(self, input): - """ Save converted file to a separate directory. """ - if self.subfolder is None: - return - - dirname, filename = os.path.split(input) - new_dir = os.path.join(dirname, self.subfolder) - new_path = os.path.join(new_dir, filename) - - if not os.path.exists(new_dir): - os.mkdir(new_dir) - - with open(input, "r") as f: - text = f.read() - with open(new_path, "w") as f: - f.write(re.sub(r"\n#\sIn\[(([0-9]+)|(\s))\]:\n{2}", "", text)) - - os.remove(input) - - -SAVE_PROGRESS_INDICATOR_FILE = ".nbautoexport" +def version_callback(value: bool): + if value: + typer.echo(__version__) + raise typer.Exit() -def post_save(model, os_path, contents_manager): - """Post-save hook for converting notebooks to other formats using Jupyter nbconvert and saving - in a subfolder. - The following arguments are standard for Jupyter post-save hooks. See [Jupyter Documentation]( - https://jupyter-notebook.readthedocs.io/en/stable/extending/savehooks.html). +@app.callback() +def main(): + """Exports Jupyter notebooks to various file formats (.py, .html, and more) upon save, + automatically. - Args: - model (dict): the model representing the file. See [Jupyter documentation]( - https://jupyter-notebook.readthedocs.io/en/stable/extending/contents.html#data-model). - os_path (str): the filesystem path to the file just written - contents_manager (FileContentsManager): FileContentsManager instance that hook is bound to + Use the install command to configure a notebooks directory to be watched. """ - # only do this for notebooks - if model["type"] != "notebook": - return - - # only do this if we've added the special indicator file to the working directory - cwd = os.path.dirname(os_path) - save_progress_indicator = os.path.join(cwd, SAVE_PROGRESS_INDICATOR_FILE) - should_convert = os.path.exists(save_progress_indicator) + pass - if should_convert: - with open(save_progress_indicator, "r") as f: - save_settings = f.read() - if len(save_settings) > 0: - save_settings = json.loads(save_settings) - else: - save_settings = {} - - subfolder_type = save_settings.get("organize_by", "notebook") - export_formats = save_settings.get("export_formats", ["script"]) - converter = NbConvertApp() - - for export_format in export_formats: - if subfolder_type == "notebook": - d, fname = os.path.split(os_path) - subfolder = os.path.splitext(fname)[0] - - elif subfolder_type == "extension": - subfolder = export_format - - converter.postprocessor = CopyToSubfolderPostProcessor(subfolder=subfolder) - converter.export_format = export_format - converter.initialize() - converter.notebooks = [os_path] - converter.convert_notebooks() +@app.command() +def clean( + directory: Path = typer.Argument( + ..., exists=True, file_okay=False, dir_okay=True, writable=True + ) +): + """Remove subfolders/files not matching .nbautoconvert configuration. + """ + sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json") -def initialize_post_save_hook(c: Config): - # >>> nbautoexport initialize, version=[{version}] >>> - try: - from nbautoexport import post_save + to_remove = find_unwanted_outputs(directory, config) - if callable(c.FileContentsManager.post_save_hook): - old_post_save = c.FileContentsManager.post_save_hook + typer.echo("Removing following files:") + for path in to_remove: + typer.echo(" " + path) + if path.is_dir(): + for subpath in path.iterdir(): + typer.echo(" " + subpath) - def _post_save(model, os_path, contents_manager): - old_post_save(model=model, os_path=os_path, contents_manager=contents_manager) - post_save(model=model, os_path=os_path, contents_manager=contents_manager) + delete = typer.confirm("Are you sure you want to delete these files?") + if not delete: + typer.echo("Not deleting") + raise typer.Abort() - c.FileContentsManager.post_save_hook = _post_save + for path in to_remove: + if path.is_dir(): + shutil.rmtree(path) else: - c.FileContentsManager.post_save_hook = post_save - except Exception: - pass - # <<< nbautoexport initialize <<< - pass # need this line for above comment to be included in function source + os.remove(path) + typer.echo("Files deleted.") -post_save_hook_initialize_block = textwrap.dedent( - "".join(getsourcelines(initialize_post_save_hook)[0][1:-1]).format(version=__version__) -) - -block_regex = re.compile( - r"# >>> nbautoexport initialize.*# <<< nbautoexport initialize <<<\n?", - re.DOTALL, # dot matches newline -) -version_regex = re.compile(r"(?<=# >>> nbautoexport initialize, version=\[).*(?=\] >>>)") - +@app.command() +def convert( + input: Path = typer.Argument(..., exists=True, file_okay=True, dir_okay=True, writable=True), + export_formats: List[ExportFormat] = typer.Option( + ["script"], + "--export-format", + "-f", + show_default=True, + help=( + """File format(s) to save for each notebook. Options are 'script', 'html', 'markdown', """ + """and 'rst'. Multiple formats should be provided using multiple flags, e.g., '-f """ + """script-f html -f markdown'.""" + ), + ), + organize_by: OrganizeBy = typer.Option( + "extension", + "--organize-by", + "-b", + show_default=True, + help=( + """Whether to save exported file(s) in a folder per notebook or a folder per extension. """ + """Options are 'notebook' or 'extension'.""" + ), + ), +): + """Convert notebook(s) using specified configuration options. -def install_post_save_hook(): - """Splices the post save hook into the global Jupyter configuration file + INPUT is the path to a notebook to be converted, or a directory containing notebooks to be + converted. """ - config_dir = jupyter_config_dir() - config_path = (Path(config_dir) / "jupyter_notebook_config.py").expanduser().resolve() - - if not config_path.exists(): - logger.debug(f"No existing Jupyter configuration detected at {config_path}. Creating...") - config_path.parent.mkdir(exist_ok=True, parents=True) - with config_path.open("w") as fp: - fp.write(post_save_hook_initialize_block) - logger.info("nbautoexport post-save hook installed.") - return - - # If config exists, check for existing nbautoexport initialize block and install as appropriate - logger.debug(f"Detected existing Jupyter configuration at {config_path}") - - with config_path.open("r") as fp: - config = fp.read() - - if block_regex.search(config): - logger.debug("Detected existing nbautoexport post-save hook.") - - version_match = version_regex.search(config) - if version_match: - existing_version = version_match.group() - logger.debug(f"Existing post-save hook is version {existing_version}") - else: - existing_version = "" - logger.debug("Existing post-save hook predates versioning.") - - if parse_version(existing_version) < parse_version(__version__): - logger.info(f"Updating nbautoexport post-save hook with version {__version__}...") - with config_path.open("w") as fp: - # Open as w replaces existing file. We're replacing entire config. - fp.write(block_regex.sub(post_save_hook_initialize_block, config)) - else: - logger.debug("No changes made.") - return + config = NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by) + if input.is_dir(): + for notebook_path in input.glob("*.ipynb"): + export_notebook(notebook_path, config=config) else: - logger.info("Installing post-save hook.") - with config_path.open("a") as fp: - # Open as a just appends. We append block at the end of existing file. - fp.write("\n" + post_save_hook_initialize_block) - - logger.info("nbautoexport post-save hook installed.") + export_notebook(notebook_path, config=config) -def install_sentinel( - export_formats: List[ExportFormat], organize_by: OrganizeBy, directory: Path, overwrite: bool +@app.command() +def export( + input: Path = typer.Argument( + "extension", exists=True, file_okay=True, dir_okay=True, writable=True + ) ): - """Writes the configuration file to a specified directory. + """Convert notebook(s) using existing configuration file. - Args: - export_formats: A list of `nbconvert`-supported export formats to write on each save - organize_by: Whether to organize exported files by notebook filename or in folders by extension - directory: The directory containing the notebooks to monitor - overwrite: Overwrite an existing sentinel file if one exists - """ - sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE + INPUT is the path to a notebook to be converted, or a directory containing notebooks to be + converted. - if sentinel_path.exists() and (not overwrite): - raise FileExistsError( - f"""Detected existing autoexport configuration at {sentinel_path}. """ - """If you wish to overwrite, use the --overwrite flag.""" - ) + A .nbautoconvert configuration file is required to be in the same directory as the notebook(s). + """ + if input.is_dir(): + sentinel_path = input / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json") + for notebook_path in input.glob("*.ipynb"): + export_notebook(notebook_path, config=config) else: - config = { - "export_formats": export_formats, - "organize_by": organize_by, - } - - logger.info(f"Creating configuration file at {sentinel_path}") - logger.info(f"\n{json.dumps(config, indent=2)}") - with sentinel_path.open("w") as fp: - json.dump(config, fp) - - -def version_callback(value: bool): - if value: - typer.echo(__version__) - raise typer.Exit() + sentinel_path = input.parent / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json") + export_notebook(notebook_path, config=config) @app.command() def install( directory: Path = typer.Argument( - "notebooks", exists=True, file_okay=False, dir_okay=True, writable=True + "extension", exists=True, file_okay=False, dir_okay=True, writable=True ), export_formats: List[ExportFormat] = typer.Option( ["script"], @@ -258,6 +158,11 @@ def install( """Options are 'notebook' or 'extension'.""" ), ), + autoclean: bool = typer.Option( + False, + show_default=True, + help="Whether to automatically delete files in subfolders that don't match configuration.", + ), overwrite: bool = typer.Option( False, "--overwrite", @@ -273,11 +178,8 @@ def install( None, "--version", callback=version_callback, is_eager=True, help="Show version" ), ): - """Exports Jupyter notebooks to various file formats (.py, .html, and more) upon save, - automatically. - - This command creates a .nbautoexport configuration file in DIRECTORY. Defaults to - "./notebooks/" + """ + Create a .nbautoexport configuration file in DIRECTORY. Defaults to "./notebooks/" """ if verbose: logging.basicConfig(level=logging.DEBUG) diff --git a/nbautoexport/sentinel.py b/nbautoexport/sentinel.py new file mode 100644 index 0000000..7b3eebd --- /dev/null +++ b/nbautoexport/sentinel.py @@ -0,0 +1,97 @@ +from enum import Enum +from pathlib import Path +from typing import List + +from pydantic import BaseModel + +from nbautoexport.utils import logger + + +SAVE_PROGRESS_INDICATOR_FILE = ".nbautoexport" + + +class ExportFormat(str, Enum): + html = "html" + latex = "latex" + pdf = "pdf" + slides = "slides" + markdown = "markdown" + asciidoc = "asciidoc" + script = "script" + notebook = "notebook" + + @classmethod + def has_value(cls, value): + return any(level for level in cls if level.value == value) + + +class OrganizeBy(str, Enum): + notebook = "notebook" + extension = "extension" + + +class NbAutoexportConfig(BaseModel): + export_formats: List[ExportFormat] = ["script"] + organize_by: OrganizeBy = "extension" + autoclean: bool = False + + +def install_sentinel( + export_formats: List[ExportFormat], organize_by: OrganizeBy, directory: Path, overwrite: bool +): + """Writes the configuration file to a specified directory. + + Args: + export_formats: A list of `nbconvert`-supported export formats to write on each save + organize_by: Whether to organize exported files by notebook filename or in folders by extension + directory: The directory containing the notebooks to monitor + overwrite: Overwrite an existing sentinel file if one exists + """ + sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE + + if sentinel_path.exists() and (not overwrite): + raise FileExistsError( + f"""Detected existing autoexport configuration at {sentinel_path}. """ + """If you wish to overwrite, use the --overwrite flag.""" + ) + else: + config = NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by) + + logger.info(f"Creating configuration file at {sentinel_path}") + logger.info(f"\n{config.json(indent=2)}") + with sentinel_path.open("w") as fp: + fp.write(config.json(indent=2)) + + +def find_unwanted_outputs(directory: Path, config: NbAutoexportConfig) -> List[Path]: + """Determine which subfolders or files in a given directory are not expected to be generated by + current notebooks in that directory and given NbAutoexportConfig. + + Args: + directory (Path): directory of notebooks + config (NbAutoexportConfig): nbautoexport configuration + + Returns: + List[Path]: list of paths + """ + notebook_paths = directory.glob("*.ipynb") + notebook_names = (f.stem for f in notebook_paths) + + to_remove = [] + subfolders = (f for f in directory.iterdir() if f.is_dir()) + if config.organize_by == "notebook": + for subfolder in subfolders: + if subfolder.name not in notebook_names: + to_remove.append(subfolder) + elif config.organize_by == "extension": + for subfolder in subfolders: + if subfolder.name in config.export_formats: + # Check for individual files to remove + for subfolder_file in subfolder.glob("*"): + if subfolder_file.stem not in notebook_names: + to_remove.append(subfolder_file) + else: + # Otherwise to remove entire subfolder + to_remove.append(subfolder) + + return to_remove diff --git a/nbautoexport/utils.py b/nbautoexport/utils.py new file mode 100644 index 0000000..45219fe --- /dev/null +++ b/nbautoexport/utils.py @@ -0,0 +1,5 @@ +import logging +from nbautoexport._version import get_versions + +logger = logging.getLogger("nbautoexport") +__version__ = get_versions()["version"] diff --git a/requirements.txt b/requirements.txt index 16ec0d1..1b62dbd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ jupyter-contrib-nbextensions>=0.5.1 nbconvert>=5.6.1 +pydantic typer>=0.2.1 From deb08415809a1560f9a6d0c8b91f26edb572e682 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Thu, 25 Jun 2020 09:14:45 -0700 Subject: [PATCH 02/25] Rename autoclean to just clean --- nbautoexport/convert.py | 7 +++++-- nbautoexport/nbautoexport.py | 2 +- nbautoexport/sentinel.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nbautoexport/convert.py b/nbautoexport/convert.py index 13d397c..7899a97 100644 --- a/nbautoexport/convert.py +++ b/nbautoexport/convert.py @@ -15,12 +15,15 @@ class CopyToSubfolderPostProcessor(PostProcessorBase): - def __init__(self, subfolder=""): + def __init__(self, subfolder=None): self.subfolder = subfolder super(CopyToSubfolderPostProcessor, self).__init__() def postprocess(self, input: str): """ Save converted file to a separate directory. """ + if self.subfolder is None: + return + input = Path(input) new_dir = input.parent / self.subfolder @@ -65,7 +68,7 @@ def post_save(model: dict, os_path: str, contents_manager: FileContentsManager): ) export_notebook(os_path, config=config) - if config.autoclean: + if config.clean: to_remove = find_unwanted_outputs(cwd, config) for path in to_remove: if path.is_dir(): diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index df4719c..ff8dc25 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -158,7 +158,7 @@ def install( """Options are 'notebook' or 'extension'.""" ), ), - autoclean: bool = typer.Option( + clean: bool = typer.Option( False, show_default=True, help="Whether to automatically delete files in subfolders that don't match configuration.", diff --git a/nbautoexport/sentinel.py b/nbautoexport/sentinel.py index 7b3eebd..1da4c07 100644 --- a/nbautoexport/sentinel.py +++ b/nbautoexport/sentinel.py @@ -33,7 +33,7 @@ class OrganizeBy(str, Enum): class NbAutoexportConfig(BaseModel): export_formats: List[ExportFormat] = ["script"] organize_by: OrganizeBy = "extension" - autoclean: bool = False + clean: bool = False def install_sentinel( From f3094b10996da550021bec34b8cbff91c7d8d6c4 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Thu, 25 Jun 2020 22:19:49 -0700 Subject: [PATCH 03/25] Fix unit tests --- nbautoexport/convert.py | 12 ++--- nbautoexport/nbautoexport.py | 13 +++-- tests/test_install_post_save_hook.py | 75 ++++++++++++++-------------- tests/test_nbautoexport.py | 21 +++----- tests/test_post_save.py | 3 +- 5 files changed, 63 insertions(+), 61 deletions(-) diff --git a/nbautoexport/convert.py b/nbautoexport/convert.py index 7899a97..55ca3a8 100644 --- a/nbautoexport/convert.py +++ b/nbautoexport/convert.py @@ -27,7 +27,7 @@ def postprocess(self, input: str): input = Path(input) new_dir = input.parent / self.subfolder - new_dir.mkdir(new_dir, exist_ok=True) + new_dir.mkdir(exist_ok=True) new_path = new_dir / input.name with input.open("r") as f: @@ -81,14 +81,14 @@ def export_notebook(notebook_path: Path, config: NbAutoexportConfig): converter = NbConvertApp() for export_format in config.export_formats: - if config.subfolder_type == "notebook": + if config.organize_by == "notebook": subfolder = notebook_path.stem - elif config.subfolder_type == "extension": - subfolder = export_format + elif config.organize_by == "extension": + subfolder = export_format.value converter.postprocessor = CopyToSubfolderPostProcessor(subfolder=subfolder) - converter.export_format = export_format + converter.export_format = export_format.value converter.initialize() - converter.notebooks = [notebook_path] + converter.notebooks = [str(notebook_path)] converter.convert_notebooks() diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index ff8dc25..5ab5f24 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -28,7 +28,15 @@ def version_callback(value: bool): @app.callback() -def main(): +def main( + version: bool = typer.Option( + None, + "--version", + callback=version_callback, + is_eager=True, + help="Show nbautoexport version.", + ), +): """Exports Jupyter notebooks to various file formats (.py, .html, and more) upon save, automatically. @@ -174,9 +182,6 @@ def install( verbose: bool = typer.Option( False, "--verbose", "-v", is_flag=True, show_default=True, help="Verbose mode" ), - version: bool = typer.Option( - None, "--version", callback=version_callback, is_eager=True, help="Show version" - ), ): """ Create a .nbautoexport configuration file in DIRECTORY. Defaults to "./notebooks/" diff --git a/tests/test_install_post_save_hook.py b/tests/test_install_post_save_hook.py index 1fb2680..26869e1 100644 --- a/tests/test_install_post_save_hook.py +++ b/tests/test_install_post_save_hook.py @@ -8,6 +8,7 @@ from nbautoexport import __version__ import nbautoexport as nbautoexport_root import nbautoexport.nbautoexport as nbautoexport +from nbautoexport import convert, jupyter_config def test_parse_version(): @@ -19,7 +20,7 @@ def test_parse_version(): def test_initialize_block_content(): - init_block = nbautoexport.post_save_hook_initialize_block + init_block = jupyter_config.post_save_hook_initialize_block assert __version__ in init_block assert init_block.startswith("# >>> nbautoexport initialize") assert init_block.endswith("# <<< nbautoexport initialize <<<\n") @@ -33,8 +34,8 @@ def test_initialize_block_regexes(): print('good night world!') """ ) - assert nbautoexport.block_regex.search(config_no_nbautoexport) is None - assert nbautoexport.version_regex.search(config_no_nbautoexport) is None + assert jupyter_config.block_regex.search(config_no_nbautoexport) is None + assert jupyter_config.version_regex.search(config_no_nbautoexport) is None config_no_version = textwrap.dedent( """\ @@ -48,8 +49,8 @@ def test_initialize_block_regexes(): """ ) - assert nbautoexport.block_regex.search(config_no_version) is not None - assert nbautoexport.version_regex.search(config_no_version) is None + assert jupyter_config.block_regex.search(config_no_version) is not None + assert jupyter_config.version_regex.search(config_no_version) is None config_with_version = textwrap.dedent( """\ @@ -62,8 +63,8 @@ def test_initialize_block_regexes(): print('good night world!') """ ) - assert nbautoexport.block_regex.search(config_with_version) is not None - version_match = nbautoexport.version_regex.search(config_with_version) + assert jupyter_config.block_regex.search(config_with_version) is not None + version_match = jupyter_config.version_regex.search(config_with_version) assert version_match is not None assert version_match.group() == "old_and_tired" @@ -72,19 +73,19 @@ def test_install_hook_no_config(tmp_path, monkeypatch): def mock_jupyter_config_dir(): return str(tmp_path) - monkeypatch.setattr(nbautoexport, "jupyter_config_dir", mock_jupyter_config_dir) + monkeypatch.setattr(jupyter_config, "jupyter_config_dir", mock_jupyter_config_dir) config_path = tmp_path / "jupyter_notebook_config.py" assert not config_path.exists() - nbautoexport.install_post_save_hook() + jupyter_config.install_post_save_hook() assert config_path.exists() with config_path.open("r") as fp: config = fp.read() - assert config == nbautoexport.post_save_hook_initialize_block + assert config == jupyter_config.post_save_hook_initialize_block def test_install_hook_missing_config_dir(tmp_path, monkeypatch): @@ -93,42 +94,42 @@ def test_install_hook_missing_config_dir(tmp_path, monkeypatch): def mock_jupyter_config_dir(): return str(config_dir) - monkeypatch.setattr(nbautoexport, "jupyter_config_dir", mock_jupyter_config_dir) + monkeypatch.setattr(jupyter_config, "jupyter_config_dir", mock_jupyter_config_dir) config_path = config_dir / "jupyter_notebook_config.py" assert not config_dir.exists() assert not config_path.exists() - nbautoexport.install_post_save_hook() + jupyter_config.install_post_save_hook() assert config_dir.exists() assert config_path.exists() with config_path.open("r") as fp: config = fp.read() - assert config == nbautoexport.post_save_hook_initialize_block + assert config == jupyter_config.post_save_hook_initialize_block def test_install_hook_existing_config_no_hook(tmp_path, monkeypatch): def mock_jupyter_config_dir(): return str(tmp_path) - monkeypatch.setattr(nbautoexport, "jupyter_config_dir", mock_jupyter_config_dir) + monkeypatch.setattr(jupyter_config, "jupyter_config_dir", mock_jupyter_config_dir) config_path = tmp_path / "jupyter_notebook_config.py" with config_path.open("w") as fp: fp.write("print('hello world!')") - nbautoexport.install_post_save_hook() + jupyter_config.install_post_save_hook() assert config_path.exists() with config_path.open("r") as fp: config = fp.read() assert config == ( - "print('hello world!')" + "\n" + nbautoexport.post_save_hook_initialize_block + "print('hello world!')" + "\n" + jupyter_config.post_save_hook_initialize_block ) @@ -136,7 +137,7 @@ def test_install_hook_replace_hook_no_version(tmp_path, monkeypatch): def mock_jupyter_config_dir(): return str(tmp_path) - monkeypatch.setattr(nbautoexport, "jupyter_config_dir", mock_jupyter_config_dir) + monkeypatch.setattr(jupyter_config, "jupyter_config_dir", mock_jupyter_config_dir) config_path = tmp_path / "jupyter_notebook_config.py" @@ -153,14 +154,14 @@ def mock_jupyter_config_dir(): with config_path.open("w") as fp: fp.write(textwrap.dedent(old_config_text)) - nbautoexport.install_post_save_hook() + jupyter_config.install_post_save_hook() assert config_path.exists() with config_path.open("r") as fp: config = fp.read() assert config == ( "print('hello world!')\n\n" - + nbautoexport.post_save_hook_initialize_block + + jupyter_config.post_save_hook_initialize_block + "\nprint('good night world!')\n" ) assert "old_and_tired()" not in config @@ -171,7 +172,7 @@ def test_install_hook_replace_hook_older_version(tmp_path, monkeypatch): def mock_jupyter_config_dir(): return str(tmp_path) - monkeypatch.setattr(nbautoexport, "jupyter_config_dir", mock_jupyter_config_dir) + monkeypatch.setattr(jupyter_config, "jupyter_config_dir", mock_jupyter_config_dir) config_path = tmp_path / "jupyter_notebook_config.py" @@ -188,14 +189,14 @@ def mock_jupyter_config_dir(): with config_path.open("w") as fp: fp.write(textwrap.dedent(old_config_text)) - nbautoexport.install_post_save_hook() + jupyter_config.install_post_save_hook() assert config_path.exists() with config_path.open("r") as fp: config = fp.read() assert config == ( "print('hello world!')\n\n" - + nbautoexport.post_save_hook_initialize_block + + jupyter_config.post_save_hook_initialize_block + "\nprint('good night world!')\n" ) assert "old_and_tired()" not in config @@ -205,17 +206,17 @@ def mock_jupyter_config_dir(): def test_initialize_post_save_binding(): """Test that post_save hook can be successfully bound to a Jupyter config. """ - jupyter_config = Config(FileContentsManager=FileContentsManager()) - nbautoexport.initialize_post_save_hook(jupyter_config) - assert isinstance(jupyter_config.FileContentsManager, FileContentsManager) - assert jupyter_config.FileContentsManager.post_save_hook is nbautoexport.post_save + jupyter_config_obj = Config(FileContentsManager=FileContentsManager()) + jupyter_config.initialize_post_save_hook(jupyter_config_obj) + assert isinstance(jupyter_config_obj.FileContentsManager, FileContentsManager) + assert jupyter_config_obj.FileContentsManager.post_save_hook is convert.post_save def test_initialize_post_save_execution(monkeypatch): """Test that bound post_save hook with given signature can be successfully run. """ - jupyter_config = Config(FileContentsManager=FileContentsManager()) + jupyter_config_obj = Config(FileContentsManager=FileContentsManager()) def mocked_post_save(model, os_path, contents_manager): """Append a token to os_path to certify that function ran. @@ -224,12 +225,12 @@ def mocked_post_save(model, os_path, contents_manager): monkeypatch.setattr(nbautoexport_root, "post_save", mocked_post_save) - nbautoexport.initialize_post_save_hook(jupyter_config) + jupyter_config.initialize_post_save_hook(jupyter_config_obj) - assert isinstance(jupyter_config.FileContentsManager, FileContentsManager) - assert callable(jupyter_config.FileContentsManager.post_save_hook) + assert isinstance(jupyter_config_obj.FileContentsManager, FileContentsManager) + assert callable(jupyter_config_obj.FileContentsManager.post_save_hook) os_path_list = [] - jupyter_config.FileContentsManager.run_post_save_hook(model=None, os_path=os_path_list) + jupyter_config_obj.FileContentsManager.run_post_save_hook(model=None, os_path=os_path_list) assert os_path_list == ["nbautoexport"] @@ -237,14 +238,14 @@ def test_initialize_post_save_existing(monkeypatch): """Test that handling of existing post_save hook works properly. """ - jupyter_config = Config(FileContentsManager=FileContentsManager()) + jupyter_config_obj = Config(FileContentsManager=FileContentsManager()) def old_post_save(model, os_path, contents_manager): """Append a token to os_path to certify that function ran. """ os_path.append("old_post_save") - jupyter_config.FileContentsManager.post_save_hook = old_post_save + jupyter_config_obj.FileContentsManager.post_save_hook = old_post_save def mocked_post_save(model, os_path, contents_manager): """Append a token to os_path to certify that function ran. @@ -253,10 +254,10 @@ def mocked_post_save(model, os_path, contents_manager): monkeypatch.setattr(nbautoexport_root, "post_save", mocked_post_save) - nbautoexport.initialize_post_save_hook(jupyter_config) + jupyter_config.initialize_post_save_hook(jupyter_config_obj) - assert isinstance(jupyter_config.FileContentsManager, FileContentsManager) - assert callable(jupyter_config.FileContentsManager.post_save_hook) + assert isinstance(jupyter_config_obj.FileContentsManager, FileContentsManager) + assert callable(jupyter_config_obj.FileContentsManager.post_save_hook) os_path_list = [] - jupyter_config.FileContentsManager.run_post_save_hook(model=None, os_path=os_path_list) + jupyter_config_obj.FileContentsManager.run_post_save_hook(model=None, os_path=os_path_list) assert os_path_list == ["old_post_save", "nbautoexport"] diff --git a/tests/test_nbautoexport.py b/tests/test_nbautoexport.py index 39dddef..df2f037 100644 --- a/tests/test_nbautoexport.py +++ b/tests/test_nbautoexport.py @@ -6,7 +6,8 @@ from typer.testing import CliRunner from nbautoexport import __version__ -from nbautoexport.nbautoexport import app, ExportFormat, install_sentinel +from nbautoexport.nbautoexport import app +from nbautoexport.sentinel import ExportFormat, install_sentinel, NbAutoexportConfig def test_cli(): @@ -27,7 +28,7 @@ def test_version(): def test_invalid_export_format(): runner = CliRunner() - result = runner.invoke(app, ["-f", "invalid-output-format"]) + result = runner.invoke(app, ["install", "-f", "invalid-output-format"]) assert result.exit_code == 2 assert ( "Error: Invalid value for '--export-format' / '-f': invalid choice: invalid-output-format" @@ -45,7 +46,7 @@ def test_export_format_compatibility(): def test_invalid_organize_by(): runner = CliRunner() - result = runner.invoke(app, ["-b", "invalid-organize-by"]) + result = runner.invoke(app, ["install", "-b", "invalid-organize-by"]) assert result.exit_code == 2 assert ( "Invalid value for '--organize-by' / '-b': invalid choice: invalid-organize-by" @@ -56,7 +57,7 @@ def test_invalid_organize_by(): def test_refuse_overwrite(tmp_path): (tmp_path / ".nbautoexport").touch() runner = CliRunner() - result = runner.invoke(app, [str(tmp_path)]) + result = runner.invoke(app, ["install", str(tmp_path)]) assert result.exit_code == 1 assert "Detected existing autoexport configuration at" in result.output @@ -65,7 +66,7 @@ def test_force_overwrite(tmp_path): (tmp_path / ".nbautoexport").touch() runner = CliRunner() result = runner.invoke( - app, [str(tmp_path), "-o", "-f", "script", "-f", "html", "-b", "notebook"] + app, ["install", str(tmp_path), "-o", "-f", "script", "-f", "html", "-b", "notebook"] ) print(result.output) print(result.exit_code) @@ -73,10 +74,7 @@ def test_force_overwrite(tmp_path): with (tmp_path / ".nbautoexport").open("r") as fp: config = json.load(fp) - expected_config = { - "export_formats": ["script", "html"], - "organize_by": "notebook", - } + expected_config = NbAutoexportConfig(export_formats=["script", "html"], organize_by="notebook") assert config == expected_config @@ -86,8 +84,5 @@ def test_install_sentinel(tmp_path): with (tmp_path / ".nbautoexport").open("r") as fp: config = json.load(fp) - expected_config = { - "export_formats": export_formats, - "organize_by": "notebook", - } + expected_config = NbAutoexportConfig(export_formats=export_formats, organize_by="notebook") assert config == expected_config diff --git a/tests/test_post_save.py b/tests/test_post_save.py index d618d9a..7c65ab1 100644 --- a/tests/test_post_save.py +++ b/tests/test_post_save.py @@ -3,7 +3,8 @@ import shutil import sys -from nbautoexport.nbautoexport import post_save, SAVE_PROGRESS_INDICATOR_FILE +from nbautoexport.convert import post_save +from nbautoexport.sentinel import SAVE_PROGRESS_INDICATOR_FILE NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" From e25d1ac5d6be2c547e3fd74b731b2dbc746026d6 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Thu, 25 Jun 2020 22:20:33 -0700 Subject: [PATCH 04/25] Rename test scripts --- tests/{test_nbautoexport.py => test_cli.py} | 0 tests/{test_post_save.py => test_convert.py} | 0 tests/{test_install_post_save_hook.py => test_jupyter_config.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/{test_nbautoexport.py => test_cli.py} (100%) rename tests/{test_post_save.py => test_convert.py} (100%) rename tests/{test_install_post_save_hook.py => test_jupyter_config.py} (100%) diff --git a/tests/test_nbautoexport.py b/tests/test_cli.py similarity index 100% rename from tests/test_nbautoexport.py rename to tests/test_cli.py diff --git a/tests/test_post_save.py b/tests/test_convert.py similarity index 100% rename from tests/test_post_save.py rename to tests/test_convert.py diff --git a/tests/test_install_post_save_hook.py b/tests/test_jupyter_config.py similarity index 100% rename from tests/test_install_post_save_hook.py rename to tests/test_jupyter_config.py From ffd9ddb3a8e02043a69d7d4794e45ca230b2844f Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Fri, 26 Jun 2020 10:09:20 -0700 Subject: [PATCH 05/25] New approach to cleaning --- nbautoexport/__main__.py | 3 ++ nbautoexport/convert.py | 19 +++++--- nbautoexport/nbautoexport.py | 57 ++++++++++++++--------- nbautoexport/sentinel.py | 84 +++++++++++++++++---------------- requirements.txt | 2 +- tests/test_cli.py | 90 ++++++++++-------------------------- tests/test_cli_install.py | 71 ++++++++++++++++++++++++++++ tests/test_convert.py | 10 ++-- 8 files changed, 196 insertions(+), 140 deletions(-) create mode 100644 nbautoexport/__main__.py create mode 100644 tests/test_cli_install.py diff --git a/nbautoexport/__main__.py b/nbautoexport/__main__.py new file mode 100644 index 0000000..108b25f --- /dev/null +++ b/nbautoexport/__main__.py @@ -0,0 +1,3 @@ +from nbautoexport.nbautoexport import app + +app(prog_name="python -m nbautoexport") diff --git a/nbautoexport/convert.py b/nbautoexport/convert.py index 55ca3a8..b2ef73a 100644 --- a/nbautoexport/convert.py +++ b/nbautoexport/convert.py @@ -1,14 +1,12 @@ import os from pathlib import Path import re -import shutil from nbconvert.nbconvertapp import NbConvertApp from nbconvert.postprocessors.base import PostProcessorBase from notebook.services.contents.filemanager import FileContentsManager from nbautoexport.sentinel import ( - find_unwanted_outputs, NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE, ) @@ -69,12 +67,19 @@ def post_save(model: dict, os_path: str, contents_manager: FileContentsManager): export_notebook(os_path, config=config) if config.clean: - to_remove = find_unwanted_outputs(cwd, config) + # Remove files that are not notebooks or expected files + notebook_paths = cwd.glob("*.ipynb") + expected_exports = [cwd / export for export in config.expected_exports(notebook_paths)] + subfiles = (f for f in cwd.rglob("*") if f.is_file()) + to_remove = set(subfiles).difference(notebook_paths).difference(expected_exports) for path in to_remove: - if path.is_dir(): - shutil.rmtree(path) - else: - os.remove(path) + os.remove(path) + + # Remove empty subdirectories + subfolders = (d for d in cwd.iterdir() if d.is_dir()) + for subfolder in subfolders: + if not any(subfolder.iterdir()): + subfolder.rmdir() def export_notebook(notebook_path: Path, config: NbAutoexportConfig): diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index 5ab5f24..9a8a7dc 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -1,7 +1,6 @@ import logging import os from pathlib import Path -import shutil from typing import List import typer @@ -10,7 +9,6 @@ from nbautoexport.convert import export_notebook from nbautoexport.sentinel import ( ExportFormat, - find_unwanted_outputs, install_sentinel, NbAutoexportConfig, OrganizeBy, @@ -49,32 +47,51 @@ def main( def clean( directory: Path = typer.Argument( ..., exists=True, file_okay=False, dir_okay=True, writable=True - ) + ), + yes: bool = typer.Option( + False, "--yes", "-y", help="Assume 'yes' answer to confirmation prompt to delete files." + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Show files that would be removed, without actually removing." + ), ): - """Remove subfolders/files not matching .nbautoconvert configuration. + """Remove subfolders/files not matching .nbautoconvert configuration and existing notebooks. """ sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json") - to_remove = find_unwanted_outputs(directory, config) + # Remove files that are not notebooks or expected files + notebook_paths = sorted(directory.glob("*.ipynb")) + expected_exports = [directory / export for export in config.expected_exports(notebook_paths)] + subfiles = (f for f in directory.glob("**/*") if f.is_file()) + checkpoints = (f for f in directory.glob(".ipynb_checkpoints/*") if f.is_file()) + to_remove = ( + set(subfiles) + .difference(notebook_paths) + .difference(expected_exports) + .difference(checkpoints) + .difference([sentinel_path]) + ) typer.echo("Removing following files:") - for path in to_remove: - typer.echo(" " + path) - if path.is_dir(): - for subpath in path.iterdir(): - typer.echo(" " + subpath) + for path in sorted(to_remove): + typer.echo(f" {path}") + + if dry_run: + typer.echo("Dry run completed. Exiting...") + typer.Exit() - delete = typer.confirm("Are you sure you want to delete these files?") - if not delete: - typer.echo("Not deleting") - raise typer.Abort() + if not yes: + typer.confirm("Are you sure you want to delete these files?", abort=True) for path in to_remove: - if path.is_dir(): - shutil.rmtree(path) - else: - os.remove(path) + os.remove(path) + + # Remove empty subdirectories + subfolders = (d for d in directory.iterdir() if d.is_dir()) + for subfolder in subfolders: + if not any(subfolder.iterdir()): + subfolder.rmdir() typer.echo("Files deleted.") @@ -196,7 +213,3 @@ def install( raise typer.Exit(code=1) install_post_save_hook() - - -if __name__ == "__main__": - app() diff --git a/nbautoexport/sentinel.py b/nbautoexport/sentinel.py index 1da4c07..75c75e6 100644 --- a/nbautoexport/sentinel.py +++ b/nbautoexport/sentinel.py @@ -1,11 +1,11 @@ from enum import Enum from pathlib import Path -from typing import List +from typing import Iterable, List from pydantic import BaseModel from nbautoexport.utils import logger - +from nbconvert.exporters import get_exporter, PythonExporter SAVE_PROGRESS_INDICATOR_FILE = ".nbautoexport" @@ -21,7 +21,16 @@ class ExportFormat(str, Enum): notebook = "notebook" @classmethod - def has_value(cls, value): + def get_extension(cls, value: str) -> str: + exporter = get_exporter(cls(value).value) + return exporter().file_extension + + @staticmethod + def get_script_extensions() -> List[str]: + return [exporter().file_extension for exporter in [PythonExporter]] + + @classmethod + def has_value(cls, value: str) -> bool: return any(level for level in cls if level.value == value) @@ -31,10 +40,41 @@ class OrganizeBy(str, Enum): class NbAutoexportConfig(BaseModel): - export_formats: List[ExportFormat] = ["script"] - organize_by: OrganizeBy = "extension" + export_formats: List[ExportFormat] = [ExportFormat.script] + organize_by: OrganizeBy = OrganizeBy.extension clean: bool = False + def expected_exports(self, notebook_paths: Iterable[Path]) -> List[Path]: + notebook_names: List[str] = [notebook.stem for notebook in notebook_paths] + if self.organize_by == OrganizeBy.notebook: + export_paths = [ + Path(notebook) / f"{notebook}{ExportFormat.get_extension(export_format)}" + for notebook in notebook_names + for export_format in self.export_formats + ] + # special case for script, since it depends on language + if ExportFormat.script in self.export_formats: + export_paths += [ + Path(notebook) / f"{notebook}{extension}" + for notebook in notebook_names + for extension in ExportFormat.get_script_extensions() + ] + elif self.organize_by == OrganizeBy.extension: + export_paths = [ + Path(export_format.value) + / f"{notebook}{ExportFormat.get_extension(export_format)}" + for notebook in notebook_names + for export_format in self.export_formats + ] + # special case for script, since it depends on language + if ExportFormat.script in self.export_formats: + export_paths += [ + Path(ExportFormat.script.value) / f"{notebook}{extension}" + for notebook in notebook_names + for extension in ExportFormat.get_script_extensions() + ] + return sorted(export_paths) + def install_sentinel( export_formats: List[ExportFormat], organize_by: OrganizeBy, directory: Path, overwrite: bool @@ -61,37 +101,3 @@ def install_sentinel( logger.info(f"\n{config.json(indent=2)}") with sentinel_path.open("w") as fp: fp.write(config.json(indent=2)) - - -def find_unwanted_outputs(directory: Path, config: NbAutoexportConfig) -> List[Path]: - """Determine which subfolders or files in a given directory are not expected to be generated by - current notebooks in that directory and given NbAutoexportConfig. - - Args: - directory (Path): directory of notebooks - config (NbAutoexportConfig): nbautoexport configuration - - Returns: - List[Path]: list of paths - """ - notebook_paths = directory.glob("*.ipynb") - notebook_names = (f.stem for f in notebook_paths) - - to_remove = [] - subfolders = (f for f in directory.iterdir() if f.is_dir()) - if config.organize_by == "notebook": - for subfolder in subfolders: - if subfolder.name not in notebook_names: - to_remove.append(subfolder) - elif config.organize_by == "extension": - for subfolder in subfolders: - if subfolder.name in config.export_formats: - # Check for individual files to remove - for subfolder_file in subfolder.glob("*"): - if subfolder_file.stem not in notebook_names: - to_remove.append(subfolder_file) - else: - # Otherwise to remove entire subfolder - to_remove.append(subfolder) - - return to_remove diff --git a/requirements.txt b/requirements.txt index 1b62dbd..577c68f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ jupyter-contrib-nbextensions>=0.5.1 nbconvert>=5.6.1 pydantic -typer>=0.2.1 +typer>=0.3.0 diff --git a/tests/test_cli.py b/tests/test_cli.py index df2f037..8c7b7e6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,17 +1,21 @@ -"""Tests for `nbautoexport` package.""" +import subprocess -import json - -from nbconvert.exporters import get_export_names from typer.testing import CliRunner -from nbautoexport import __version__ from nbautoexport.nbautoexport import app -from nbautoexport.sentinel import ExportFormat, install_sentinel, NbAutoexportConfig +from nbautoexport import __version__ -def test_cli(): - """Test the CLI.""" +def test_main(): + """Test the CLI main callback.""" + runner = CliRunner() + result = runner.invoke(app) + assert result.exit_code == 0 + assert "Exports Jupyter notebooks to various file formats" in result.output + + +def test_main_help(): + """Test the CLI main callback with --help flag.""" runner = CliRunner() result = runner.invoke(app, ["--help"]) assert result.exit_code == 0 @@ -19,70 +23,24 @@ def test_cli(): def test_version(): - """Test the CLI.""" + """Test the CLI main callback with --version flag.""" runner = CliRunner() result = runner.invoke(app, ["--version"]) assert result.exit_code == 0 assert result.output.strip() == __version__ -def test_invalid_export_format(): - runner = CliRunner() - result = runner.invoke(app, ["install", "-f", "invalid-output-format"]) - assert result.exit_code == 2 - assert ( - "Error: Invalid value for '--export-format' / '-f': invalid choice: invalid-output-format" - in result.output - ) - - -def test_export_format_compatibility(): - """Test that export formats are compatible with Jupyter nbautoconvert. - """ - nbconvert_export_names = get_export_names() - for export_format in ExportFormat: - assert export_format.value in nbconvert_export_names +def test_main_python_m(): + result = subprocess.run(["python", "-m", "nbautoexport"], capture_output=True, text=True) + assert result.returncode == 0 + assert "Exports Jupyter notebooks to various file formats" in result.stdout + assert result.stdout.startswith("Usage: python -m nbautoexport") + assert "Usage: __main__.py" not in result.stdout -def test_invalid_organize_by(): - runner = CliRunner() - result = runner.invoke(app, ["install", "-b", "invalid-organize-by"]) - assert result.exit_code == 2 - assert ( - "Invalid value for '--organize-by' / '-b': invalid choice: invalid-organize-by" - in result.output +def test_version_python_m(): + result = subprocess.run( + ["python", "-m", "nbautoexport", "--version"], capture_output=True, text=True ) - - -def test_refuse_overwrite(tmp_path): - (tmp_path / ".nbautoexport").touch() - runner = CliRunner() - result = runner.invoke(app, ["install", str(tmp_path)]) - assert result.exit_code == 1 - assert "Detected existing autoexport configuration at" in result.output - - -def test_force_overwrite(tmp_path): - (tmp_path / ".nbautoexport").touch() - runner = CliRunner() - result = runner.invoke( - app, ["install", str(tmp_path), "-o", "-f", "script", "-f", "html", "-b", "notebook"] - ) - print(result.output) - print(result.exit_code) - assert result.exit_code == 0 - with (tmp_path / ".nbautoexport").open("r") as fp: - config = json.load(fp) - - expected_config = NbAutoexportConfig(export_formats=["script", "html"], organize_by="notebook") - assert config == expected_config - - -def test_install_sentinel(tmp_path): - export_formats = ["script", "html"] - install_sentinel(export_formats, organize_by="notebook", directory=tmp_path, overwrite=False) - with (tmp_path / ".nbautoexport").open("r") as fp: - config = json.load(fp) - - expected_config = NbAutoexportConfig(export_formats=export_formats, organize_by="notebook") - assert config == expected_config + assert result.returncode == 0 + assert result.stdout.strip() == __version__ diff --git a/tests/test_cli_install.py b/tests/test_cli_install.py new file mode 100644 index 0000000..1dbb824 --- /dev/null +++ b/tests/test_cli_install.py @@ -0,0 +1,71 @@ +"""Tests for `nbautoexport` package.""" + +import json + +from nbconvert.exporters import get_export_names +from typer.testing import CliRunner + +from nbautoexport.nbautoexport import app +from nbautoexport.sentinel import ExportFormat, install_sentinel, NbAutoexportConfig + + +def test_invalid_export_format(): + runner = CliRunner() + result = runner.invoke(app, ["install", "-f", "invalid-output-format"]) + assert result.exit_code == 2 + assert ( + "Error: Invalid value for '--export-format' / '-f': invalid choice: invalid-output-format" + in result.output + ) + + +def test_export_format_compatibility(): + """Test that export formats are compatible with Jupyter nbautoconvert. + """ + nbconvert_export_names = get_export_names() + for export_format in ExportFormat: + assert export_format.value in nbconvert_export_names + + +def test_invalid_organize_by(): + runner = CliRunner() + result = runner.invoke(app, ["install", "-b", "invalid-organize-by"]) + assert result.exit_code == 2 + assert ( + "Invalid value for '--organize-by' / '-b': invalid choice: invalid-organize-by" + in result.output + ) + + +def test_refuse_overwrite(tmp_path): + (tmp_path / ".nbautoexport").touch() + runner = CliRunner() + result = runner.invoke(app, ["install", str(tmp_path)]) + assert result.exit_code == 1 + assert "Detected existing autoexport configuration at" in result.output + + +def test_force_overwrite(tmp_path): + (tmp_path / ".nbautoexport").touch() + runner = CliRunner() + result = runner.invoke( + app, ["install", str(tmp_path), "-o", "-f", "script", "-f", "html", "-b", "notebook"] + ) + print(result.output) + print(result.exit_code) + assert result.exit_code == 0 + with (tmp_path / ".nbautoexport").open("r") as fp: + config = json.load(fp) + + expected_config = NbAutoexportConfig(export_formats=["script", "html"], organize_by="notebook") + assert config == expected_config + + +def test_install_sentinel(tmp_path): + export_formats = ["script", "html"] + install_sentinel(export_formats, organize_by="notebook", directory=tmp_path, overwrite=False) + with (tmp_path / ".nbautoexport").open("r") as fp: + config = json.load(fp) + + expected_config = NbAutoexportConfig(export_formats=export_formats, organize_by="notebook") + assert config == expected_config diff --git a/tests/test_convert.py b/tests/test_convert.py index 7c65ab1..15afc3d 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -9,7 +9,7 @@ NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" -def test_no_sentinel(tmp_path): +def test_post_save_no_sentinel(tmp_path): notebook_path = tmp_path / "the_notebook.ipynb" sentinel_path = tmp_path / SAVE_PROGRESS_INDICATOR_FILE shutil.copy(NOTEBOOK_FILE, notebook_path) @@ -20,7 +20,7 @@ def test_no_sentinel(tmp_path): assert set(tmp_path.iterdir()) == {notebook_path} -def test_organize_by_notebook(tmp_path, monkeypatch): +def test_post_save_organize_by_notebook(tmp_path, monkeypatch): notebook_path = tmp_path / "the_notebook.ipynb" sentinel_path = tmp_path / SAVE_PROGRESS_INDICATOR_FILE shutil.copy(NOTEBOOK_FILE, notebook_path) @@ -42,7 +42,7 @@ def test_organize_by_notebook(tmp_path, monkeypatch): assert (tmp_path / "the_notebook" / "the_notebook.html").exists() -def test_organize_by_extension(tmp_path, monkeypatch): +def test_post_save_organize_by_extension(tmp_path, monkeypatch): notebook_path = tmp_path / "the_notebook.ipynb" sentinel_path = tmp_path / SAVE_PROGRESS_INDICATOR_FILE shutil.copy(NOTEBOOK_FILE, notebook_path) @@ -66,7 +66,7 @@ def test_organize_by_extension(tmp_path, monkeypatch): assert (tmp_path / "html" / "the_notebook.html").exists() -def test_type_file(tmp_path, monkeypatch): +def test_post_save_type_file(tmp_path, monkeypatch): notebook_path = tmp_path / "the_notebook.ipynb" sentinel_path = tmp_path / SAVE_PROGRESS_INDICATOR_FILE shutil.copy(NOTEBOOK_FILE, notebook_path) @@ -84,7 +84,7 @@ def test_type_file(tmp_path, monkeypatch): } -def test_type_directory(tmp_path, monkeypatch): +def test_post_save_type_directory(tmp_path, monkeypatch): notebook_path = tmp_path / "the_notebook.ipynb" sentinel_path = tmp_path / SAVE_PROGRESS_INDICATOR_FILE shutil.copy(NOTEBOOK_FILE, notebook_path) From e2edc588118743fd34f63da0a58333b7454a2b4a Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Fri, 26 Jun 2020 17:32:55 -0700 Subject: [PATCH 06/25] Rename convert module to export --- nbautoexport/__init__.py | 2 +- nbautoexport/{convert.py => export.py} | 0 nbautoexport/nbautoexport.py | 2 +- tests/t_cli_clean.py | 68 +++++++++++++++++++++++ tests/{test_convert.py => test_export.py} | 2 +- tests/test_jupyter_config.py | 4 +- 6 files changed, 73 insertions(+), 5 deletions(-) rename nbautoexport/{convert.py => export.py} (100%) create mode 100644 tests/t_cli_clean.py rename tests/{test_convert.py => test_export.py} (98%) diff --git a/nbautoexport/__init__.py b/nbautoexport/__init__.py index 2c4c3e6..bcbb331 100644 --- a/nbautoexport/__init__.py +++ b/nbautoexport/__init__.py @@ -1,4 +1,4 @@ -from nbautoexport.convert import post_save +from nbautoexport.export import post_save from nbautoexport.utils import __version__ __all__ = [post_save, __version__] diff --git a/nbautoexport/convert.py b/nbautoexport/export.py similarity index 100% rename from nbautoexport/convert.py rename to nbautoexport/export.py diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index 9a8a7dc..4e38424 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -6,7 +6,7 @@ import typer from nbautoexport.jupyter_config import install_post_save_hook -from nbautoexport.convert import export_notebook +from nbautoexport.export import export_notebook from nbautoexport.sentinel import ( ExportFormat, install_sentinel, diff --git a/tests/t_cli_clean.py b/tests/t_cli_clean.py new file mode 100644 index 0000000..3b4d2f8 --- /dev/null +++ b/tests/t_cli_clean.py @@ -0,0 +1,68 @@ +import itertools +from pathlib import Path +import shutil + +import pytest +from typer.testing import CliRunner + +from nbautoexport.nbautoexport import app +from nbautoexport.sentinel import ( + ExportFormat, + NbAutoexportConfig, + SAVE_PROGRESS_INDICATOR_FILE, + find_unwanted_outputs, +) + +NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" + + +@pytest.fixture() +def notebooks_dir(tmp_path): + notebooks = [f"the_notebook_{n}" for n in range(3)] + for nb in notebooks: + shutil.copy(NOTEBOOK_FILE, tmp_path / f"{nb}.ipynb") + + # organize_by notebook + nb_subfolder = tmp_path / nb + nb_subfolder.mkdir() + (nb_subfolder / f"{nb}.py").touch() + + # organize_by extension + for export_format in ExportFormat: + format_subfolder = tmp_path / export_format.value + format_subfolder.mkdir(exist_ok=True) + (format_subfolder / f"{nb}.ext").touch() + + return tmp_path + + +@pytest.mark.parametrize( + "export_formats, organize_by", + itertools.product([["script", "html"]], ["notebook", "extension"]), +) +def test_clean(notebooks_dir, export_formats, organize_by): + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + with sentinel_path.open("w") as fp: + fp.write(NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by).json()) + + result = CliRunner().invoke(app, ["clean", str(notebooks_dir), "--yes"]) + print("---") + print(result.stdout) + print("---") + assert result.exit_code == 0 + + +def test_clean_no_directory(): + result = CliRunner().invoke(app, ["clean"]) + + assert result.exit_code == 2 + assert "Error: Missing argument 'DIRECTORY'." in result.stdout + + +@pytest.mark.parametrize( + "export_formats, organize_by", + itertools.product([["script", "html"]], ["notebook", "extension"]), +) +def test_find(notebooks_dir, export_formats, organize_by): + config = NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by) + result = find_unwanted_outputs(notebooks_dir, config) diff --git a/tests/test_convert.py b/tests/test_export.py similarity index 98% rename from tests/test_convert.py rename to tests/test_export.py index 15afc3d..8713242 100644 --- a/tests/test_convert.py +++ b/tests/test_export.py @@ -3,7 +3,7 @@ import shutil import sys -from nbautoexport.convert import post_save +from nbautoexport.export import post_save from nbautoexport.sentinel import SAVE_PROGRESS_INDICATOR_FILE NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" diff --git a/tests/test_jupyter_config.py b/tests/test_jupyter_config.py index 26869e1..b90b5a0 100644 --- a/tests/test_jupyter_config.py +++ b/tests/test_jupyter_config.py @@ -8,7 +8,7 @@ from nbautoexport import __version__ import nbautoexport as nbautoexport_root import nbautoexport.nbautoexport as nbautoexport -from nbautoexport import convert, jupyter_config +from nbautoexport import export, jupyter_config def test_parse_version(): @@ -209,7 +209,7 @@ def test_initialize_post_save_binding(): jupyter_config_obj = Config(FileContentsManager=FileContentsManager()) jupyter_config.initialize_post_save_hook(jupyter_config_obj) assert isinstance(jupyter_config_obj.FileContentsManager, FileContentsManager) - assert jupyter_config_obj.FileContentsManager.post_save_hook is convert.post_save + assert jupyter_config_obj.FileContentsManager.post_save_hook is export.post_save def test_initialize_post_save_execution(monkeypatch): From ae1344b710439b767282fa4f6f84d103f087b700 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Fri, 26 Jun 2020 22:29:24 -0700 Subject: [PATCH 07/25] Simplify cleaning; fix pdf convert bug --- nbautoexport/export.py | 29 +++++++++++----- nbautoexport/nbautoexport.py | 66 ++++++++++++++++++++---------------- nbautoexport/sentinel.py | 42 +++++++++++++++++++++-- 3 files changed, 96 insertions(+), 41 deletions(-) diff --git a/nbautoexport/export.py b/nbautoexport/export.py index b2ef73a..f6ba4af 100644 --- a/nbautoexport/export.py +++ b/nbautoexport/export.py @@ -7,18 +7,20 @@ from notebook.services.contents.filemanager import FileContentsManager from nbautoexport.sentinel import ( + ExportFormat, NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE, ) class CopyToSubfolderPostProcessor(PostProcessorBase): - def __init__(self, subfolder=None): + def __init__(self, subfolder: str, export_format: ExportFormat): self.subfolder = subfolder + self.export_format = export_format super(CopyToSubfolderPostProcessor, self).__init__() def postprocess(self, input: str): - """ Save converted file to a separate directory. """ + """ Save converted file to a separate directory, removing cell numbers.""" if self.subfolder is None: return @@ -28,12 +30,22 @@ def postprocess(self, input: str): new_dir.mkdir(exist_ok=True) new_path = new_dir / input.name + if self.export_format == ExportFormat.pdf: + # Can't read pdf file as unicode, skip rest of postprocessing and just copy + input.replace(new_path) + return + + # Rewrite converted file to new path, removing cell numbers with input.open("r") as f: text = f.read() - with new_path.open("w") as f: f.write(re.sub(r"\n#\sIn\[(([0-9]+)|(\s))\]:\n{2}", "", text)) + # For markdown files, we also need to move the assets directory, for stuff like images + if self.export_format == ExportFormat.markdown: + assets_dir = input.parent / f"{input.stem}_files" + assets_dir.replace(new_dir / f"{input.stem}_files") + os.remove(input) @@ -68,11 +80,8 @@ def post_save(model: dict, os_path: str, contents_manager: FileContentsManager): if config.clean: # Remove files that are not notebooks or expected files - notebook_paths = cwd.glob("*.ipynb") - expected_exports = [cwd / export for export in config.expected_exports(notebook_paths)] - subfiles = (f for f in cwd.rglob("*") if f.is_file()) - to_remove = set(subfiles).difference(notebook_paths).difference(expected_exports) - for path in to_remove: + files_to_clean = config.files_to_clean(cwd) + for path in files_to_clean: os.remove(path) # Remove empty subdirectories @@ -92,7 +101,9 @@ def export_notebook(notebook_path: Path, config: NbAutoexportConfig): elif config.organize_by == "extension": subfolder = export_format.value - converter.postprocessor = CopyToSubfolderPostProcessor(subfolder=subfolder) + converter.postprocessor = CopyToSubfolderPostProcessor( + subfolder=subfolder, export_format=export_format + ) converter.export_format = export_format.value converter.initialize() converter.notebooks = [str(notebook_path)] diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index 4e38424..9c32987 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -1,6 +1,7 @@ import logging import os from pathlib import Path +import sys from typing import List import typer @@ -8,6 +9,8 @@ from nbautoexport.jupyter_config import install_post_save_hook from nbautoexport.export import export_notebook from nbautoexport.sentinel import ( + DEFAULT_EXPORT_FORMATS, + DEFAULT_ORGANIZE_BY, ExportFormat, install_sentinel, NbAutoexportConfig, @@ -19,6 +22,12 @@ app = typer.Typer() +def validate_sentinel_path(path: Path): + if not path.exists(): + typer.echo(f"Error: Missing expected nbautoexport config file [{path.resolve()}].") + raise typer.Exit(code=1) + + def version_callback(value: bool): if value: typer.echo(__version__) @@ -58,48 +67,43 @@ def clean( """Remove subfolders/files not matching .nbautoconvert configuration and existing notebooks. """ sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE + validate_sentinel_path(sentinel_path) + config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json") - # Remove files that are not notebooks or expected files - notebook_paths = sorted(directory.glob("*.ipynb")) - expected_exports = [directory / export for export in config.expected_exports(notebook_paths)] - subfiles = (f for f in directory.glob("**/*") if f.is_file()) - checkpoints = (f for f in directory.glob(".ipynb_checkpoints/*") if f.is_file()) - to_remove = ( - set(subfiles) - .difference(notebook_paths) - .difference(expected_exports) - .difference(checkpoints) - .difference([sentinel_path]) - ) - - typer.echo("Removing following files:") - for path in sorted(to_remove): + files_to_clean = config.files_to_clean(directory) + + typer.echo("Identified following files to clean up:") + for path in sorted(files_to_clean): typer.echo(f" {path}") if dry_run: - typer.echo("Dry run completed. Exiting...") - typer.Exit() + typer.echo("Dry run completed. Exiting.") + raise typer.Exit(code=0) if not yes: typer.confirm("Are you sure you want to delete these files?", abort=True) - for path in to_remove: + typer.echo("Removing identified files...") + for path in files_to_clean: os.remove(path) # Remove empty subdirectories + typer.echo("Removing empty subdirectories...") subfolders = (d for d in directory.iterdir() if d.is_dir()) for subfolder in subfolders: if not any(subfolder.iterdir()): + typer.echo(f" {subfolder}") subfolder.rmdir() - typer.echo("Files deleted.") + + typer.echo("Cleaning complete.") @app.command() def convert( input: Path = typer.Argument(..., exists=True, file_okay=True, dir_okay=True, writable=True), export_formats: List[ExportFormat] = typer.Option( - ["script"], + DEFAULT_EXPORT_FORMATS, "--export-format", "-f", show_default=True, @@ -110,7 +114,7 @@ def convert( ), ), organize_by: OrganizeBy = typer.Option( - "extension", + DEFAULT_ORGANIZE_BY, "--organize-by", "-b", show_default=True, @@ -125,19 +129,18 @@ def convert( INPUT is the path to a notebook to be converted, or a directory containing notebooks to be converted. """ + sys.argv = [sys.argv[0]] config = NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by) if input.is_dir(): for notebook_path in input.glob("*.ipynb"): export_notebook(notebook_path, config=config) else: - export_notebook(notebook_path, config=config) + export_notebook(input, config=config) @app.command() def export( - input: Path = typer.Argument( - "extension", exists=True, file_okay=True, dir_okay=True, writable=True - ) + input: Path = typer.Argument(..., exists=True, file_okay=True, dir_okay=True, writable=True) ): """Convert notebook(s) using existing configuration file. @@ -146,13 +149,16 @@ def export( A .nbautoconvert configuration file is required to be in the same directory as the notebook(s). """ + sys.argv = [sys.argv[0]] if input.is_dir(): sentinel_path = input / SAVE_PROGRESS_INDICATOR_FILE - config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json") - for notebook_path in input.glob("*.ipynb"): - export_notebook(notebook_path, config=config) + notebook_paths = input.glob("*.ipynb") else: sentinel_path = input.parent / SAVE_PROGRESS_INDICATOR_FILE + notebook_paths = [input] + + validate_sentinel_path(sentinel_path) + for notebook_path in notebook_paths: config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json") export_notebook(notebook_path, config=config) @@ -163,7 +169,7 @@ def install( "extension", exists=True, file_okay=False, dir_okay=True, writable=True ), export_formats: List[ExportFormat] = typer.Option( - ["script"], + DEFAULT_EXPORT_FORMATS, "--export-format", "-f", show_default=True, @@ -174,7 +180,7 @@ def install( ), ), organize_by: OrganizeBy = typer.Option( - "notebook", + DEFAULT_ORGANIZE_BY, "--organize-by", "-b", show_default=True, diff --git a/nbautoexport/sentinel.py b/nbautoexport/sentinel.py index 75c75e6..7acc98e 100644 --- a/nbautoexport/sentinel.py +++ b/nbautoexport/sentinel.py @@ -7,7 +7,10 @@ from nbautoexport.utils import logger from nbconvert.exporters import get_exporter, PythonExporter + SAVE_PROGRESS_INDICATOR_FILE = ".nbautoexport" +DEFAULT_EXPORT_FORMATS = ["script"] +DEFAULT_ORGANIZE_BY = "extension" class ExportFormat(str, Enum): @@ -40,11 +43,20 @@ class OrganizeBy(str, Enum): class NbAutoexportConfig(BaseModel): - export_formats: List[ExportFormat] = [ExportFormat.script] - organize_by: OrganizeBy = OrganizeBy.extension + export_formats: List[ExportFormat] = [ExportFormat(fmt) for fmt in DEFAULT_EXPORT_FORMATS] + organize_by: OrganizeBy = OrganizeBy(DEFAULT_ORGANIZE_BY) clean: bool = False def expected_exports(self, notebook_paths: Iterable[Path]) -> List[Path]: + """Given paths to a set of notebook files, return list of paths of files that nbautoexport + would be expected to export to given this configuration. + + Args: + notebook_paths (Iterable[Path]): iterable of notebook file paths + + Returns: + List[Path]: list of expected nbautoexport output files, relative to notebook files + """ notebook_names: List[str] = [notebook.stem for notebook in notebook_paths] if self.organize_by == OrganizeBy.notebook: export_paths = [ @@ -75,6 +87,32 @@ def expected_exports(self, notebook_paths: Iterable[Path]) -> List[Path]: ] return sorted(export_paths) + def files_to_clean(self, directory: Path) -> List[Path]: + """Given path to a notebooks directory watched by nbautoexport, find all files that are not + expected exports by current nbautoexport configuration and existing notebooks, or other + expected Jupyter or nbautoexport files. + + Args: + directory (Path): notebooks directory to find files to clean up + + Returns: + List[Path]: list of files to clean up + """ + notebook_paths = list(directory.glob("*.ipynb")) + expected_exports = [directory / export for export in self.expected_exports(notebook_paths)] + subfiles = (f for f in directory.glob("**/*") if f.is_file()) + checkpoints = (f for f in directory.glob(".ipynb_checkpoints/*") if f.is_file()) + sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE + + to_clean = ( + set(subfiles) + .difference(notebook_paths) + .difference(expected_exports) + .difference(checkpoints) + .difference([sentinel_path]) + ) + return sorted(to_clean) + def install_sentinel( export_formats: List[ExportFormat], organize_by: OrganizeBy, directory: Path, overwrite: bool From 0f2d2d14a2f2c3a5c79529843f040cf56fbfe737 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Sat, 27 Jun 2020 00:24:36 -0700 Subject: [PATCH 08/25] Lots of tests --- nbautoexport/export.py | 34 ++++---- nbautoexport/nbautoexport.py | 3 - nbautoexport/sentinel.py | 17 ++-- nbautoexport/utils.py | 13 +++ tests/t_cli_clean.py | 68 --------------- tests/test_cli_clean.py | 155 +++++++++++++++++++++++++++++++++++ tests/test_cli_convert.py | 108 ++++++++++++++++++++++++ tests/test_cli_export.py | 144 ++++++++++++++++++++++++++++++++ tests/test_cli_install.py | 11 +-- tests/test_export.py | 142 +++++++++++++++++++++----------- tests/test_jupyter_config.py | 12 +++ tests/test_sentinel.py | 33 ++++++++ 12 files changed, 587 insertions(+), 153 deletions(-) delete mode 100644 tests/t_cli_clean.py create mode 100644 tests/test_cli_clean.py create mode 100644 tests/test_cli_convert.py create mode 100644 tests/test_cli_export.py create mode 100644 tests/test_sentinel.py diff --git a/nbautoexport/export.py b/nbautoexport/export.py index f6ba4af..ed63395 100644 --- a/nbautoexport/export.py +++ b/nbautoexport/export.py @@ -11,6 +11,7 @@ NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE, ) +from nbautoexport.utils import cleared_argv class CopyToSubfolderPostProcessor(PostProcessorBase): @@ -92,19 +93,20 @@ def post_save(model: dict, os_path: str, contents_manager: FileContentsManager): def export_notebook(notebook_path: Path, config: NbAutoexportConfig): - converter = NbConvertApp() - - for export_format in config.export_formats: - if config.organize_by == "notebook": - subfolder = notebook_path.stem - - elif config.organize_by == "extension": - subfolder = export_format.value - - converter.postprocessor = CopyToSubfolderPostProcessor( - subfolder=subfolder, export_format=export_format - ) - converter.export_format = export_format.value - converter.initialize() - converter.notebooks = [str(notebook_path)] - converter.convert_notebooks() + with cleared_argv(): + converter = NbConvertApp() + + for export_format in config.export_formats: + if config.organize_by == "notebook": + subfolder = notebook_path.stem + + elif config.organize_by == "extension": + subfolder = export_format.value + + converter.postprocessor = CopyToSubfolderPostProcessor( + subfolder=subfolder, export_format=export_format + ) + converter.export_format = export_format.value + converter.initialize() + converter.notebooks = [str(notebook_path)] + converter.convert_notebooks() diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index 9c32987..533f553 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -1,7 +1,6 @@ import logging import os from pathlib import Path -import sys from typing import List import typer @@ -129,7 +128,6 @@ def convert( INPUT is the path to a notebook to be converted, or a directory containing notebooks to be converted. """ - sys.argv = [sys.argv[0]] config = NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by) if input.is_dir(): for notebook_path in input.glob("*.ipynb"): @@ -149,7 +147,6 @@ def export( A .nbautoconvert configuration file is required to be in the same directory as the notebook(s). """ - sys.argv = [sys.argv[0]] if input.is_dir(): sentinel_path = input / SAVE_PROGRESS_INDICATOR_FILE notebook_paths = input.glob("*.ipynb") diff --git a/nbautoexport/sentinel.py b/nbautoexport/sentinel.py index 7acc98e..2c7c2be 100644 --- a/nbautoexport/sentinel.py +++ b/nbautoexport/sentinel.py @@ -1,6 +1,6 @@ from enum import Enum from pathlib import Path -from typing import Iterable, List +from typing import Iterable, List, Optional from pydantic import BaseModel @@ -14,18 +14,23 @@ class ExportFormat(str, Enum): + asciidoc = "asciidoc" html = "html" latex = "latex" - pdf = "pdf" - slides = "slides" markdown = "markdown" - asciidoc = "asciidoc" - script = "script" notebook = "notebook" + pdf = "pdf" + rst = "rst" + script = "script" + slides = "slides" @classmethod - def get_extension(cls, value: str) -> str: + def get_extension(cls, value: str, language: Optional[str] = None) -> str: + if cls(value) == cls.script and language == "python": + return PythonExporter().file_extension exporter = get_exporter(cls(value).value) + if cls(value) == cls.notebook: + return f".nbconvert{exporter().file_extension}" return exporter().file_extension @staticmethod diff --git a/nbautoexport/utils.py b/nbautoexport/utils.py index 45219fe..20e0b69 100644 --- a/nbautoexport/utils.py +++ b/nbautoexport/utils.py @@ -1,5 +1,18 @@ +from contextlib import contextmanager import logging +import sys + from nbautoexport._version import get_versions logger = logging.getLogger("nbautoexport") __version__ = get_versions()["version"] + + +@contextmanager +def cleared_argv(): + prev_argv = [arg for arg in sys.argv] + sys.argv = [sys.argv[0]] + try: + yield + finally: + sys.argv = prev_argv diff --git a/tests/t_cli_clean.py b/tests/t_cli_clean.py deleted file mode 100644 index 3b4d2f8..0000000 --- a/tests/t_cli_clean.py +++ /dev/null @@ -1,68 +0,0 @@ -import itertools -from pathlib import Path -import shutil - -import pytest -from typer.testing import CliRunner - -from nbautoexport.nbautoexport import app -from nbautoexport.sentinel import ( - ExportFormat, - NbAutoexportConfig, - SAVE_PROGRESS_INDICATOR_FILE, - find_unwanted_outputs, -) - -NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" - - -@pytest.fixture() -def notebooks_dir(tmp_path): - notebooks = [f"the_notebook_{n}" for n in range(3)] - for nb in notebooks: - shutil.copy(NOTEBOOK_FILE, tmp_path / f"{nb}.ipynb") - - # organize_by notebook - nb_subfolder = tmp_path / nb - nb_subfolder.mkdir() - (nb_subfolder / f"{nb}.py").touch() - - # organize_by extension - for export_format in ExportFormat: - format_subfolder = tmp_path / export_format.value - format_subfolder.mkdir(exist_ok=True) - (format_subfolder / f"{nb}.ext").touch() - - return tmp_path - - -@pytest.mark.parametrize( - "export_formats, organize_by", - itertools.product([["script", "html"]], ["notebook", "extension"]), -) -def test_clean(notebooks_dir, export_formats, organize_by): - sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE - with sentinel_path.open("w") as fp: - fp.write(NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by).json()) - - result = CliRunner().invoke(app, ["clean", str(notebooks_dir), "--yes"]) - print("---") - print(result.stdout) - print("---") - assert result.exit_code == 0 - - -def test_clean_no_directory(): - result = CliRunner().invoke(app, ["clean"]) - - assert result.exit_code == 2 - assert "Error: Missing argument 'DIRECTORY'." in result.stdout - - -@pytest.mark.parametrize( - "export_formats, organize_by", - itertools.product([["script", "html"]], ["notebook", "extension"]), -) -def test_find(notebooks_dir, export_formats, organize_by): - config = NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by) - result = find_unwanted_outputs(notebooks_dir, config) diff --git a/tests/test_cli_clean.py b/tests/test_cli_clean.py new file mode 100644 index 0000000..6447c99 --- /dev/null +++ b/tests/test_cli_clean.py @@ -0,0 +1,155 @@ +from pathlib import Path +import shutil + +import pytest +from typer.testing import CliRunner + +from nbautoexport.nbautoexport import app +from nbautoexport.sentinel import ( + ExportFormat, + NbAutoexportConfig, + SAVE_PROGRESS_INDICATOR_FILE, +) + +NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" + +EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] +UNEXPECTED_NOTEBOOK = "a_walk_to_remember" +EXPECTED_FORMATS = ["markdown", "html"] +UNEXPECTED_FORMAT = "latex" + + +@pytest.fixture() +def notebooks_dir(tmp_path): + notebooks = EXPECTED_NOTEBOOKS + [UNEXPECTED_NOTEBOOK] + export_formats = [ExportFormat(fmt) for fmt in EXPECTED_FORMATS + [UNEXPECTED_FORMAT]] + for nb in notebooks: + shutil.copy(NOTEBOOK_FILE, tmp_path / f"{nb}.ipynb") + + # organize_by notebook + nb_subfolder = tmp_path / nb + nb_subfolder.mkdir() + for fmt in export_formats: + (nb_subfolder / f"{nb}{ExportFormat.get_extension(fmt)}").touch() + + # organize_by extension + for fmt in export_formats: + format_subfolder = tmp_path / fmt.value + format_subfolder.mkdir(exist_ok=True) + (format_subfolder / f"{nb}{ExportFormat.get_extension(fmt)}").touch() + + (tmp_path / f"{UNEXPECTED_NOTEBOOK}.ipynb").unlink() + + return tmp_path + + +@pytest.mark.parametrize("need_confirmation", [True, False]) +def test_clean_organize_by_notebook(notebooks_dir, need_confirmation): + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + with sentinel_path.open("w") as fp: + fp.write( + NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook").json() + ) + + if need_confirmation: + result = CliRunner().invoke(app, ["clean", str(notebooks_dir)], input="y") + else: + result = CliRunner().invoke(app, ["clean", str(notebooks_dir), "--yes"]) + assert result.exit_code == 0 + + expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} + expected_export_dirs = {notebooks_dir / nb for nb in EXPECTED_NOTEBOOKS} + expected_export_files = { + notebooks_dir / nb / f"{nb}{ExportFormat.get_extension(fmt)}" + for nb in EXPECTED_NOTEBOOKS + for fmt in EXPECTED_FORMATS + } + + all_expected = ( + expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} + ) + assert set(notebooks_dir.glob("**/*")) == all_expected + + +@pytest.mark.parametrize("need_confirmation", [True, False]) +def test_clean_organize_by_extension(notebooks_dir, need_confirmation): + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + with sentinel_path.open("w") as fp: + fp.write( + NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension").json() + ) + + if need_confirmation: + result = CliRunner().invoke(app, ["clean", str(notebooks_dir)], input="y") + else: + result = CliRunner().invoke(app, ["clean", str(notebooks_dir), "--yes"]) + assert result.exit_code == 0 + + expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} + expected_export_dirs = {notebooks_dir / fmt for fmt in EXPECTED_FORMATS} + expected_export_files = { + notebooks_dir / fmt / f"{nb}{ExportFormat.get_extension(fmt)}" + for nb in EXPECTED_NOTEBOOKS + for fmt in EXPECTED_FORMATS + } + + all_expected = ( + expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} + ) + assert set(notebooks_dir.glob("**/*")) == all_expected + + +def test_clean_abort(notebooks_dir): + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + with sentinel_path.open("w") as fp: + fp.write(NbAutoexportConfig(ExportFormat=EXPECTED_FORMATS).json()) + + starting_files = set(notebooks_dir.glob("**/*")) + + result = CliRunner().invoke(app, ["clean", str(notebooks_dir)], input="n") + assert result.exit_code == 1 + assert result.stdout.endswith("Aborted!\n") + + ending_files = set(notebooks_dir.glob("**/*")) + + # no files deleted + assert starting_files == ending_files + + +def test_clean_dry_run(notebooks_dir): + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + with sentinel_path.open("w") as fp: + fp.write(NbAutoexportConfig(ExportFormat=EXPECTED_FORMATS).json()) + + starting_files = set(notebooks_dir.glob("**/*")) + + result = CliRunner().invoke(app, ["clean", str(notebooks_dir), "--dry-run"]) + assert result.exit_code == 0 + + ending_files = set(notebooks_dir.glob("**/*")) + + # no files deleted + assert starting_files == ending_files + + +def test_clean_no_directory_error(): + result = CliRunner().invoke(app, ["clean"]) + + assert result.exit_code == 2 + assert "Error: Missing argument 'DIRECTORY'." in result.stdout + + +def test_clean_missing_config_error(notebooks_dir): + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + + starting_files = set(notebooks_dir.glob("**/*")) + + result = CliRunner().invoke(app, ["clean", str(notebooks_dir)]) + assert result.exit_code == 1 + assert "Error: Missing expected nbautoexport config file" in result.stdout + assert str(sentinel_path.resolve()) in result.stdout + + ending_files = set(notebooks_dir.glob("**/*")) + + # no files deleted + assert starting_files == ending_files diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py new file mode 100644 index 0000000..1c28c0e --- /dev/null +++ b/tests/test_cli_convert.py @@ -0,0 +1,108 @@ +from pathlib import Path +import shutil + +import pytest +from typer.testing import CliRunner + +from nbautoexport.nbautoexport import app +from nbautoexport.sentinel import ExportFormat + +NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" + +EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] +EXPECTED_FORMATS = ["script", "html"] + + +@pytest.fixture() +def notebooks_dir(tmp_path): + notebooks = EXPECTED_NOTEBOOKS + for nb in notebooks: + shutil.copy(NOTEBOOK_FILE, tmp_path / f"{nb}.ipynb") + return tmp_path + + +def test_convert_dir_organize_by_extension(notebooks_dir): + cmd_list = ["convert", str(notebooks_dir), "-b", "extension"] + for fmt in EXPECTED_FORMATS: + cmd_list.append("-f") + cmd_list.append(fmt) + result = CliRunner().invoke(app, cmd_list) + assert result.exit_code == 0 + + expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} + expected_export_dirs = {notebooks_dir / fmt for fmt in EXPECTED_FORMATS} + expected_export_files = { + notebooks_dir / fmt / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" + for nb in EXPECTED_NOTEBOOKS + for fmt in EXPECTED_FORMATS + } + + all_expected = expected_notebooks | expected_export_dirs | expected_export_files + assert set(notebooks_dir.glob("**/*")) == all_expected + + +def test_convert_dir_organize_by_notebook(notebooks_dir): + cmd_list = ["convert", str(notebooks_dir), "-b", "notebook"] + for fmt in EXPECTED_FORMATS: + cmd_list.append("-f") + cmd_list.append(fmt) + result = CliRunner().invoke(app, cmd_list) + assert result.exit_code == 0 + + expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} + expected_export_dirs = {notebooks_dir / nb for nb in EXPECTED_NOTEBOOKS} + expected_export_files = { + notebooks_dir / nb / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" + for nb in EXPECTED_NOTEBOOKS + for fmt in EXPECTED_FORMATS + } + + all_expected = expected_notebooks | expected_export_dirs | expected_export_files + assert set(notebooks_dir.glob("**/*")) == all_expected + + +def test_convert_single_organize_by_extension(notebooks_dir): + nb = EXPECTED_NOTEBOOKS[0] + cmd_list = ["convert", str(notebooks_dir / f"{nb}.ipynb"), "-b", "extension"] + for fmt in EXPECTED_FORMATS: + cmd_list.append("-f") + cmd_list.append(fmt) + result = CliRunner().invoke(app, cmd_list) + assert result.exit_code == 0 + + expected_notebooks = {notebooks_dir / f"{nb_}.ipynb" for nb_ in EXPECTED_NOTEBOOKS} + expected_export_dirs = {notebooks_dir / fmt for fmt in EXPECTED_FORMATS} + expected_export_files = { + notebooks_dir / fmt / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" + for fmt in EXPECTED_FORMATS + } + + all_expected = expected_notebooks | expected_export_dirs | expected_export_files + assert set(notebooks_dir.glob("**/*")) == all_expected + + +def test_convert_single_organize_by_notebook(notebooks_dir): + nb = EXPECTED_NOTEBOOKS[0] + cmd_list = ["convert", str(notebooks_dir / f"{nb}.ipynb"), "-b", "notebook"] + for fmt in EXPECTED_FORMATS: + cmd_list.append("-f") + cmd_list.append(fmt) + result = CliRunner().invoke(app, cmd_list) + assert result.exit_code == 0 + + expected_notebooks = {notebooks_dir / f"{nb_}.ipynb" for nb_ in EXPECTED_NOTEBOOKS} + expected_export_dirs = {notebooks_dir / nb} + expected_export_files = { + notebooks_dir / nb / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" + for fmt in EXPECTED_FORMATS + } + + all_expected = expected_notebooks | expected_export_dirs | expected_export_files + assert set(notebooks_dir.glob("**/*")) == all_expected + + +def test_convert_no_input(): + result = CliRunner().invoke(app, ["convert"]) + + assert result.exit_code == 2 + assert "Error: Missing argument 'INPUT'." in result.stdout diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py new file mode 100644 index 0000000..260ec3d --- /dev/null +++ b/tests/test_cli_export.py @@ -0,0 +1,144 @@ +from pathlib import Path +import shutil + +import pytest +from typer.testing import CliRunner + +from nbautoexport.nbautoexport import app +from nbautoexport.sentinel import ( + ExportFormat, + NbAutoexportConfig, + SAVE_PROGRESS_INDICATOR_FILE, +) + +NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" + +EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] +EXPECTED_FORMATS = ["script", "html"] + + +@pytest.fixture() +def notebooks_dir(tmp_path): + notebooks = EXPECTED_NOTEBOOKS + for nb in notebooks: + shutil.copy(NOTEBOOK_FILE, tmp_path / f"{nb}.ipynb") + return tmp_path + + +def test_export_dir_organize_by_extension(notebooks_dir): + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + with sentinel_path.open("w") as fp: + fp.write( + NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension").json() + ) + + result = CliRunner().invoke(app, ["export", str(notebooks_dir)]) + assert result.exit_code == 0 + + expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} + expected_export_dirs = {notebooks_dir / fmt for fmt in EXPECTED_FORMATS} + expected_export_files = { + notebooks_dir / fmt / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" + for nb in EXPECTED_NOTEBOOKS + for fmt in EXPECTED_FORMATS + } + + all_expected = ( + expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} + ) + assert set(notebooks_dir.glob("**/*")) == all_expected + + +def test_export_dir_organize_by_notebook(notebooks_dir): + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + with sentinel_path.open("w") as fp: + fp.write( + NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook").json() + ) + + result = CliRunner().invoke(app, ["export", str(notebooks_dir)]) + assert result.exit_code == 0 + + expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} + expected_export_dirs = {notebooks_dir / nb for nb in EXPECTED_NOTEBOOKS} + expected_export_files = { + notebooks_dir / nb / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" + for nb in EXPECTED_NOTEBOOKS + for fmt in EXPECTED_FORMATS + } + + all_expected = ( + expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} + ) + assert set(notebooks_dir.glob("**/*")) == all_expected + + +def test_export_single_organize_by_extension(notebooks_dir): + nb = EXPECTED_NOTEBOOKS[0] + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + with sentinel_path.open("w") as fp: + fp.write( + NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension").json() + ) + + result = CliRunner().invoke(app, ["export", str(notebooks_dir / f"{nb}.ipynb")]) + assert result.exit_code == 0 + + expected_notebooks = {notebooks_dir / f"{nb_}.ipynb" for nb_ in EXPECTED_NOTEBOOKS} + expected_export_dirs = {notebooks_dir / fmt for fmt in EXPECTED_FORMATS} + expected_export_files = { + notebooks_dir / fmt / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" + for fmt in EXPECTED_FORMATS + } + + all_expected = ( + expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} + ) + assert set(notebooks_dir.glob("**/*")) == all_expected + + +def test_export_single_organize_by_notebook(notebooks_dir): + nb = EXPECTED_NOTEBOOKS[0] + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + with sentinel_path.open("w") as fp: + fp.write( + NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook").json() + ) + + result = CliRunner().invoke(app, ["export", str(notebooks_dir / f"{nb}.ipynb")]) + assert result.exit_code == 0 + + expected_notebooks = {notebooks_dir / f"{nb_}.ipynb" for nb_ in EXPECTED_NOTEBOOKS} + expected_export_dirs = {notebooks_dir / nb} + expected_export_files = { + notebooks_dir / nb / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" + for fmt in EXPECTED_FORMATS + } + + all_expected = ( + expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} + ) + assert set(notebooks_dir.glob("**/*")) == all_expected + + +def test_export_no_input(): + result = CliRunner().invoke(app, ["export"]) + + assert result.exit_code == 2 + assert "Error: Missing argument 'INPUT'." in result.stdout + + +def test_export_missing_config_error(notebooks_dir): + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + + starting_files = set(notebooks_dir.glob("**/*")) + + result = CliRunner().invoke(app, ["export", str(notebooks_dir)]) + assert result.exit_code == 1 + assert "Error: Missing expected nbautoexport config file" in result.stdout + assert str(sentinel_path.resolve()) in result.stdout + + ending_files = set(notebooks_dir.glob("**/*")) + + # no files deleted + assert starting_files == ending_files diff --git a/tests/test_cli_install.py b/tests/test_cli_install.py index 1dbb824..1b9e082 100644 --- a/tests/test_cli_install.py +++ b/tests/test_cli_install.py @@ -2,11 +2,10 @@ import json -from nbconvert.exporters import get_export_names from typer.testing import CliRunner from nbautoexport.nbautoexport import app -from nbautoexport.sentinel import ExportFormat, install_sentinel, NbAutoexportConfig +from nbautoexport.sentinel import install_sentinel, NbAutoexportConfig def test_invalid_export_format(): @@ -19,14 +18,6 @@ def test_invalid_export_format(): ) -def test_export_format_compatibility(): - """Test that export formats are compatible with Jupyter nbautoconvert. - """ - nbconvert_export_names = get_export_names() - for export_format in ExportFormat: - assert export_format.value in nbconvert_export_names - - def test_invalid_organize_by(): runner = CliRunner() result = runner.invoke(app, ["install", "-b", "invalid-organize-by"]) diff --git a/tests/test_export.py b/tests/test_export.py index 8713242..2d59188 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1,102 +1,144 @@ import json from pathlib import Path import shutil -import sys -from nbautoexport.export import post_save -from nbautoexport.sentinel import SAVE_PROGRESS_INDICATOR_FILE +import pytest + +from nbautoexport.export import export_notebook, post_save +from nbautoexport.sentinel import ExportFormat, NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" -def test_post_save_no_sentinel(tmp_path): - notebook_path = tmp_path / "the_notebook.ipynb" - sentinel_path = tmp_path / SAVE_PROGRESS_INDICATOR_FILE - shutil.copy(NOTEBOOK_FILE, notebook_path) +@pytest.fixture() +def notebooks_dir(tmp_path): + shutil.copy(NOTEBOOK_FILE, tmp_path / "the_notebook.ipynb") + return tmp_path + + +def test_export_notebook_by_extension(notebooks_dir): + notebook_path = notebooks_dir / "the_notebook.ipynb" + config = NbAutoexportConfig( + export_formats=[fmt for fmt in ExportFormat], organize_by="extension" + ) + export_notebook(notebook_path, config) + + expected_export_dirs = {notebooks_dir / fmt.value for fmt in ExportFormat} + expected_export_files = { + notebooks_dir + / fmt.value + / f"{notebook_path.stem}{ExportFormat.get_extension(fmt, language='python')}" + for fmt in ExportFormat + } + all_expected = {notebook_path} | expected_export_dirs | expected_export_files + assert all_expected.issubset(set(notebooks_dir.glob("**/*"))) + + +def test_export_notebook_by_notebook(notebooks_dir): + notebook_path = notebooks_dir / "the_notebook.ipynb" + config = NbAutoexportConfig( + export_formats=[fmt for fmt in ExportFormat], organize_by="notebook" + ) + export_notebook(notebook_path, config) + + expected_export_dirs = {notebooks_dir / notebook_path.stem} + expected_export_files = { + notebooks_dir + / notebook_path.stem + / f"{notebook_path.stem}{ExportFormat.get_extension(fmt, language='python')}" + for fmt in ExportFormat + } + all_expected = {notebook_path} | expected_export_dirs | expected_export_files + assert all_expected.issubset(set(notebooks_dir.glob("**/*"))) + + +def test_post_save_no_sentinel(notebooks_dir): + """Test that post_save does nothing with no sentinel file. + """ + notebook_path = notebooks_dir / "the_notebook.ipynb" + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE assert notebook_path.exists() assert not sentinel_path.exists() post_save(model={"type": "notebook"}, os_path=str(notebook_path), contents_manager=None) - assert set(tmp_path.iterdir()) == {notebook_path} + assert set(notebooks_dir.iterdir()) == {notebook_path} -def test_post_save_organize_by_notebook(tmp_path, monkeypatch): - notebook_path = tmp_path / "the_notebook.ipynb" - sentinel_path = tmp_path / SAVE_PROGRESS_INDICATOR_FILE - shutil.copy(NOTEBOOK_FILE, notebook_path) +def test_post_save_organize_by_notebook(notebooks_dir): + notebook_path = notebooks_dir / "the_notebook.ipynb" + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE with sentinel_path.open("w") as fp: - json.dump({"export_formats": ["script", "html"], "organize_by": "notebook"}, fp) - - monkeypatch.setattr(sys, "argv", [""]) # prevent pytest args from being passed to nbconvert + json.dump( + NbAutoexportConfig(export_formats=["script", "html"], organize_by="notebook").dict(), + fp, + ) assert notebook_path.exists() assert sentinel_path.exists() post_save(model={"type": "notebook"}, os_path=str(notebook_path), contents_manager=None) - assert set(tmp_path.iterdir()) == { + assert set(notebooks_dir.iterdir()) == { sentinel_path, # sentinel file notebook_path, # original ipynb - tmp_path / "the_notebook", # converted notebook directory + notebooks_dir / "the_notebook", # converted notebook directory } - assert (tmp_path / "the_notebook").is_dir() - assert (tmp_path / "the_notebook" / "the_notebook.py").exists() - assert (tmp_path / "the_notebook" / "the_notebook.html").exists() + assert (notebooks_dir / "the_notebook").is_dir() + assert (notebooks_dir / "the_notebook" / "the_notebook.py").exists() + assert (notebooks_dir / "the_notebook" / "the_notebook.html").exists() -def test_post_save_organize_by_extension(tmp_path, monkeypatch): - notebook_path = tmp_path / "the_notebook.ipynb" - sentinel_path = tmp_path / SAVE_PROGRESS_INDICATOR_FILE - shutil.copy(NOTEBOOK_FILE, notebook_path) +def test_post_save_organize_by_extension(notebooks_dir): + notebook_path = notebooks_dir / "the_notebook.ipynb" + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE with sentinel_path.open("w") as fp: - json.dump({"export_formats": ["script", "html"], "organize_by": "extension"}, fp) - - monkeypatch.setattr(sys, "argv", [""]) # prevent pytest args from being passed to nbconvert + json.dump( + NbAutoexportConfig(export_formats=["script", "html"], organize_by="extension").dict(), + fp, + ) assert notebook_path.exists() assert sentinel_path.exists() post_save(model={"type": "notebook"}, os_path=str(notebook_path), contents_manager=None) - assert set(tmp_path.iterdir()) == { + assert set(notebooks_dir.iterdir()) == { sentinel_path, # sentinel file notebook_path, # original ipynb - tmp_path / "script", # converted notebook directory - tmp_path / "html", # converted notebook directory + notebooks_dir / "script", # converted notebook directory + notebooks_dir / "html", # converted notebook directory } - assert (tmp_path / "script").is_dir() - assert (tmp_path / "html").is_dir() - assert (tmp_path / "script" / "the_notebook.py").exists() - assert (tmp_path / "html" / "the_notebook.html").exists() + assert (notebooks_dir / "script").is_dir() + assert (notebooks_dir / "html").is_dir() + assert (notebooks_dir / "script" / "the_notebook.py").exists() + assert (notebooks_dir / "html" / "the_notebook.html").exists() -def test_post_save_type_file(tmp_path, monkeypatch): - notebook_path = tmp_path / "the_notebook.ipynb" - sentinel_path = tmp_path / SAVE_PROGRESS_INDICATOR_FILE - shutil.copy(NOTEBOOK_FILE, notebook_path) +def test_post_save_type_file(notebooks_dir): + """Test that post_save should do nothing if model type is 'file'. + """ + notebook_path = notebooks_dir / "the_notebook.ipynb" + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE with sentinel_path.open("w") as fp: - json.dump({"export_formats": ["script", "html"], "organize_by": "extension"}, fp) - - monkeypatch.setattr(sys, "argv", [""]) # prevent pytest args from being passed to nbconvert + json.dump(NbAutoexportConfig().dict(), fp) assert notebook_path.exists() assert sentinel_path.exists() post_save(model={"type": "file"}, os_path=str(notebook_path), contents_manager=None) - assert set(tmp_path.iterdir()) == { + assert set(notebooks_dir.iterdir()) == { sentinel_path, # sentinel file notebook_path, # original ipynb } -def test_post_save_type_directory(tmp_path, monkeypatch): - notebook_path = tmp_path / "the_notebook.ipynb" - sentinel_path = tmp_path / SAVE_PROGRESS_INDICATOR_FILE - shutil.copy(NOTEBOOK_FILE, notebook_path) +def test_post_save_type_directory(notebooks_dir): + """Test that post_save should do nothing if model type is 'directory'. + """ + notebook_path = notebooks_dir / "the_notebook.ipynb" + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE with sentinel_path.open("w") as fp: - json.dump({"export_formats": ["script", "html"], "organize_by": "extension"}, fp) - - monkeypatch.setattr(sys, "argv", [""]) # prevent pytest args from being passed to nbconvert + json.dump(NbAutoexportConfig().dict(), fp) assert notebook_path.exists() assert sentinel_path.exists() post_save(model={"type": "directory"}, os_path=str(notebook_path), contents_manager=None) - assert set(tmp_path.iterdir()) == { + assert set(notebooks_dir.iterdir()) == { sentinel_path, # sentinel file notebook_path, # original ipynb } diff --git a/tests/test_jupyter_config.py b/tests/test_jupyter_config.py index b90b5a0..e941e4f 100644 --- a/tests/test_jupyter_config.py +++ b/tests/test_jupyter_config.py @@ -261,3 +261,15 @@ def mocked_post_save(model, os_path, contents_manager): os_path_list = [] jupyter_config_obj.FileContentsManager.run_post_save_hook(model=None, os_path=os_path_list) assert os_path_list == ["old_post_save", "nbautoexport"] + + +def test_initialize_post_save_import_error_caught(monkeypatch): + """Test that bound post_save hook with given signature can be successfully run. + """ + + jupyter_config_obj = Config(FileContentsManager=FileContentsManager()) + + monkeypatch.delattr(nbautoexport_root, "post_save") + + # Expect: ImportError: cannot import name 'post_save' from 'nbautoexport' + jupyter_config.initialize_post_save_hook(jupyter_config_obj) diff --git a/tests/test_sentinel.py b/tests/test_sentinel.py new file mode 100644 index 0000000..b65745b --- /dev/null +++ b/tests/test_sentinel.py @@ -0,0 +1,33 @@ +from nbconvert.exporters import get_export_names + +from nbautoexport.sentinel import ExportFormat + + +def test_export_format_compatibility(): + """Test that export formats are compatible with Jupyter nbautoconvert. + """ + nbconvert_export_names = get_export_names() + for export_format in ExportFormat: + assert export_format.value in nbconvert_export_names + + +def test_export_format_extensions(): + for level in ExportFormat: + extension = ExportFormat.get_extension(level) + assert isinstance(extension, str) + assert extension.startswith(".") + assert len(extension) > 1 + + script_extensions = ExportFormat.get_script_extensions() + assert len(script_extensions) > 0 + for extension in script_extensions: + assert isinstance(extension, str) + assert extension.startswith(".") + assert len(extension) > 1 + + +def test_export_format_has_value(): + for level in ExportFormat: + assert ExportFormat.has_value(level.value) + + assert not ExportFormat.has_value("paper") From fcdfdfdbb54d487ff5aa560d74a2f9a58ee2f3c6 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Sat, 27 Jun 2020 00:55:17 -0700 Subject: [PATCH 09/25] Test post_save cleaning --- tests/test_export.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_export.py b/tests/test_export.py index 2d59188..b3d46d2 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -142,3 +142,37 @@ def test_post_save_type_directory(notebooks_dir): sentinel_path, # sentinel file notebook_path, # original ipynb } + + +def test_post_save_clean(notebooks_dir): + notebook_path = notebooks_dir / "the_notebook.ipynb" + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + with sentinel_path.open("w") as fp: + json.dump( + NbAutoexportConfig( + export_formats=["script"], organize_by="extension", clean=True + ).dict(), + fp, + ) + + org_by_notebook_dir = notebooks_dir / "the_notebook" + org_by_notebook_dir.mkdir() + org_by_notebook_script = org_by_notebook_dir / "the_notebook.py" + org_by_notebook_script.touch() + html_dir = notebooks_dir / "html" + html_dir.mkdir() + html_file = html_dir / "the_notebook.html" + html_file.touch() + + for path in [org_by_notebook_dir, org_by_notebook_script, html_dir, html_file]: + assert path.exists() + + post_save(model={"type": "notebook"}, os_path=str(notebook_path), contents_manager=None) + + all_expected = { + notebook_path, + sentinel_path, + notebooks_dir / "script", + notebooks_dir / "script" / "the_notebook.py", + } + assert set(notebooks_dir.glob("**/*")) == all_expected From 1674e44a58c0d9dc8525a76b9390312710600912 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Sat, 27 Jun 2020 00:55:35 -0700 Subject: [PATCH 10/25] cleared_argv tests --- nbautoexport/utils.py | 3 +++ tests/test_utils.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/test_utils.py diff --git a/nbautoexport/utils.py b/nbautoexport/utils.py index 20e0b69..c4cf7f4 100644 --- a/nbautoexport/utils.py +++ b/nbautoexport/utils.py @@ -10,6 +10,9 @@ @contextmanager def cleared_argv(): + """Context manager that temporarily clears sys.argv. Useful for wrapping nbconvert so + unexpected arguments from outer program (e.g., nbautoexport) aren't passed to nbconvert. + """ prev_argv = [arg for arg in sys.argv] sys.argv = [sys.argv[0]] try: diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..c50c65e --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,35 @@ +import sys + +import pytest + +from nbautoexport.utils import cleared_argv + + +def test_cleared_argv(monkeypatch): + """cleared_argv context manager clears sys.argv and restores it on exit + """ + mocked_argv = ["nbautoexport", "convert", "the_notebook.ipynb", "-f", "script"] + monkeypatch.setattr(sys, "argv", mocked_argv) + + assert sys.argv == mocked_argv + + with cleared_argv(): + assert sys.argv == mocked_argv[:1] + + assert sys.argv == mocked_argv + + +def test_cleared_argv_with_error(monkeypatch): + """cleared_argv context manager restores sys.argv even with exception + """ + mocked_argv = ["nbautoexport", "convert", "the_notebook.ipynb", "-f", "script"] + monkeypatch.setattr(sys, "argv", mocked_argv) + + assert sys.argv == mocked_argv + + with pytest.raises(Exception): + with cleared_argv(): + assert sys.argv == mocked_argv[:1] + raise Exception + + assert sys.argv == mocked_argv From b1b05e161cf67cbb58a2d34b577f518d406dcf8a Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Sat, 27 Jun 2020 01:00:24 -0700 Subject: [PATCH 11/25] Add handling of images for asciidoc and rst --- nbautoexport/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbautoexport/export.py b/nbautoexport/export.py index ed63395..caf5ed1 100644 --- a/nbautoexport/export.py +++ b/nbautoexport/export.py @@ -42,8 +42,8 @@ def postprocess(self, input: str): with new_path.open("w") as f: f.write(re.sub(r"\n#\sIn\[(([0-9]+)|(\s))\]:\n{2}", "", text)) - # For markdown files, we also need to move the assets directory, for stuff like images - if self.export_format == ExportFormat.markdown: + # For some formats, we also need to move the assets directory, for stuff like images + if self.export_format in [ExportFormat.asciidoc, ExportFormat.markdown, ExportFormat.rst]: assets_dir = input.parent / f"{input.stem}_files" assets_dir.replace(new_dir / f"{input.stem}_files") From 00193b5002d3182ead18f2940186de44d72291ff Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Sat, 27 Jun 2020 16:20:20 -0700 Subject: [PATCH 12/25] Updated notebook identification and script extension handling --- nbautoexport/clean.py | 82 +++++++++++++++++++++++++++ nbautoexport/export.py | 18 +++--- nbautoexport/nbautoexport.py | 13 +++-- nbautoexport/sentinel.py | 86 ++++------------------------ nbautoexport/utils.py | 49 ++++++++++++++++ tests/conftest.py | 9 +++ tests/test_clean.py | 38 +++++++++++++ tests/test_cli_clean.py | 84 ++++++++++++--------------- tests/test_cli_convert.py | 89 ++++++++++++++++------------- tests/test_cli_export.py | 107 ++++++++++++++--------------------- tests/test_export.py | 63 ++++++++++----------- tests/test_sentinel.py | 7 +-- tests/test_utils.py | 34 ++++++++++- 13 files changed, 402 insertions(+), 277 deletions(-) create mode 100644 nbautoexport/clean.py create mode 100644 tests/conftest.py create mode 100644 tests/test_clean.py diff --git a/nbautoexport/clean.py b/nbautoexport/clean.py new file mode 100644 index 0000000..0d51a43 --- /dev/null +++ b/nbautoexport/clean.py @@ -0,0 +1,82 @@ +from typing import Iterable, List +from pathlib import Path +from nbautoexport.utils import find_notebooks, JupyterNotebook +from nbautoexport.sentinel import ( + ExportFormat, + NbAutoexportConfig, + OrganizeBy, + SAVE_PROGRESS_INDICATOR_FILE, +) + +FORMATS_WITH_IMAGE_DIR = [ + ExportFormat.asciidoc, + ExportFormat.latex, + ExportFormat.markdown, + ExportFormat.rst, +] + + +def notebook_exports_generator( + notebook: JupyterNotebook, export_format: ExportFormat, organize_by: OrganizeBy +) -> Iterable[Path]: + if organize_by == OrganizeBy.notebook: + subfolder = notebook.path.parent / notebook.name + elif organize_by == OrganizeBy.extension: + subfolder = notebook.path.parent / export_format.value + yield subfolder + yield subfolder / f"{notebook.name}{ExportFormat.get_extension(export_format, notebook)}" + if export_format in FORMATS_WITH_IMAGE_DIR: + image_dir = subfolder / f"{notebook.name}_files" + if image_dir.exists(): + yield image_dir + yield from image_dir.iterdir() + + +def get_expected_exports( + notebooks: Iterable[JupyterNotebook], config: NbAutoexportConfig +) -> List[Path]: + """Given an iterable of Jupyter notebooks, return list of paths of files that nbautoexport + would be expected to export to given this configuration. + + Args: + notebooks (Iterable[JupyterNotebooks]): iterable of notebooks + + Returns: + List[Path]: list of expected nbautoexport output files, relative to notebook files + """ + + export_paths = set() + for notebook in notebooks: + for export_format in config.export_formats: + export_paths.update( + notebook_exports_generator(notebook, export_format, config.organize_by) + ) + return sorted(export_paths) + + +def find_files_to_clean(directory: Path, config: NbAutoexportConfig) -> List[Path]: + """Given path to a notebooks directory watched by nbautoexport, find all files that are not + expected exports by current nbautoexport configuration and existing notebooks, or other + expected Jupyter or nbautoexport files. + + Args: + directory (Path): notebooks directory to find files to clean up + + Returns: + List[Path]: list of files to clean up + """ + notebooks = find_notebooks(directory) + expected_exports = [directory / export for export in get_expected_exports(notebooks, config)] + checkpoints = (f for f in directory.glob(".ipynb_checkpoints/*") if f.is_file()) + sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE + + subfiles = (f for f in directory.glob("**/*") if f.is_file()) + + to_clean = ( + set(subfiles) + .difference(nb.path for nb in notebooks) + .difference(expected_exports) + .difference(checkpoints) + .difference([sentinel_path]) + ) + return sorted(to_clean) diff --git a/nbautoexport/export.py b/nbautoexport/export.py index caf5ed1..46ff59f 100644 --- a/nbautoexport/export.py +++ b/nbautoexport/export.py @@ -1,4 +1,3 @@ -import os from pathlib import Path import re @@ -6,6 +5,7 @@ from nbconvert.postprocessors.base import PostProcessorBase from notebook.services.contents.filemanager import FileContentsManager +from nbautoexport.clean import find_files_to_clean, FORMATS_WITH_IMAGE_DIR from nbautoexport.sentinel import ( ExportFormat, NbAutoexportConfig, @@ -43,11 +43,16 @@ def postprocess(self, input: str): f.write(re.sub(r"\n#\sIn\[(([0-9]+)|(\s))\]:\n{2}", "", text)) # For some formats, we also need to move the assets directory, for stuff like images - if self.export_format in [ExportFormat.asciidoc, ExportFormat.markdown, ExportFormat.rst]: + if self.export_format in FORMATS_WITH_IMAGE_DIR: assets_dir = input.parent / f"{input.stem}_files" - assets_dir.replace(new_dir / f"{input.stem}_files") + if assets_dir.exists() and assets_dir.is_dir(): + new_assets_dir = new_dir / f"{input.stem}_files" + new_assets_dir.mkdir(exist_ok=True) + for asset in assets_dir.iterdir(): + asset.rename(new_assets_dir / asset.name) + assets_dir.rmdir() - os.remove(input) + input.unlink() def post_save(model: dict, os_path: str, contents_manager: FileContentsManager): @@ -81,9 +86,9 @@ def post_save(model: dict, os_path: str, contents_manager: FileContentsManager): if config.clean: # Remove files that are not notebooks or expected files - files_to_clean = config.files_to_clean(cwd) + files_to_clean = find_files_to_clean(cwd, config) for path in files_to_clean: - os.remove(path) + path.unlink() # Remove empty subdirectories subfolders = (d for d in cwd.iterdir() if d.is_dir()) @@ -99,7 +104,6 @@ def export_notebook(notebook_path: Path, config: NbAutoexportConfig): for export_format in config.export_formats: if config.organize_by == "notebook": subfolder = notebook_path.stem - elif config.organize_by == "extension": subfolder = export_format.value diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index 533f553..f591869 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -1,12 +1,12 @@ import logging -import os from pathlib import Path from typing import List import typer -from nbautoexport.jupyter_config import install_post_save_hook +from nbautoexport.clean import find_files_to_clean from nbautoexport.export import export_notebook +from nbautoexport.jupyter_config import install_post_save_hook from nbautoexport.sentinel import ( DEFAULT_EXPORT_FORMATS, DEFAULT_ORGANIZE_BY, @@ -70,7 +70,7 @@ def clean( config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json") - files_to_clean = config.files_to_clean(directory) + files_to_clean = find_files_to_clean(directory, config) typer.echo("Identified following files to clean up:") for path in sorted(files_to_clean): @@ -85,12 +85,17 @@ def clean( typer.echo("Removing identified files...") for path in files_to_clean: - os.remove(path) + if path.is_file(): + path.unlink() # Remove empty subdirectories typer.echo("Removing empty subdirectories...") subfolders = (d for d in directory.iterdir() if d.is_dir()) for subfolder in subfolders: + for subsubfolder in subfolder.iterdir(): + if subsubfolder.is_dir() and not any(subsubfolder.iterdir()): + typer.echo(f" {subsubfolder}") + subsubfolder.rmdir() if not any(subfolder.iterdir()): typer.echo(f" {subfolder}") subfolder.rmdir() diff --git a/nbautoexport/sentinel.py b/nbautoexport/sentinel.py index 2c7c2be..cce6bfc 100644 --- a/nbautoexport/sentinel.py +++ b/nbautoexport/sentinel.py @@ -1,11 +1,11 @@ from enum import Enum from pathlib import Path -from typing import Iterable, List, Optional +from typing import List, Optional from pydantic import BaseModel -from nbautoexport.utils import logger -from nbconvert.exporters import get_exporter, PythonExporter +from nbautoexport.utils import JupyterNotebook, logger +from nbconvert.exporters import get_exporter SAVE_PROGRESS_INDICATOR_FILE = ".nbautoexport" @@ -25,18 +25,17 @@ class ExportFormat(str, Enum): slides = "slides" @classmethod - def get_extension(cls, value: str, language: Optional[str] = None) -> str: - if cls(value) == cls.script and language == "python": - return PythonExporter().file_extension + def get_extension(cls, value: str, notebook: Optional[JupyterNotebook] = None) -> str: + # Script format needs notebook to determine appropriate language's extension + if cls(value) == cls.script and notebook is not None: + return notebook.get_script_extension() + exporter = get_exporter(cls(value).value) + if cls(value) == cls.notebook: return f".nbconvert{exporter().file_extension}" return exporter().file_extension - @staticmethod - def get_script_extensions() -> List[str]: - return [exporter().file_extension for exporter in [PythonExporter]] - @classmethod def has_value(cls, value: str) -> bool: return any(level for level in cls if level.value == value) @@ -52,71 +51,8 @@ class NbAutoexportConfig(BaseModel): organize_by: OrganizeBy = OrganizeBy(DEFAULT_ORGANIZE_BY) clean: bool = False - def expected_exports(self, notebook_paths: Iterable[Path]) -> List[Path]: - """Given paths to a set of notebook files, return list of paths of files that nbautoexport - would be expected to export to given this configuration. - - Args: - notebook_paths (Iterable[Path]): iterable of notebook file paths - - Returns: - List[Path]: list of expected nbautoexport output files, relative to notebook files - """ - notebook_names: List[str] = [notebook.stem for notebook in notebook_paths] - if self.organize_by == OrganizeBy.notebook: - export_paths = [ - Path(notebook) / f"{notebook}{ExportFormat.get_extension(export_format)}" - for notebook in notebook_names - for export_format in self.export_formats - ] - # special case for script, since it depends on language - if ExportFormat.script in self.export_formats: - export_paths += [ - Path(notebook) / f"{notebook}{extension}" - for notebook in notebook_names - for extension in ExportFormat.get_script_extensions() - ] - elif self.organize_by == OrganizeBy.extension: - export_paths = [ - Path(export_format.value) - / f"{notebook}{ExportFormat.get_extension(export_format)}" - for notebook in notebook_names - for export_format in self.export_formats - ] - # special case for script, since it depends on language - if ExportFormat.script in self.export_formats: - export_paths += [ - Path(ExportFormat.script.value) / f"{notebook}{extension}" - for notebook in notebook_names - for extension in ExportFormat.get_script_extensions() - ] - return sorted(export_paths) - - def files_to_clean(self, directory: Path) -> List[Path]: - """Given path to a notebooks directory watched by nbautoexport, find all files that are not - expected exports by current nbautoexport configuration and existing notebooks, or other - expected Jupyter or nbautoexport files. - - Args: - directory (Path): notebooks directory to find files to clean up - - Returns: - List[Path]: list of files to clean up - """ - notebook_paths = list(directory.glob("*.ipynb")) - expected_exports = [directory / export for export in self.expected_exports(notebook_paths)] - subfiles = (f for f in directory.glob("**/*") if f.is_file()) - checkpoints = (f for f in directory.glob(".ipynb_checkpoints/*") if f.is_file()) - sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE - - to_clean = ( - set(subfiles) - .difference(notebook_paths) - .difference(expected_exports) - .difference(checkpoints) - .difference([sentinel_path]) - ) - return sorted(to_clean) + class Config: + extra = "forbid" def install_sentinel( diff --git a/nbautoexport/utils.py b/nbautoexport/utils.py index c4cf7f4..d50cd66 100644 --- a/nbautoexport/utils.py +++ b/nbautoexport/utils.py @@ -1,6 +1,13 @@ from contextlib import contextmanager +import json import logging +from pathlib import Path import sys +from warnings import warn + +from pydantic import BaseModel +from nbconvert.exporters import get_export_names, get_exporter +import nbformat from nbautoexport._version import get_versions @@ -8,6 +15,48 @@ __version__ = get_versions()["version"] +class JupyterNotebook(BaseModel): + path: Path + metadata: nbformat.notebooknode.NotebookNode + + def get_script_extension(self): + # Match logic of nbconvert.exporters.script.ScriptExporter + # Order of precedence is: nb_convert_exporter, language, file_extension, .txt + lang_info = self.metadata.get("language_info", {}) + if "nbconvert_exporter" in lang_info: + return get_exporter(lang_info.nbconvert_exporter)().file_extension + if "name" in lang_info and lang_info.name in get_export_names(): + exporter = get_exporter(lang_info.name)().file_extension + return lang_info.get("file_extension", ".txt") + + @property + def name(self): + return self.path.stem + + @classmethod + def from_file(cls, path): + notebook = nbformat.read(path, as_version=nbformat.NO_CONVERT) + nbformat.validate(notebook) + return cls(path=path, metadata=notebook.metadata) + + +def find_notebooks(directory: Path): + notebooks = [] + for subfile in directory.iterdir(): + if subfile.is_file() and subfile.name: + try: + notebook = nbformat.read(subfile, as_version=nbformat.NO_CONVERT) + nbformat.validate(notebook) + notebooks.append(JupyterNotebook(path=subfile, metadata=notebook.metadata)) + except Exception as e: + if subfile.suffix.lower() == ".ipynb": + warn( + f"Error reading {subfile.resolve()} as Jupyter Notebook: " + + f"[{type(e).__name__}] {e}" + ) + return notebooks + + @contextmanager def cleared_argv(): """Context manager that temporarily clears sys.argv. Useful for wrapping nbconvert so diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a00e6ec --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,9 @@ +from pathlib import Path + +import pytest +from nbautoexport.utils import JupyterNotebook + + +@pytest.fixture(scope="session") +def notebook_asset(): + return JupyterNotebook.from_file(Path(__file__).parent / "assets" / "the_notebook.ipynb") diff --git a/tests/test_clean.py b/tests/test_clean.py new file mode 100644 index 0000000..067b0c6 --- /dev/null +++ b/tests/test_clean.py @@ -0,0 +1,38 @@ +import itertools +import shutil +import pytest + +from nbautoexport.clean import notebook_exports_generator +from nbautoexport.export import export_notebook +from nbautoexport.sentinel import ExportFormat, NbAutoexportConfig, OrganizeBy +from nbautoexport.utils import find_notebooks + +EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] +EXPECTED_FORMATS = ["script", "html"] + + +@pytest.fixture() +def notebooks_dir(tmp_path, notebook_asset): + notebooks = EXPECTED_NOTEBOOKS + for nb in notebooks: + shutil.copy(notebook_asset.path, tmp_path / f"{nb}.ipynb") + return tmp_path + + +@pytest.mark.parametrize("export_format, organize_by", itertools.product(ExportFormat, OrganizeBy)) +def test_notebook_exports_generator(notebooks_dir, export_format, organize_by): + """Test that notebook_exports_generator matches what export_notebook produces. + """ + notebook = find_notebooks(notebooks_dir)[0] + notebook_files = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} + + config = NbAutoexportConfig(export_formats=[export_format], organize_by=organize_by) + export_notebook(notebook.path, config) + + predicted_exports = { + notebooks_dir / export + for export in notebook_exports_generator(notebook, export_format, organize_by) + } + + actual_exports = set(notebooks_dir.glob("**/*")).difference(notebook_files) + assert predicted_exports == actual_exports diff --git a/tests/test_cli_clean.py b/tests/test_cli_clean.py index 6447c99..b0a28bf 100644 --- a/tests/test_cli_clean.py +++ b/tests/test_cli_clean.py @@ -1,42 +1,44 @@ -from pathlib import Path import shutil import pytest from typer.testing import CliRunner +from nbautoexport.clean import get_expected_exports from nbautoexport.nbautoexport import app -from nbautoexport.sentinel import ( - ExportFormat, - NbAutoexportConfig, - SAVE_PROGRESS_INDICATOR_FILE, -) - -NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" +from nbautoexport.sentinel import ExportFormat, NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE +from nbautoexport.utils import JupyterNotebook EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] UNEXPECTED_NOTEBOOK = "a_walk_to_remember" -EXPECTED_FORMATS = ["markdown", "html"] +EXPECTED_FORMATS = ["script", "html"] UNEXPECTED_FORMAT = "latex" @pytest.fixture() -def notebooks_dir(tmp_path): +def notebooks_dir(tmp_path, notebook_asset): notebooks = EXPECTED_NOTEBOOKS + [UNEXPECTED_NOTEBOOK] export_formats = [ExportFormat(fmt) for fmt in EXPECTED_FORMATS + [UNEXPECTED_FORMAT]] - for nb in notebooks: - shutil.copy(NOTEBOOK_FILE, tmp_path / f"{nb}.ipynb") + for nb_name in notebooks: + shutil.copy(notebook_asset.path, tmp_path / f"{nb_name}.ipynb") + nb = JupyterNotebook.from_file(tmp_path / f"{nb_name}.ipynb") # organize_by notebook - nb_subfolder = tmp_path / nb + nb_subfolder = tmp_path / nb.name nb_subfolder.mkdir() for fmt in export_formats: - (nb_subfolder / f"{nb}{ExportFormat.get_extension(fmt)}").touch() + (nb_subfolder / f"{nb.name}{ExportFormat.get_extension(fmt, nb)}").touch() # organize_by extension for fmt in export_formats: format_subfolder = tmp_path / fmt.value format_subfolder.mkdir(exist_ok=True) - (format_subfolder / f"{nb}{ExportFormat.get_extension(fmt)}").touch() + (format_subfolder / f"{nb.name}{ExportFormat.get_extension(fmt, nb)}").touch() + + # add latex image dir + (nb_subfolder / f"{nb.name}_files").mkdir() + (nb_subfolder / f"{nb.name}_files" / f"{nb.name}_1_1.png").touch() + (tmp_path / "latex" / f"{nb.name}_files").mkdir() + (tmp_path / "latex" / f"{nb.name}_files" / f"{nb.name}_1_1.png").touch() (tmp_path / f"{UNEXPECTED_NOTEBOOK}.ipynb").unlink() @@ -44,12 +46,11 @@ def notebooks_dir(tmp_path): @pytest.mark.parametrize("need_confirmation", [True, False]) -def test_clean_organize_by_notebook(notebooks_dir, need_confirmation): +def test_clean_organize_by_extension(notebooks_dir, need_confirmation): sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension") with sentinel_path.open("w") as fp: - fp.write( - NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook").json() - ) + fp.write(config.json()) if need_confirmation: result = CliRunner().invoke(app, ["clean", str(notebooks_dir)], input="y") @@ -57,27 +58,21 @@ def test_clean_organize_by_notebook(notebooks_dir, need_confirmation): result = CliRunner().invoke(app, ["clean", str(notebooks_dir), "--yes"]) assert result.exit_code == 0 - expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} - expected_export_dirs = {notebooks_dir / nb for nb in EXPECTED_NOTEBOOKS} - expected_export_files = { - notebooks_dir / nb / f"{nb}{ExportFormat.get_extension(fmt)}" - for nb in EXPECTED_NOTEBOOKS - for fmt in EXPECTED_FORMATS - } - - all_expected = ( - expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} - ) + expected_notebooks = [ + JupyterNotebook.from_file(notebooks_dir / f"{nb}.ipynb") for nb in EXPECTED_NOTEBOOKS + ] + expected_exports = set(get_expected_exports(expected_notebooks, config)) + + all_expected = {nb.path for nb in expected_notebooks} | expected_exports | {sentinel_path} assert set(notebooks_dir.glob("**/*")) == all_expected @pytest.mark.parametrize("need_confirmation", [True, False]) -def test_clean_organize_by_extension(notebooks_dir, need_confirmation): +def test_clean_organize_by_notebook(notebooks_dir, need_confirmation): sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook") with sentinel_path.open("w") as fp: - fp.write( - NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension").json() - ) + fp.write(config.json()) if need_confirmation: result = CliRunner().invoke(app, ["clean", str(notebooks_dir)], input="y") @@ -85,24 +80,19 @@ def test_clean_organize_by_extension(notebooks_dir, need_confirmation): result = CliRunner().invoke(app, ["clean", str(notebooks_dir), "--yes"]) assert result.exit_code == 0 - expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} - expected_export_dirs = {notebooks_dir / fmt for fmt in EXPECTED_FORMATS} - expected_export_files = { - notebooks_dir / fmt / f"{nb}{ExportFormat.get_extension(fmt)}" - for nb in EXPECTED_NOTEBOOKS - for fmt in EXPECTED_FORMATS - } - - all_expected = ( - expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} - ) + expected_notebooks = [ + JupyterNotebook.from_file(notebooks_dir / f"{nb}.ipynb") for nb in EXPECTED_NOTEBOOKS + ] + expected_exports = set(get_expected_exports(expected_notebooks, config)) + + all_expected = {nb.path for nb in expected_notebooks} | expected_exports | {sentinel_path} assert set(notebooks_dir.glob("**/*")) == all_expected def test_clean_abort(notebooks_dir): sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE with sentinel_path.open("w") as fp: - fp.write(NbAutoexportConfig(ExportFormat=EXPECTED_FORMATS).json()) + fp.write(NbAutoexportConfig(export_formats=EXPECTED_FORMATS).json()) starting_files = set(notebooks_dir.glob("**/*")) @@ -119,7 +109,7 @@ def test_clean_abort(notebooks_dir): def test_clean_dry_run(notebooks_dir): sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE with sentinel_path.open("w") as fp: - fp.write(NbAutoexportConfig(ExportFormat=EXPECTED_FORMATS).json()) + fp.write(NbAutoexportConfig(export_formats=EXPECTED_FORMATS).json()) starting_files = set(notebooks_dir.glob("**/*")) diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 1c28c0e..dd74ae5 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -1,23 +1,23 @@ -from pathlib import Path import shutil import pytest from typer.testing import CliRunner +from nbautoexport.clean import get_expected_exports from nbautoexport.nbautoexport import app -from nbautoexport.sentinel import ExportFormat +from nbautoexport.sentinel import NbAutoexportConfig +from nbautoexport.utils import find_notebooks -NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] EXPECTED_FORMATS = ["script", "html"] @pytest.fixture() -def notebooks_dir(tmp_path): +def notebooks_dir(tmp_path, notebook_asset): notebooks = EXPECTED_NOTEBOOKS for nb in notebooks: - shutil.copy(NOTEBOOK_FILE, tmp_path / f"{nb}.ipynb") + shutil.copy(notebook_asset.path, tmp_path / f"{nb}.ipynb") return tmp_path @@ -29,15 +29,19 @@ def test_convert_dir_organize_by_extension(notebooks_dir): result = CliRunner().invoke(app, cmd_list) assert result.exit_code == 0 - expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} - expected_export_dirs = {notebooks_dir / fmt for fmt in EXPECTED_FORMATS} - expected_export_files = { - notebooks_dir / fmt / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" - for nb in EXPECTED_NOTEBOOKS - for fmt in EXPECTED_FORMATS - } + expected_notebooks = find_notebooks(notebooks_dir) + assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + + expected_notebook_files = {nb.path for nb in expected_notebooks} + expected_exports = set( + get_expected_exports( + expected_notebooks, + NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension"), + ) + ) + + all_expected = expected_notebook_files | expected_exports - all_expected = expected_notebooks | expected_export_dirs | expected_export_files assert set(notebooks_dir.glob("**/*")) == all_expected @@ -49,55 +53,62 @@ def test_convert_dir_organize_by_notebook(notebooks_dir): result = CliRunner().invoke(app, cmd_list) assert result.exit_code == 0 - expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} - expected_export_dirs = {notebooks_dir / nb for nb in EXPECTED_NOTEBOOKS} - expected_export_files = { - notebooks_dir / nb / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" - for nb in EXPECTED_NOTEBOOKS - for fmt in EXPECTED_FORMATS - } + expected_notebooks = find_notebooks(notebooks_dir) + assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + + expected_notebook_files = {nb.path for nb in expected_notebooks} + expected_exports = set( + get_expected_exports( + expected_notebooks, + NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook"), + ) + ) - all_expected = expected_notebooks | expected_export_dirs | expected_export_files + all_expected = expected_notebook_files | expected_exports assert set(notebooks_dir.glob("**/*")) == all_expected def test_convert_single_organize_by_extension(notebooks_dir): - nb = EXPECTED_NOTEBOOKS[0] - cmd_list = ["convert", str(notebooks_dir / f"{nb}.ipynb"), "-b", "extension"] + expected_notebooks = find_notebooks(notebooks_dir) + nb = expected_notebooks[0] + + cmd_list = ["convert", str(notebooks_dir / f"{nb.name}.ipynb"), "-b", "extension"] for fmt in EXPECTED_FORMATS: cmd_list.append("-f") cmd_list.append(fmt) result = CliRunner().invoke(app, cmd_list) assert result.exit_code == 0 - expected_notebooks = {notebooks_dir / f"{nb_}.ipynb" for nb_ in EXPECTED_NOTEBOOKS} - expected_export_dirs = {notebooks_dir / fmt for fmt in EXPECTED_FORMATS} - expected_export_files = { - notebooks_dir / fmt / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" - for fmt in EXPECTED_FORMATS - } + expected_notebook_files = {nb_.path for nb_ in expected_notebooks} + expected_exports = set( + get_expected_exports( + [nb], NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension"), + ) + ) - all_expected = expected_notebooks | expected_export_dirs | expected_export_files + all_expected = expected_notebook_files | expected_exports assert set(notebooks_dir.glob("**/*")) == all_expected def test_convert_single_organize_by_notebook(notebooks_dir): - nb = EXPECTED_NOTEBOOKS[0] - cmd_list = ["convert", str(notebooks_dir / f"{nb}.ipynb"), "-b", "notebook"] + expected_notebooks = find_notebooks(notebooks_dir) + nb = expected_notebooks[0] + + cmd_list = ["convert", str(notebooks_dir / f"{nb.name}.ipynb"), "-b", "notebook"] for fmt in EXPECTED_FORMATS: cmd_list.append("-f") cmd_list.append(fmt) result = CliRunner().invoke(app, cmd_list) assert result.exit_code == 0 - expected_notebooks = {notebooks_dir / f"{nb_}.ipynb" for nb_ in EXPECTED_NOTEBOOKS} - expected_export_dirs = {notebooks_dir / nb} - expected_export_files = { - notebooks_dir / nb / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" - for fmt in EXPECTED_FORMATS - } + expected_notebook_files = {nb_.path for nb_ in expected_notebooks} + expected_exports = set( + get_expected_exports( + [nb], NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook"), + ) + ) - all_expected = expected_notebooks | expected_export_dirs | expected_export_files + all_expected = expected_notebook_files | expected_exports assert set(notebooks_dir.glob("**/*")) == all_expected diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index 260ec3d..901980d 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -1,123 +1,98 @@ -from pathlib import Path import shutil import pytest from typer.testing import CliRunner +from nbautoexport.clean import get_expected_exports from nbautoexport.nbautoexport import app -from nbautoexport.sentinel import ( - ExportFormat, - NbAutoexportConfig, - SAVE_PROGRESS_INDICATOR_FILE, -) - -NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" +from nbautoexport.sentinel import NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE +from nbautoexport.utils import find_notebooks EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] EXPECTED_FORMATS = ["script", "html"] @pytest.fixture() -def notebooks_dir(tmp_path): +def notebooks_dir(tmp_path, notebook_asset): notebooks = EXPECTED_NOTEBOOKS for nb in notebooks: - shutil.copy(NOTEBOOK_FILE, tmp_path / f"{nb}.ipynb") + shutil.copy(notebook_asset.path, tmp_path / f"{nb}.ipynb") return tmp_path def test_export_dir_organize_by_extension(notebooks_dir): sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension") with sentinel_path.open("w") as fp: - fp.write( - NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension").json() - ) + fp.write(config.json()) result = CliRunner().invoke(app, ["export", str(notebooks_dir)]) assert result.exit_code == 0 - expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} - expected_export_dirs = {notebooks_dir / fmt for fmt in EXPECTED_FORMATS} - expected_export_files = { - notebooks_dir / fmt / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" - for nb in EXPECTED_NOTEBOOKS - for fmt in EXPECTED_FORMATS - } - - all_expected = ( - expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} - ) + expected_notebooks = find_notebooks(notebooks_dir) + assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + + expected_notebook_files = {nb.path for nb in expected_notebooks} + expected_exports = set(get_expected_exports(expected_notebooks, config)) + + all_expected = expected_notebook_files | expected_exports | {sentinel_path} assert set(notebooks_dir.glob("**/*")) == all_expected def test_export_dir_organize_by_notebook(notebooks_dir): sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook") with sentinel_path.open("w") as fp: - fp.write( - NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook").json() - ) + fp.write(config.json()) result = CliRunner().invoke(app, ["export", str(notebooks_dir)]) assert result.exit_code == 0 - expected_notebooks = {notebooks_dir / f"{nb}.ipynb" for nb in EXPECTED_NOTEBOOKS} - expected_export_dirs = {notebooks_dir / nb for nb in EXPECTED_NOTEBOOKS} - expected_export_files = { - notebooks_dir / nb / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" - for nb in EXPECTED_NOTEBOOKS - for fmt in EXPECTED_FORMATS - } - - all_expected = ( - expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} - ) + expected_notebooks = find_notebooks(notebooks_dir) + assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + + expected_notebook_files = {nb.path for nb in expected_notebooks} + expected_exports = set(get_expected_exports(expected_notebooks, config)) + + all_expected = expected_notebook_files | expected_exports | {sentinel_path} assert set(notebooks_dir.glob("**/*")) == all_expected def test_export_single_organize_by_extension(notebooks_dir): - nb = EXPECTED_NOTEBOOKS[0] + expected_notebooks = find_notebooks(notebooks_dir) + nb = expected_notebooks[0] + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension") with sentinel_path.open("w") as fp: - fp.write( - NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension").json() - ) + fp.write(config.json()) - result = CliRunner().invoke(app, ["export", str(notebooks_dir / f"{nb}.ipynb")]) + result = CliRunner().invoke(app, ["export", str(nb.path)]) assert result.exit_code == 0 - expected_notebooks = {notebooks_dir / f"{nb_}.ipynb" for nb_ in EXPECTED_NOTEBOOKS} - expected_export_dirs = {notebooks_dir / fmt for fmt in EXPECTED_FORMATS} - expected_export_files = { - notebooks_dir / fmt / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" - for fmt in EXPECTED_FORMATS - } + expected_notebook_files = {nb_.path for nb_ in expected_notebooks} + expected_exports = set(get_expected_exports([nb], config)) - all_expected = ( - expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} - ) + all_expected = expected_notebook_files | expected_exports | {sentinel_path} assert set(notebooks_dir.glob("**/*")) == all_expected def test_export_single_organize_by_notebook(notebooks_dir): - nb = EXPECTED_NOTEBOOKS[0] + expected_notebooks = find_notebooks(notebooks_dir) + nb = expected_notebooks[0] + sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook") with sentinel_path.open("w") as fp: - fp.write( - NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook").json() - ) + fp.write(config.json()) - result = CliRunner().invoke(app, ["export", str(notebooks_dir / f"{nb}.ipynb")]) + result = CliRunner().invoke(app, ["export", str(nb.path)]) assert result.exit_code == 0 - expected_notebooks = {notebooks_dir / f"{nb_}.ipynb" for nb_ in EXPECTED_NOTEBOOKS} - expected_export_dirs = {notebooks_dir / nb} - expected_export_files = { - notebooks_dir / nb / f"{nb}{ExportFormat.get_extension(fmt, language='python')}" - for fmt in EXPECTED_FORMATS - } + expected_notebook_files = {nb_.path for nb_ in expected_notebooks} + expected_exports = set(get_expected_exports([nb], config)) - all_expected = ( - expected_notebooks | expected_export_dirs | expected_export_files | {sentinel_path} - ) + all_expected = expected_notebook_files | expected_exports | {sentinel_path} assert set(notebooks_dir.glob("**/*")) == all_expected diff --git a/tests/test_export.py b/tests/test_export.py index b3d46d2..7af7517 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1,54 +1,49 @@ import json -from pathlib import Path import shutil import pytest +from nbautoexport.clean import FORMATS_WITH_IMAGE_DIR from nbautoexport.export import export_notebook, post_save from nbautoexport.sentinel import ExportFormat, NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE - -NOTEBOOK_FILE = Path(__file__).parent / "assets" / "the_notebook.ipynb" +from nbautoexport.utils import JupyterNotebook @pytest.fixture() -def notebooks_dir(tmp_path): - shutil.copy(NOTEBOOK_FILE, tmp_path / "the_notebook.ipynb") +def notebooks_dir(tmp_path, notebook_asset): + shutil.copy(notebook_asset.path, tmp_path / "the_notebook.ipynb") return tmp_path -def test_export_notebook_by_extension(notebooks_dir): - notebook_path = notebooks_dir / "the_notebook.ipynb" +@pytest.mark.parametrize("organize_by", ["extension", "notebook"]) +def test_export_notebook_by_extension(notebooks_dir, organize_by): + """Test that export notebook works. Explicitly write out expected files, because tests for + get_expected_exports will compare against export_notebook. + """ + notebook = JupyterNotebook.from_file(notebooks_dir / "the_notebook.ipynb") config = NbAutoexportConfig( - export_formats=[fmt for fmt in ExportFormat], organize_by="extension" + export_formats=[fmt for fmt in ExportFormat], organize_by=organize_by ) - export_notebook(notebook_path, config) - - expected_export_dirs = {notebooks_dir / fmt.value for fmt in ExportFormat} - expected_export_files = { - notebooks_dir - / fmt.value - / f"{notebook_path.stem}{ExportFormat.get_extension(fmt, language='python')}" - for fmt in ExportFormat - } - all_expected = {notebook_path} | expected_export_dirs | expected_export_files - assert all_expected.issubset(set(notebooks_dir.glob("**/*"))) + export_notebook(notebook.path, config) + expected_exports = set() + for fmt in ExportFormat: + if organize_by == "extension": + subfolder = notebooks_dir / fmt.value + elif organize_by == "notebook": + subfolder = notebooks_dir / notebook.name + extension = ExportFormat.get_extension(fmt, notebook) -def test_export_notebook_by_notebook(notebooks_dir): - notebook_path = notebooks_dir / "the_notebook.ipynb" - config = NbAutoexportConfig( - export_formats=[fmt for fmt in ExportFormat], organize_by="notebook" - ) - export_notebook(notebook_path, config) - - expected_export_dirs = {notebooks_dir / notebook_path.stem} - expected_export_files = { - notebooks_dir - / notebook_path.stem - / f"{notebook_path.stem}{ExportFormat.get_extension(fmt, language='python')}" - for fmt in ExportFormat - } - all_expected = {notebook_path} | expected_export_dirs | expected_export_files + expected_exports.add(subfolder) # subfolder + expected_exports.add(subfolder / f"{notebook.name}{extension}") # export file + + if fmt in FORMATS_WITH_IMAGE_DIR: + image_subfolder = subfolder / f"{notebook.name}_files" + + expected_exports.add(image_subfolder) # image subdir + expected_exports.add(image_subfolder / f"{notebook.name}_1_1.png") # image file + + all_expected = {notebook.path} | expected_exports assert all_expected.issubset(set(notebooks_dir.glob("**/*"))) diff --git a/tests/test_sentinel.py b/tests/test_sentinel.py index b65745b..5e7d30b 100644 --- a/tests/test_sentinel.py +++ b/tests/test_sentinel.py @@ -11,16 +11,15 @@ def test_export_format_compatibility(): assert export_format.value in nbconvert_export_names -def test_export_format_extensions(): +def test_export_format_extensions(notebook_asset): for level in ExportFormat: extension = ExportFormat.get_extension(level) assert isinstance(extension, str) assert extension.startswith(".") assert len(extension) > 1 - script_extensions = ExportFormat.get_script_extensions() - assert len(script_extensions) > 0 - for extension in script_extensions: + for level in ExportFormat: + extension = ExportFormat.get_extension(level, notebook_asset) assert isinstance(extension, str) assert extension.startswith(".") assert len(extension) > 1 diff --git a/tests/test_utils.py b/tests/test_utils.py index c50c65e..dbfd868 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,7 +2,39 @@ import pytest -from nbautoexport.utils import cleared_argv +from nbautoexport.utils import cleared_argv, find_notebooks + + +def test_get_script_extensions(notebook_asset, monkeypatch): + # get extension from metadata.language_info.nbconvert_exporter + assert notebook_asset.get_script_extension() == ".py" + + # fall back to metadata.language_info.name + monkeypatch.delitem(notebook_asset.metadata.language_info, "nbconvert_exporter") + assert notebook_asset.get_script_extension() == ".py" + + # metadata.language_info.name isn't known, fall back to metadata.language_info.file_extension + monkeypatch.setitem(notebook_asset.metadata.language_info, "name", "mongoose") + assert notebook_asset.get_script_extension() == ".py" + + # metadata.language_info.name is missing, fall back to metadata.language_info.file_extension + monkeypatch.delitem(notebook_asset.metadata.language_info, "name") + assert notebook_asset.get_script_extension() == ".py" + + # metadata.language_info.file_extension is missing, fall back to ".txt" + monkeypatch.delitem(notebook_asset.metadata.language_info, "file_extension") + assert notebook_asset.get_script_extension() == ".txt" + + # metadata.language_info is missing, fall back to ".txt" + monkeypatch.delitem(notebook_asset.metadata, "language_info") + assert notebook_asset.get_script_extension() == ".txt" + + +def test_find_notebooks_warning(tmp_path): + bad_notebook_path = tmp_path / "the_journal.ipynb" + bad_notebook_path.touch() + with pytest.warns(Warning, match="Error reading"): + find_notebooks(tmp_path) def test_cleared_argv(monkeypatch): From a1c1eb48acbe0153b028f998397f169c30741799 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Sat, 27 Jun 2020 16:43:53 -0700 Subject: [PATCH 13/25] Change get_extension to have consistent signature --- nbautoexport/clean.py | 26 +++++++++++++++++++++++++- nbautoexport/sentinel.py | 17 ++--------------- nbautoexport/utils.py | 3 +-- tests/test_cli_clean.py | 6 +++--- tests/test_export.py | 4 ++-- tests/test_sentinel.py | 9 ++------- 6 files changed, 35 insertions(+), 30 deletions(-) diff --git a/nbautoexport/clean.py b/nbautoexport/clean.py index 0d51a43..01fad10 100644 --- a/nbautoexport/clean.py +++ b/nbautoexport/clean.py @@ -1,5 +1,8 @@ from typing import Iterable, List from pathlib import Path + +from nbconvert.exporters import get_exporter + from nbautoexport.utils import find_notebooks, JupyterNotebook from nbautoexport.sentinel import ( ExportFormat, @@ -16,6 +19,27 @@ ] +def get_extension(notebook: JupyterNotebook, export_format: str) -> str: + """Given a notebook and export format, return expected export file extension. + + Args: + notebook (JupyterNotebook): notebook to determine extension for + export_format (str): export format name + + Returns: + str: file extension, e.g., '.py' + """ + # Script format needs notebook to determine appropriate language's extension + if ExportFormat(export_format) == ExportFormat.script: + return notebook.get_script_extension() + + exporter = get_exporter(ExportFormat(export_format).value) + + if ExportFormat(export_format) == ExportFormat.notebook: + return f".nbconvert{exporter().file_extension}" + return exporter().file_extension + + def notebook_exports_generator( notebook: JupyterNotebook, export_format: ExportFormat, organize_by: OrganizeBy ) -> Iterable[Path]: @@ -24,7 +48,7 @@ def notebook_exports_generator( elif organize_by == OrganizeBy.extension: subfolder = notebook.path.parent / export_format.value yield subfolder - yield subfolder / f"{notebook.name}{ExportFormat.get_extension(export_format, notebook)}" + yield subfolder / f"{notebook.name}{get_extension(notebook,export_format)}" if export_format in FORMATS_WITH_IMAGE_DIR: image_dir = subfolder / f"{notebook.name}_files" if image_dir.exists(): diff --git a/nbautoexport/sentinel.py b/nbautoexport/sentinel.py index cce6bfc..eff5000 100644 --- a/nbautoexport/sentinel.py +++ b/nbautoexport/sentinel.py @@ -1,11 +1,10 @@ from enum import Enum from pathlib import Path -from typing import List, Optional +from typing import List from pydantic import BaseModel -from nbautoexport.utils import JupyterNotebook, logger -from nbconvert.exporters import get_exporter +from nbautoexport.utils import logger SAVE_PROGRESS_INDICATOR_FILE = ".nbautoexport" @@ -24,18 +23,6 @@ class ExportFormat(str, Enum): script = "script" slides = "slides" - @classmethod - def get_extension(cls, value: str, notebook: Optional[JupyterNotebook] = None) -> str: - # Script format needs notebook to determine appropriate language's extension - if cls(value) == cls.script and notebook is not None: - return notebook.get_script_extension() - - exporter = get_exporter(cls(value).value) - - if cls(value) == cls.notebook: - return f".nbconvert{exporter().file_extension}" - return exporter().file_extension - @classmethod def has_value(cls, value: str) -> bool: return any(level for level in cls if level.value == value) diff --git a/nbautoexport/utils.py b/nbautoexport/utils.py index d50cd66..65846b5 100644 --- a/nbautoexport/utils.py +++ b/nbautoexport/utils.py @@ -1,5 +1,4 @@ from contextlib import contextmanager -import json import logging from pathlib import Path import sys @@ -26,7 +25,7 @@ def get_script_extension(self): if "nbconvert_exporter" in lang_info: return get_exporter(lang_info.nbconvert_exporter)().file_extension if "name" in lang_info and lang_info.name in get_export_names(): - exporter = get_exporter(lang_info.name)().file_extension + return get_exporter(lang_info.name)().file_extension return lang_info.get("file_extension", ".txt") @property diff --git a/tests/test_cli_clean.py b/tests/test_cli_clean.py index b0a28bf..ac1fcb2 100644 --- a/tests/test_cli_clean.py +++ b/tests/test_cli_clean.py @@ -3,7 +3,7 @@ import pytest from typer.testing import CliRunner -from nbautoexport.clean import get_expected_exports +from nbautoexport.clean import get_expected_exports, get_extension from nbautoexport.nbautoexport import app from nbautoexport.sentinel import ExportFormat, NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE from nbautoexport.utils import JupyterNotebook @@ -26,13 +26,13 @@ def notebooks_dir(tmp_path, notebook_asset): nb_subfolder = tmp_path / nb.name nb_subfolder.mkdir() for fmt in export_formats: - (nb_subfolder / f"{nb.name}{ExportFormat.get_extension(fmt, nb)}").touch() + (nb_subfolder / f"{nb.name}{get_extension(nb, fmt)}").touch() # organize_by extension for fmt in export_formats: format_subfolder = tmp_path / fmt.value format_subfolder.mkdir(exist_ok=True) - (format_subfolder / f"{nb.name}{ExportFormat.get_extension(fmt, nb)}").touch() + (format_subfolder / f"{nb.name}{get_extension(nb, fmt)}").touch() # add latex image dir (nb_subfolder / f"{nb.name}_files").mkdir() diff --git a/tests/test_export.py b/tests/test_export.py index 7af7517..ca32546 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -3,7 +3,7 @@ import pytest -from nbautoexport.clean import FORMATS_WITH_IMAGE_DIR +from nbautoexport.clean import FORMATS_WITH_IMAGE_DIR, get_extension from nbautoexport.export import export_notebook, post_save from nbautoexport.sentinel import ExportFormat, NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE from nbautoexport.utils import JupyterNotebook @@ -32,7 +32,7 @@ def test_export_notebook_by_extension(notebooks_dir, organize_by): subfolder = notebooks_dir / fmt.value elif organize_by == "notebook": subfolder = notebooks_dir / notebook.name - extension = ExportFormat.get_extension(fmt, notebook) + extension = get_extension(notebook, fmt) expected_exports.add(subfolder) # subfolder expected_exports.add(subfolder / f"{notebook.name}{extension}") # export file diff --git a/tests/test_sentinel.py b/tests/test_sentinel.py index 5e7d30b..372d7ab 100644 --- a/tests/test_sentinel.py +++ b/tests/test_sentinel.py @@ -1,5 +1,6 @@ from nbconvert.exporters import get_export_names +from nbautoexport.clean import get_extension from nbautoexport.sentinel import ExportFormat @@ -13,13 +14,7 @@ def test_export_format_compatibility(): def test_export_format_extensions(notebook_asset): for level in ExportFormat: - extension = ExportFormat.get_extension(level) - assert isinstance(extension, str) - assert extension.startswith(".") - assert len(extension) > 1 - - for level in ExportFormat: - extension = ExportFormat.get_extension(level, notebook_asset) + extension = get_extension(notebook_asset, level) assert isinstance(extension, str) assert extension.startswith(".") assert len(extension) > 1 From 6ef14ca445eb51b472e4f0b7928e2a63c20d2ba8 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Sat, 27 Jun 2020 17:20:57 -0700 Subject: [PATCH 14/25] Documentation --- nbautoexport/clean.py | 14 ++++++++-- nbautoexport/export.py | 6 +++++ nbautoexport/nbautoexport.py | 50 ++++++++++++++++++++++++------------ nbautoexport/sentinel.py | 3 ++- nbautoexport/utils.py | 11 +++++++- 5 files changed, 63 insertions(+), 21 deletions(-) diff --git a/nbautoexport/clean.py b/nbautoexport/clean.py index 01fad10..62652e3 100644 --- a/nbautoexport/clean.py +++ b/nbautoexport/clean.py @@ -19,7 +19,7 @@ ] -def get_extension(notebook: JupyterNotebook, export_format: str) -> str: +def get_extension(notebook: JupyterNotebook, export_format: ExportFormat) -> str: """Given a notebook and export format, return expected export file extension. Args: @@ -43,12 +43,22 @@ def get_extension(notebook: JupyterNotebook, export_format: str) -> str: def notebook_exports_generator( notebook: JupyterNotebook, export_format: ExportFormat, organize_by: OrganizeBy ) -> Iterable[Path]: + """[summary] + + Args: + notebook (JupyterNotebook): notebook to get export paths for + export_format (ExportFormat): export format + organize_by (OrganizeBy): type of subfolder approach + + Returns: + Iterable[Path]: expected export paths given notebook and configuration options + """ if organize_by == OrganizeBy.notebook: subfolder = notebook.path.parent / notebook.name elif organize_by == OrganizeBy.extension: subfolder = notebook.path.parent / export_format.value yield subfolder - yield subfolder / f"{notebook.name}{get_extension(notebook,export_format)}" + yield subfolder / f"{notebook.name}{get_extension(notebook, export_format)}" if export_format in FORMATS_WITH_IMAGE_DIR: image_dir = subfolder / f"{notebook.name}_files" if image_dir.exists(): diff --git a/nbautoexport/export.py b/nbautoexport/export.py index 46ff59f..19a585a 100644 --- a/nbautoexport/export.py +++ b/nbautoexport/export.py @@ -98,6 +98,12 @@ def post_save(model: dict, os_path: str, contents_manager: FileContentsManager): def export_notebook(notebook_path: Path, config: NbAutoexportConfig): + """Export a given notebook file given configuration. + + Args: + notebook_path (Path): path to notebook to export with nbconvert + config (NbAutoexportConfig): configuration + """ with cleared_argv(): converter = NbConvertApp() diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index f591869..7d20a0d 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -8,6 +8,7 @@ from nbautoexport.export import export_notebook from nbautoexport.jupyter_config import install_post_save_hook from nbautoexport.sentinel import ( + DEFAULT_CLEAN, DEFAULT_EXPORT_FORMATS, DEFAULT_ORGANIZE_BY, ExportFormat, @@ -54,7 +55,7 @@ def main( @app.command() def clean( directory: Path = typer.Argument( - ..., exists=True, file_okay=False, dir_okay=True, writable=True + ..., exists=True, file_okay=False, dir_okay=True, writable=True, help="Directory to clean." ), yes: bool = typer.Option( False, "--yes", "-y", help="Assume 'yes' answer to confirmation prompt to delete files." @@ -105,7 +106,14 @@ def clean( @app.command() def convert( - input: Path = typer.Argument(..., exists=True, file_okay=True, dir_okay=True, writable=True), + input: Path = typer.Argument( + ..., + exists=True, + file_okay=True, + dir_okay=True, + writable=True, + help="Path to notebook file or directory of notebook files to convert.", + ), export_formats: List[ExportFormat] = typer.Option( DEFAULT_EXPORT_FORMATS, "--export-format", @@ -129,9 +137,6 @@ def convert( ), ): """Convert notebook(s) using specified configuration options. - - INPUT is the path to a notebook to be converted, or a directory containing notebooks to be - converted. """ config = NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by) if input.is_dir(): @@ -143,13 +148,17 @@ def convert( @app.command() def export( - input: Path = typer.Argument(..., exists=True, file_okay=True, dir_okay=True, writable=True) + input: Path = typer.Argument( + ..., + exists=True, + file_okay=True, + dir_okay=True, + writable=True, + help="Path to notebook file or directory of notebook files to export.", + ) ): """Convert notebook(s) using existing configuration file. - INPUT is the path to a notebook to be converted, or a directory containing notebooks to be - converted. - A .nbautoconvert configuration file is required to be in the same directory as the notebook(s). """ if input.is_dir(): @@ -168,7 +177,12 @@ def export( @app.command() def install( directory: Path = typer.Argument( - "extension", exists=True, file_okay=False, dir_okay=True, writable=True + "notebooks", + exists=True, + file_okay=False, + dir_okay=True, + writable=True, + help="Path to directory of notebook files to watch with nbautoexport.", ), export_formats: List[ExportFormat] = typer.Option( DEFAULT_EXPORT_FORMATS, @@ -176,9 +190,9 @@ def install( "-f", show_default=True, help=( - """File format(s) to save for each notebook. Options are 'script', 'html', 'markdown', """ - """and 'rst'. Multiple formats should be provided using multiple flags, e.g., '-f """ - """script-f html -f markdown'.""" + "File format(s) to save for each notebook. Options are 'script', 'html', 'markdown', " + + "and 'rst'. Multiple formats should be provided using multiple flags, e.g., " + + "'-f script -f html -f markdown'." ), ), organize_by: OrganizeBy = typer.Option( @@ -187,14 +201,16 @@ def install( "-b", show_default=True, help=( - """Whether to save exported file(s) in a folder per notebook or a folder per extension. """ - """Options are 'notebook' or 'extension'.""" + "Whether to save exported file(s) in a folder per notebook or a folder per extension. " + + "Options are 'notebook' or 'extension'." + "" ), ), clean: bool = typer.Option( - False, + DEFAULT_CLEAN, show_default=True, - help="Whether to automatically delete files in subfolders that don't match configuration.", + help="Whether to automatically delete files that don't match expected exports given " + + "notebooks and configuration.", ), overwrite: bool = typer.Option( False, diff --git a/nbautoexport/sentinel.py b/nbautoexport/sentinel.py index eff5000..1952999 100644 --- a/nbautoexport/sentinel.py +++ b/nbautoexport/sentinel.py @@ -10,6 +10,7 @@ SAVE_PROGRESS_INDICATOR_FILE = ".nbautoexport" DEFAULT_EXPORT_FORMATS = ["script"] DEFAULT_ORGANIZE_BY = "extension" +DEFAULT_CLEAN = False class ExportFormat(str, Enum): @@ -36,7 +37,7 @@ class OrganizeBy(str, Enum): class NbAutoexportConfig(BaseModel): export_formats: List[ExportFormat] = [ExportFormat(fmt) for fmt in DEFAULT_EXPORT_FORMATS] organize_by: OrganizeBy = OrganizeBy(DEFAULT_ORGANIZE_BY) - clean: bool = False + clean: bool = DEFAULT_CLEAN class Config: extra = "forbid" diff --git a/nbautoexport/utils.py b/nbautoexport/utils.py index 65846b5..95654c7 100644 --- a/nbautoexport/utils.py +++ b/nbautoexport/utils.py @@ -2,6 +2,7 @@ import logging from pathlib import Path import sys +from typing import List from warnings import warn from pydantic import BaseModel @@ -39,7 +40,15 @@ def from_file(cls, path): return cls(path=path, metadata=notebook.metadata) -def find_notebooks(directory: Path): +def find_notebooks(directory: Path) -> List[JupyterNotebook]: + """Finds Jupyter notebooks in a directory. Not recursive. + + Args: + directory (Path): directory to search for notebook files + + Returns: + List[JupyterNotebook]: notebooks found + """ notebooks = [] for subfile in directory.iterdir(): if subfile.is_file() and subfile.name: From 43ecd34dc6a3c74b80d3830c218cc6b41160015b Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Sat, 27 Jun 2020 17:22:07 -0700 Subject: [PATCH 15/25] Change default organize_by back to notebook. Will change separately --- nbautoexport/sentinel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbautoexport/sentinel.py b/nbautoexport/sentinel.py index 1952999..4012634 100644 --- a/nbautoexport/sentinel.py +++ b/nbautoexport/sentinel.py @@ -9,7 +9,7 @@ SAVE_PROGRESS_INDICATOR_FILE = ".nbautoexport" DEFAULT_EXPORT_FORMATS = ["script"] -DEFAULT_ORGANIZE_BY = "extension" +DEFAULT_ORGANIZE_BY = "notebook" DEFAULT_CLEAN = False From a7c79cf1994194ecef07cd64e510c96eabf09cee Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Wed, 1 Jul 2020 15:38:45 -0700 Subject: [PATCH 16/25] Fix relative working directory bug --- nbautoexport/clean.py | 4 +-- nbautoexport/utils.py | 12 +++++++ tests/test_clean.py | 5 +-- tests/test_cli_clean.py | 80 ++++++++++++++++++++++++++++------------- 4 files changed, 71 insertions(+), 30 deletions(-) diff --git a/nbautoexport/clean.py b/nbautoexport/clean.py index 62652e3..0a8e602 100644 --- a/nbautoexport/clean.py +++ b/nbautoexport/clean.py @@ -99,8 +99,8 @@ def find_files_to_clean(directory: Path, config: NbAutoexportConfig) -> List[Pat Returns: List[Path]: list of files to clean up """ - notebooks = find_notebooks(directory) - expected_exports = [directory / export for export in get_expected_exports(notebooks, config)] + notebooks: List[JupyterNotebook] = find_notebooks(directory) + expected_exports: List[Path] = get_expected_exports(notebooks, config) checkpoints = (f for f in directory.glob(".ipynb_checkpoints/*") if f.is_file()) sentinel_path = directory / SAVE_PROGRESS_INDICATOR_FILE diff --git a/nbautoexport/utils.py b/nbautoexport/utils.py index 95654c7..e50b254 100644 --- a/nbautoexport/utils.py +++ b/nbautoexport/utils.py @@ -1,5 +1,6 @@ from contextlib import contextmanager import logging +import os from pathlib import Path import sys from typing import List @@ -76,3 +77,14 @@ def cleared_argv(): yield finally: sys.argv = prev_argv + + +@contextmanager +def working_directory(directory: Path): + """Changes working directory and returns to previous on exit.""" + prev_cwd = Path.cwd() + os.chdir(directory) + try: + yield + finally: + os.chdir(prev_cwd) diff --git a/tests/test_clean.py b/tests/test_clean.py index 067b0c6..8236900 100644 --- a/tests/test_clean.py +++ b/tests/test_clean.py @@ -29,10 +29,7 @@ def test_notebook_exports_generator(notebooks_dir, export_format, organize_by): config = NbAutoexportConfig(export_formats=[export_format], organize_by=organize_by) export_notebook(notebook.path, config) - predicted_exports = { - notebooks_dir / export - for export in notebook_exports_generator(notebook, export_format, organize_by) - } + predicted_exports = set(notebook_exports_generator(notebook, export_format, organize_by)) actual_exports = set(notebooks_dir.glob("**/*")).difference(notebook_files) assert predicted_exports == actual_exports diff --git a/tests/test_cli_clean.py b/tests/test_cli_clean.py index ac1fcb2..35283a1 100644 --- a/tests/test_cli_clean.py +++ b/tests/test_cli_clean.py @@ -1,3 +1,5 @@ +from itertools import product +from pathlib import Path import shutil import pytest @@ -6,7 +8,7 @@ from nbautoexport.clean import get_expected_exports, get_extension from nbautoexport.nbautoexport import app from nbautoexport.sentinel import ExportFormat, NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE -from nbautoexport.utils import JupyterNotebook +from nbautoexport.utils import JupyterNotebook, working_directory EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] UNEXPECTED_NOTEBOOK = "a_walk_to_remember" @@ -45,10 +47,12 @@ def notebooks_dir(tmp_path, notebook_asset): return tmp_path -@pytest.mark.parametrize("need_confirmation", [True, False]) -def test_clean_organize_by_extension(notebooks_dir, need_confirmation): +@pytest.mark.parametrize( + "need_confirmation, organize_by", product([True, False], ["extension", "notebook"]) +) +def test_clean(notebooks_dir, need_confirmation, organize_by): sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE - config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension") + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by) with sentinel_path.open("w") as fp: fp.write(config.json()) @@ -67,26 +71,54 @@ def test_clean_organize_by_extension(notebooks_dir, need_confirmation): assert set(notebooks_dir.glob("**/*")) == all_expected -@pytest.mark.parametrize("need_confirmation", [True, False]) -def test_clean_organize_by_notebook(notebooks_dir, need_confirmation): - sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE - config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook") - with sentinel_path.open("w") as fp: - fp.write(config.json()) - - if need_confirmation: - result = CliRunner().invoke(app, ["clean", str(notebooks_dir)], input="y") - else: - result = CliRunner().invoke(app, ["clean", str(notebooks_dir), "--yes"]) - assert result.exit_code == 0 - - expected_notebooks = [ - JupyterNotebook.from_file(notebooks_dir / f"{nb}.ipynb") for nb in EXPECTED_NOTEBOOKS - ] - expected_exports = set(get_expected_exports(expected_notebooks, config)) - - all_expected = {nb.path for nb in expected_notebooks} | expected_exports | {sentinel_path} - assert set(notebooks_dir.glob("**/*")) == all_expected +@pytest.mark.parametrize("organize_by", ["extension", "notebook"]) +def test_clean_relative(notebooks_dir, organize_by): + """ Test that cleaning works relative to current working directory. + """ + with working_directory(notebooks_dir): + sentinel_path = Path(SAVE_PROGRESS_INDICATOR_FILE) + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by) + with sentinel_path.open("w") as fp: + fp.write(config.json()) + + result = CliRunner().invoke(app, ["clean", "."], input="y") + assert result.exit_code == 0 + + expected_notebooks = [ + JupyterNotebook.from_file(f"{nb}.ipynb") for nb in EXPECTED_NOTEBOOKS + ] + expected_exports = set(get_expected_exports(expected_notebooks, config)) + + all_expected = {nb.path for nb in expected_notebooks} | expected_exports | {sentinel_path} + assert set(Path().glob("**/*")) == all_expected + + +@pytest.mark.parametrize("organize_by", ["extension", "notebook"]) +def test_clean_relative_subdirectory(notebooks_dir, organize_by): + """ Test that cleaning works for subdirectory relative to current working directory. + """ + with working_directory(notebooks_dir): + # Set up subdirectory + subdir = Path("subdir") + subdir.mkdir() + for subfile in Path().iterdir(): + shutil.move(str(subfile), str(subdir)) + + sentinel_path = subdir / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by) + with sentinel_path.open("w") as fp: + fp.write(config.json()) + + result = CliRunner().invoke(app, ["clean", "subdir"], input="y") + assert result.exit_code == 0 + + expected_notebooks = [ + JupyterNotebook.from_file(subdir / f"{nb}.ipynb") for nb in EXPECTED_NOTEBOOKS + ] + expected_exports = set(get_expected_exports(expected_notebooks, config)) + + all_expected = {nb.path for nb in expected_notebooks} | expected_exports | {sentinel_path} + assert set(subdir.glob("**/*")) == all_expected def test_clean_abort(notebooks_dir): From 019902fc78acd50d28d15d17daf6a1be187a780e Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Wed, 1 Jul 2020 15:46:11 -0700 Subject: [PATCH 17/25] Test for working_directory context manager --- tests/test_utils.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index dbfd868..74f3ff0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,8 +1,9 @@ +from pathlib import Path import sys import pytest -from nbautoexport.utils import cleared_argv, find_notebooks +from nbautoexport.utils import cleared_argv, find_notebooks, working_directory def test_get_script_extensions(notebook_asset, monkeypatch): @@ -65,3 +66,11 @@ def test_cleared_argv_with_error(monkeypatch): raise Exception assert sys.argv == mocked_argv + + +def test_working_directory(tmp_path): + cwd = Path.cwd() + assert cwd != tmp_path + with working_directory(tmp_path): + assert Path.cwd() == tmp_path + assert Path.cwd() == cwd From 78bd46b9ec5430d48ee1348ccb41c3f159cbe8bc Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Wed, 1 Jul 2020 18:19:27 -0700 Subject: [PATCH 18/25] Merge convert and export commands --- nbautoexport/nbautoexport.py | 92 +++++++-------- nbautoexport/utils.py | 3 + tests/test_cli_convert.py | 119 ------------------- tests/test_cli_export.py | 213 ++++++++++++++++++++++++++--------- tests/test_utils.py | 31 ++++- 5 files changed, 241 insertions(+), 217 deletions(-) delete mode 100644 tests/test_cli_convert.py diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index 7d20a0d..55274f3 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import List +from typing import List, Optional import typer @@ -17,7 +17,7 @@ OrganizeBy, SAVE_PROGRESS_INDICATOR_FILE, ) -from nbautoexport.utils import __version__ +from nbautoexport.utils import __version__, find_notebooks app = typer.Typer() @@ -105,72 +105,79 @@ def clean( @app.command() -def convert( +def export( input: Path = typer.Argument( ..., exists=True, file_okay=True, dir_okay=True, writable=True, - help="Path to notebook file or directory of notebook files to convert.", + help="Path to notebook file or directory of notebook files to export.", ), - export_formats: List[ExportFormat] = typer.Option( - DEFAULT_EXPORT_FORMATS, + export_formats: Optional[List[ExportFormat]] = typer.Option( + None, "--export-format", "-f", show_default=True, help=( - """File format(s) to save for each notebook. Options are 'script', 'html', 'markdown', """ - """and 'rst'. Multiple formats should be provided using multiple flags, e.g., '-f """ - """script-f html -f markdown'.""" + "File format(s) to save for each notebook. Multiple formats should be provided using " + "multiple flags, e.g., '-f script -f html -f markdown'. Provided values will override " + "existing .nbautoexport config files. If neither provided, defaults to " + f"{DEFAULT_EXPORT_FORMATS}." ), ), - organize_by: OrganizeBy = typer.Option( - DEFAULT_ORGANIZE_BY, + organize_by: Optional[OrganizeBy] = typer.Option( + None, "--organize-by", "-b", show_default=True, help=( - """Whether to save exported file(s) in a folder per notebook or a folder per extension. """ - """Options are 'notebook' or 'extension'.""" + "Whether to save exported file(s) in a subfolder per notebook or per export format. " + "Provided values will override existing .nbautoexport config files. If neither " + f"provided, defaults to '{DEFAULT_ORGANIZE_BY}'." ), ), ): - """Convert notebook(s) using specified configuration options. - """ - config = NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by) - if input.is_dir(): - for notebook_path in input.glob("*.ipynb"): - export_notebook(notebook_path, config=config) - else: - export_notebook(input, config=config) + """Manually export notebook or directory of notebooks. + An .nbautoexport configuration file in same directory as notebook(s) will be used if it + exists. Configuration options specified by command-line options will override configuration + file. If no existing configuration option exists and no values are provided, default values + will be used. -@app.command() -def export( - input: Path = typer.Argument( - ..., - exists=True, - file_okay=True, - dir_okay=True, - writable=True, - help="Path to notebook file or directory of notebook files to export.", - ) -): - """Convert notebook(s) using existing configuration file. - - A .nbautoconvert configuration file is required to be in the same directory as the notebook(s). + The export command will not do cleaning, regardless of the 'clean' setting in an .nbautoexport + configuration file. """ if input.is_dir(): sentinel_path = input / SAVE_PROGRESS_INDICATOR_FILE - notebook_paths = input.glob("*.ipynb") + notebook_paths = [nb.path for nb in find_notebooks(input)] else: sentinel_path = input.parent / SAVE_PROGRESS_INDICATOR_FILE notebook_paths = [input] - validate_sentinel_path(sentinel_path) - for notebook_path in notebook_paths: + # Configuration: input options override existing sentinel file + if sentinel_path.exists(): + typer.echo(f"Reading existing configuration file from {sentinel_path} ...") config = NbAutoexportConfig.parse_file(path=sentinel_path, content_type="application/json") + + # Overrides + if len(export_formats) > 0: + typer.echo(f"Overriding config with specified export formats: {export_formats}") + config.export_formats = export_formats + if organize_by is not None: + typer.echo(f"Overriding config with specified organization strategy: {export_formats}") + config.organize_by = organize_by + else: + typer.echo("No configuration found. Using command options as configuration ...") + if len(export_formats) == 0: + typer.echo(f"No export formats specified. Using default: {DEFAULT_EXPORT_FORMATS}") + export_formats = DEFAULT_EXPORT_FORMATS + if organize_by is None: + typer.echo(f"No organize-by specified. Using default: {DEFAULT_ORGANIZE_BY}") + organize_by = DEFAULT_ORGANIZE_BY + config = NbAutoexportConfig(export_formats=export_formats, organize_by=organize_by) + + for notebook_path in notebook_paths: export_notebook(notebook_path, config=config) @@ -190,9 +197,8 @@ def install( "-f", show_default=True, help=( - "File format(s) to save for each notebook. Options are 'script', 'html', 'markdown', " - + "and 'rst'. Multiple formats should be provided using multiple flags, e.g., " - + "'-f script -f html -f markdown'." + "File format(s) to save for each notebook. Multiple formats should be provided using " + "multiple flags, e.g., '-f script -f html -f markdown'." ), ), organize_by: OrganizeBy = typer.Option( @@ -201,9 +207,7 @@ def install( "-b", show_default=True, help=( - "Whether to save exported file(s) in a folder per notebook or a folder per extension. " - + "Options are 'notebook' or 'extension'." - "" + "Whether to save exported file(s) in a subfolder per notebook or per export format. " ), ), clean: bool = typer.Option( diff --git a/nbautoexport/utils.py b/nbautoexport/utils.py index e50b254..8d8e55c 100644 --- a/nbautoexport/utils.py +++ b/nbautoexport/utils.py @@ -40,6 +40,9 @@ def from_file(cls, path): nbformat.validate(notebook) return cls(path=path, metadata=notebook.metadata) + def __hash__(self): + return hash(self.json()) + def find_notebooks(directory: Path) -> List[JupyterNotebook]: """Finds Jupyter notebooks in a directory. Not recursive. diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py deleted file mode 100644 index dd74ae5..0000000 --- a/tests/test_cli_convert.py +++ /dev/null @@ -1,119 +0,0 @@ -import shutil - -import pytest -from typer.testing import CliRunner - -from nbautoexport.clean import get_expected_exports -from nbautoexport.nbautoexport import app -from nbautoexport.sentinel import NbAutoexportConfig -from nbautoexport.utils import find_notebooks - - -EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] -EXPECTED_FORMATS = ["script", "html"] - - -@pytest.fixture() -def notebooks_dir(tmp_path, notebook_asset): - notebooks = EXPECTED_NOTEBOOKS - for nb in notebooks: - shutil.copy(notebook_asset.path, tmp_path / f"{nb}.ipynb") - return tmp_path - - -def test_convert_dir_organize_by_extension(notebooks_dir): - cmd_list = ["convert", str(notebooks_dir), "-b", "extension"] - for fmt in EXPECTED_FORMATS: - cmd_list.append("-f") - cmd_list.append(fmt) - result = CliRunner().invoke(app, cmd_list) - assert result.exit_code == 0 - - expected_notebooks = find_notebooks(notebooks_dir) - assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) - - expected_notebook_files = {nb.path for nb in expected_notebooks} - expected_exports = set( - get_expected_exports( - expected_notebooks, - NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension"), - ) - ) - - all_expected = expected_notebook_files | expected_exports - - assert set(notebooks_dir.glob("**/*")) == all_expected - - -def test_convert_dir_organize_by_notebook(notebooks_dir): - cmd_list = ["convert", str(notebooks_dir), "-b", "notebook"] - for fmt in EXPECTED_FORMATS: - cmd_list.append("-f") - cmd_list.append(fmt) - result = CliRunner().invoke(app, cmd_list) - assert result.exit_code == 0 - - expected_notebooks = find_notebooks(notebooks_dir) - assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) - - expected_notebook_files = {nb.path for nb in expected_notebooks} - expected_exports = set( - get_expected_exports( - expected_notebooks, - NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook"), - ) - ) - - all_expected = expected_notebook_files | expected_exports - assert set(notebooks_dir.glob("**/*")) == all_expected - - -def test_convert_single_organize_by_extension(notebooks_dir): - expected_notebooks = find_notebooks(notebooks_dir) - nb = expected_notebooks[0] - - cmd_list = ["convert", str(notebooks_dir / f"{nb.name}.ipynb"), "-b", "extension"] - for fmt in EXPECTED_FORMATS: - cmd_list.append("-f") - cmd_list.append(fmt) - result = CliRunner().invoke(app, cmd_list) - assert result.exit_code == 0 - - expected_notebook_files = {nb_.path for nb_ in expected_notebooks} - expected_exports = set( - get_expected_exports( - [nb], NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension"), - ) - ) - - all_expected = expected_notebook_files | expected_exports - assert set(notebooks_dir.glob("**/*")) == all_expected - - -def test_convert_single_organize_by_notebook(notebooks_dir): - expected_notebooks = find_notebooks(notebooks_dir) - nb = expected_notebooks[0] - - cmd_list = ["convert", str(notebooks_dir / f"{nb.name}.ipynb"), "-b", "notebook"] - for fmt in EXPECTED_FORMATS: - cmd_list.append("-f") - cmd_list.append(fmt) - result = CliRunner().invoke(app, cmd_list) - assert result.exit_code == 0 - - expected_notebook_files = {nb_.path for nb_ in expected_notebooks} - expected_exports = set( - get_expected_exports( - [nb], NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook"), - ) - ) - - all_expected = expected_notebook_files | expected_exports - assert set(notebooks_dir.glob("**/*")) == all_expected - - -def test_convert_no_input(): - result = CliRunner().invoke(app, ["convert"]) - - assert result.exit_code == 2 - assert "Error: Missing argument 'INPUT'." in result.stdout diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index 901980d..bf14ffa 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -1,3 +1,5 @@ +from itertools import chain, product +from pathlib import Path import shutil import pytest @@ -5,8 +7,12 @@ from nbautoexport.clean import get_expected_exports from nbautoexport.nbautoexport import app -from nbautoexport.sentinel import NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE -from nbautoexport.utils import find_notebooks +from nbautoexport.sentinel import ( + NbAutoexportConfig, + DEFAULT_EXPORT_FORMATS, + SAVE_PROGRESS_INDICATOR_FILE, +) +from nbautoexport.utils import find_notebooks, working_directory EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] EXPECTED_FORMATS = ["script", "html"] @@ -20,100 +26,201 @@ def notebooks_dir(tmp_path, notebook_asset): return tmp_path -def test_export_dir_organize_by_extension(notebooks_dir): - sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE - config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension") - with sentinel_path.open("w") as fp: - fp.write(config.json()) - - result = CliRunner().invoke(app, ["export", str(notebooks_dir)]) - assert result.exit_code == 0 - +@pytest.mark.parametrize("input_type", ["dir", "notebook"]) +def test_export_no_config_no_cli_opts(notebooks_dir, input_type): + """Test export command with no config file and no CLI options. Should use default options. + """ expected_notebooks = find_notebooks(notebooks_dir) assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + if input_type == "dir": + expected_to_convert = expected_notebooks + input_path = str(notebooks_dir) + elif input_type == "notebook": + expected_to_convert = expected_notebooks[:1] + input_path = str(expected_notebooks[0].path) + + result = CliRunner().invoke(app, ["export", input_path]) + assert result.exit_code == 0 + expected_notebook_files = {nb.path for nb in expected_notebooks} - expected_exports = set(get_expected_exports(expected_notebooks, config)) + expected_exports = set(get_expected_exports(expected_to_convert, NbAutoexportConfig())) - all_expected = expected_notebook_files | expected_exports | {sentinel_path} + all_expected = expected_notebook_files | expected_exports assert set(notebooks_dir.glob("**/*")) == all_expected -def test_export_dir_organize_by_notebook(notebooks_dir): - sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE - config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook") - with sentinel_path.open("w") as fp: - fp.write(config.json()) +@pytest.mark.parametrize( + "input_type, organize_by", product(["dir", "notebook"], ["extension", "notebook"]) +) +def test_export_no_config_with_cli_opts(notebooks_dir, input_type, organize_by): + """Test export command with no config file and CLI options. Should use CLI options. + """ + expected_notebooks = find_notebooks(notebooks_dir) + assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + + if input_type == "dir": + expected_to_convert = expected_notebooks + input_path = str(notebooks_dir) + elif input_type == "notebook": + expected_to_convert = expected_notebooks[:1] + input_path = str(expected_notebooks[0].path) - result = CliRunner().invoke(app, ["export", str(notebooks_dir)]) + flags = list(chain(["-b", organize_by], *(["-f", fmt] for fmt in EXPECTED_FORMATS))) + result = CliRunner().invoke(app, ["export", input_path] + flags) assert result.exit_code == 0 - expected_notebooks = find_notebooks(notebooks_dir) - assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + assert set(EXPECTED_FORMATS) != set(DEFAULT_EXPORT_FORMATS) # make sure test is meaningful + expected_config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by) expected_notebook_files = {nb.path for nb in expected_notebooks} - expected_exports = set(get_expected_exports(expected_notebooks, config)) + expected_exports = set(get_expected_exports(expected_to_convert, expected_config)) - all_expected = expected_notebook_files | expected_exports | {sentinel_path} + all_expected = expected_notebook_files | expected_exports assert set(notebooks_dir.glob("**/*")) == all_expected -def test_export_single_organize_by_extension(notebooks_dir): +@pytest.mark.parametrize( + "input_type, organize_by", product(["dir", "notebook"], ["extension", "notebook"]) +) +def test_export_with_config_no_cli_opts(notebooks_dir, input_type, organize_by): + """Test that export works with a config and no CLI options. Should use config options. + """ expected_notebooks = find_notebooks(notebooks_dir) - nb = expected_notebooks[0] + assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + + if input_type == "dir": + expected_to_convert = expected_notebooks + input_path = str(notebooks_dir) + elif input_type == "notebook": + expected_to_convert = expected_notebooks[:1] + input_path = str(expected_notebooks[0].path) sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE - config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="extension") + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by) with sentinel_path.open("w") as fp: fp.write(config.json()) - result = CliRunner().invoke(app, ["export", str(nb.path)]) + result = CliRunner().invoke(app, ["export", input_path]) assert result.exit_code == 0 - expected_notebook_files = {nb_.path for nb_ in expected_notebooks} - expected_exports = set(get_expected_exports([nb], config)) + expected_notebook_files = {nb.path for nb in expected_notebooks} + expected_exports = set(get_expected_exports(expected_to_convert, config)) all_expected = expected_notebook_files | expected_exports | {sentinel_path} assert set(notebooks_dir.glob("**/*")) == all_expected -def test_export_single_organize_by_notebook(notebooks_dir): +@pytest.mark.parametrize( + "input_type, organize_by", product(["dir", "notebook"], ["extension", "notebook"]) +) +def test_export_with_config_with_cli_opts(notebooks_dir, input_type, organize_by): + """Test that export works with both config and CLI options. CLI options should overide config. + """ expected_notebooks = find_notebooks(notebooks_dir) - nb = expected_notebooks[0] + assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + + if input_type == "dir": + expected_to_convert = expected_notebooks + input_path = str(notebooks_dir) + elif input_type == "notebook": + expected_to_convert = expected_notebooks[:1] + input_path = str(expected_notebooks[0].path) sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE - config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by="notebook") + written_config = NbAutoexportConfig() with sentinel_path.open("w") as fp: - fp.write(config.json()) + fp.write(written_config.json()) - result = CliRunner().invoke(app, ["export", str(nb.path)]) + expected_config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by) + assert expected_config != written_config + + flags = list(chain(["-b", organize_by], *(["-f", fmt] for fmt in EXPECTED_FORMATS))) + result = CliRunner().invoke(app, ["export", input_path] + flags) assert result.exit_code == 0 - expected_notebook_files = {nb_.path for nb_ in expected_notebooks} - expected_exports = set(get_expected_exports([nb], config)) + expected_notebook_files = {nb.path for nb in expected_notebooks} + expected_exports = set(get_expected_exports(expected_to_convert, expected_config)) + + expected_exports_from_written = set(get_expected_exports(expected_to_convert, written_config)) + assert expected_exports != expected_exports_from_written all_expected = expected_notebook_files | expected_exports | {sentinel_path} assert set(notebooks_dir.glob("**/*")) == all_expected -def test_export_no_input(): +@pytest.mark.parametrize( + "input_type, organize_by", product(["dir", "notebook"], ["extension", "notebook"]) +) +def test_export_relative(notebooks_dir, input_type, organize_by): + """ Test that export works relative to current working directory. + """ + with working_directory(notebooks_dir): + expected_notebooks = find_notebooks(Path()) + assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + + sentinel_path = Path(SAVE_PROGRESS_INDICATOR_FILE) + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by) + with sentinel_path.open("w") as fp: + fp.write(config.json()) + + if input_type == "dir": + expected_to_convert = expected_notebooks + input_path = "." + elif input_type == "notebook": + expected_to_convert = expected_notebooks[:1] + input_path = f"{expected_notebooks[0].path.name}" + + result = CliRunner().invoke(app, ["export", input_path]) + assert result.exit_code == 0 + + expected_notebook_files = {nb.path for nb in expected_notebooks} + expected_exports = set(get_expected_exports(expected_to_convert, config)) + + all_expected = expected_notebook_files | expected_exports | {sentinel_path} + assert set(Path().glob("**/*")) == all_expected + + +@pytest.mark.parametrize( + "input_type, organize_by", product(["dir", "notebook"], ["extension", "notebook"]) +) +def test_clean_relative_subdirectory(notebooks_dir, input_type, organize_by): + """ Test that export works for subdirectory relative to current working directory. + """ + with working_directory(notebooks_dir): + # Set up subdirectory + subdir = Path("subdir") + subdir.mkdir() + for subfile in Path().iterdir(): + shutil.move(str(subfile), str(subdir)) + + sentinel_path = subdir / SAVE_PROGRESS_INDICATOR_FILE + config = NbAutoexportConfig(export_formats=EXPECTED_FORMATS, organize_by=organize_by) + with sentinel_path.open("w") as fp: + fp.write(config.json()) + + expected_notebooks = find_notebooks(subdir) + assert len(expected_notebooks) == len(EXPECTED_NOTEBOOKS) + + if input_type == "dir": + expected_to_convert = expected_notebooks + input_path = "subdir" + elif input_type == "notebook": + expected_to_convert = expected_notebooks[:1] + input_path = str(subdir / f"{expected_notebooks[0].path.name}") + + result = CliRunner().invoke(app, ["export", input_path]) + assert result.exit_code == 0 + + expected_notebook_files = {nb.path for nb in expected_notebooks} + expected_exports = set(get_expected_exports(expected_to_convert, config)) + + all_expected = expected_notebook_files | expected_exports | {sentinel_path} + assert set(subdir.glob("**/*")) == all_expected + + +def test_export_no_input_error(): result = CliRunner().invoke(app, ["export"]) assert result.exit_code == 2 assert "Error: Missing argument 'INPUT'." in result.stdout - - -def test_export_missing_config_error(notebooks_dir): - sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE - - starting_files = set(notebooks_dir.glob("**/*")) - - result = CliRunner().invoke(app, ["export", str(notebooks_dir)]) - assert result.exit_code == 1 - assert "Error: Missing expected nbautoexport config file" in result.stdout - assert str(sentinel_path.resolve()) in result.stdout - - ending_files = set(notebooks_dir.glob("**/*")) - - # no files deleted - assert starting_files == ending_files diff --git a/tests/test_utils.py b/tests/test_utils.py index 74f3ff0..2d3a5d1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,12 @@ +import json from pathlib import Path +import shutil import sys import pytest -from nbautoexport.utils import cleared_argv, find_notebooks, working_directory +from nbautoexport.sentinel import NbAutoexportConfig, SAVE_PROGRESS_INDICATOR_FILE +from nbautoexport.utils import JupyterNotebook, cleared_argv, find_notebooks, working_directory def test_get_script_extensions(notebook_asset, monkeypatch): @@ -31,6 +34,32 @@ def test_get_script_extensions(notebook_asset, monkeypatch): assert notebook_asset.get_script_extension() == ".txt" +def test_find_notebooks(tmp_path, notebook_asset): + shutil.copy(notebook_asset.path, tmp_path / "the_notebook_0.ipynb") + shutil.copy(notebook_asset.path, tmp_path / "the_notebook_1.ipynb") + expected_notebooks = [ + JupyterNotebook.from_file(tmp_path / "the_notebook_0.ipynb"), + JupyterNotebook.from_file(tmp_path / "the_notebook_1.ipynb"), + ] + + # Non-notebook files + (tmp_path / "the_journal.txt").touch() + with (tmp_path / "the_log.json").open("w") as fp: + json.dump( + { + "LOG ENTRY: SOL 61": "How come Aquaman can control whales?", + "LOG ENTRY: SOL 381": "That makes me a pirate! A space pirate!", + }, + fp, + ) + with (tmp_path / SAVE_PROGRESS_INDICATOR_FILE).open("w") as fp: + fp.write(NbAutoexportConfig().json()) + + found_notebooks = find_notebooks(tmp_path) + + assert set(found_notebooks) == set(expected_notebooks) + + def test_find_notebooks_warning(tmp_path): bad_notebook_path = tmp_path / "the_journal.ipynb" bad_notebook_path.touch() From 4a15445d59a991d380529af955deb65195a07d86 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Wed, 1 Jul 2020 18:45:16 -0700 Subject: [PATCH 19/25] clean and export handling no notebooks --- nbautoexport/nbautoexport.py | 9 +++++++++ tests/test_cli_clean.py | 15 +++++++++++++++ tests/test_cli_export.py | 15 +++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/nbautoexport/nbautoexport.py b/nbautoexport/nbautoexport.py index 55274f3..d1ebef2 100644 --- a/nbautoexport/nbautoexport.py +++ b/nbautoexport/nbautoexport.py @@ -73,6 +73,10 @@ def clean( files_to_clean = find_files_to_clean(directory, config) + if len(files_to_clean) == 0: + typer.echo("No files identified for cleaning. Exiting.") + raise typer.Exit(code=0) + typer.echo("Identified following files to clean up:") for path in sorted(files_to_clean): typer.echo(f" {path}") @@ -151,6 +155,11 @@ def export( if input.is_dir(): sentinel_path = input / SAVE_PROGRESS_INDICATOR_FILE notebook_paths = [nb.path for nb in find_notebooks(input)] + + if len(notebook_paths) == 0: + typer.echo(f"No notebooks found in directory [{input}]. Exiting.") + raise typer.Exit(code=1) + else: sentinel_path = input.parent / SAVE_PROGRESS_INDICATOR_FILE notebook_paths = [input] diff --git a/tests/test_cli_clean.py b/tests/test_cli_clean.py index 35283a1..f8ffcbf 100644 --- a/tests/test_cli_clean.py +++ b/tests/test_cli_clean.py @@ -70,6 +70,11 @@ def test_clean(notebooks_dir, need_confirmation, organize_by): all_expected = {nb.path for nb in expected_notebooks} | expected_exports | {sentinel_path} assert set(notebooks_dir.glob("**/*")) == all_expected + # Run clean again, there should be nothing to do + result_rerun = CliRunner().invoke(app, ["clean", str(notebooks_dir)]) + assert result_rerun.exit_code == 0 + assert result_rerun.stdout.strip().endswith("No files identified for cleaning. Exiting.") + @pytest.mark.parametrize("organize_by", ["extension", "notebook"]) def test_clean_relative(notebooks_dir, organize_by): @@ -92,6 +97,11 @@ def test_clean_relative(notebooks_dir, organize_by): all_expected = {nb.path for nb in expected_notebooks} | expected_exports | {sentinel_path} assert set(Path().glob("**/*")) == all_expected + # Run clean again, there should be nothing to do + result_rerun = CliRunner().invoke(app, ["clean", "."]) + assert result_rerun.exit_code == 0 + assert result_rerun.stdout.strip().endswith("No files identified for cleaning. Exiting.") + @pytest.mark.parametrize("organize_by", ["extension", "notebook"]) def test_clean_relative_subdirectory(notebooks_dir, organize_by): @@ -120,6 +130,11 @@ def test_clean_relative_subdirectory(notebooks_dir, organize_by): all_expected = {nb.path for nb in expected_notebooks} | expected_exports | {sentinel_path} assert set(subdir.glob("**/*")) == all_expected + # Run clean again, there should be nothing to do + result_rerun = CliRunner().invoke(app, ["clean", "subdir"]) + assert result_rerun.exit_code == 0 + assert result_rerun.stdout.strip().endswith("No files identified for cleaning. Exiting.") + def test_clean_abort(notebooks_dir): sentinel_path = notebooks_dir / SAVE_PROGRESS_INDICATOR_FILE diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index bf14ffa..131d39f 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -219,6 +219,21 @@ def test_clean_relative_subdirectory(notebooks_dir, input_type, organize_by): assert set(subdir.glob("**/*")) == all_expected +def test_export_dir_no_notebooks_error(tmp_path): + assert len(list(tmp_path.iterdir())) == 0 + result = CliRunner().invoke(app, ["export", str(tmp_path)]) + assert result.exit_code == 1 + assert result.stdout.startswith("No notebooks found in directory") + + +def test_export_notebook_doesnt_exist_error(tmp_path): + nonexistent_notebook = tmp_path / "anne_hughes_diary.ipynb" + assert not nonexistent_notebook.exists() + result = CliRunner().invoke(app, ["export", str(nonexistent_notebook)]) + assert result.exit_code == 2 + assert "does not exist" in result.stdout + + def test_export_no_input_error(): result = CliRunner().invoke(app, ["export"]) From 2d8855406145a7556f61f1a91f57f5c25fa1c41d Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Wed, 1 Jul 2020 18:46:55 -0700 Subject: [PATCH 20/25] Remove redundant test dir specification to allow for individual tests --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 874af78..82519cd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,3 @@ [pytest] testpaths = tests -addopts = --cov=. tests --cov-report=term --cov-report=html --cov-report=xml +addopts = --cov=. --cov-report=term --cov-report=html --cov-report=xml From 5eef6a16bc9f12203304ea4776beee74896c5d78 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Wed, 1 Jul 2020 22:54:03 -0700 Subject: [PATCH 21/25] Use miniconda and install pandoc and texlive --- .editorconfig | 3 +++ .github/workflows/tests.yml | 20 ++++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.editorconfig b/.editorconfig index d4a2c44..937e6fe 100644 --- a/.editorconfig +++ b/.editorconfig @@ -19,3 +19,6 @@ insert_final_newline = false [Makefile] indent_style = tab + +[*.yml] +indent_size = 2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 163b9e2..f2c7f03 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,21 +21,33 @@ jobs: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - name: Set up Python ${{ matrix.python-version }} with Miniconda + uses: goanpeca/setup-miniconda@v1 with: python-version: ${{ matrix.python-version }} + channels: conda-forge - name: Install dependencies + shell: bash -l {0} run: | + which python + python --version python -m pip install --upgrade pip - pip install -r requirements-dev.txt + python -m pip install -r requirements-dev.txt + conda install pandoc tectonic + mkdir bin + ln -s $(which tectonic) bin/xelatex + echo "::add-path::$(pwd)/bin" + # pandoc is for markdown-based export formats. + # tectonic is a TeX distribution. a LaTeX engine is needed for pdf export format. - name: Lint package + shell: bash -l {0} run: | make lint - name: Run tests + shell: bash -l {0} run: | make test @@ -44,4 +56,4 @@ jobs: with: file: ./coverage.xml fail_ci_if_error: true - if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == 3.8 }} + if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.8 From 5c5b4f526966ddbe8dcb1d7f27c57c4eae01080c Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Thu, 2 Jul 2020 00:09:08 -0700 Subject: [PATCH 22/25] Remove test for pdf --- .github/workflows/tests.yml | 7 +------ tests/test_clean.py | 7 +++++-- tests/test_export.py | 9 +++++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f2c7f03..e8eab6b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,12 +34,7 @@ jobs: python --version python -m pip install --upgrade pip python -m pip install -r requirements-dev.txt - conda install pandoc tectonic - mkdir bin - ln -s $(which tectonic) bin/xelatex - echo "::add-path::$(pwd)/bin" - # pandoc is for markdown-based export formats. - # tectonic is a TeX distribution. a LaTeX engine is needed for pdf export format. + conda install pandoc - name: Lint package shell: bash -l {0} diff --git a/tests/test_clean.py b/tests/test_clean.py index 8236900..54bdd01 100644 --- a/tests/test_clean.py +++ b/tests/test_clean.py @@ -7,8 +7,9 @@ from nbautoexport.sentinel import ExportFormat, NbAutoexportConfig, OrganizeBy from nbautoexport.utils import find_notebooks +EXPORT_FORMATS_TO_TEST = [fmt for fmt in ExportFormat if fmt != ExportFormat.pdf] + EXPECTED_NOTEBOOKS = [f"the_notebook_{n}" for n in range(3)] -EXPECTED_FORMATS = ["script", "html"] @pytest.fixture() @@ -19,7 +20,9 @@ def notebooks_dir(tmp_path, notebook_asset): return tmp_path -@pytest.mark.parametrize("export_format, organize_by", itertools.product(ExportFormat, OrganizeBy)) +@pytest.mark.parametrize( + "export_format, organize_by", itertools.product(EXPORT_FORMATS_TO_TEST, OrganizeBy) +) def test_notebook_exports_generator(notebooks_dir, export_format, organize_by): """Test that notebook_exports_generator matches what export_notebook produces. """ diff --git a/tests/test_export.py b/tests/test_export.py index ca32546..74f488d 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -9,6 +9,9 @@ from nbautoexport.utils import JupyterNotebook +EXPORT_FORMATS_TO_TEST = [fmt for fmt in ExportFormat if fmt != ExportFormat.pdf] + + @pytest.fixture() def notebooks_dir(tmp_path, notebook_asset): shutil.copy(notebook_asset.path, tmp_path / "the_notebook.ipynb") @@ -16,14 +19,12 @@ def notebooks_dir(tmp_path, notebook_asset): @pytest.mark.parametrize("organize_by", ["extension", "notebook"]) -def test_export_notebook_by_extension(notebooks_dir, organize_by): +def test_export_notebook(notebooks_dir, organize_by): """Test that export notebook works. Explicitly write out expected files, because tests for get_expected_exports will compare against export_notebook. """ notebook = JupyterNotebook.from_file(notebooks_dir / "the_notebook.ipynb") - config = NbAutoexportConfig( - export_formats=[fmt for fmt in ExportFormat], organize_by=organize_by - ) + config = NbAutoexportConfig(export_formats=EXPORT_FORMATS_TO_TEST, organize_by=organize_by) export_notebook(notebook.path, config) expected_exports = set() From 5095d2e2c70df9d71eb3addb3cf7a180b54892c0 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Thu, 2 Jul 2020 08:55:50 -0700 Subject: [PATCH 23/25] subprocess capture_output not available in python 3.6 --- tests/test_cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 8c7b7e6..3c18768 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -40,7 +40,10 @@ def test_main_python_m(): def test_version_python_m(): result = subprocess.run( - ["python", "-m", "nbautoexport", "--version"], capture_output=True, text=True + ["python", "-m", "nbautoexport", "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, ) assert result.returncode == 0 assert result.stdout.strip() == __version__ From 1412c155d60ad719176381f4db2b940fab27e1f8 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Thu, 2 Jul 2020 09:05:06 -0700 Subject: [PATCH 24/25] Fix change to test --- tests/test_cli.py | 9 +++++++-- tests/test_export.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 3c18768..91ca9d8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -31,7 +31,12 @@ def test_version(): def test_main_python_m(): - result = subprocess.run(["python", "-m", "nbautoexport"], capture_output=True, text=True) + result = subprocess.run( + ["python", "-m", "nbautoexport"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) assert result.returncode == 0 assert "Exports Jupyter notebooks to various file formats" in result.stdout assert result.stdout.startswith("Usage: python -m nbautoexport") @@ -43,7 +48,7 @@ def test_version_python_m(): ["python", "-m", "nbautoexport", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, + universal_newlines=True, ) assert result.returncode == 0 assert result.stdout.strip() == __version__ diff --git a/tests/test_export.py b/tests/test_export.py index 74f488d..1730136 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -28,7 +28,7 @@ def test_export_notebook(notebooks_dir, organize_by): export_notebook(notebook.path, config) expected_exports = set() - for fmt in ExportFormat: + for fmt in EXPORT_FORMATS_TO_TEST: if organize_by == "extension": subfolder = notebooks_dir / fmt.value elif organize_by == "notebook": From a44bae02838dd70466f3b99162bb6112311cfdbe Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Thu, 2 Jul 2020 09:37:22 -0700 Subject: [PATCH 25/25] rename to replace to work with Windows --- nbautoexport/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbautoexport/export.py b/nbautoexport/export.py index 19a585a..bb45357 100644 --- a/nbautoexport/export.py +++ b/nbautoexport/export.py @@ -49,7 +49,7 @@ def postprocess(self, input: str): new_assets_dir = new_dir / f"{input.stem}_files" new_assets_dir.mkdir(exist_ok=True) for asset in assets_dir.iterdir(): - asset.rename(new_assets_dir / asset.name) + asset.replace(new_assets_dir / asset.name) assets_dir.rmdir() input.unlink()