From 875eea35a4803dbd6089d2732b3881a55dadc421 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Thu, 16 Jul 2020 07:10:38 +0100 Subject: [PATCH] :tada: --- .gitignore | 12 ++ .isort.cfg | 2 + .pre-commit-config.yaml | 21 ++++ AUTHORS.rst | 13 +++ CONTRIBUTING.rst | 127 +++++++++++++++++++++ HISTORY.rst | 8 ++ LICENSE | 21 ++++ MANIFEST.in | 11 ++ Makefile | 85 ++++++++++++++ README.rst | 31 ++++++ azure-pipelines.yml | 45 ++++++++ docs/Makefile | 20 ++++ docs/authors.rst | 1 + docs/conf.py | 154 ++++++++++++++++++++++++++ docs/contributing.rst | 1 + docs/history.rst | 1 + docs/index.rst | 20 ++++ docs/installation.rst | 51 +++++++++ docs/make.bat | 36 ++++++ docs/readme.rst | 1 + docs/usage.rst | 7 ++ nbqa/__init__.py | 3 + nbqa/__main__.py | 71 ++++++++++++ nbqa/put_magics_back_in.py | 12 ++ nbqa/replace_magics.py | 12 ++ nbqa/replace_source.py | 31 ++++++ nbqa/save_source.py | 28 +++++ pytest.ini | 1 + requirements-dev.txt | 7 ++ setup.cfg | 4 + setup.py | 48 ++++++++ tests/__init__.py | 0 tests/conftest.py | 20 ++++ tests/data/clean_notebook.ipynb | 41 +++++++ tests/data/notebook_for_testing.ipynb | 87 +++++++++++++++ tests/test_black.py | 39 +++++++ tests/test_flake8_works.py | 33 ++++++ tests/test_return_code.py | 61 ++++++++++ tox.ini | 14 +++ 39 files changed, 1180 insertions(+) create mode 100644 .gitignore create mode 100644 .isort.cfg create mode 100644 .pre-commit-config.yaml create mode 100644 AUTHORS.rst create mode 100644 CONTRIBUTING.rst create mode 100644 HISTORY.rst create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 README.rst create mode 100644 azure-pipelines.yml create mode 100644 docs/Makefile create mode 100644 docs/authors.rst create mode 100755 docs/conf.py create mode 100644 docs/contributing.rst create mode 100644 docs/history.rst create mode 100644 docs/index.rst create mode 100644 docs/installation.rst create mode 100644 docs/make.bat create mode 100644 docs/readme.rst create mode 100644 docs/usage.rst create mode 100644 nbqa/__init__.py create mode 100644 nbqa/__main__.py create mode 100644 nbqa/put_magics_back_in.py create mode 100644 nbqa/replace_magics.py create mode 100644 nbqa/replace_source.py create mode 100644 nbqa/save_source.py create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/data/clean_notebook.ipynb create mode 100644 tests/data/notebook_for_testing.ipynb create mode 100755 tests/test_black.py create mode 100644 tests/test_flake8_works.py create mode 100644 tests/test_return_code.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..ec91bc5b --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +*.pyc +.ipynb_checkpoints +venv +test-output.xml +.coverage +coverage.xml +htmlcov/ +junit +*.egg* +docs/_build +dist +.tox diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000..b9fb3f3e --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +profile=black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..274aeef3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.1.0 + hooks: + - id: end-of-file-fixer + - id: requirements-txt-fixer + - id: trailing-whitespace + - id: debug-statements +- repo: https://github.com/python/black + rev: 19.10b0 + hooks: + - id: black + language_version: python3 +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.3 + hooks: + - id: flake8 +- repo: https://github.com/timothycrosley/isort + rev: 5.0.9 + hooks: + - id: isort diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 00000000..96d8a391 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,13 @@ +======= +Credits +======= + +Development Lead +---------------- + +* Marco Gorelli + +Contributors +------------ + +None yet. Why not be the first? diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 00000000..6e3f9eb5 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,127 @@ +.. highlight:: shell + +============ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every little bit +helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +Report Bugs +~~~~~~~~~~~ + +Report bugs at https://github.com/MarcoGorelli/nbqa/issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +Fix Bugs +~~~~~~~~ + +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help +wanted" is open to whoever wants to implement it. + +Implement Features +~~~~~~~~~~~~~~~~~~ + +Look through the GitHub issues for features. Anything tagged with "enhancement" +and "help wanted" is open to whoever wants to implement it. + +Write Documentation +~~~~~~~~~~~~~~~~~~~ + +nbQA could always use more documentation, whether as part of the +official nbQA docs, in docstrings, or even on the web in blog posts, +articles, and such. + +Submit Feedback +~~~~~~~~~~~~~~~ + +The best way to send feedback is to file an issue at https://github.com/MarcoGorelli/nbqa/issues. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up `nbqa` for local development. + +1. Fork the `nbqa` repo on GitHub. +2. Clone your fork locally:: + + $ git clone git@github.com:your_name_here/nbqa.git nbqa-dev + +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: + + $ mkvirtualenv nbqa + $ cd nbqa-dev/ + $ python setup.py develop + +4. Create a branch for local development:: + + $ git checkout -b name-of-your-bugfix-or-feature + + Now you can make your changes locally. + +5. When you're done making changes, check that your changes pass tests, + including testing other Python versions with tox:: + + $ python setup.py test or pytest + $ tox + + To get flake8 and tox, just pip install them into your virtualenv. + +6. Commit your changes and push your branch to GitHub:: + + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + +7. Submit a pull request through the GitHub website. + +Pull Request Guidelines +----------------------- + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 3.6, 3.7 and 3.8, and for PyPy. Check + https://travis-ci.com/MarcoGorelli/nbqa/pull_requests + and make sure that the tests pass for all supported Python versions. + +Tips +---- + +To run a subset of tests:: + + + $ python -m unittest tests.test_nbqa + +Deploying +--------- + +A reminder for the maintainers on how to deploy. +Make sure all your changes are committed (including an entry in HISTORY.rst). +Then run:: + +$ bump2version patch # possible: major / minor / patch +$ git push +$ git push --tags + +Travis will then deploy to PyPI if tests pass. diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 00000000..9a31c2f5 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,8 @@ +======= +History +======= + +0.1.0 (2020-07-13) +------------------ + +* First release on PyPI. diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..b7fd64e7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020, Marco Gorelli + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..965b2dda --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,11 @@ +include AUTHORS.rst +include CONTRIBUTING.rst +include HISTORY.rst +include LICENSE +include README.rst + +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..3beace69 --- /dev/null +++ b/Makefile @@ -0,0 +1,85 @@ +.PHONY: clean clean-test clean-pyc clean-build docs help +.DEFAULT_GOAL := help + +define BROWSER_PYSCRIPT +import os, webbrowser, sys + +from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT + +define PRINT_HELP_PYSCRIPT +import re, sys + +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print("%-20s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT + +BROWSER := python -c "$$BROWSER_PYSCRIPT" + +help: + @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + +clean-build: ## remove build artifacts + rm -fr build/ + rm -fr dist/ + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + +clean-pyc: ## remove Python file artifacts + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: ## remove test and coverage artifacts + rm -fr .tox/ + rm -f .coverage + rm -fr htmlcov/ + rm -fr .pytest_cache + +lint: ## check style with flake8 + flake8 nbqa tests + +test: ## run tests quickly with the default Python + python setup.py test + +test-all: ## run tests on every Python version with tox + tox + +coverage: ## check code coverage quickly with the default Python + coverage run --source nbqa setup.py test + coverage report -m + coverage html + $(BROWSER) htmlcov/index.html + +docs: ## generate Sphinx HTML documentation, including API docs + rm -f docs/nbqa.rst + rm -f docs/modules.rst + sphinx-apidoc -o docs/ nbqa + $(MAKE) -C docs clean + $(MAKE) -C docs html + $(BROWSER) docs/_build/html/index.html + +servedocs: docs ## compile the docs watching for changes + watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . + +release: dist ## package and upload a release + twine upload dist/* + +dist: clean ## builds source and wheel package + python setup.py sdist + python setup.py bdist_wheel + ls -l dist + +install: clean ## install the package to the active Python's site-packages + python setup.py install diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..c32c6a58 --- /dev/null +++ b/README.rst @@ -0,0 +1,31 @@ +==== +nbQA +==== + +.. image:: https://dev.azure.com/megorelli/megorelli/_apis/build/status/MarcoGorelli.nbQA?branchName=master + :target: https://dev.azure.com/megorelli/megorelli/_build/latest?definitionId=1&branchName=master + +.. image:: https://img.shields.io/azure-devops/coverage/megorelli/megorelli/1/master.svg + :target: https://dev.azure.com/megorelli/megorelli/_build/latest?definitionId=1&branchName=master + +Adapter to run any code-quality tool on a Jupyter notebook. + +E.g. just as you would run `flake8` against a Python file + +.. code-block:: bash + + flake8 my_python_file.py + +you can now just as easily run it against a Jupyter notebook by prepending `nbqa`: + +.. code-block:: bash + + nbqa flake8 my_notebook.ipynb + +Installation: nbQA has no dependencies, so as long as you have Python3.6+, you can do + +.. code-block:: bash + + pip install -U nbqa + +and it'll work without conflicting with any of your existing installs! diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..496b3062 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,45 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: +- master + +pool: + vmImage: 'ubuntu-latest' +strategy: + matrix: + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + Python38: + python.version: '3.8' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + displayName: 'Install dependencies' + +- script: | + black . --check + isort . --check-only + flake8 + displayName: 'static analysis' + +- script: | + pytest --junitxml=junit/test-results.xml --cov=nbqa --cov-report=xml --cov-report=html + displayName: 'pytest' + +- task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' + reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..052bce1d --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = nbqa +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/authors.rst b/docs/authors.rst new file mode 100644 index 00000000..e122f914 --- /dev/null +++ b/docs/authors.rst @@ -0,0 +1 @@ +.. include:: ../AUTHORS.rst diff --git a/docs/conf.py b/docs/conf.py new file mode 100755 index 00000000..5ec7b88f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +# +# nbqa documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 9 13:47:02 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another +# directory, add these directories to sys.path here. If the directory is +# relative to the documentation root, use os.path.abspath to make it +# absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath("..")) + +import nbqa # noqa + +# -- General configuration --------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ["sphinx.ext.autodoc", "sphinx.ext.viewcode"] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "nbQA" +copyright = "2020, Marco Gorelli" +author = "Marco Gorelli" + +# The version info for the project you're documenting, acts as replacement +# for |version| and |release|, also used in various other places throughout +# the built documents. +# +# The short X.Y version. +version = nbqa.__version__ +# The full version, including alpha/beta/rc tags. +release = nbqa.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "alabaster" + +# Theme options are theme-specific and customize the look and feel of a +# theme further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + + +# -- Options for HTMLHelp output --------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = "nbqadoc" + + +# -- Options for LaTeX output ------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto, manual, or own class]). +latex_documents = [ + (master_doc, "nbqa.tex", "nbQA Documentation", "Marco Gorelli", "manual"), +] + + +# -- Options for manual page output ------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "nbqa", "nbQA Documentation", [author], 1)] + + +# -- Options for Texinfo output ---------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "nbqa", + "nbQA Documentation", + author, + "nbqa", + "One line description of project.", + "Miscellaneous", + ), +] diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 00000000..e582053e --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 00000000..25064996 --- /dev/null +++ b/docs/history.rst @@ -0,0 +1 @@ +.. include:: ../HISTORY.rst diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..fe206e17 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +Welcome to nbQA's documentation! +====================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + readme + installation + usage + modules + contributing + authors + history + +Indices and tables +================== +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 00000000..60387f5c --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,51 @@ +.. highlight:: shell + +============ +Installation +============ + + +Stable release +-------------- + +To install nbQA, run this command in your terminal: + +.. code-block:: console + + $ pip install nbqa + +This is the preferred method to install nbQA, as it will always install the most recent stable release. + +If you don't have `pip`_ installed, this `Python installation guide`_ can guide +you through the process. + +.. _pip: https://pip.pypa.io +.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ + + +From sources +------------ + +The sources for nbQA can be downloaded from the `Github repo`_. + +You can either clone the public repository: + +.. code-block:: console + + $ git clone git://github.com/MarcoGorelli/nbqa + +Or download the `tarball`_: + +.. code-block:: console + + $ curl -OJL https://github.com/MarcoGorelli/nbqa/tarball/master + +Once you have a copy of the source, you can install it with: + +.. code-block:: console + + $ python setup.py install + + +.. _Github repo: https://github.com/MarcoGorelli/nbqa +.. _tarball: https://github.com/MarcoGorelli/nbqa/tarball/master diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..36f77ffd --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=python -msphinx +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=nbqa + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The Sphinx module was not found. Make sure you have Sphinx installed, + echo.then set the SPHINXBUILD environment variable to point to the full + echo.path of the 'sphinx-build' executable. Alternatively you may add the + echo.Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 00000000..72a33558 --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1 @@ +.. include:: ../README.rst diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 00000000..ed7f397f --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,7 @@ +===== +Usage +===== + +To use nbQA in a project:: + + import nbqa diff --git a/nbqa/__init__.py b/nbqa/__init__.py new file mode 100644 index 00000000..a10f5cfa --- /dev/null +++ b/nbqa/__init__.py @@ -0,0 +1,3 @@ +__author__ = """Marco Gorelli""" +__email__ = "m.e.gorelli@gmail.com" +__version__ = "0.1.0" diff --git a/nbqa/__main__.py b/nbqa/__main__.py new file mode 100644 index 00000000..910d11ce --- /dev/null +++ b/nbqa/__main__.py @@ -0,0 +1,71 @@ +import re +import subprocess +import sys +from pathlib import Path + +from nbqa import put_magics_back_in, replace_magics, replace_source, save_source + + +def main(args=None): + + import argparse + + parser = argparse.ArgumentParser(description="Process some integers.") + parser.add_argument("command") + parser.add_argument("root_dir", default=".", nargs="?") + args, kwargs = parser.parse_known_args(args) + command = args.command + root_dir = args.root_dir + + if not Path(root_dir).is_dir(): + notebooks = [Path(root_dir)] + else: + notebooks = Path(root_dir).rglob("*.ipynb") + output_code = 0 + for notebook in notebooks: + if "ipynb_checkpoints" in str(notebook): + continue + + tempfile = save_source.main(notebook) + replace_magics.main(tempfile) + + output = subprocess.run( + [command, tempfile, *kwargs], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + if output_code == 0: + output_code = output.returncode + + # replace ending, convert to str + out = output.stdout.decode().replace(tempfile, notebook.name) + err = output.stderr.decode().replace(tempfile, notebook.name) + + with open(tempfile, "r") as handle: + cells = handle.readlines() + mapping = {} + cell_no = 0 + cell_count = None + for n, i in enumerate(cells): + if i == "# %%\n": + cell_no += 1 + cell_count = 0 + else: + cell_count += 1 + mapping[n + 1] = f"cell_{cell_no}:{cell_count}" + out = re.sub( + rf"(?<={notebook.name}:)\d+", lambda x: str(mapping[int(x.group())]), out, + ) + + sys.stdout.write(out) + sys.stderr.write(err) + + put_magics_back_in.main(tempfile) + + replace_source.main(tempfile, notebook) + + sys.exit(output_code) + + +if __name__ == "__main__": + main() diff --git a/nbqa/put_magics_back_in.py b/nbqa/put_magics_back_in.py new file mode 100644 index 00000000..465a8704 --- /dev/null +++ b/nbqa/put_magics_back_in.py @@ -0,0 +1,12 @@ +import re + + +def main(path): + + with open(path, "r") as handle: + file = handle.read() + + file = re.sub(r"# (%%\w+)", r"\1", file) + + with open(path, "w") as handle: + handle.write(file) diff --git a/nbqa/replace_magics.py b/nbqa/replace_magics.py new file mode 100644 index 00000000..905d181b --- /dev/null +++ b/nbqa/replace_magics.py @@ -0,0 +1,12 @@ +import re + + +def main(path): + + with open(path, "r") as handle: + file = handle.read() + + file = re.sub(r"(%%\w+)", r"# \1", file) + + with open(path, "w") as handle: + handle.write(file) diff --git a/nbqa/replace_source.py b/nbqa/replace_source.py new file mode 100644 index 00000000..b9d87572 --- /dev/null +++ b/nbqa/replace_source.py @@ -0,0 +1,31 @@ +import json + + +def main(python_file, notebook): + with open(notebook, "r") as handle: + notebook_json = json.load(handle) + + with open(python_file, "r") as handle: + pyfile = handle.read() + + pycells = pyfile[len("# %%") :].split("\n\n\n# %%") + + new_sources = [] + for i in pycells: + cells = i.splitlines(True) + if cells[0] == "\n": + new_sources.append({"source": cells[1:], "cell_type": "code"}) + else: + new_sources.append({"source": cells[1:], "cell_type": "markdown"}) + + new_sources = [ + { + **{key: val for key, val in i.items() if i != "source"}, + **{"source": new_sources[n]["source"]}, + } + for n, i in enumerate(notebook_json["cells"]) + ] + notebook_json.update({"cells": new_sources}) + with open(notebook, "w") as handle: + json.dump(notebook_json, handle, indent=1, ensure_ascii=False) + handle.write("\n") diff --git a/nbqa/save_source.py b/nbqa/save_source.py new file mode 100644 index 00000000..2655a570 --- /dev/null +++ b/nbqa/save_source.py @@ -0,0 +1,28 @@ +import json +import tempfile + +CODE_SEPARATOR = "\n\n# %%\n" + +MARKDOWN_SEPARATOR = "\n\n# %% [markdown]\n" + + +def main(path): + + _, filename = tempfile.mkstemp(suffix=".py") + + with open(path, "r") as handel: + notebook = json.load(handel) + + cells = notebook["cells"] + + result = [ + f"{CODE_SEPARATOR}{''.join(i['source'])}\n" + if i["cell_type"] == "code" + else f"{MARKDOWN_SEPARATOR}{''.join(i['source'])}\n" + for i in cells + ] + + with open(filename, "w") as handle: + handle.write("".join(result)[len("\n\n") : -len("\n")]) + + return filename diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..eea2c180 --- /dev/null +++ b/pytest.ini @@ -0,0 +1 @@ +[pytest] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..92413c45 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +black==19.10b0 +flake8==3.8.3 +pre-commit==2.6.0 +pytest==5.4.3 +pytest-azurepipelines==0.8.0 +pytest-cov==2.10.0 +Sphinx==3.1.2 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..48438a59 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +ignore=E203,E503,W504 +max-line-length=120 +exclude=venv,.* diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..ab948913 --- /dev/null +++ b/setup.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +"""The setup script.""" + +from setuptools import find_packages, setup + +with open("README.rst") as readme_file: + readme = readme_file.read() + +with open("HISTORY.rst") as history_file: + history = history_file.read() + +requirements = [] + +setup_requirements = [] + +test_requirements = [] + +setup( + author="Marco Gorelli", + author_email="m.e.gorelli@gmail.com", + python_requires=">=3.6", + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + ], + description="Run any Python code quality tool on a Jupyter Notebook!", + entry_points={"console_scripts": ["nbqa=nbqa.__main__:main"]}, + install_requires=requirements, + license="MIT license", + long_description=readme + "\n\n" + history, + include_package_data=True, + keywords="nbqa", + name="nbqa", + packages=find_packages(include=["nbqa", "nbqa.*"]), + setup_requires=setup_requirements, + test_suite="tests", + tests_require=test_requirements, + url="https://github.com/MarcoGorelli/nbQA", + version="0.1.1", + zip_safe=False, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..c271792d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,20 @@ +import shutil +from pathlib import Path + +import pytest + + +@pytest.fixture +def tmp_notebook_for_testing(tmpdir): + """ + Make temporary copy of test notebook before it's operated on, then revert it. + """ + filename = Path("tests/data") / "notebook_for_testing.ipynb" + temp_file = Path(tmpdir) / "tmp.ipynb" + shutil.copy( + str(filename), str(temp_file), + ) + yield filename + shutil.copy( + str(temp_file), str(filename), + ) diff --git a/tests/data/clean_notebook.ipynb b/tests/data/clean_notebook.ipynb new file mode 100644 index 00000000..1003cda5 --- /dev/null +++ b/tests/data/clean_notebook.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "assert 1 + 1 == 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/data/notebook_for_testing.ipynb b/tests/data/notebook_for_testing.ipynb new file mode 100644 index 00000000..365981fe --- /dev/null +++ b/tests/data/notebook_for_testing.ipynb @@ -0,0 +1,87 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "import numpy as np\n", + "\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Some markdown cell containing \\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 9 µs, sys: 6 µs, total: 15 µs\n", + "Wall time: 17.9 µs\n" + ] + }, + { + "data": { + "text/plain": [ + "'hello 3'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "def hello(name: str = \"world\\n\"):\n", + "\n", + " return f'hello {name}'\n", + "\n", + "\n", + "hello(3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/test_black.py b/tests/test_black.py new file mode 100755 index 00000000..86b1bc9e --- /dev/null +++ b/tests/test_black.py @@ -0,0 +1,39 @@ +import difflib + +import pytest + +from nbqa.__main__ import main + + +def test_black_works(tmp_notebook_for_testing, capsys): + """ + Check black works. Should only reformat code cells. + """ + # check diff + with open(tmp_notebook_for_testing, "r") as handle: + before = handle.readlines() + with pytest.raises(SystemExit): + main(["black", "tests/data/notebook_for_testing.ipynb"]) + with open(tmp_notebook_for_testing, "r") as handle: + after = handle.readlines() + result = "".join(difflib.unified_diff(before, after)) + expected = ( + "--- \n" + "+++ \n" + "@@ -48,7 +48,7 @@\n" + ' "%%time\\n",\n' + ' "def hello(name: str = \\"world\\\\n\\"):\\n",\n' + ' "\\n",\n- " return f\'hello {name}\'\\n",\n' + '+ " return f\\"hello {name}\\"\\n",\n "\\n",\n' + ' "\\n",\n "hello(3)"\n' + ) + assert result == expected + + # check out and err + out, err = capsys.readouterr() + expected_out = "" + expected_err = ( + "reformatted notebook_for_testing.ipynb\nAll done! ✨ 🍰 ✨\n1 file reformatted.\n" + ) + assert out == expected_out + assert err == expected_err diff --git a/tests/test_flake8_works.py b/tests/test_flake8_works.py new file mode 100644 index 00000000..69e4c7b0 --- /dev/null +++ b/tests/test_flake8_works.py @@ -0,0 +1,33 @@ +import difflib + +import pytest + +from nbqa.__main__ import main + + +def test_flake8_works(tmp_notebook_for_testing, capsys): + """ + Check flake8 works. Shouldn't alter the notebook content. + """ + # check diff + with open(tmp_notebook_for_testing, "r") as handle: + before = handle.readlines() + with pytest.raises(SystemExit): + main(["flake8"]) + + with open(tmp_notebook_for_testing, "r") as handle: + after = handle.readlines() + result = "".join(difflib.unified_diff(before, after)) + expected = "" + assert result == expected + + # check out and err + out, err = capsys.readouterr() + expected_out = ( + "notebook_for_testing.ipynb:cell_1:1:1: F401 'pandas as pd' imported but unused\n" + "notebook_for_testing.ipynb:cell_1:3:1: F401 'numpy as np' imported but unused\n" + "notebook_for_testing.ipynb:cell_1:5:1: F401 'os' imported but unused\n" + ) + expected_err = "" + assert out == expected_out + assert err == expected_err diff --git a/tests/test_return_code.py b/tests/test_return_code.py new file mode 100644 index 00000000..308dbb97 --- /dev/null +++ b/tests/test_return_code.py @@ -0,0 +1,61 @@ +import subprocess + + +def test_flake8_return_code(): + output = subprocess.run( + ["python", "-m", "nbqa", "flake8", "tests/data/notebook_for_testing.ipynb"] + ) + result = output.returncode + expected = 1 + assert result == expected + + output = subprocess.run( + ["python", "-m", "nbqa", "flake8", "tests/data/clean_notebook.ipynb"] + ) + result = output.returncode + expected = 0 + assert result == expected + + +def test_black_return_code(): + output = subprocess.run( + [ + "python", + "-m", + "nbqa", + "black", + "tests/data/notebook_for_testing.ipynb", + "--check", + ] + ) + result = output.returncode + expected = 1 + assert result == expected + + output = subprocess.run( + ["python", "-m", "nbqa", "black", "tests/data/clean_notebook.ipynb", "--check"] + ) + result = output.returncode + expected = 0 + assert result == expected + + output = subprocess.run(["python", "-m", "nbqa", "black", "--check"]) + result = output.returncode + expected = 1 + assert result == expected + + output = subprocess.run( + [ + "python", + "-m", + "nbqa", + "black", + "tests/data/clean_notebook.ipynb", + "--check", + "-l", + "1", + ] + ) + result = output.returncode + expected = 1 + assert result == expected diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..79ebf853 --- /dev/null +++ b/tox.ini @@ -0,0 +1,14 @@ +[tox] +envlist = py36, py37, py38 + +[travis] +python = + 3.8: py38 + 3.7: py37 + 3.6: py36 + +[testenv] +setenv = + PYTHONPATH = {toxinidir} + +commands = python setup.py test