From 3dac4fd87a8e807218cfe915fc6144d08f03e5c3 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Thu, 13 Jan 2022 12:19:31 -0300 Subject: [PATCH 01/34] specify major and minor identifier to match branch names when merge is done and bump versions accordingly --- .github/workflows/publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 850e3745..8411c524 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,6 +20,8 @@ jobs: id: git-version with: release-branch: main + minor-identifier: feature/ + major-identifier: breaking/ publish: runs-on: ubuntu-latest From 5a6033e9ae64ec097d6f8e0ae61a1eaca6ce0239 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Thu, 13 Jan 2022 12:20:53 -0300 Subject: [PATCH 02/34] update dependencies versions --- poetry.lock | 130 ++++++++++++++++++++++++++++------------------------ 1 file changed, 71 insertions(+), 59 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1ffa620e..a3c9cdde 100644 --- a/poetry.lock +++ b/poetry.lock @@ -19,17 +19,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "21.2.0" +version = "21.4.0" description = "Classes Without Boilerplate" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "black" @@ -150,7 +150,7 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.24" +version = "3.1.26" description = "GitPython is a python library used to interact with Git repositories" category = "main" optional = false @@ -158,11 +158,10 @@ python-versions = ">=3.7" [package.dependencies] gitdb = ">=4.0.1,<5" -typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\""} [[package]] name = "importlib-metadata" -version = "4.9.0" +version = "4.10.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -174,7 +173,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -293,11 +292,11 @@ i18n = ["babel (>=2.9.0)"] [[package]] name = "mkdocs-autorefs" -version = "0.3.0" +version = "0.3.1" description = "Automatically link across pages in MkDocs." category = "dev" optional = false -python-versions = ">=3.6,<4.0" +python-versions = ">=3.6.2,<4.0.0" [package.dependencies] Markdown = ">=3.3,<4.0" @@ -316,7 +315,7 @@ mkdocs = ">=1.2" [[package]] name = "mkdocs-material" -version = "8.1.2" +version = "8.1.6" description = "A Material Design theme for MkDocs" category = "dev" optional = false @@ -401,11 +400,11 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [[package]] name = "platformdirs" -version = "2.4.0" +version = "2.4.1" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] @@ -441,7 +440,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pydantic" -version = "1.8.2" +version = "1.9.0" description = "Data validation and settings management using python 3.6 type hinting" category = "main" optional = false @@ -478,7 +477,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pygments" -version = "2.10.0" +version = "2.11.2" description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false @@ -588,7 +587,7 @@ pyyaml = "*" [[package]] name = "rich" -version = "10.16.1" +version = "10.16.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false @@ -712,15 +711,15 @@ watchmedo = ["PyYAML (>=3.10)"] [[package]] name = "zipp" -version = "3.6.0" +version = "3.7.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] [metadata] lock-version = "1.1" @@ -737,8 +736,8 @@ atomicwrites = [ {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, - {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] black = [ {file = "black-21.12b0-py3-none-any.whl", hash = "sha256:a615e69ae185e08fdd73e4715e260e2479c861b5740057fde6e8b4e3b7dd589f"}, @@ -822,12 +821,12 @@ gitdb = [ {file = "gitdb-4.0.9.tar.gz", hash = "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa"}, ] gitpython = [ - {file = "GitPython-3.1.24-py3-none-any.whl", hash = "sha256:dc0a7f2f697657acc8d7f89033e8b1ea94dd90356b2983bca89dc8d2ab3cc647"}, - {file = "GitPython-3.1.24.tar.gz", hash = "sha256:df83fdf5e684fef7c6ee2c02fc68a5ceb7e7e759d08b694088d0cacb4eba59e5"}, + {file = "GitPython-3.1.26-py3-none-any.whl", hash = "sha256:26ac35c212d1f7b16036361ca5cff3ec66e11753a0d677fb6c48fa4e1a9dd8d6"}, + {file = "GitPython-3.1.26.tar.gz", hash = "sha256:fc8868f63a2e6d268fb25f481995ba185a85a66fcad126f039323ff6635669ee"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.9.0-py3-none-any.whl", hash = "sha256:e8b45564028bc25f8c99f546616112a6df5de6655893d7eb74c9a99680dc9751"}, - {file = "importlib_metadata-4.9.0.tar.gz", hash = "sha256:ee50794eccb0ec340adbc838344ebb9a6ff2bcba78f752d31fc716497e2149d6"}, + {file = "importlib_metadata-4.10.0-py3-none-any.whl", hash = "sha256:b7cf7d3fef75f1e4c80a96ca660efbd51473d7e8f39b5ab9210febc7809012a4"}, + {file = "importlib_metadata-4.10.0.tar.gz", hash = "sha256:92a8b58ce734b2a4494878e0ecf7d79ccd7a128b5fc6014c401e0b61f006f0f6"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -898,16 +897,16 @@ mkdocs = [ {file = "mkdocs-1.2.3.tar.gz", hash = "sha256:89f5a094764381cda656af4298727c9f53dc3e602983087e1fe96ea1df24f4c1"}, ] mkdocs-autorefs = [ - {file = "mkdocs-autorefs-0.3.0.tar.gz", hash = "sha256:2f89556eb2107d72e3aff41b04dcaaf1125d407a33b8027fbc982137d248d37d"}, - {file = "mkdocs_autorefs-0.3.0-py3-none-any.whl", hash = "sha256:261875003e49b5d708993fd2792a69d624cbc8cf7de49e96c81d3d9825977ca4"}, + {file = "mkdocs-autorefs-0.3.1.tar.gz", hash = "sha256:12baad29359f468b44d980ed35b713715409097a1d8e3d0ef90962db95205eda"}, + {file = "mkdocs_autorefs-0.3.1-py3-none-any.whl", hash = "sha256:f0fd7c115eaafda7fb16bf5ff5d70eda55d7c0599eac64f8b25eacf864312a85"}, ] mkdocs-coverage = [ {file = "mkdocs-coverage-0.2.5.tar.gz", hash = "sha256:6a9a796a50b873a62f273d4ac43f41f1f96c1c6c2154038ab6072e253818490a"}, {file = "mkdocs_coverage-0.2.5-py3-none-any.whl", hash = "sha256:d94a5c020ff37bd59b7bd27424df1ce57e5437597a627edcf6116366f96997e6"}, ] mkdocs-material = [ - {file = "mkdocs-material-8.1.2.tar.gz", hash = "sha256:83b73d62b11cbc97c3b05de44f2fad3b96edf3fe2cf977851972e2c25348893b"}, - {file = "mkdocs_material-8.1.2-py2.py3-none-any.whl", hash = "sha256:2fe84abc86daa3cb5b5e7e0438c20c6e2a99c1f4a7527c783b0f8b1fb4fb840a"}, + {file = "mkdocs-material-8.1.6.tar.gz", hash = "sha256:12eb74faf018950f51261a773f9bea12cc296ec4bdbb2c8cf74102ee35b6df79"}, + {file = "mkdocs_material-8.1.6-py2.py3-none-any.whl", hash = "sha256:b2303413e3154502759f90ee2720b451be8855f769c385d8fb06a93ce54aafe2"}, ] mkdocs-material-extensions = [ {file = "mkdocs-material-extensions-1.0.3.tar.gz", hash = "sha256:bfd24dfdef7b41c312ede42648f9eb83476ea168ec163b613f9abd12bbfddba2"}, @@ -955,8 +954,8 @@ pathspec = [ {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, ] platformdirs = [ - {file = "platformdirs-2.4.0-py3-none-any.whl", hash = "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d"}, - {file = "platformdirs-2.4.0.tar.gz", hash = "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2"}, + {file = "platformdirs-2.4.1-py3-none-any.whl", hash = "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca"}, + {file = "platformdirs-2.4.1.tar.gz", hash = "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, @@ -971,28 +970,41 @@ pycodestyle = [ {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, ] pydantic = [ - {file = "pydantic-1.8.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:05ddfd37c1720c392f4e0d43c484217b7521558302e7069ce8d318438d297739"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a7c6002203fe2c5a1b5cbb141bb85060cbff88c2d78eccbc72d97eb7022c43e4"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:589eb6cd6361e8ac341db97602eb7f354551482368a37f4fd086c0733548308e"}, - {file = "pydantic-1.8.2-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:10e5622224245941efc193ad1d159887872776df7a8fd592ed746aa25d071840"}, - {file = "pydantic-1.8.2-cp36-cp36m-win_amd64.whl", hash = "sha256:99a9fc39470010c45c161a1dc584997f1feb13f689ecf645f59bb4ba623e586b"}, - {file = "pydantic-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a83db7205f60c6a86f2c44a61791d993dff4b73135df1973ecd9eed5ea0bda20"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:41b542c0b3c42dc17da70554bc6f38cbc30d7066d2c2815a94499b5684582ecb"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1"}, - {file = "pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:18b5ea242dd3e62dbf89b2b0ec9ba6c7b5abaf6af85b95a97b00279f65845a23"}, - {file = "pydantic-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:234a6c19f1c14e25e362cb05c68afb7f183eb931dd3cd4605eafff055ebbf287"}, - {file = "pydantic-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:021ea0e4133e8c824775a0cfe098677acf6fa5a3cbf9206a376eed3fc09302cd"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e710876437bc07bd414ff453ac8ec63d219e7690128d925c6e82889d674bb505"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:ac8eed4ca3bd3aadc58a13c2aa93cd8a884bcf21cb019f8cfecaae3b6ce3746e"}, - {file = "pydantic-1.8.2-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4a03cbbe743e9c7247ceae6f0d8898f7a64bb65800a45cbdc52d65e370570820"}, - {file = "pydantic-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:8621559dcf5afacf0069ed194278f35c255dc1a1385c28b32dd6c110fd6531b3"}, - {file = "pydantic-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8b223557f9510cf0bfd8b01316bf6dd281cf41826607eada99662f5e4963f316"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:244ad78eeb388a43b0c927e74d3af78008e944074b7d0f4f696ddd5b2af43c62"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:05ef5246a7ffd2ce12a619cbb29f3307b7c4509307b1b49f456657b43529dc6f"}, - {file = "pydantic-1.8.2-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:54cd5121383f4a461ff7644c7ca20c0419d58052db70d8791eacbbe31528916b"}, - {file = "pydantic-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:4be75bebf676a5f0f87937c6ddb061fa39cbea067240d98e298508c1bda6f3f3"}, - {file = "pydantic-1.8.2-py3-none-any.whl", hash = "sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833"}, - {file = "pydantic-1.8.2.tar.gz", hash = "sha256:26464e57ccaafe72b7ad156fdaa4e9b9ef051f69e175dbbb463283000c05ab7b"}, + {file = "pydantic-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cb23bcc093697cdea2708baae4f9ba0e972960a835af22560f6ae4e7e47d33f5"}, + {file = "pydantic-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1d5278bd9f0eee04a44c712982343103bba63507480bfd2fc2790fa70cd64cf4"}, + {file = "pydantic-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab624700dc145aa809e6f3ec93fb8e7d0f99d9023b713f6a953637429b437d37"}, + {file = "pydantic-1.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8d7da6f1c1049eefb718d43d99ad73100c958a5367d30b9321b092771e96c25"}, + {file = "pydantic-1.9.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3c3b035103bd4e2e4a28da9da7ef2fa47b00ee4a9cf4f1a735214c1bcd05e0f6"}, + {file = "pydantic-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3011b975c973819883842c5ab925a4e4298dffccf7782c55ec3580ed17dc464c"}, + {file = "pydantic-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:086254884d10d3ba16da0588604ffdc5aab3f7f09557b998373e885c690dd398"}, + {file = "pydantic-1.9.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0fe476769acaa7fcddd17cadd172b156b53546ec3614a4d880e5d29ea5fbce65"}, + {file = "pydantic-1.9.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8e9dcf1ac499679aceedac7e7ca6d8641f0193c591a2d090282aaf8e9445a46"}, + {file = "pydantic-1.9.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1e4c28f30e767fd07f2ddc6f74f41f034d1dd6bc526cd59e63a82fe8bb9ef4c"}, + {file = "pydantic-1.9.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:c86229333cabaaa8c51cf971496f10318c4734cf7b641f08af0a6fbf17ca3054"}, + {file = "pydantic-1.9.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:c0727bda6e38144d464daec31dff936a82917f431d9c39c39c60a26567eae3ed"}, + {file = "pydantic-1.9.0-cp36-cp36m-win_amd64.whl", hash = "sha256:dee5ef83a76ac31ab0c78c10bd7d5437bfdb6358c95b91f1ba7ff7b76f9996a1"}, + {file = "pydantic-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9c9bdb3af48e242838f9f6e6127de9be7063aad17b32215ccc36a09c5cf1070"}, + {file = "pydantic-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ee7e3209db1e468341ef41fe263eb655f67f5c5a76c924044314e139a1103a2"}, + {file = "pydantic-1.9.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b6037175234850ffd094ca77bf60fb54b08b5b22bc85865331dd3bda7a02fa1"}, + {file = "pydantic-1.9.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b2571db88c636d862b35090ccf92bf24004393f85c8870a37f42d9f23d13e032"}, + {file = "pydantic-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8b5ac0f1c83d31b324e57a273da59197c83d1bb18171e512908fe5dc7278a1d6"}, + {file = "pydantic-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bbbc94d0c94dd80b3340fc4f04fd4d701f4b038ebad72c39693c794fd3bc2d9d"}, + {file = "pydantic-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e0896200b6a40197405af18828da49f067c2fa1f821491bc8f5bde241ef3f7d7"}, + {file = "pydantic-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bdfdadb5994b44bd5579cfa7c9b0e1b0e540c952d56f627eb227851cda9db77"}, + {file = "pydantic-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:574936363cd4b9eed8acdd6b80d0143162f2eb654d96cb3a8ee91d3e64bf4cf9"}, + {file = "pydantic-1.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c556695b699f648c58373b542534308922c46a1cda06ea47bc9ca45ef5b39ae6"}, + {file = "pydantic-1.9.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f947352c3434e8b937e3aa8f96f47bdfe6d92779e44bb3f41e4c213ba6a32145"}, + {file = "pydantic-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5e48ef4a8b8c066c4a31409d91d7ca372a774d0212da2787c0d32f8045b1e034"}, + {file = "pydantic-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:96f240bce182ca7fe045c76bcebfa0b0534a1bf402ed05914a6f1dadff91877f"}, + {file = "pydantic-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:815ddebb2792efd4bba5488bc8fde09c29e8ca3227d27cf1c6990fc830fd292b"}, + {file = "pydantic-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c5b77947b9e85a54848343928b597b4f74fc364b70926b3c4441ff52620640c"}, + {file = "pydantic-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c68c3bc88dbda2a6805e9a142ce84782d3930f8fdd9655430d8576315ad97ce"}, + {file = "pydantic-1.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a79330f8571faf71bf93667d3ee054609816f10a259a109a0738dac983b23c3"}, + {file = "pydantic-1.9.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f5a64b64ddf4c99fe201ac2724daada8595ada0d102ab96d019c1555c2d6441d"}, + {file = "pydantic-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a733965f1a2b4090a5238d40d983dcd78f3ecea221c7af1497b845a9709c1721"}, + {file = "pydantic-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cc6a4cb8a118ffec2ca5fcb47afbacb4f16d0ab8b7350ddea5e8ef7bcc53a16"}, + {file = "pydantic-1.9.0-py3-none-any.whl", hash = "sha256:085ca1de245782e9b46cefcf99deecc67d418737a1fd3f6a4f511344b613a5b3"}, + {file = "pydantic-1.9.0.tar.gz", hash = "sha256:742645059757a56ecd886faf4ed2441b9c0cd406079c2b4bee51bcc3fbcd510a"}, ] pydocstyle = [ {file = "pydocstyle-6.1.1-py3-none-any.whl", hash = "sha256:6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4"}, @@ -1003,8 +1015,8 @@ pyflakes = [ {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, ] pygments = [ - {file = "Pygments-2.10.0-py3-none-any.whl", hash = "sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380"}, - {file = "Pygments-2.10.0.tar.gz", hash = "sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6"}, + {file = "Pygments-2.11.2-py3-none-any.whl", hash = "sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65"}, + {file = "Pygments-2.11.2.tar.gz", hash = "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a"}, ] pymdown-extensions = [ {file = "pymdown-extensions-9.1.tar.gz", hash = "sha256:74247f2c80f1d9e3c7242abe1c16317da36c6f26c7ad4b8a7f457f0ec20f0365"}, @@ -1070,8 +1082,8 @@ pyyaml-env-tag = [ {file = "pyyaml_env_tag-0.1.tar.gz", hash = "sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb"}, ] rich = [ - {file = "rich-10.16.1-py3-none-any.whl", hash = "sha256:bbe04dd6ac09e4b00d22cb1051aa127beaf6e16c3d8687b026e96d3fca6aad52"}, - {file = "rich-10.16.1.tar.gz", hash = "sha256:4949e73de321784ef6664ebbc854ac82b20ff60b2865097b93f3b9b41e30da27"}, + {file = "rich-10.16.2-py3-none-any.whl", hash = "sha256:c59d73bd804c90f747c8d7b1d023b88f2a9ac2454224a4aeaf959b21eeb42d03"}, + {file = "rich-10.16.2.tar.gz", hash = "sha256:720974689960e06c2efdb54327f8bf0cdbdf4eae4ad73b6c94213cad405c371b"}, ] shellingham = [ {file = "shellingham-1.4.0-py2.py3-none-any.whl", hash = "sha256:536b67a0697f2e4af32ab176c00a50ac2899c5a05e0d8e2dadac8e58888283f9"}, @@ -1139,6 +1151,6 @@ watchdog = [ {file = "watchdog-2.1.6.tar.gz", hash = "sha256:a36e75df6c767cbf46f61a91c70b3ba71811dfa0aca4a324d9407a06a8b7a2e7"}, ] zipp = [ - {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, - {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"}, + {file = "zipp-3.7.0-py3-none-any.whl", hash = "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375"}, + {file = "zipp-3.7.0.tar.gz", hash = "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d"}, ] From 89b96dfaa69e2e32b9de5b94cdfc548795da160f Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Thu, 13 Jan 2022 15:17:37 -0300 Subject: [PATCH 03/34] add tomli as dependency for parsing `pyproject.toml` --- poetry.lock | 10 +++++----- pyproject.toml | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index a3c9cdde..3026f2bd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -643,9 +643,9 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tomli" -version = "1.2.3" +version = "0.2.10" description = "A lil' TOML parser" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -724,7 +724,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "e7b1b2993e98255ef82299adaf0828d67c2798d2e9c8854d310f8d9057e3557d" +content-hash = "649505e5cbac9517769e00e3feb67836ef5677a0862594a0dc7ec26cb2924144" [metadata.files] astunparse = [ @@ -1106,8 +1106,8 @@ toml = [ {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] tomli = [ - {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, - {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, + {file = "tomli-0.2.10-py3-none-any.whl", hash = "sha256:92e0d13492eb5d2c747e276db1fce381e0d50ace4906da410db9adb9c5267176"}, + {file = "tomli-0.2.10.tar.gz", hash = "sha256:bcc8c2345da5a39f83a63f8b63b239779a33066b603b28f463d84d8ca981bd40"}, ] typer = [ {file = "typer-0.3.2-py3-none-any.whl", hash = "sha256:ba58b920ce851b12a2d790143009fa00ac1d05b3ff3257061ff69dbdfc3d161b"}, diff --git a/pyproject.toml b/pyproject.toml index a11515bd..85fccd9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ typer = "^0.3.2" rich = "^10.6.0" pydantic = "^1.8.2" GitPython = "^3.1.24" +tomli = "^0.2.6" [tool.poetry.dev-dependencies] From e4661751301a848cef62254406a4f7fbd9a4e63e Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Thu, 13 Jan 2022 15:18:19 -0300 Subject: [PATCH 04/34] move logic to finding common parent to function of its own --- databooks/common.py | 7 ++++++- databooks/conflicts.py | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/databooks/common.py b/databooks/common.py index d2b0547e..9bd49445 100644 --- a/databooks/common.py +++ b/databooks/common.py @@ -2,7 +2,7 @@ import json from itertools import chain from pathlib import Path -from typing import List +from typing import Iterable, List from databooks import JupyterNotebook @@ -32,3 +32,8 @@ def expand_paths(paths: List[Path], ignore: List[str]) -> List[Path]: for p in paths if not any(p.match(i) for i in ignore) and p.exists() and p.suffix == ".ipynb" ] + + +def find_common_parent(paths: Iterable[Path]) -> Path: + """Find common parent amongst several file paths.""" + return max(set.intersection(*[set(p.parents) for p in paths])) diff --git a/databooks/conflicts.py b/databooks/conflicts.py index f0f6ebf0..e617c3df 100644 --- a/databooks/conflicts.py +++ b/databooks/conflicts.py @@ -7,7 +7,7 @@ from git import Repo -from databooks.common import write_notebook +from databooks.common import find_common_parent, write_notebook from databooks.data_models.notebook import Cell, Cells, JupyterNotebook from databooks.git_utils import ConflictFile, get_conflict_blobs, get_repo from databooks.logging import get_logger, set_verbose @@ -29,7 +29,7 @@ def path2conflicts( raise ValueError( "Expected either notebook files, a directory or glob expression." ) - common_parent = max(set.intersection(*[set(p.parents) for p in nb_paths])) + common_parent = find_common_parent(nb_paths) repo = get_repo(common_parent) if repo is None else repo return [ file From d862dd84354b73f3f8f0ffccac25160ae07df6e6 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Thu, 13 Jan 2022 17:06:29 -0300 Subject: [PATCH 05/34] make sure to use absolute paths (resolve) when computing the common parent --- databooks/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databooks/common.py b/databooks/common.py index 9bd49445..bbc54951 100644 --- a/databooks/common.py +++ b/databooks/common.py @@ -36,4 +36,4 @@ def expand_paths(paths: List[Path], ignore: List[str]) -> List[Path]: def find_common_parent(paths: Iterable[Path]) -> Path: """Find common parent amongst several file paths.""" - return max(set.intersection(*[set(p.parents) for p in paths])) + return max(set.intersection(*[set(p.resolve().parents) for p in paths])) From 8579f475451cac3862cf485615e503bbf8ac7d9a Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Thu, 13 Jan 2022 17:10:26 -0300 Subject: [PATCH 06/34] add function to find files along dirpaths (will be needed to look for config files) --- databooks/config.py | 41 +++++++++++++++++++++++++++++++++++++++++ tests/test_config.py | 19 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 databooks/config.py create mode 100644 tests/test_config.py diff --git a/databooks/config.py b/databooks/config.py new file mode 100644 index 00000000..b08b113e --- /dev/null +++ b/databooks/config.py @@ -0,0 +1,41 @@ +"""Configuration functions, and settings objects.""" +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Union + +import tomli +from pydantic import BaseSettings +from pydantic.env_settings import SettingsSourceCallable + +from databooks.common import find_common_parent +from databooks.git_utils import get_repo +from databooks.logging import get_logger + +TOML_CONFIG_FILE = "pyproject.toml" +INI_CONFIG_FILE = "settings.ini" + +ConfigFields = Dict[str, Any] + +logger = get_logger(__file__) + + +def _find_file(filename: str, start: Path, finish: Path) -> Union[Path, None]: + """ + Recursively find file along directory path, from the end (child) directory to start. + + :param filename: File name to locate + :param start: Start (parent) directory + :param finish: Finish (child) directory + :return: File path + """ + if not start.is_dir() or not finish.is_dir(): + raise ValueError("Parameters `start` and `finish` must be directory paths.") + + if finish.samefile(start): + logger.debug(f"No file found between {start} and {finish}.") + return None + elif (finish / filename).is_file(): + return finish / filename + else: + return _find_file(filename=filename, start=start, finish=finish.parent) + + diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..02bc9bb5 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,19 @@ +from pathlib import Path + +from py._path.local import LocalPath + +from databooks.config import _find_file + + +def test__find_file(tmpdir: LocalPath) -> None: + """Find file based on name, and search path.""" + filename = "SAMPLE_FILE.ext" + + start_dir = Path(tmpdir) + end_dir = start_dir / "to" / "some" / "dir" + end_dir.mkdir(parents=True) + (start_dir / "to" / filename).touch() + + filepath = _find_file(filename=filename, start=start_dir, finish=end_dir) + assert filepath == start_dir / "to" / filename + assert filepath.is_file() From c60a6e53b709770dbc31da1a02a056269dca2e7e Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Fri, 14 Jan 2022 10:58:44 -0300 Subject: [PATCH 07/34] add log in case no notebooks are found (signal to user that nothing was done) --- databooks/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/databooks/cli.py b/databooks/cli.py index cf3347d0..8e8e9d84 100644 --- a/databooks/cli.py +++ b/databooks/cli.py @@ -77,6 +77,10 @@ def meta( "Expected either notebook files, a directory or glob expression." ) nb_paths = expand_paths(paths=paths, ignore=ignore) + if not nb_paths: + logger.info("No notebooks found. Nothing to do.") + raise Exit() + if not bool(prefix + suffix) and not check: if not overwrite: raise BadParameter( From 1465f973ee4ca6ab0d29ce81e2e2cf9f1fa73f84 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Fri, 14 Jan 2022 23:34:30 -0300 Subject: [PATCH 08/34] use env var to determine verbosity of the logger (easier to debug) - revert `get_logger` to previous code --- databooks/logging.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/databooks/logging.py b/databooks/logging.py index 9fbe2e52..729df3f7 100644 --- a/databooks/logging.py +++ b/databooks/logging.py @@ -1,12 +1,14 @@ """Logging helper functions.""" import logging +import os from rich.logging import RichHandler -def get_logger(name: str, level: str = "INFO") -> logging.Logger: +def get_logger(name: str) -> logging.Logger: """Get logger with rich configuration.""" + level = os.getenv("LOG_LEVEL", logging.INFO) logging.basicConfig( level=level, format="%(message)s", From 5f507a650fdcba6cd97ed9354b9786390b6c6432 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Fri, 14 Jan 2022 23:35:06 -0300 Subject: [PATCH 09/34] reduce verbosity level when finding repo (avoid polluting normal CLI use) --- databooks/git_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databooks/git_utils.py b/databooks/git_utils.py index e2ba9fc3..98a55eab 100644 --- a/databooks/git_utils.py +++ b/databooks/git_utils.py @@ -32,7 +32,7 @@ class ConflictFile: def get_repo(path: Path = Path.cwd()) -> Repo: """Find git repo in current or parent directories.""" repo = Repo(path=path, search_parent_directories=True) - logger.info(f"Repo found at: {repo.working_dir}") + logger.debug(f"Repo found at: {repo.working_dir}") return repo From 41f53cc050e025b1e7247f298ecd0d811fcfb278 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 16:51:31 -0300 Subject: [PATCH 10/34] bugfix find_file (first check if file is present in the anchor before exiting) and add function to find config --- databooks/config.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/databooks/config.py b/databooks/config.py index b08b113e..ac533595 100644 --- a/databooks/config.py +++ b/databooks/config.py @@ -1,10 +1,6 @@ """Configuration functions, and settings objects.""" from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Union - -import tomli -from pydantic import BaseSettings -from pydantic.env_settings import SettingsSourceCallable +from typing import Any, Dict, List, Optional from databooks.common import find_common_parent from databooks.git_utils import get_repo @@ -18,7 +14,7 @@ logger = get_logger(__file__) -def _find_file(filename: str, start: Path, finish: Path) -> Union[Path, None]: +def _find_file(filename: str, start: Path, finish: Path) -> Optional[Path]: """ Recursively find file along directory path, from the end (child) directory to start. @@ -28,14 +24,31 @@ def _find_file(filename: str, start: Path, finish: Path) -> Union[Path, None]: :return: File path """ if not start.is_dir() or not finish.is_dir(): - raise ValueError("Parameters `start` and `finish` must be directory paths.") + raise ValueError("Parameters `start` and `finish` must be directories.") - if finish.samefile(start): - logger.debug(f"No file found between {start} and {finish}.") + if start.resolve() not in [finish, *finish.resolve().parents]: + logger.debug( + f"Parameter `start` is not a parent directory of `finish` (for {start} and" + f" {finish}). Cannot find {filename}." + ) return None - elif (finish / filename).is_file(): + + if (finish / filename).is_file(): return finish / filename + elif finish.samefile(start): + logger.debug(f"{filename} not found between {start} and {finish}.") + return None else: return _find_file(filename=filename, start=start, finish=finish.parent) +def find_config(target_paths: List[Path], config_filename: str) -> Optional[Path]: + """Find configuration file from CLI target paths.""" + common_parent = find_common_parent(paths=target_paths) + repo_dir = get_repo().working_dir + + return _find_file( + filename=config_filename, + start=Path(repo_dir) if repo_dir is not None else Path(common_parent.anchor), + finish=common_parent, + ) From 8f2bd5b345881f8fe3fe34dc029885a74e38f259 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 17:27:23 -0300 Subject: [PATCH 11/34] move find_file to common utils, as it can be used for both finding config and repo -> current implementation of `get_repo` would recursively look for git repo until program interruption --- databooks/common.py | 37 ++++++++++++++++++++++++++++++++++++- databooks/config.py | 36 ++++-------------------------------- databooks/git_utils.py | 6 +++++- tests/test_common.py | 31 +++++++++++++++++++++++++++++++ tests/test_config.py | 19 ------------------- 5 files changed, 76 insertions(+), 53 deletions(-) create mode 100644 tests/test_common.py delete mode 100644 tests/test_config.py diff --git a/databooks/common.py b/databooks/common.py index bbc54951..18d91241 100644 --- a/databooks/common.py +++ b/databooks/common.py @@ -2,9 +2,12 @@ import json from itertools import chain from pathlib import Path -from typing import Iterable, List +from typing import Iterable, List, Optional from databooks import JupyterNotebook +from databooks.logging import get_logger + +logger = get_logger(__file__) def write_notebook(nb: JupyterNotebook, path: Path) -> None: @@ -37,3 +40,35 @@ def expand_paths(paths: List[Path], ignore: List[str]) -> List[Path]: def find_common_parent(paths: Iterable[Path]) -> Path: """Find common parent amongst several file paths.""" return max(set.intersection(*[set(p.resolve().parents) for p in paths])) + + +def find_obj( + obj_name: str, start: Path, finish: Path, is_dir: bool = False +) -> Optional[Path]: + """ + Recursively find file along directory path, from the end (child) directory to start. + + :param obj_name: File name to locate + :param start: Start (parent) directory + :param finish: Finish (child) directory + :param is_dir: Whether object is a directory or a file + :return: File path + """ + if not start.is_dir() or not finish.is_dir(): + raise ValueError("Parameters `start` and `finish` must be directories.") + + if start.resolve() not in [finish, *finish.resolve().parents]: + logger.debug( + f"Parameter `start` is not a parent directory of `finish` (for {start} and" + f" {finish}). Cannot find {obj_name}." + ) + return None + + is_obj = (finish / obj_name).is_dir() if is_dir else (finish / obj_name).is_file() + if is_obj: + return finish / obj_name + elif finish.samefile(start): + logger.debug(f"{obj_name} not found between {start} and {finish}.") + return None + else: + return find_obj(obj_name=obj_name, start=start, finish=finish.parent) diff --git a/databooks/config.py b/databooks/config.py index ac533595..d8378302 100644 --- a/databooks/config.py +++ b/databooks/config.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from databooks.common import find_common_parent +from databooks.common import find_common_parent, find_obj from databooks.git_utils import get_repo from databooks.logging import get_logger @@ -14,41 +14,13 @@ logger = get_logger(__file__) -def _find_file(filename: str, start: Path, finish: Path) -> Optional[Path]: - """ - Recursively find file along directory path, from the end (child) directory to start. - - :param filename: File name to locate - :param start: Start (parent) directory - :param finish: Finish (child) directory - :return: File path - """ - if not start.is_dir() or not finish.is_dir(): - raise ValueError("Parameters `start` and `finish` must be directories.") - - if start.resolve() not in [finish, *finish.resolve().parents]: - logger.debug( - f"Parameter `start` is not a parent directory of `finish` (for {start} and" - f" {finish}). Cannot find {filename}." - ) - return None - - if (finish / filename).is_file(): - return finish / filename - elif finish.samefile(start): - logger.debug(f"{filename} not found between {start} and {finish}.") - return None - else: - return _find_file(filename=filename, start=start, finish=finish.parent) - - -def find_config(target_paths: List[Path], config_filename: str) -> Optional[Path]: +def get_config(target_paths: List[Path], config_filename: str) -> Optional[Path]: """Find configuration file from CLI target paths.""" common_parent = find_common_parent(paths=target_paths) repo_dir = get_repo().working_dir - return _find_file( - filename=config_filename, + return find_obj( + obj_name=config_filename, start=Path(repo_dir) if repo_dir is not None else Path(common_parent.anchor), finish=common_parent, ) diff --git a/databooks/git_utils.py b/databooks/git_utils.py index 98a55eab..19ec4901 100644 --- a/databooks/git_utils.py +++ b/databooks/git_utils.py @@ -5,6 +5,7 @@ from git import Blob, Git, Repo # type: ignore +from databooks.common import find_obj from databooks.logging import get_logger logger = get_logger(name=__file__) @@ -31,7 +32,10 @@ class ConflictFile: def get_repo(path: Path = Path.cwd()) -> Repo: """Find git repo in current or parent directories.""" - repo = Repo(path=path, search_parent_directories=True) + repo_dir = find_obj( + obj_name=".git", start=Path(path.anchor), finish=path, is_dir=True + ) + repo = Repo(path=repo_dir) logger.debug(f"Repo found at: {repo.working_dir}") return repo diff --git a/tests/test_common.py b/tests/test_common.py new file mode 100644 index 00000000..5d5e215a --- /dev/null +++ b/tests/test_common.py @@ -0,0 +1,31 @@ +from pathlib import Path + +from py._path.local import LocalPath + +from databooks.common import find_file + + +def test_find_file(tmpdir: LocalPath) -> None: + """Find file based on name, and search path.""" + filename = "SAMPLE_FILE.ext" + + start_dir = Path(tmpdir) + end_dir = start_dir / "to" / "some" / "dir" + end_dir.mkdir(parents=True) + (start_dir / "to" / filename).touch() + + filepath = find_file(filename=filename, start=start_dir, finish=end_dir) + assert filepath == start_dir / "to" / filename + assert filepath.is_file() + + +def test_find_file__missing(tmpdir: LocalPath) -> None: + """Return `None` when looking for file along path.""" + filename = "SAMPLE_FILE.ext" + + start_dir = Path(tmpdir) + end_dir = start_dir / "to" / "some" / "dir" + end_dir.mkdir(parents=True) + + filepath = find_file(filename=filename, start=start_dir, finish=end_dir) + assert filepath is None diff --git a/tests/test_config.py b/tests/test_config.py deleted file mode 100644 index 02bc9bb5..00000000 --- a/tests/test_config.py +++ /dev/null @@ -1,19 +0,0 @@ -from pathlib import Path - -from py._path.local import LocalPath - -from databooks.config import _find_file - - -def test__find_file(tmpdir: LocalPath) -> None: - """Find file based on name, and search path.""" - filename = "SAMPLE_FILE.ext" - - start_dir = Path(tmpdir) - end_dir = start_dir / "to" / "some" / "dir" - end_dir.mkdir(parents=True) - (start_dir / "to" / filename).touch() - - filepath = _find_file(filename=filename, start=start_dir, finish=end_dir) - assert filepath == start_dir / "to" / filename - assert filepath.is_file() From d41aa9613e93897d0ecf7e77b10ea5f2aa29abdc Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 17:36:30 -0300 Subject: [PATCH 12/34] fix tests - point to directory not a file --- tests/test_git_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_git_utils.py b/tests/test_git_utils.py index 1e2f6a76..9494d3dc 100644 --- a/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -50,8 +50,8 @@ def init_repo_conflicts( def test_get_repo() -> None: """Find git repository.""" - filepath = Path(__file__) - repo = get_repo(filepath) + curr_dir = Path(__file__).parent + repo = get_repo(curr_dir) assert repo.working_dir is not None assert isinstance(repo, Repo) assert Path(repo.working_dir).stem == "databooks" From 70f922c1e4f2ebdd6d8520124540d56ade86b1a0 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 17:37:43 -0300 Subject: [PATCH 13/34] refactor tests for `find_file` to reflect new name in `common.py` --- tests/test_common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_common.py b/tests/test_common.py index 5d5e215a..ba7564d5 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -2,10 +2,10 @@ from py._path.local import LocalPath -from databooks.common import find_file +from databooks.common import find_obj -def test_find_file(tmpdir: LocalPath) -> None: +def test_find_obj(tmpdir: LocalPath) -> None: """Find file based on name, and search path.""" filename = "SAMPLE_FILE.ext" @@ -14,12 +14,12 @@ def test_find_file(tmpdir: LocalPath) -> None: end_dir.mkdir(parents=True) (start_dir / "to" / filename).touch() - filepath = find_file(filename=filename, start=start_dir, finish=end_dir) + filepath = find_obj(obj_name=filename, start=start_dir, finish=end_dir) assert filepath == start_dir / "to" / filename assert filepath.is_file() -def test_find_file__missing(tmpdir: LocalPath) -> None: +def test_find_obj__missing(tmpdir: LocalPath) -> None: """Return `None` when looking for file along path.""" filename = "SAMPLE_FILE.ext" @@ -27,5 +27,5 @@ def test_find_file__missing(tmpdir: LocalPath) -> None: end_dir = start_dir / "to" / "some" / "dir" end_dir.mkdir(parents=True) - filepath = find_file(filename=filename, start=start_dir, finish=end_dir) + filepath = find_obj(obj_name=filename, start=start_dir, finish=end_dir) assert filepath is None From cc175c52d2b610a571f234a51a006deedd6286b2 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 17:38:29 -0300 Subject: [PATCH 14/34] include sensible defaults for `expand_paths (make `ignore` param optional) --- databooks/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databooks/common.py b/databooks/common.py index 18d91241..da695e3f 100644 --- a/databooks/common.py +++ b/databooks/common.py @@ -16,7 +16,7 @@ def write_notebook(nb: JupyterNotebook, path: Path) -> None: json.dump(nb.dict(), fp=f, indent=2) -def expand_paths(paths: List[Path], ignore: List[str]) -> List[Path]: +def expand_paths(paths: List[Path], ignore: List[str] = ["!*"]) -> List[Path]: """ Get paths of existing file from list of directory or file paths. From 803dbae8deabc3ccc192a11357d4b55487c95b42 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 17:41:45 -0300 Subject: [PATCH 15/34] add leading `_` to `version_callback` to indicate "private methods" --- databooks/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/databooks/cli.py b/databooks/cli.py index 8e8e9d84..26324744 100644 --- a/databooks/cli.py +++ b/databooks/cli.py @@ -25,9 +25,9 @@ app = Typer() -def version_callback(value: bool) -> None: +def _version_callback(show_version: bool) -> None: """Return application version.""" - if value: + if show_version: echo("databooks version: " + _DISTRIBUTION_METADATA["Version"]) raise Exit() @@ -35,7 +35,7 @@ def version_callback(value: bool) -> None: @app.callback() def callback( # noqa: D103 version: Optional[bool] = Option( - None, "--version", callback=version_callback, is_eager=True + None, "--version", callback=_version_callback, is_eager=True ) ) -> None: ... From 7660d9f76edc6d61a876deae0972123c41ad5799 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 19:04:31 -0300 Subject: [PATCH 16/34] add config callback to inject configuration into cli command defaults - also reimplement `help` to execute eagerly otherwise we get an error with missing eager required parameter `paths` --- databooks/cli.py | 56 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/databooks/cli.py b/databooks/cli.py index 26324744..2333b17f 100644 --- a/databooks/cli.py +++ b/databooks/cli.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import List, Optional +import tomli from rich.progress import ( BarColumn, Progress, @@ -11,9 +12,10 @@ TextColumn, TimeElapsedColumn, ) -from typer import Argument, BadParameter, Exit, Option, Typer, echo +from typer import Argument, BadParameter, Context, Exit, Option, Typer, echo from databooks.common import expand_paths +from databooks.config import TOML_CONFIG_FILE, get_config from databooks.conflicts import conflicts2nbs, path2conflicts from databooks.logging import get_logger from databooks.metadata import clear_all @@ -32,6 +34,41 @@ def _version_callback(show_version: bool) -> None: raise Exit() +def _help_callback(ctx: Context, show_help: Optional[bool]) -> None: + """Reimplement `help` command to execute eagerly.""" + if show_help: + echo(ctx.command.get_help(ctx)) + raise Exit() + + +def _config_callback(ctx: Context, config_path: Optional[Path]) -> Optional[Path]: + """Get config file and inject values into context to override default args.""" + config_path = ( + get_config( + target_paths=expand_paths(paths=[Path(p) for p in ctx.params["paths"]]), + config_filename=TOML_CONFIG_FILE, + ) + if config_path is None and "paths" in ctx.params + else config_path + ) + logger.debug(f"Loading config file from: {config_path}") + + ctx.default_map = ctx.default_map or {} # initialize defaults + + if config_path is not None: # config may not be specified + with config_path.open("r") as f: + conf = ( + tomli.load(f) + .get("tool", {}) + .get("databooks", {}) + .get(ctx.command.name, {}) + ) + # Merge configuration + ctx.default_map.update({k.replace("-", "_"): v for k, v in conf.items()}) + + return config_path + + @app.callback() def callback( # noqa: D103 version: Optional[bool] = Option( @@ -45,9 +82,9 @@ def callback( # noqa: D103 callback.__doc__ = _DISTRIBUTION_METADATA["Summary"] -@app.command() +@app.command(add_help_option=False) def meta( - paths: List[Path] = Argument(..., help="Path(s) of notebook files"), + paths: List[Path] = Argument(..., is_eager=True, help="Path(s) of notebook files"), ignore: List[str] = Option(["!*"], help="Glob expression(s) of files to ignore"), prefix: str = Option("", help="Prefix to add to filepath when writing files"), suffix: str = Option("", help="Suffix to add to filepath when writing files"), @@ -70,6 +107,19 @@ def meta( verbose: bool = Option( False, "--verbose", "-v", help="Log processed files in console" ), + config: Optional[Path] = Option( + None, + "--config", + "-c", + is_eager=True, + callback=_config_callback, + resolve_path=True, + exists=True, + help="Get CLI options from configuration file", + ), + help: Optional[bool] = Option( + None, is_eager=True, callback=_help_callback, help="Show this message and exit" + ), ) -> None: """Clear both notebook and cell metadata.""" if any(path.suffix not in ("", ".ipynb") for path in paths): From 79befb6be952b476b815c694c25028aface305b6 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 19:09:25 -0300 Subject: [PATCH 17/34] add config tests - reading and overriding with cli args --- tests/test_cli.py | 41 ++++++++++++++++++++++++++++++ tests/test_data_models/__init__.py | 0 2 files changed, 41 insertions(+) create mode 100644 tests/test_data_models/__init__.py diff --git a/tests/test_cli.py b/tests/test_cli.py index 3bdf12f3..0eeb12e5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ import logging from importlib.metadata import version from pathlib import Path +from textwrap import dedent from _pytest.logging import LogCaptureFixture from py._path.local import LocalPath @@ -18,6 +19,18 @@ from tests.test_data_models.test_notebook import TestJupyterNotebook # type: ignore from tests.test_git_utils import init_repo_conflicts +SAMPLE_CONFIG = dedent( + """ + [tool.databooks.meta] + rm-outs=true + rm_exec=false + overwrite=true + + [tool.databooks.fix] + metadata-head=false + """ +) + runner = CliRunner() @@ -75,6 +88,34 @@ def test_meta__check(tmpdir: LocalPath, caplog: LogCaptureFixture) -> None: assert logs[0].message == "Found unwanted metadata in 1 out of 1 files" +def test_meta__config(tmpdir: LocalPath) -> None: + """Retrieve and parse configuration.""" + read_path = tmpdir.mkdir("notebooks") / "test_meta_nb.ipynb" # type: ignore + write_notebook(nb=TestJupyterNotebook().jupyter_notebook, path=read_path) + + config_path = tmpdir / "pyproject.toml" # type: ignore + config_path.write_text(SAMPLE_CONFIG, encoding="utf-8") + + nb_read = JupyterNotebook.parse_file(path=read_path) + # Take arguments from config file + result = runner.invoke(app, ["meta", str(read_path), "--config", str(config_path)]) + nb_write = JupyterNotebook.parse_file(path=read_path) + + assert result.exit_code == 0 + assert nb_read != nb_write, "Notebook was not overwritten" + assert all(c.outputs == [] for c in nb_write.cells) + assert all(c.execution_count is not None for c in nb_write.cells) + + # Override config file arguments + result = runner.invoke( + app, ["meta", str(read_path), "--rm-exec", "--config", str(config_path)] + ) + nb_write = JupyterNotebook.parse_file(path=read_path) + + assert result.exit_code == 0 + assert all(c.execution_count is None for c in nb_write.cells) + + def test_fix(tmpdir: LocalPath) -> None: """Fix notebook conflicts.""" # Setup diff --git a/tests/test_data_models/__init__.py b/tests/test_data_models/__init__.py new file mode 100644 index 00000000..e69de29b From c79347b3b28f45883f9029d1d19414fcdd83db07 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 19:34:58 -0300 Subject: [PATCH 18/34] bugfix - pass all arguments in recursion when looking for objects --- databooks/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/databooks/common.py b/databooks/common.py index da695e3f..5c23b542 100644 --- a/databooks/common.py +++ b/databooks/common.py @@ -71,4 +71,6 @@ def find_obj( logger.debug(f"{obj_name} not found between {start} and {finish}.") return None else: - return find_obj(obj_name=obj_name, start=start, finish=finish.parent) + return find_obj( + obj_name=obj_name, start=start, finish=finish.parent, is_dir=is_dir + ) From 96e56669db444a6a15d2c351ddb3cefe4b80e513 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 19:37:27 -0300 Subject: [PATCH 19/34] add config options for `fix` command - add `config`, make `paths` an eager argument and reimplement `help` eagerly --- databooks/cli.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/databooks/cli.py b/databooks/cli.py index 2333b17f..1b239c9c 100644 --- a/databooks/cli.py +++ b/databooks/cli.py @@ -179,9 +179,11 @@ def meta( ) -@app.command() +@app.command(add_help_option=False) def fix( - paths: List[Path] = Argument(..., help="Path(s) of notebook files with conflicts"), + paths: List[Path] = Argument( + ..., is_eager=True, help="Path(s) of notebook files with conflicts" + ), ignore: List[str] = Option(["!*"], help="Glob expression(s) of files to ignore"), metadata_head: bool = Option( True, help="Whether or not to keep the metadata from the head/current notebook" @@ -204,6 +206,19 @@ def fix( help="Interactively resolve the conflicts (not implemented)", ), verbose: bool = Option(False, help="Log processed files in console"), + config: Optional[Path] = Option( + None, + "--config", + "-c", + is_eager=True, + callback=_config_callback, + resolve_path=True, + exists=True, + help="Get CLI options from configuration file", + ), + help: Optional[bool] = Option( + None, is_eager=True, callback=_help_callback, help="Show this message and exit" + ), ) -> None: """ Fix git conflicts for notebooks. From 4a2fc17d11fd0ad3ba1aa29c2ecc3463dfa4414a Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 20:56:42 -0300 Subject: [PATCH 20/34] remove unnecessary parameter when testing `fix` --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0eeb12e5..bad66f03 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -152,7 +152,7 @@ def test_fix(tmpdir: LocalPath) -> None: id_other = conflict_files[0].last_log # Run CLI and check conflict resolution - result = runner.invoke(app, ["fix", str(tmpdir), "--cell-fields-ignore", "id"]) + result = runner.invoke(app, ["fix", str(tmpdir)]) fixed_notebook = JupyterNotebook.parse_file(path=tmpdir / nb_path) assert len(conflict_files) == 1 From 44bd9b73025b847f87411bc64cd872efab711b1a Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 21:20:00 -0300 Subject: [PATCH 21/34] change notebook metadata for comparison to demonstrate metadata selection when fixing conflicts --- tests/test_cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index bad66f03..123772fa 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,5 @@ import logging +from copy import deepcopy from importlib.metadata import version from pathlib import Path from textwrap import dedent @@ -137,6 +138,8 @@ def test_fix(tmpdir: LocalPath) -> None: source="extra", ) notebook_2.cells = notebook_2.cells + [extra_cell] + notebook_2.nbformat += 1 + notebook_2.nbformat_minor += 1 git_repo = init_repo_conflicts( tmpdir=tmpdir, From 8198953f59a249039364681563d64be8a6b9bccb Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 21:20:32 -0300 Subject: [PATCH 22/34] simplify logic for comparing metatada obtained with expected --- tests/test_cli.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 123772fa..8e19dfea 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -161,11 +161,9 @@ def test_fix(tmpdir: LocalPath) -> None: assert len(conflict_files) == 1 assert result.exit_code == 0 - # Add `tags` since we use `databooks.data_models.base.resolve` with default - # `ignore_none = True` - assert fixed_notebook.metadata == NotebookMetadata( - **notebook_1.metadata.dict(), **{"tags": []} - ) + expected_metadata = deepcopy(notebook_2.metadata.dict()) + expected_metadata.update(notebook_1.metadata.dict()) + assert fixed_notebook.metadata == NotebookMetadata(**expected_metadata) assert fixed_notebook.nbformat == notebook_1.nbformat assert fixed_notebook.nbformat_minor == notebook_1.nbformat_minor assert fixed_notebook.cells == notebook_1.cells + [ From f8cb47680cf0bd971180df12e392a8741a83b407 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 21:23:01 -0300 Subject: [PATCH 23/34] add test for `fix` using config --- tests/test_cli.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 8e19dfea..3b377908 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -184,3 +184,74 @@ def test_fix(tmpdir: LocalPath) -> None: cell_type="markdown", ), ] + +def test_fix__config(tmpdir: LocalPath) -> None: + """Fix notebook conflicts.""" + # Setup + config_path = tmpdir / "pyproject.toml" # type: ignore + config_path.write_text(SAMPLE_CONFIG, encoding="utf-8") + + nb_path = Path("test_conflicts_nb.ipynb") + notebook_1 = TestJupyterNotebook().jupyter_notebook + notebook_2 = TestJupyterNotebook().jupyter_notebook + + notebook_1.metadata = NotebookMetadata( + kernelspec=dict( + display_name="different_kernel_display_name", name="kernel_name" + ), + field_to_remove=["Field to remove"], + another_field_to_remove="another field", + ) + + extra_cell = Cell( + cell_type="raw", + metadata=CellMetadata(random_meta=["meta"]), + source="extra", + ) + notebook_2.cells = notebook_2.cells + [extra_cell] + notebook_2.nbformat += 1 + notebook_2.nbformat_minor += 1 + + git_repo = init_repo_conflicts( + tmpdir=tmpdir, + filename=nb_path, + contents_main=notebook_1.json(), + contents_other=notebook_2.json(), + commit_message_main="Notebook from main", + commit_message_other="Notebook from other", + ) + + conflict_files = get_conflict_blobs(repo=git_repo) + id_main = conflict_files[0].first_log + id_other = conflict_files[0].last_log + + # Run CLI and check conflict resolution + result = runner.invoke(app, ["fix", str(tmpdir), "--config", str(config_path)]) + fixed_notebook = JupyterNotebook.parse_file(path=tmpdir / nb_path) + + assert len(conflict_files) == 1 + assert result.exit_code == 0 + + expected_metadata = deepcopy(notebook_1.metadata.dict()) + expected_metadata.update(notebook_2.metadata.dict()) + assert fixed_notebook.metadata == NotebookMetadata(**expected_metadata) + assert fixed_notebook.nbformat == notebook_2.nbformat + assert fixed_notebook.nbformat_minor == notebook_2.nbformat_minor + assert fixed_notebook.cells == notebook_1.cells + [ + Cell( + metadata=CellMetadata(git_hash=id_main), + source=[f"`<<<<<<< {id_main}`"], + cell_type="markdown", + ), + Cell( + source=["`=======`"], + cell_type="markdown", + metadata=CellMetadata(), + ), + extra_cell, + Cell( + metadata=CellMetadata(git_hash=id_other), + source=[f"`>>>>>>> {id_other}`"], + cell_type="markdown", + ), + ] \ No newline at end of file From 64665c2883600702d7845d8f656c45ad33ea68b2 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 22:33:10 -0300 Subject: [PATCH 24/34] add docs about config file - what it is and how to use it --- docs/usage/configuration.md | 66 +++++++++++++++++++++++++++++++++++++ mkdocs.yml | 4 +++ 2 files changed, 70 insertions(+) create mode 100644 docs/usage/configuration.md diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md new file mode 100644 index 00000000..5bed02ef --- /dev/null +++ b/docs/usage/configuration.md @@ -0,0 +1,66 @@ +# Configuration + +Instead of passing the same parameters every time when running a command, it is also +possible to set up a configuration that will be read and override the defaults. The order +of priority (from higher priority to lower) + +1. User input arguments in the CLI +2. Configuration file +3. Defaults + +So it's still possible to override the configuration file via CLI parameters (as expected). + +## What can I configure? + +All CLI parameters are actually configurable, so you can pass specify anything that is +also available to you via the UI, with one exception: the required `PATHS` argument. +This is because the `PATHS` argument is also used for finding your configuration (see +[how can I use it](#how-can-i-use-it) for more information). + +!!! info + Remember that flags are parsed as boolean values. So you can specify `--verbose` on + the configuration as `verbose=true`. + +## How does it look like? + +The configuration file is a `pyproject.toml` file that you can place at the root of your +project. There, you can specify values for either command under the `[tool.databooks.]`. + +So if, for example, the desired behavior is + +- `databooks meta` + - Remove outputs + - Don't remove execution count + - Always overwrite files +- `databooks fix` + - Keep notebook metadata from `base` (not `head`) + +The `pyproject.toml` file would look like + +```toml +[tool.databooks.meta] +rm-outs = true +rm_exec = false +overwrite = true + +[tool.databooks.fix] +metadata-head = false +``` + +## How can I use it? + +There are 2 ways to specify the configuration file: explicitly and implicitly. You can +explicitly specify the `pyproject.toml` via the `--config` parameter. If none is specified, +then `databooks` will look for a `pyproject.toml` in your project. + +`databooks` will look for the configuration file by first finding the common directory +between all the target paths and from there recursively go to the parent directories +until either finding the configuration file or the root of the git repo. That way, you can +have multiple configuration files and depending on where your notebooks are located the +correct values will be used (think [monorepo](https://en.wikipedia.org/wiki/Monorepo)). + +!!! tip + `databooks` has a `verbose` concept that will print more information to the terminal + if desired. For debugging purposes one can still increase the verbosity by setting + and environment variable `LOG_LEVEL` to `DEBUG`. That way, one can get information, + among many other things, of the configuration file used. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 0782b4c8..c7312f3d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,6 +8,9 @@ repo_name: "datarootsio/databooks" markdown_extensions: - def_list + - admonition + - pymdownx.details + - pymdownx.superfences - pymdownx.snippets: check_paths: true - pymdownx.tasklist: @@ -53,6 +56,7 @@ nav: - CLI: usage/cli.md - CI/CD: usage/cicd.md - Pre-commit hooks: usage/precommit.md + - Configuration: usage/configuration.md - CLI: CLI.md - API: - Overview: API.md From e801cfebcf982b9040fac5f7da457da6a2ae2260 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sat, 15 Jan 2022 22:36:47 -0300 Subject: [PATCH 25/34] lint tests --- tests/test_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 3b377908..5e0e3e51 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -185,6 +185,7 @@ def test_fix(tmpdir: LocalPath) -> None: ), ] + def test_fix__config(tmpdir: LocalPath) -> None: """Fix notebook conflicts.""" # Setup @@ -254,4 +255,4 @@ def test_fix__config(tmpdir: LocalPath) -> None: source=[f"`>>>>>>> {id_other}`"], cell_type="markdown", ), - ] \ No newline at end of file + ] From 13e56ee03e7b292a0b6b4c088a08dab269b4448d Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sun, 16 Jan 2022 11:44:00 -0300 Subject: [PATCH 26/34] update README required packages --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0fb44194..045adaae 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ The key features include: - [Rich](https://rich.readthedocs.io/en/latest/) - [Pydantic](https://pydantic-docs.helpmanual.io/) - [GitPython](https://gitpython.readthedocs.io/en/stable/tutorial.html) +- [Tomli](https://github.com/hukkin/tomli) ## Installation From f9757fdcd8ac5b9009093a8d5f9d9ef326d89bf0 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sun, 16 Jan 2022 14:44:22 -0300 Subject: [PATCH 27/34] update CLI docs --- docs/CLI.md | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/CLI.md b/docs/CLI.md index 1acb809e..1c6b65f4 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -5,9 +5,7 @@ hide: # `databooks` -Databooks - set of helpers to ease collaboration of data scientists - using Jupyter Notebooks. Easily resolve git conflicts and remove metadata to reduce - the number of conflicts. +A CLI tool that minimises friction when versioning and collaborating on Jupyter notebooks. **Usage**: @@ -24,13 +22,13 @@ $ databooks [OPTIONS] COMMAND [ARGS]... **Commands**: -* `diff`: Show differences between notebooks (not... -* `fix`: Fix git conflicts for notebooks by getting... -* `meta`: Clear both notebook and cell metadata +* `diff`: Show differences between notebooks (not implemented) +* `fix`: Fix git conflicts for notebooks. +* `meta`: Clear both notebook and cell metadata. ## `databooks diff` -Show differences between notebooks (not implemented) +Show differences between notebooks (not implemented). **Usage**: @@ -44,9 +42,11 @@ $ databooks diff [OPTIONS] ## `databooks fix` -Fix git conflicts for notebooks by getting unmerged blobs from git index - comparing them and returning a valid notebook with the differences - - see [git docs](https://git-scm.com/docs/git-ls-files) +Fix git conflicts for notebooks. + +Perform by getting the unmerged blobs from git index, comparing them and returning + a valid notebook summarizing the differences - see + [git docs](https://git-scm.com/docs/git-ls-files). **Usage**: @@ -61,15 +61,17 @@ $ databooks fix [OPTIONS] PATHS... **Options**: * `--ignore TEXT`: Glob expression(s) of files to ignore [default: !*] -* `--metadata-first / --no-metadata-first`: Whether or not to keep the metadata from the first/current notebook [default: True] -* `--cells-first / --no-cells-first`: Whether to keep the cells from the first or last notebook. Omit to keep both +* `--metadata-head / --no-metadata-head`: Whether or not to keep the metadata from the head/current notebook [default: True] +* `--cells-head / --no-cells-head`: Whether to keep the cells from the head/base notebook. Omit to keep both +* `--cell-fields-ignore TEXT`: Cell fields to remove before comparing cells [default: id, execution_count] * `-i, --interactive`: Interactively resolve the conflicts (not implemented) [default: False] * `--verbose / --no-verbose`: Log processed files in console [default: False] -* `--help`: Show this message and exit. +* `-c, --config PATH`: Get CLI options from configuration file +* `--help / --no-help`: Show this message and exit ## `databooks meta` -Clear both notebook and cell metadata +Clear both notebook and cell metadata. **Usage**: @@ -90,7 +92,9 @@ $ databooks meta [OPTIONS] PATHS... * `--rm-exec / --no-rm-exec`: Whether to remove the cell execution counts [default: True] * `--nb-meta-keep TEXT`: Notebook metadata fields to keep [default: ] * `--cell-meta-keep TEXT`: Cells metadata fields to keep [default: ] +* `--cell-fields-keep TEXT`: Other (excluding `execution_counts` and `outputs`) cell fields to keep [default: ] * `-w, --overwrite`: Confirm overwrite of files [default: False] * `--check`: Don't write files but check whether there is unwanted metadata [default: False] * `-v, --verbose`: Log processed files in console [default: False] -* `--help`: Show this message and exit. +* `-c, --config PATH`: Get CLI options from configuration file +* `--help / --no-help`: Show this message and exit From cb0c6a2f7d37a64e8f65d7e082e2751dc59a6197 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sun, 16 Jan 2022 20:01:40 -0300 Subject: [PATCH 28/34] fix: add descriptive docstrings for tests --- tests/test_cli.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 5e0e3e51..338added 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -43,7 +43,7 @@ def test_version_callback() -> None: def test_meta(tmpdir: LocalPath) -> None: - """Fix notebook conflicts.""" + """Remove notebook metadata.""" read_path = tmpdir.mkdir("notebooks") / "test_meta_nb.ipynb" # type: ignore write_notebook(nb=TestJupyterNotebook().jupyter_notebook, path=read_path) @@ -72,7 +72,7 @@ def test_meta(tmpdir: LocalPath) -> None: def test_meta__check(tmpdir: LocalPath, caplog: LogCaptureFixture) -> None: - """Fix notebook conflicts.""" + """Report on existing notebook metadata (both when it is and isn't present).""" caplog.set_level(logging.INFO) read_path = tmpdir.mkdir("notebooks") / "test_meta_nb.ipynb" # type: ignore @@ -88,9 +88,18 @@ def test_meta__check(tmpdir: LocalPath, caplog: LogCaptureFixture) -> None: assert nb_read == nb_write assert logs[0].message == "Found unwanted metadata in 1 out of 1 files" + # Clean notebook and check again + runner.invoke(app, ["meta", str(read_path), "--overwrite"]) + result = runner.invoke(app, ["meta", str(read_path), "--check"]) + + logs = list(caplog.records) + assert result.exit_code == 0 + assert len(logs) == 4 + assert logs[-1].message == "No unwanted metadata!" + def test_meta__config(tmpdir: LocalPath) -> None: - """Retrieve and parse configuration.""" + """Check notebook metadata with configuration overriding defaults.""" read_path = tmpdir.mkdir("notebooks") / "test_meta_nb.ipynb" # type: ignore write_notebook(nb=TestJupyterNotebook().jupyter_notebook, path=read_path) @@ -187,7 +196,7 @@ def test_fix(tmpdir: LocalPath) -> None: def test_fix__config(tmpdir: LocalPath) -> None: - """Fix notebook conflicts.""" + """Fix notebook conflicts with configuration overriding defaults.""" # Setup config_path = tmpdir / "pyproject.toml" # type: ignore config_path.write_text(SAMPLE_CONFIG, encoding="utf-8") From b71e5129344a66c0e2ce78cda59712bb34536e9b Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sun, 16 Jan 2022 20:11:48 -0300 Subject: [PATCH 29/34] remove unnecessary ellipsis - docstrings are enough for classes/functions to be valid --- databooks/data_models/base.py | 5 +---- databooks/data_models/notebook.py | 4 ---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/databooks/data_models/base.py b/databooks/data_models/base.py index dc2cd7e6..714c4836 100644 --- a/databooks/data_models/base.py +++ b/databooks/data_models/base.py @@ -28,8 +28,7 @@ class DiffModel(Protocol, Iterable): is_diff: bool def resolve(self, *args: Any, **kwargs: Any) -> DatabooksBase: - """Return a valid base object.""" - ... + """Protocol method that returns a valid base object.""" class BaseCells(UserList, Generic[T]): @@ -40,8 +39,6 @@ def resolve(self, **kwargs: Any) -> list: """Return valid notebook cells from differences.""" raise NotImplementedError - ... - @overload def resolve( diff --git a/databooks/data_models/notebook.py b/databooks/data_models/notebook.py index a6ff7af4..8d12b2fd 100644 --- a/databooks/data_models/notebook.py +++ b/databooks/data_models/notebook.py @@ -31,14 +31,10 @@ class NotebookMetadata(DatabooksBase): """Notebook metadata. Empty by default but can accept extra fields.""" - ... - class CellMetadata(DatabooksBase): """Cell metadata. Empty by default but can accept extra fields.""" - ... - class Cell(DatabooksBase): """ From 5e5a34d9b5b8ba8bb1f9c81b0655d9fbd16d1d42 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sun, 16 Jan 2022 20:13:14 -0300 Subject: [PATCH 30/34] raise custom error instead of letting python raise default to add more context --- databooks/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/databooks/common.py b/databooks/common.py index 5c23b542..fd3818dc 100644 --- a/databooks/common.py +++ b/databooks/common.py @@ -39,6 +39,8 @@ def expand_paths(paths: List[Path], ignore: List[str] = ["!*"]) -> List[Path]: def find_common_parent(paths: Iterable[Path]) -> Path: """Find common parent amongst several file paths.""" + if not paths: + raise ValueError(f"Expected non-empty `paths`, got {paths}.") return max(set.intersection(*[set(p.resolve().parents) for p in paths])) From 73abcd326e833ad05d1ebe1c4eef73755fd27600 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sun, 16 Jan 2022 20:15:26 -0300 Subject: [PATCH 31/34] add `rglob` parameter when expanding paths to give more flexibility to function (instead of hardcoding `.ipynb`) and add debugging logs in case function returns an empty list (may introduce bugs) --- databooks/common.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/databooks/common.py b/databooks/common.py index fd3818dc..45379824 100644 --- a/databooks/common.py +++ b/databooks/common.py @@ -16,26 +16,35 @@ def write_notebook(nb: JupyterNotebook, path: Path) -> None: json.dump(nb.dict(), fp=f, indent=2) -def expand_paths(paths: List[Path], ignore: List[str] = ["!*"]) -> List[Path]: +def expand_paths( + paths: List[Path], *, ignore: List[str] = ["!*"], rglob: str = "*.ipynb" +) -> List[Path]: """ Get paths of existing file from list of directory or file paths. :param paths: Paths to consider (can be directories or files) :param ignore: Glob expressions of files to ignore - :return: List of existing paths for notebooks + :param rglob: Glob expression for expanding directory paths and filtering out + existing file paths (i.e.: to retrieve only notebooks) + :return: List of existing file paths """ - paths = list( + filepaths = list( chain.from_iterable( - list(path.rglob("*.ipynb")) if path.is_dir() else [path] for path in paths + list(path.rglob(rglob)) if path.is_dir() else [path] for path in paths ) ) - - return [ + valid_filepaths = [ p - for p in paths - if not any(p.match(i) for i in ignore) and p.exists() and p.suffix == ".ipynb" + for p in filepaths + if not any(p.match(i) for i in ignore) and p.is_file() and p.match(rglob) ] + if not valid_filepaths: + logger.debug( + f"There are no files in {paths} (ignoring {ignore}) that match `{rglob}`." + ) + return valid_filepaths + def find_common_parent(paths: Iterable[Path]) -> Path: """Find common parent amongst several file paths.""" From 4be53cecb3170a3ca49252ebee2c3b72a3816e95 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sun, 16 Jan 2022 20:17:41 -0300 Subject: [PATCH 32/34] bugfix - when expanding paths to look for config, we were filtering out all non-`ipynb` file - pass `rglob=*` to include all and only look for config if any expanded filepaths are found --- databooks/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/databooks/cli.py b/databooks/cli.py index 1b239c9c..48ea2c3c 100644 --- a/databooks/cli.py +++ b/databooks/cli.py @@ -43,12 +43,15 @@ def _help_callback(ctx: Context, show_help: Optional[bool]) -> None: def _config_callback(ctx: Context, config_path: Optional[Path]) -> Optional[Path]: """Get config file and inject values into context to override default args.""" + target_paths = expand_paths( + paths=[Path(p) for p in ctx.params.get("paths", ())], rglob="*" + ) config_path = ( get_config( - target_paths=expand_paths(paths=[Path(p) for p in ctx.params["paths"]]), + target_paths=target_paths, config_filename=TOML_CONFIG_FILE, ) - if config_path is None and "paths" in ctx.params + if config_path is None and target_paths else config_path ) logger.debug(f"Loading config file from: {config_path}") @@ -65,7 +68,6 @@ def _config_callback(ctx: Context, config_path: Optional[Path]) -> Optional[Path ) # Merge configuration ctx.default_map.update({k.replace("-", "_"): v for k, v in conf.items()}) - return config_path From 5c279d5b4d63290d922b2bcf12d057b0a419b1d6 Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sun, 16 Jan 2022 20:19:06 -0300 Subject: [PATCH 33/34] change log to include passed paths - easier to find typos in args or bugs --- databooks/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databooks/cli.py b/databooks/cli.py index 48ea2c3c..6a154a61 100644 --- a/databooks/cli.py +++ b/databooks/cli.py @@ -130,7 +130,7 @@ def meta( ) nb_paths = expand_paths(paths=paths, ignore=ignore) if not nb_paths: - logger.info("No notebooks found. Nothing to do.") + logger.info(f"No notebooks found in {paths}. Nothing to do.") raise Exit() if not bool(prefix + suffix) and not check: From 59bcacc8c1c751f546474d6ab78f919d9f440f5b Mon Sep 17 00:00:00 2001 From: Murilo Cunha Date: Sun, 16 Jan 2022 20:19:56 -0300 Subject: [PATCH 34/34] add tests that catch bad parameters, or when no action is required --- tests/test_cli.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 338added..533d552f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -126,6 +126,43 @@ def test_meta__config(tmpdir: LocalPath) -> None: assert all(c.execution_count is None for c in nb_write.cells) +def test_meta__script(tmpdir: LocalPath) -> None: + """Raise `typer.BadParameter` when passing a script instead of a notebook.""" + py_path = tmpdir.mkdir("files") / "a_script.py" # type: ignore + py_path.write_text("# some python code", encoding="utf-8") + + result = runner.invoke(app, ["meta", str(py_path)]) + assert result.exit_code == 2 + assert ( + "Expected either notebook files, a directory or glob expression." + in result.output + ) + + +def test_meta__no_overwrite(tmpdir: LocalPath) -> None: + """Raise `typer.BadParameter` when no `--overwrite` and no prefix nor suffix.""" + nb_path = tmpdir.mkdir("notebooks") / "test_meta_nb.ipynb" # type: ignore + write_notebook(nb=TestJupyterNotebook().jupyter_notebook, path=nb_path) + + result = runner.invoke(app, ["meta", str(nb_path)]) + assert ( + "No prefix nor suffix were passed. Please specify `--overwrite` or `-w` to" + " overwrite files." in result.output + ) + + +def test_meta__no_notebooks_found(tmpdir: LocalPath, caplog: LogCaptureFixture) -> None: + """Log that no notebook was found in the paths passed.""" + caplog.set_level(logging.INFO) + nb_path = tmpdir.mkdir("notebooks") / "inexistent_nb.ipynb" # type: ignore + + result = runner.invoke(app, ["meta", str(nb_path), "--check"]) + logs = list(caplog.records) + assert result.exit_code == 0 + assert len(logs) == 1 + assert logs[0].message == f"No notebooks found in {[Path(nb_path)]}. Nothing to do." + + def test_fix(tmpdir: LocalPath) -> None: """Fix notebook conflicts.""" # Setup