From 3cd0da48769298d7edbeadd4fdcbdcd191e6c0b7 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 10:15:37 -0500 Subject: [PATCH 01/79] add gitpython as dependency --- poetry.lock | 87 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 4 +++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 42603f6f..dbb0a60a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -72,6 +72,14 @@ category = "main" optional = false python-versions = ">=3.6, <3.7" +[[package]] +name = "ddt" +version = "1.4.4" +description = "Data-Driven/Decorated Tests" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "flake8" version = "3.9.2" @@ -133,6 +141,53 @@ python-versions = "*" [package.dependencies] flake8 = "*" +[[package]] +name = "gitdb" +version = "4.0.9" +description = "Git Object Database" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitdb2" +version = "4.0.2" +description = "A mirror package for gitdb" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +gitdb = ">=4.0.1" + +[[package]] +name = "gitpython" +version = "3.0.0" +description = "Python Git Library" +category = "main" +optional = false +python-versions = ">=3.0, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +ddt = ">=1.1.1" +gitdb = ">=0.6.4" +gitdb2 = ">=2.0.0" + +[[package]] +name = "gitpython" +version = "3.1.27" +description = "GitPython is a python library used to interact with Git repositories" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +gitdb = ">=4.0.1,<5" +typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} + [[package]] name = "importlib-metadata" version = "4.8.3" @@ -326,6 +381,14 @@ python-versions = ">=3.5" [package.dependencies] pytest = ">=3.0.0" +[[package]] +name = "smmap" +version = "5.0.0" +description = "A pure Python implementation of a sliding window memory map manager" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "snowballstemmer" version = "2.2.0" @@ -381,7 +444,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.2" -content-hash = "137b130c84cdc39d2856217674905ddb4185a179303d7b9e358bd1c71bfc868b" +content-hash = "9e96bdd7ab8525079bd8453f4dff4db0b5b82004268a24ee9ff844f8482220de" [metadata.files] atomicwrites = [ @@ -429,6 +492,10 @@ dataclasses = [ {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, ] +ddt = [ + {file = "ddt-1.4.4-py2.py3-none-any.whl", hash = "sha256:439f163a149d47761bcb11fefdd7327c7bc9262fa1c53f5929f643a6faa6d454"}, + {file = "ddt-1.4.4.tar.gz", hash = "sha256:8de39a69730442dc835e4d33f9d2e33043ff91151c8d18086959ee556febb9f8"}, +] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, @@ -449,6 +516,20 @@ flake8-polyfill = [ {file = "flake8-polyfill-1.0.2.tar.gz", hash = "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda"}, {file = "flake8_polyfill-1.0.2-py2.py3-none-any.whl", hash = "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9"}, ] +gitdb = [ + {file = "gitdb-4.0.9-py3-none-any.whl", hash = "sha256:8033ad4e853066ba6ca92050b9df2f89301b8fc8bf7e9324d412a63f8bf1a8fd"}, + {file = "gitdb-4.0.9.tar.gz", hash = "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa"}, +] +gitdb2 = [ + {file = "gitdb2-4.0.2-py3-none-any.whl", hash = "sha256:a1c974e5fab8c2c90192c1367c81cbc54baec04244bda1816e9c8ab377d1cba3"}, + {file = "gitdb2-4.0.2.tar.gz", hash = "sha256:0986cb4003de743f2b3aba4c828edd1ab58ce98e1c4a8acf72ef02760d4beb4e"}, +] +gitpython = [ + {file = "GitPython-3.0.0-py3-none-any.whl", hash = "sha256:556b64796c5e268b35e3b431a429e813ad54aa178e1baaec2a9ba82e8575a89e"}, + {file = "GitPython-3.0.0.tar.gz", hash = "sha256:629867ebf609cef21bb9d849039e281e25963fb7d714a2f6bacc1ecce4800293"}, + {file = "GitPython-3.1.27-py3-none-any.whl", hash = "sha256:5b68b000463593e05ff2b261acff0ff0972df8ab1b70d3cdbd41b546c8b8fc3d"}, + {file = "GitPython-3.1.27.tar.gz", hash = "sha256:1c885ce809e8ba2d88a29befeb385fcea06338d3640712b59ca623c220bb5704"}, +] importlib-metadata = [ {file = "importlib_metadata-4.8.3-py3-none-any.whl", hash = "sha256:65a9576a5b2d58ca44d133c42a241905cc45e34d2c06fd5ba2bafa221e5d7b5e"}, {file = "importlib_metadata-4.8.3.tar.gz", hash = "sha256:766abffff765960fcc18003801f7044eb6755ffae4521c8e8ce8e83b9c9b0668"}, @@ -517,6 +598,10 @@ pytest-snapshot = [ {file = "pytest-snapshot-0.8.0.tar.gz", hash = "sha256:cf84c88c3e0b4ae08ae797d9ccdc32715b64dd68b2da40f575db56956ed23326"}, {file = "pytest_snapshot-0.8.0-py3-none-any.whl", hash = "sha256:cc75e1a89a394ff790a7032e4eec7cc64aee0fef4c6c11eab038c991532619e6"}, ] +smmap = [ + {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, + {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, +] snowballstemmer = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, diff --git a/pyproject.toml b/pyproject.toml index 9a7daa1c..210e6ba5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,10 @@ black = ">=21.10b0" # These dependencies were selected because they are already used by black. click = ">=7.1.2" toml = ">=0.10.1" +GitPython = [ + {version=">=3.1", python=">=3.7"}, + {version="<=3.0", python=">=3.6,<3.7"} +] # flake8 plugins should be listed here (in alphabetical order) flake8-black = ">=0.2.1" From 9b92744edb3c0f4d7fe53fb41cdeecf967733f8b Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 13:12:39 -0500 Subject: [PATCH 02/79] get git import working --- poetry.lock | 75 ++++++++++++++++++++++++++++++-------------------- pyproject.toml | 5 ++-- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/poetry.lock b/poetry.lock index dbb0a60a..5ac1f694 100644 --- a/poetry.lock +++ b/poetry.lock @@ -46,7 +46,7 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "click" -version = "8.0.3" +version = "8.0.4" description = "Composable command line interface toolkit" category = "main" optional = false @@ -96,16 +96,16 @@ pyflakes = ">=2.3.0,<2.4.0" [[package]] name = "flake8-black" -version = "0.2.4" +version = "0.3.1" description = "flake8 plugin to call black as a code style validator" category = "main" optional = false python-versions = "*" [package.dependencies] -black = "*" +black = ">=22.1.0" flake8 = ">=3.0.0" -toml = "*" +tomli = "*" [[package]] name = "flake8-docstrings" @@ -154,14 +154,14 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitdb2" -version = "4.0.2" -description = "A mirror package for gitdb" +version = "2.0.6" +description = "Git Object Database" category = "main" optional = false -python-versions = "*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] -gitdb = ">=4.0.1" +smmap2 = ">=2.0.0" [[package]] name = "gitpython" @@ -338,7 +338,7 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "6.2.5" +version = "7.0.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false @@ -353,15 +353,15 @@ iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" py = ">=1.8.2" -toml = "*" +tomli = ">=1.0.0" [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] [[package]] name = "pytest-click" -version = "1.0.2" -description = "Py.test plugin for Click" +version = "1.1.0" +description = "Pytest plugin for Click" category = "dev" optional = false python-versions = "*" @@ -372,7 +372,7 @@ pytest = ">=5.0" [[package]] name = "pytest-snapshot" -version = "0.8.0" +version = "0.8.1" description = "A plugin for snapshot testing with pytest." category = "dev" optional = false @@ -389,6 +389,17 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "smmap2" +version = "3.0.1" +description = "A mirror package for smmap" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +smmap = ">=3.0.1" + [[package]] name = "snowballstemmer" version = "2.2.0" @@ -423,7 +434,7 @@ python-versions = ">=3.6" [[package]] name = "typing-extensions" -version = "4.0.1" +version = "4.1.1" description = "Backported and Experimental Type Hints for Python 3.6+" category = "main" optional = false @@ -444,7 +455,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.2" -content-hash = "9e96bdd7ab8525079bd8453f4dff4db0b5b82004268a24ee9ff844f8482220de" +content-hash = "e03f86500f9ec341d70d3f92744b062f7bc6ce9d9bf8032333078d9922d8a8ce" [metadata.files] atomicwrites = [ @@ -481,8 +492,8 @@ black = [ {file = "black-22.1.0.tar.gz", hash = "sha256:a7c0192d35635f6fc1174be575cb7915e92e5dd629ee79fdaf0dcfa41a80afb5"}, ] click = [ - {file = "click-8.0.3-py3-none-any.whl", hash = "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3"}, - {file = "click-8.0.3.tar.gz", hash = "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b"}, + {file = "click-8.0.4-py3-none-any.whl", hash = "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1"}, + {file = "click-8.0.4.tar.gz", hash = "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb"}, ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, @@ -501,8 +512,8 @@ flake8 = [ {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] flake8-black = [ - {file = "flake8-black-0.2.4.tar.gz", hash = "sha256:a7871bfd1cbff431a1fc91ba60ae154510c80f575e6b9a2bbb13dfb4650afd22"}, - {file = "flake8_black-0.2.4-py3-none-any.whl", hash = "sha256:0a70dfd97c8439827f365dc6dbc6c8c9cc087f0833625c6cc6848ff7876256be"}, + {file = "flake8-black-0.3.1.tar.gz", hash = "sha256:d0e9af6c2790fbbd3e3ffe579e2d185a5eb5419c233040d155957c4cacf2074d"}, + {file = "flake8_black-0.3.1-py3-none-any.whl", hash = "sha256:7824329dcb93ff167d22846cf61b6f0ce60546650fc3db33776a33094839efda"}, ] flake8-docstrings = [ {file = "flake8-docstrings-1.6.0.tar.gz", hash = "sha256:9fe7c6a306064af8e62a055c2f61e9eb1da55f84bb39caef2b84ce53708ac34b"}, @@ -521,8 +532,8 @@ gitdb = [ {file = "gitdb-4.0.9.tar.gz", hash = "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa"}, ] gitdb2 = [ - {file = "gitdb2-4.0.2-py3-none-any.whl", hash = "sha256:a1c974e5fab8c2c90192c1367c81cbc54baec04244bda1816e9c8ab377d1cba3"}, - {file = "gitdb2-4.0.2.tar.gz", hash = "sha256:0986cb4003de743f2b3aba4c828edd1ab58ce98e1c4a8acf72ef02760d4beb4e"}, + {file = "gitdb2-2.0.6-py2.py3-none-any.whl", hash = "sha256:96bbb507d765a7f51eb802554a9cfe194a174582f772e0d89f4e87288c288b7b"}, + {file = "gitdb2-2.0.6.tar.gz", hash = "sha256:1b6df1433567a51a4a9c1a5a0de977aa351a405cc56d7d35f3388bad1f630350"}, ] gitpython = [ {file = "GitPython-3.0.0-py3-none-any.whl", hash = "sha256:556b64796c5e268b35e3b431a429e813ad54aa178e1baaec2a9ba82e8575a89e"}, @@ -587,21 +598,25 @@ pyparsing = [ {file = "pyparsing-3.0.7.tar.gz", hash = "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea"}, ] pytest = [ - {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, - {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, + {file = "pytest-7.0.1-py3-none-any.whl", hash = "sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db"}, + {file = "pytest-7.0.1.tar.gz", hash = "sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171"}, ] pytest-click = [ - {file = "pytest_click-1.0.2-py3-none-any.whl", hash = "sha256:ac298e15a31d8892d9d063c3be82bb9877aaa11a4944e05e541ecc753258e474"}, - {file = "pytest_click-1.0.2.tar.gz", hash = "sha256:b40b8435adde7a0d931352036a3882b13cd424fce2ffd7be6c87665050259be5"}, + {file = "pytest_click-1.1.0-py3-none-any.whl", hash = "sha256:eade4742c2f02c345e78a32534a43e8db04acf98d415090539dacc880b7cd0e9"}, + {file = "pytest_click-1.1.0.tar.gz", hash = "sha256:fdd9f6721f877dda021e7c5dc73e70aecd37e5ed23ec6820f8a7b3fd7b4f8d30"}, ] pytest-snapshot = [ - {file = "pytest-snapshot-0.8.0.tar.gz", hash = "sha256:cf84c88c3e0b4ae08ae797d9ccdc32715b64dd68b2da40f575db56956ed23326"}, - {file = "pytest_snapshot-0.8.0-py3-none-any.whl", hash = "sha256:cc75e1a89a394ff790a7032e4eec7cc64aee0fef4c6c11eab038c991532619e6"}, + {file = "pytest-snapshot-0.8.1.tar.gz", hash = "sha256:0f8872d56bc3ceacb465967072b059a36714898a37c9eb1c75cd4054110106f2"}, + {file = "pytest_snapshot-0.8.1-py3-none-any.whl", hash = "sha256:ccb72c8e40dd1ec96b40caf0d328a9e9124b91d6a06204ad47d67403d83a4fd2"}, ] smmap = [ {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, ] +smmap2 = [ + {file = "smmap2-3.0.1-py3-none-any.whl", hash = "sha256:0cb6ea470b1ad9a65a02ca7f4c7ae601861f7dd24a43812ca51cfca2892bb524"}, + {file = "smmap2-3.0.1.tar.gz", hash = "sha256:44cc8bdaf96442dbb9a8e2e14377d074b3d0eea292eee3c95c8c449b6c92c557"}, +] snowballstemmer = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, @@ -641,8 +656,8 @@ typed-ast = [ {file = "typed_ast-1.5.2.tar.gz", hash = "sha256:525a2d4088e70a9f75b08b3f87a51acc9cde640e19cc523c7e41aa355564ae27"}, ] typing-extensions = [ - {file = "typing_extensions-4.0.1-py3-none-any.whl", hash = "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b"}, - {file = "typing_extensions-4.0.1.tar.gz", hash = "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e"}, + {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"}, + {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, ] zipp = [ {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, diff --git a/pyproject.toml b/pyproject.toml index 210e6ba5..f8b6fd5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ GitPython = [ {version=">=3.1", python=">=3.7"}, {version="<=3.0", python=">=3.6,<3.7"} ] +gitdb2 = {version="^2.0.0", python=">=3.6,<3.7"} # for py 3.6, pin this down # flake8 plugins should be listed here (in alphabetical order) flake8-black = ">=0.2.1" @@ -61,8 +62,8 @@ exclude = ''' ''' [tool.pytest.ini_options] -addopts = "--doctest-modules" -norecursedirs = "*__snapshots" +# addopts = "--doctest-modules" +# norecursedirs = "*__snapshots" [tool.ni-python-styleguide] extend_exclude = "*__snapshots/*/*input.py" From 3f920ffab32e96d7366e2c6a35904f5c305f3b50 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 13:13:25 -0500 Subject: [PATCH 03/79] allow windows style paths --- ni_python_styleguide/_utils/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ni_python_styleguide/_utils/lint.py b/ni_python_styleguide/_utils/lint.py index b4229ec6..55b334cb 100644 --- a/ni_python_styleguide/_utils/lint.py +++ b/ni_python_styleguide/_utils/lint.py @@ -48,7 +48,7 @@ class Parser: """Lint errors parser.""" __MATCHER = re.compile( - r"^(?P[\w\\/\.\-]+):(?P\d+):(?P\d+): (?P\w+) (?P.+)" + r"^(?P[\w\\/\.\-\:]+):(?P\d+):(?P\d+): (?P\w+) (?P.+)" ) @staticmethod From 5fc3d2ae9760468a8afa6d7d6fdb81d02655a907 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 13:14:31 -0500 Subject: [PATCH 04/79] format --- .../unicode_in_files/input.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/unicode_in_files/input.py b/tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/unicode_in_files/input.py index 398af279..b8cb656d 100644 --- a/tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/unicode_in_files/input.py +++ b/tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/unicode_in_files/input.py @@ -13,6 +13,7 @@ def problem_chars(self): """Return stored string with a unicode char.""" return self._problem_chars + def method_withBadName_andParams(my_normal_param, myBadlyNamedParam, my_other_Bad_param): """Provide example where black will want to split out result.""" return 5 + 7 From 9053cd0860740d466a7c1ae2ca0d9f45bef0ac0b Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 13:26:09 -0500 Subject: [PATCH 05/79] add support for handling cases with no ignored errors --- .../_acknowledge_existing_errors/__init__.py | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 9b47d015..7e6922a9 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -32,6 +32,25 @@ def _filter_suppresion_from_line(line: str): return line +def _get_lint_errors_to_process( + exclude, app_import_names, extend_ignore, file_or_dir, *_, excluded_errors=None +): + lint_errors = _lint.get_lint_output( + format=None, + qs_or_vs=None, + exclude=exclude, + app_import_names=app_import_names, + extend_ignore=extend_ignore, + file_or_dir=file_or_dir, + ).splitlines() + parsed_errors = map(_lint_errors_parser.parse, lint_errors) + parsed_errors = list(filter(None, parsed_errors)) + if excluded_errors is None: + excluded_errors = EXCLUDED_ERRORS + lint_errors_to_process = [error for error in parsed_errors if error.code not in excluded_errors] + return lint_errors_to_process + + def acknowledge_lint_errors( exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=False ): @@ -70,16 +89,8 @@ def acknowledge_lint_errors( # re-apply suppressions on correct lines remove_auto_suppressions_from_file(bad_file) - current_lint_errors = _utils.lint.get_errors_to_process( - exclude=exclude, - app_import_names=app_import_names, - extend_ignore=extend_ignore, - file_or_dir=file_or_dir, - excluded_errors=EXCLUDED_ERRORS, - ) - - _suppress_errors_in_file( - bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING + current_lint_errors = _get_lint_errors_to_process( + exclude, app_import_names, extend_ignore, [bad_file] ) changed = _format.format_check(bad_file) @@ -96,7 +107,7 @@ def acknowledge_lint_errors( def remove_auto_suppressions_from_file(file: pathlib.Path): """Remove auto-suppressions from file.""" - lines = file.read_text(encoding=_utils.DEFAULT_ENCODING).splitlines() + lines = file.read_text(encoding=DEFAULT_ENCODING).splitlines() stripped_lines = [_filter_suppresion_from_line(line) for line in lines] file.write_text("\n".join(stripped_lines) + "\n", encoding=_utils.DEFAULT_ENCODING) From 8acb5aa5d8a9bb0c4b7bd800282473d132e414f6 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 13:30:08 -0500 Subject: [PATCH 06/79] get initial setup working --- ni_python_styleguide/_cli.py | 26 ++++ ni_python_styleguide/_fix.py | 131 ++++++++++++++++++ ni_python_styleguide/_git_utils.py | 16 +++ .../unicode_in_files/output.py | 1 + .../basic_example/input.py | 15 ++ .../basic_example/output.py | 19 +++ tests/test_cli/test_fix.py | 28 ++++ 7 files changed, 236 insertions(+) create mode 100644 ni_python_styleguide/_fix.py create mode 100644 ni_python_styleguide/_git_utils.py create mode 100644 tests/test_cli/fix_test_cases__snapshots/basic_example/input.py create mode 100644 tests/test_cli/fix_test_cases__snapshots/basic_example/output.py create mode 100644 tests/test_cli/test_fix.py diff --git a/ni_python_styleguide/_cli.py b/ni_python_styleguide/_cli.py index 77e13725..3ddf553e 100644 --- a/ni_python_styleguide/_cli.py +++ b/ni_python_styleguide/_cli.py @@ -1,7 +1,9 @@ import click +import pathlib import toml from ni_python_styleguide import _acknowledge_existing_errors +from ni_python_styleguide import _fix from ni_python_styleguide import _lint @@ -158,3 +160,27 @@ def acknowledge_existing_violations(obj, extend_ignore, file_or_dir, aggressive) file_or_dir=file_or_dir, aggressive=aggressive, ) + + +@main.command() +@click.option( + "--extend-ignore", + type=str, + help="Comma-separated list of errors and warnings to ignore (or skip)", +) +@click.argument("file_or_dir", nargs=-1) +@click.option( + "--aggressive", + is_flag=True, + help="Remove any existing suppressions, fix what can be fixed, and suppress remaining.", +) +@click.pass_obj +def fix(obj, extend_ignore, file_or_dir, aggressive): + """Fix basic linter/formatting errors in file(s)/directory(s) given.""" # noqa: D4 + _fix.fix( + exclude=obj["EXCLUDE"], + app_import_names=obj["APP_IMPORT_NAMES"], + extend_ignore=extend_ignore, + file_or_dir=file_or_dir or [pathlib.Path.cwd()], + aggressive=aggressive, + ) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py new file mode 100644 index 00000000..b6734269 --- /dev/null +++ b/ni_python_styleguide/_fix.py @@ -0,0 +1,131 @@ +import fileinput +import logging +import pathlib +from collections import defaultdict +from typing import Iterable, List + +from ni_python_styleguide import _acknowledge_existing_errors, _format +from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser + +# from ni_python_styleguide import _git_utils + +_module_logger = logging.getLogger(__name__) + + +def _split_imports_line(lines: str, *_, **__): + r"""Split multi-import lines to multiple lines. + + >>> _split_imports_line("import os, collections\n") + 'import os\nimport collections\n' + + >>> _split_imports_line("import os\n") + 'import os\n' + + >>> _split_imports_line("from ni_python_styleguide import _acknowledge_existing_errors, _format") + 'from ni_python_styleguide import _acknowledge_existing_errors\nfrom ni_python_styleguide import _format\n' + + >>> _split_imports_line("from ni_python_styleguide import _acknowledge_existing_errors") + 'from ni_python_styleguide import _acknowledge_existing_errors\n' + + >>> _split_imports_line("import os, collections\nimport pathlib") + 'import os\nimport collections\nimport pathlib\n' + + >>> _split_imports_line("import os, collections\nimport pathlib, os") + 'import os\nimport collections\nimport pathlib\nimport os\n' + """ + result_parts = [] + for line in lines.splitlines(): + first, _, rest = line.partition(",") + if not rest or "import" not in line: + result_parts.append(line) + continue + prefix, first = " ".join(first.split()[:-1]), first.split()[-1] + split_up = [first] + rest.split(",") + result_parts.extend([prefix + " " + part.strip() for part in split_up]) + result = "\n".join(result_parts) + if result: + return result + "\n" + return result + + +def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): + _module_logger("Splitting import lines in file: %s", file) + hashed_offending_lines = set(offending_lines) + with fileinput.input(file, inplace=True) as py_file: + for line_no, line in py_file: + if line_no in hashed_offending_lines: + print(_split_imports_line(line)) + else: + print(line) + + +def _sort_imports(): + pass + + +def _remove_unused_non_fist_party_imports(lines: str, *_, **__): + return lines + + +CODE_TO_HANDLER_MAPPING = { + "I100": _sort_imports, + "I201": _sort_imports, + "F401": _remove_unused_non_fist_party_imports, + "E401": _split_imports, +} + + +def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=False, diff=False): + """Fix basic linter errors and format.""" + if aggressive: + raise Exception("--aggressive is not implemented yet") + lint_errors_to_process = _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [pathlib.Path(file_or_dir_) for file_or_dir_ in file_or_dir or "."], + excluded_errors=[], # we fix black errors, so we don't need to filter it. + ) + + lint_errors_by_file = defaultdict(list) + for error in lint_errors_to_process: + lint_errors_by_file[pathlib.Path(error.file)].append(error) + + failed_files = [] + for bad_file, errors_in_file in lint_errors_by_file.items(): + errors_in_file: List[_lint_errors_parser.LintError] + try: + _format.format(bad_file) + line_to_codes_mapping = defaultdict(set) + for error in errors_in_file: + # humans talk 1-based, enumerate is 0-based + line_to_codes_mapping[int(error.line) - 1].add(error.code) + # in_memory_file = deque(bad_file.read_text().splitlines()) + with fileinput.FileInput(files=[str(bad_file)], inplace=True) as f: + for line_no, line in enumerate(f): + if not any( + [ + line_no in line_to_codes_mapping, + any( + [ + code in CODE_TO_HANDLER_MAPPING + for code in line_to_codes_mapping.get(line_no, []) + ] + ), + ] + ): + print(line) + continue + + working_line = line + for handler in [_split_imports_line, _remove_unused_non_fist_party_imports]: + working_line = handler(working_line) + print(working_line, end='') + _format.format(bad_file) + except AttributeError as e: + failed_files.append((bad_file, e)) + if failed_files: + raise Exception( + "Failed to format files:\n" + + "\n".join([f"{file}: {error}" for file, error in failed_files]) + ) diff --git a/ni_python_styleguide/_git_utils.py b/ni_python_styleguide/_git_utils.py new file mode 100644 index 00000000..00ececb7 --- /dev/null +++ b/ni_python_styleguide/_git_utils.py @@ -0,0 +1,16 @@ +import fnmatch +import pathlib +import os + +from git import repo + + +def get_tracked_files(cwd: os.PathLike, *_, branch: str = "main", filter: str = "*.*"): + """Get all Git tracked files under cwd.""" + cwd = pathlib.Path(cwd) + _repo = repo.Repo(str(cwd), search_parent_directories=True) + repo_root = pathlib.Path(_repo.working_dir) + output = _repo.git.ls_tree(str(cwd), r=branch, name_only=True).splitlines() + for line in output: + if line.startswith(str(cwd)) and fnmatch.fnmatch(line, filter): + yield repo_root / line diff --git a/tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/unicode_in_files/output.py b/tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/unicode_in_files/output.py index 8899523f..163662e7 100644 --- a/tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/unicode_in_files/output.py +++ b/tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/unicode_in_files/output.py @@ -13,6 +13,7 @@ def problem_chars(self): """Return stored string with a unicode char.""" return self._problem_chars + def method_withBadName_andParams(my_normal_param, myBadlyNamedParam, my_other_Bad_param): # noqa N802: function name 'method_withBadName_andParams' should be lowercase (auto-generated noqa) # noqa N803: argument name 'myBadlyNamedParam' should be lowercase (auto-generated noqa) """Provide example where black will want to split out result.""" return 5 + 7 diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py new file mode 100644 index 00000000..20e33db1 --- /dev/null +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py @@ -0,0 +1,15 @@ +from typing import Iterable, List, Hashable +from os import path, access + + +class Foo(object): # comment should get moved + o = 5 + i = 1 + + def __init__(self, o: Iterable[int]) -> None: + this_method = 1 + + serperator = path.sep + + has_a_lot_of_empty_lines = 2 + pass diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py new file mode 100644 index 00000000..ac14b60d --- /dev/null +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py @@ -0,0 +1,19 @@ +from typing import Iterable +from typing import List +from typing import Hashable +from os import path +from os import access + + +class Foo(object): # comment should get moved + o = 5 + + i = 1 + + def __init__(self, o: Iterable[int]) -> None: + this_method = 1 + + serperator = path.sep + + has_a_lot_of_empty_lines = 2 + pass diff --git a/tests/test_cli/test_fix.py b/tests/test_cli/test_fix.py new file mode 100644 index 00000000..45b43865 --- /dev/null +++ b/tests/test_cli/test_fix.py @@ -0,0 +1,28 @@ +"""Tests for the "fix" subcommand of ni-python-styleguide.""" + +import pathlib +import shutil + +import pytest + +from ni_python_styleguide import _fix + + +TEST_CASE_DIR = pathlib.Path(__file__).parent.absolute() / "fix_test_cases__snapshots" + + +@pytest.mark.parametrize("test_dir", [x for x in TEST_CASE_DIR.iterdir() if x.is_dir()]) +def test_given_bad_input__fix__produces_expected_output_simple( + test_dir, snapshot, tmp_path, styleguide_command +): + """Test that suppresion yields expected_output file.""" + in_file = test_dir / "input.py" + test_file = tmp_path / "input.py" + shutil.copyfile(in_file, test_file) + + output = styleguide_command(command="fix") + + assert output.exit_code in (True, 0), f"Error in running:\n{output}" + result = test_file.read_text(encoding="UTF-8") + snapshot.snapshot_dir = test_dir + snapshot.assert_match(result, "output.py") From c0ae649674cb45501e398214916297a07d048dd0 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 13:40:14 -0500 Subject: [PATCH 07/79] move git_utils to _utils.git --- ni_python_styleguide/{_git_utils.py => _utils/git.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename ni_python_styleguide/{_git_utils.py => _utils/git.py} (100%) diff --git a/ni_python_styleguide/_git_utils.py b/ni_python_styleguide/_utils/git.py similarity index 100% rename from ni_python_styleguide/_git_utils.py rename to ni_python_styleguide/_utils/git.py index 00ececb7..409f3005 100644 --- a/ni_python_styleguide/_git_utils.py +++ b/ni_python_styleguide/_utils/git.py @@ -1,6 +1,6 @@ import fnmatch -import pathlib import os +import pathlib from git import repo From 33238c17f257381d354bbbe013bd7360a560fbd3 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 13:40:40 -0500 Subject: [PATCH 08/79] un-needed --- tests/test_cli/test_fix.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_cli/test_fix.py b/tests/test_cli/test_fix.py index 45b43865..e2989eba 100644 --- a/tests/test_cli/test_fix.py +++ b/tests/test_cli/test_fix.py @@ -5,8 +5,6 @@ import pytest -from ni_python_styleguide import _fix - TEST_CASE_DIR = pathlib.Path(__file__).parent.absolute() / "fix_test_cases__snapshots" From 1f80f5fe8c646c80d5b2e6e87208654fa4692611 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 13:40:49 -0500 Subject: [PATCH 09/79] fix imports --- ni_python_styleguide/_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ni_python_styleguide/_cli.py b/ni_python_styleguide/_cli.py index 3ddf553e..51fc0d31 100644 --- a/ni_python_styleguide/_cli.py +++ b/ni_python_styleguide/_cli.py @@ -1,5 +1,6 @@ -import click import pathlib + +import click import toml from ni_python_styleguide import _acknowledge_existing_errors From 3abb6255790ec242091df85af99aebd64a44657c Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 13:40:58 -0500 Subject: [PATCH 10/79] remove the blank lines --- ni_python_styleguide/_fix.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index b6734269..0773e51a 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -4,10 +4,10 @@ from collections import defaultdict from typing import Iterable, List +import ni_python_styleguide._utils from ni_python_styleguide import _acknowledge_existing_errors, _format from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser -# from ni_python_styleguide import _git_utils _module_logger = logging.getLogger(__name__) @@ -21,8 +21,10 @@ def _split_imports_line(lines: str, *_, **__): >>> _split_imports_line("import os\n") 'import os\n' - >>> _split_imports_line("from ni_python_styleguide import _acknowledge_existing_errors, _format") - 'from ni_python_styleguide import _acknowledge_existing_errors\nfrom ni_python_styleguide import _format\n' + >>> _split_imports_line("from ni_python_styleguide import" + ... " _acknowledge_existing_errors, _format") + 'from ni_python_styleguide import _acknowledge_existing_errors\n" + ... "from ni_python_styleguide import _format\n' >>> _split_imports_line("from ni_python_styleguide import _acknowledge_existing_errors") 'from ni_python_styleguide import _acknowledge_existing_errors\n' @@ -114,13 +116,13 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa ), ] ): - print(line) + print(line, end="") continue working_line = line for handler in [_split_imports_line, _remove_unused_non_fist_party_imports]: working_line = handler(working_line) - print(working_line, end='') + print(working_line, end="") _format.format(bad_file) except AttributeError as e: failed_files.append((bad_file, e)) From c06a5f2e057937fc9be7fe6456983c81cc8bdcfa Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 13:41:40 -0500 Subject: [PATCH 11/79] cleanup some more for the linter --- .../test_cli/fix_test_cases__snapshots/basic_example/input.py | 2 ++ .../test_cli/fix_test_cases__snapshots/basic_example/output.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py index 20e33db1..e1f884c9 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py @@ -1,3 +1,4 @@ +"""Provide example cases of imports that need sorting and a file that needs formatted.""" from typing import Iterable, List, Hashable from os import path, access @@ -7,6 +8,7 @@ class Foo(object): # comment should get moved i = 1 def __init__(self, o: Iterable[int]) -> None: + """Test class.""" this_method = 1 serperator = path.sep diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py index ac14b60d..dbfa9775 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py @@ -1,3 +1,4 @@ +"""Provide example cases of imports that need sorting and a file that needs formatted.""" from typing import Iterable from typing import List from typing import Hashable @@ -7,10 +8,10 @@ class Foo(object): # comment should get moved o = 5 - i = 1 def __init__(self, o: Iterable[int]) -> None: + """Test class.""" this_method = 1 serperator = path.sep From b81dd3735b54fa5607e3a2bd6c359294bb95647d Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 15:02:27 -0500 Subject: [PATCH 12/79] extract multiline string helpers to _utils --- .../_acknowledge_existing_errors/__init__.py | 7 ++++- ni_python_styleguide/_fix.py | 1 - ni_python_styleguide/_utils/__init__.py | 2 +- ni_python_styleguide/_utils/string_helpers.py | 26 ++++++------------- .../test_acknowledge_existing_errors.py | 1 + tests/test_cli/test_utils.py | 1 - 6 files changed, 16 insertions(+), 22 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 7e6922a9..99f2a2f6 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -4,7 +4,9 @@ from collections import defaultdict from ni_python_styleguide import _format +from ni_python_styleguide import _lint from ni_python_styleguide import _utils +from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser _module_logger = logging.getLogger(__name__) @@ -92,6 +94,9 @@ def acknowledge_lint_errors( current_lint_errors = _get_lint_errors_to_process( exclude, app_import_names, extend_ignore, [bad_file] ) + _suppress_errors_in_file( + bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING + ) changed = _format.format_check(bad_file) if not changed: # are we done? @@ -107,7 +112,7 @@ def acknowledge_lint_errors( def remove_auto_suppressions_from_file(file: pathlib.Path): """Remove auto-suppressions from file.""" - lines = file.read_text(encoding=DEFAULT_ENCODING).splitlines() + lines = file.read_text(encoding=_utils.DEFAULT_ENCODING).splitlines() stripped_lines = [_filter_suppresion_from_line(line) for line in lines] file.write_text("\n".join(stripped_lines) + "\n", encoding=_utils.DEFAULT_ENCODING) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 0773e51a..313d7522 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -4,7 +4,6 @@ from collections import defaultdict from typing import Iterable, List -import ni_python_styleguide._utils from ni_python_styleguide import _acknowledge_existing_errors, _format from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser diff --git a/ni_python_styleguide/_utils/__init__.py b/ni_python_styleguide/_utils/__init__.py index 96c4e689..0c75474a 100644 --- a/ni_python_styleguide/_utils/__init__.py +++ b/ni_python_styleguide/_utils/__init__.py @@ -1,4 +1,4 @@ -from . import lint # noqa: F401 +from . import git # noqa: F401 from . import string_helpers # noqa: F401 DEFAULT_ENCODING = "UTF-8" diff --git a/ni_python_styleguide/_utils/string_helpers.py b/ni_python_styleguide/_utils/string_helpers.py index 339fba9f..bfdd10c4 100644 --- a/ni_python_styleguide/_utils/string_helpers.py +++ b/ni_python_styleguide/_utils/string_helpers.py @@ -1,5 +1,4 @@ import pathlib -from typing import List, Optional import ni_python_styleguide._utils @@ -7,19 +6,11 @@ class InMultiLineStringChecker: """Provide utility methods to decide if line is within a multiline string.""" - def __init__(self, error_file: Optional[str] = None, *_, lines: Optional[List[str]] = None): + def __init__(self, error_file): """Cache off whether each line is in a multiline string or not.""" + self._error_file = pathlib.Path(error_file) self._values = [] - if error_file: - self._error_file = pathlib.Path(error_file) - self._load_lines() - else: - self._error_file = None - if not lines: - raise Exception( - "Error, must provide either path to `error_file` or provide `lines`" - ) - self._set_lines(lines) + self._load_lines() @property def values(self): @@ -34,9 +25,12 @@ def in_multiline_string(self, lineno): def _count_multiline_string_endings_in_line(line): return line.count('"""'), line.count("'''") - def _set_lines(self, lines): + def _load_lines(self): + in_file = self._error_file.read_text( + encoding=ni_python_styleguide._utils.DEFAULT_ENCODING + ).splitlines() current_count = [0, 0] - for line in lines: + for line in in_file: type1, type2 = InMultiLineStringChecker._count_multiline_string_endings_in_line(line) current_count[0] += type1 current_count[1] += type2 @@ -51,7 +45,3 @@ def _set_lines(self, lines): any([part % 2 == 1 for part in current_count]) or code_part_of_line.strip().endswith("\\") ) - - def _load_lines(self): - in_file = self._error_file.read_text(encoding=ni_python_styleguide._utils.DEFAULT_ENCODING) - self._set_lines(in_file.splitlines()) diff --git a/tests/test_cli/test_acknowledge_existing_errors.py b/tests/test_cli/test_acknowledge_existing_errors.py index ed2a6b0f..1a7ee27e 100644 --- a/tests/test_cli/test_acknowledge_existing_errors.py +++ b/tests/test_cli/test_acknowledge_existing_errors.py @@ -10,6 +10,7 @@ ) + @pytest.mark.parametrize("test_dir", [x for x in TEST_CASE_DIR.iterdir() if x.is_dir()]) def test_given_bad_input_produces_expected_output_simple( test_dir, snapshot, tmp_path, styleguide_command diff --git a/tests/test_cli/test_utils.py b/tests/test_cli/test_utils.py index e9ef7b3d..869d8c60 100644 --- a/tests/test_cli/test_utils.py +++ b/tests/test_cli/test_utils.py @@ -1,4 +1,3 @@ -"""Test the _utils submodule.""" import pytest from ni_python_styleguide import _utils From 0b6e87034b4353205cd5b62fd47585b46cbdd9ce Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 15:03:40 -0500 Subject: [PATCH 13/79] typo --- tests/test_cli/fix_test_cases__snapshots/basic_example/input.py | 2 +- .../test_cli/fix_test_cases__snapshots/basic_example/output.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py index e1f884c9..d98880d5 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py @@ -11,7 +11,7 @@ def __init__(self, o: Iterable[int]) -> None: """Test class.""" this_method = 1 - serperator = path.sep + seperator = path.sep has_a_lot_of_empty_lines = 2 pass diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py index dbfa9775..1c8560b8 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py @@ -14,7 +14,7 @@ def __init__(self, o: Iterable[int]) -> None: """Test class.""" this_method = 1 - serperator = path.sep + seperator = path.sep has_a_lot_of_empty_lines = 2 pass From 0a3520304f5d66a462fa621b920d7a4e5275f7ee Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 15:32:29 -0500 Subject: [PATCH 14/79] format --- tests/test_cli/test_acknowledge_existing_errors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cli/test_acknowledge_existing_errors.py b/tests/test_cli/test_acknowledge_existing_errors.py index 1a7ee27e..ed2a6b0f 100644 --- a/tests/test_cli/test_acknowledge_existing_errors.py +++ b/tests/test_cli/test_acknowledge_existing_errors.py @@ -10,7 +10,6 @@ ) - @pytest.mark.parametrize("test_dir", [x for x in TEST_CASE_DIR.iterdir() if x.is_dir()]) def test_given_bad_input_produces_expected_output_simple( test_dir, snapshot, tmp_path, styleguide_command From 0ba226fc4001200e3afefdaf64019f83814bdc73 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 17:06:10 -0500 Subject: [PATCH 15/79] make these work right --- ni_python_styleguide/_fix.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 313d7522..3bea4419 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -22,8 +22,7 @@ def _split_imports_line(lines: str, *_, **__): >>> _split_imports_line("from ni_python_styleguide import" ... " _acknowledge_existing_errors, _format") - 'from ni_python_styleguide import _acknowledge_existing_errors\n" - ... "from ni_python_styleguide import _format\n' + 'from ni_python_styleguide import _acknowledge_existing_errors\nfrom ni_python_styleguide import _format\n' >>> _split_imports_line("from ni_python_styleguide import _acknowledge_existing_errors") 'from ni_python_styleguide import _acknowledge_existing_errors\n' @@ -33,9 +32,12 @@ def _split_imports_line(lines: str, *_, **__): >>> _split_imports_line("import os, collections\nimport pathlib, os") 'import os\nimport collections\nimport pathlib\nimport os\n' - """ + + >>> _split_imports_line("\n") + '\n' + """ # noqa W505: long lines... result_parts = [] - for line in lines.splitlines(): + for line in lines.splitlines(keepends=True): first, _, rest = line.partition(",") if not rest or "import" not in line: result_parts.append(line) @@ -44,8 +46,8 @@ def _split_imports_line(lines: str, *_, **__): split_up = [first] + rest.split(",") result_parts.extend([prefix + " " + part.strip() for part in split_up]) result = "\n".join(result_parts) - if result: - return result + "\n" + if result.strip(): + return result.rstrip() + "\n" return result From b54f7119af8bce75c505e1547f117969e2532c53 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 17:07:20 -0500 Subject: [PATCH 16/79] extract fixing multi-import lines --- ni_python_styleguide/_fix.py | 40 +++++++++++++++++------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 3bea4419..88f57624 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -62,7 +62,7 @@ def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): print(line) -def _sort_imports(): +def _sort_imports(file: pathlib.Path): pass @@ -78,6 +78,14 @@ def _remove_unused_non_fist_party_imports(lines: str, *_, **__): } +def _handle_multiple_import_lines(bad_file, line_to_codes_mapping): + with fileinput.FileInput(files=[str(bad_file)], inplace=True) as f: + for line in f: + working_line = line + working_line = _split_imports_line(working_line) + print(working_line, end="") + + def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=False, diff=False): """Fix basic linter errors and format.""" if aggressive: @@ -104,27 +112,17 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa # humans talk 1-based, enumerate is 0-based line_to_codes_mapping[int(error.line) - 1].add(error.code) # in_memory_file = deque(bad_file.read_text().splitlines()) - with fileinput.FileInput(files=[str(bad_file)], inplace=True) as f: - for line_no, line in enumerate(f): - if not any( - [ - line_no in line_to_codes_mapping, - any( - [ - code in CODE_TO_HANDLER_MAPPING - for code in line_to_codes_mapping.get(line_no, []) - ] - ), - ] - ): - print(line, end="") - continue - - working_line = line - for handler in [_split_imports_line, _remove_unused_non_fist_party_imports]: - working_line = handler(working_line) - print(working_line, end="") + + _format.format(bad_file) + _handle_multiple_import_lines(bad_file, line_to_codes_mapping) _format.format(bad_file) + remaining_lint_errors_in_file = _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [bad_file], + excluded_errors=[], # we fix black errors, so we don't need to filter it. + ) except AttributeError as e: failed_files.append((bad_file, e)) if failed_files: From 7c4c1dce592938c332dcd1c1bd6a4f3a64211bba Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 17:08:24 -0500 Subject: [PATCH 17/79] add sorting imports --- ni_python_styleguide/_fix.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 88f57624..f56dc876 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -4,6 +4,8 @@ from collections import defaultdict from typing import Iterable, List +import isort + from ni_python_styleguide import _acknowledge_existing_errors, _format from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser @@ -63,7 +65,9 @@ def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): def _sort_imports(file: pathlib.Path): - pass + raw = file.read_text() + output = isort.code(raw, multi_line_output =3, line_length=1) + file.write_text(output) def _remove_unused_non_fist_party_imports(lines: str, *_, **__): @@ -112,7 +116,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa # humans talk 1-based, enumerate is 0-based line_to_codes_mapping[int(error.line) - 1].add(error.code) # in_memory_file = deque(bad_file.read_text().splitlines()) - + _sort_imports(bad_file) _format.format(bad_file) _handle_multiple_import_lines(bad_file, line_to_codes_mapping) _format.format(bad_file) From c3bcddce8a184dffaea21372fc46c58d77b4be9d Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 17:09:44 -0500 Subject: [PATCH 18/79] format --- ni_python_styleguide/_fix.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index f56dc876..464d9c79 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -66,7 +66,7 @@ def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): def _sort_imports(file: pathlib.Path): raw = file.read_text() - output = isort.code(raw, multi_line_output =3, line_length=1) + output = isort.code(raw, multi_line_output=3, line_length=1) file.write_text(output) @@ -120,12 +120,14 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa _format.format(bad_file) _handle_multiple_import_lines(bad_file, line_to_codes_mapping) _format.format(bad_file) - remaining_lint_errors_in_file = _acknowledge_existing_errors._get_lint_errors_to_process( - exclude, - app_import_names, - extend_ignore, - [bad_file], - excluded_errors=[], # we fix black errors, so we don't need to filter it. + remaining_lint_errors_in_file = ( + _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [bad_file], + excluded_errors=[], # we fix black errors, so we don't need to filter it. + ) ) except AttributeError as e: failed_files.append((bad_file, e)) From 2a647cd4811ba5a9ffe7ac16e0fbfe613a0a1c61 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 17:09:56 -0500 Subject: [PATCH 19/79] add isort as dep --- poetry.lock | 20 +++++++++++++++++++- pyproject.toml | 2 ++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 5ac1f694..5a9cf42b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -213,6 +213,20 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "isort" +version = "5.10.1" +description = "A Python utility / library to sort Python imports." +category = "main" +optional = false +python-versions = ">=3.6.1,<4.0" + +[package.extras] +pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +requirements_deprecated_finder = ["pipreqs", "pip-api"] +colors = ["colorama (>=0.4.3,<0.5.0)"] +plugins = ["setuptools"] + [[package]] name = "mccabe" version = "0.6.1" @@ -455,7 +469,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.2" -content-hash = "e03f86500f9ec341d70d3f92744b062f7bc6ce9d9bf8032333078d9922d8a8ce" +content-hash = "6c017bbb5c1ee35ee76ad4c60e075632bb6d6e811d71250de8ea47f93491c37e" [metadata.files] atomicwrites = [ @@ -549,6 +563,10 @@ iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] +isort = [ + {file = "isort-5.10.1-py3-none-any.whl", hash = "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7"}, + {file = "isort-5.10.1.tar.gz", hash = "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"}, +] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, diff --git a/pyproject.toml b/pyproject.toml index f8b6fd5a..21c29b86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,8 @@ GitPython = [ {version="<=3.0", python=">=3.6,<3.7"} ] gitdb2 = {version="^2.0.0", python=">=3.6,<3.7"} # for py 3.6, pin this down +isort = {version=">=5.10", python=">=3.6.2,<4"} + # flake8 plugins should be listed here (in alphabetical order) flake8-black = ">=0.2.1" From 6e66b8f25856d76a59e8ffea448d1d97c3d77afa Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 17:10:13 -0500 Subject: [PATCH 20/79] make the linter happy --- tests/test_cli/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cli/test_utils.py b/tests/test_cli/test_utils.py index 869d8c60..e9ef7b3d 100644 --- a/tests/test_cli/test_utils.py +++ b/tests/test_cli/test_utils.py @@ -1,3 +1,4 @@ +"""Test the _utils submodule.""" import pytest from ni_python_styleguide import _utils From f377be6bba20078064273252c8e8169fcad90e6d Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 25 Mar 2022 17:10:26 -0500 Subject: [PATCH 21/79] add some more test cases --- .../fix_test_cases__snapshots/basic_example/input.py | 2 ++ .../fix_test_cases__snapshots/basic_example/output.py | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py index d98880d5..2db64dd4 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py @@ -1,5 +1,7 @@ """Provide example cases of imports that need sorting and a file that needs formatted.""" from typing import Iterable, List, Hashable +import pytest +import pathlib from os import path, access diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py index 1c8560b8..acabcbb1 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py @@ -1,9 +1,12 @@ """Provide example cases of imports that need sorting and a file that needs formatted.""" +import pathlib +from os import access +from os import path +from typing import Hashable from typing import Iterable from typing import List -from typing import Hashable -from os import path -from os import access + +import pytest class Foo(object): # comment should get moved From 6a083502bc6f343cf8690cc9ca11e5121b14b1e1 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 14:26:07 -0500 Subject: [PATCH 22/79] remove some dead code --- ni_python_styleguide/_fix.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 464d9c79..770d6614 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -70,19 +70,7 @@ def _sort_imports(file: pathlib.Path): file.write_text(output) -def _remove_unused_non_fist_party_imports(lines: str, *_, **__): - return lines - - -CODE_TO_HANDLER_MAPPING = { - "I100": _sort_imports, - "I201": _sort_imports, - "F401": _remove_unused_non_fist_party_imports, - "E401": _split_imports, -} - - -def _handle_multiple_import_lines(bad_file, line_to_codes_mapping): +def _handle_multiple_import_lines(bad_file): with fileinput.FileInput(files=[str(bad_file)], inplace=True) as f: for line in f: working_line = line @@ -115,10 +103,9 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa for error in errors_in_file: # humans talk 1-based, enumerate is 0-based line_to_codes_mapping[int(error.line) - 1].add(error.code) - # in_memory_file = deque(bad_file.read_text().splitlines()) _sort_imports(bad_file) _format.format(bad_file) - _handle_multiple_import_lines(bad_file, line_to_codes_mapping) + _handle_multiple_import_lines(bad_file) _format.format(bad_file) remaining_lint_errors_in_file = ( _acknowledge_existing_errors._get_lint_errors_to_process( From 614b99afecc8ad592565ef616dcbb009331cb01d Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 14:45:14 -0500 Subject: [PATCH 23/79] make it ignore imports in docstrings --- ni_python_styleguide/_fix.py | 11 ++++++-- ni_python_styleguide/_utils/string_helpers.py | 26 +++++++++++++------ .../basic_example/input.py | 6 +++++ .../basic_example/output.py | 6 +++++ 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 770d6614..7264ddfc 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -8,6 +8,7 @@ from ni_python_styleguide import _acknowledge_existing_errors, _format from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser +from ni_python_styleguide import _utils _module_logger = logging.getLogger(__name__) @@ -70,10 +71,16 @@ def _sort_imports(file: pathlib.Path): file.write_text(output) -def _handle_multiple_import_lines(bad_file): +def _handle_multiple_import_lines(bad_file: pathlib.Path): + multiline_string_checker = _utils.string_helpers.InMultiLineStringChecker( + lines=bad_file.read_text(encoding=_utils.DEFAULT_ENCODING).splitlines() + ) with fileinput.FileInput(files=[str(bad_file)], inplace=True) as f: - for line in f: + for line_no, line in enumerate(f): working_line = line + if multiline_string_checker.in_multiline_string(line_no + 1): + print(working_line, end="") + continue working_line = _split_imports_line(working_line) print(working_line, end="") diff --git a/ni_python_styleguide/_utils/string_helpers.py b/ni_python_styleguide/_utils/string_helpers.py index bfdd10c4..339fba9f 100644 --- a/ni_python_styleguide/_utils/string_helpers.py +++ b/ni_python_styleguide/_utils/string_helpers.py @@ -1,4 +1,5 @@ import pathlib +from typing import List, Optional import ni_python_styleguide._utils @@ -6,11 +7,19 @@ class InMultiLineStringChecker: """Provide utility methods to decide if line is within a multiline string.""" - def __init__(self, error_file): + def __init__(self, error_file: Optional[str] = None, *_, lines: Optional[List[str]] = None): """Cache off whether each line is in a multiline string or not.""" - self._error_file = pathlib.Path(error_file) self._values = [] - self._load_lines() + if error_file: + self._error_file = pathlib.Path(error_file) + self._load_lines() + else: + self._error_file = None + if not lines: + raise Exception( + "Error, must provide either path to `error_file` or provide `lines`" + ) + self._set_lines(lines) @property def values(self): @@ -25,12 +34,9 @@ def in_multiline_string(self, lineno): def _count_multiline_string_endings_in_line(line): return line.count('"""'), line.count("'''") - def _load_lines(self): - in_file = self._error_file.read_text( - encoding=ni_python_styleguide._utils.DEFAULT_ENCODING - ).splitlines() + def _set_lines(self, lines): current_count = [0, 0] - for line in in_file: + for line in lines: type1, type2 = InMultiLineStringChecker._count_multiline_string_endings_in_line(line) current_count[0] += type1 current_count[1] += type2 @@ -45,3 +51,7 @@ def _load_lines(self): any([part % 2 == 1 for part in current_count]) or code_part_of_line.strip().endswith("\\") ) + + def _load_lines(self): + in_file = self._error_file.read_text(encoding=ni_python_styleguide._utils.DEFAULT_ENCODING) + self._set_lines(in_file.splitlines()) diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py index 2db64dd4..e6877a5d 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py @@ -4,6 +4,12 @@ import pathlib from os import path, access +""" +imports in multiline strings are left alone +>>> import foo, bar +import beef, chicken +""" + class Foo(object): # comment should get moved o = 5 diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py index acabcbb1..4b076d69 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py @@ -8,6 +8,12 @@ import pytest +""" +imports in multiline strings are left alone +>>> import foo, bar +import beef, chicken +""" + class Foo(object): # comment should get moved o = 5 From e5f7dd0f4b53548409a0bf95c111b3bb9b7d6fb6 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 15:05:23 -0500 Subject: [PATCH 24/79] skip if line does not have import for from as a statement --- ni_python_styleguide/_fix.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 7264ddfc..6ae5fe93 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -42,7 +42,13 @@ def _split_imports_line(lines: str, *_, **__): result_parts = [] for line in lines.splitlines(keepends=True): first, _, rest = line.partition(",") - if not rest or "import" not in line: + if not all( + [ + rest, + "import " in line, + line.strip().startswith("import ") or line.strip().startswith("from "), + ] + ): result_parts.append(line) continue prefix, first = " ".join(first.split()[:-1]), first.split()[-1] From 10d0cbebfabff7de4a33eb8b8745e86b100cf73c Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 15:16:41 -0500 Subject: [PATCH 25/79] get it mostly working --- ni_python_styleguide/_fix.py | 6 +- .../more_complicated_example/input.py | 128 +++++++++++++++++ .../more_complicated_example/output.py | 134 ++++++++++++++++++ 3 files changed, 265 insertions(+), 3 deletions(-) create mode 100644 tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py create mode 100644 tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 6ae5fe93..3982ae97 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -71,9 +71,9 @@ def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): print(line) -def _sort_imports(file: pathlib.Path): +def _sort_imports(file: pathlib.Path, app_import_names): raw = file.read_text() - output = isort.code(raw, multi_line_output=3, line_length=1) + output = isort.code(raw, multi_line_output=3, line_length=1, known_first_party=app_import_names) file.write_text(output) @@ -116,7 +116,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa for error in errors_in_file: # humans talk 1-based, enumerate is 0-based line_to_codes_mapping[int(error.line) - 1].add(error.code) - _sort_imports(bad_file) + _sort_imports(bad_file, app_import_names=app_import_names) _format.format(bad_file) _handle_multiple_import_lines(bad_file) _format.format(bad_file) diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py new file mode 100644 index 00000000..121f8cbb --- /dev/null +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py @@ -0,0 +1,128 @@ +from collections import defaultdict +from ni_python_styleguide import _acknowledge_existing_errors, _format, _utils +from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser +from typing import Iterable, List +import fileinput +import isort +import logging +import pathlib + +_module_logger = logging.getLogger(__name__) + + +def _split_imports_line(lines: str, *_, **__): + r"""Split multi-import lines to multiple lines. + + >>> _split_imports_line("import os, collections\n") + 'import os\nimport collections\n' + + >>> _split_imports_line("import os\n") + 'import os\n' + + >>> _split_imports_line("from ni_python_styleguide import" + ... " _acknowledge_existing_errors, _format") + 'from ni_python_styleguide import _acknowledge_existing_errors\nfrom ni_python_styleguide import _format\n' + + >>> _split_imports_line("from ni_python_styleguide import _acknowledge_existing_errors") + 'from ni_python_styleguide import _acknowledge_existing_errors\n' + + >>> _split_imports_line("import os, collections\nimport pathlib") + 'import os\nimport collections\nimport pathlib\n' + + >>> _split_imports_line("import os, collections\nimport pathlib, os") + 'import os\nimport collections\nimport pathlib\nimport os\n' + + >>> _split_imports_line("\n") + '\n' + """ # noqa W505: long lines... + result_parts = [] + for line in lines.splitlines(keepends=True): + first, _, rest = line.partition(",") + if not rest or "import" not in line: + result_parts.append(line) + continue + prefix, first = " ".join(first.split()[:-1]), first.split()[-1] + split_up = [first] + rest.split(",") + result_parts.extend([prefix + " " + part.strip() for part in split_up]) + result = "\n".join(result_parts) + if result.strip(): + return result.rstrip() + "\n" + return result + + +def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): + _module_logger("Splitting import lines in file: %s", file) + hashed_offending_lines = set(offending_lines) + with fileinput.input(file, inplace=True) as py_file: + for line_no, line in py_file: + if line_no in hashed_offending_lines: + print(_split_imports_line(line)) + else: + print(line) + + +def _sort_imports(file: pathlib.Path): + raw = file.read_text() + output = isort.code(raw, multi_line_output=3, line_length=1, sections="STDLIB,THIRDPARTY,FIRSTPARTY") + file.write_text(output) + + +def _handle_multiple_import_lines(bad_file: pathlib.Path): + multiline_string_checker = _utils.string_helpers.InMultiLineStringChecker( + lines=bad_file.read_text(encoding=_utils.DEFAULT_ENCODING).splitlines() + ) + with fileinput.FileInput(files=[str(bad_file)], inplace=True) as f: + for line_no, line in enumerate(f): + working_line = line + if multiline_string_checker.in_multiline_string(line_no + 1): + print(working_line, end="") + continue + working_line = _split_imports_line(working_line) + print(working_line, end="") + + +def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=False, diff=False): + """Fix basic linter errors and format.""" + if aggressive: + raise Exception("--aggressive is not implemented yet") + lint_errors_to_process = _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [pathlib.Path(file_or_dir_) for file_or_dir_ in file_or_dir or "."], + excluded_errors=[], # we fix black errors, so we don't need to filter it. + ) + + lint_errors_by_file = defaultdict(list) + for error in lint_errors_to_process: + lint_errors_by_file[pathlib.Path(error.file)].append(error) + + failed_files = [] + for bad_file, errors_in_file in lint_errors_by_file.items(): + errors_in_file: List[_lint_errors_parser.LintError] + try: + _format.format(bad_file) + line_to_codes_mapping = defaultdict(set) + for error in errors_in_file: + # humans talk 1-based, enumerate is 0-based + line_to_codes_mapping[int(error.line) - 1].add(error.code) + _sort_imports(bad_file) + _format.format(bad_file) + _handle_multiple_import_lines(bad_file) + _format.format(bad_file) + remaining_lint_errors_in_file = ( + _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [bad_file], + excluded_errors=[], # we fix black errors, so we don't need to filter it. + ) + ) + except AttributeError as e: + failed_files.append((bad_file, e)) + if failed_files: + raise Exception( + "Failed to format files:\n" + + "\n".join([f"{file}: {error}" for file, error in failed_files]) + ) diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py new file mode 100644 index 00000000..5310465a --- /dev/null +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py @@ -0,0 +1,134 @@ +import fileinput +import logging +import pathlib +from collections import defaultdict +from typing import Iterable +from typing import List + +import isort +from ni_python_styleguide import _acknowledge_existing_errors +from ni_python_styleguide import _format +from ni_python_styleguide import _utils +from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser + +_module_logger = logging.getLogger(__name__) + + +def _split_imports_line(lines: str, *_, **__): + r"""Split multi-import lines to multiple lines. + + >>> _split_imports_line("import os, collections\n") + 'import os\nimport collections\n' + + >>> _split_imports_line("import os\n") + 'import os\n' + + >>> _split_imports_line("from ni_python_styleguide import" + ... " _acknowledge_existing_errors, _format") + 'from ni_python_styleguide import _acknowledge_existing_errors\nfrom ni_python_styleguide import _format\n' + + >>> _split_imports_line("from ni_python_styleguide import _acknowledge_existing_errors") + 'from ni_python_styleguide import _acknowledge_existing_errors\n' + + >>> _split_imports_line("import os, collections\nimport pathlib") + 'import os\nimport collections\nimport pathlib\n' + + >>> _split_imports_line("import os, collections\nimport pathlib, os") + 'import os\nimport collections\nimport pathlib\nimport os\n' + + >>> _split_imports_line("\n") + '\n' + """ # noqa W505: long lines... + result_parts = [] + for line in lines.splitlines(keepends=True): + first, _, rest = line.partition(",") + if not rest or "import" not in line: + result_parts.append(line) + continue + prefix, first = " ".join(first.split()[:-1]), first.split()[-1] + split_up = [first] + rest.split(",") + result_parts.extend([prefix + " " + part.strip() for part in split_up]) + result = "\n".join(result_parts) + if result.strip(): + return result.rstrip() + "\n" + return result + + +def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): + _module_logger("Splitting import lines in file: %s", file) + hashed_offending_lines = set(offending_lines) + with fileinput.input(file, inplace=True) as py_file: + for line_no, line in py_file: + if line_no in hashed_offending_lines: + print(_split_imports_line(line)) + else: + print(line) + + +def _sort_imports(file: pathlib.Path): + raw = file.read_text() + output = isort.code( + raw, multi_line_output=3, line_length=1, sections="STDLIB,THIRDPARTY,FIRSTPARTY" + ) + file.write_text(output) + + +def _handle_multiple_import_lines(bad_file: pathlib.Path): + multiline_string_checker = _utils.string_helpers.InMultiLineStringChecker( + lines=bad_file.read_text(encoding=_utils.DEFAULT_ENCODING).splitlines() + ) + with fileinput.FileInput(files=[str(bad_file)], inplace=True) as f: + for line_no, line in enumerate(f): + working_line = line + if multiline_string_checker.in_multiline_string(line_no + 1): + print(working_line, end="") + continue + working_line = _split_imports_line(working_line) + print(working_line, end="") + + +def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=False, diff=False): + """Fix basic linter errors and format.""" + if aggressive: + raise Exception("--aggressive is not implemented yet") + lint_errors_to_process = _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [pathlib.Path(file_or_dir_) for file_or_dir_ in file_or_dir or "."], + excluded_errors=[], # we fix black errors, so we don't need to filter it. + ) + + lint_errors_by_file = defaultdict(list) + for error in lint_errors_to_process: + lint_errors_by_file[pathlib.Path(error.file)].append(error) + + failed_files = [] + for bad_file, errors_in_file in lint_errors_by_file.items(): + errors_in_file: List[_lint_errors_parser.LintError] + try: + _format.format(bad_file) + line_to_codes_mapping = defaultdict(set) + for error in errors_in_file: + # humans talk 1-based, enumerate is 0-based + line_to_codes_mapping[int(error.line) - 1].add(error.code) + _sort_imports(bad_file) + _format.format(bad_file) + _handle_multiple_import_lines(bad_file) + _format.format(bad_file) + remaining_lint_errors_in_file = ( + _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [bad_file], + excluded_errors=[], # we fix black errors, so we don't need to filter it. + ) + ) + except AttributeError as e: + failed_files.append((bad_file, e)) + if failed_files: + raise Exception( + "Failed to format files:\n" + + "\n".join([f"{file}: {error}" for file, error in failed_files]) + ) From 34df13ca7ebc4881c61283e24d154cdd88dc41b9 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 15:39:49 -0500 Subject: [PATCH 26/79] add the pyproject.toml file to the test env for getting the app name --- tests/test_cli/test_fix.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_cli/test_fix.py b/tests/test_cli/test_fix.py index e2989eba..298a545e 100644 --- a/tests/test_cli/test_fix.py +++ b/tests/test_cli/test_fix.py @@ -8,10 +8,17 @@ TEST_CASE_DIR = pathlib.Path(__file__).parent.absolute() / "fix_test_cases__snapshots" +@pytest.fixture() +def example_pyproject_file(tmp_path): + in_file = next((parent / "pyproject.toml" for parent in TEST_CASE_DIR.parents if (parent / "pyproject.toml").is_file()), "pyproject.toml") + out_file = tmp_path / "pyproject.toml" + shutil.copyfile(in_file, out_file) + return out_file + @pytest.mark.parametrize("test_dir", [x for x in TEST_CASE_DIR.iterdir() if x.is_dir()]) def test_given_bad_input__fix__produces_expected_output_simple( - test_dir, snapshot, tmp_path, styleguide_command + test_dir, snapshot, tmp_path, styleguide_command, example_pyproject_file ): """Test that suppresion yields expected_output file.""" in_file = test_dir / "input.py" From f8f97f2726df47becaccdaaca3e95b4c925a0b55 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 15:40:54 -0500 Subject: [PATCH 27/79] cleanup the test case --- .../more_complicated_example/input.py | 2 +- .../more_complicated_example/output.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py index 121f8cbb..900df533 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py @@ -63,7 +63,7 @@ def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): def _sort_imports(file: pathlib.Path): raw = file.read_text() - output = isort.code(raw, multi_line_output=3, line_length=1, sections="STDLIB,THIRDPARTY,FIRSTPARTY") + output = isort.code(raw, multi_line_output=3, line_length=1) file.write_text(output) diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py index 5310465a..234f4f9e 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py @@ -67,9 +67,7 @@ def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): def _sort_imports(file: pathlib.Path): raw = file.read_text() - output = isort.code( - raw, multi_line_output=3, line_length=1, sections="STDLIB,THIRDPARTY,FIRSTPARTY" - ) + output = isort.code(raw, multi_line_output=3, line_length=1) file.write_text(output) From da9430d2bb28abb3add479d172d3e5b3a85c418a Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 15:41:37 -0500 Subject: [PATCH 28/79] get app_import_names passing down to sort correctly --- ni_python_styleguide/_fix.py | 7 ++++++- .../more_complicated_example/output.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 3982ae97..4bcbb022 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -73,7 +73,12 @@ def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): def _sort_imports(file: pathlib.Path, app_import_names): raw = file.read_text() - output = isort.code(raw, multi_line_output=3, line_length=1, known_first_party=app_import_names) + output = isort.code( + raw, + multi_line_output=3, + line_length=1, + known_first_party=filter(None, app_import_names.split(",")), + ) file.write_text(output) diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py index 234f4f9e..41152db0 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py @@ -6,6 +6,7 @@ from typing import List import isort + from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _format from ni_python_styleguide import _utils From 96326b92aae0b1f22c0cc1b06fc85f6089f018f2 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 15:42:02 -0500 Subject: [PATCH 29/79] format --- tests/test_cli/test_fix.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_cli/test_fix.py b/tests/test_cli/test_fix.py index 298a545e..3ee82767 100644 --- a/tests/test_cli/test_fix.py +++ b/tests/test_cli/test_fix.py @@ -8,9 +8,17 @@ TEST_CASE_DIR = pathlib.Path(__file__).parent.absolute() / "fix_test_cases__snapshots" + @pytest.fixture() def example_pyproject_file(tmp_path): - in_file = next((parent / "pyproject.toml" for parent in TEST_CASE_DIR.parents if (parent / "pyproject.toml").is_file()), "pyproject.toml") + in_file = next( + ( + parent / "pyproject.toml" + for parent in TEST_CASE_DIR.parents + if (parent / "pyproject.toml").is_file() + ), + "pyproject.toml", + ) out_file = tmp_path / "pyproject.toml" shutil.copyfile(in_file, out_file) return out_file From 4c50922e80625f490cc6aac7ceba69e141a6117f Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 16:19:43 -0500 Subject: [PATCH 30/79] extract handling single file --- .../_acknowledge_existing_errors/__init__.py | 80 ++++++++++++------- 1 file changed, 49 insertions(+), 31 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 99f2a2f6..0227af33 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -75,41 +75,59 @@ def acknowledge_lint_errors( failed_files = [] for bad_file, errors_in_file in lint_errors_by_file.items(): - _suppress_errors_in_file(bad_file, errors_in_file, encoding=_utils.DEFAULT_ENCODING) - - if aggressive: - - # some cases are expected to take up to 4 passes, making this 2x rounded - per_file_format_iteration_limit = 10 - for _ in range(per_file_format_iteration_limit): - # format the files - this may move the suppression off the correct lines - # Note: due to Github pycodestyle#868, we have to format, change, format - # (check if that time made changes) - # - else we wind up with lambda's going un-suppressed - # and/or not re-formatted (to fail later) - _format.format(bad_file) - - # re-apply suppressions on correct lines - remove_auto_suppressions_from_file(bad_file) - current_lint_errors = _get_lint_errors_to_process( - exclude, app_import_names, extend_ignore, [bad_file] - ) - _suppress_errors_in_file( - bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING - ) - - changed = _format.format_check(bad_file) - if not changed: # are we done? - break - else: - failed_files.append( - f"Could not handle suppressions/formatting of file {bad_file} after maximum number of tries ({per_file_format_iteration_limit})" - ) - _module_logger.warning("Max tries reached on %s", bad_file) + try: + suppress_errors_in_single_file( + exclude, + app_import_names, + extend_ignore, + aggressive, + bad_file, + errors_in_file, + ) + except RuntimeError: + failed_files.append(bad_file) if failed_files: raise RuntimeError("Could not handle some files:\n" + "\n\n".join(failed_files) + "\n\n\n") +def suppress_errors_in_single_file( + exclude, app_import_names, extend_ignore, aggressive, bad_file, errors_in_file +): + """Suppress `errors_in_file` in `bad_file`. + + if `aggressive` try repeatedly to suppress and format until stable state is reached. + """ + _suppress_errors_in_file(bad_file, errors_in_file, encoding=_utils.DEFAULT_ENCODING) + + if aggressive: + # some cases are expected to take up to 4 passes, making this 2x rounded + per_file_format_iteration_limit = 10 + for _ in range(per_file_format_iteration_limit): + # format the files - this may move the suppression off the correct lines + # Note: due to Github pycodestyle#868, we have to format, change, format + # (check if that time made changes) + # - else we wind up with lambda's going un-suppressed + # and/or not re-formatted (to fail later) + _format.format(bad_file) + + # re-apply suppressions on correct lines + remove_auto_suppressions_from_file(bad_file) + current_lint_errors = _get_lint_errors_to_process( + exclude, app_import_names, extend_ignore, [bad_file] + ) + _suppress_errors_in_file( + bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING + ) + + changed = _format.format_check(bad_file) + if not changed: # are we done? + break + else: + raise RuntimeError( + f"Could not suppress/format {bad_file} after {per_file_format_iteration_limit} tries" + ) + + def remove_auto_suppressions_from_file(file: pathlib.Path): """Remove auto-suppressions from file.""" lines = file.read_text(encoding=_utils.DEFAULT_ENCODING).splitlines() From 496c8185a19000fcb70e89c56060f6f820dff913 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 16:21:07 -0500 Subject: [PATCH 31/79] setup for --aggressive --- ni_python_styleguide/_fix.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 4bcbb022..f95aacdd 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -98,13 +98,23 @@ def _handle_multiple_import_lines(bad_file: pathlib.Path): def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=False, diff=False): """Fix basic linter errors and format.""" + file_or_dir = file_or_dir or ["."] if aggressive: - raise Exception("--aggressive is not implemented yet") + all_files = [] + for file_or_dir_ in file_or_dir: + if pathlib.Path(file_or_dir).is_dir(): + all_files.extend(_utils.git.get_tracked_files(file_or_dir, filter="*.py")) + else: + all_files.append(pathlib.Path(file_or_dir)) + for file in all_files: + if not file.is_file(): # doesn't really exist... + continue + _acknowledge_existing_errors.remove_auto_suppressions_from_file(file) lint_errors_to_process = _acknowledge_existing_errors._get_lint_errors_to_process( exclude, app_import_names, extend_ignore, - [pathlib.Path(file_or_dir_) for file_or_dir_ in file_or_dir or "."], + [pathlib.Path(file_or_dir_) for file_or_dir_ in file_or_dir], excluded_errors=[], # we fix black errors, so we don't need to filter it. ) @@ -131,9 +141,17 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa app_import_names, extend_ignore, [bad_file], - excluded_errors=[], # we fix black errors, so we don't need to filter it. ) ) + if remaining_lint_errors_in_file: + _acknowledge_existing_errors.suppress_errors_in_single_file( + exclude=exclude, + app_import_names=app_import_names, + extend_ignore=extend_ignore, + aggressive=aggressive, + bad_file=bad_file, + errors_in_file=remaining_lint_errors_in_file, + ) except AttributeError as e: failed_files.append((bad_file, e)) if failed_files: From f5de92d45d0bb78849aebe5b05a6c1ee7bc080dc Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 16:21:35 -0500 Subject: [PATCH 32/79] make the linter happy --- tests/test_cli/test_fix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cli/test_fix.py b/tests/test_cli/test_fix.py index 3ee82767..84bbe541 100644 --- a/tests/test_cli/test_fix.py +++ b/tests/test_cli/test_fix.py @@ -11,6 +11,7 @@ @pytest.fixture() def example_pyproject_file(tmp_path): + """Provide the current projects pyproject.toml file to tests as an example.""" in_file = next( ( parent / "pyproject.toml" From 1b4320960accfd1317ec3c9cb39ecf600ccb2008 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 16:23:59 -0500 Subject: [PATCH 33/79] we can dog-food ourselves! --- ni_python_styleguide/_fix.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index f95aacdd..15db39c5 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -2,14 +2,15 @@ import logging import pathlib from collections import defaultdict -from typing import Iterable, List +from typing import Iterable +from typing import List import isort -from ni_python_styleguide import _acknowledge_existing_errors, _format -from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser +from ni_python_styleguide import _acknowledge_existing_errors +from ni_python_styleguide import _format from ni_python_styleguide import _utils - +from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser _module_logger = logging.getLogger(__name__) From ab7ea2013c8a5c6414312c63961805af12b1d449 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 16:25:15 -0500 Subject: [PATCH 34/79] only suppress remaining if --aggressive --- ni_python_styleguide/_fix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 15db39c5..0667b9fd 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -144,7 +144,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa [bad_file], ) ) - if remaining_lint_errors_in_file: + if remaining_lint_errors_in_file and aggressive: _acknowledge_existing_errors.suppress_errors_in_single_file( exclude=exclude, app_import_names=app_import_names, From e6d2ea46ae5c0b53fcf6b085a172ce5c0a2381f9 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 16:51:54 -0500 Subject: [PATCH 35/79] get snapshots to where linter is happy with output without --aggressive --- .../basic_example/input.py | 12 ++++++++++-- .../basic_example/output.py | 11 +++++++++-- .../more_complicated_example/input.py | 3 ++- .../more_complicated_example/output.py | 15 +++++++-------- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py index e6877a5d..324661bb 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py @@ -12,6 +12,8 @@ class Foo(object): # comment should get moved + """Provide dummy code.""" + o = 5 i = 1 @@ -19,7 +21,13 @@ def __init__(self, o: Iterable[int]) -> None: """Test class.""" this_method = 1 - seperator = path.sep + self.seperator = path.sep * this_method + - has_a_lot_of_empty_lines = 2 pass + + @pytest.mark.skip(reason="not a test") + def method_to_use_everything(self, x: Iterable[List[Hashable]], p: pathlib.Path): + "Strange method." + x = access(p) + return x diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py index 4b076d69..d9ce0996 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py @@ -16,6 +16,8 @@ class Foo(object): # comment should get moved + """Provide dummy code.""" + o = 5 i = 1 @@ -23,7 +25,12 @@ def __init__(self, o: Iterable[int]) -> None: """Test class.""" this_method = 1 - seperator = path.sep + self.seperator = path.sep * this_method - has_a_lot_of_empty_lines = 2 pass + + @pytest.mark.skip(reason="not a test") + def method_to_use_everything(self, x: Iterable[List[Hashable]], p: pathlib.Path): + "Strange method." + x = access(p) + return x diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py index 900df533..ad43995e 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py @@ -1,3 +1,4 @@ +"""Provide a more complex example with some corner cases.""" from collections import defaultdict from ni_python_styleguide import _acknowledge_existing_errors, _format, _utils from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser @@ -110,7 +111,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa _format.format(bad_file) _handle_multiple_import_lines(bad_file) _format.format(bad_file) - remaining_lint_errors_in_file = ( + _ = ( _acknowledge_existing_errors._get_lint_errors_to_process( exclude, app_import_names, diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py index 41152db0..3b9f854a 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py @@ -1,3 +1,4 @@ +"""Provide a more complex example with some corner cases.""" import fileinput import logging import pathlib @@ -115,14 +116,12 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa _format.format(bad_file) _handle_multiple_import_lines(bad_file) _format.format(bad_file) - remaining_lint_errors_in_file = ( - _acknowledge_existing_errors._get_lint_errors_to_process( - exclude, - app_import_names, - extend_ignore, - [bad_file], - excluded_errors=[], # we fix black errors, so we don't need to filter it. - ) + _ = _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [bad_file], + excluded_errors=[], # we fix black errors, so we don't need to filter it. ) except AttributeError as e: failed_files.append((bad_file, e)) From e171e2a5145ecff852870077d16c462b99c5c5f7 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 16:59:10 -0500 Subject: [PATCH 36/79] fix var reference --- ni_python_styleguide/_fix.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 0667b9fd..5c6c27ab 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -103,10 +103,10 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa if aggressive: all_files = [] for file_or_dir_ in file_or_dir: - if pathlib.Path(file_or_dir).is_dir(): - all_files.extend(_utils.git.get_tracked_files(file_or_dir, filter="*.py")) + if pathlib.Path(file_or_dir_).is_dir(): + all_files.extend(_utils.git.get_tracked_files(file_or_dir_, filter="*.py")) else: - all_files.append(pathlib.Path(file_or_dir)) + all_files.append(pathlib.Path(file_or_dir_)) for file in all_files: if not file.is_file(): # doesn't really exist... continue From bfe916e6554034137f6704f87915e3347a0169b6 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 16:59:21 -0500 Subject: [PATCH 37/79] add --aggressive cases --- .../basic_example/output__aggressive.py | 36 +++++ .../output__aggressive.py | 132 ++++++++++++++++++ tests/test_cli/test_fix.py | 21 ++- 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 tests/test_cli/fix_test_cases__snapshots/basic_example/output__aggressive.py create mode 100644 tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/output__aggressive.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/output__aggressive.py new file mode 100644 index 00000000..d9ce0996 --- /dev/null +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/output__aggressive.py @@ -0,0 +1,36 @@ +"""Provide example cases of imports that need sorting and a file that needs formatted.""" +import pathlib +from os import access +from os import path +from typing import Hashable +from typing import Iterable +from typing import List + +import pytest + +""" +imports in multiline strings are left alone +>>> import foo, bar +import beef, chicken +""" + + +class Foo(object): # comment should get moved + """Provide dummy code.""" + + o = 5 + i = 1 + + def __init__(self, o: Iterable[int]) -> None: + """Test class.""" + this_method = 1 + + self.seperator = path.sep * this_method + + pass + + @pytest.mark.skip(reason="not a test") + def method_to_use_everything(self, x: Iterable[List[Hashable]], p: pathlib.Path): + "Strange method." + x = access(p) + return x diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py new file mode 100644 index 00000000..3b9f854a --- /dev/null +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py @@ -0,0 +1,132 @@ +"""Provide a more complex example with some corner cases.""" +import fileinput +import logging +import pathlib +from collections import defaultdict +from typing import Iterable +from typing import List + +import isort + +from ni_python_styleguide import _acknowledge_existing_errors +from ni_python_styleguide import _format +from ni_python_styleguide import _utils +from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser + +_module_logger = logging.getLogger(__name__) + + +def _split_imports_line(lines: str, *_, **__): + r"""Split multi-import lines to multiple lines. + + >>> _split_imports_line("import os, collections\n") + 'import os\nimport collections\n' + + >>> _split_imports_line("import os\n") + 'import os\n' + + >>> _split_imports_line("from ni_python_styleguide import" + ... " _acknowledge_existing_errors, _format") + 'from ni_python_styleguide import _acknowledge_existing_errors\nfrom ni_python_styleguide import _format\n' + + >>> _split_imports_line("from ni_python_styleguide import _acknowledge_existing_errors") + 'from ni_python_styleguide import _acknowledge_existing_errors\n' + + >>> _split_imports_line("import os, collections\nimport pathlib") + 'import os\nimport collections\nimport pathlib\n' + + >>> _split_imports_line("import os, collections\nimport pathlib, os") + 'import os\nimport collections\nimport pathlib\nimport os\n' + + >>> _split_imports_line("\n") + '\n' + """ # noqa W505: long lines... + result_parts = [] + for line in lines.splitlines(keepends=True): + first, _, rest = line.partition(",") + if not rest or "import" not in line: + result_parts.append(line) + continue + prefix, first = " ".join(first.split()[:-1]), first.split()[-1] + split_up = [first] + rest.split(",") + result_parts.extend([prefix + " " + part.strip() for part in split_up]) + result = "\n".join(result_parts) + if result.strip(): + return result.rstrip() + "\n" + return result + + +def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): + _module_logger("Splitting import lines in file: %s", file) + hashed_offending_lines = set(offending_lines) + with fileinput.input(file, inplace=True) as py_file: + for line_no, line in py_file: + if line_no in hashed_offending_lines: + print(_split_imports_line(line)) + else: + print(line) + + +def _sort_imports(file: pathlib.Path): + raw = file.read_text() + output = isort.code(raw, multi_line_output=3, line_length=1) + file.write_text(output) + + +def _handle_multiple_import_lines(bad_file: pathlib.Path): + multiline_string_checker = _utils.string_helpers.InMultiLineStringChecker( + lines=bad_file.read_text(encoding=_utils.DEFAULT_ENCODING).splitlines() + ) + with fileinput.FileInput(files=[str(bad_file)], inplace=True) as f: + for line_no, line in enumerate(f): + working_line = line + if multiline_string_checker.in_multiline_string(line_no + 1): + print(working_line, end="") + continue + working_line = _split_imports_line(working_line) + print(working_line, end="") + + +def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=False, diff=False): + """Fix basic linter errors and format.""" + if aggressive: + raise Exception("--aggressive is not implemented yet") + lint_errors_to_process = _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [pathlib.Path(file_or_dir_) for file_or_dir_ in file_or_dir or "."], + excluded_errors=[], # we fix black errors, so we don't need to filter it. + ) + + lint_errors_by_file = defaultdict(list) + for error in lint_errors_to_process: + lint_errors_by_file[pathlib.Path(error.file)].append(error) + + failed_files = [] + for bad_file, errors_in_file in lint_errors_by_file.items(): + errors_in_file: List[_lint_errors_parser.LintError] + try: + _format.format(bad_file) + line_to_codes_mapping = defaultdict(set) + for error in errors_in_file: + # humans talk 1-based, enumerate is 0-based + line_to_codes_mapping[int(error.line) - 1].add(error.code) + _sort_imports(bad_file) + _format.format(bad_file) + _handle_multiple_import_lines(bad_file) + _format.format(bad_file) + _ = _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [bad_file], + excluded_errors=[], # we fix black errors, so we don't need to filter it. + ) + except AttributeError as e: + failed_files.append((bad_file, e)) + if failed_files: + raise Exception( + "Failed to format files:\n" + + "\n".join([f"{file}: {error}" for file, error in failed_files]) + ) diff --git a/tests/test_cli/test_fix.py b/tests/test_cli/test_fix.py index 84bbe541..22ec2cc9 100644 --- a/tests/test_cli/test_fix.py +++ b/tests/test_cli/test_fix.py @@ -5,7 +5,6 @@ import pytest - TEST_CASE_DIR = pathlib.Path(__file__).parent.absolute() / "fix_test_cases__snapshots" @@ -40,3 +39,23 @@ def test_given_bad_input__fix__produces_expected_output_simple( result = test_file.read_text(encoding="UTF-8") snapshot.snapshot_dir = test_dir snapshot.assert_match(result, "output.py") + + +@pytest.mark.parametrize( # noqa F811: redefinition of unused 'test_given_bad_input__fix__produces_expected_output_simple' from line 27 (auto-generated noqa) + "test_dir", [x for x in TEST_CASE_DIR.iterdir() if x.is_dir()] +) +def test_given_bad_input__fix__produces_expected_output_simple( + test_dir, snapshot, tmp_path, styleguide_command, example_pyproject_file +): + """Test that suppresion yields expected_output file.""" + in_file = test_dir / "input.py" + test_file = tmp_path / "input.py" + shutil.copyfile(in_file, test_file) + + output = styleguide_command(command="fix") + output = styleguide_command(command="fix", command_args=["--aggressive"]) + + assert output.exit_code in (True, 0), f"Error in running:\n{output}" + result = test_file.read_text(encoding="UTF-8") + snapshot.snapshot_dir = test_dir + snapshot.assert_match(result, "output__aggressive.py") From ec5f6871a0b9c461c6ab445d99f24404ccadaa11 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 17:53:27 -0500 Subject: [PATCH 38/79] format --- ni_python_styleguide/_utils/git.py | 10 +++++++++- .../more_complicated_example/input.py | 14 ++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ni_python_styleguide/_utils/git.py b/ni_python_styleguide/_utils/git.py index 409f3005..5089531a 100644 --- a/ni_python_styleguide/_utils/git.py +++ b/ni_python_styleguide/_utils/git.py @@ -2,7 +2,15 @@ import os import pathlib -from git import repo +from git import GitError, repo + + +def find_repo_root(cwd: os.PathLike): + try: + _repo = repo.Repo(str(cwd), search_parent_directories=True) + except GitError: + return None + return _repo.git_dir def get_tracked_files(cwd: os.PathLike, *_, branch: str = "main", filter: str = "*.*"): diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py index ad43995e..acfcfa5d 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py @@ -111,14 +111,12 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa _format.format(bad_file) _handle_multiple_import_lines(bad_file) _format.format(bad_file) - _ = ( - _acknowledge_existing_errors._get_lint_errors_to_process( - exclude, - app_import_names, - extend_ignore, - [bad_file], - excluded_errors=[], # we fix black errors, so we don't need to filter it. - ) + _ = _acknowledge_existing_errors._get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [bad_file], + excluded_errors=[], # we fix black errors, so we don't need to filter it. ) except AttributeError as e: failed_files.append((bad_file, e)) From 5073deee037323296e81b95a7db363c5c1ef1364 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 18:19:43 -0500 Subject: [PATCH 39/79] remove unused code --- ni_python_styleguide/_fix.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 5c6c27ab..06343ff2 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -61,17 +61,6 @@ def _split_imports_line(lines: str, *_, **__): return result -def _split_imports(file: pathlib.Path, offending_lines: Iterable[int]): - _module_logger("Splitting import lines in file: %s", file) - hashed_offending_lines = set(offending_lines) - with fileinput.input(file, inplace=True) as py_file: - for line_no, line in py_file: - if line_no in hashed_offending_lines: - print(_split_imports_line(line)) - else: - print(line) - - def _sort_imports(file: pathlib.Path, app_import_names): raw = file.read_text() output = isort.code( From fbf349231190c16a0441ce2a85b2b1d170484391 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 18:20:30 -0500 Subject: [PATCH 40/79] handle if dir is not in a git repo --- ni_python_styleguide/_fix.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 06343ff2..7b4213b7 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -92,10 +92,14 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa if aggressive: all_files = [] for file_or_dir_ in file_or_dir: - if pathlib.Path(file_or_dir_).is_dir(): - all_files.extend(_utils.git.get_tracked_files(file_or_dir_, filter="*.py")) + file_path = pathlib.Path(file_or_dir_) + if file_path.is_dir(): + if _utils.git.find_repo_root(file_or_dir_): + all_files.extend(_utils.git.get_tracked_files(file_or_dir_, filter="*.py")) + else: + all_files.extend(file_path.rglob("*.py")) else: - all_files.append(pathlib.Path(file_or_dir_)) + all_files.append(file_path) for file in all_files: if not file.is_file(): # doesn't really exist... continue From c444813f86e38481053fe0e9ef9ad7295273054e Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 18:21:16 -0500 Subject: [PATCH 41/79] use the working tree dir --- ni_python_styleguide/_utils/git.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ni_python_styleguide/_utils/git.py b/ni_python_styleguide/_utils/git.py index 5089531a..225a9d7f 100644 --- a/ni_python_styleguide/_utils/git.py +++ b/ni_python_styleguide/_utils/git.py @@ -6,11 +6,12 @@ def find_repo_root(cwd: os.PathLike): + """Find the Git repo path or return None.""" try: _repo = repo.Repo(str(cwd), search_parent_directories=True) except GitError: return None - return _repo.git_dir + return _repo.working_tree_dir def get_tracked_files(cwd: os.PathLike, *_, branch: str = "main", filter: str = "*.*"): From a97127e43a80e689459a9c356bcc0922947a288f Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 18:21:52 -0500 Subject: [PATCH 42/79] make black condense isort's output even with long comment after --- ni_python_styleguide/_fix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 7b4213b7..05dc77bb 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -126,7 +126,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa # humans talk 1-based, enumerate is 0-based line_to_codes_mapping[int(error.line) - 1].add(error.code) _sort_imports(bad_file, app_import_names=app_import_names) - _format.format(bad_file) + _format.format(bad_file, "--line-length=300") # condense any split lines _handle_multiple_import_lines(bad_file) _format.format(bad_file) remaining_lint_errors_in_file = ( From 1db0982b307ba52e1d8e2c934ef35ef179c3637a Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 18:22:09 -0500 Subject: [PATCH 43/79] handle comments --- ni_python_styleguide/_fix.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 05dc77bb..64bd96bb 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -2,7 +2,6 @@ import logging import pathlib from collections import defaultdict -from typing import Iterable from typing import List import isort @@ -42,20 +41,25 @@ def _split_imports_line(lines: str, *_, **__): """ # noqa W505: long lines... result_parts = [] for line in lines.splitlines(keepends=True): - first, _, rest = line.partition(",") + code_portion_of_line, *non_code = line.split("#", maxsplit=1) + first, _, rest = code_portion_of_line.partition(",") if not all( [ rest, - "import " in line, - line.strip().startswith("import ") or line.strip().startswith("from "), + "import " in code_portion_of_line, + code_portion_of_line.strip().startswith("import ") + or code_portion_of_line.strip().startswith("from "), ] ): - result_parts.append(line) + result_parts.append(code_portion_of_line) continue prefix, first = " ".join(first.split()[:-1]), first.split()[-1] split_up = [first] + rest.split(",") result_parts.extend([prefix + " " + part.strip() for part in split_up]) - result = "\n".join(result_parts) + suffix = "" + if non_code: + suffix = "#" + "".join(non_code) + result = "\n".join(result_parts) + suffix if result.strip(): return result.rstrip() + "\n" return result From 7bf06210f2350f3feef5152ec0aad6e19046244f Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 18:22:24 -0500 Subject: [PATCH 44/79] cleanup tests --- tests/test_cli/test_fix.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_cli/test_fix.py b/tests/test_cli/test_fix.py index 22ec2cc9..e15a995e 100644 --- a/tests/test_cli/test_fix.py +++ b/tests/test_cli/test_fix.py @@ -7,6 +7,8 @@ TEST_CASE_DIR = pathlib.Path(__file__).parent.absolute() / "fix_test_cases__snapshots" +TEST_CASES = [x for x in TEST_CASE_DIR.iterdir() if x.is_dir() and (x / "input.py").is_file()] + @pytest.fixture() def example_pyproject_file(tmp_path): @@ -24,7 +26,7 @@ def example_pyproject_file(tmp_path): return out_file -@pytest.mark.parametrize("test_dir", [x for x in TEST_CASE_DIR.iterdir() if x.is_dir()]) +@pytest.mark.parametrize("test_dir", TEST_CASES) def test_given_bad_input__fix__produces_expected_output_simple( test_dir, snapshot, tmp_path, styleguide_command, example_pyproject_file ): @@ -41,10 +43,8 @@ def test_given_bad_input__fix__produces_expected_output_simple( snapshot.assert_match(result, "output.py") -@pytest.mark.parametrize( # noqa F811: redefinition of unused 'test_given_bad_input__fix__produces_expected_output_simple' from line 27 (auto-generated noqa) - "test_dir", [x for x in TEST_CASE_DIR.iterdir() if x.is_dir()] -) -def test_given_bad_input__fix__produces_expected_output_simple( +@pytest.mark.parametrize("test_dir", TEST_CASES) +def test_given_bad_input__fix__produces_expected_output_aggressive( test_dir, snapshot, tmp_path, styleguide_command, example_pyproject_file ): """Test that suppresion yields expected_output file.""" @@ -52,7 +52,6 @@ def test_given_bad_input__fix__produces_expected_output_simple( test_file = tmp_path / "input.py" shutil.copyfile(in_file, test_file) - output = styleguide_command(command="fix") output = styleguide_command(command="fix", command_args=["--aggressive"]) assert output.exit_code in (True, 0), f"Error in running:\n{output}" From 450a7fc58f11ea72dfe6755106d30d95b73d4d6c Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 18:22:47 -0500 Subject: [PATCH 45/79] add example case of previously suppressed that is no longer needed --- .../fix_test_cases__snapshots/basic_example/input.py | 7 +++++-- .../fix_test_cases__snapshots/basic_example/output.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py index 324661bb..25372d48 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/input.py @@ -1,5 +1,9 @@ """Provide example cases of imports that need sorting and a file that needs formatted.""" -from typing import Iterable, List, Hashable +from typing import ( + Iterable, + List, + Hashable, # noqa F401: un-used import comment that is actually used, should get removed in --aggressive (auto-generated noqa) +) import pytest import pathlib from os import path, access @@ -23,7 +27,6 @@ def __init__(self, o: Iterable[int]) -> None: self.seperator = path.sep * this_method - pass @pytest.mark.skip(reason="not a test") diff --git a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py index d9ce0996..139568e0 100644 --- a/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/basic_example/output.py @@ -2,7 +2,9 @@ import pathlib from os import access from os import path -from typing import Hashable +from typing import ( + Hashable, +) # noqa F401: un-used import comment that is actually used, should get removed in --aggressive (auto-generated noqa) from typing import Iterable from typing import List From 8b9beb384390e3e8a660bf17fba07f90d3656bcd Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 18:27:43 -0500 Subject: [PATCH 46/79] revert testing change --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 21c29b86..3f5c761d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,8 +64,8 @@ exclude = ''' ''' [tool.pytest.ini_options] -# addopts = "--doctest-modules" -# norecursedirs = "*__snapshots" +addopts = "--doctest-modules" +norecursedirs = "*__snapshots" [tool.ni-python-styleguide] extend_exclude = "*__snapshots/*/*input.py" From 4a986ad9dc6846e0094991f432e03b69c57e4afe Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 18:36:03 -0500 Subject: [PATCH 47/79] =?UTF-8?q?=E2=80=BE\=5F(=E3=83=84)=5F/=E2=80=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- poetry.lock | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5a9cf42b..9b7c2aff 100644 --- a/poetry.lock +++ b/poetry.lock @@ -469,7 +469,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.2" -content-hash = "6c017bbb5c1ee35ee76ad4c60e075632bb6d6e811d71250de8ea47f93491c37e" +content-hash = "a0bca0f47291a96c5c5a85a21ae88305bd1be635293db0b15ff9992df45562dd" [metadata.files] atomicwrites = [ diff --git a/pyproject.toml b/pyproject.toml index 3f5c761d..59462e41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ GitPython = [ {version="<=3.0", python=">=3.6,<3.7"} ] gitdb2 = {version="^2.0.0", python=">=3.6,<3.7"} # for py 3.6, pin this down -isort = {version=">=5.10", python=">=3.6.2,<4"} +isort = {version=">=5.10", python=">=3.6,<4"} # flake8 plugins should be listed here (in alphabetical order) From 243ed8aeca11f03e1c6c4494830258a9a40473b8 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 31 Mar 2022 21:11:19 -0500 Subject: [PATCH 48/79] try also locking gitdb --- poetry.lock | 49 ++++++++++++++++++++++++++++++------------------- pyproject.toml | 10 +++++++--- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9b7c2aff..e70590b7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -72,14 +72,6 @@ category = "main" optional = false python-versions = ">=3.6, <3.7" -[[package]] -name = "ddt" -version = "1.4.4" -description = "Data-Driven/Decorated Tests" -category = "main" -optional = false -python-versions = "*" - [[package]] name = "flake8" version = "3.9.2" @@ -152,6 +144,17 @@ python-versions = ">=3.6" [package.dependencies] smmap = ">=3.0.1,<6" +[[package]] +name = "gitdb" +version = "0.6.4" +description = "Git Object Database" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +smmap = ">=0.8.5" + [[package]] name = "gitdb2" version = "2.0.6" @@ -165,15 +168,24 @@ smmap2 = ">=2.0.0" [[package]] name = "gitpython" -version = "3.0.0" +version = "2.1.15" +description = "Python Git Library" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +gitdb2 = ">=2,<3" + +[[package]] +name = "gitpython" +version = "3.0.7" description = "Python Git Library" category = "main" optional = false -python-versions = ">=3.0, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.4" [package.dependencies] -ddt = ">=1.1.1" -gitdb = ">=0.6.4" gitdb2 = ">=2.0.0" [[package]] @@ -469,7 +481,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.2" -content-hash = "a0bca0f47291a96c5c5a85a21ae88305bd1be635293db0b15ff9992df45562dd" +content-hash = "011ef16fc877c3bfa885654632f1161009219024045b868c554d5e155ab323bc" [metadata.files] atomicwrites = [ @@ -517,10 +529,6 @@ dataclasses = [ {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, ] -ddt = [ - {file = "ddt-1.4.4-py2.py3-none-any.whl", hash = "sha256:439f163a149d47761bcb11fefdd7327c7bc9262fa1c53f5929f643a6faa6d454"}, - {file = "ddt-1.4.4.tar.gz", hash = "sha256:8de39a69730442dc835e4d33f9d2e33043ff91151c8d18086959ee556febb9f8"}, -] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, @@ -544,14 +552,17 @@ flake8-polyfill = [ gitdb = [ {file = "gitdb-4.0.9-py3-none-any.whl", hash = "sha256:8033ad4e853066ba6ca92050b9df2f89301b8fc8bf7e9324d412a63f8bf1a8fd"}, {file = "gitdb-4.0.9.tar.gz", hash = "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa"}, + {file = "gitdb-0.6.4.tar.gz", hash = "sha256:a3ebbc27be035a2e874ed904df516e35f4a29a778a764385de09de9e0f139658"}, ] gitdb2 = [ {file = "gitdb2-2.0.6-py2.py3-none-any.whl", hash = "sha256:96bbb507d765a7f51eb802554a9cfe194a174582f772e0d89f4e87288c288b7b"}, {file = "gitdb2-2.0.6.tar.gz", hash = "sha256:1b6df1433567a51a4a9c1a5a0de977aa351a405cc56d7d35f3388bad1f630350"}, ] gitpython = [ - {file = "GitPython-3.0.0-py3-none-any.whl", hash = "sha256:556b64796c5e268b35e3b431a429e813ad54aa178e1baaec2a9ba82e8575a89e"}, - {file = "GitPython-3.0.0.tar.gz", hash = "sha256:629867ebf609cef21bb9d849039e281e25963fb7d714a2f6bacc1ecce4800293"}, + {file = "GitPython-2.1.15-py2.py3-none-any.whl", hash = "sha256:23b4de99c5fc1564701301cead04e16aa3e47019034668327206d1b52e1e54f7"}, + {file = "GitPython-2.1.15.tar.gz", hash = "sha256:838c38d882a97b2fd146dbd4eb9baaf857cb65ebaa64e4c06fca48aaa491e1c6"}, + {file = "GitPython-3.0.7-py3-none-any.whl", hash = "sha256:99c77677f31f255e130f3fed4c8e0eebb35f1a09df98ff965fff6774f71688cf"}, + {file = "GitPython-3.0.7.tar.gz", hash = "sha256:99cd0403cecd8a13b95d2e045b9fcaa7837137fcc5ec3105f2c413305d82c143"}, {file = "GitPython-3.1.27-py3-none-any.whl", hash = "sha256:5b68b000463593e05ff2b261acff0ff0972df8ab1b70d3cdbd41b546c8b8fc3d"}, {file = "GitPython-3.1.27.tar.gz", hash = "sha256:1c885ce809e8ba2d88a29befeb385fcea06338d3640712b59ca623c220bb5704"}, ] diff --git a/pyproject.toml b/pyproject.toml index 59462e41..b214ef24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,10 +23,14 @@ black = ">=21.10b0" click = ">=7.1.2" toml = ">=0.10.1" GitPython = [ - {version=">=3.1", python=">=3.7"}, - {version="<=3.0", python=">=3.6,<3.7"} + {version=">=3.0", python=">=3.7"}, + {version="<3.0", python=">=3.6,<3.7"} +] +gitdb2 = {version="^2.0.6", python=">=3.6,<3.7"} # for py 3.6, pin this down +gitdb = [ + {version="^0.6", python=">=3.6,<3.7"}, + {version="^4.0.1", python=">=3.7"}, ] -gitdb2 = {version="^2.0.0", python=">=3.6,<3.7"} # for py 3.6, pin this down isort = {version=">=5.10", python=">=3.6,<4"} From b76d55700b32cbce75231ef420f6d3d13e9de4ec Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Fri, 8 Apr 2022 10:57:42 -0500 Subject: [PATCH 49/79] don't use git --- ni_python_styleguide/_utils/git.py | 25 ------- poetry.lock | 113 +---------------------------- pyproject.toml | 9 --- 3 files changed, 1 insertion(+), 146 deletions(-) delete mode 100644 ni_python_styleguide/_utils/git.py diff --git a/ni_python_styleguide/_utils/git.py b/ni_python_styleguide/_utils/git.py deleted file mode 100644 index 225a9d7f..00000000 --- a/ni_python_styleguide/_utils/git.py +++ /dev/null @@ -1,25 +0,0 @@ -import fnmatch -import os -import pathlib - -from git import GitError, repo - - -def find_repo_root(cwd: os.PathLike): - """Find the Git repo path or return None.""" - try: - _repo = repo.Repo(str(cwd), search_parent_directories=True) - except GitError: - return None - return _repo.working_tree_dir - - -def get_tracked_files(cwd: os.PathLike, *_, branch: str = "main", filter: str = "*.*"): - """Get all Git tracked files under cwd.""" - cwd = pathlib.Path(cwd) - _repo = repo.Repo(str(cwd), search_parent_directories=True) - repo_root = pathlib.Path(_repo.working_dir) - output = _repo.git.ls_tree(str(cwd), r=branch, name_only=True).splitlines() - for line in output: - if line.startswith(str(cwd)) and fnmatch.fnmatch(line, filter): - yield repo_root / line diff --git a/poetry.lock b/poetry.lock index e70590b7..cd2268f4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -133,73 +133,6 @@ python-versions = "*" [package.dependencies] flake8 = "*" -[[package]] -name = "gitdb" -version = "4.0.9" -description = "Git Object Database" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -smmap = ">=3.0.1,<6" - -[[package]] -name = "gitdb" -version = "0.6.4" -description = "Git Object Database" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -smmap = ">=0.8.5" - -[[package]] -name = "gitdb2" -version = "2.0.6" -description = "Git Object Database" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -smmap2 = ">=2.0.0" - -[[package]] -name = "gitpython" -version = "2.1.15" -description = "Python Git Library" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -gitdb2 = ">=2,<3" - -[[package]] -name = "gitpython" -version = "3.0.7" -description = "Python Git Library" -category = "main" -optional = false -python-versions = ">=3.4" - -[package.dependencies] -gitdb2 = ">=2.0.0" - -[[package]] -name = "gitpython" -version = "3.1.27" -description = "GitPython is a python library used to interact with Git repositories" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -gitdb = ">=4.0.1,<5" -typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} - [[package]] name = "importlib-metadata" version = "4.8.3" @@ -407,25 +340,6 @@ python-versions = ">=3.5" [package.dependencies] pytest = ">=3.0.0" -[[package]] -name = "smmap" -version = "5.0.0" -description = "A pure Python implementation of a sliding window memory map manager" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "smmap2" -version = "3.0.1" -description = "A mirror package for smmap" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -smmap = ">=3.0.1" - [[package]] name = "snowballstemmer" version = "2.2.0" @@ -481,7 +395,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = ">=3.6.2" -content-hash = "011ef16fc877c3bfa885654632f1161009219024045b868c554d5e155ab323bc" +content-hash = "19c3ef66a1d276e07a3ee11c76993bba231aa23d1f57a5bbbd7f685a58212dca" [metadata.files] atomicwrites = [ @@ -549,23 +463,6 @@ flake8-polyfill = [ {file = "flake8-polyfill-1.0.2.tar.gz", hash = "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda"}, {file = "flake8_polyfill-1.0.2-py2.py3-none-any.whl", hash = "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9"}, ] -gitdb = [ - {file = "gitdb-4.0.9-py3-none-any.whl", hash = "sha256:8033ad4e853066ba6ca92050b9df2f89301b8fc8bf7e9324d412a63f8bf1a8fd"}, - {file = "gitdb-4.0.9.tar.gz", hash = "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa"}, - {file = "gitdb-0.6.4.tar.gz", hash = "sha256:a3ebbc27be035a2e874ed904df516e35f4a29a778a764385de09de9e0f139658"}, -] -gitdb2 = [ - {file = "gitdb2-2.0.6-py2.py3-none-any.whl", hash = "sha256:96bbb507d765a7f51eb802554a9cfe194a174582f772e0d89f4e87288c288b7b"}, - {file = "gitdb2-2.0.6.tar.gz", hash = "sha256:1b6df1433567a51a4a9c1a5a0de977aa351a405cc56d7d35f3388bad1f630350"}, -] -gitpython = [ - {file = "GitPython-2.1.15-py2.py3-none-any.whl", hash = "sha256:23b4de99c5fc1564701301cead04e16aa3e47019034668327206d1b52e1e54f7"}, - {file = "GitPython-2.1.15.tar.gz", hash = "sha256:838c38d882a97b2fd146dbd4eb9baaf857cb65ebaa64e4c06fca48aaa491e1c6"}, - {file = "GitPython-3.0.7-py3-none-any.whl", hash = "sha256:99c77677f31f255e130f3fed4c8e0eebb35f1a09df98ff965fff6774f71688cf"}, - {file = "GitPython-3.0.7.tar.gz", hash = "sha256:99cd0403cecd8a13b95d2e045b9fcaa7837137fcc5ec3105f2c413305d82c143"}, - {file = "GitPython-3.1.27-py3-none-any.whl", hash = "sha256:5b68b000463593e05ff2b261acff0ff0972df8ab1b70d3cdbd41b546c8b8fc3d"}, - {file = "GitPython-3.1.27.tar.gz", hash = "sha256:1c885ce809e8ba2d88a29befeb385fcea06338d3640712b59ca623c220bb5704"}, -] importlib-metadata = [ {file = "importlib_metadata-4.8.3-py3-none-any.whl", hash = "sha256:65a9576a5b2d58ca44d133c42a241905cc45e34d2c06fd5ba2bafa221e5d7b5e"}, {file = "importlib_metadata-4.8.3.tar.gz", hash = "sha256:766abffff765960fcc18003801f7044eb6755ffae4521c8e8ce8e83b9c9b0668"}, @@ -638,14 +535,6 @@ pytest-snapshot = [ {file = "pytest-snapshot-0.8.1.tar.gz", hash = "sha256:0f8872d56bc3ceacb465967072b059a36714898a37c9eb1c75cd4054110106f2"}, {file = "pytest_snapshot-0.8.1-py3-none-any.whl", hash = "sha256:ccb72c8e40dd1ec96b40caf0d328a9e9124b91d6a06204ad47d67403d83a4fd2"}, ] -smmap = [ - {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, - {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, -] -smmap2 = [ - {file = "smmap2-3.0.1-py3-none-any.whl", hash = "sha256:0cb6ea470b1ad9a65a02ca7f4c7ae601861f7dd24a43812ca51cfca2892bb524"}, - {file = "smmap2-3.0.1.tar.gz", hash = "sha256:44cc8bdaf96442dbb9a8e2e14377d074b3d0eea292eee3c95c8c449b6c92c557"}, -] snowballstemmer = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, diff --git a/pyproject.toml b/pyproject.toml index b214ef24..425562d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,15 +22,6 @@ black = ">=21.10b0" # These dependencies were selected because they are already used by black. click = ">=7.1.2" toml = ">=0.10.1" -GitPython = [ - {version=">=3.0", python=">=3.7"}, - {version="<3.0", python=">=3.6,<3.7"} -] -gitdb2 = {version="^2.0.6", python=">=3.6,<3.7"} # for py 3.6, pin this down -gitdb = [ - {version="^0.6", python=">=3.6,<3.7"}, - {version="^4.0.1", python=">=3.7"}, -] isort = {version=">=5.10", python=">=3.6,<4"} From 77d8f24dc7825db2a2bdc6f4fd397e8cf8a860a4 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 09:43:12 -0500 Subject: [PATCH 50/79] remove deleted import --- ni_python_styleguide/_utils/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ni_python_styleguide/_utils/__init__.py b/ni_python_styleguide/_utils/__init__.py index 0c75474a..5d57ce50 100644 --- a/ni_python_styleguide/_utils/__init__.py +++ b/ni_python_styleguide/_utils/__init__.py @@ -1,4 +1,3 @@ -from . import git # noqa: F401 from . import string_helpers # noqa: F401 DEFAULT_ENCODING = "UTF-8" From b077660fdcd1ff4bd933052e08d3f9b54662eead Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 09:43:36 -0500 Subject: [PATCH 51/79] just rglob with filter for excludes list --- ni_python_styleguide/_fix.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 64bd96bb..b2ee3656 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -2,6 +2,7 @@ import logging import pathlib from collections import defaultdict +from fnmatch import fnmatch from typing import List import isort @@ -90,7 +91,7 @@ def _handle_multiple_import_lines(bad_file: pathlib.Path): print(working_line, end="") -def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=False, diff=False): +def fix(exclude: str, app_import_names: str, extend_ignore, file_or_dir, *_, aggressive=False, diff=False): """Fix basic linter errors and format.""" file_or_dir = file_or_dir or ["."] if aggressive: @@ -98,12 +99,10 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa for file_or_dir_ in file_or_dir: file_path = pathlib.Path(file_or_dir_) if file_path.is_dir(): - if _utils.git.find_repo_root(file_or_dir_): - all_files.extend(_utils.git.get_tracked_files(file_or_dir_, filter="*.py")) - else: - all_files.extend(file_path.rglob("*.py")) + all_files.extend(file_path.rglob("*.py")) else: all_files.append(file_path) + all_files = filter(lambda o: not any([fnmatch(o, exclude_) for exclude_ in exclude.split(",")]), all_files) for file in all_files: if not file.is_file(): # doesn't really exist... continue From 453c9eb6578e8674e7951e40f20e4996c185d5ad Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 11:58:07 -0500 Subject: [PATCH 52/79] unextract handling the file --- .../_acknowledge_existing_errors/__init__.py | 31 ++++--------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 0227af33..109fbc67 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -75,29 +75,7 @@ def acknowledge_lint_errors( failed_files = [] for bad_file, errors_in_file in lint_errors_by_file.items(): - try: - suppress_errors_in_single_file( - exclude, - app_import_names, - extend_ignore, - aggressive, - bad_file, - errors_in_file, - ) - except RuntimeError: - failed_files.append(bad_file) - if failed_files: - raise RuntimeError("Could not handle some files:\n" + "\n\n".join(failed_files) + "\n\n\n") - - -def suppress_errors_in_single_file( - exclude, app_import_names, extend_ignore, aggressive, bad_file, errors_in_file -): - """Suppress `errors_in_file` in `bad_file`. - - if `aggressive` try repeatedly to suppress and format until stable state is reached. - """ - _suppress_errors_in_file(bad_file, errors_in_file, encoding=_utils.DEFAULT_ENCODING) + _suppress_errors_in_file(bad_file, errors_in_file, encoding=_utils.DEFAULT_ENCODING) if aggressive: # some cases are expected to take up to 4 passes, making this 2x rounded @@ -123,9 +101,12 @@ def suppress_errors_in_single_file( if not changed: # are we done? break else: - raise RuntimeError( - f"Could not suppress/format {bad_file} after {per_file_format_iteration_limit} tries" + failed_files.append( + f"Could not handle suppressions/formatting of file {bad_file} after maximum number of tries ({per_file_format_iteration_limit})" ) + _module_logger.warning("Max tries reached on %s", bad_file) + if failed_files: + raise RuntimeError("Could not handle some files:\n" + "\n\n".join(failed_files) + "\n\n\n") def remove_auto_suppressions_from_file(file: pathlib.Path): From 5e6c1681b4ef62dc60e45a92dfa82fcc5c10a4e2 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 11:59:35 -0500 Subject: [PATCH 53/79] update fix to match --- ni_python_styleguide/_fix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index b2ee3656..67280f41 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -141,12 +141,12 @@ def fix(exclude: str, app_import_names: str, extend_ignore, file_or_dir, *_, agg ) ) if remaining_lint_errors_in_file and aggressive: - _acknowledge_existing_errors.suppress_errors_in_single_file( + _acknowledge_existing_errors.acknowledge_lint_errors( exclude=exclude, app_import_names=app_import_names, extend_ignore=extend_ignore, aggressive=aggressive, - bad_file=bad_file, + file_or_dir=[bad_file], errors_in_file=remaining_lint_errors_in_file, ) except AttributeError as e: From c891708e4a447eb93e70c5f1e8068386263b4042 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 12:20:17 -0500 Subject: [PATCH 54/79] refactor getting lint output to utils --- .../_acknowledge_existing_errors/__init__.py | 13 +--- ni_python_styleguide/_utils/__init__.py | 1 + ni_python_styleguide/_utils/lint.py | 67 +------------------ 3 files changed, 5 insertions(+), 76 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 109fbc67..ceced94d 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -37,20 +37,9 @@ def _filter_suppresion_from_line(line: str): def _get_lint_errors_to_process( exclude, app_import_names, extend_ignore, file_or_dir, *_, excluded_errors=None ): - lint_errors = _lint.get_lint_output( - format=None, - qs_or_vs=None, - exclude=exclude, - app_import_names=app_import_names, - extend_ignore=extend_ignore, - file_or_dir=file_or_dir, - ).splitlines() - parsed_errors = map(_lint_errors_parser.parse, lint_errors) - parsed_errors = list(filter(None, parsed_errors)) if excluded_errors is None: excluded_errors = EXCLUDED_ERRORS - lint_errors_to_process = [error for error in parsed_errors if error.code not in excluded_errors] - return lint_errors_to_process + return _utils.lint.get_lint_errors_to_process(exclude, app_import_names, extend_ignore, file_or_dir, excluded_errors) def acknowledge_lint_errors( diff --git a/ni_python_styleguide/_utils/__init__.py b/ni_python_styleguide/_utils/__init__.py index 5d57ce50..75a4f5a3 100644 --- a/ni_python_styleguide/_utils/__init__.py +++ b/ni_python_styleguide/_utils/__init__.py @@ -1,3 +1,4 @@ from . import string_helpers # noqa: F401 +from . import lint # noqa: F401 DEFAULT_ENCODING = "UTF-8" diff --git a/ni_python_styleguide/_utils/lint.py b/ni_python_styleguide/_utils/lint.py index 55b334cb..7b120075 100644 --- a/ni_python_styleguide/_utils/lint.py +++ b/ni_python_styleguide/_utils/lint.py @@ -1,12 +1,7 @@ -import logging -import re -from collections import namedtuple - +from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser from ni_python_styleguide import _lint - -def get_errors_to_process(exclude, app_import_names, extend_ignore, file_or_dir, excluded_errors): - """Get lint errors to process.""" +def get_lint_errors_to_process(exclude, app_import_names, extend_ignore, file_or_dir, excluded_errors): lint_errors = _lint.get_lint_output( format=None, qs_or_vs=None, @@ -15,63 +10,7 @@ def get_errors_to_process(exclude, app_import_names, extend_ignore, file_or_dir, extend_ignore=extend_ignore, file_or_dir=file_or_dir, ).splitlines() - parsed_errors = map(parse, lint_errors) + parsed_errors = map(_lint_errors_parser.parse, lint_errors) parsed_errors = list(filter(None, parsed_errors)) lint_errors_to_process = [error for error in parsed_errors if error.code not in excluded_errors] return lint_errors_to_process - - -LintError = namedtuple("LintError", ["file", "line", "column", "code", "explanation"]) - - -def parse(line): - r""" - Parse line into :class:`LintError`. - - >>> parse(r'source\arfile.py:55:16: BLK100 Black would make changes.') - LintError(file='source\\arfile.py', line=55, column=16, code='BLK100', explanation='Black would make changes.') - - >>> parse(r"source\rpmfile\__init__.py:13:1: F401 'functools.wraps' imported but unused") - LintError(file='source\\rpmfile\\__init__.py', line=13, column=1, code='F401', explanation="'functools.wraps' imported but unused") - - >>> parse(r"expected_output.py:77:6: N802 function name 'method_withBadName_with_bad_params_on_multiple_lines_1' should be lowercase") - LintError(file='expected_output.py', line=77, column=6, code='N802', explanation="function name 'method_withBadName_with_bad_params_on_multiple_lines_1' should be lowercase") - - >>> parse(r"./tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/doc_line_tests/expected_output.py:1:1: D100 Missing docstring in public module") - LintError(file='./tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/doc_line_tests/expected_output.py', line=1, column=1, code='D100', explanation='Missing docstring in public module') - """ # NOQA W505: doc line too long (115 > 100 characters) - p = Parser() - return p.parse(line) - - -class Parser: - """Lint errors parser.""" - - __MATCHER = re.compile( - r"^(?P[\w\\/\.\-\:]+):(?P\d+):(?P\d+): (?P\w+) (?P.+)" - ) - - @staticmethod - def _to_lint_error(file: str, line: str, column: str, code: str, explanation: str, **kwargs): - return LintError( - file=file, - line=int(line), - column=int(column), - code=code, - explanation=explanation, - **kwargs - ) - - def parse(self, line): - """Parse `line` and return a :class:`LintError`. - - :param line: the line to parse - :return: lint error as metada object - :rtype: LintError - """ - data = Parser.__MATCHER.search(line) - logging.debug("parsing line: %s, yielded %s", line, data) - if not data: - return None - result = Parser._to_lint_error(**data.groupdict()) - return result From 298c164fd4f32f2e41b741d87d7bda7959e814ca Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 12:42:33 -0500 Subject: [PATCH 55/79] finish extracting --- .../_acknowledge_existing_errors/__init__.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index ceced94d..8c2e2c04 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -34,14 +34,6 @@ def _filter_suppresion_from_line(line: str): return line -def _get_lint_errors_to_process( - exclude, app_import_names, extend_ignore, file_or_dir, *_, excluded_errors=None -): - if excluded_errors is None: - excluded_errors = EXCLUDED_ERRORS - return _utils.lint.get_lint_errors_to_process(exclude, app_import_names, extend_ignore, file_or_dir, excluded_errors) - - def acknowledge_lint_errors( exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=False ): @@ -50,7 +42,7 @@ def acknowledge_lint_errors( Excluded error (reason): BLK100 - run black """ - lint_errors_to_process = _utils.lint.get_errors_to_process( + lint_errors_to_process = _utils.lint.get_lint_errors_to_process( exclude, app_import_names, extend_ignore, @@ -79,9 +71,14 @@ def acknowledge_lint_errors( # re-apply suppressions on correct lines remove_auto_suppressions_from_file(bad_file) - current_lint_errors = _get_lint_errors_to_process( - exclude, app_import_names, extend_ignore, [bad_file] + current_lint_errors = _utils.lint.get_lint_errors_to_process( + exclude=exclude, + app_import_names=app_import_names, + extend_ignore=extend_ignore, + file_or_dir=file_or_dir, + excluded_errors=EXCLUDED_ERRORS, ) + _suppress_errors_in_file( bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING ) From 6949a853d63cf24e17e459d43c8011d927921379 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 12:42:37 -0500 Subject: [PATCH 56/79] format --- ni_python_styleguide/_fix.py | 16 +++++++++++++--- ni_python_styleguide/_utils/__init__.py | 2 +- ni_python_styleguide/_utils/lint.py | 5 ++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 67280f41..7e8636e1 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -91,7 +91,15 @@ def _handle_multiple_import_lines(bad_file: pathlib.Path): print(working_line, end="") -def fix(exclude: str, app_import_names: str, extend_ignore, file_or_dir, *_, aggressive=False, diff=False): +def fix( + exclude: str, + app_import_names: str, + extend_ignore, + file_or_dir, + *_, + aggressive=False, + diff=False, +): """Fix basic linter errors and format.""" file_or_dir = file_or_dir or ["."] if aggressive: @@ -102,12 +110,14 @@ def fix(exclude: str, app_import_names: str, extend_ignore, file_or_dir, *_, agg all_files.extend(file_path.rglob("*.py")) else: all_files.append(file_path) - all_files = filter(lambda o: not any([fnmatch(o, exclude_) for exclude_ in exclude.split(",")]), all_files) + all_files = filter( + lambda o: not any([fnmatch(o, exclude_) for exclude_ in exclude.split(",")]), all_files + ) for file in all_files: if not file.is_file(): # doesn't really exist... continue _acknowledge_existing_errors.remove_auto_suppressions_from_file(file) - lint_errors_to_process = _acknowledge_existing_errors._get_lint_errors_to_process( + lint_errors_to_process = _acknowledge_existing_errors._utils.lint.get_lint_errors_to_process( exclude, app_import_names, extend_ignore, diff --git a/ni_python_styleguide/_utils/__init__.py b/ni_python_styleguide/_utils/__init__.py index 75a4f5a3..775df7b1 100644 --- a/ni_python_styleguide/_utils/__init__.py +++ b/ni_python_styleguide/_utils/__init__.py @@ -1,4 +1,4 @@ from . import string_helpers # noqa: F401 -from . import lint # noqa: F401 +from . import lint # noqa: F401 DEFAULT_ENCODING = "UTF-8" diff --git a/ni_python_styleguide/_utils/lint.py b/ni_python_styleguide/_utils/lint.py index 7b120075..fdb8471d 100644 --- a/ni_python_styleguide/_utils/lint.py +++ b/ni_python_styleguide/_utils/lint.py @@ -1,7 +1,10 @@ from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser from ni_python_styleguide import _lint -def get_lint_errors_to_process(exclude, app_import_names, extend_ignore, file_or_dir, excluded_errors): + +def get_lint_errors_to_process( + exclude, app_import_names, extend_ignore, file_or_dir, excluded_errors +): lint_errors = _lint.get_lint_output( format=None, qs_or_vs=None, From 5e38cf39dc337b220798466cc60f6b64e7c4b61c Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 12:44:33 -0500 Subject: [PATCH 57/79] fix terminology --- ni_python_styleguide/_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ni_python_styleguide/_cli.py b/ni_python_styleguide/_cli.py index 51fc0d31..b7379842 100644 --- a/ni_python_styleguide/_cli.py +++ b/ni_python_styleguide/_cli.py @@ -150,7 +150,7 @@ def lint(obj, format, extend_ignore, file_or_dir): ) @click.pass_obj def acknowledge_existing_violations(obj, extend_ignore, file_or_dir, aggressive): - """Lint existing violations and suppress. + """Lint existing violations and acknowledge. Use this command to acknowledge violations in existing code to allow for enforcing new code. """ @@ -173,7 +173,7 @@ def acknowledge_existing_violations(obj, extend_ignore, file_or_dir, aggressive) @click.option( "--aggressive", is_flag=True, - help="Remove any existing suppressions, fix what can be fixed, and suppress remaining.", + help="Remove any existing acknowledgments, fix what can be fixed, and re-acknowledges remaining.", ) @click.pass_obj def fix(obj, extend_ignore, file_or_dir, aggressive): From 558c9352bb50c3054681ab90930f81fe62418b87 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 12:54:06 -0500 Subject: [PATCH 58/79] move lint errors_parser in with linter utils --- ni_python_styleguide/_utils/lint.py | 64 ++++++++++++++++++++++- tests/test_cli/test_lint_errors_parser.py | 1 + 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/ni_python_styleguide/_utils/lint.py b/ni_python_styleguide/_utils/lint.py index fdb8471d..d50a13bf 100644 --- a/ni_python_styleguide/_utils/lint.py +++ b/ni_python_styleguide/_utils/lint.py @@ -1,3 +1,6 @@ +import logging +import ni_python_styleguide + from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser from ni_python_styleguide import _lint @@ -13,7 +16,66 @@ def get_lint_errors_to_process( extend_ignore=extend_ignore, file_or_dir=file_or_dir, ).splitlines() - parsed_errors = map(_lint_errors_parser.parse, lint_errors) + parsed_errors = map(parse, lint_errors) parsed_errors = list(filter(None, parsed_errors)) lint_errors_to_process = [error for error in parsed_errors if error.code not in excluded_errors] return lint_errors_to_process + + +import re +from collections import namedtuple + +LintError = namedtuple("LintError", ["file", "line", "column", "code", "explanation"]) + + +def parse(line): + r""" + Parse line into :class:`LintError`. + + >>> parse(r'source\arfile.py:55:16: BLK100 Black would make changes.') + LintError(file='source\\arfile.py', line=55, column=16, code='BLK100', explanation='Black would make changes.') + + >>> parse(r"source\rpmfile\__init__.py:13:1: F401 'functools.wraps' imported but unused") + LintError(file='source\\rpmfile\\__init__.py', line=13, column=1, code='F401', explanation="'functools.wraps' imported but unused") + + >>> parse(r"expected_output.py:77:6: N802 function name 'method_withBadName_with_bad_params_on_multiple_lines_1' should be lowercase") + LintError(file='expected_output.py', line=77, column=6, code='N802', explanation="function name 'method_withBadName_with_bad_params_on_multiple_lines_1' should be lowercase") + + >>> parse(r"./tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/doc_line_tests/expected_output.py:1:1: D100 Missing docstring in public module") + LintError(file='./tests/test_cli/acknowledge_existing_errors_test_cases__snapshots/doc_line_tests/expected_output.py', line=1, column=1, code='D100', explanation='Missing docstring in public module') + """ # NOQA W505: doc line too long (115 > 100 characters) + p = Parser() + return p.parse(line) + + +class Parser: + """Lint errors parser.""" + + __MATCHER = re.compile( + r"^(?P[\w\\/\.\-\:]+):(?P\d+):(?P\d+): (?P\w+) (?P.+)" + ) + + @staticmethod + def _to_lint_error(file: str, line: str, column: str, code: str, explanation: str, **kwargs): + return LintError( + file=file, + line=int(line), + column=int(column), + code=code, + explanation=explanation, + **kwargs + ) + + def parse(self, line): + """Parse `line` and return a :class:`LintError`. + + :param line: the line to parse + :return: lint error as metada object + :rtype: LintError + """ + data = Parser.__MATCHER.search(line) + logging.debug("parsing line: %s, yielded %s", line, data) + if not data: + return None + result = Parser._to_lint_error(**data.groupdict()) + return result diff --git a/tests/test_cli/test_lint_errors_parser.py b/tests/test_cli/test_lint_errors_parser.py index e7a29e38..dbf6f484 100644 --- a/tests/test_cli/test_lint_errors_parser.py +++ b/tests/test_cli/test_lint_errors_parser.py @@ -2,6 +2,7 @@ import pytest +import ni_python_styleguide._acknowledge_existing_errors._lint_errors_parser import ni_python_styleguide._utils.lint EXAMPLE_LINT_ERROR_LINES = [ # noqa W505 From 9c24309802d64b98a26d04e8d9dee3a21abeb875 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 12:59:55 -0500 Subject: [PATCH 59/79] update more occurances --- ni_python_styleguide/_acknowledge_existing_errors/__init__.py | 1 - ni_python_styleguide/_fix.py | 4 ++-- ni_python_styleguide/_utils/__init__.py | 2 +- ni_python_styleguide/_utils/lint.py | 3 +-- .../more_complicated_example/input.py | 4 ++-- .../more_complicated_example/output.py | 4 ++-- .../more_complicated_example/output__aggressive.py | 4 ++-- 7 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 8c2e2c04..32c10ced 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -6,7 +6,6 @@ from ni_python_styleguide import _format from ni_python_styleguide import _lint from ni_python_styleguide import _utils -from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser _module_logger = logging.getLogger(__name__) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 7e8636e1..cc7ed6b4 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -10,7 +10,6 @@ from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _format from ni_python_styleguide import _utils -from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser _module_logger = logging.getLogger(__name__) @@ -143,11 +142,12 @@ def fix( _handle_multiple_import_lines(bad_file) _format.format(bad_file) remaining_lint_errors_in_file = ( - _acknowledge_existing_errors._get_lint_errors_to_process( + _utils.lint.get_lint_errors_to_process( exclude, app_import_names, extend_ignore, [bad_file], + exclude=[], ) ) if remaining_lint_errors_in_file and aggressive: diff --git a/ni_python_styleguide/_utils/__init__.py b/ni_python_styleguide/_utils/__init__.py index 775df7b1..96c4e689 100644 --- a/ni_python_styleguide/_utils/__init__.py +++ b/ni_python_styleguide/_utils/__init__.py @@ -1,4 +1,4 @@ -from . import string_helpers # noqa: F401 from . import lint # noqa: F401 +from . import string_helpers # noqa: F401 DEFAULT_ENCODING = "UTF-8" diff --git a/ni_python_styleguide/_utils/lint.py b/ni_python_styleguide/_utils/lint.py index d50a13bf..33636915 100644 --- a/ni_python_styleguide/_utils/lint.py +++ b/ni_python_styleguide/_utils/lint.py @@ -1,7 +1,6 @@ import logging -import ni_python_styleguide -from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser +import ni_python_styleguide from ni_python_styleguide import _lint diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py index acfcfa5d..927c8918 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py @@ -86,7 +86,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa """Fix basic linter errors and format.""" if aggressive: raise Exception("--aggressive is not implemented yet") - lint_errors_to_process = _acknowledge_existing_errors._get_lint_errors_to_process( + lint_errors_to_process = _utils.lint.get_lint_errors_to_process( exclude, app_import_names, extend_ignore, @@ -111,7 +111,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa _format.format(bad_file) _handle_multiple_import_lines(bad_file) _format.format(bad_file) - _ = _acknowledge_existing_errors._get_lint_errors_to_process( + _ = _utils.lint.get_lint_errors_to_process( exclude, app_import_names, extend_ignore, diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py index 3b9f854a..9d5f1d33 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py @@ -91,7 +91,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa """Fix basic linter errors and format.""" if aggressive: raise Exception("--aggressive is not implemented yet") - lint_errors_to_process = _acknowledge_existing_errors._get_lint_errors_to_process( + lint_errors_to_process = _utils.lint.get_lint_errors_to_process( exclude, app_import_names, extend_ignore, @@ -116,7 +116,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa _format.format(bad_file) _handle_multiple_import_lines(bad_file) _format.format(bad_file) - _ = _acknowledge_existing_errors._get_lint_errors_to_process( + _ = _utils.lint.get_lint_errors_to_process( exclude, app_import_names, extend_ignore, diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py index 3b9f854a..9d5f1d33 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py @@ -91,7 +91,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa """Fix basic linter errors and format.""" if aggressive: raise Exception("--aggressive is not implemented yet") - lint_errors_to_process = _acknowledge_existing_errors._get_lint_errors_to_process( + lint_errors_to_process = _utils.lint.get_lint_errors_to_process( exclude, app_import_names, extend_ignore, @@ -116,7 +116,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa _format.format(bad_file) _handle_multiple_import_lines(bad_file) _format.format(bad_file) - _ = _acknowledge_existing_errors._get_lint_errors_to_process( + _ = _utils.lint.get_lint_errors_to_process( exclude, app_import_names, extend_ignore, From dc5c5a02588ef03c4501d93507259e9b1631fe71 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 13:01:16 -0500 Subject: [PATCH 60/79] correct call --- ni_python_styleguide/_fix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index cc7ed6b4..e66b755a 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -147,7 +147,7 @@ def fix( app_import_names, extend_ignore, [bad_file], - exclude=[], + excluded_errors=[], ) ) if remaining_lint_errors_in_file and aggressive: From 7b4a57c66a97a0c89c1f0d699d8205729b801835 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 13:02:31 -0500 Subject: [PATCH 61/79] dog-food the fix command --- ni_python_styleguide/_fix.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index e66b755a..ad553664 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -141,14 +141,12 @@ def fix( _format.format(bad_file, "--line-length=300") # condense any split lines _handle_multiple_import_lines(bad_file) _format.format(bad_file) - remaining_lint_errors_in_file = ( - _utils.lint.get_lint_errors_to_process( - exclude, - app_import_names, - extend_ignore, - [bad_file], - excluded_errors=[], - ) + remaining_lint_errors_in_file = _utils.lint.get_lint_errors_to_process( + exclude, + app_import_names, + extend_ignore, + [bad_file], + excluded_errors=[], ) if remaining_lint_errors_in_file and aggressive: _acknowledge_existing_errors.acknowledge_lint_errors( From fa4179a1ba83c0f45dfe70fe09af10b15f2bdf54 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 13:05:17 -0500 Subject: [PATCH 62/79] fix another reference --- ni_python_styleguide/_fix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index ad553664..0094a8f6 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -130,7 +130,7 @@ def fix( failed_files = [] for bad_file, errors_in_file in lint_errors_by_file.items(): - errors_in_file: List[_lint_errors_parser.LintError] + errors_in_file: List[_utils.lint.LintError] try: _format.format(bad_file) line_to_codes_mapping = defaultdict(set) From 77da8030e6edf9913b5a7ab3379eeabcfca94c4d Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 13:05:56 -0500 Subject: [PATCH 63/79] document public api --- ni_python_styleguide/_utils/lint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ni_python_styleguide/_utils/lint.py b/ni_python_styleguide/_utils/lint.py index 33636915..3b0e909b 100644 --- a/ni_python_styleguide/_utils/lint.py +++ b/ni_python_styleguide/_utils/lint.py @@ -7,6 +7,7 @@ def get_lint_errors_to_process( exclude, app_import_names, extend_ignore, file_or_dir, excluded_errors ): + """Get lint errors to process.""" lint_errors = _lint.get_lint_output( format=None, qs_or_vs=None, From 1a6e878eec1262d85b84b383fd273f26c3ab61f7 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 18 Apr 2022 13:09:04 -0500 Subject: [PATCH 64/79] remove broken import --- tests/test_cli/test_lint_errors_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cli/test_lint_errors_parser.py b/tests/test_cli/test_lint_errors_parser.py index dbf6f484..e7a29e38 100644 --- a/tests/test_cli/test_lint_errors_parser.py +++ b/tests/test_cli/test_lint_errors_parser.py @@ -2,7 +2,6 @@ import pytest -import ni_python_styleguide._acknowledge_existing_errors._lint_errors_parser import ni_python_styleguide._utils.lint EXAMPLE_LINT_ERROR_LINES = [ # noqa W505 From 4efba7eceb12bb621ed56296e01e02660d16cffa Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 21 Apr 2022 15:28:44 -0500 Subject: [PATCH 65/79] fix indentation --- .../_acknowledge_existing_errors/__init__.py | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 32c10ced..f74113b9 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -57,39 +57,39 @@ def acknowledge_lint_errors( for bad_file, errors_in_file in lint_errors_by_file.items(): _suppress_errors_in_file(bad_file, errors_in_file, encoding=_utils.DEFAULT_ENCODING) - if aggressive: - # some cases are expected to take up to 4 passes, making this 2x rounded - per_file_format_iteration_limit = 10 - for _ in range(per_file_format_iteration_limit): - # format the files - this may move the suppression off the correct lines - # Note: due to Github pycodestyle#868, we have to format, change, format - # (check if that time made changes) - # - else we wind up with lambda's going un-suppressed - # and/or not re-formatted (to fail later) - _format.format(bad_file) - - # re-apply suppressions on correct lines - remove_auto_suppressions_from_file(bad_file) - current_lint_errors = _utils.lint.get_lint_errors_to_process( - exclude=exclude, - app_import_names=app_import_names, - extend_ignore=extend_ignore, - file_or_dir=file_or_dir, - excluded_errors=EXCLUDED_ERRORS, - ) - - _suppress_errors_in_file( - bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING - ) - - changed = _format.format_check(bad_file) - if not changed: # are we done? - break - else: - failed_files.append( - f"Could not handle suppressions/formatting of file {bad_file} after maximum number of tries ({per_file_format_iteration_limit})" - ) - _module_logger.warning("Max tries reached on %s", bad_file) + if aggressive: + # some cases are expected to take up to 4 passes, making this 2x rounded + per_file_format_iteration_limit = 10 + for _ in range(per_file_format_iteration_limit): + # format the files - this may move the suppression off the correct lines + # Note: due to Github pycodestyle#868, we have to format, change, format + # (check if that time made changes) + # - else we wind up with lambda's going un-suppressed + # and/or not re-formatted (to fail later) + _format.format(bad_file) + + # re-apply suppressions on correct lines + remove_auto_suppressions_from_file(bad_file) + current_lint_errors = _utils.lint.get_lint_errors_to_process( + exclude=exclude, + app_import_names=app_import_names, + extend_ignore=extend_ignore, + file_or_dir=file_or_dir, + excluded_errors=EXCLUDED_ERRORS, + ) + + _suppress_errors_in_file( + bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING + ) + + changed = _format.format_check(bad_file) + if not changed: # are we done? + break + else: + failed_files.append( + f"Could not handle suppressions/formatting of file {bad_file} after maximum number of tries ({per_file_format_iteration_limit})" + ) + _module_logger.warning("Max tries reached on %s", bad_file) if failed_files: raise RuntimeError("Could not handle some files:\n" + "\n\n".join(failed_files) + "\n\n\n") From e0a953481344c9927481a8eb18f800e18b232b03 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 21 Apr 2022 15:29:18 -0500 Subject: [PATCH 66/79] flip the indentation --- .../_acknowledge_existing_errors/__init__.py | 68 ++++++++++--------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index f74113b9..ddb9f5a6 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -57,39 +57,41 @@ def acknowledge_lint_errors( for bad_file, errors_in_file in lint_errors_by_file.items(): _suppress_errors_in_file(bad_file, errors_in_file, encoding=_utils.DEFAULT_ENCODING) - if aggressive: - # some cases are expected to take up to 4 passes, making this 2x rounded - per_file_format_iteration_limit = 10 - for _ in range(per_file_format_iteration_limit): - # format the files - this may move the suppression off the correct lines - # Note: due to Github pycodestyle#868, we have to format, change, format - # (check if that time made changes) - # - else we wind up with lambda's going un-suppressed - # and/or not re-formatted (to fail later) - _format.format(bad_file) - - # re-apply suppressions on correct lines - remove_auto_suppressions_from_file(bad_file) - current_lint_errors = _utils.lint.get_lint_errors_to_process( - exclude=exclude, - app_import_names=app_import_names, - extend_ignore=extend_ignore, - file_or_dir=file_or_dir, - excluded_errors=EXCLUDED_ERRORS, - ) - - _suppress_errors_in_file( - bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING - ) - - changed = _format.format_check(bad_file) - if not changed: # are we done? - break - else: - failed_files.append( - f"Could not handle suppressions/formatting of file {bad_file} after maximum number of tries ({per_file_format_iteration_limit})" - ) - _module_logger.warning("Max tries reached on %s", bad_file) + if not aggressive: + continue + + # some cases are expected to take up to 4 passes, making this 2x rounded + per_file_format_iteration_limit = 10 + for _ in range(per_file_format_iteration_limit): + # format the files - this may move the suppression off the correct lines + # Note: due to Github pycodestyle#868, we have to format, change, format + # (check if that time made changes) + # - else we wind up with lambda's going un-suppressed + # and/or not re-formatted (to fail later) + _format.format(bad_file) + + # re-apply suppressions on correct lines + remove_auto_suppressions_from_file(bad_file) + current_lint_errors = _utils.lint.get_lint_errors_to_process( + exclude=exclude, + app_import_names=app_import_names, + extend_ignore=extend_ignore, + file_or_dir=file_or_dir, + excluded_errors=EXCLUDED_ERRORS, + ) + + _suppress_errors_in_file( + bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING + ) + + changed = _format.format_check(bad_file) + if not changed: # are we done? + break + else: + failed_files.append( + f"Could not handle suppressions/formatting of file {bad_file} after maximum number of tries ({per_file_format_iteration_limit})" + ) + _module_logger.warning("Max tries reached on %s", bad_file) if failed_files: raise RuntimeError("Could not handle some files:\n" + "\n\n".join(failed_files) + "\n\n\n") From 12eebc373d3d8836c85005825c27a5397387fa39 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 21 Apr 2022 15:46:01 -0500 Subject: [PATCH 67/79] remove some unused imports --- ni_python_styleguide/_acknowledge_existing_errors/__init__.py | 1 - ni_python_styleguide/_utils/lint.py | 1 - .../more_complicated_example/output__aggressive.py | 1 - 3 files changed, 3 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index ddb9f5a6..9cbbf78e 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -4,7 +4,6 @@ from collections import defaultdict from ni_python_styleguide import _format -from ni_python_styleguide import _lint from ni_python_styleguide import _utils _module_logger = logging.getLogger(__name__) diff --git a/ni_python_styleguide/_utils/lint.py b/ni_python_styleguide/_utils/lint.py index 3b0e909b..11f45cdf 100644 --- a/ni_python_styleguide/_utils/lint.py +++ b/ni_python_styleguide/_utils/lint.py @@ -1,6 +1,5 @@ import logging -import ni_python_styleguide from ni_python_styleguide import _lint diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py index 9d5f1d33..4002c1ac 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py @@ -8,7 +8,6 @@ import isort -from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _format from ni_python_styleguide import _utils from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser From 622c82a075acb494fb2c70f8fea88fd27b7f1b88 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 21 Apr 2022 16:39:33 -0500 Subject: [PATCH 68/79] fix imports --- ni_python_styleguide/_utils/lint.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ni_python_styleguide/_utils/lint.py b/ni_python_styleguide/_utils/lint.py index 11f45cdf..cf39e1f9 100644 --- a/ni_python_styleguide/_utils/lint.py +++ b/ni_python_styleguide/_utils/lint.py @@ -1,4 +1,6 @@ import logging +import re +from collections import namedtuple from ni_python_styleguide import _lint @@ -21,9 +23,6 @@ def get_lint_errors_to_process( return lint_errors_to_process -import re -from collections import namedtuple - LintError = namedtuple("LintError", ["file", "line", "column", "code", "explanation"]) From b5ddf54724dedb021c32657eb9923f7ddc8117e3 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 21 Apr 2022 16:42:47 -0500 Subject: [PATCH 69/79] update tests --- .../more_complicated_example/output__aggressive.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py index 4002c1ac..9d5f1d33 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py @@ -8,6 +8,7 @@ import isort +from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _format from ni_python_styleguide import _utils from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser From 9c458caaf0f29fed1c6328a4cd862599a016dfc7 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 27 Apr 2022 09:24:27 -0500 Subject: [PATCH 70/79] handle merge differences --- .../_acknowledge_existing_errors/__init__.py | 7 +++---- ni_python_styleguide/_utils/lint.py | 4 +--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 9cbbf78e..63f0b41c 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -40,7 +40,7 @@ def acknowledge_lint_errors( Excluded error (reason): BLK100 - run black """ - lint_errors_to_process = _utils.lint.get_lint_errors_to_process( + lint_errors_to_process = _utils.lint.get_errors_to_process( exclude, app_import_names, extend_ignore, @@ -56,8 +56,7 @@ def acknowledge_lint_errors( for bad_file, errors_in_file in lint_errors_by_file.items(): _suppress_errors_in_file(bad_file, errors_in_file, encoding=_utils.DEFAULT_ENCODING) - if not aggressive: - continue + if aggressive: # some cases are expected to take up to 4 passes, making this 2x rounded per_file_format_iteration_limit = 10 @@ -71,7 +70,7 @@ def acknowledge_lint_errors( # re-apply suppressions on correct lines remove_auto_suppressions_from_file(bad_file) - current_lint_errors = _utils.lint.get_lint_errors_to_process( + current_lint_errors = _utils.lint.get_errors_to_process( exclude=exclude, app_import_names=app_import_names, extend_ignore=extend_ignore, diff --git a/ni_python_styleguide/_utils/lint.py b/ni_python_styleguide/_utils/lint.py index cf39e1f9..55b334cb 100644 --- a/ni_python_styleguide/_utils/lint.py +++ b/ni_python_styleguide/_utils/lint.py @@ -5,9 +5,7 @@ from ni_python_styleguide import _lint -def get_lint_errors_to_process( - exclude, app_import_names, extend_ignore, file_or_dir, excluded_errors -): +def get_errors_to_process(exclude, app_import_names, extend_ignore, file_or_dir, excluded_errors): """Get lint errors to process.""" lint_errors = _lint.get_lint_output( format=None, From 76ccc165f1add7d624e85506bf729f99f4cf32b6 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 2 May 2022 09:29:21 -0500 Subject: [PATCH 71/79] make the diff smaller --- ni_python_styleguide/_acknowledge_existing_errors/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 63f0b41c..20140114 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -56,7 +56,8 @@ def acknowledge_lint_errors( for bad_file, errors_in_file in lint_errors_by_file.items(): _suppress_errors_in_file(bad_file, errors_in_file, encoding=_utils.DEFAULT_ENCODING) - if aggressive: + if not aggressive: + continue # some cases are expected to take up to 4 passes, making this 2x rounded per_file_format_iteration_limit = 10 From 0256d6f5808b394a6d134fa77e007a933cbb340d Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 2 May 2022 09:29:40 -0500 Subject: [PATCH 72/79] fix reference to get lint output --- ni_python_styleguide/_acknowledge_existing_errors/__init__.py | 2 +- ni_python_styleguide/_fix.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 20140114..94269827 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -71,7 +71,7 @@ def acknowledge_lint_errors( # re-apply suppressions on correct lines remove_auto_suppressions_from_file(bad_file) - current_lint_errors = _utils.lint.get_errors_to_process( + current_lint_errors = _utils.lint.get_errors_to_process( exclude=exclude, app_import_names=app_import_names, extend_ignore=extend_ignore, diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 0094a8f6..0582954d 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -116,7 +116,7 @@ def fix( if not file.is_file(): # doesn't really exist... continue _acknowledge_existing_errors.remove_auto_suppressions_from_file(file) - lint_errors_to_process = _acknowledge_existing_errors._utils.lint.get_lint_errors_to_process( + lint_errors_to_process = _acknowledge_existing_errors._utils.lint.get_errors_to_process( exclude, app_import_names, extend_ignore, From 17c38078f18599ecf03f41f27ec25dadc63bb3d5 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 2 May 2022 09:30:59 -0500 Subject: [PATCH 73/79] flip that back around --- .../_acknowledge_existing_errors/__init__.py | 68 +++++++++---------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py index 94269827..354a6fc8 100644 --- a/ni_python_styleguide/_acknowledge_existing_errors/__init__.py +++ b/ni_python_styleguide/_acknowledge_existing_errors/__init__.py @@ -56,41 +56,39 @@ def acknowledge_lint_errors( for bad_file, errors_in_file in lint_errors_by_file.items(): _suppress_errors_in_file(bad_file, errors_in_file, encoding=_utils.DEFAULT_ENCODING) - if not aggressive: - continue - - # some cases are expected to take up to 4 passes, making this 2x rounded - per_file_format_iteration_limit = 10 - for _ in range(per_file_format_iteration_limit): - # format the files - this may move the suppression off the correct lines - # Note: due to Github pycodestyle#868, we have to format, change, format - # (check if that time made changes) - # - else we wind up with lambda's going un-suppressed - # and/or not re-formatted (to fail later) - _format.format(bad_file) - - # re-apply suppressions on correct lines - remove_auto_suppressions_from_file(bad_file) - current_lint_errors = _utils.lint.get_errors_to_process( - exclude=exclude, - app_import_names=app_import_names, - extend_ignore=extend_ignore, - file_or_dir=file_or_dir, - excluded_errors=EXCLUDED_ERRORS, - ) - - _suppress_errors_in_file( - bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING - ) - - changed = _format.format_check(bad_file) - if not changed: # are we done? - break - else: - failed_files.append( - f"Could not handle suppressions/formatting of file {bad_file} after maximum number of tries ({per_file_format_iteration_limit})" - ) - _module_logger.warning("Max tries reached on %s", bad_file) + if aggressive: + # some cases are expected to take up to 4 passes, making this 2x rounded + per_file_format_iteration_limit = 10 + for _ in range(per_file_format_iteration_limit): + # format the files - this may move the suppression off the correct lines + # Note: due to Github pycodestyle#868, we have to format, change, format + # (check if that time made changes) + # - else we wind up with lambda's going un-suppressed + # and/or not re-formatted (to fail later) + _format.format(bad_file) + + # re-apply suppressions on correct lines + remove_auto_suppressions_from_file(bad_file) + current_lint_errors = _utils.lint.get_errors_to_process( + exclude=exclude, + app_import_names=app_import_names, + extend_ignore=extend_ignore, + file_or_dir=file_or_dir, + excluded_errors=EXCLUDED_ERRORS, + ) + + _suppress_errors_in_file( + bad_file, current_lint_errors, encoding=_utils.DEFAULT_ENCODING + ) + + changed = _format.format_check(bad_file) + if not changed: # are we done? + break + else: + failed_files.append( + f"Could not handle suppressions/formatting of file {bad_file} after maximum number of tries ({per_file_format_iteration_limit})" + ) + _module_logger.warning("Max tries reached on %s", bad_file) if failed_files: raise RuntimeError("Could not handle some files:\n" + "\n\n".join(failed_files) + "\n\n\n") From c9b989f1f10ac71be1c33c8e9b5ac59c92388e41 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 2 May 2022 10:04:03 -0500 Subject: [PATCH 74/79] fix more references --- ni_python_styleguide/_fix.py | 2 +- .../more_complicated_example/input.py | 4 ++-- .../more_complicated_example/output.py | 4 ++-- .../more_complicated_example/output__aggressive.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 0582954d..38ece722 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -141,7 +141,7 @@ def fix( _format.format(bad_file, "--line-length=300") # condense any split lines _handle_multiple_import_lines(bad_file) _format.format(bad_file) - remaining_lint_errors_in_file = _utils.lint.get_lint_errors_to_process( + remaining_lint_errors_in_file = _utils.lint.get_errors_to_process( exclude, app_import_names, extend_ignore, diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py index 927c8918..459e1ea1 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py @@ -86,7 +86,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa """Fix basic linter errors and format.""" if aggressive: raise Exception("--aggressive is not implemented yet") - lint_errors_to_process = _utils.lint.get_lint_errors_to_process( + lint_errors_to_process = _utils.lint.get_errors_to_process( exclude, app_import_names, extend_ignore, @@ -111,7 +111,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa _format.format(bad_file) _handle_multiple_import_lines(bad_file) _format.format(bad_file) - _ = _utils.lint.get_lint_errors_to_process( + _ = _utils.lint.get_errors_to_process( exclude, app_import_names, extend_ignore, diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py index 9d5f1d33..1c81c86c 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py @@ -91,7 +91,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa """Fix basic linter errors and format.""" if aggressive: raise Exception("--aggressive is not implemented yet") - lint_errors_to_process = _utils.lint.get_lint_errors_to_process( + lint_errors_to_process = _utils.lint.get_errors_to_process( exclude, app_import_names, extend_ignore, @@ -116,7 +116,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa _format.format(bad_file) _handle_multiple_import_lines(bad_file) _format.format(bad_file) - _ = _utils.lint.get_lint_errors_to_process( + _ = _utils.lint.get_errors_to_process( exclude, app_import_names, extend_ignore, diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py index 9d5f1d33..1c81c86c 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py @@ -91,7 +91,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa """Fix basic linter errors and format.""" if aggressive: raise Exception("--aggressive is not implemented yet") - lint_errors_to_process = _utils.lint.get_lint_errors_to_process( + lint_errors_to_process = _utils.lint.get_errors_to_process( exclude, app_import_names, extend_ignore, @@ -116,7 +116,7 @@ def fix(exclude, app_import_names, extend_ignore, file_or_dir, *_, aggressive=Fa _format.format(bad_file) _handle_multiple_import_lines(bad_file) _format.format(bad_file) - _ = _utils.lint.get_lint_errors_to_process( + _ = _utils.lint.get_errors_to_process( exclude, app_import_names, extend_ignore, From 53c8f73dfb5e4dc816c029b376e63e0a741ddf0f Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 2 May 2022 10:08:09 -0500 Subject: [PATCH 75/79] fix ni-python-styleguide linter errors --- .../fix_test_cases__snapshots/more_complicated_example/output.py | 1 - .../more_complicated_example/output__aggressive.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py index 1c81c86c..f69cf07a 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py @@ -8,7 +8,6 @@ import isort -from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _format from ni_python_styleguide import _utils from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py index 1c81c86c..f69cf07a 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py @@ -8,7 +8,6 @@ import isort -from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _format from ni_python_styleguide import _utils from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser From dd9cfff39e8a5738c68628eac6541d761f1d2ce3 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 2 May 2022 10:10:23 -0500 Subject: [PATCH 76/79] update tests --- .../fix_test_cases__snapshots/more_complicated_example/output.py | 1 + .../more_complicated_example/output__aggressive.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py index f69cf07a..1c81c86c 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py @@ -8,6 +8,7 @@ import isort +from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _format from ni_python_styleguide import _utils from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py index f69cf07a..1c81c86c 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py @@ -8,6 +8,7 @@ import isort +from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _format from ni_python_styleguide import _utils from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser From c6b6d421715d418365850c4478bd6b8858eeb71b Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 2 May 2022 10:15:34 -0500 Subject: [PATCH 77/79] remove it from the input --- .../fix_test_cases__snapshots/more_complicated_example/input.py | 2 +- .../more_complicated_example/output.py | 1 - .../more_complicated_example/output__aggressive.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py index 459e1ea1..21ad476a 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/input.py @@ -1,6 +1,6 @@ """Provide a more complex example with some corner cases.""" from collections import defaultdict -from ni_python_styleguide import _acknowledge_existing_errors, _format, _utils +from ni_python_styleguide import _format, _utils from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser from typing import Iterable, List import fileinput diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py index 1c81c86c..f69cf07a 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output.py @@ -8,7 +8,6 @@ import isort -from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _format from ni_python_styleguide import _utils from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser diff --git a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py index 1c81c86c..f69cf07a 100644 --- a/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py +++ b/tests/test_cli/fix_test_cases__snapshots/more_complicated_example/output__aggressive.py @@ -8,7 +8,6 @@ import isort -from ni_python_styleguide import _acknowledge_existing_errors from ni_python_styleguide import _format from ni_python_styleguide import _utils from ni_python_styleguide._acknowledge_existing_errors import _lint_errors_parser From 6ad06a46740fea355484074bb29353da2ace9eba Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 2 May 2022 11:10:01 -0500 Subject: [PATCH 78/79] fix verbage --- ni_python_styleguide/_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ni_python_styleguide/_cli.py b/ni_python_styleguide/_cli.py index b7379842..f36ffc7b 100644 --- a/ni_python_styleguide/_cli.py +++ b/ni_python_styleguide/_cli.py @@ -173,7 +173,7 @@ def acknowledge_existing_violations(obj, extend_ignore, file_or_dir, aggressive) @click.option( "--aggressive", is_flag=True, - help="Remove any existing acknowledgments, fix what can be fixed, and re-acknowledges remaining.", + help="Remove any existing acknowledgments, fix what can be fixed, and re-acknowledge remaining.", ) @click.pass_obj def fix(obj, extend_ignore, file_or_dir, aggressive): From 2c0d20dc15a6b38c13fe6a7d021d639b626c64d9 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Mon, 2 May 2022 11:10:11 -0500 Subject: [PATCH 79/79] remove unsupported argument --- ni_python_styleguide/_fix.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ni_python_styleguide/_fix.py b/ni_python_styleguide/_fix.py index 38ece722..f0168628 100644 --- a/ni_python_styleguide/_fix.py +++ b/ni_python_styleguide/_fix.py @@ -97,7 +97,6 @@ def fix( file_or_dir, *_, aggressive=False, - diff=False, ): """Fix basic linter errors and format.""" file_or_dir = file_or_dir or ["."]