From 357d2079749eaee999641eff4a3e6633f3acacef Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 18 Dec 2022 17:04:48 +0100 Subject: [PATCH] fIx isort problems introduced by recent isort release (#28434) The recent isort changed their mind on sorting the imports. This change follows the change and bumps isort to latest released version (isort has no install_requires on its own so bumping min version has no effect on other dependencies) This change adds a number of isort:skip_file, isort:off, isort:skips in order to handle a very annoying bug in isort, that no matter how much you try, it sometimes treat "known first party" packages differently - depending on how many files it processes at a time. We should be able to restore it after this bug is fixed: https://github.com/PyCQA/isort/issues/2045 This change also updates the common.sql API to skip them from isort for the very same reason (depending on how many files are modified, the isort order might change. (cherry picked from commit f115b207bc844c10569b2df6fc9acfa32a3c7f41) GitOrigin-RevId: d7db157adf1ad1fc687ce6e203d71fab7f495f25 --- .pre-commit-config.yaml | 10 +++++----- docker_tests/test_docker_compose_quick_start.py | 4 ++++ docker_tests/test_examples_of_prod_image_building.py | 3 +++ docker_tests/test_prod_image.py | 3 +++ docs/build_docs.py | 6 ++++++ docs/exts/docs_build/dev_index_generator.py | 3 +++ docs/exts/docs_build/errors.py | 4 +++- docs/publish_docs.py | 3 +++ kubernetes_tests/test_kubernetes_executor.py | 2 +- kubernetes_tests/test_other_executors.py | 2 +- pyproject.toml | 1 + setup.py | 5 ++++- 12 files changed, 37 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 810ac0fc1bb..0405cbd8617 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -146,6 +146,11 @@ repos: - --fuzzy-match-generates-todo files: > \.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$ + - repo: https://github.com/PyCQA/isort + rev: 5.11.2 + hooks: + - id: isort + name: Run isort to sort imports in Python files # Keep version of black in sync wit blacken-docs and pre-commit-hook-names - repo: https://github.com/psf/black rev: 22.12.0 @@ -233,11 +238,6 @@ repos: entry: yamllint -c yamllint-config.yml --strict types: [yaml] exclude: ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/(?:templates|files)/.*\.yaml$|openapi/.*\.yaml$|^\.pre-commit-config\.yaml$|^airflow/_vendor/ - - repo: https://github.com/PyCQA/isort - rev: 5.10.1 - hooks: - - id: isort - name: Run isort to sort imports in Python files - repo: https://github.com/pycqa/pydocstyle rev: 6.1.1 hooks: diff --git a/docker_tests/test_docker_compose_quick_start.py b/docker_tests/test_docker_compose_quick_start.py index fd553ed1756..4754aac3299 100644 --- a/docker_tests/test_docker_compose_quick_start.py +++ b/docker_tests/test_docker_compose_quick_start.py @@ -27,10 +27,14 @@ from unittest import mock import requests + +# isort:off (needed to workaround isort bug) from docker_tests.command_utils import run_command from docker_tests.constants import SOURCE_ROOT from docker_tests.docker_tests_utils import docker_image +# isort:on (needed to workaround isort bug) + AIRFLOW_WWW_USER_USERNAME = os.environ.get("_AIRFLOW_WWW_USER_USERNAME", "airflow") AIRFLOW_WWW_USER_PASSWORD = os.environ.get("_AIRFLOW_WWW_USER_PASSWORD", "airflow") DAG_ID = "example_bash_operator" diff --git a/docker_tests/test_examples_of_prod_image_building.py b/docker_tests/test_examples_of_prod_image_building.py index 978473e64e4..858e4c1e8fc 100644 --- a/docker_tests/test_examples_of_prod_image_building.py +++ b/docker_tests/test_examples_of_prod_image_building.py @@ -25,9 +25,12 @@ import pytest import requests +# isort:off (needed to workaround isort bug) from docker_tests.command_utils import run_command from docker_tests.constants import SOURCE_ROOT +# isort:on (needed to workaround isort bug) + DOCKER_EXAMPLES_DIR = SOURCE_ROOT / "docs" / "docker-stack" / "docker-examples" diff --git a/docker_tests/test_prod_image.py b/docker_tests/test_prod_image.py index f1d7fbd0180..8ad2d89a9f5 100644 --- a/docker_tests/test_prod_image.py +++ b/docker_tests/test_prod_image.py @@ -24,6 +24,7 @@ import pytest +# isort:off (needed to workaround isort bug) from docker_tests.command_utils import run_command from docker_tests.constants import SOURCE_ROOT from docker_tests.docker_tests_utils import ( @@ -32,6 +33,8 @@ run_bash_in_docker, run_python_in_docker, ) + +# isort:on (needed to workaround isort bug) from setup import PREINSTALLED_PROVIDERS INSTALLED_PROVIDER_PATH = SOURCE_ROOT / "scripts" / "ci" / "installed_providers.txt" diff --git a/docs/build_docs.py b/docs/build_docs.py index cd6c83249d4..273858be763 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -15,6 +15,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +""" +Builds documentation and runs spell checking + +# isort:skip_file (needed to workaround isort bug) +""" from __future__ import annotations import argparse @@ -33,6 +38,7 @@ from docs.exts.docs_build.github_action_utils import with_group from docs.exts.docs_build.package_filter import process_package_filters from docs.exts.docs_build.spelling_checks import SpellingError, display_spelling_error_summary + from rich.console import Console from tabulate import tabulate diff --git a/docs/exts/docs_build/dev_index_generator.py b/docs/exts/docs_build/dev_index_generator.py index f423ed20f77..0aee91b8f4a 100644 --- a/docs/exts/docs_build/dev_index_generator.py +++ b/docs/exts/docs_build/dev_index_generator.py @@ -23,8 +23,11 @@ import jinja2 +# isort:off (needed to workaround isort bug) from docs.exts.provider_yaml_utils import load_package_data +# isort:on (needed to workaround isort bug) + CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) DOCS_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir)) BUILD_DIR = os.path.abspath(os.path.join(DOCS_DIR, "_build")) diff --git a/docs/exts/docs_build/errors.py b/docs/exts/docs_build/errors.py index 187b89c15aa..321379896ca 100644 --- a/docs/exts/docs_build/errors.py +++ b/docs/exts/docs_build/errors.py @@ -23,7 +23,9 @@ from rich.console import Console from airflow.utils.code_utils import prepare_code_snippet -from docs.exts.docs_build.code_utils import CONSOLE_WIDTH + +from docs.exts.docs_build.code_utils import CONSOLE_WIDTH # isort:skip (needed to workaround isort bug) + CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__))) DOCS_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir)) diff --git a/docs/publish_docs.py b/docs/publish_docs.py index bf337995043..9a17aa29ef4 100755 --- a/docs/publish_docs.py +++ b/docs/publish_docs.py @@ -21,10 +21,13 @@ import argparse import os +# isort:off (needed to workaround isort bug) from exts.docs_build.docs_builder import AirflowDocsBuilder from exts.docs_build.package_filter import process_package_filters from exts.provider_yaml_utils import load_package_data +# isort:on (needed to workaround isort bug) + AIRFLOW_SITE_DIR = os.environ.get("AIRFLOW_SITE_DIRECTORY") diff --git a/kubernetes_tests/test_kubernetes_executor.py b/kubernetes_tests/test_kubernetes_executor.py index 7f40a976103..95afb25590f 100644 --- a/kubernetes_tests/test_kubernetes_executor.py +++ b/kubernetes_tests/test_kubernetes_executor.py @@ -20,7 +20,7 @@ import pytest -from kubernetes_tests.test_base import EXECUTOR, TestBase +from kubernetes_tests.test_base import EXECUTOR, TestBase # isort:skip (needed to workaround isort bug) @pytest.mark.skipif(EXECUTOR != "KubernetesExecutor", reason="Only runs on KubernetesExecutor") diff --git a/kubernetes_tests/test_other_executors.py b/kubernetes_tests/test_other_executors.py index f517fdc82fd..42a8258871a 100644 --- a/kubernetes_tests/test_other_executors.py +++ b/kubernetes_tests/test_other_executors.py @@ -20,7 +20,7 @@ import pytest -from kubernetes_tests.test_base import EXECUTOR, TestBase +from kubernetes_tests.test_base import EXECUTOR, TestBase # isort:skip (needed to workaround isort bug) # These tests are here because only KubernetesExecutor can run the tests in diff --git a/pyproject.toml b/pyproject.toml index 39cfef27e47..a40b7d91799 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,5 +35,6 @@ known_first_party = ["airflow", "airflow_breeze", "docker_tests", "docs", "kuber # The test_python.py is needed because adding __future__.annotations breaks runtime checks that are # needed for the test to work skip = ["build", ".tox", "venv", "tests/decorators/test_python.py"] +lines_between_types = 0 skip_glob = ["*.pyi"] profile = "black" diff --git a/setup.py b/setup.py index 2484b2076d0..c7af47ffcc5 100644 --- a/setup.py +++ b/setup.py @@ -377,7 +377,10 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve "freezegun", "gitpython", "ipdb", - "isort", + # make sure that we are using stable sorting order from 5.* version (some changes were introduced + # in 5.11.3. Black is not compatible yet, so we need to limit isort + # we can remove the limit when black and isort agree on the order + "isort==5.11.2", "jira", "jsondiff", "mongomock",