diff --git a/.github/workflows/ci-amd-arm.yml b/.github/workflows/ci-amd-arm.yml index ab34ba25882bb..777e47e586a28 100644 --- a/.github/workflows/ci-amd-arm.yml +++ b/.github/workflows/ci-amd-arm.yml @@ -91,7 +91,6 @@ jobs: kubernetes-versions-list-as-string: >- ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} - mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} platform: ${{ steps.selective-checks.outputs.platform }} @@ -115,7 +114,7 @@ jobs: run-go-sdk-tests: ${{ steps.selective-checks.outputs.run-go-sdk-tests }} run-helm-tests: ${{ steps.selective-checks.outputs.run-helm-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} - run-mypy: ${{ steps.selective-checks.outputs.run-mypy }} + run-mypy-providers: ${{ steps.selective-checks.outputs.run-mypy-providers }} run-remote-logging-elasticsearch-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }} run-remote-logging-s3-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }} run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} @@ -307,8 +306,6 @@ jobs: with: runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} - run-mypy: ${{ needs.build-info.outputs.run-mypy }} - mypy-checks: ${{ needs.build-info.outputs.mypy-checks }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} branch: ${{ needs.build-info.outputs.default-branch }} canary-run: ${{ needs.build-info.outputs.canary-run }} @@ -333,6 +330,51 @@ jobs: DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + mypy-providers: + timeout-minutes: 45 + name: "MyPy providers checks" + needs: [build-info, build-ci-images] + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + if: needs.build-info.outputs.run-mypy-providers == 'true' + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Free up disk space" + shell: bash + run: ./scripts/tools/free_up_disk_space.sh + - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ needs.build-info.outputs.platform }} + python: "${{ needs.build-info.outputs.default-python-version }}" + use-uv: ${{ needs.build-info.outputs.use-uv }} + make-mnt-writeable-and-cleanup: true + id: breeze + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek + with: + python-version: ${{steps.breeze.outputs.host-python-version}} + platform: ${{ needs.build-info.outputs.platform }} + save-cache: false + - name: "MyPy checks for providers" + run: prek --color always --verbose --stage manual mypy-providers --all-files + env: + VERBOSE: "false" + COLUMNS: "202" + SKIP_GROUP_OUTPUT: "true" + DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} + RUFF_FORMAT: "github" + INCLUDE_MYPY_VOLUME: "false" + providers: name: "provider distributions tests" uses: ./.github/workflows/test-providers.yml @@ -895,6 +937,7 @@ jobs: - build-prod-images - ci-image-checks - generate-constraints + - mypy-providers - providers - tests-helm - tests-integration-system diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 9d21ad2f92c40..1f25943204179 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -28,14 +28,6 @@ on: # yamllint disable-line rule:truthy description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string - run-mypy: - description: "Whether to run mypy checks (true/false)" - required: true - type: string - mypy-checks: - description: "List of folders to run mypy checks on" - required: false - type: string python-versions-list-as-string: description: "The list of python versions as string separated by spaces" required: true @@ -169,55 +161,6 @@ jobs: run: cat ~/.cache/prek/prek.log || true if: failure() - mypy: - timeout-minutes: 45 - name: "MyPy checks" - runs-on: ${{ fromJSON(inputs.runners) }} - if: inputs.run-mypy == 'true' - strategy: - fail-fast: false - matrix: - mypy-check: ${{ fromJSON(inputs.mypy-checks) }} - env: - PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: "Free up disk space" - shell: bash - run: ./scripts/tools/free_up_disk_space.sh - - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" - uses: ./.github/actions/prepare_breeze_and_image - with: - platform: ${{ inputs.platform }} - python: "${{ inputs.default-python-version }}" - use-uv: ${{ inputs.use-uv }} - make-mnt-writeable-and-cleanup: true - id: breeze - - name: "Install prek" - uses: ./.github/actions/install-prek - id: prek - with: - python-version: ${{steps.breeze.outputs.host-python-version}} - platform: ${{ inputs.platform }} - save-cache: false - - name: "MyPy checks for ${{ matrix.mypy-check }}" - run: prek --color always --verbose --stage manual "$MYPY_CHECK" --all-files - env: - VERBOSE: "false" - COLUMNS: "202" - SKIP_GROUP_OUTPUT: "true" - DEFAULT_BRANCH: ${{ inputs.branch }} - RUFF_FORMAT: "github" - INCLUDE_MYPY_VOLUME: "false" - MYPY_CHECK: ${{ matrix.mypy-check }} - build-docs: timeout-minutes: 150 name: "Build documentation" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d9eaf4abe0ea..5140ae80fe5e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1014,39 +1014,23 @@ repos: ^uv\.lock$ pass_filenames: false require_serial: true - ## ADD MOST PREK HOOK ABOVE THAT LINE - # The below prek hooks are those requiring CI image to be built - ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE - id: mypy-dev - stages: ['pre-push'] name: Run mypy for dev language: python - entry: ./scripts/ci/prek/mypy.py - files: ^dev/.*\.py$|^scripts/.*\.py$ - require_serial: true - - id: mypy-dev - stages: ['manual'] - name: Run mypy for dev (manual) - language: python - entry: ./scripts/ci/prek/mypy_folder.py dev scripts + entry: ./scripts/ci/prek/mypy_local_folder.py dev scripts pass_filenames: false files: ^.*\.py$ require_serial: true - id: mypy-devel-common - stages: ['pre-push'] name: Run mypy for devel-common language: python - entry: ./scripts/ci/prek/mypy.py - files: ^devel-common/.*\.py$ - require_serial: true - - id: mypy-devel-common - stages: ['manual'] - name: Run mypy for devel-common (manual) - language: python - entry: ./scripts/ci/prek/mypy_folder.py devel-common + entry: ./scripts/ci/prek/mypy_local_folder.py devel-common pass_filenames: false files: ^.*\.py$ require_serial: true + ## ADD MOST PREK HOOK ABOVE THAT LINE + # The below prek hooks are those requiring CI image to be built + ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE - id: check-template-fields-valid name: Check templated fields mapped in operators/sensors language: python diff --git a/AGENTS.md b/AGENTS.md index 89bbd76620f6a..b9ef07b381d46 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,8 @@ - **Run other suites of tests** `breeze testing ` (test groups: `airflow-ctl-tests`, `docker-compose-tests`, `task-sdk-tests`) - **Run scripts tests:** `uv run --project scripts pytest scripts/tests/ -xvs` - **Run Airflow CLI:** `breeze run airflow dags list` -- **Type-check:** `breeze run mypy path/to/code` +- **Type-check (non-providers):** `uv run --project --with "apache-airflow-devel-common[mypy]" mypy path/to/code` +- **Type-check (providers):** `breeze run mypy path/to/code` - **Lint with ruff only:** `prek run ruff --from-ref ` - **Format with ruff only:** `prek run ruff-format --from-ref ` - **Run regular (fast) static checks:** `prek run --from-ref --stage pre-commit` @@ -147,7 +148,7 @@ code review checklist in [`.github/instructions/code-review.instructions.md`](.g 3. Confirm the code follows the project's coding standards and architecture boundaries described in this file. 4. Run regular (fast) static checks (`prek run --from-ref --stage pre-commit`) - and fix any failures. + and fix any failures. This includes mypy checks for non-provider projects (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common). 5. Run manual (slower) checks (`prek run --from-ref --stage manual`) and fix any failures. 6. Run relevant individual tests and confirm they pass. 7. Find which tests to run for the changes with selective-checks and run those tests in parallel to confirm they pass and check for CI-specific issues. diff --git a/airflow-core/.pre-commit-config.yaml b/airflow-core/.pre-commit-config.yaml index 5d2e3e7fe2b9b..0567187dbf686 100644 --- a/airflow-core/.pre-commit-config.yaml +++ b/airflow-core/.pre-commit-config.yaml @@ -221,23 +221,15 @@ repos: additional_dependencies: ['pnpm@10.25.0'] pass_filenames: true require_serial: true - ## ADD MOST PREK HOOK ABOVE THAT LINE - # The below prek hooks are those requiring CI image to be built - id: mypy-airflow-core - stages: ['pre-push'] name: Run mypy for airflow-core language: python - entry: ../scripts/ci/prek/mypy.py - files: ^.*\.py$ - require_serial: true - - id: mypy-airflow-core - stages: ['manual'] - name: Run mypy for airflow-core (manual) - language: python - entry: ../scripts/ci/prek/mypy_folder.py airflow-core + entry: ../scripts/ci/prek/mypy_local_folder.py airflow-core pass_filenames: false files: ^.*\.py$ require_serial: true + ## ADD MOST PREK HOOK ABOVE THAT LINE + # The below prek hooks are those requiring CI image to be built - id: generate-openapi-spec name: Generate the FastAPI API spec language: python diff --git a/airflow-core/src/airflow/utils/log/non_caching_file_handler.py b/airflow-core/src/airflow/utils/log/non_caching_file_handler.py index aa0ca9864e2ea..ad3c0dbe27974 100644 --- a/airflow-core/src/airflow/utils/log/non_caching_file_handler.py +++ b/airflow-core/src/airflow/utils/log/non_caching_file_handler.py @@ -25,7 +25,7 @@ def make_file_io_non_caching(io: IO[str]) -> IO[str]: try: fd = io.fileno() - os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) + os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) # type: ignore[attr-defined] except Exception: # in case either file descriptor cannot be retrieved or fadvise is not available # we should simply return the wrapper retrieved by FileHandler's open method diff --git a/airflow-ctl/.pre-commit-config.yaml b/airflow-ctl/.pre-commit-config.yaml index e63268b077ef8..c45a1985ec129 100644 --- a/airflow-ctl/.pre-commit-config.yaml +++ b/airflow-ctl/.pre-commit-config.yaml @@ -25,21 +25,9 @@ repos: - repo: local hooks: - id: mypy-airflow-ctl - stages: ['pre-push'] name: Run mypy for airflow-ctl language: python - entry: ../scripts/ci/prek/mypy.py - files: - (?x) - ^src/airflowctl/.*\.py$| - ^tests/.*\.py$ - exclude: .*generated.py - require_serial: true - - id: mypy-airflow-ctl - stages: ['manual'] - name: Run mypy for airflow-ctl (manual) - language: python - entry: ../scripts/ci/prek/mypy_folder.py airflow-ctl + entry: ../scripts/ci/prek/mypy_local_folder.py airflow-ctl pass_filenames: false files: ^.*\.py$ require_serial: true diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 1d009b92cefb4..23256c6c0e026 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -173,17 +173,18 @@ But you can run prek hooks manually as needed. prek - Run only mypy check on your staged airflow and dev files by specifying the - ``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be specified): + ``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be specified). + For non-provider projects, mypy runs locally via ``uv`` (no breeze image needed): .. code-block:: bash - prek mypy-airflow-core mypy-dev --stage pre-push + prek mypy-airflow-core mypy-dev - Run only mypy airflow checks on all "airflow-core" files by using: .. code-block:: bash - prek mypy-airflow-core --all-files --stage pre-push + prek mypy-airflow-core --all-files - Run all pre-commit stage hooks on all files by using: @@ -279,41 +280,40 @@ them manually by running ``prek --stage manual ``. Mypy checks ----------- -When we run mypy checks locally when pushing a change to PR, the ``mypy-*`` checks is run, ``mypy-airflow``, -``mypy-dev``, ``mypy-providers``, ``mypy-airflow-ctl``, depending on the files you are changing. The mypy checks -are run by passing those changed files to mypy. This is way faster than running checks for all files (even -if mypy cache is used - especially when you change a file in Airflow core that is imported and used by many -files). You also need to have ``breeze ci-image build --python 3.10`` built locally to run the mypy checks. +When we run mypy checks locally, the ``mypy-*`` checks run depending on the files you are changing: +``mypy-airflow-core``, ``mypy-dev``, ``mypy-providers``, ``mypy-task-sdk``, ``mypy-airflow-ctl``, etc. -However, in some cases, it produces different results than when running checks for the whole set -of files, because ``mypy`` does not even know that some types are defined in other files and it might not -be able to follow imports properly if they are dynamic. Therefore in CI we run ``mypy`` check for whole -directories (``airflow`` - excluding providers, ``providers``, ``dev`` and ``docs``) to make sure -that we catch all ``mypy`` errors - so you can experience different results when running mypy locally and -in CI. If you want to run mypy checks for all files locally, you can do it by running the following -command (example for ``airflow`` files): +For **non-provider projects** (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common), mypy +runs locally using the ``uv`` virtualenv — no breeze CI image is needed. These checks run as regular +prek hooks in the ``pre-commit`` stage, checking whole directories at once. This means they run both +as part of local commits and as part of regular static checks in CI (not as separate mypy CI jobs). +You can also run mypy directly. Use ``--frozen`` to avoid updating ``uv.lock``: .. code-block:: bash - prek --stage manual mypy- --all-files + uv run --frozen --project --with "apache-airflow-devel-common[mypy]" mypy path/to/code -For example: +To run the prek hook for a specific project (example for ``airflow-core`` files): .. code-block:: bash - prek --stage manual mypy-airflow --all-files + prek mypy-airflow-core --all-files To show unused mypy ignores for any providers/airflow etc, eg: run below command: .. code-block:: bash + export SHOW_UNUSED_MYPY_WARNINGS=true - prek --stage manual mypy-airflow --all-files + prek mypy-airflow-core --all-files + +For non-provider projects, the local mypy cache is stored in ``.mypy_cache`` at the repo root. + +For **providers**, mypy still runs via breeze (``breeze run mypy``) as a separate CI job and requires +``breeze ci-image build --python 3.10`` to be built locally. Providers use a separate docker-volume +(called ``mypy-cache-volume``) that keeps the cache of last MyPy execution. -MyPy uses a separate docker-volume (called ``mypy-cache-volume``) that keeps the cache of last MyPy -execution in order to speed MyPy checks up (sometimes by order of magnitude). While in most cases MyPy -will handle refreshing the cache when and if needed, there are some cases when it won't (cache invalidation -is the hard problem in computer science). This might happen for example when we upgrade MyPY. In such -cases you might need to manually remove the cache volume by running ``breeze down --cleanup-mypy-cache``. +To clear all mypy caches (both local ``.mypy_cache`` and the Docker volume), run +``breeze down --cleanup-mypy-cache``. ----------- diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 5c70a495afc59..f10ed0ec0bd3c 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -932,6 +932,10 @@ def down(preserve_volumes: bool, cleanup_mypy_cache: bool, cleanup_build_cache: if cleanup_mypy_cache: command_to_execute = ["docker", "volume", "rm", "--force", "mypy-cache-volume"] run_command(command_to_execute) + local_mypy_cache = AIRFLOW_ROOT_PATH / ".mypy_cache" + if local_mypy_cache.exists(): + console_print(f"\n[info]Removing local mypy cache: {local_mypy_cache}\n") + shutil.rmtree(local_mypy_cache) if cleanup_build_cache: command_to_execute = ["docker", "volume", "rm", "--force", "airflow-cache-volume"] run_command(command_to_execute) @@ -1070,6 +1074,10 @@ def doctor(ctx): console_print("\n[info]Cleaning mypy cache...\n") command_to_execute = ["docker", "volume", "rm", "--force", "mypy-cache-volume"] run_command(command_to_execute) + local_mypy_cache = AIRFLOW_ROOT_PATH / ".mypy_cache" + if local_mypy_cache.exists(): + console_print(f"\n[info]Removing local mypy cache: {local_mypy_cache}\n") + shutil.rmtree(local_mypy_cache) console_print("\n[info]Cleaning build cache...\n") command_to_execute = ["docker", "volume", "rm", "--force", "airflow-cache-volume"] diff --git a/dev/breeze/src/airflow_breeze/commands/registry_commands.py b/dev/breeze/src/airflow_breeze/commands/registry_commands.py index f818a16ea01a9..0655deb51d182 100644 --- a/dev/breeze/src/airflow_breeze/commands/registry_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/registry_commands.py @@ -160,7 +160,7 @@ def _read_provider_yaml_info(provider_id: str) -> tuple[str, list[str]]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] provider_yaml_path = _find_provider_yaml(provider_id) with open(provider_yaml_path) as f: diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 25f9a945473cb..553f0379b4cc1 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -533,7 +533,7 @@ def load_pyproject_toml(pyproject_toml_file_path: Path) -> dict[str, Any]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] toml_content = pyproject_toml_file_path.read_text() syntax = Syntax(toml_content, "toml", theme="ansi_dark", line_numbers=True) try: diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 99e74fcfa5039..6a756289db649 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -870,58 +870,21 @@ def _should_be_run(self, source_area: FileGroupForCi) -> bool: return False @cached_property - def mypy_checks(self) -> list[str]: - checks_to_run: list[str] = [] - if ( - self._matching_files(FileGroupForCi.DEVEL_TOML_FILES, CI_FILE_GROUP_MATCHES) - and self._default_branch == "main" - ): - return [ - "mypy-airflow-core", - "mypy-providers", - "mypy-dev", - "mypy-task-sdk", - "mypy-devel-common", - "mypy-airflow-ctl", - ] - if ( - self._matching_files(FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES, CI_FILE_GROUP_MATCHES) - or self.full_tests_needed - ): - checks_to_run.append("mypy-airflow-core") - if ( + def run_mypy_providers(self) -> bool: + # Non-provider mypy checks run as part of regular static checks (prek hooks). + # Only provider mypy needs a separate CI job (requires the CI Docker image with breeze). + return ( self._matching_files(FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES) or self._matching_files( FileGroupForCi.ALL_PROVIDERS_DISTRIBUTION_CONFIG_FILES, CI_FILE_GROUP_MATCHES ) or self._are_all_providers_affected() - ) and self._default_branch == "main": - checks_to_run.append("mypy-providers") - if ( - self._matching_files(FileGroupForCi.ALL_DEV_PYTHON_FILES, CI_FILE_GROUP_MATCHES) - or self.full_tests_needed - ): - checks_to_run.append("mypy-dev") - if ( - self._matching_files(FileGroupForCi.TASK_SDK_FILES, CI_FILE_GROUP_MATCHES) - or self.full_tests_needed - ): - checks_to_run.append("mypy-task-sdk") - if ( - self._matching_files(FileGroupForCi.ALL_DEVEL_COMMON_PYTHON_FILES, CI_FILE_GROUP_MATCHES) - or self.full_tests_needed - ): - checks_to_run.append("mypy-devel-common") - if ( - self._matching_files(FileGroupForCi.ALL_AIRFLOW_CTL_PYTHON_FILES, CI_FILE_GROUP_MATCHES) + or ( + self._matching_files(FileGroupForCi.DEVEL_TOML_FILES, CI_FILE_GROUP_MATCHES) + and self._default_branch == "main" + ) or self.full_tests_needed - ): - checks_to_run.append("mypy-airflow-ctl") - return checks_to_run - - @cached_property - def run_mypy(self) -> bool: - return self.mypy_checks != [] + ) and self._default_branch == "main" @cached_property def run_python_scans(self) -> bool: @@ -1499,6 +1462,23 @@ def skip_prek_hooks(self) -> str: # only skip provider validation if none of the provider.yaml and provider # python files changed because validation also walks through all the provider python files prek_hooks_to_skip.add("check-provider-yaml-valid") + # Non-provider mypy checks run as prek hooks in static checks. + # Skip them when their relevant files haven't changed, unless devel-common + # pyproject.toml changes on main (which affects all mypy checks). + if not ( + self._matching_files(FileGroupForCi.DEVEL_TOML_FILES, CI_FILE_GROUP_MATCHES) + and self._default_branch == "main" + ): + if not self._matching_files(FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-airflow-core") + if not self._matching_files(FileGroupForCi.ALL_DEV_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-dev") + if not self._matching_files(FileGroupForCi.TASK_SDK_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-task-sdk") + if not self._matching_files(FileGroupForCi.ALL_DEVEL_COMMON_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-devel-common") + if not self._matching_files(FileGroupForCi.ALL_AIRFLOW_CTL_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-airflow-ctl") return ",".join(sorted(prek_hooks_to_skip)) @cached_property diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index b48062f93d164..2a11a4ef29d2a 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -384,7 +384,7 @@ def test_apply_version_suffix_to_provider_pyproject_toml( try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] from unittest.mock import patch # Get the original provider details @@ -474,7 +474,7 @@ def test_apply_version_suffix_to_non_provider_pyproject_tomls( try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] distribution_paths = [AIRFLOW_ROOT_PATH / distribution for distribution in distributions] original_pyproject_toml_paths = [path / "pyproject.toml" for path in distribution_paths] original_contents = [path.read_text() for path in original_pyproject_toml_paths] diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 5d6ff3768722e..90e5551ce3377 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -98,59 +98,57 @@ _get_test_list_as_json(_split_list(sorted(LIST_OF_ALL_PROVIDER_TESTS), 5)) ) -ALL_MYPY_CHECKS_ARRAY = [ - "mypy-airflow-core", - "mypy-providers", - "mypy-dev", - "mypy-task-sdk", - "mypy-devel-common", - "mypy-airflow-ctl", -] - -ALL_MYPY_CHECKS = str(ALL_MYPY_CHECKS_ARRAY) - -ALL_MYPY_CHECKS_EXCEPT_PROVIDERS = str( - [check for check in ALL_MYPY_CHECKS_ARRAY if check != "mypy-providers"] -) ALL_SKIPPED_COMMITS_ON_NO_CI_IMAGE = ( "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED = "identity,update-uv-lock" ALL_SKIPPED_COMMITS_IF_NO_UI = ( - "identity,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "identity,mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" +) +ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = ( + "identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,update-uv-lock" ) -ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = "identity,lint-helm-chart,update-uv-lock" ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = ( - "identity,lint-helm-chart,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = ( "check-provider-yaml-valid,identity," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = ( "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS = ( "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = ( - "check-provider-yaml-valid,flynt,identity,lint-helm-chart,update-uv-lock" + "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED = ( "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) @@ -244,8 +242,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": None, "providers-test-types-list-as-strings-in-json": None, "individual-providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="No tests on simple change", ) @@ -278,8 +275,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests should be run when API file changed", ) @@ -303,8 +299,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests should be run when fastapi files change", ) @@ -328,8 +323,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests should run when API test files change", ) @@ -349,7 +343,11 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-unit-tests": "true", "run-amazon-tests": "false", "docs-build": "true", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( _get_test_list_as_json( @@ -358,8 +356,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ), "providers-test-types-list-as-strings-in-json": None, "individual-providers-test-types-list-as-strings-in-json": None, - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", "skip-providers-tests": "true", }, id="Only Serialization tests", @@ -388,8 +385,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests and docs should run on API change", ) @@ -431,8 +427,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Selected Providers and docs should run", ) @@ -474,8 +469,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", "skip-providers-tests": "false", }, id="Selected Providers should run when system tests are modified", @@ -521,8 +515,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", "skip-providers-tests": "false", }, id="Selected Providers and docs should run when both system tests and tests are modified", @@ -568,8 +561,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", "skip-providers-tests": "false", }, id="Selected Providers and docs should run when both system tests and tests are modified for more than one provider", @@ -595,8 +587,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": None, "providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Only docs builds should run - no tests needed", ) @@ -619,13 +610,16 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-task-sdk-integration-tests": "true", "docs-build": "true", "full-tests-needed": "false", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "skip-providers-tests": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": "['mypy-providers', 'mypy-task-sdk']", + "run-mypy-providers": "true", }, id="Task SDK source file changed - Task SDK, Core and provider tests should run", ) @@ -651,8 +645,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Task SDK integration tests files changed - " "Task SDK integration tests and prod image build should run but no other tests", @@ -676,11 +669,14 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-airflow-ctl-integration-tests": "true", "docs-build": "true", "full-tests-needed": "false", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-ctl']", + "run-mypy-providers": "false", }, id="Airflow CTL source file changed - Airflow CTL tests should run", ) @@ -706,8 +702,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Airflow CTL integration tests files changed - " "Airflow CTL integration tests and prod image build should run but no other tests", @@ -748,8 +743,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Helm tests, providers (both upstream and downstream)," "kubernetes tests and docs should run", @@ -812,8 +806,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): }, ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Helm tests, http and all relevant providers, kubernetes tests and " "docs should run even if unimportant files were added", @@ -880,8 +873,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Docs should run even if unimportant files were added and prod image " "should be build for chart changes", @@ -907,8 +899,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "true", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run and upgrading to newer requirements as dependencies change", ) @@ -947,8 +938,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Providers tests run including amazon tests if only amazon provider.yaml files changed", ), @@ -980,8 +970,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Providers tests run without amazon tests if no amazon file changed", ), @@ -1018,8 +1007,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Providers tests run including amazon tests if amazon provider files changed", ), @@ -1042,7 +1030,11 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "false", "run-kubernetes-tests": "false", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "identity,lint-helm-chart," + "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( [{"description": "Always", "test_types": "Always"}] @@ -1055,8 +1047,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core', 'mypy-providers']", + "run-mypy-providers": "true", }, id="Only Always and common providers tests should run when only common.io and tests/always changed", ), @@ -1078,8 +1069,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests to run when standard operator changed", ), @@ -1103,8 +1093,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests should be run when tests/utils/ change", ) @@ -1132,8 +1121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "testable-core-integrations": "['kerberos', 'otel', 'redis']", "testable-providers-integrations": "['celery', 'cassandra', 'drill', 'elasticsearch', 'tinkerpop', 'kafka', " "'mongo', 'pinot', 'qdrant', 'redis', 'trino', 'ydb']", - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests should be run when devel-common/ change", ) @@ -1153,8 +1141,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "full-tests-needed": "false", "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS, "upgrade-to-newer-dependencies": "false", - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", "run-helm-tests": "false", "run-ui-tests": "true", "run-ui-e2e-tests": "true", @@ -1184,8 +1171,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": None, "providers-test-types-list-as-strings-in-json": None, "individual-providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Run docs-build for RELEASE_NOTES.rst", ), @@ -1202,13 +1188,16 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-unit-tests": "false", "run-amazon-tests": "false", "docs-build": "true", - "skip-prek-hooks": "check-provider-yaml-valid,flynt,identity,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock", + "skip-prek-hooks": ( + "check-provider-yaml-valid,flynt,identity," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": None, "providers-test-types-list-as-strings-in-json": None, "individual-providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Run docs-build for chart/RELEASE_NOTES.rst", ), @@ -1230,8 +1219,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": None, "providers-test-types-list-as-strings-in-json": None, "individual-providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Run docs-build for SECURITY.md", ), @@ -1244,8 +1232,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("devel-common/pyproject.toml",), { - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All mypy checks should run when devel-common/pyproject.toml changes", ) @@ -1526,8 +1513,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including all providers when git provider is changed" "(special case for now)", @@ -1561,8 +1547,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including all providers when full tests are needed, " "and all versions are required.", @@ -1596,8 +1581,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including all providers when full tests are needed " "but with single python and kubernetes if `default versions only` label is set", @@ -1631,8 +1615,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", "run-ui-e2e-tests": "true", }, id="Everything should run including all providers when full tests are needed " @@ -1668,8 +1651,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including all providers when full tests are needed " "but with single python and kubernetes if `latest versions only` label is set", @@ -1704,8 +1686,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including full providers when full " "tests are needed even with different label set as well", @@ -1738,8 +1719,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including full providers when " "full tests are needed even if no files are changed", @@ -1766,8 +1746,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "skip-prek-hooks": All_SKIPPED_COMMITS_IF_NON_MAIN_BRANCH, "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS_EXCEPT_PROVIDERS, + "run-mypy-providers": "false", }, id="Everything should run except Providers and lint prek " "when full tests are needed for non-main branch", @@ -1809,8 +1788,7 @@ def test_expected_output_full_tests_needed( "full-tests-needed": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Nothing should run if only non-important files changed", ), @@ -1835,8 +1813,7 @@ def test_expected_output_full_tests_needed( "core-test-types-list-as-strings-in-json": json.dumps( [{"description": "Always", "test_types": "Always"}] ), - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="No Helm tests, No providers no lint charts, should run if " "only chart/providers changed in non-main but PROD image should be built", @@ -1863,8 +1840,7 @@ def test_expected_output_full_tests_needed( "core-test-types-list-as-strings-in-json": json.dumps( [{"description": "Always...CLI", "test_types": "Always CLI"}] ), - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, id="Only CLI tests and Kubernetes tests should run if cli/chart files changed in non-main branch", ), @@ -1887,8 +1863,7 @@ def test_expected_output_full_tests_needed( "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, id="All tests except Providers and helm lint prek " "should run if core file changed in non-main branch", @@ -1930,8 +1905,7 @@ def test_expected_output_pull_request_v2_7( "upgrade-to-newer-dependencies": "false", "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED, "core-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="No tests run on push if only text non-doc files changed", ), @@ -1951,8 +1925,7 @@ def test_expected_output_pull_request_v2_7( "docs-list-as-string": None, "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="No tests run on push if only text non-doc files changed in non-main branch", ), @@ -1973,8 +1946,7 @@ def test_expected_output_pull_request_v2_7( "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", "run-ui-e2e-tests": "false", }, id="All tests run on push if core file changed", @@ -2029,8 +2001,7 @@ def test_expected_output_push( "upgrade-to-newer-dependencies": "false", "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED, "core-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Nothing should run if only non-important files changed", ), @@ -2047,13 +2018,16 @@ def test_expected_output_push( "skip-providers-tests": "true", "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( [{"description": "Always", "test_types": "Always"}] ), - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, id="Only Always and docs build should run if only system tests changed", ), @@ -2080,7 +2054,10 @@ def test_expected_output_push( "apache.kafka cncf.kubernetes common.compat common.messaging common.sql databricks facebook google hashicorp http microsoft.azure " "microsoft.mssql mysql openlineage oracle postgres " "presto salesforce samba sftp ssh standard trino", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_UI, + "skip-prek-hooks": ( + "identity,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( @@ -2099,8 +2076,7 @@ def test_expected_output_push( } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core', 'mypy-providers']", + "run-mypy-providers": "true", }, id="CLI tests and Google-related provider tests should run if cli/chart files changed but " "prod image should be build too and k8s tests too", @@ -2117,12 +2093,15 @@ def test_expected_output_push( "skip-providers-tests": "true", "docs-build": "true", "docs-list-as-string": "apache-airflow", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, id="Tests for all airflow core types except providers should run if model file changed", ), @@ -2141,8 +2120,7 @@ def test_expected_output_push( "upgrade-to-newer-dependencies": "false", "skip-prek-hooks": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", ), @@ -2179,8 +2157,7 @@ def test_expected_output_push( } ] ), - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Trigger openlineage and related providers tests when Assets files changed", ), @@ -2243,8 +2220,7 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(mock_get, gith "skip-prek-hooks": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, "upgrade-to-newer-dependencies": ("true" if github_event == GithubEvents.SCHEDULE else "false"), "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, str(stderr), ) @@ -2293,8 +2269,7 @@ def test_files_provided_trigger_full_build_for_any_event_type(mock_get, github_e "skip-prek-hooks": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, "upgrade-to-newer-dependencies": ("true" if github_event == GithubEvents.SCHEDULE else "false"), "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, str(stderr), ) @@ -2607,8 +2582,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("README.md",), { - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, "main", (), @@ -2617,8 +2591,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("airflow-core/src/airflow/cli/file.py",), { - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, "main", (), @@ -2627,8 +2600,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("airflow-core/src/airflow/models/file.py",), { - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, "main", (), @@ -2637,8 +2609,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("task-sdk/src/airflow/sdk/a_file.py",), { - "run-mypy": "true", - "mypy-checks": "['mypy-providers', 'mypy-task-sdk']", + "run-mypy-providers": "true", }, "main", (), @@ -2647,8 +2618,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("dev/a_package/a_file.py",), { - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, "main", (), @@ -2657,8 +2627,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("readme.md",), { - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, "main", ("full tests needed",), diff --git a/devel-common/src/sphinx_exts/docs_build/package_filter.py b/devel-common/src/sphinx_exts/docs_build/package_filter.py index f4d7c037232cf..3cd83951be838 100644 --- a/devel-common/src/sphinx_exts/docs_build/package_filter.py +++ b/devel-common/src/sphinx_exts/docs_build/package_filter.py @@ -64,7 +64,7 @@ def find_packages_to_build(available_packages: list[str], package_filters: list[ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] read_toml = tomllib.loads(pyproject_toml_path.read_text()) package_name = read_toml["project"]["name"] if package_name == "apache-airflow": diff --git a/scripts/ci/prek/check_extra_packages_ref.py b/scripts/ci/prek/check_extra_packages_ref.py index 6e3bd8bcd6b2f..009e4d6c66266 100755 --- a/scripts/ci/prek/check_extra_packages_ref.py +++ b/scripts/ci/prek/check_extra_packages_ref.py @@ -39,7 +39,7 @@ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] EXTRA_PACKAGES_REF_FILE = AIRFLOW_ROOT_PATH / "airflow-core" / "docs" / "extra-packages-ref.rst" PYPROJECT_TOML_FILE_PATH = AIRFLOW_ROOT_PATH / "pyproject.toml" diff --git a/scripts/ci/prek/check_shared_distributions_structure.py b/scripts/ci/prek/check_shared_distributions_structure.py index ac08c28924b36..3a7366bd300cd 100755 --- a/scripts/ci/prek/check_shared_distributions_structure.py +++ b/scripts/ci/prek/check_shared_distributions_structure.py @@ -35,7 +35,7 @@ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] from common_prek_utils import AIRFLOW_ROOT_PATH, console diff --git a/scripts/ci/prek/check_shared_distributions_usage.py b/scripts/ci/prek/check_shared_distributions_usage.py index c5759d76d73f2..5a734fa255a3f 100755 --- a/scripts/ci/prek/check_shared_distributions_usage.py +++ b/scripts/ci/prek/check_shared_distributions_usage.py @@ -39,7 +39,7 @@ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] from common_prek_utils import AIRFLOW_ROOT_PATH, console, insert_documentation diff --git a/scripts/ci/prek/check_version_consistency.py b/scripts/ci/prek/check_version_consistency.py index ebcf457d0da5f..2711b55cccd39 100755 --- a/scripts/ci/prek/check_version_consistency.py +++ b/scripts/ci/prek/check_version_consistency.py @@ -32,7 +32,7 @@ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] from common_prek_utils import ( AIRFLOW_CORE_SOURCES_PATH, diff --git a/scripts/ci/prek/mypy_local_folder.py b/scripts/ci/prek/mypy_local_folder.py new file mode 100755 index 0000000000000..e578079b11fa8 --- /dev/null +++ b/scripts/ci/prek/mypy_local_folder.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.10,<3.11" +# dependencies = [ +# "rich>=13.6.0", +# ] +# /// +"""Run mypy on entire folders using local virtualenv (uv) instead of breeze. + +Used for non-provider projects: airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common. +""" + +from __future__ import annotations + +import os +import re +import shlex +import subprocess +import sys + +from common_prek_utils import ( + AIRFLOW_ROOT_PATH, +) + +CI = os.environ.get("CI") + +try: + from rich.console import Console + + console = Console(width=400, color_system="standard") +except ImportError: + console = None # type: ignore[assignment] + +if __name__ not in ("__main__", "__mp_main__"): + raise SystemExit( + "This file is intended to be executed as an executable program. You cannot use it as a module." + f"To run this script, run the ./{__file__} command" + ) + +ALLOWED_FOLDERS = [ + "airflow-core", + "dev", + "scripts", + "devel-common", + "task-sdk", + "airflow-ctl", +] + +# Map folder(s) to the uv project to use for running mypy. +# When multiple folders are checked together (e.g. dev + scripts), the first folder's project is used. +FOLDER_TO_PROJECT = { + "airflow-core": "airflow-core", + "task-sdk": "task-sdk", + "airflow-ctl": "airflow-ctl", + "devel-common": "devel-common", + "dev": "dev", + "scripts": "scripts", +} + +if len(sys.argv) < 2: + if console: + console.print(f"[yellow]You need to specify the folder to test as parameter: {ALLOWED_FOLDERS}\n") + else: + print(f"You need to specify the folder to test as parameter: {ALLOWED_FOLDERS}") + sys.exit(1) + +mypy_folders = sys.argv[1:] + +show_unused_warnings = os.environ.get("SHOW_UNUSED_MYPY_WARNINGS", "false") +show_unreachable_warnings = os.environ.get("SHOW_UNREACHABLE_MYPY_WARNINGS", "false") + +for mypy_folder in mypy_folders: + if mypy_folder not in ALLOWED_FOLDERS: + if console: + console.print( + f"\n[red]ERROR: Folder `{mypy_folder}` is wrong.[/]\n\n" + f"All folders passed should be one of those: {ALLOWED_FOLDERS}\n" + ) + else: + print( + f"\nERROR: Folder `{mypy_folder}` is wrong.\n\n" + f"All folders passed should be one of those: {ALLOWED_FOLDERS}\n" + ) + sys.exit(1) + +exclude_regexps = [ + re.compile(x) + for x in [ + r"^.*/node_modules/.*", + r"^.*\\..*", + r"^.*/src/airflow/__init__.py$", + ] +] + + +def get_all_files(folder: str) -> list[str]: + files_to_check = [] + python_file_paths = (AIRFLOW_ROOT_PATH / folder).resolve().rglob("*.py") + for file in python_file_paths: + if ( + file.name not in ("conftest.py",) + and not any(x.match(file.as_posix()) for x in exclude_regexps) + and not any(part.startswith(".") for part in file.parts) + ): + files_to_check.append(file.relative_to(AIRFLOW_ROOT_PATH).as_posix()) + return files_to_check + + +all_files_to_check: list[str] = [] +for mypy_folder in mypy_folders: + all_files_to_check.extend(get_all_files(mypy_folder)) + +if not all_files_to_check: + print("No files to test. Quitting") + sys.exit(0) + +# Write file list +mypy_file_list = AIRFLOW_ROOT_PATH / "files" / "mypy_files.txt" +mypy_file_list.parent.mkdir(parents=True, exist_ok=True) +mypy_file_list.write_text("\n".join(all_files_to_check)) + +if console: + console.print(f"[info]You can check the list of files in:[/] {mypy_file_list}") +else: + print(f"You can check the list of files in: {mypy_file_list}") + +file_argument_local = f"@{mypy_file_list}" +file_argument_ci = "@/files/mypy_files.txt" + +project = FOLDER_TO_PROJECT.get(mypy_folders[0], "devel-common") + +mypy_extra_args: list[str] = [] + +if show_unused_warnings == "true": + if console: + console.print("[info]Running mypy with --warn-unused-ignores") + else: + print("Running mypy with --warn-unused-ignores") + mypy_extra_args.append("--warn-unused-ignores") + +if show_unreachable_warnings == "true": + if console: + console.print("[info]Running mypy with --warn-unreachable") + else: + print("Running mypy with --warn-unreachable") + mypy_extra_args.append("--warn-unreachable") + +if console: + console.print(f"[magenta]Running mypy for folders: {mypy_folders}[/]") +else: + print(f"Running mypy for folders: {mypy_folders}") + +if CI: + # In CI, run inside the breeze Docker image to avoid needing a local environment + # and to not change uv.lock or synchronize dependencies. + from common_prek_utils import ( + initialize_breeze_prek, + run_command_via_breeze_shell, + ) + + initialize_breeze_prek(__name__, __file__) + + mypy_cmd = f"TERM=ansi mypy {shlex.quote(file_argument_ci)} {' '.join(mypy_extra_args)}" + result = run_command_via_breeze_shell( + cmd=["bash", "-c", mypy_cmd], + warn_image_upgrade_needed=True, + extra_env={ + "INCLUDE_MYPY_VOLUME": "false", + "MOUNT_SOURCES": "selected", + }, + ) +else: + # Locally, run via uv with --frozen to not update the lock file. + cmd = [ + "uv", + "run", + "--frozen", + "--project", + project, + "--with", + "apache-airflow-devel-common[mypy]", + "mypy", + file_argument_local, + *mypy_extra_args, + ] + + result = subprocess.run( + cmd, + cwd=str(AIRFLOW_ROOT_PATH), + check=False, + env={**os.environ, "TERM": "ansi"}, + ) + +if result.returncode != 0: + msg = ( + "Mypy check failed. You can run mypy locally with:\n" + f" prek run mypy-{mypy_folders[0]} --all-files\n" + "Or directly with:\n" + f' uv run --project {project} --with "apache-airflow-devel-common[mypy]" mypy \n' + "You can also clear the mypy cache with:\n" + " breeze down --cleanup-mypy-cache\n" + ) + if console: + console.print(f"[yellow]{msg}") + else: + print(msg) +sys.exit(result.returncode) diff --git a/scripts/ci/prek/update_airflow_pyproject_toml.py b/scripts/ci/prek/update_airflow_pyproject_toml.py index cccae46867ed1..ff3ee1e1fe244 100755 --- a/scripts/ci/prek/update_airflow_pyproject_toml.py +++ b/scripts/ci/prek/update_airflow_pyproject_toml.py @@ -85,7 +85,7 @@ def get_optional_dependencies(pyproject_toml_path: Path) -> list[str]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] airflow_core_toml_dict = tomllib.loads(pyproject_toml_path.read_text()) return airflow_core_toml_dict["project"]["optional-dependencies"].keys() @@ -112,7 +112,7 @@ def _read_toml(path: Path) -> dict[str, Any]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] return tomllib.loads(path.read_text()) diff --git a/scripts/ci/prek/update_providers_dependencies.py b/scripts/ci/prek/update_providers_dependencies.py index 9e89084f7b0e3..b114e55a04726 100755 --- a/scripts/ci/prek/update_providers_dependencies.py +++ b/scripts/ci/prek/update_providers_dependencies.py @@ -69,7 +69,7 @@ def load_pyproject_toml(pyproject_toml_file_path: Path) -> dict[str, Any]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] return tomllib.loads(pyproject_toml_file_path.read_text()) diff --git a/scripts/tools/initialize_virtualenv.py b/scripts/tools/initialize_virtualenv.py index fff6fbb4b6227..a6edf0268dca7 100755 --- a/scripts/tools/initialize_virtualenv.py +++ b/scripts/tools/initialize_virtualenv.py @@ -62,7 +62,7 @@ def get_dependency_groups(pyproject_toml_path: Path) -> list[str]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] airflow_core_toml_dict = tomllib.loads(pyproject_toml_path.read_text()) return airflow_core_toml_dict["dependency-groups"].keys() diff --git a/task-sdk/.pre-commit-config.yaml b/task-sdk/.pre-commit-config.yaml index 315e0ea8a133f..df1be38500f71 100644 --- a/task-sdk/.pre-commit-config.yaml +++ b/task-sdk/.pre-commit-config.yaml @@ -56,17 +56,9 @@ repos: pass_filenames: false files: ^src/airflow/sdk/definitions/dag\.py$|^src/airflow/sdk/definitions/decorators/task_group\.py$ - id: mypy-task-sdk - stages: ['pre-push'] name: Run mypy for task-sdk language: python - entry: ../scripts/ci/prek/mypy.py - files: ^.*\.py$ - require_serial: true - - id: mypy-task-sdk - stages: ['manual'] - name: Run mypy for task-sdk (manual) - language: python - entry: ../scripts/ci/prek/mypy_folder.py task-sdk + entry: ../scripts/ci/prek/mypy_local_folder.py task-sdk pass_filenames: false files: ^.*\.py$ require_serial: true