From 01a0962572f86805c0da0595609af3fab7cd945c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 6 Apr 2026 18:22:18 +0200 Subject: [PATCH] Run non-provider mypy checks as regular prek static checks instead of separate CI jobs Non-provider mypy checks (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common) now run locally via uv as regular prek hooks in the pre-commit stage, instead of running as separate mypy CI jobs in the CI image checks workflow. This means they run as part of the regular static checks job in CI and automatically on every local commit. The folder-level mypy checks (which check entire directories at once for comprehensive cross-file type checking) replace the previous file-level incremental checks. Provider mypy checks still run via breeze as a dedicated CI job, now embedded directly in the main CI workflow (ci-amd-arm.yml) instead of being dispatched through the ci-image-checks reusable workflow. The selective checks logic skips non-provider mypy hooks when their relevant files haven't changed, unless devel-common/pyproject.toml changes on main (which affects all mypy configurations). --- .github/workflows/ci-amd-arm.yml | 51 +++- .github/workflows/ci-image-checks.yml | 57 ---- .pre-commit-config.yaml | 26 +- AGENTS.md | 5 +- airflow-core/.pre-commit-config.yaml | 14 +- .../utils/log/non_caching_file_handler.py | 2 +- airflow-ctl/.pre-commit-config.yaml | 14 +- contributing-docs/08_static_code_checks.rst | 48 ++-- .../commands/developer_commands.py | 8 + .../commands/registry_commands.py | 2 +- .../src/airflow_breeze/utils/packages.py | 2 +- .../airflow_breeze/utils/selective_checks.py | 72 ++--- dev/breeze/tests/test_packages.py | 4 +- dev/breeze/tests/test_selective_checks.py | 261 ++++++++---------- .../sphinx_exts/docs_build/package_filter.py | 2 +- scripts/ci/prek/check_extra_packages_ref.py | 2 +- .../check_shared_distributions_structure.py | 2 +- .../prek/check_shared_distributions_usage.py | 2 +- scripts/ci/prek/check_version_consistency.py | 2 +- scripts/ci/prek/mypy_local_folder.py | 223 +++++++++++++++ .../ci/prek/update_airflow_pyproject_toml.py | 4 +- .../ci/prek/update_providers_dependencies.py | 2 +- scripts/tools/initialize_virtualenv.py | 2 +- task-sdk/.pre-commit-config.yaml | 10 +- 24 files changed, 470 insertions(+), 347 deletions(-) create mode 100755 scripts/ci/prek/mypy_local_folder.py diff --git a/.github/workflows/ci-amd-arm.yml b/.github/workflows/ci-amd-arm.yml index ab34ba25882bb..777e47e586a28 100644 --- a/.github/workflows/ci-amd-arm.yml +++ b/.github/workflows/ci-amd-arm.yml @@ -91,7 +91,6 @@ jobs: kubernetes-versions-list-as-string: >- ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} - mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} platform: ${{ steps.selective-checks.outputs.platform }} @@ -115,7 +114,7 @@ jobs: run-go-sdk-tests: ${{ steps.selective-checks.outputs.run-go-sdk-tests }} run-helm-tests: ${{ steps.selective-checks.outputs.run-helm-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} - run-mypy: ${{ steps.selective-checks.outputs.run-mypy }} + run-mypy-providers: ${{ steps.selective-checks.outputs.run-mypy-providers }} run-remote-logging-elasticsearch-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }} run-remote-logging-s3-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }} run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} @@ -307,8 +306,6 @@ jobs: with: runners: ${{ needs.build-info.outputs.runner-type }} platform: ${{ needs.build-info.outputs.platform }} - run-mypy: ${{ needs.build-info.outputs.run-mypy }} - mypy-checks: ${{ needs.build-info.outputs.mypy-checks }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} branch: ${{ needs.build-info.outputs.default-branch }} canary-run: ${{ needs.build-info.outputs.canary-run }} @@ -333,6 +330,51 @@ jobs: DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + mypy-providers: + timeout-minutes: 45 + name: "MyPy providers checks" + needs: [build-info, build-ci-images] + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + if: needs.build-info.outputs.run-mypy-providers == 'true' + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Free up disk space" + shell: bash + run: ./scripts/tools/free_up_disk_space.sh + - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ needs.build-info.outputs.platform }} + python: "${{ needs.build-info.outputs.default-python-version }}" + use-uv: ${{ needs.build-info.outputs.use-uv }} + make-mnt-writeable-and-cleanup: true + id: breeze + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek + with: + python-version: ${{steps.breeze.outputs.host-python-version}} + platform: ${{ needs.build-info.outputs.platform }} + save-cache: false + - name: "MyPy checks for providers" + run: prek --color always --verbose --stage manual mypy-providers --all-files + env: + VERBOSE: "false" + COLUMNS: "202" + SKIP_GROUP_OUTPUT: "true" + DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} + RUFF_FORMAT: "github" + INCLUDE_MYPY_VOLUME: "false" + providers: name: "provider distributions tests" uses: ./.github/workflows/test-providers.yml @@ -895,6 +937,7 @@ jobs: - build-prod-images - ci-image-checks - generate-constraints + - mypy-providers - providers - tests-helm - tests-integration-system diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 9d21ad2f92c40..1f25943204179 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -28,14 +28,6 @@ on: # yamllint disable-line rule:truthy description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string - run-mypy: - description: "Whether to run mypy checks (true/false)" - required: true - type: string - mypy-checks: - description: "List of folders to run mypy checks on" - required: false - type: string python-versions-list-as-string: description: "The list of python versions as string separated by spaces" required: true @@ -169,55 +161,6 @@ jobs: run: cat ~/.cache/prek/prek.log || true if: failure() - mypy: - timeout-minutes: 45 - name: "MyPy checks" - runs-on: ${{ fromJSON(inputs.runners) }} - if: inputs.run-mypy == 'true' - strategy: - fail-fast: false - matrix: - mypy-check: ${{ fromJSON(inputs.mypy-checks) }} - env: - PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: "Free up disk space" - shell: bash - run: ./scripts/tools/free_up_disk_space.sh - - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" - uses: ./.github/actions/prepare_breeze_and_image - with: - platform: ${{ inputs.platform }} - python: "${{ inputs.default-python-version }}" - use-uv: ${{ inputs.use-uv }} - make-mnt-writeable-and-cleanup: true - id: breeze - - name: "Install prek" - uses: ./.github/actions/install-prek - id: prek - with: - python-version: ${{steps.breeze.outputs.host-python-version}} - platform: ${{ inputs.platform }} - save-cache: false - - name: "MyPy checks for ${{ matrix.mypy-check }}" - run: prek --color always --verbose --stage manual "$MYPY_CHECK" --all-files - env: - VERBOSE: "false" - COLUMNS: "202" - SKIP_GROUP_OUTPUT: "true" - DEFAULT_BRANCH: ${{ inputs.branch }} - RUFF_FORMAT: "github" - INCLUDE_MYPY_VOLUME: "false" - MYPY_CHECK: ${{ matrix.mypy-check }} - build-docs: timeout-minutes: 150 name: "Build documentation" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d9eaf4abe0ea..5140ae80fe5e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1014,39 +1014,23 @@ repos: ^uv\.lock$ pass_filenames: false require_serial: true - ## ADD MOST PREK HOOK ABOVE THAT LINE - # The below prek hooks are those requiring CI image to be built - ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE - id: mypy-dev - stages: ['pre-push'] name: Run mypy for dev language: python - entry: ./scripts/ci/prek/mypy.py - files: ^dev/.*\.py$|^scripts/.*\.py$ - require_serial: true - - id: mypy-dev - stages: ['manual'] - name: Run mypy for dev (manual) - language: python - entry: ./scripts/ci/prek/mypy_folder.py dev scripts + entry: ./scripts/ci/prek/mypy_local_folder.py dev scripts pass_filenames: false files: ^.*\.py$ require_serial: true - id: mypy-devel-common - stages: ['pre-push'] name: Run mypy for devel-common language: python - entry: ./scripts/ci/prek/mypy.py - files: ^devel-common/.*\.py$ - require_serial: true - - id: mypy-devel-common - stages: ['manual'] - name: Run mypy for devel-common (manual) - language: python - entry: ./scripts/ci/prek/mypy_folder.py devel-common + entry: ./scripts/ci/prek/mypy_local_folder.py devel-common pass_filenames: false files: ^.*\.py$ require_serial: true + ## ADD MOST PREK HOOK ABOVE THAT LINE + # The below prek hooks are those requiring CI image to be built + ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE - id: check-template-fields-valid name: Check templated fields mapped in operators/sensors language: python diff --git a/AGENTS.md b/AGENTS.md index 89bbd76620f6a..b9ef07b381d46 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,8 @@ - **Run other suites of tests** `breeze testing ` (test groups: `airflow-ctl-tests`, `docker-compose-tests`, `task-sdk-tests`) - **Run scripts tests:** `uv run --project scripts pytest scripts/tests/ -xvs` - **Run Airflow CLI:** `breeze run airflow dags list` -- **Type-check:** `breeze run mypy path/to/code` +- **Type-check (non-providers):** `uv run --project --with "apache-airflow-devel-common[mypy]" mypy path/to/code` +- **Type-check (providers):** `breeze run mypy path/to/code` - **Lint with ruff only:** `prek run ruff --from-ref ` - **Format with ruff only:** `prek run ruff-format --from-ref ` - **Run regular (fast) static checks:** `prek run --from-ref --stage pre-commit` @@ -147,7 +148,7 @@ code review checklist in [`.github/instructions/code-review.instructions.md`](.g 3. Confirm the code follows the project's coding standards and architecture boundaries described in this file. 4. Run regular (fast) static checks (`prek run --from-ref --stage pre-commit`) - and fix any failures. + and fix any failures. This includes mypy checks for non-provider projects (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common). 5. Run manual (slower) checks (`prek run --from-ref --stage manual`) and fix any failures. 6. Run relevant individual tests and confirm they pass. 7. Find which tests to run for the changes with selective-checks and run those tests in parallel to confirm they pass and check for CI-specific issues. diff --git a/airflow-core/.pre-commit-config.yaml b/airflow-core/.pre-commit-config.yaml index 5d2e3e7fe2b9b..0567187dbf686 100644 --- a/airflow-core/.pre-commit-config.yaml +++ b/airflow-core/.pre-commit-config.yaml @@ -221,23 +221,15 @@ repos: additional_dependencies: ['pnpm@10.25.0'] pass_filenames: true require_serial: true - ## ADD MOST PREK HOOK ABOVE THAT LINE - # The below prek hooks are those requiring CI image to be built - id: mypy-airflow-core - stages: ['pre-push'] name: Run mypy for airflow-core language: python - entry: ../scripts/ci/prek/mypy.py - files: ^.*\.py$ - require_serial: true - - id: mypy-airflow-core - stages: ['manual'] - name: Run mypy for airflow-core (manual) - language: python - entry: ../scripts/ci/prek/mypy_folder.py airflow-core + entry: ../scripts/ci/prek/mypy_local_folder.py airflow-core pass_filenames: false files: ^.*\.py$ require_serial: true + ## ADD MOST PREK HOOK ABOVE THAT LINE + # The below prek hooks are those requiring CI image to be built - id: generate-openapi-spec name: Generate the FastAPI API spec language: python diff --git a/airflow-core/src/airflow/utils/log/non_caching_file_handler.py b/airflow-core/src/airflow/utils/log/non_caching_file_handler.py index aa0ca9864e2ea..ad3c0dbe27974 100644 --- a/airflow-core/src/airflow/utils/log/non_caching_file_handler.py +++ b/airflow-core/src/airflow/utils/log/non_caching_file_handler.py @@ -25,7 +25,7 @@ def make_file_io_non_caching(io: IO[str]) -> IO[str]: try: fd = io.fileno() - os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) + os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) # type: ignore[attr-defined] except Exception: # in case either file descriptor cannot be retrieved or fadvise is not available # we should simply return the wrapper retrieved by FileHandler's open method diff --git a/airflow-ctl/.pre-commit-config.yaml b/airflow-ctl/.pre-commit-config.yaml index e63268b077ef8..c45a1985ec129 100644 --- a/airflow-ctl/.pre-commit-config.yaml +++ b/airflow-ctl/.pre-commit-config.yaml @@ -25,21 +25,9 @@ repos: - repo: local hooks: - id: mypy-airflow-ctl - stages: ['pre-push'] name: Run mypy for airflow-ctl language: python - entry: ../scripts/ci/prek/mypy.py - files: - (?x) - ^src/airflowctl/.*\.py$| - ^tests/.*\.py$ - exclude: .*generated.py - require_serial: true - - id: mypy-airflow-ctl - stages: ['manual'] - name: Run mypy for airflow-ctl (manual) - language: python - entry: ../scripts/ci/prek/mypy_folder.py airflow-ctl + entry: ../scripts/ci/prek/mypy_local_folder.py airflow-ctl pass_filenames: false files: ^.*\.py$ require_serial: true diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 1d009b92cefb4..23256c6c0e026 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -173,17 +173,18 @@ But you can run prek hooks manually as needed. prek - Run only mypy check on your staged airflow and dev files by specifying the - ``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be specified): + ``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be specified). + For non-provider projects, mypy runs locally via ``uv`` (no breeze image needed): .. code-block:: bash - prek mypy-airflow-core mypy-dev --stage pre-push + prek mypy-airflow-core mypy-dev - Run only mypy airflow checks on all "airflow-core" files by using: .. code-block:: bash - prek mypy-airflow-core --all-files --stage pre-push + prek mypy-airflow-core --all-files - Run all pre-commit stage hooks on all files by using: @@ -279,41 +280,40 @@ them manually by running ``prek --stage manual ``. Mypy checks ----------- -When we run mypy checks locally when pushing a change to PR, the ``mypy-*`` checks is run, ``mypy-airflow``, -``mypy-dev``, ``mypy-providers``, ``mypy-airflow-ctl``, depending on the files you are changing. The mypy checks -are run by passing those changed files to mypy. This is way faster than running checks for all files (even -if mypy cache is used - especially when you change a file in Airflow core that is imported and used by many -files). You also need to have ``breeze ci-image build --python 3.10`` built locally to run the mypy checks. +When we run mypy checks locally, the ``mypy-*`` checks run depending on the files you are changing: +``mypy-airflow-core``, ``mypy-dev``, ``mypy-providers``, ``mypy-task-sdk``, ``mypy-airflow-ctl``, etc. -However, in some cases, it produces different results than when running checks for the whole set -of files, because ``mypy`` does not even know that some types are defined in other files and it might not -be able to follow imports properly if they are dynamic. Therefore in CI we run ``mypy`` check for whole -directories (``airflow`` - excluding providers, ``providers``, ``dev`` and ``docs``) to make sure -that we catch all ``mypy`` errors - so you can experience different results when running mypy locally and -in CI. If you want to run mypy checks for all files locally, you can do it by running the following -command (example for ``airflow`` files): +For **non-provider projects** (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common), mypy +runs locally using the ``uv`` virtualenv — no breeze CI image is needed. These checks run as regular +prek hooks in the ``pre-commit`` stage, checking whole directories at once. This means they run both +as part of local commits and as part of regular static checks in CI (not as separate mypy CI jobs). +You can also run mypy directly. Use ``--frozen`` to avoid updating ``uv.lock``: .. code-block:: bash - prek --stage manual mypy- --all-files + uv run --frozen --project --with "apache-airflow-devel-common[mypy]" mypy path/to/code -For example: +To run the prek hook for a specific project (example for ``airflow-core`` files): .. code-block:: bash - prek --stage manual mypy-airflow --all-files + prek mypy-airflow-core --all-files To show unused mypy ignores for any providers/airflow etc, eg: run below command: .. code-block:: bash + export SHOW_UNUSED_MYPY_WARNINGS=true - prek --stage manual mypy-airflow --all-files + prek mypy-airflow-core --all-files + +For non-provider projects, the local mypy cache is stored in ``.mypy_cache`` at the repo root. + +For **providers**, mypy still runs via breeze (``breeze run mypy``) as a separate CI job and requires +``breeze ci-image build --python 3.10`` to be built locally. Providers use a separate docker-volume +(called ``mypy-cache-volume``) that keeps the cache of last MyPy execution. -MyPy uses a separate docker-volume (called ``mypy-cache-volume``) that keeps the cache of last MyPy -execution in order to speed MyPy checks up (sometimes by order of magnitude). While in most cases MyPy -will handle refreshing the cache when and if needed, there are some cases when it won't (cache invalidation -is the hard problem in computer science). This might happen for example when we upgrade MyPY. In such -cases you might need to manually remove the cache volume by running ``breeze down --cleanup-mypy-cache``. +To clear all mypy caches (both local ``.mypy_cache`` and the Docker volume), run +``breeze down --cleanup-mypy-cache``. ----------- diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 5c70a495afc59..f10ed0ec0bd3c 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -932,6 +932,10 @@ def down(preserve_volumes: bool, cleanup_mypy_cache: bool, cleanup_build_cache: if cleanup_mypy_cache: command_to_execute = ["docker", "volume", "rm", "--force", "mypy-cache-volume"] run_command(command_to_execute) + local_mypy_cache = AIRFLOW_ROOT_PATH / ".mypy_cache" + if local_mypy_cache.exists(): + console_print(f"\n[info]Removing local mypy cache: {local_mypy_cache}\n") + shutil.rmtree(local_mypy_cache) if cleanup_build_cache: command_to_execute = ["docker", "volume", "rm", "--force", "airflow-cache-volume"] run_command(command_to_execute) @@ -1070,6 +1074,10 @@ def doctor(ctx): console_print("\n[info]Cleaning mypy cache...\n") command_to_execute = ["docker", "volume", "rm", "--force", "mypy-cache-volume"] run_command(command_to_execute) + local_mypy_cache = AIRFLOW_ROOT_PATH / ".mypy_cache" + if local_mypy_cache.exists(): + console_print(f"\n[info]Removing local mypy cache: {local_mypy_cache}\n") + shutil.rmtree(local_mypy_cache) console_print("\n[info]Cleaning build cache...\n") command_to_execute = ["docker", "volume", "rm", "--force", "airflow-cache-volume"] diff --git a/dev/breeze/src/airflow_breeze/commands/registry_commands.py b/dev/breeze/src/airflow_breeze/commands/registry_commands.py index f818a16ea01a9..0655deb51d182 100644 --- a/dev/breeze/src/airflow_breeze/commands/registry_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/registry_commands.py @@ -160,7 +160,7 @@ def _read_provider_yaml_info(provider_id: str) -> tuple[str, list[str]]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] provider_yaml_path = _find_provider_yaml(provider_id) with open(provider_yaml_path) as f: diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 25f9a945473cb..553f0379b4cc1 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -533,7 +533,7 @@ def load_pyproject_toml(pyproject_toml_file_path: Path) -> dict[str, Any]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] toml_content = pyproject_toml_file_path.read_text() syntax = Syntax(toml_content, "toml", theme="ansi_dark", line_numbers=True) try: diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 99e74fcfa5039..6a756289db649 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -870,58 +870,21 @@ def _should_be_run(self, source_area: FileGroupForCi) -> bool: return False @cached_property - def mypy_checks(self) -> list[str]: - checks_to_run: list[str] = [] - if ( - self._matching_files(FileGroupForCi.DEVEL_TOML_FILES, CI_FILE_GROUP_MATCHES) - and self._default_branch == "main" - ): - return [ - "mypy-airflow-core", - "mypy-providers", - "mypy-dev", - "mypy-task-sdk", - "mypy-devel-common", - "mypy-airflow-ctl", - ] - if ( - self._matching_files(FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES, CI_FILE_GROUP_MATCHES) - or self.full_tests_needed - ): - checks_to_run.append("mypy-airflow-core") - if ( + def run_mypy_providers(self) -> bool: + # Non-provider mypy checks run as part of regular static checks (prek hooks). + # Only provider mypy needs a separate CI job (requires the CI Docker image with breeze). + return ( self._matching_files(FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES) or self._matching_files( FileGroupForCi.ALL_PROVIDERS_DISTRIBUTION_CONFIG_FILES, CI_FILE_GROUP_MATCHES ) or self._are_all_providers_affected() - ) and self._default_branch == "main": - checks_to_run.append("mypy-providers") - if ( - self._matching_files(FileGroupForCi.ALL_DEV_PYTHON_FILES, CI_FILE_GROUP_MATCHES) - or self.full_tests_needed - ): - checks_to_run.append("mypy-dev") - if ( - self._matching_files(FileGroupForCi.TASK_SDK_FILES, CI_FILE_GROUP_MATCHES) - or self.full_tests_needed - ): - checks_to_run.append("mypy-task-sdk") - if ( - self._matching_files(FileGroupForCi.ALL_DEVEL_COMMON_PYTHON_FILES, CI_FILE_GROUP_MATCHES) - or self.full_tests_needed - ): - checks_to_run.append("mypy-devel-common") - if ( - self._matching_files(FileGroupForCi.ALL_AIRFLOW_CTL_PYTHON_FILES, CI_FILE_GROUP_MATCHES) + or ( + self._matching_files(FileGroupForCi.DEVEL_TOML_FILES, CI_FILE_GROUP_MATCHES) + and self._default_branch == "main" + ) or self.full_tests_needed - ): - checks_to_run.append("mypy-airflow-ctl") - return checks_to_run - - @cached_property - def run_mypy(self) -> bool: - return self.mypy_checks != [] + ) and self._default_branch == "main" @cached_property def run_python_scans(self) -> bool: @@ -1499,6 +1462,23 @@ def skip_prek_hooks(self) -> str: # only skip provider validation if none of the provider.yaml and provider # python files changed because validation also walks through all the provider python files prek_hooks_to_skip.add("check-provider-yaml-valid") + # Non-provider mypy checks run as prek hooks in static checks. + # Skip them when their relevant files haven't changed, unless devel-common + # pyproject.toml changes on main (which affects all mypy checks). + if not ( + self._matching_files(FileGroupForCi.DEVEL_TOML_FILES, CI_FILE_GROUP_MATCHES) + and self._default_branch == "main" + ): + if not self._matching_files(FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-airflow-core") + if not self._matching_files(FileGroupForCi.ALL_DEV_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-dev") + if not self._matching_files(FileGroupForCi.TASK_SDK_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-task-sdk") + if not self._matching_files(FileGroupForCi.ALL_DEVEL_COMMON_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-devel-common") + if not self._matching_files(FileGroupForCi.ALL_AIRFLOW_CTL_PYTHON_FILES, CI_FILE_GROUP_MATCHES): + prek_hooks_to_skip.add("mypy-airflow-ctl") return ",".join(sorted(prek_hooks_to_skip)) @cached_property diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index b48062f93d164..2a11a4ef29d2a 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -384,7 +384,7 @@ def test_apply_version_suffix_to_provider_pyproject_toml( try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] from unittest.mock import patch # Get the original provider details @@ -474,7 +474,7 @@ def test_apply_version_suffix_to_non_provider_pyproject_tomls( try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] distribution_paths = [AIRFLOW_ROOT_PATH / distribution for distribution in distributions] original_pyproject_toml_paths = [path / "pyproject.toml" for path in distribution_paths] original_contents = [path.read_text() for path in original_pyproject_toml_paths] diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 5d6ff3768722e..90e5551ce3377 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -98,59 +98,57 @@ _get_test_list_as_json(_split_list(sorted(LIST_OF_ALL_PROVIDER_TESTS), 5)) ) -ALL_MYPY_CHECKS_ARRAY = [ - "mypy-airflow-core", - "mypy-providers", - "mypy-dev", - "mypy-task-sdk", - "mypy-devel-common", - "mypy-airflow-ctl", -] - -ALL_MYPY_CHECKS = str(ALL_MYPY_CHECKS_ARRAY) - -ALL_MYPY_CHECKS_EXCEPT_PROVIDERS = str( - [check for check in ALL_MYPY_CHECKS_ARRAY if check != "mypy-providers"] -) ALL_SKIPPED_COMMITS_ON_NO_CI_IMAGE = ( "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED = "identity,update-uv-lock" ALL_SKIPPED_COMMITS_IF_NO_UI = ( - "identity,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "identity,mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" +) +ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = ( + "identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,update-uv-lock" ) -ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = "identity,lint-helm-chart,update-uv-lock" ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = ( - "identity,lint-helm-chart,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + "identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = ( "check-provider-yaml-valid,identity," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = ( "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS = ( "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = ( - "check-provider-yaml-valid,flynt,identity,lint-helm-chart,update-uv-lock" + "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,update-uv-lock" ) ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED = ( "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" ) @@ -244,8 +242,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": None, "providers-test-types-list-as-strings-in-json": None, "individual-providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="No tests on simple change", ) @@ -278,8 +275,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests should be run when API file changed", ) @@ -303,8 +299,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests should be run when fastapi files change", ) @@ -328,8 +323,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests should run when API test files change", ) @@ -349,7 +343,11 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-unit-tests": "true", "run-amazon-tests": "false", "docs-build": "true", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( _get_test_list_as_json( @@ -358,8 +356,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ), "providers-test-types-list-as-strings-in-json": None, "individual-providers-test-types-list-as-strings-in-json": None, - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", "skip-providers-tests": "true", }, id="Only Serialization tests", @@ -388,8 +385,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests and docs should run on API change", ) @@ -431,8 +427,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Selected Providers and docs should run", ) @@ -474,8 +469,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", "skip-providers-tests": "false", }, id="Selected Providers should run when system tests are modified", @@ -521,8 +515,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", "skip-providers-tests": "false", }, id="Selected Providers and docs should run when both system tests and tests are modified", @@ -568,8 +561,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", "skip-providers-tests": "false", }, id="Selected Providers and docs should run when both system tests and tests are modified for more than one provider", @@ -595,8 +587,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": None, "providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Only docs builds should run - no tests needed", ) @@ -619,13 +610,16 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-task-sdk-integration-tests": "true", "docs-build": "true", "full-tests-needed": "false", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "skip-providers-tests": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": "['mypy-providers', 'mypy-task-sdk']", + "run-mypy-providers": "true", }, id="Task SDK source file changed - Task SDK, Core and provider tests should run", ) @@ -651,8 +645,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Task SDK integration tests files changed - " "Task SDK integration tests and prod image build should run but no other tests", @@ -676,11 +669,14 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-airflow-ctl-integration-tests": "true", "docs-build": "true", "full-tests-needed": "false", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-core,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-ctl']", + "run-mypy-providers": "false", }, id="Airflow CTL source file changed - Airflow CTL tests should run", ) @@ -706,8 +702,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, "skip-providers-tests": "true", "upgrade-to-newer-dependencies": "false", - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Airflow CTL integration tests files changed - " "Airflow CTL integration tests and prod image build should run but no other tests", @@ -748,8 +743,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Helm tests, providers (both upstream and downstream)," "kubernetes tests and docs should run", @@ -812,8 +806,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): }, ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Helm tests, http and all relevant providers, kubernetes tests and " "docs should run even if unimportant files were added", @@ -880,8 +873,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Docs should run even if unimportant files were added and prod image " "should be build for chart changes", @@ -907,8 +899,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "true", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run and upgrading to newer requirements as dependencies change", ) @@ -947,8 +938,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Providers tests run including amazon tests if only amazon provider.yaml files changed", ), @@ -980,8 +970,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Providers tests run without amazon tests if no amazon file changed", ), @@ -1018,8 +1007,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-providers']", + "run-mypy-providers": "true", }, id="Providers tests run including amazon tests if amazon provider files changed", ), @@ -1042,7 +1030,11 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "false", "run-kubernetes-tests": "false", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "identity,lint-helm-chart," + "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( [{"description": "Always", "test_types": "Always"}] @@ -1055,8 +1047,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core', 'mypy-providers']", + "run-mypy-providers": "true", }, id="Only Always and common providers tests should run when only common.io and tests/always changed", ), @@ -1078,8 +1069,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests to run when standard operator changed", ), @@ -1103,8 +1093,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests should be run when tests/utils/ change", ) @@ -1132,8 +1121,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "testable-core-integrations": "['kerberos', 'otel', 'redis']", "testable-providers-integrations": "['celery', 'cassandra', 'drill', 'elasticsearch', 'tinkerpop', 'kafka', " "'mongo', 'pinot', 'qdrant', 'redis', 'trino', 'ydb']", - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All tests should be run when devel-common/ change", ) @@ -1153,8 +1141,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "full-tests-needed": "false", "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS, "upgrade-to-newer-dependencies": "false", - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", "run-helm-tests": "false", "run-ui-tests": "true", "run-ui-e2e-tests": "true", @@ -1184,8 +1171,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": None, "providers-test-types-list-as-strings-in-json": None, "individual-providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Run docs-build for RELEASE_NOTES.rst", ), @@ -1202,13 +1188,16 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-unit-tests": "false", "run-amazon-tests": "false", "docs-build": "true", - "skip-prek-hooks": "check-provider-yaml-valid,flynt,identity,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock", + "skip-prek-hooks": ( + "check-provider-yaml-valid,flynt,identity," + "mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": None, "providers-test-types-list-as-strings-in-json": None, "individual-providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Run docs-build for chart/RELEASE_NOTES.rst", ), @@ -1230,8 +1219,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": None, "providers-test-types-list-as-strings-in-json": None, "individual-providers-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Run docs-build for SECURITY.md", ), @@ -1244,8 +1232,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("devel-common/pyproject.toml",), { - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="All mypy checks should run when devel-common/pyproject.toml changes", ) @@ -1526,8 +1513,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including all providers when git provider is changed" "(special case for now)", @@ -1561,8 +1547,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including all providers when full tests are needed, " "and all versions are required.", @@ -1596,8 +1581,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including all providers when full tests are needed " "but with single python and kubernetes if `default versions only` label is set", @@ -1631,8 +1615,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", "run-ui-e2e-tests": "true", }, id="Everything should run including all providers when full tests are needed " @@ -1668,8 +1651,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including all providers when full tests are needed " "but with single python and kubernetes if `latest versions only` label is set", @@ -1704,8 +1686,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including full providers when full " "tests are needed even with different label set as well", @@ -1738,8 +1719,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "individual-providers-test-types-list-as-strings-in-json": LIST_OF_ALL_PROVIDER_TESTS_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="Everything should run including full providers when " "full tests are needed even if no files are changed", @@ -1766,8 +1746,7 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "skip-prek-hooks": All_SKIPPED_COMMITS_IF_NON_MAIN_BRANCH, "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS_EXCEPT_PROVIDERS, + "run-mypy-providers": "false", }, id="Everything should run except Providers and lint prek " "when full tests are needed for non-main branch", @@ -1809,8 +1788,7 @@ def test_expected_output_full_tests_needed( "full-tests-needed": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Nothing should run if only non-important files changed", ), @@ -1835,8 +1813,7 @@ def test_expected_output_full_tests_needed( "core-test-types-list-as-strings-in-json": json.dumps( [{"description": "Always", "test_types": "Always"}] ), - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="No Helm tests, No providers no lint charts, should run if " "only chart/providers changed in non-main but PROD image should be built", @@ -1863,8 +1840,7 @@ def test_expected_output_full_tests_needed( "core-test-types-list-as-strings-in-json": json.dumps( [{"description": "Always...CLI", "test_types": "Always CLI"}] ), - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, id="Only CLI tests and Kubernetes tests should run if cli/chart files changed in non-main branch", ), @@ -1887,8 +1863,7 @@ def test_expected_output_full_tests_needed( "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, id="All tests except Providers and helm lint prek " "should run if core file changed in non-main branch", @@ -1930,8 +1905,7 @@ def test_expected_output_pull_request_v2_7( "upgrade-to-newer-dependencies": "false", "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED, "core-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="No tests run on push if only text non-doc files changed", ), @@ -1951,8 +1925,7 @@ def test_expected_output_pull_request_v2_7( "docs-list-as-string": None, "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="No tests run on push if only text non-doc files changed in non-main branch", ), @@ -1973,8 +1946,7 @@ def test_expected_output_pull_request_v2_7( "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", "run-ui-e2e-tests": "false", }, id="All tests run on push if core file changed", @@ -2029,8 +2001,7 @@ def test_expected_output_push( "upgrade-to-newer-dependencies": "false", "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED, "core-test-types-list-as-strings-in-json": None, - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Nothing should run if only non-important files changed", ), @@ -2047,13 +2018,16 @@ def test_expected_output_push( "skip-providers-tests": "true", "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( [{"description": "Always", "test_types": "Always"}] ), - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, id="Only Always and docs build should run if only system tests changed", ), @@ -2080,7 +2054,10 @@ def test_expected_output_push( "apache.kafka cncf.kubernetes common.compat common.messaging common.sql databricks facebook google hashicorp http microsoft.azure " "microsoft.mssql mysql openlineage oracle postgres " "presto salesforce samba sftp ssh standard trino", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_UI, + "skip-prek-hooks": ( + "identity,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": json.dumps( @@ -2099,8 +2076,7 @@ def test_expected_output_push( } ] ), - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core', 'mypy-providers']", + "run-mypy-providers": "true", }, id="CLI tests and Google-related provider tests should run if cli/chart files changed but " "prod image should be build too and k8s tests too", @@ -2117,12 +2093,15 @@ def test_expected_output_push( "skip-providers-tests": "true", "docs-build": "true", "docs-list-as-string": "apache-airflow", - "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS, + "skip-prek-hooks": ( + "check-provider-yaml-valid,identity,lint-helm-chart," + "mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk," + "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock" + ), "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, id="Tests for all airflow core types except providers should run if model file changed", ), @@ -2141,8 +2120,7 @@ def test_expected_output_push( "upgrade-to-newer-dependencies": "false", "skip-prek-hooks": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", ), @@ -2179,8 +2157,7 @@ def test_expected_output_push( } ] ), - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, id="Trigger openlineage and related providers tests when Assets files changed", ), @@ -2243,8 +2220,7 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(mock_get, gith "skip-prek-hooks": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, "upgrade-to-newer-dependencies": ("true" if github_event == GithubEvents.SCHEDULE else "false"), "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, str(stderr), ) @@ -2293,8 +2269,7 @@ def test_files_provided_trigger_full_build_for_any_event_type(mock_get, github_e "skip-prek-hooks": ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED, "upgrade-to-newer-dependencies": ("true" if github_event == GithubEvents.SCHEDULE else "false"), "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, str(stderr), ) @@ -2607,8 +2582,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("README.md",), { - "run-mypy": "false", - "mypy-checks": "[]", + "run-mypy-providers": "false", }, "main", (), @@ -2617,8 +2591,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("airflow-core/src/airflow/cli/file.py",), { - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, "main", (), @@ -2627,8 +2600,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("airflow-core/src/airflow/models/file.py",), { - "run-mypy": "true", - "mypy-checks": "['mypy-airflow-core']", + "run-mypy-providers": "false", }, "main", (), @@ -2637,8 +2609,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("task-sdk/src/airflow/sdk/a_file.py",), { - "run-mypy": "true", - "mypy-checks": "['mypy-providers', 'mypy-task-sdk']", + "run-mypy-providers": "true", }, "main", (), @@ -2647,8 +2618,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("dev/a_package/a_file.py",), { - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, "main", (), @@ -2657,8 +2627,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs pytest.param( ("readme.md",), { - "run-mypy": "true", - "mypy-checks": ALL_MYPY_CHECKS, + "run-mypy-providers": "true", }, "main", ("full tests needed",), diff --git a/devel-common/src/sphinx_exts/docs_build/package_filter.py b/devel-common/src/sphinx_exts/docs_build/package_filter.py index f4d7c037232cf..3cd83951be838 100644 --- a/devel-common/src/sphinx_exts/docs_build/package_filter.py +++ b/devel-common/src/sphinx_exts/docs_build/package_filter.py @@ -64,7 +64,7 @@ def find_packages_to_build(available_packages: list[str], package_filters: list[ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] read_toml = tomllib.loads(pyproject_toml_path.read_text()) package_name = read_toml["project"]["name"] if package_name == "apache-airflow": diff --git a/scripts/ci/prek/check_extra_packages_ref.py b/scripts/ci/prek/check_extra_packages_ref.py index 6e3bd8bcd6b2f..009e4d6c66266 100755 --- a/scripts/ci/prek/check_extra_packages_ref.py +++ b/scripts/ci/prek/check_extra_packages_ref.py @@ -39,7 +39,7 @@ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] EXTRA_PACKAGES_REF_FILE = AIRFLOW_ROOT_PATH / "airflow-core" / "docs" / "extra-packages-ref.rst" PYPROJECT_TOML_FILE_PATH = AIRFLOW_ROOT_PATH / "pyproject.toml" diff --git a/scripts/ci/prek/check_shared_distributions_structure.py b/scripts/ci/prek/check_shared_distributions_structure.py index ac08c28924b36..3a7366bd300cd 100755 --- a/scripts/ci/prek/check_shared_distributions_structure.py +++ b/scripts/ci/prek/check_shared_distributions_structure.py @@ -35,7 +35,7 @@ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] from common_prek_utils import AIRFLOW_ROOT_PATH, console diff --git a/scripts/ci/prek/check_shared_distributions_usage.py b/scripts/ci/prek/check_shared_distributions_usage.py index c5759d76d73f2..5a734fa255a3f 100755 --- a/scripts/ci/prek/check_shared_distributions_usage.py +++ b/scripts/ci/prek/check_shared_distributions_usage.py @@ -39,7 +39,7 @@ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] from common_prek_utils import AIRFLOW_ROOT_PATH, console, insert_documentation diff --git a/scripts/ci/prek/check_version_consistency.py b/scripts/ci/prek/check_version_consistency.py index ebcf457d0da5f..2711b55cccd39 100755 --- a/scripts/ci/prek/check_version_consistency.py +++ b/scripts/ci/prek/check_version_consistency.py @@ -32,7 +32,7 @@ try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] from common_prek_utils import ( AIRFLOW_CORE_SOURCES_PATH, diff --git a/scripts/ci/prek/mypy_local_folder.py b/scripts/ci/prek/mypy_local_folder.py new file mode 100755 index 0000000000000..e578079b11fa8 --- /dev/null +++ b/scripts/ci/prek/mypy_local_folder.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.10,<3.11" +# dependencies = [ +# "rich>=13.6.0", +# ] +# /// +"""Run mypy on entire folders using local virtualenv (uv) instead of breeze. + +Used for non-provider projects: airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common. +""" + +from __future__ import annotations + +import os +import re +import shlex +import subprocess +import sys + +from common_prek_utils import ( + AIRFLOW_ROOT_PATH, +) + +CI = os.environ.get("CI") + +try: + from rich.console import Console + + console = Console(width=400, color_system="standard") +except ImportError: + console = None # type: ignore[assignment] + +if __name__ not in ("__main__", "__mp_main__"): + raise SystemExit( + "This file is intended to be executed as an executable program. You cannot use it as a module." + f"To run this script, run the ./{__file__} command" + ) + +ALLOWED_FOLDERS = [ + "airflow-core", + "dev", + "scripts", + "devel-common", + "task-sdk", + "airflow-ctl", +] + +# Map folder(s) to the uv project to use for running mypy. +# When multiple folders are checked together (e.g. dev + scripts), the first folder's project is used. +FOLDER_TO_PROJECT = { + "airflow-core": "airflow-core", + "task-sdk": "task-sdk", + "airflow-ctl": "airflow-ctl", + "devel-common": "devel-common", + "dev": "dev", + "scripts": "scripts", +} + +if len(sys.argv) < 2: + if console: + console.print(f"[yellow]You need to specify the folder to test as parameter: {ALLOWED_FOLDERS}\n") + else: + print(f"You need to specify the folder to test as parameter: {ALLOWED_FOLDERS}") + sys.exit(1) + +mypy_folders = sys.argv[1:] + +show_unused_warnings = os.environ.get("SHOW_UNUSED_MYPY_WARNINGS", "false") +show_unreachable_warnings = os.environ.get("SHOW_UNREACHABLE_MYPY_WARNINGS", "false") + +for mypy_folder in mypy_folders: + if mypy_folder not in ALLOWED_FOLDERS: + if console: + console.print( + f"\n[red]ERROR: Folder `{mypy_folder}` is wrong.[/]\n\n" + f"All folders passed should be one of those: {ALLOWED_FOLDERS}\n" + ) + else: + print( + f"\nERROR: Folder `{mypy_folder}` is wrong.\n\n" + f"All folders passed should be one of those: {ALLOWED_FOLDERS}\n" + ) + sys.exit(1) + +exclude_regexps = [ + re.compile(x) + for x in [ + r"^.*/node_modules/.*", + r"^.*\\..*", + r"^.*/src/airflow/__init__.py$", + ] +] + + +def get_all_files(folder: str) -> list[str]: + files_to_check = [] + python_file_paths = (AIRFLOW_ROOT_PATH / folder).resolve().rglob("*.py") + for file in python_file_paths: + if ( + file.name not in ("conftest.py",) + and not any(x.match(file.as_posix()) for x in exclude_regexps) + and not any(part.startswith(".") for part in file.parts) + ): + files_to_check.append(file.relative_to(AIRFLOW_ROOT_PATH).as_posix()) + return files_to_check + + +all_files_to_check: list[str] = [] +for mypy_folder in mypy_folders: + all_files_to_check.extend(get_all_files(mypy_folder)) + +if not all_files_to_check: + print("No files to test. Quitting") + sys.exit(0) + +# Write file list +mypy_file_list = AIRFLOW_ROOT_PATH / "files" / "mypy_files.txt" +mypy_file_list.parent.mkdir(parents=True, exist_ok=True) +mypy_file_list.write_text("\n".join(all_files_to_check)) + +if console: + console.print(f"[info]You can check the list of files in:[/] {mypy_file_list}") +else: + print(f"You can check the list of files in: {mypy_file_list}") + +file_argument_local = f"@{mypy_file_list}" +file_argument_ci = "@/files/mypy_files.txt" + +project = FOLDER_TO_PROJECT.get(mypy_folders[0], "devel-common") + +mypy_extra_args: list[str] = [] + +if show_unused_warnings == "true": + if console: + console.print("[info]Running mypy with --warn-unused-ignores") + else: + print("Running mypy with --warn-unused-ignores") + mypy_extra_args.append("--warn-unused-ignores") + +if show_unreachable_warnings == "true": + if console: + console.print("[info]Running mypy with --warn-unreachable") + else: + print("Running mypy with --warn-unreachable") + mypy_extra_args.append("--warn-unreachable") + +if console: + console.print(f"[magenta]Running mypy for folders: {mypy_folders}[/]") +else: + print(f"Running mypy for folders: {mypy_folders}") + +if CI: + # In CI, run inside the breeze Docker image to avoid needing a local environment + # and to not change uv.lock or synchronize dependencies. + from common_prek_utils import ( + initialize_breeze_prek, + run_command_via_breeze_shell, + ) + + initialize_breeze_prek(__name__, __file__) + + mypy_cmd = f"TERM=ansi mypy {shlex.quote(file_argument_ci)} {' '.join(mypy_extra_args)}" + result = run_command_via_breeze_shell( + cmd=["bash", "-c", mypy_cmd], + warn_image_upgrade_needed=True, + extra_env={ + "INCLUDE_MYPY_VOLUME": "false", + "MOUNT_SOURCES": "selected", + }, + ) +else: + # Locally, run via uv with --frozen to not update the lock file. + cmd = [ + "uv", + "run", + "--frozen", + "--project", + project, + "--with", + "apache-airflow-devel-common[mypy]", + "mypy", + file_argument_local, + *mypy_extra_args, + ] + + result = subprocess.run( + cmd, + cwd=str(AIRFLOW_ROOT_PATH), + check=False, + env={**os.environ, "TERM": "ansi"}, + ) + +if result.returncode != 0: + msg = ( + "Mypy check failed. You can run mypy locally with:\n" + f" prek run mypy-{mypy_folders[0]} --all-files\n" + "Or directly with:\n" + f' uv run --project {project} --with "apache-airflow-devel-common[mypy]" mypy \n' + "You can also clear the mypy cache with:\n" + " breeze down --cleanup-mypy-cache\n" + ) + if console: + console.print(f"[yellow]{msg}") + else: + print(msg) +sys.exit(result.returncode) diff --git a/scripts/ci/prek/update_airflow_pyproject_toml.py b/scripts/ci/prek/update_airflow_pyproject_toml.py index cccae46867ed1..ff3ee1e1fe244 100755 --- a/scripts/ci/prek/update_airflow_pyproject_toml.py +++ b/scripts/ci/prek/update_airflow_pyproject_toml.py @@ -85,7 +85,7 @@ def get_optional_dependencies(pyproject_toml_path: Path) -> list[str]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] airflow_core_toml_dict = tomllib.loads(pyproject_toml_path.read_text()) return airflow_core_toml_dict["project"]["optional-dependencies"].keys() @@ -112,7 +112,7 @@ def _read_toml(path: Path) -> dict[str, Any]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] return tomllib.loads(path.read_text()) diff --git a/scripts/ci/prek/update_providers_dependencies.py b/scripts/ci/prek/update_providers_dependencies.py index 9e89084f7b0e3..b114e55a04726 100755 --- a/scripts/ci/prek/update_providers_dependencies.py +++ b/scripts/ci/prek/update_providers_dependencies.py @@ -69,7 +69,7 @@ def load_pyproject_toml(pyproject_toml_file_path: Path) -> dict[str, Any]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] return tomllib.loads(pyproject_toml_file_path.read_text()) diff --git a/scripts/tools/initialize_virtualenv.py b/scripts/tools/initialize_virtualenv.py index fff6fbb4b6227..a6edf0268dca7 100755 --- a/scripts/tools/initialize_virtualenv.py +++ b/scripts/tools/initialize_virtualenv.py @@ -62,7 +62,7 @@ def get_dependency_groups(pyproject_toml_path: Path) -> list[str]: try: import tomllib except ImportError: - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] airflow_core_toml_dict = tomllib.loads(pyproject_toml_path.read_text()) return airflow_core_toml_dict["dependency-groups"].keys() diff --git a/task-sdk/.pre-commit-config.yaml b/task-sdk/.pre-commit-config.yaml index 315e0ea8a133f..df1be38500f71 100644 --- a/task-sdk/.pre-commit-config.yaml +++ b/task-sdk/.pre-commit-config.yaml @@ -56,17 +56,9 @@ repos: pass_filenames: false files: ^src/airflow/sdk/definitions/dag\.py$|^src/airflow/sdk/definitions/decorators/task_group\.py$ - id: mypy-task-sdk - stages: ['pre-push'] name: Run mypy for task-sdk language: python - entry: ../scripts/ci/prek/mypy.py - files: ^.*\.py$ - require_serial: true - - id: mypy-task-sdk - stages: ['manual'] - name: Run mypy for task-sdk (manual) - language: python - entry: ../scripts/ci/prek/mypy_folder.py task-sdk + entry: ../scripts/ci/prek/mypy_local_folder.py task-sdk pass_filenames: false files: ^.*\.py$ require_serial: true