Run non-provider mypy as regular prek static checks instead of separate CI jobs

.github/workflows/ci-amd-arm.yml

@@ -91,7 +91,6 @@ jobs:
       kubernetes-versions-list-as-string: >-
         ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
       latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
-      mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }}
       mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
       mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
       platform: ${{ steps.selective-checks.outputs.platform }}
@@ -115,7 +114,7 @@ jobs:
       run-go-sdk-tests: ${{ steps.selective-checks.outputs.run-go-sdk-tests }}
       run-helm-tests: ${{ steps.selective-checks.outputs.run-helm-tests }}
       run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
-      run-mypy: ${{ steps.selective-checks.outputs.run-mypy }}
+      run-mypy-providers: ${{ steps.selective-checks.outputs.run-mypy-providers }}
       run-remote-logging-elasticsearch-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }}
       run-remote-logging-s3-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }}
       run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
@@ -307,8 +306,6 @@ jobs:
     with:
       runners: ${{ needs.build-info.outputs.runner-type }}
       platform: ${{ needs.build-info.outputs.platform }}
-      run-mypy: ${{ needs.build-info.outputs.run-mypy }}
-      mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       canary-run: ${{ needs.build-info.outputs.canary-run }}
@@ -333,6 +330,51 @@ jobs:
       DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
       SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
+  mypy-providers:
+    timeout-minutes: 45
+    name: "MyPy providers checks"
+    needs: [build-info, build-ci-images]
+    runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+    if: needs.build-info.outputs.run-mypy-providers == 'true'
+    env:
+      PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}"
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          persist-credentials: false
+      - name: "Free up disk space"
+        shell: bash
+        run: ./scripts/tools/free_up_disk_space.sh
+      - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}"
+        uses: ./.github/actions/prepare_breeze_and_image
+        with:
+          platform: ${{ needs.build-info.outputs.platform }}
+          python: "${{ needs.build-info.outputs.default-python-version }}"
+          use-uv: ${{ needs.build-info.outputs.use-uv }}
+          make-mnt-writeable-and-cleanup: true
+        id: breeze
+      - name: "Install prek"
+        uses: ./.github/actions/install-prek
+        id: prek
+        with:
+          python-version: ${{steps.breeze.outputs.host-python-version}}
+          platform: ${{ needs.build-info.outputs.platform }}
+          save-cache: false
+      - name: "MyPy checks for providers"
+        run: prek --color always --verbose --stage manual mypy-providers --all-files
+        env:
+          VERBOSE: "false"
+          COLUMNS: "202"
+          SKIP_GROUP_OUTPUT: "true"
+          DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+          RUFF_FORMAT: "github"
+          INCLUDE_MYPY_VOLUME: "false"
+
   providers:
     name: "provider distributions tests"
     uses: ./.github/workflows/test-providers.yml
@@ -895,6 +937,7 @@ jobs:
       - build-prod-images
       - ci-image-checks
       - generate-constraints
+      - mypy-providers
       - providers
       - tests-helm
       - tests-integration-system

.github/workflows/ci-image-checks.yml

@@ -28,14 +28,6 @@ on:  # yamllint disable-line rule:truthy
         description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
         required: true
         type: string
-      run-mypy:
-        description: "Whether to run mypy checks (true/false)"
-        required: true
-        type: string
-      mypy-checks:
-        description: "List of folders to run mypy checks on"
-        required: false
-        type: string
       python-versions-list-as-string:
         description: "The list of python versions as string separated by spaces"
         required: true
@@ -169,55 +161,6 @@ jobs:
         run: cat ~/.cache/prek/prek.log || true
         if: failure()
 
-  mypy:
-    timeout-minutes: 45
-    name: "MyPy checks"
-    runs-on: ${{ fromJSON(inputs.runners) }}
-    if: inputs.run-mypy == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        mypy-check: ${{ fromJSON(inputs.mypy-checks) }}
-    env:
-      PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - name: "Cleanup repo"
-        shell: bash
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          persist-credentials: false
-      - name: "Free up disk space"
-        shell: bash
-        run: ./scripts/tools/free_up_disk_space.sh
-      - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
-        uses: ./.github/actions/prepare_breeze_and_image
-        with:
-          platform: ${{ inputs.platform }}
-          python: "${{ inputs.default-python-version }}"
-          use-uv: ${{ inputs.use-uv }}
-          make-mnt-writeable-and-cleanup: true
-        id: breeze
-      - name: "Install prek"
-        uses: ./.github/actions/install-prek
-        id: prek
-        with:
-          python-version: ${{steps.breeze.outputs.host-python-version}}
-          platform: ${{ inputs.platform }}
-          save-cache: false
-      - name: "MyPy checks for ${{ matrix.mypy-check }}"
-        run: prek --color always --verbose --stage manual "$MYPY_CHECK" --all-files
-        env:
-          VERBOSE: "false"
-          COLUMNS: "202"
-          SKIP_GROUP_OUTPUT: "true"
-          DEFAULT_BRANCH: ${{ inputs.branch }}
-          RUFF_FORMAT: "github"
-          INCLUDE_MYPY_VOLUME: "false"
-          MYPY_CHECK: ${{ matrix.mypy-check }}
-
   build-docs:
     timeout-minutes: 150
     name: "Build documentation"

.pre-commit-config.yaml

@@ -1014,39 +1014,23 @@ repos:
           ^uv\.lock$
         pass_filenames: false
         require_serial: true
-        ## ADD MOST PREK HOOK ABOVE THAT LINE
-        # The below prek hooks are those requiring CI image to be built
-        ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
       - id: mypy-dev
-        stages: ['pre-push']
         name: Run mypy for dev
         language: python
-        entry: ./scripts/ci/prek/mypy.py
-        files: ^dev/.*\.py$|^scripts/.*\.py$
-        require_serial: true
-      - id: mypy-dev
-        stages: ['manual']
-        name: Run mypy for dev (manual)
-        language: python
-        entry: ./scripts/ci/prek/mypy_folder.py dev scripts
+        entry: ./scripts/ci/prek/mypy_local_folder.py dev scripts
         pass_filenames: false
         files: ^.*\.py$
         require_serial: true
       - id: mypy-devel-common
-        stages: ['pre-push']
         name: Run mypy for devel-common
         language: python
-        entry: ./scripts/ci/prek/mypy.py
-        files: ^devel-common/.*\.py$
-        require_serial: true
-      - id: mypy-devel-common
-        stages: ['manual']
-        name: Run mypy for devel-common (manual)
-        language: python
-        entry: ./scripts/ci/prek/mypy_folder.py devel-common
+        entry: ./scripts/ci/prek/mypy_local_folder.py devel-common
         pass_filenames: false
         files: ^.*\.py$
         require_serial: true
+      ## ADD MOST PREK HOOK ABOVE THAT LINE
+      # The below prek hooks are those requiring CI image to be built
+      ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
       - id: check-template-fields-valid
         name: Check templated fields mapped in operators/sensors
         language: python

AGENTS.md

@@ -30,7 +30,8 @@
 - **Run other suites of tests** `breeze testing <test_group>` (test groups: `airflow-ctl-tests`, `docker-compose-tests`, `task-sdk-tests`)
 - **Run scripts tests:** `uv run --project scripts pytest scripts/tests/ -xvs`
 - **Run Airflow CLI:** `breeze run airflow dags list`
-- **Type-check:** `breeze run mypy path/to/code`
+- **Type-check (non-providers):** `uv run --project <PROJECT> --with "apache-airflow-devel-common[mypy]" mypy path/to/code`
+- **Type-check (providers):** `breeze run mypy path/to/code`
 - **Lint with ruff only:** `prek run ruff --from-ref <target_branch>`
 - **Format with ruff only:** `prek run ruff-format --from-ref <target_branch>`
 - **Run regular (fast) static checks:** `prek run --from-ref <target_branch> --stage pre-commit`
@@ -147,7 +148,7 @@ code review checklist in [`.github/instructions/code-review.instructions.md`](.g
 3. Confirm the code follows the project's coding standards and architecture boundaries
    described in this file.
 4. Run regular (fast) static checks (`prek run --from-ref <target_branch> --stage pre-commit`)
-   and fix any failures.
+   and fix any failures. This includes mypy checks for non-provider projects (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common).
 5. Run manual (slower) checks (`prek run --from-ref <target_branch> --stage manual`) and fix any failures.
 6. Run relevant individual tests and confirm they pass.
 7. Find which tests to run for the changes with selective-checks and run those tests in parallel to confirm they pass and check for CI-specific issues.

airflow-core/.pre-commit-config.yaml

@@ -221,23 +221,15 @@ repos:
         additional_dependencies: ['pnpm@10.25.0']
         pass_filenames: true
         require_serial: true
-        ## ADD MOST PREK HOOK ABOVE THAT LINE
-        # The below prek hooks are those requiring CI image to be built
       - id: mypy-airflow-core
-        stages: ['pre-push']
         name: Run mypy for airflow-core
         language: python
-        entry: ../scripts/ci/prek/mypy.py
-        files: ^.*\.py$
-        require_serial: true
-      - id: mypy-airflow-core
-        stages: ['manual']
-        name: Run mypy for airflow-core (manual)
-        language: python
-        entry: ../scripts/ci/prek/mypy_folder.py airflow-core
+        entry: ../scripts/ci/prek/mypy_local_folder.py airflow-core
         pass_filenames: false
         files: ^.*\.py$
         require_serial: true
+      ## ADD MOST PREK HOOK ABOVE THAT LINE
+      # The below prek hooks are those requiring CI image to be built
       - id: generate-openapi-spec
         name: Generate the FastAPI API spec
         language: python

airflow-core/src/airflow/utils/log/non_caching_file_handler.py

@@ -25,7 +25,7 @@
 def make_file_io_non_caching(io: IO[str]) -> IO[str]:
     try:
         fd = io.fileno()
-        os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
+        os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)  # type: ignore[attr-defined]
     except Exception:
         # in case either file descriptor cannot be retrieved or fadvise is not available
         # we should simply return the wrapper retrieved by FileHandler's open method

airflow-ctl/.pre-commit-config.yaml

@@ -25,21 +25,9 @@ repos:
   - repo: local
     hooks:
       - id: mypy-airflow-ctl
-        stages: ['pre-push']
         name: Run mypy for airflow-ctl
         language: python
-        entry: ../scripts/ci/prek/mypy.py
-        files:
-          (?x)
-          ^src/airflowctl/.*\.py$|
-          ^tests/.*\.py$
-        exclude: .*generated.py
-        require_serial: true
-      - id: mypy-airflow-ctl
-        stages: ['manual']
-        name: Run mypy for airflow-ctl (manual)
-        language: python
-        entry: ../scripts/ci/prek/mypy_folder.py airflow-ctl
+        entry: ../scripts/ci/prek/mypy_local_folder.py airflow-ctl
         pass_filenames: false
         files: ^.*\.py$
         require_serial: true

contributing-docs/08_static_code_checks.rst

@@ -173,17 +173,18 @@ But you can run prek hooks manually as needed.
     prek
 
 -   Run only mypy check on your staged airflow and dev files by specifying the
-    ``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be specified):
+    ``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be specified).
+    For non-provider projects, mypy runs locally via ``uv`` (no breeze image needed):
 
 .. code-block:: bash
 
-    prek mypy-airflow-core mypy-dev  --stage pre-push
+    prek mypy-airflow-core mypy-dev
 
 -   Run only mypy airflow checks on all "airflow-core" files by using:
 
 .. code-block:: bash
 
-    prek mypy-airflow-core --all-files --stage pre-push
+    prek mypy-airflow-core --all-files
 
 -   Run all pre-commit stage hooks on all files by using:
 
@@ -279,41 +280,40 @@ them manually by running ``prek --stage manual <hook-id>``.
 Mypy checks
 -----------
 
-When we run mypy checks locally when pushing a change to PR, the ``mypy-*`` checks is run, ``mypy-airflow``,
-``mypy-dev``, ``mypy-providers``, ``mypy-airflow-ctl``, depending on the files you are changing. The mypy checks
-are run by passing those changed files to mypy. This is way faster than running checks for all files (even
-if mypy cache is used - especially when you change a file in Airflow core that is imported and used by many
-files). You also need to have ``breeze ci-image build --python 3.10`` built locally to run the mypy checks.
+When we run mypy checks locally, the ``mypy-*`` checks run depending on the files you are changing:
+``mypy-airflow-core``, ``mypy-dev``, ``mypy-providers``, ``mypy-task-sdk``, ``mypy-airflow-ctl``, etc.
 
-However, in some cases, it produces different results than when running checks for the whole set
-of files, because ``mypy`` does not even know that some types are defined in other files and it might not
-be able to follow imports properly if they are dynamic. Therefore in CI we run ``mypy`` check for whole
-directories (``airflow`` - excluding providers, ``providers``, ``dev`` and ``docs``) to make sure
-that we catch all ``mypy`` errors - so you can experience different results when running mypy locally and
-in CI. If you want to run mypy checks for all files locally, you can do it by running the following
-command (example for ``airflow`` files):
+For **non-provider projects** (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common), mypy
+runs locally using the ``uv`` virtualenv — no breeze CI image is needed. These checks run as regular
+prek hooks in the ``pre-commit`` stage, checking whole directories at once. This means they run both
+as part of local commits and as part of regular static checks in CI (not as separate mypy CI jobs).
+You can also run mypy directly. Use ``--frozen`` to avoid updating ``uv.lock``:
 
 .. code-block:: bash
 
-  prek --stage manual mypy-<FOLDER> --all-files
+  uv run --frozen --project <PROJECT> --with "apache-airflow-devel-common[mypy]" mypy path/to/code
 
-For example:
+To run the prek hook for a specific project (example for ``airflow-core`` files):
 
 .. code-block:: bash
 
-  prek --stage manual mypy-airflow --all-files
+  prek mypy-airflow-core --all-files
 
 To show unused mypy ignores for any providers/airflow etc, eg: run below command:
 
 .. code-block:: bash
+
   export SHOW_UNUSED_MYPY_WARNINGS=true
-  prek --stage manual mypy-airflow --all-files
+  prek mypy-airflow-core --all-files
+
+For non-provider projects, the local mypy cache is stored in ``.mypy_cache`` at the repo root.
+
+For **providers**, mypy still runs via breeze (``breeze run mypy``) as a separate CI job and requires
+``breeze ci-image build --python 3.10`` to be built locally. Providers use a separate docker-volume
+(called ``mypy-cache-volume``) that keeps the cache of last MyPy execution.
 
-MyPy uses a separate docker-volume (called ``mypy-cache-volume``) that keeps the cache of last MyPy
-execution in order to speed MyPy checks up (sometimes by order of magnitude). While in most cases MyPy
-will handle refreshing the cache when and if needed, there are some cases when it won't (cache invalidation
-is the hard problem in computer science). This might happen for example when we upgrade MyPY. In such
-cases you might need to manually remove the cache volume by running ``breeze down --cleanup-mypy-cache``.
+To clear all mypy caches (both local ``.mypy_cache`` and the Docker volume), run
+``breeze down --cleanup-mypy-cache``.
 
 -----------
 

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -932,6 +932,10 @@ def down(preserve_volumes: bool, cleanup_mypy_cache: bool, cleanup_build_cache:
     if cleanup_mypy_cache:
         command_to_execute = ["docker", "volume", "rm", "--force", "mypy-cache-volume"]
         run_command(command_to_execute)
+        local_mypy_cache = AIRFLOW_ROOT_PATH / ".mypy_cache"
+        if local_mypy_cache.exists():
+            console_print(f"\n[info]Removing local mypy cache: {local_mypy_cache}\n")
+            shutil.rmtree(local_mypy_cache)
     if cleanup_build_cache:
         command_to_execute = ["docker", "volume", "rm", "--force", "airflow-cache-volume"]
         run_command(command_to_execute)
@@ -1070,6 +1074,10 @@ def doctor(ctx):
         console_print("\n[info]Cleaning mypy cache...\n")
         command_to_execute = ["docker", "volume", "rm", "--force", "mypy-cache-volume"]
         run_command(command_to_execute)
+        local_mypy_cache = AIRFLOW_ROOT_PATH / ".mypy_cache"
+        if local_mypy_cache.exists():
+            console_print(f"\n[info]Removing local mypy cache: {local_mypy_cache}\n")
+            shutil.rmtree(local_mypy_cache)
 
         console_print("\n[info]Cleaning build cache...\n")
         command_to_execute = ["docker", "volume", "rm", "--force", "airflow-cache-volume"]

dev/breeze/src/airflow_breeze/commands/registry_commands.py

@@ -160,7 +160,7 @@ def _read_provider_yaml_info(provider_id: str) -> tuple[str, list[str]]:
     try:
         import tomllib
     except ImportError:
-        import tomli as tomllib
+        import tomli as tomllib  # type: ignore[no-redef]
 
     provider_yaml_path = _find_provider_yaml(provider_id)
     with open(provider_yaml_path) as f: