[v3-2-test] Isolate non-provider mypy hooks per distribution with ded…

.pre-commit-config.yaml

@@ -1021,19 +1021,19 @@ repos:
           ^uv\.lock$
         pass_filenames: false
         require_serial: true
-      - id: mypy-dev
-        name: Run mypy for dev
+      - id: mypy-devel-common
+        name: Run mypy for devel-common
         language: python
-        entry: ./scripts/ci/prek/mypy_local_folder.py dev scripts
+        entry: ./scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py devel-common
         pass_filenames: false
         files: ^.*\.py$
         require_serial: true
-      - id: mypy-devel-common
-        name: Run mypy for devel-common
+      - id: check-shared-mypy-hooks
+        name: Every shared/<dist> has a mypy-shared-<dist> prek hook
         language: python
-        entry: ./scripts/ci/prek/mypy_local_folder.py devel-common
+        entry: ./scripts/ci/prek/check_shared_mypy_hooks.py
         pass_filenames: false
-        files: ^.*\.py$
+        files: ^shared/.*|^scripts/ci/prek/check_shared_mypy_hooks\.py$
         require_serial: true
       ## ADD MOST PREK HOOK ABOVE THAT LINE
       # The below prek hooks are those requiring CI image to be built

AGENTS.md

@@ -30,7 +30,7 @@
 - **Run other suites of tests** `breeze testing <test_group>` (test groups: `airflow-ctl-tests`, `docker-compose-tests`, `task-sdk-tests`)
 - **Run scripts tests:** `uv run --project scripts pytest scripts/tests/ -xvs`
 - **Run Airflow CLI:** `breeze run airflow dags list`
-- **Type-check (non-providers):** first run `uv sync --frozen --project <PROJECT>` to align the local virtualenv with `uv.lock` (the dependency set CI uses), then `uv run --frozen --project <PROJECT> --with "apache-airflow-devel-common[mypy]" mypy path/to/code`
+- **Type-check (non-providers):** run the prek hook — `prek run mypy-<project> --all-files` (e.g. `mypy-airflow-core`, `mypy-task-sdk`, `mypy-shared-logging`; each `shared/<dist>` workspace member has its own `mypy-shared-<dist>` hook). The hook uses a dedicated virtualenv and mypy cache under `.build/mypy-venvs/<hook>/` and `.build/mypy-caches/<hook>/`; mypy itself is installed from `uv.lock` via the `mypy` dependency group (`uv sync --group mypy`), so it never mutates your project `.venv`. Clear with `breeze down --cleanup-mypy-cache`.
 - **Type-check (providers):** `breeze run mypy path/to/code`
 - **Lint with ruff only:** `prek run ruff --from-ref <target_branch>`
 - **Format with ruff only:** `prek run ruff-format --from-ref <target_branch>`

airflow-core/.pre-commit-config.yaml

@@ -16,7 +16,7 @@
 # under the License.
 ---
 default_stages: [pre-commit, pre-push]
-minimum_prek_version: '0.2.0'
+minimum_prek_version: '0.3.4'
 default_language_version:
   python: python3
   node: 22.19.0
@@ -224,7 +224,7 @@ repos:
       - id: mypy-airflow-core
         name: Run mypy for airflow-core
         language: python
-        entry: ../scripts/ci/prek/mypy_local_folder.py airflow-core
+        entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-core
         pass_filenames: false
         files: ^.*\.py$
         require_serial: true

airflow-core/pyproject.toml

@@ -311,6 +311,10 @@ dev = [
 docs = [
     "apache-airflow-devel-common[docs]"
 ]
+mypy = [
+    "apache-airflow-devel-common[mypy]",
+]
+
 
 
 [tool.uv]

airflow-ctl-tests/.pre-commit-config.yaml

@@ -16,7 +16,7 @@
 # under the License.
 ---
 default_stages: [pre-commit, pre-push]
-minimum_prek_version: '0.2.0'
+minimum_prek_version: '0.3.4'
 default_language_version:
   python: python3
 repos:
@@ -30,3 +30,10 @@ repos:
         files:
           (?x)
           ^tests/airflowctl_tests/.*\.py$
+      - id: mypy-airflow-ctl-tests
+        name: Run mypy for airflow-ctl-tests
+        language: python
+        entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-ctl-tests
+        pass_filenames: false
+        files: ^.*\.py$
+        require_serial: true

airflow-ctl-tests/pyproject.toml

@@ -76,3 +76,8 @@ exclude = ["*"]
 
 [tool.hatch.build.targets.wheel]
 bypass-selection = true
+
+[dependency-groups]
+mypy = [
+    "apache-airflow-devel-common[mypy]",
+]

airflow-ctl/.pre-commit-config.yaml

@@ -16,7 +16,7 @@
 # under the License.
 ---
 default_stages: [pre-commit, pre-push]
-minimum_prek_version: '0.2.0'
+minimum_prek_version: '0.3.4'
 default_language_version:
   python: python3
   node: 22.19.0
@@ -27,7 +27,7 @@ repos:
       - id: mypy-airflow-ctl
         name: Run mypy for airflow-ctl
         language: python
-        entry: ../scripts/ci/prek/mypy_local_folder.py airflow-ctl
+        entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-ctl
         pass_filenames: false
         files: ^.*\.py$
         require_serial: true

airflow-ctl/pyproject.toml

@@ -151,6 +151,10 @@ codegen = [
 ]
 
 # uv run --verbose --group codegen --project apache-airflow-ctl --directory airflow-ctl/ datamodel-codegen --url="http://0.0.0.0:28080/auth/openapi.json" --output=src/airflowctl/api/datamodels/auth_generated.py
+mypy = [
+    "apache-airflow-devel-common[mypy]",
+]
+
 [tool.datamodel-codegen]
 capitalise-enum-members=true # `State.RUNNING` not `State.running`
 disable-timestamp=true

airflow-e2e-tests/.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+default_stages: [pre-commit, pre-push]
+minimum_prek_version: '0.3.4'
+default_language_version:
+  python: python3
+repos:
+  - repo: local
+    hooks:
+      - id: mypy-airflow-e2e-tests
+        name: Run mypy for airflow-e2e-tests
+        language: python
+        entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py airflow-e2e-tests
+        pass_filenames: false
+        files: ^.*\.py$
+        require_serial: true

airflow-e2e-tests/pyproject.toml

@@ -77,3 +77,8 @@ exclude = ["*"]
 
 [tool.hatch.build.targets.wheel]
 bypass-selection = true
+
+[dependency-groups]
+mypy = [
+    "apache-airflow-devel-common[mypy]",
+]

chart/.pre-commit-config.yaml

@@ -16,7 +16,7 @@
 # under the License.
 ---
 default_stages: [pre-commit, pre-push]
-minimum_prek_version: '0.2.0'
+minimum_prek_version: '0.3.4'
 default_language_version:
   python: python3
   node: 22.19.0

contributing-docs/08_static_code_checks.rst

@@ -281,47 +281,59 @@ Mypy checks
 -----------
 
 When we run mypy checks locally, the ``mypy-*`` checks run depending on the files you are changing:
-``mypy-airflow-core``, ``mypy-dev``, ``mypy-providers``, ``mypy-task-sdk``, ``mypy-airflow-ctl``, etc.
+``mypy-airflow-core``, ``mypy-dev``, ``mypy-providers``, ``mypy-scripts``, ``mypy-task-sdk``,
+``mypy-airflow-ctl``, ``mypy-devel-common``, ``mypy-airflow-ctl-tests``, ``mypy-helm-tests``,
+``mypy-airflow-e2e-tests``, ``mypy-task-sdk-integration-tests``, ``mypy-docker-tests``,
+``mypy-kubernetes-tests``, and one ``mypy-shared-<dist>`` hook per ``shared/<dist>`` workspace
+distribution (e.g. ``mypy-shared-configuration``, ``mypy-shared-logging``).
 
-For **non-provider projects** (airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common), mypy
-runs locally using the ``uv`` virtualenv — no breeze CI image is needed. These checks run as regular
-prek hooks in the ``pre-commit`` stage, checking whole directories at once. This means they run both
-as part of local commits and as part of regular static checks in CI (not as separate mypy CI jobs).
+For **non-provider projects**, mypy runs locally using ``uv`` — no breeze CI image is needed. These
+checks run as regular prek hooks in the ``pre-commit`` stage, checking whole directories at once. This
+means they run both as part of local commits and as part of regular static checks in CI (not as
+separate mypy CI jobs).
 
-Before running mypy directly (or via the ``mypy-*`` prek hooks), synchronize your local virtualenv
-with ``uv.lock`` so it matches the dependency set CI uses — otherwise mypy may pick up a different
-set of installed packages than CI and produce results that diverge from CI:
+Each non-provider ``mypy-*`` hook uses a **dedicated virtualenv and mypy cache** under ``.build/`` so
+running mypy never mutates your regular project ``.venv`` and each hook keeps a stable, CI-aligned
+dependency set:
 
-.. code-block:: bash
+- virtualenvs: ``.build/mypy-venvs/<hook-name>/``
+- mypy caches: ``.build/mypy-caches/<hook-name>/``
 
-  uv sync --frozen --project <PROJECT>
+Adding a new shared library
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Then run mypy directly. Use ``--frozen`` so ``uv`` does not update ``uv.lock``:
+Every ``shared/<dist>`` workspace member has its own ``mypy-shared-<dist>`` prek hook so it is
+type-checked in isolation against its own dependency set. When you add a new shared library under
+``shared/<new-dist>/``, you also need to:
 
-.. code-block:: bash
+1. Add a ``[dependency-groups]`` section with ``mypy = ["apache-airflow-devel-common[mypy]"]`` in
+   ``shared/<new-dist>/pyproject.toml`` (so ``uv sync --group mypy`` installs mypy into the hook's
+   dedicated virtualenv).
+2. Create ``shared/<new-dist>/.pre-commit-config.yaml`` with a ``mypy-shared-<new-dist>`` hook
+   entry that calls ``../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py shared/<new-dist>``.
 
-  uv run --frozen --project <PROJECT> --with "apache-airflow-devel-common[mypy]" mypy path/to/code
+The ``check-shared-mypy-hooks`` prek hook enforces step 2 — it fails and prints the exact config
+contents to add when any ``shared/<dist>`` is missing its dedicated mypy hook.
 
 To run the prek hook for a specific project (example for ``airflow-core`` files):
 
 .. code-block:: bash
 
   prek mypy-airflow-core --all-files
 
-To show unused mypy ignores for any providers/airflow etc, eg: run below command:
+To show unused mypy ignores, run:
 
 .. code-block:: bash
 
   export SHOW_UNUSED_MYPY_WARNINGS=true
   prek mypy-airflow-core --all-files
 
-For non-provider projects, the local mypy cache is stored in ``.mypy_cache`` at the repo root.
-
 For **providers**, mypy still runs via breeze (``breeze run mypy``) as a separate CI job and requires
 ``breeze ci-image build --python 3.10`` to be built locally. Providers use a separate docker-volume
 (called ``mypy-cache-volume``) that keeps the cache of last MyPy execution.
 
-To clear all mypy caches (both local ``.mypy_cache`` and the Docker volume), run
+To clear all mypy caches (the Docker volume used by providers, any legacy repo-root ``.mypy_cache``,
+and the per-hook venvs + caches under ``.build/mypy-venvs/`` and ``.build/mypy-caches/``), run
 ``breeze down --cleanup-mypy-cache``.
 
 -----------

dev/.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+default_stages: [pre-commit, pre-push]
+minimum_prek_version: '0.3.4'
+default_language_version:
+  python: python3
+repos:
+  - repo: local
+    hooks:
+      - id: mypy-dev
+        name: Run mypy for dev
+        language: python
+        entry: ../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py dev
+        pass_filenames: false
+        files: ^.*\.py$
+        require_serial: true

dev/breeze/doc/03_developer_tasks.rst

@@ -321,7 +321,7 @@ For example, this following command:
 
 .. code-block:: bash
 
-     prek mypy-airflow
+     prek mypy-airflow-core
 
 will run mypy check for currently staged files inside ``airflow/`` excluding providers.
 .. _breeze-dev:running-prek-in-breeze:
@@ -349,7 +349,7 @@ re-run latest prek hooks on your changes, but it can take a long time (few minut
 
 .. code-block:: bash
 
-     prek mypy-airflow --all-files
+     prek mypy-airflow-core --all-files
 
 The above will run mypy check for all files.
 
@@ -358,7 +358,7 @@ specifying (can be multiple times) ``--file`` flag.
 
 .. code-block:: bash
 
-     prek mypy-airflow --file airflow/utils/code_utils.py --file airflow/utils/timeout.py
+     prek mypy-airflow-core --file airflow/utils/code_utils.py --file airflow/utils/timeout.py
 
 The above will run mypy check for those to files (note: autocomplete should work for the file selection).
 
@@ -370,7 +370,7 @@ of commits you choose.
 
 .. code-block:: bash
 
-     prek mypy-airflow --last-commit
+     prek mypy-airflow-core --last-commit
 
 The above will run mypy check for all files in the last commit in your branch.
 
@@ -383,9 +383,17 @@ in ``--from-ref`` and ``--to-ref`` flags.
 
 .. note::
 
-    When you run static checks, some of the artifacts (mypy_cache) is stored in docker-compose volume
-    so that it can speed up static checks execution significantly. However, sometimes, the cache might
-    get broken, in which case you should run ``breeze down`` to clean up the cache.
+    When you run static checks, some of the artifacts (mypy_cache) is stored to speed up static
+    checks execution significantly:
+
+    - The providers ``mypy-providers`` hook runs via Breeze and stores its cache in the
+      ``mypy-cache-volume`` docker-compose volume.
+    - Each non-provider ``mypy-*`` hook uses its own dedicated virtualenv and mypy cache under
+      ``.build/mypy-venvs/<hook>/`` and ``.build/mypy-caches/<hook>/``; mypy itself is installed
+      from the workspace ``uv.lock`` via the ``mypy`` dependency group (``uv sync --group mypy``).
+
+    If the cache gets broken, run ``breeze down --cleanup-mypy-cache`` which wipes the docker
+    volume and every per-hook ``.build/mypy-venvs/`` and ``.build/mypy-caches/`` directory.
 
 .. note::
 

dev/breeze/doc/ci/04_selective_checks.md

@@ -68,9 +68,20 @@ We have the following Groups of files for CI that determine which tests are run:
 * `All Python files` - if none of the Python file changed, that indicates that we should not run unit tests
 * `All source files` - if none of the sources change, that indicates that we should probably not build
   an image and run any image-based static checks
-* `All Airflow Python files` - files that are checked by `mypy-airflow` static checks
+* `All Airflow Python files` - files that are checked by `mypy-airflow-core` static checks
 * `All Providers Python files` - files that are checked by `mypy-providers` static checks
 * `All Dev Python files` - files that are checked by `mypy-dev` static checks
+* `All Scripts Python files` - files that are checked by `mypy-scripts` static checks
+* `Task SDK files` - files that are checked by `mypy-task-sdk` static checks
+* `All Airflow CTL Python files` - files that are checked by `mypy-airflow-ctl` static checks
+* `All Devel Common Python files` - files that are checked by `mypy-devel-common` static checks
+* `All Helm Tests Python files` / `All Docker Tests Python files` /
+  `All Kubernetes Tests Python files` / `All Airflow E2E Tests Python files` /
+  `Airflow CTL Integration Test files` / `Task SDK Integration Test files` - files that are
+  checked by the respective `mypy-helm-tests` / `mypy-docker-tests` / `mypy-kubernetes-tests` /
+  `mypy-airflow-e2e-tests` / `mypy-airflow-ctl-tests` / `mypy-task-sdk-integration-tests` hooks
+* `shared/<dist>/**/*.py` - each `shared/<dist>` workspace member has its own `mypy-shared-<dist>`
+  hook (selective-checks enumerates them at runtime)
 * `All Provider Yaml files` - all provider yaml files
 
 We have a number of `TEST_TYPES` that can be selectively disabled/enabled based on the
@@ -140,8 +151,17 @@ when some files are not changed. Those are the rules implemented:
 * If "full tests" mode is detected, no more prek hooks are skipped - we run all of them
 * The following checks are skipped if those files are not changed:
   * if no `All Providers Python files` changed - `mypy-providers` check is skipped
-  * if no `All Airflow Python files` changed - `mypy-airflow` check is skipped
+  * if no `All Airflow Python files` changed - `mypy-airflow-core` check is skipped
   * if no `All Dev Python files` changed - `mypy-dev` check is skipped
+  * if no `All Scripts Python files` changed - `mypy-scripts` check is skipped
+  * if no `Task SDK files` changed - `mypy-task-sdk` check is skipped
+  * if no `All Airflow CTL Python files` changed - `mypy-airflow-ctl` check is skipped
+  * if no `All Devel Common Python files` changed - `mypy-devel-common` check is skipped
+  * if no files under the matching folder changed, the corresponding per-folder mypy hook
+    is skipped (`mypy-airflow-ctl-tests`, `mypy-helm-tests`, `mypy-airflow-e2e-tests`,
+    `mypy-task-sdk-integration-tests`, `mypy-docker-tests`, `mypy-kubernetes-tests`)
+  * for each `shared/<dist>` workspace member, `mypy-shared-<dist>` is skipped when no
+    file under `shared/<dist>/` changed (enumerated at runtime)
   * if no `UI files` changed - `ts-compile-format-lint-ui` check is skipped
   * if no `WWW files` changed - `ts-compile-format-lint-www` check is skipped
   * if no `All Python files` changed - `flynt` check is skipped

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -936,6 +936,11 @@ def down(preserve_volumes: bool, cleanup_mypy_cache: bool, cleanup_build_cache:
         if local_mypy_cache.exists():
             console_print(f"\n[info]Removing local mypy cache: {local_mypy_cache}\n")
             shutil.rmtree(local_mypy_cache)
+        for subdir in ("mypy-venvs", "mypy-caches"):
+            hook_dir = AIRFLOW_ROOT_PATH / ".build" / subdir
+            if hook_dir.exists():
+                console_print(f"\n[info]Removing dedicated mypy {subdir}: {hook_dir}\n")
+                shutil.rmtree(hook_dir)
     if cleanup_build_cache:
         command_to_execute = ["docker", "volume", "rm", "--force", "airflow-cache-volume"]
         run_command(command_to_execute)