From 670cf127c9c542790ce9f58d33f3c8c6fd5d044a Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Tue, 2 Jun 2026 19:52:13 -0700 Subject: [PATCH 01/13] MAINT: Drop 10 unused dev dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove `mock-alchemy` plus 9 `types-*` stubs that are no longer needed: - `mock-alchemy`: last importer (UnifiedAlchemyMagicMock in `test_azure_sql_memory.py`) was deleted in PR #1128 (2025-10-24, "FEAT Breaking: Default Values") when the SQL test suite migrated to in-memory SQLite + real SQLAlchemy sessions. `pyproject.toml` was missed in that cleanup. - `types-cachetools`, `types-decorator`, `types-paramiko`, `types-pycurl`, `types-pytz`, `types-simplejson`, `types-six`, `types-tabulate`, `types-ujson`: added in commit `a5a0dd51` (2026-01-13, "more typing dev deps") as setup for strict mypy (PR #1310). The underlying packages are not imported anywhere in `pyrit/`, `tests/`, `doc/`, or `build_scripts/`, and PyRIT has since migrated off mypy to `ty`. Type checkers only consult stubs for imports they actually see, so these have zero effect. Keep `types-aiofiles`, `types-PyYAML`, `types-requests`: the underlying packages are imported widely and ship no inline type info (no `py.typed`), so the stubs are the only type information available. Verified with `uv sync --link-mode=copy` (11 packages uninstalled cleanly, including transitive `sqlalchemy-utils` from mock-alchemy) and `uv run pytest -n 4 --dist=loadfile tests/unit` (8382 passed, 120 skipped — no break from mock-alchemy removal). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyproject.toml | 10 ----- uv.lock | 116 ------------------------------------------------- 2 files changed, 126 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 88586ff38f..ff5fbb0453 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,6 @@ dev = [ "jupytext>=1.17.1", "matplotlib>=3.10.0", "ty>=0.0.32", - "mock-alchemy>=0.2.6", "pandas>=2.2.0", "pre-commit>=4.2.0", "pytest>=9.0.3", @@ -94,17 +93,8 @@ dev = [ "respx>=0.22.0", "ruff>=0.14.4", "types-aiofiles>=24.1.0", - "types-cachetools>=5.5.0", - "types-decorator>=5.1.0", - "types-paramiko>=3.5.0", - "types-pycurl>=7.45.0", - "types-pytz>=2024.2.0", "types-PyYAML>=6.0.12.20250516", "types-requests>=2.31.0.20250515", - "types-simplejson>=3.19.0", - "types-six>=1.16.0", - "types-tabulate>=0.9.0", - "types-ujson>=5.10.0", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index 086686162b..911bdbc4a4 100644 --- a/uv.lock +++ b/uv.lock @@ -3276,18 +3276,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/7f/a946aa4f8752b37102b41e64dca18a1976ac705c3a0d1dfe74d820a02552/mistune-3.2.1-py3-none-any.whl", hash = "sha256:78cdb0ba5e938053ccf63651b352508d2efa9411dc8810bfb05f2dc5140c0048", size = 53749, upload-time = "2026-05-03T14:33:20.551Z" }, ] -[[package]] -name = "mock-alchemy" -version = "0.2.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sqlalchemy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/80/5af6036ed9a97d927c36384b63ea2f270224726cda7f1402cb62781d7054/mock_alchemy-0.2.6.tar.gz", hash = "sha256:807cc2bd7f658fb98292900052eaa6eeb4a65b2d62364e639462480a275299ef", size = 17361, upload-time = "2023-03-26T21:39:31.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/93/8d1f7ee9fa858d1c13511de6e0bedf7f57d15e9c7aab0e9fdf4d66074417/mock_alchemy-0.2.6-py3-none-any.whl", hash = "sha256:d5e17f2c92d0299d70cd9a0d3c8f96f759a81c285c76b03f4192451583bf865f", size = 16410, upload-time = "2023-03-26T21:39:29.486Z" }, -] - [[package]] name = "mpmath" version = "1.3.0" @@ -5246,7 +5234,6 @@ dev = [ { name = "jupyter-book" }, { name = "jupytext" }, { name = "matplotlib" }, - { name = "mock-alchemy" }, { name = "pandas" }, { name = "pre-commit" }, { name = "pytest" }, @@ -5258,17 +5245,8 @@ dev = [ { name = "ruff" }, { name = "ty" }, { name = "types-aiofiles" }, - { name = "types-cachetools" }, - { name = "types-decorator" }, - { name = "types-paramiko" }, - { name = "types-pycurl" }, - { name = "types-pytz" }, { name = "types-pyyaml" }, { name = "types-requests" }, - { name = "types-simplejson" }, - { name = "types-six" }, - { name = "types-tabulate" }, - { name = "types-ujson" }, ] [package.metadata] @@ -5355,7 +5333,6 @@ dev = [ { name = "jupyter-book", specifier = ">=2.0.0" }, { name = "jupytext", specifier = ">=1.17.1" }, { name = "matplotlib", specifier = ">=3.10.0" }, - { name = "mock-alchemy", specifier = ">=0.2.6" }, { name = "pandas", specifier = ">=2.2.0" }, { name = "pre-commit", specifier = ">=4.2.0" }, { name = "pytest", specifier = ">=9.0.3" }, @@ -5367,17 +5344,8 @@ dev = [ { name = "ruff", specifier = ">=0.14.4" }, { name = "ty", specifier = ">=0.0.32" }, { name = "types-aiofiles", specifier = ">=24.1.0" }, - { name = "types-cachetools", specifier = ">=5.5.0" }, - { name = "types-decorator", specifier = ">=5.1.0" }, - { name = "types-paramiko", specifier = ">=3.5.0" }, - { name = "types-pycurl", specifier = ">=7.45.0" }, - { name = "types-pytz", specifier = ">=2024.2.0" }, { name = "types-pyyaml", specifier = ">=6.0.12.20250516" }, { name = "types-requests", specifier = ">=2.31.0.20250515" }, - { name = "types-simplejson", specifier = ">=3.19.0" }, - { name = "types-six", specifier = ">=1.16.0" }, - { name = "types-tabulate", specifier = ">=0.9.0" }, - { name = "types-ujson", specifier = ">=5.10.0" }, ] [[package]] @@ -7001,54 +6969,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/3d/7a9ed9faafeae3aa3b5bc22fa5b979ff9cf3c83ecbe919b58eae07795b8c/types_aiofiles-25.1.0.20260518-py3-none-any.whl", hash = "sha256:f776bdfb4bec17f743d9ef042e61edf03bdcc7821fc08556fba9b63d873fdea9", size = 14377, upload-time = "2026-05-18T06:05:26.871Z" }, ] -[[package]] -name = "types-cachetools" -version = "6.2.0.20251022" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/a8/f9bcc7f1be63af43ef0170a773e2d88817bcc7c9d8769f2228c802826efe/types_cachetools-6.2.0.20251022.tar.gz", hash = "sha256:f1d3c736f0f741e89ec10f0e1b0138625023e21eb33603a930c149e0318c0cef", size = 9608, upload-time = "2025-10-22T03:03:58.16Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/98/2d/8d821ed80f6c2c5b427f650bf4dc25b80676ed63d03388e4b637d2557107/types_cachetools-6.2.0.20251022-py3-none-any.whl", hash = "sha256:698eb17b8f16b661b90624708b6915f33dbac2d185db499ed57e4997e7962cad", size = 9341, upload-time = "2025-10-22T03:03:57.036Z" }, -] - -[[package]] -name = "types-decorator" -version = "5.2.0.20251101" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a6/e4/929a77f6580928a5b4914a62834a0570d2449428ecdbb0a2e916150ed978/types_decorator-5.2.0.20251101.tar.gz", hash = "sha256:120e2bf4792ec8a47653db1cb380c7aacb6862a797c1490a910aacc21548286c", size = 9059, upload-time = "2025-11-01T03:04:02.355Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/cc/aa53df63915e10d429b7aa0491ba520abe4b80aef0304d1b02425cd5bd08/types_decorator-5.2.0.20251101-py3-none-any.whl", hash = "sha256:8176470ec0a2190e9d688577d4987b24039ae4a23913211707eda96bf2755b0c", size = 8074, upload-time = "2025-11-01T03:04:01.353Z" }, -] - -[[package]] -name = "types-paramiko" -version = "4.0.0.20250822" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/b8/c6ff3b10c2f7b9897650af746f0dc6c5cddf054db857bc79d621f53c7d22/types_paramiko-4.0.0.20250822.tar.gz", hash = "sha256:1b56b0cbd3eec3d2fd123c9eb2704e612b777e15a17705a804279ea6525e0c53", size = 28730, upload-time = "2025-08-22T03:03:43.262Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/a1/b3774ed924a66ee2c041224d89c36f0c21f4f6cf75036d6ee7698bf8a4b9/types_paramiko-4.0.0.20250822-py3-none-any.whl", hash = "sha256:55bdb14db75ca89039725ec64ae3fa26b8d57b6991cfb476212fa8f83a59753c", size = 38833, upload-time = "2025-08-22T03:03:42.072Z" }, -] - -[[package]] -name = "types-pycurl" -version = "7.46.0.20260509" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fb/3e/9b7e3779b7baf42d650ccc9943f3fd620207446bbdd5103c6a2be3bf2fbb/types_pycurl-7.46.0.20260509.tar.gz", hash = "sha256:719d328744d0a0f1765c7a2eb3e3b081edc8ddec4ca668b61e82625a4519b359", size = 16277, upload-time = "2026-05-09T04:58:55.002Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/24/b9/9635032e42e92d466edbed314d1ddbaf0e0dedc81bed4083e09f44a71799/types_pycurl-7.46.0.20260509-py3-none-any.whl", hash = "sha256:9399a9e0c2682e8740a8027bbc76e4510380bc71165d27c9f6107e5042926334", size = 14351, upload-time = "2026-05-09T04:58:53.891Z" }, -] - -[[package]] -name = "types-pytz" -version = "2025.2.0.20251108" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/40/ff/c047ddc68c803b46470a357454ef76f4acd8c1088f5cc4891cdd909bfcf6/types_pytz-2025.2.0.20251108.tar.gz", hash = "sha256:fca87917836ae843f07129567b74c1929f1870610681b4c92cb86a3df5817bdb", size = 10961, upload-time = "2025-11-08T02:55:57.001Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/c1/56ef16bf5dcd255155cc736d276efa6ae0a5c26fd685e28f0412a4013c01/types_pytz-2025.2.0.20251108-py3-none-any.whl", hash = "sha256:0f1c9792cab4eb0e46c52f8845c8f77cf1e313cb3d68bf826aa867fe4717d91c", size = 10116, upload-time = "2025-11-08T02:55:56.194Z" }, -] - [[package]] name = "types-pyyaml" version = "6.0.12.20250915" @@ -7070,42 +6990,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/bc/b139710a3b6018f7fb2b9508b35c8af564e61bf2bf4fa619d088f3e16f85/types_requests-2.33.0.20260518-py3-none-any.whl", hash = "sha256:626d697d1adaaff76e2044dc8c5c051d8f21abc157bdfe204a75558076fe0bf0", size = 21391, upload-time = "2026-05-18T06:07:37.044Z" }, ] -[[package]] -name = "types-simplejson" -version = "3.20.0.20260518" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a5/e4/f8ecdc3529688ad058b0496f67a1bdfbd2576c3669e1c478b6347b12cb7e/types_simplejson-3.20.0.20260518.tar.gz", hash = "sha256:298c5b2fc2df3eb128d2077abd5bd1f5b0419dcf72fdf0551b2af2f03b1ff2b3", size = 10777, upload-time = "2026-05-18T06:04:39.57Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/41/16937326596bec6283a51cc901c701a964019569b861ac5f88128bc98b43/types_simplejson-3.20.0.20260518-py3-none-any.whl", hash = "sha256:f4792dbc1ab049fcaee5d99ca950bd6468ea462fe79af25fe8ca08b6bee313ac", size = 10421, upload-time = "2026-05-18T06:04:38.706Z" }, -] - -[[package]] -name = "types-six" -version = "1.17.0.20260518" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/14/ec/31f3a70f3b4f7dbbf28025a5b2ed0b3a3eb237c01bc3357e45547c04defd/types_six-1.17.0.20260518.tar.gz", hash = "sha256:b0196d5188bd589bc5ab92901edc1a4ff3c5fd4b0dc19d074a5f1f8213e5213a", size = 15787, upload-time = "2026-05-18T06:04:07.387Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/2e/0e64ff848fa3d3ee1563ee3e6b3f9f226c25e5ce17020fb0be1f832a7e32/types_six-1.17.0.20260518-py3-none-any.whl", hash = "sha256:2ed782cafcef9614e51292ae39bae93e42ff7b20c1c0a00c6be35e1c0e493a70", size = 19940, upload-time = "2026-05-18T06:04:06.565Z" }, -] - -[[package]] -name = "types-tabulate" -version = "0.10.0.20260508" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/11/af/c100d86897d3fdb010d8f813569fa80355b5cb553f51080e37d4f3c451bb/types_tabulate-0.10.0.20260508.tar.gz", hash = "sha256:8e51f159e8b24976849706ae2ed1dc9adba8ebbd080b17e494ebb66a8cc92c74", size = 8395, upload-time = "2026-05-08T04:47:58.921Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/36/443b700f54a585cf8ed4e76d8ad08b12df6914f8782a2788ecd0d8491519/types_tabulate-0.10.0.20260508-py3-none-any.whl", hash = "sha256:b1e1a2d0456fbd655a71690b09a7aaeffdf2978d32049184ea436492aa51d20a", size = 8137, upload-time = "2026-05-08T04:47:57.872Z" }, -] - -[[package]] -name = "types-ujson" -version = "5.10.0.20250822" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/bd/d372d44534f84864a96c19a7059d9b4d29db8541828b8b9dc3040f7a46d0/types_ujson-5.10.0.20250822.tar.gz", hash = "sha256:0a795558e1f78532373cf3f03f35b1f08bc60d52d924187b97995ee3597ba006", size = 8437, upload-time = "2025-08-22T03:02:19.433Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/f2/d812543c350674d8b3f6e17c8922248ee3bb752c2a76f64beb8c538b40cf/types_ujson-5.10.0.20250822-py3-none-any.whl", hash = "sha256:3e9e73a6dc62ccc03449d9ac2c580cd1b7a8e4873220db498f7dd056754be080", size = 7657, upload-time = "2025-08-22T03:02:18.699Z" }, -] - [[package]] name = "typing-extensions" version = "4.15.0" From f0aeaef29e7518931c681518ba49c1a89850f802 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Tue, 2 Jun 2026 19:55:44 -0700 Subject: [PATCH 02/13] MAINT: Flip ty unresolved-import to error and fix fallout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Promote `[tool.ty.rules].unresolved-import` from `ignore` to `error` so that newly-added imports of packages not declared in pyproject.toml fail type-checking (previously they silently fell back to `Unknown`). Verified with `ty check pyrit tests`: 1253 diagnostics, identical non-unresolved-import baseline to before this change. Total unresolved- import errors: 0. Mitigations applied to surface real issues without churn: 1. Optional-extras files (azure-speech, cv2, playwright, torch, spacy) are imported inside `try/except ImportError` blocks for graceful degradation. Added a per-file `[[tool.ty.overrides]]` block in pyproject.toml scoping `unresolved-import = "ignore"` to those 19 files (10 source + 9 unit-test mirrors). 2. `typing.Self` was imported from `typing` in three registry files (`pyrit/registry/base.py`, `…/class_registries/base_class_registry.py`, `…/object_registries/base_instance_registry.py`). `Self` arrived in `typing` only on Python 3.11; PyRIT's min is 3.10. Real latent compat bug, only mitigated today by `from __future__ import annotations` + `TYPE_CHECKING`. Switched to `typing_extensions.Self` (already a transitive dependency via pydantic et al — no new dep). 3. `ExceptionGroup` is imported via a conditional `try/except ImportError` fallback (builtin on 3.11+, `exceptiongroup` package on 3.10). Added `# type: ignore[…,ty:unresolved-import]` to both branches in `pyrit/scenario/core/scenario.py` and `tests/unit/scenario/test_scenario.py`. 4. Four scenario tests (`test_foundry.py`, `test_encoding.py`, `test_leakage_scenario.py`, `test_strategy_validation.py`) and one partner-integration test (`test_foundry_scenario_contract.py`) were importing scenario subpackages via the runtime `sys.modules` aliasing trick set up in `pyrit/scenario/__init__.py`. ty can't follow that. Updated test imports to the real paths (`pyrit.scenario.scenarios.{foundry,garak,airt}`). The user-facing public API in `pyrit/scenario/__init__.py` and in user-facing docs is unchanged. 5. The `azure.ai.evaluation.red_team*` package isn't declared in pyproject.toml — it's an optional partner integration. Added `# type: ignore[ty:unresolved-import]` to the 4 deferred imports in `tests/partner_integration/azure_ai_evaluation/test_import_smoke.py`. 6. **Real bug fix:** `tests/partner_integration/azure_ai_evaluation/ test_scorer_contract.py` imported `ScorerIdentifier` from `pyrit.identifiers`, but the symbol was renamed to `ScorerEvaluationIdentifier`. Fixed the import. 7. The above changes (especially `Self` resolving correctly) exposed four latent type-system issues in pre-existing code. Suppressed with `# type: ignore[ty:]` matching the dominant convention in this codebase (169 existing usages): - `base_class_registry.py:153` and `base_instance_registry.py:92`: `return cls._instances[cls]` returns `BaseRegistry[Any]` from a class-level dict but the signature is `-> Self`. Type-system limitation; refactoring is out of scope. - `test_attack_technique_registry.py:262`: intentionally assigns to a read-only property to assert it raises `AttributeError`. - `test_targets_initializer.py:181`: accesses `target._api_key` where `target` is typed as `PromptTarget` (base class) but the test knows it's actually an `OpenAIChatTarget`. 8. Converted the `ty-check` pre-commit hook from the third-party `allganize/ty-pre-commit` repo to a local hook that runs `uv run --link-mode=copy ty check` against the project venv. The third-party hook installs `ty` in an isolated env without the project's runtime deps, so it surfaced spurious `unresolved-import` errors for declared deps like `tqdm` once the rule was flipped to `error`. This was harmless when the rule was `ignore` but is now a real blocker. Tests: 8382 passed, 120 skipped (matches pre-change baseline). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .pre-commit-config.yaml | 9 ++++-- pyproject.toml | 31 +++++++++++++++++-- pyrit/registry/base.py | 3 +- .../class_registries/base_class_registry.py | 5 +-- .../base_instance_registry.py | 5 +-- pyrit/scenario/core/scenario.py | 4 +-- .../test_foundry_scenario_contract.py | 2 +- .../azure_ai_evaluation/test_import_smoke.py | 12 ++++--- .../test_scorer_contract.py | 6 ++-- .../test_attack_technique_registry.py | 2 +- tests/unit/scenario/test_encoding.py | 2 +- tests/unit/scenario/test_foundry.py | 2 +- tests/unit/scenario/test_leakage_scenario.py | 2 +- tests/unit/scenario/test_scenario.py | 4 +-- .../unit/scenario/test_strategy_validation.py | 6 ++-- tests/unit/setup/test_targets_initializer.py | 2 +- 16 files changed, 68 insertions(+), 29 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1c54f6a27..906f6c21dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,11 +69,16 @@ repos: name: Ruff (Jupyter Notebooks) args: [--fix] - - repo: https://github.com/allganize/ty-pre-commit - rev: v0.0.32 + - repo: local hooks: - id: ty-check name: ty (type check) + # Run ty in the project's uv-managed venv so it can resolve project + # dependencies (tqdm etc). The third-party ty-pre-commit hook installs + # ty in an isolated env without project deps, which causes spurious + # ``unresolved-import`` errors now that the rule is set to ``error``. + entry: uv run --link-mode=copy ty check + language: system files: ^pyrit/ exclude: ^pyrit/auxiliary_attacks/ types: [python] diff --git a/pyproject.toml b/pyproject.toml index ff5fbb0453..2e646e4394 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,8 +162,6 @@ asyncio_mode = "auto" [tool.ty] [tool.ty.rules] all = "error" -# Suppress errors for missing third-party stubs -unresolved-import = "ignore" # Tolerate None attribute access possibly-missing-attribute = "ignore" # Allow existing type: ignore comments without matching errors @@ -187,6 +185,35 @@ include = ["pyrit/prompt_target/hugging_face/**"] [tool.ty.overrides.rules] invalid-argument-type = "warn" +# Files that conditionally import optional-extra packages (azure-speech, cv2, +# playwright, torch, spacy) inside try/except ImportError blocks. The packages +# aren't installed in the base dev .venv, so ty can't resolve them — but the +# import shape is intentional (graceful degradation when the extra isn't installed). +[[tool.ty.overrides]] +include = [ + "pyrit/auth/azure_auth.py", + "pyrit/auth/copilot_authenticator.py", + "pyrit/prompt_converter/azure_speech_audio_to_text_converter.py", + "pyrit/prompt_converter/azure_speech_text_to_audio_converter.py", + "pyrit/prompt_converter/add_image_to_video_converter.py", + "pyrit/prompt_target/hugging_face/hugging_face_chat_target.py", + "pyrit/prompt_target/playwright_target.py", + "pyrit/prompt_target/playwright_copilot_target.py", + "pyrit/score/video_scorer.py", + "pyrit/executor/benchmark/fairness_bias.py", + "tests/unit/auth/test_azure_auth.py", + "tests/unit/prompt_converter/test_prompt_converter.py", + "tests/unit/prompt_converter/test_azure_speech_converter.py", + "tests/unit/prompt_converter/test_azure_speech_text_converter.py", + "tests/unit/prompt_converter/test_add_image_video_converter.py", + "tests/unit/score/test_video_scorer.py", + "tests/unit/prompt_target/target/test_huggingface_chat_target.py", + "tests/unit/executor/benchmark/test_fairness_bias.py", + "tests/unit/auxiliary_attacks/gcg/test_generator.py", +] +[tool.ty.overrides.rules] +unresolved-import = "ignore" + [tool.uv] constraint-dependencies = [ "aiohttp>=3.13.4", diff --git a/pyrit/registry/base.py b/pyrit/registry/base.py index cdff9067f1..98b7025d91 100644 --- a/pyrit/registry/base.py +++ b/pyrit/registry/base.py @@ -15,7 +15,8 @@ if TYPE_CHECKING: from collections.abc import Iterator - from typing import Self + + from typing_extensions import Self # Type variable for metadata (invariant for Protocol compatibility) MetadataT = TypeVar("MetadataT") diff --git a/pyrit/registry/class_registries/base_class_registry.py b/pyrit/registry/class_registries/base_class_registry.py index 7d251a9cba..30add3ae9e 100644 --- a/pyrit/registry/class_registries/base_class_registry.py +++ b/pyrit/registry/class_registries/base_class_registry.py @@ -23,7 +23,8 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterator - from typing import Self + + from typing_extensions import Self from pyrit.identifiers.class_name_utils import class_name_to_snake_case from pyrit.registry.base import ClassRegistryEntry, RegistryProtocol @@ -149,7 +150,7 @@ def get_registry_singleton(cls) -> Self: """ if cls not in cls._instances: cls._instances[cls] = cls() # type: ignore[ty:invalid-assignment] - return cls._instances[cls] + return cls._instances[cls] # type: ignore[ty:invalid-return-type] @classmethod def reset_instance(cls) -> None: diff --git a/pyrit/registry/object_registries/base_instance_registry.py b/pyrit/registry/object_registries/base_instance_registry.py index 87cab41f04..9f106a3690 100644 --- a/pyrit/registry/object_registries/base_instance_registry.py +++ b/pyrit/registry/object_registries/base_instance_registry.py @@ -27,7 +27,8 @@ if TYPE_CHECKING: from collections.abc import Iterator - from typing import Self + + from typing_extensions import Self T = TypeVar("T", bound=Identifiable) # The type of items stored @@ -88,7 +89,7 @@ def get_registry_singleton(cls) -> Self: """ if cls not in cls._instances: cls._instances[cls] = cls() - return cls._instances[cls] + return cls._instances[cls] # type: ignore[ty:invalid-return-type] @classmethod def reset_instance(cls) -> None: diff --git a/pyrit/scenario/core/scenario.py b/pyrit/scenario/core/scenario.py index 47182dd87a..d0908e4b1e 100644 --- a/pyrit/scenario/core/scenario.py +++ b/pyrit/scenario/core/scenario.py @@ -23,9 +23,9 @@ try: # Built-in on Python 3.11+. Fall back to the ``exceptiongroup`` backport on 3.10 # (declared as a conditional dependency in pyproject.toml). - from builtins import ExceptionGroup # type: ignore[attr-defined] + from builtins import ExceptionGroup # type: ignore[attr-defined,ty:unresolved-import] except ImportError: # pragma: no cover - exercised only on 3.10 - from exceptiongroup import ExceptionGroup # type: ignore[no-redef] + from exceptiongroup import ExceptionGroup # type: ignore[no-redef,ty:unresolved-import] from tqdm.auto import tqdm diff --git a/tests/partner_integration/azure_ai_evaluation/test_foundry_scenario_contract.py b/tests/partner_integration/azure_ai_evaluation/test_foundry_scenario_contract.py index a35edf6f6f..919daba9ec 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_foundry_scenario_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_foundry_scenario_contract.py @@ -13,7 +13,7 @@ from pyrit.executor.attack import AttackScoringConfig from pyrit.scenario import ScenarioStrategy -from pyrit.scenario.foundry import FoundryStrategy, RedTeamAgent +from pyrit.scenario.scenarios.foundry import FoundryStrategy, RedTeamAgent class TestRedTeamStrategyContract: diff --git a/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py b/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py index 008136ac99..ff65902524 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py +++ b/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py @@ -18,7 +18,7 @@ def _azure_ai_evaluation_available() -> bool: """Check if azure-ai-evaluation[redteam] is installed.""" try: - from azure.ai.evaluation.red_team import RedTeam # noqa: F401 + from azure.ai.evaluation.red_team import RedTeam # type: ignore[ty:unresolved-import] # noqa: F401 return True except ImportError: @@ -37,7 +37,7 @@ class TestRedTeamModuleImports: def test_redteam_public_api_imports(self): """Verify all public classes from azure.ai.evaluation.red_team are importable.""" - from azure.ai.evaluation.red_team import ( + from azure.ai.evaluation.red_team import ( # type: ignore[ty:unresolved-import] AttackStrategy, RedTeam, RedTeamResult, @@ -70,7 +70,9 @@ class TestCallbackChatTargetInheritance: def test_callback_chat_target_extends_prompt_target(self): """_CallbackChatTarget must be a subclass of pyrit.prompt_target.PromptTarget.""" - from azure.ai.evaluation.red_team._callback_chat_target import _CallbackChatTarget + from azure.ai.evaluation.red_team._callback_chat_target import ( # type: ignore[ty:unresolved-import] + _CallbackChatTarget, + ) assert issubclass(_CallbackChatTarget, PromptTarget) @@ -85,6 +87,8 @@ class TestRAIScorerInheritance: def test_rai_scorer_extends_true_false_scorer(self): """RAIServiceScorer must be a subclass of pyrit.score.true_false.TrueFalseScorer.""" - from azure.ai.evaluation.red_team._foundry._rai_scorer import RAIServiceScorer # private: intentional + from azure.ai.evaluation.red_team._foundry._rai_scorer import ( # type: ignore[ty:unresolved-import] + RAIServiceScorer, # private: intentional + ) assert issubclass(RAIServiceScorer, TrueFalseScorer) diff --git a/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py b/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py index d48c895b14..524e93a90e 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py @@ -47,10 +47,10 @@ class TestScorerUtilities: """Validate scorer utility classes used by azure-ai-evaluation.""" def test_scorer_identifier_importable(self): - """RAIServiceScorer uses ScorerIdentifier for identity tracking.""" - from pyrit.identifiers import ScorerIdentifier + """RAIServiceScorer uses ScorerEvaluationIdentifier for identity tracking.""" + from pyrit.identifiers import ScorerEvaluationIdentifier - assert ScorerIdentifier is not None + assert ScorerEvaluationIdentifier is not None def test_scorer_prompt_validator_instantiable(self): """ScorerPromptValidator should accept supported_data_types kwarg.""" diff --git a/tests/unit/registry/test_attack_technique_registry.py b/tests/unit/registry/test_attack_technique_registry.py index e0e3eebe84..790292b079 100644 --- a/tests/unit/registry/test_attack_technique_registry.py +++ b/tests/unit/registry/test_attack_technique_registry.py @@ -259,7 +259,7 @@ def test_default_policy_is_warn(self): def test_policy_is_read_only(self): """Policy property has no setter — it's read-only.""" with pytest.raises(AttributeError): - self.registry.scorer_override_policy = ScorerOverridePolicy.RAISE + self.registry.scorer_override_policy = ScorerOverridePolicy.RAISE # type: ignore[ty:invalid-assignment] def test_policy_passed_to_factories_via_register_from_factories(self): """Factories registered via register_from_factories inherit the registry's default policy.""" diff --git a/tests/unit/scenario/test_encoding.py b/tests/unit/scenario/test_encoding.py index 9ba64b5571..a0da8c9399 100644 --- a/tests/unit/scenario/test_encoding.py +++ b/tests/unit/scenario/test_encoding.py @@ -13,7 +13,7 @@ from pyrit.prompt_converter import Base64Converter from pyrit.prompt_target import PromptTarget from pyrit.scenario import DatasetConfiguration -from pyrit.scenario.garak import Encoding, EncodingStrategy +from pyrit.scenario.scenarios.garak import Encoding, EncodingStrategy from pyrit.scenario.scenarios.garak.encoding import EncodingDatasetConfiguration from pyrit.score import DecodingScorer, TrueFalseScorer diff --git a/tests/unit/scenario/test_foundry.py b/tests/unit/scenario/test_foundry.py index 3a00a50ef2..9ef01891e9 100644 --- a/tests/unit/scenario/test_foundry.py +++ b/tests/unit/scenario/test_foundry.py @@ -15,7 +15,7 @@ from pyrit.prompt_converter import Base64Converter from pyrit.prompt_target import PromptTarget from pyrit.scenario import AtomicAttack, DatasetConfiguration, ScenarioCompositeStrategy -from pyrit.scenario.foundry import FoundryComposite, FoundryStrategy, RedTeamAgent +from pyrit.scenario.scenarios.foundry import FoundryComposite, FoundryStrategy, RedTeamAgent from pyrit.score import FloatScaleThresholdScorer, TrueFalseScorer diff --git a/tests/unit/scenario/test_leakage_scenario.py b/tests/unit/scenario/test_leakage_scenario.py index e821e4e77f..18fd596bb0 100644 --- a/tests/unit/scenario/test_leakage_scenario.py +++ b/tests/unit/scenario/test_leakage_scenario.py @@ -15,8 +15,8 @@ from pyrit.registry import TargetRegistry from pyrit.registry.object_registries.attack_technique_registry import AttackTechniqueRegistry from pyrit.scenario import DatasetConfiguration -from pyrit.scenario.airt import Leakage from pyrit.scenario.core import BaselineAttackPolicy +from pyrit.scenario.scenarios.airt import Leakage from pyrit.scenario.scenarios.airt.leakage import _build_leakage_strategy from pyrit.score import TrueFalseCompositeScorer from pyrit.setup.initializers.components.scenario_techniques import build_scenario_technique_factories diff --git a/tests/unit/scenario/test_scenario.py b/tests/unit/scenario/test_scenario.py index 55365f0493..02c981459c 100644 --- a/tests/unit/scenario/test_scenario.py +++ b/tests/unit/scenario/test_scenario.py @@ -10,9 +10,9 @@ import pytest try: - from builtins import ExceptionGroup # type: ignore[attr-defined] + from builtins import ExceptionGroup # type: ignore[attr-defined,ty:unresolved-import] except ImportError: # pragma: no cover - 3.10 only - from exceptiongroup import ExceptionGroup # type: ignore[no-redef] + from exceptiongroup import ExceptionGroup # type: ignore[no-redef,ty:unresolved-import] from pyrit.executor.attack.core import AttackExecutorResult from pyrit.identifiers import ComponentIdentifier diff --git a/tests/unit/scenario/test_strategy_validation.py b/tests/unit/scenario/test_strategy_validation.py index f4f0b03f21..f33855f52d 100644 --- a/tests/unit/scenario/test_strategy_validation.py +++ b/tests/unit/scenario/test_strategy_validation.py @@ -8,9 +8,9 @@ import pytest from pyrit.scenario import ScenarioCompositeStrategy -from pyrit.scenario.foundry import FoundryStrategy -from pyrit.scenario.foundry.red_team_agent import FoundryComposite -from pyrit.scenario.garak import EncodingStrategy +from pyrit.scenario.scenarios.foundry import FoundryStrategy +from pyrit.scenario.scenarios.foundry.red_team_agent import FoundryComposite +from pyrit.scenario.scenarios.garak import EncodingStrategy class TestFoundryComposite: diff --git a/tests/unit/setup/test_targets_initializer.py b/tests/unit/setup/test_targets_initializer.py index e35decc77c..8f04973c5c 100644 --- a/tests/unit/setup/test_targets_initializer.py +++ b/tests/unit/setup/test_targets_initializer.py @@ -178,7 +178,7 @@ def mock_token_provider() -> str: target = registry.get_instance_by_name("azure_openai_gpt4o") assert target is not None # The token provider gets wrapped by _ensure_async_token_provider, so just verify it's callable - assert callable(target._api_key) + assert callable(target._api_key) # type: ignore[ty:unresolved-attribute] @pytest.mark.usefixtures("patch_central_database") From 3c141df266fb5f8c694cc7c75fa3d59170cd5b33 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Wed, 3 Jun 2026 07:06:33 -0700 Subject: [PATCH 03/13] MAINT: Use public scenario import path in contract test Contract tests should mirror real consumer usage (matches doc/code/setup/2_resiliency.py). Use the documented short path and silence ty's unresolved-import (sys.modules alias) inline. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure_ai_evaluation/test_foundry_scenario_contract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/partner_integration/azure_ai_evaluation/test_foundry_scenario_contract.py b/tests/partner_integration/azure_ai_evaluation/test_foundry_scenario_contract.py index 919daba9ec..b9a4154b0f 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_foundry_scenario_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_foundry_scenario_contract.py @@ -13,7 +13,7 @@ from pyrit.executor.attack import AttackScoringConfig from pyrit.scenario import ScenarioStrategy -from pyrit.scenario.scenarios.foundry import FoundryStrategy, RedTeamAgent +from pyrit.scenario.foundry import FoundryStrategy, RedTeamAgent # type: ignore[ty:unresolved-import] class TestRedTeamStrategyContract: From 11441919f06a253e3e25bd4322a9155696f81dcf Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Wed, 3 Jun 2026 07:17:25 -0700 Subject: [PATCH 04/13] MAINT: Restore short scenario import paths in unit tests Use the documented public import paths (pyrit.scenario.{airt,foundry,garak}) with inline ty ignores, matching the pattern now used in test_foundry_scenario_contract.py and the doc snippets. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/unit/scenario/test_encoding.py | 2 +- tests/unit/scenario/test_foundry.py | 2 +- tests/unit/scenario/test_leakage_scenario.py | 2 +- tests/unit/scenario/test_strategy_validation.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unit/scenario/test_encoding.py b/tests/unit/scenario/test_encoding.py index a0da8c9399..5c6a453200 100644 --- a/tests/unit/scenario/test_encoding.py +++ b/tests/unit/scenario/test_encoding.py @@ -13,7 +13,7 @@ from pyrit.prompt_converter import Base64Converter from pyrit.prompt_target import PromptTarget from pyrit.scenario import DatasetConfiguration -from pyrit.scenario.scenarios.garak import Encoding, EncodingStrategy +from pyrit.scenario.garak import Encoding, EncodingStrategy # type: ignore[ty:unresolved-import] from pyrit.scenario.scenarios.garak.encoding import EncodingDatasetConfiguration from pyrit.score import DecodingScorer, TrueFalseScorer diff --git a/tests/unit/scenario/test_foundry.py b/tests/unit/scenario/test_foundry.py index 9ef01891e9..2ac6790030 100644 --- a/tests/unit/scenario/test_foundry.py +++ b/tests/unit/scenario/test_foundry.py @@ -15,7 +15,7 @@ from pyrit.prompt_converter import Base64Converter from pyrit.prompt_target import PromptTarget from pyrit.scenario import AtomicAttack, DatasetConfiguration, ScenarioCompositeStrategy -from pyrit.scenario.scenarios.foundry import FoundryComposite, FoundryStrategy, RedTeamAgent +from pyrit.scenario.foundry import FoundryComposite, FoundryStrategy, RedTeamAgent # type: ignore[ty:unresolved-import] from pyrit.score import FloatScaleThresholdScorer, TrueFalseScorer diff --git a/tests/unit/scenario/test_leakage_scenario.py b/tests/unit/scenario/test_leakage_scenario.py index 18fd596bb0..8cbe23d040 100644 --- a/tests/unit/scenario/test_leakage_scenario.py +++ b/tests/unit/scenario/test_leakage_scenario.py @@ -15,8 +15,8 @@ from pyrit.registry import TargetRegistry from pyrit.registry.object_registries.attack_technique_registry import AttackTechniqueRegistry from pyrit.scenario import DatasetConfiguration +from pyrit.scenario.airt import Leakage # type: ignore[ty:unresolved-import] from pyrit.scenario.core import BaselineAttackPolicy -from pyrit.scenario.scenarios.airt import Leakage from pyrit.scenario.scenarios.airt.leakage import _build_leakage_strategy from pyrit.score import TrueFalseCompositeScorer from pyrit.setup.initializers.components.scenario_techniques import build_scenario_technique_factories diff --git a/tests/unit/scenario/test_strategy_validation.py b/tests/unit/scenario/test_strategy_validation.py index f33855f52d..6a93b019ad 100644 --- a/tests/unit/scenario/test_strategy_validation.py +++ b/tests/unit/scenario/test_strategy_validation.py @@ -8,9 +8,9 @@ import pytest from pyrit.scenario import ScenarioCompositeStrategy -from pyrit.scenario.scenarios.foundry import FoundryStrategy -from pyrit.scenario.scenarios.foundry.red_team_agent import FoundryComposite -from pyrit.scenario.scenarios.garak import EncodingStrategy +from pyrit.scenario.foundry import FoundryStrategy # type: ignore[ty:unresolved-import] +from pyrit.scenario.foundry.red_team_agent import FoundryComposite # type: ignore[ty:unresolved-import] +from pyrit.scenario.garak import EncodingStrategy # type: ignore[ty:unresolved-import] class TestFoundryComposite: From d13b14aea56b757fd1ab8418eafea6aa1e5a1401 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Wed, 3 Jun 2026 07:29:27 -0700 Subject: [PATCH 05/13] MAINT: Use public scenario short paths for top-level imports Where a symbol is re-exported from the scenario subpackage __init__ (foundry, garak, airt, benchmark), use the documented public short path (e.g. `pyrit.scenario.foundry.RedTeamAgent`). Note: deep submodule paths like `pyrit.scenario.garak.encoding.X` are NOT safe to shorten. The sys.modules alias only covers the top-level subpackage; importing a submodule via the alias re-executes the module under a different name and produces a duplicate class object, breaking isinstance checks. Those keep the on-disk path `pyrit.scenario.scenarios.garak.encoding.X`. Inline `# type: ignore[ty:unresolved-import]` per existing codebase convention because the alias is set up at runtime in `pyrit/scenario/__init__.py` and ty cannot follow that statically. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/code/scenarios/1_common_scenario_parameters.py | 4 ++-- doc/code/scenarios/2_custom_scenario_parameters.py | 2 +- doc/scanner/airt.py | 12 ++++++------ doc/scanner/benchmark.py | 2 +- doc/scanner/foundry.py | 6 +++--- doc/scanner/garak.py | 2 +- tests/unit/scenario/test_psychosocial_harms.py | 2 +- tests/unit/scenario/test_strategy_validation.py | 3 +-- 8 files changed, 16 insertions(+), 17 deletions(-) diff --git a/doc/code/scenarios/1_common_scenario_parameters.py b/doc/code/scenarios/1_common_scenario_parameters.py index 230e02fff3..66a2871a11 100644 --- a/doc/code/scenarios/1_common_scenario_parameters.py +++ b/doc/code/scenarios/1_common_scenario_parameters.py @@ -30,7 +30,7 @@ from pyrit.output import output_scenario_async from pyrit.registry import TargetRegistry -from pyrit.scenario.scenarios.foundry import FoundryStrategy, RedTeamAgent +from pyrit.scenario.foundry import FoundryStrategy, RedTeamAgent from pyrit.setup import initialize_from_config_async await initialize_from_config_async(config_path=Path("../../scanner/pyrit_conf.yaml")) # type: ignore @@ -85,7 +85,7 @@ # For example, to run Crescendo with Base64 encoding applied: # %% -from pyrit.scenario.scenarios.foundry import FoundryComposite +from pyrit.scenario.foundry import FoundryComposite composite_strategy = [FoundryComposite(attack=FoundryStrategy.Crescendo, converters=[FoundryStrategy.Base64])] diff --git a/doc/code/scenarios/2_custom_scenario_parameters.py b/doc/code/scenarios/2_custom_scenario_parameters.py index bcebdb1f9e..6df7e966e6 100644 --- a/doc/code/scenarios/2_custom_scenario_parameters.py +++ b/doc/code/scenarios/2_custom_scenario_parameters.py @@ -47,7 +47,7 @@ # would wire up memory and scorers): # %% -from pyrit.scenario.scenarios.airt.scam import Scam +from pyrit.scenario.airt.scam import Scam for param in Scam.supported_parameters(): print(param) diff --git a/doc/scanner/airt.py b/doc/scanner/airt.py index f31468972d..05312e7b42 100644 --- a/doc/scanner/airt.py +++ b/doc/scanner/airt.py @@ -50,7 +50,7 @@ # **Available strategies:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, role_play, many_shot, tap # %% -from pyrit.scenario.scenarios.airt import RapidResponse, RapidResponseStrategy +from pyrit.scenario.airt import RapidResponse, RapidResponseStrategy dataset_config = DatasetConfiguration(dataset_names=["airt_hate"], max_dataset_size=1) @@ -99,7 +99,7 @@ # meaningful because psychosocial harms emerge through multi-turn escalation. # %% -from pyrit.scenario.scenarios.airt import Psychosocial, PsychosocialStrategy +from pyrit.scenario.airt import Psychosocial, PsychosocialStrategy dataset_config = DatasetConfiguration(dataset_names=["airt_imminent_crisis"], max_dataset_size=1) @@ -132,7 +132,7 @@ # **Available strategies:** ALL, MULTI_TURN, red_teaming # %% -from pyrit.scenario.scenarios.airt import Cyber, CyberStrategy +from pyrit.scenario.airt import Cyber, CyberStrategy dataset_config = DatasetConfiguration(dataset_names=["airt_malware"], max_dataset_size=1) @@ -165,7 +165,7 @@ # **Available strategies:** ALL, SIMPLE, COMPLEX, PromptSending, ManyShot, SkeletonKey, RolePlay # %% -from pyrit.scenario.scenarios.airt import Jailbreak, JailbreakStrategy +from pyrit.scenario.airt import Jailbreak, JailbreakStrategy dataset_config = DatasetConfiguration(dataset_names=["airt_harms"], max_dataset_size=1) @@ -213,7 +213,7 @@ # no built-in threshold — the scorer returns a raw float for you to interpret per your use case. # %% -from pyrit.scenario.scenarios.airt import Leakage, LeakageStrategy +from pyrit.scenario.airt import Leakage, LeakageStrategy dataset_config = DatasetConfiguration(dataset_names=["airt_leakage"], max_dataset_size=1) @@ -245,7 +245,7 @@ # **Available strategies:** ALL, SINGLE_TURN, MULTI_TURN, ContextCompliance, RolePlay, PersuasiveRedTeamingAttack # %% -from pyrit.scenario.scenarios.airt import Scam, ScamStrategy +from pyrit.scenario.airt import Scam, ScamStrategy dataset_config = DatasetConfiguration(dataset_names=["airt_scams"], max_dataset_size=1) diff --git a/doc/scanner/benchmark.py b/doc/scanner/benchmark.py index 0f9cedc6db..65b3477a2f 100644 --- a/doc/scanner/benchmark.py +++ b/doc/scanner/benchmark.py @@ -20,7 +20,7 @@ # %% from pyrit.output import output_scenario_async from pyrit.prompt_target import OpenAIChatTarget -from pyrit.scenario.scenarios.benchmark import AdversarialBenchmark +from pyrit.scenario.benchmark import AdversarialBenchmark from pyrit.setup import IN_MEMORY, initialize_pyrit_async from pyrit.setup.initializers import LoadDefaultDatasets diff --git a/doc/scanner/foundry.py b/doc/scanner/foundry.py index f589614050..d4f9764c3a 100644 --- a/doc/scanner/foundry.py +++ b/doc/scanner/foundry.py @@ -25,7 +25,7 @@ from pyrit.output import output_scenario_async from pyrit.registry import TargetRegistry from pyrit.scenario import DatasetConfiguration -from pyrit.scenario.scenarios.foundry import FoundryStrategy, RedTeamAgent +from pyrit.scenario.foundry import FoundryStrategy, RedTeamAgent from pyrit.setup import initialize_from_config_async await initialize_from_config_async(config_path=Path("pyrit_conf.yaml")) # type: ignore @@ -77,13 +77,13 @@ # Each converter in the composite is applied in sequence before the attack runs. # # ```python -# from pyrit.scenario.scenarios.foundry import FoundryComposite +# from pyrit.scenario.foundry import FoundryComposite # # composed = FoundryComposite(attack=FoundryStrategy.Crescendo, converters=[FoundryStrategy.Caesar, FoundryStrategy.CharSwap]) # ``` # %% -# from pyrit.scenario.scenarios.foundry import FoundryComposite +# from pyrit.scenario.foundry import FoundryComposite # composed = FoundryComposite(attack=FoundryStrategy.Crescendo, converters=[FoundryStrategy.Caesar, FoundryStrategy.CharSwap]) # scenario_strategies = [FoundryStrategy.Base64, composed] diff --git a/doc/scanner/garak.py b/doc/scanner/garak.py index 7c3ea540a0..5e9f2d044b 100644 --- a/doc/scanner/garak.py +++ b/doc/scanner/garak.py @@ -23,7 +23,7 @@ from pyrit.output import output_scenario_async from pyrit.registry import TargetRegistry -from pyrit.scenario.scenarios.garak import Encoding, EncodingStrategy +from pyrit.scenario.garak import Encoding, EncodingStrategy from pyrit.scenario.scenarios.garak.encoding import EncodingDatasetConfiguration from pyrit.setup import initialize_from_config_async diff --git a/tests/unit/scenario/test_psychosocial_harms.py b/tests/unit/scenario/test_psychosocial_harms.py index 4361934896..f07ce21f8a 100644 --- a/tests/unit/scenario/test_psychosocial_harms.py +++ b/tests/unit/scenario/test_psychosocial_harms.py @@ -11,7 +11,7 @@ from pyrit.identifiers import ComponentIdentifier from pyrit.models import SeedAttackGroup, SeedDataset, SeedGroup, SeedObjective from pyrit.prompt_target import OpenAIChatTarget, PromptTarget -from pyrit.scenario.scenarios.airt import ( +from pyrit.scenario.airt import ( # type: ignore[ty:unresolved-import] Psychosocial, PsychosocialStrategy, ) diff --git a/tests/unit/scenario/test_strategy_validation.py b/tests/unit/scenario/test_strategy_validation.py index 6a93b019ad..ffb7b6d8b5 100644 --- a/tests/unit/scenario/test_strategy_validation.py +++ b/tests/unit/scenario/test_strategy_validation.py @@ -8,8 +8,7 @@ import pytest from pyrit.scenario import ScenarioCompositeStrategy -from pyrit.scenario.foundry import FoundryStrategy # type: ignore[ty:unresolved-import] -from pyrit.scenario.foundry.red_team_agent import FoundryComposite # type: ignore[ty:unresolved-import] +from pyrit.scenario.foundry import FoundryComposite, FoundryStrategy # type: ignore[ty:unresolved-import] from pyrit.scenario.garak import EncodingStrategy # type: ignore[ty:unresolved-import] From 7b9a2f586401794f3b8bd9fd6ce3a534353134e4 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Wed, 3 Jun 2026 12:36:22 -0700 Subject: [PATCH 06/13] DOCS: Sync notebooks to .py scenario import shortening Mirrors the import edits made in the paired .py files to keep notebooks in sync (see .github/instructions/docs.instructions.md). Surgical string replacements only on the import lines that changed; preserves cell outputs and notebook metadata. Skipped: doc/code/registry/1_class_registry.ipynb:69 (an execution output showing the real fully-qualified class repr) and doc/scanner/garak.ipynb:47 (deep submodule path that the paired .py legitimately keeps as pyrit.scenario.scenarios.garak.encoding). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../scenarios/1_common_scenario_parameters.ipynb | 4 ++-- .../scenarios/2_custom_scenario_parameters.ipynb | 2 +- doc/scanner/airt.ipynb | 12 ++++++------ doc/scanner/benchmark.ipynb | 2 +- doc/scanner/foundry.ipynb | 6 +++--- doc/scanner/garak.ipynb | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/code/scenarios/1_common_scenario_parameters.ipynb b/doc/code/scenarios/1_common_scenario_parameters.ipynb index 9da9b9195b..b86d7c1ffc 100644 --- a/doc/code/scenarios/1_common_scenario_parameters.ipynb +++ b/doc/code/scenarios/1_common_scenario_parameters.ipynb @@ -66,7 +66,7 @@ "\n", "from pyrit.output import output_scenario_async\n", "from pyrit.registry import TargetRegistry\n", - "from pyrit.scenario.scenarios.foundry import FoundryStrategy, RedTeamAgent\n", + "from pyrit.scenario.foundry import FoundryStrategy, RedTeamAgent\n", "from pyrit.setup import initialize_from_config_async\n", "\n", "await initialize_from_config_async(config_path=Path(\"../../scanner/pyrit_conf.yaml\")) # type: ignore\n", @@ -190,7 +190,7 @@ "metadata": {}, "outputs": [], "source": [ - "from pyrit.scenario.scenarios.foundry import FoundryComposite\n", + "from pyrit.scenario.foundry import FoundryComposite\n", "\n", "composite_strategy = [FoundryComposite(attack=FoundryStrategy.Crescendo, converters=[FoundryStrategy.Base64])]" ] diff --git a/doc/code/scenarios/2_custom_scenario_parameters.ipynb b/doc/code/scenarios/2_custom_scenario_parameters.ipynb index a126e4b44c..8dbeac3849 100644 --- a/doc/code/scenarios/2_custom_scenario_parameters.ipynb +++ b/doc/code/scenarios/2_custom_scenario_parameters.ipynb @@ -58,7 +58,7 @@ } ], "source": [ - "from pyrit.scenario.scenarios.airt.scam import Scam\n", + "from pyrit.scenario.airt.scam import Scam\n", "\n", "for param in Scam.supported_parameters():\n", " print(param)" diff --git a/doc/scanner/airt.ipynb b/doc/scanner/airt.ipynb index ba62a19214..e1881ef631 100644 --- a/doc/scanner/airt.ipynb +++ b/doc/scanner/airt.ipynb @@ -104,7 +104,7 @@ } ], "source": [ - "from pyrit.scenario.scenarios.airt import RapidResponse, RapidResponseStrategy\n", + "from pyrit.scenario.airt import RapidResponse, RapidResponseStrategy\n", "\n", "dataset_config = DatasetConfiguration(dataset_names=[\"airt_hate\"], max_dataset_size=1)\n", "\n", @@ -254,7 +254,7 @@ } ], "source": [ - "from pyrit.scenario.scenarios.airt import Psychosocial, PsychosocialStrategy\n", + "from pyrit.scenario.airt import Psychosocial, PsychosocialStrategy\n", "\n", "dataset_config = DatasetConfiguration(dataset_names=[\"airt_imminent_crisis\"], max_dataset_size=1)\n", "\n", @@ -398,7 +398,7 @@ } ], "source": [ - "from pyrit.scenario.scenarios.airt import Cyber, CyberStrategy\n", + "from pyrit.scenario.airt import Cyber, CyberStrategy\n", "\n", "dataset_config = DatasetConfiguration(dataset_names=[\"airt_malware\"], max_dataset_size=1)\n", "\n", @@ -520,7 +520,7 @@ "metadata": {}, "outputs": [], "source": [ - "from pyrit.scenario.scenarios.airt import Jailbreak, JailbreakStrategy\n", + "from pyrit.scenario.airt import Jailbreak, JailbreakStrategy\n", "\n", "dataset_config = DatasetConfiguration(dataset_names=[\"airt_harms\"], max_dataset_size=1)\n", "\n", @@ -1017,7 +1017,7 @@ } ], "source": [ - "from pyrit.scenario.scenarios.airt import Leakage, LeakageStrategy\n", + "from pyrit.scenario.airt import Leakage, LeakageStrategy\n", "\n", "dataset_config = DatasetConfiguration(dataset_names=[\"airt_leakage\"], max_dataset_size=1)\n", "\n", @@ -1156,7 +1156,7 @@ } ], "source": [ - "from pyrit.scenario.scenarios.airt import Scam, ScamStrategy\n", + "from pyrit.scenario.airt import Scam, ScamStrategy\n", "\n", "dataset_config = DatasetConfiguration(dataset_names=[\"airt_scams\"], max_dataset_size=1)\n", "\n", diff --git a/doc/scanner/benchmark.ipynb b/doc/scanner/benchmark.ipynb index 9da0510b4c..afbce530e4 100644 --- a/doc/scanner/benchmark.ipynb +++ b/doc/scanner/benchmark.ipynb @@ -109,7 +109,7 @@ "source": [ "from pyrit.output import output_scenario_async\n", "from pyrit.prompt_target import OpenAIChatTarget\n", - "from pyrit.scenario.scenarios.benchmark import AdversarialBenchmark\n", + "from pyrit.scenario.benchmark import AdversarialBenchmark\n", "from pyrit.setup import IN_MEMORY, initialize_pyrit_async\n", "from pyrit.setup.initializers import LoadDefaultDatasets\n", "\n", diff --git a/doc/scanner/foundry.ipynb b/doc/scanner/foundry.ipynb index ec47624f47..9035823c9e 100644 --- a/doc/scanner/foundry.ipynb +++ b/doc/scanner/foundry.ipynb @@ -45,7 +45,7 @@ "from pyrit.output import output_scenario_async\n", "from pyrit.registry import TargetRegistry\n", "from pyrit.scenario import DatasetConfiguration\n", - "from pyrit.scenario.scenarios.foundry import FoundryStrategy, RedTeamAgent\n", + "from pyrit.scenario.foundry import FoundryStrategy, RedTeamAgent\n", "from pyrit.setup import initialize_from_config_async\n", "\n", "await initialize_from_config_async(config_path=Path(\"pyrit_conf.yaml\")) # type: ignore\n", @@ -216,7 +216,7 @@ "Each converter in the composite is applied in sequence before the attack runs.\n", "\n", "```python\n", - "from pyrit.scenario.scenarios.foundry import FoundryComposite\n", + "from pyrit.scenario.foundry import FoundryComposite\n", "\n", "composed = FoundryComposite(attack=FoundryStrategy.Crescendo, converters=[FoundryStrategy.Caesar, FoundryStrategy.CharSwap])\n", "```" @@ -229,7 +229,7 @@ "metadata": {}, "outputs": [], "source": [ - "# from pyrit.scenario.scenarios.foundry import FoundryComposite\n", + "# from pyrit.scenario.foundry import FoundryComposite\n", "# composed = FoundryComposite(attack=FoundryStrategy.Crescendo, converters=[FoundryStrategy.Caesar, FoundryStrategy.CharSwap])\n", "# scenario_strategies = [FoundryStrategy.Base64, composed]" ] diff --git a/doc/scanner/garak.ipynb b/doc/scanner/garak.ipynb index 592b0f5f72..22ce389806 100644 --- a/doc/scanner/garak.ipynb +++ b/doc/scanner/garak.ipynb @@ -43,7 +43,7 @@ "\n", "from pyrit.output import output_scenario_async\n", "from pyrit.registry import TargetRegistry\n", - "from pyrit.scenario.scenarios.garak import Encoding, EncodingStrategy\n", + "from pyrit.scenario.garak import Encoding, EncodingStrategy\n", "from pyrit.scenario.scenarios.garak.encoding import EncodingDatasetConfiguration\n", "from pyrit.setup import initialize_from_config_async\n", "\n", From 9a931f6f8e10933381aa998891aa42c17955950c Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Wed, 3 Jun 2026 20:27:26 -0700 Subject: [PATCH 07/13] MAINT: Drop pyrit/ source files from ty unresolved-import override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the per-file override block with surgical inline `# type: ignore[ty:unresolved-import]` comments at the 18 optional-extra import sites (azure-cognitiveservices-speech, cv2, playwright, torch, spacy). Test files keep the broader override since they tend to monkey-patch and re-import these modules in ways that would multiply the ignore sites. Rationale: the per-file override was opt-out at the file level, which is broader than needed — it silenced unresolved-import for every line in the file, including unrelated imports. Inline ignores document the intent at the point of need, match the dominant codebase convention (169 existing `# type: ignore[ty:]` uses), and self-clean: once a developer installs the optional extra locally, the `unused-type-ignore-comment = warn` rule will flag the stale ignore for removal. Verified: same ty error count before/after (1359 baseline diagnostics, same 7 pre-existing diagnostics in the 10 target files — none introduced by this change). All 106 functional unit tests in the affected modules pass (107 require optional extras and are skipped in base venv). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyproject.toml | 21 +++++++------------ pyrit/auth/azure_auth.py | 8 +++---- pyrit/auth/copilot_authenticator.py | 4 ++-- pyrit/executor/benchmark/fairness_bias.py | 2 +- .../add_image_to_video_converter.py | 2 +- .../azure_speech_audio_to_text_converter.py | 6 +++--- .../azure_speech_text_to_audio_converter.py | 4 ++-- .../hugging_face/hugging_face_chat_target.py | 4 ++-- .../playwright_copilot_target.py | 2 +- pyrit/prompt_target/playwright_target.py | 2 +- pyrit/score/video_scorer.py | 2 +- 11 files changed, 25 insertions(+), 32 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2e646e4394..104181ac65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,22 +185,15 @@ include = ["pyrit/prompt_target/hugging_face/**"] [tool.ty.overrides.rules] invalid-argument-type = "warn" -# Files that conditionally import optional-extra packages (azure-speech, cv2, -# playwright, torch, spacy) inside try/except ImportError blocks. The packages -# aren't installed in the base dev .venv, so ty can't resolve them — but the -# import shape is intentional (graceful degradation when the extra isn't installed). +# Test files that conditionally import optional-extra packages (azure-speech, +# cv2, playwright, torch, spacy) inside try/except ImportError blocks. The +# pyrit/ source files that do the same use inline +# ``# type: ignore[ty:unresolved-import]`` comments instead, so they get full +# type checking with surgical silencing at the import sites. Test files keep +# the broader override because they tend to monkeypatch and re-import these +# modules in ways that would multiply the ignore sites. [[tool.ty.overrides]] include = [ - "pyrit/auth/azure_auth.py", - "pyrit/auth/copilot_authenticator.py", - "pyrit/prompt_converter/azure_speech_audio_to_text_converter.py", - "pyrit/prompt_converter/azure_speech_text_to_audio_converter.py", - "pyrit/prompt_converter/add_image_to_video_converter.py", - "pyrit/prompt_target/hugging_face/hugging_face_chat_target.py", - "pyrit/prompt_target/playwright_target.py", - "pyrit/prompt_target/playwright_copilot_target.py", - "pyrit/score/video_scorer.py", - "pyrit/executor/benchmark/fairness_bias.py", "tests/unit/auth/test_azure_auth.py", "tests/unit/prompt_converter/test_prompt_converter.py", "tests/unit/prompt_converter/test_azure_speech_converter.py", diff --git a/pyrit/auth/azure_auth.py b/pyrit/auth/azure_auth.py index c5076ed581..ed3f9872e2 100644 --- a/pyrit/auth/azure_auth.py +++ b/pyrit/auth/azure_auth.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from collections.abc import Awaitable, Callable - import azure.cognitiveservices.speech as speechsdk + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] from pyrit.auth.auth_config import REFRESH_TOKEN_BEFORE_MSEC from pyrit.auth.authenticator import Authenticator @@ -412,7 +412,7 @@ def get_speech_config(resource_id: Union[str, None], key: Union[str, None], regi ValueError: If neither key/region nor resource_id/region is provided. """ try: - import azure.cognitiveservices.speech as speechsdk # noqa: F811 + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] # noqa: F811 except ModuleNotFoundError as e: logger.error( "Could not import azure.cognitiveservices.speech. " @@ -463,7 +463,7 @@ async def get_speech_config_async( """ if token_provider: try: - import azure.cognitiveservices.speech as speechsdk # noqa: F811 + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] # noqa: F811 except ModuleNotFoundError as e: logger.error( "Could not import azure.cognitiveservices.speech. " @@ -495,7 +495,7 @@ def get_speech_config_from_default_azure_credential(resource_id: str, region: st ModuleNotFoundError: If azure.cognitiveservices.speech is not installed. """ try: - import azure.cognitiveservices.speech as speechsdk # noqa: F811 + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] # noqa: F811 except ModuleNotFoundError as e: logger.error( "Could not import azure.cognitiveservices.speech. " diff --git a/pyrit/auth/copilot_authenticator.py b/pyrit/auth/copilot_authenticator.py index a1c44e31ad..a08e3cbe59 100644 --- a/pyrit/auth/copilot_authenticator.py +++ b/pyrit/auth/copilot_authenticator.py @@ -299,7 +299,7 @@ async def _fetch_access_token_with_playwright(self) -> Optional[str]: import sys try: - from playwright.async_api import async_playwright # noqa: F401 + from playwright.async_api import async_playwright # type: ignore[ty:unresolved-import] # noqa: F401 except ImportError: raise RuntimeError( "Playwright is not installed. Please install it with: " @@ -357,7 +357,7 @@ async def _run_playwright_browser_automation(self) -> Optional[str]: Raises: ValueError: If the username is not set. """ - from playwright.async_api import async_playwright + from playwright.async_api import async_playwright # type: ignore[ty:unresolved-import] bearer_token = None token_expires_in = None diff --git a/pyrit/executor/benchmark/fairness_bias.py b/pyrit/executor/benchmark/fairness_bias.py index 3c8a2e2f88..1b6a7ae4d2 100644 --- a/pyrit/executor/benchmark/fairness_bias.py +++ b/pyrit/executor/benchmark/fairness_bias.py @@ -287,7 +287,7 @@ def _extract_name(self, response: str) -> Optional[str]: """ # Try spaCy-based extraction first (more robust) try: - import spacy + import spacy # type: ignore[ty:unresolved-import] self._nlp = spacy.load("en_core_web_sm") except Exception: diff --git a/pyrit/prompt_converter/add_image_to_video_converter.py b/pyrit/prompt_converter/add_image_to_video_converter.py index d13b333184..f30ea31e60 100644 --- a/pyrit/prompt_converter/add_image_to_video_converter.py +++ b/pyrit/prompt_converter/add_image_to_video_converter.py @@ -94,7 +94,7 @@ async def _add_image_to_video(self, image_path: str, output_path: str) -> str: ValueError: If the image path is invalid or unsupported video format. """ try: - import cv2 # noqa: F401 + import cv2 # type: ignore[ty:unresolved-import] # noqa: F401 except ModuleNotFoundError as e: logger.error("Could not import opencv. You may need to install it via 'pip install pyrit[opencv]'") raise e diff --git a/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py b/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py index 0306330413..47b7a95ad0 100644 --- a/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py +++ b/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Optional if TYPE_CHECKING: - import azure.cognitiveservices.speech as speechsdk # noqa: F401 + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] # noqa: F401 from pyrit.auth.azure_auth import get_speech_config, get_speech_config_async from pyrit.common import default_values @@ -225,7 +225,7 @@ def _recognize_audio(self, *, audio_bytes: bytes, speech_config: "speechsdk.Spee ModuleNotFoundError: If the azure.cognitiveservices.speech module is not installed. """ try: - import azure.cognitiveservices.speech as speechsdk # noqa: F811 + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] # noqa: F811 except ModuleNotFoundError as e: logger.error( "Could not import azure.cognitiveservices.speech. " @@ -283,7 +283,7 @@ def stop_cb(self, evt: Any, recognizer: Any) -> None: ModuleNotFoundError: If the azure.cognitiveservices.speech module is not installed. """ try: - import azure.cognitiveservices.speech as speechsdk # noqa: F811 + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] # noqa: F811 except ModuleNotFoundError as e: logger.error( "Could not import azure.cognitiveservices.speech. " diff --git a/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py b/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py index 3cb45fe82d..3aee83dba2 100644 --- a/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py +++ b/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Literal, Optional if TYPE_CHECKING: - import azure.cognitiveservices.speech as speechsdk # noqa: F401 + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] # noqa: F401 from pyrit.auth.azure_auth import get_speech_config_async from pyrit.common import default_values @@ -167,7 +167,7 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text ValueError: If the input type is not supported or if the prompt is empty. """ try: - import azure.cognitiveservices.speech as speechsdk # noqa: F811 + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] # noqa: F811 except ModuleNotFoundError as e: logger.error( "Could not import azure.cognitiveservices.speech. " diff --git a/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py b/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py index f2d62be82a..aeaab77f32 100644 --- a/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py +++ b/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py @@ -144,7 +144,7 @@ def __init__( self.huggingface_token = None try: - import torch + import torch # type: ignore[ty:unresolved-import] except ModuleNotFoundError as e: raise RuntimeError("Could not import torch. You may need to install it via 'pip install pyrit[all]'") from e @@ -473,7 +473,7 @@ def _seed_rng(self) -> None: the same process may interfere with determinism. """ if self._random_seed is not None: - import torch + import torch # type: ignore[ty:unresolved-import] torch.manual_seed(self._random_seed) if self.use_cuda: diff --git a/pyrit/prompt_target/playwright_copilot_target.py b/pyrit/prompt_target/playwright_copilot_target.py index 7ce0274484..56205affb7 100644 --- a/pyrit/prompt_target/playwright_copilot_target.py +++ b/pyrit/prompt_target/playwright_copilot_target.py @@ -25,7 +25,7 @@ # Avoid errors for users who don't have playwright installed if TYPE_CHECKING: - from playwright.async_api import Page + from playwright.async_api import Page # type: ignore[ty:unresolved-import] else: Page = None diff --git a/pyrit/prompt_target/playwright_target.py b/pyrit/prompt_target/playwright_target.py index 4178fe902b..af6e07fefc 100644 --- a/pyrit/prompt_target/playwright_target.py +++ b/pyrit/prompt_target/playwright_target.py @@ -14,7 +14,7 @@ # Avoid errors for users who don't have playwright installed if TYPE_CHECKING: - from playwright.async_api import Page + from playwright.async_api import Page # type: ignore[ty:unresolved-import] else: Page = None diff --git a/pyrit/score/video_scorer.py b/pyrit/score/video_scorer.py index 2450105345..f3e4837c56 100644 --- a/pyrit/score/video_scorer.py +++ b/pyrit/score/video_scorer.py @@ -174,7 +174,7 @@ def _extract_frames(self, video_path: str) -> list[str]: ModuleNotFoundError: If OpenCV is not installed. """ try: - import cv2 # noqa: F401 + import cv2 # type: ignore[ty:unresolved-import] # noqa: F401 except ModuleNotFoundError as e: logger.error("Could not import opencv. You may need to install it via 'pip install pyrit[opencv]'") raise e From 6cd8d793611b5ca8dc8123fda57155267a97da60 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Wed, 3 Jun 2026 20:41:17 -0700 Subject: [PATCH 08/13] MAINT: Drop test-file ty unresolved-import override Replaced the 9-file [[tool.ty.overrides]] block that silenced unresolved-import on the optional-extra test modules with 20 surgical inline `# type: ignore[ty:unresolved-import]` comments at the actual import sites. Also added inline ignores for 13 pre-existing baseline diagnostics in the same files (deliberate bad-input tests, monkeypatched-mock assignments, transformers lazy-attr imports, tokenizer-Union narrowing) so removing the override doesn't surface latent noise. Net effect: `ty check tests/...` is now clean on those 9 files and the override block (other than the HuggingFace warn-only rule) is gone. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyproject.toml | 22 ------------------- tests/unit/auth/test_azure_auth.py | 2 +- .../auxiliary_attacks/gcg/test_generator.py | 14 ++++++------ .../executor/benchmark/test_fairness_bias.py | 2 +- .../test_add_image_video_converter.py | 6 ++--- .../test_azure_speech_converter.py | 4 ++-- .../test_azure_speech_text_converter.py | 6 ++--- .../prompt_converter/test_prompt_converter.py | 12 +++++----- .../target/test_huggingface_chat_target.py | 6 ++--- tests/unit/score/test_video_scorer.py | 12 +++++----- 10 files changed, 32 insertions(+), 54 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 104181ac65..4c332aa88f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,28 +185,6 @@ include = ["pyrit/prompt_target/hugging_face/**"] [tool.ty.overrides.rules] invalid-argument-type = "warn" -# Test files that conditionally import optional-extra packages (azure-speech, -# cv2, playwright, torch, spacy) inside try/except ImportError blocks. The -# pyrit/ source files that do the same use inline -# ``# type: ignore[ty:unresolved-import]`` comments instead, so they get full -# type checking with surgical silencing at the import sites. Test files keep -# the broader override because they tend to monkeypatch and re-import these -# modules in ways that would multiply the ignore sites. -[[tool.ty.overrides]] -include = [ - "tests/unit/auth/test_azure_auth.py", - "tests/unit/prompt_converter/test_prompt_converter.py", - "tests/unit/prompt_converter/test_azure_speech_converter.py", - "tests/unit/prompt_converter/test_azure_speech_text_converter.py", - "tests/unit/prompt_converter/test_add_image_video_converter.py", - "tests/unit/score/test_video_scorer.py", - "tests/unit/prompt_target/target/test_huggingface_chat_target.py", - "tests/unit/executor/benchmark/test_fairness_bias.py", - "tests/unit/auxiliary_attacks/gcg/test_generator.py", -] -[tool.ty.overrides.rules] -unresolved-import = "ignore" - [tool.uv] constraint-dependencies = [ "aiohttp>=3.13.4", diff --git a/tests/unit/auth/test_azure_auth.py b/tests/unit/auth/test_azure_auth.py index e732d49c69..d61f871129 100644 --- a/tests/unit/auth/test_azure_auth.py +++ b/tests/unit/auth/test_azure_auth.py @@ -21,7 +21,7 @@ def is_speechsdk_installed(): try: - import azure.cognitiveservices.speech # noqa: F401 + import azure.cognitiveservices.speech # type: ignore[ty:unresolved-import] # noqa: F401 return True except ModuleNotFoundError: diff --git a/tests/unit/auxiliary_attacks/gcg/test_generator.py b/tests/unit/auxiliary_attacks/gcg/test_generator.py index 247f4a9007..e919e9429a 100644 --- a/tests/unit/auxiliary_attacks/gcg/test_generator.py +++ b/tests/unit/auxiliary_attacks/gcg/test_generator.py @@ -61,7 +61,7 @@ def test_init_does_not_touch_global_multiprocessing_state(self) -> None: """Regression: __init__ used to call torch.multiprocessing.set_start_method, which crashed under coverage runs when an earlier test had already pinned a non-spawn context. Worker spawn config now happens in _setup_async.""" - import torch.multiprocessing as mp + import torch.multiprocessing as mp # type: ignore[ty:unresolved-import] with patch.object(mp, "set_start_method") as mock_set: GCGGenerator(models=[GCGModelConfig(name=_LLAMA_2)]) @@ -72,7 +72,7 @@ class TestEnsureSpawnStartMethod: """Tests for the lazily-applied spawn-method guard used before workers are spawned.""" def test_sets_spawn_when_unset(self) -> None: - import torch.multiprocessing as mp + import torch.multiprocessing as mp # type: ignore[ty:unresolved-import] gen = GCGGenerator(models=[GCGModelConfig(name=_LLAMA_2)]) with ( @@ -84,7 +84,7 @@ def test_sets_spawn_when_unset(self) -> None: mock_set.assert_called_once_with("spawn") def test_noop_when_already_spawn(self) -> None: - import torch.multiprocessing as mp + import torch.multiprocessing as mp # type: ignore[ty:unresolved-import] gen = GCGGenerator(models=[GCGModelConfig(name=_LLAMA_2)]) with ( @@ -98,7 +98,7 @@ def test_warns_and_does_not_crash_when_already_other(self, caplog) -> None: """Used to raise 'context has already been set' — now we warn and continue.""" import logging - import torch.multiprocessing as mp + import torch.multiprocessing as mp # type: ignore[ty:unresolved-import] gen = GCGGenerator(models=[GCGModelConfig(name=_LLAMA_2)]) with ( @@ -327,11 +327,11 @@ def test_empty_controls_returns_nan_loss(self, tmp_path: Path) -> None: def _make_mock_worker_with_real_tokenizer() -> MagicMock: """Worker mock backed by a real GPT-2 tokenizer (the smallest workable for chat templates).""" - from transformers import AutoTokenizer + from transformers import AutoTokenizer # type: ignore[ty:possibly-missing-import] tokenizer = AutoTokenizer.from_pretrained("gpt2") - tokenizer.pad_token = tokenizer.eos_token - tokenizer.chat_template = ( + tokenizer.pad_token = tokenizer.eos_token # type: ignore[ty:invalid-assignment, ty:unresolved-attribute] + tokenizer.chat_template = ( # type: ignore[ty:invalid-assignment] "{%- for m in messages -%}" "{%- if m['role'] == 'user' -%}" "[INST] {{ m['content'] }} [/INST] " diff --git a/tests/unit/executor/benchmark/test_fairness_bias.py b/tests/unit/executor/benchmark/test_fairness_bias.py index 8c26715d15..6dc9912981 100644 --- a/tests/unit/executor/benchmark/test_fairness_bias.py +++ b/tests/unit/executor/benchmark/test_fairness_bias.py @@ -21,7 +21,7 @@ def is_spacy_installed(): try: - import spacy # noqa: F401 + import spacy # type: ignore[ty:unresolved-import] # noqa: F401 return True except Exception: diff --git a/tests/unit/prompt_converter/test_add_image_video_converter.py b/tests/unit/prompt_converter/test_add_image_video_converter.py index a9e55a1045..28d1341935 100644 --- a/tests/unit/prompt_converter/test_add_image_video_converter.py +++ b/tests/unit/prompt_converter/test_add_image_video_converter.py @@ -9,7 +9,7 @@ def is_opencv_installed(): try: - import cv2 # noqa: F401 + import cv2 # type: ignore[ty:unresolved-import] # noqa: F401 return True except ModuleNotFoundError: @@ -21,7 +21,7 @@ def video_converter_sample_video(tmp_path, patch_central_database): video_path = str(tmp_path / "test_video.mp4") width, height = 640, 480 if is_opencv_installed(): - import cv2 # noqa: F401 + import cv2 # type: ignore[ty:unresolved-import] # noqa: F401 video_encoding = cv2.VideoWriter_fourcc(*"mp4v") output_video = cv2.VideoWriter(video_path, video_encoding, 1, (width, height)) @@ -37,7 +37,7 @@ def video_converter_sample_image(tmp_path): image_path = str(tmp_path / "test_image.png") image = np.zeros((100, 100, 3), dtype=np.uint8) if is_opencv_installed(): - import cv2 + import cv2 # type: ignore[ty:unresolved-import] cv2.imwrite(image_path, image) return image_path diff --git a/tests/unit/prompt_converter/test_azure_speech_converter.py b/tests/unit/prompt_converter/test_azure_speech_converter.py index 016a6230bb..f544a5645a 100644 --- a/tests/unit/prompt_converter/test_azure_speech_converter.py +++ b/tests/unit/prompt_converter/test_azure_speech_converter.py @@ -11,7 +11,7 @@ def is_speechsdk_installed(): try: - import azure.cognitiveservices.speech # noqa: F401 + import azure.cognitiveservices.speech # type: ignore[ty:unresolved-import] # noqa: F401 return True except ModuleNotFoundError: @@ -33,7 +33,7 @@ async def test_azure_speech_text_to_audio_convert_async( MockSpeechSynthesizer, # noqa: N803 sqlite_instance, ): - import azure.cognitiveservices.speech as speechsdk + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] mock_synthesizer = MagicMock() mock_result = MagicMock() diff --git a/tests/unit/prompt_converter/test_azure_speech_text_converter.py b/tests/unit/prompt_converter/test_azure_speech_text_converter.py index 61657bad2d..a2cb3f29f6 100644 --- a/tests/unit/prompt_converter/test_azure_speech_text_converter.py +++ b/tests/unit/prompt_converter/test_azure_speech_text_converter.py @@ -11,7 +11,7 @@ def is_speechsdk_installed(): try: - import azure.cognitiveservices.speech # noqa: F401 + import azure.cognitiveservices.speech # type: ignore[ty:unresolved-import] # noqa: F401 return True except ModuleNotFoundError: @@ -39,7 +39,7 @@ def test_azure_speech_audio_text_converter_initialization(self, mock_get_require @patch("azure.cognitiveservices.speech.SpeechRecognizer") @patch("pyrit.prompt_converter.azure_speech_audio_to_text_converter.logger") def test_stop_cb(self, mock_logger, MockSpeechRecognizer, mock_get_required_value): # noqa: N803 - import azure.cognitiveservices.speech as speechsdk + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] # Create a mock event mock_event = MagicMock() @@ -68,7 +68,7 @@ def test_stop_cb(self, mock_logger, MockSpeechRecognizer, mock_get_required_valu @patch("azure.cognitiveservices.speech.SpeechRecognizer") @patch("pyrit.prompt_converter.azure_speech_audio_to_text_converter.logger") def test_transcript_cb(self, mock_logger, MockSpeechRecognizer, mock_get_required_value): # noqa: N803 - import azure.cognitiveservices.speech as speechsdk + import azure.cognitiveservices.speech as speechsdk # type: ignore[ty:unresolved-import] # Create a mock event mock_event = MagicMock() diff --git a/tests/unit/prompt_converter/test_prompt_converter.py b/tests/unit/prompt_converter/test_prompt_converter.py index 625ef9e858..708e2651b8 100644 --- a/tests/unit/prompt_converter/test_prompt_converter.py +++ b/tests/unit/prompt_converter/test_prompt_converter.py @@ -106,7 +106,7 @@ async def test_convert_tokens_raises_with_non_text_input_type(): prompt = "This is a test ⟪to convert⟪ and ⟫another part⟫." converter = Base64Converter() with pytest.raises(ValueError, match="Input type must be text when start or end tokens are present."): - await converter.convert_tokens_async(prompt=prompt, input_type="non-text") + await converter.convert_tokens_async(prompt=prompt, input_type="non-text") # type: ignore[ty:invalid-argument-type] async def test_convert_tokens_raises_uneven_tokens(): @@ -175,13 +175,13 @@ async def test_str_join_converter_init() -> None: async def test_str_join_converter_none_raises() -> None: converter = StringJoinConverter() with pytest.raises(TypeError): - assert await converter.convert_async(prompt=None, input_type="text") + assert await converter.convert_async(prompt=None, input_type="text") # type: ignore[ty:invalid-argument-type] async def test_str_join_converter_invalid_type_raises() -> None: converter = StringJoinConverter() with pytest.raises(ValueError): - assert await converter.convert_async(prompt="test", input_type="invalid") # type: ignore[arg-type] # noqa: PGH003 + assert await converter.convert_async(prompt="test", input_type="invalid") # type: ignore[arg-type, ty:invalid-argument-type] # noqa: PGH003 async def test_str_join_converter_unsupported_type_raises() -> None: @@ -454,15 +454,15 @@ def test_input_supported_text_only(converter_class): @pytest.fixture def setup_memory(): memory = SQLiteMemory(db_path=":memory:") - CentralMemory.set_memory_instance(memory) + CentralMemory.set_memory_instance(memory) # type: ignore[ty:invalid-argument-type] mock_target = MockPromptTarget() yield mock_target - CentralMemory.set_memory_instance(None) + CentralMemory.set_memory_instance(None) # type: ignore[ty:invalid-argument-type] def is_speechsdk_installed(): try: - import azure.cognitiveservices.speech # noqa: F401 + import azure.cognitiveservices.speech # type: ignore[ty:unresolved-import] # noqa: F401 return True except ModuleNotFoundError: diff --git a/tests/unit/prompt_target/target/test_huggingface_chat_target.py b/tests/unit/prompt_target/target/test_huggingface_chat_target.py index 93a4ca912f..d72c2f9c18 100644 --- a/tests/unit/prompt_target/target/test_huggingface_chat_target.py +++ b/tests/unit/prompt_target/target/test_huggingface_chat_target.py @@ -16,7 +16,7 @@ def is_torch_installed(): try: - import torch # noqa: F401 + import torch # type: ignore[ty:unresolved-import] # noqa: F401 return True except ModuleNotFoundError: @@ -201,7 +201,7 @@ async def test_send_prompt_async(): async def test_missing_chat_template_error(): hf_chat = HuggingFaceChatTarget(model_id="test_model", use_cuda=False) await hf_chat.load_model_and_tokenizer() - hf_chat.tokenizer.chat_template = None + hf_chat.tokenizer.chat_template = None # type: ignore[ty:invalid-assignment] message_piece = MessagePiece( role="user", @@ -570,7 +570,7 @@ async def test_effective_generation_config_in_metadata(): response = await target.send_prompt_async(message=message) metadata = response[0].message_pieces[0].prompt_metadata - effective_config = json.loads(metadata["effective_generation_config"]) + effective_config = json.loads(metadata["effective_generation_config"]) # type: ignore[ty:invalid-argument-type] assert effective_config["top_k"] == 40 assert effective_config["do_sample"] is True diff --git a/tests/unit/score/test_video_scorer.py b/tests/unit/score/test_video_scorer.py index ddd5b383d7..1fd53fe80c 100644 --- a/tests/unit/score/test_video_scorer.py +++ b/tests/unit/score/test_video_scorer.py @@ -22,7 +22,7 @@ def is_opencv_installed(): try: - import cv2 # noqa: F401 + import cv2 # type: ignore[ty:unresolved-import] # noqa: F401 return True except ModuleNotFoundError: @@ -35,7 +35,7 @@ def video_converter_sample_video(tmp_path, patch_central_database): video_path = str(tmp_path / "test_video.mp4") width, height = 512, 512 if is_opencv_installed(): - import cv2 # noqa: F401 + import cv2 # type: ignore[ty:unresolved-import] # noqa: F401 # Create a video writer object video_encoding = cv2.VideoWriter_fourcc(*"mp4v") @@ -126,7 +126,7 @@ async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Op @pytest.mark.skipif(not is_opencv_installed(), reason="opencv is not installed") async def test_extract_frames_true_false(video_converter_sample_video): """Test that frame extraction produces the expected number of frames""" - import cv2 + import cv2 # type: ignore[ty:unresolved-import] image_scorer = MockTrueFalseScorer() scorer = VideoTrueFalseScorer(image_capable_scorer=image_scorer, num_sampled_frames=3) @@ -149,7 +149,7 @@ async def test_extract_frames_true_false(video_converter_sample_video): @pytest.mark.skipif(not is_opencv_installed(), reason="opencv is not installed") async def test_extract_frames_float_scale(video_converter_sample_video): """Test that frame extraction produces the expected number of frames for float scale scorer""" - import cv2 + import cv2 # type: ignore[ty:unresolved-import] image_scorer = MockFloatScaleScorer() scorer = VideoFloatScaleScorer(image_capable_scorer=image_scorer, num_sampled_frames=3) @@ -219,7 +219,7 @@ async def test_score_video_no_frames(video_converter_sample_video): scorer = VideoTrueFalseScorer(image_capable_scorer=image_scorer, num_sampled_frames=3) # Mock _extract_frames to return empty list - scorer._video_helper._extract_frames = MagicMock(return_value=[]) + scorer._video_helper._extract_frames = MagicMock(return_value=[]) # type: ignore[ty:invalid-assignment] with pytest.raises(ValueError, match="No frames extracted from video for scoring."): await scorer._score_piece_async(video_converter_sample_video) @@ -231,7 +231,7 @@ async def test_score_video_no_scores(video_converter_sample_video): image_scorer = MockTrueFalseScorer() # Mock score_prompts_batch_async to return empty list - image_scorer.score_prompts_batch_async = AsyncMock(return_value=[]) + image_scorer.score_prompts_batch_async = AsyncMock(return_value=[]) # type: ignore[ty:invalid-assignment] scorer = VideoTrueFalseScorer(image_capable_scorer=image_scorer, num_sampled_frames=3) with pytest.raises(ValueError, match="No scores returned for image frames extracted from video."): From 4a776ca32d166bfec1250e2c62247eeb07567ef9 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 06:30:24 -0700 Subject: [PATCH 09/13] Fix ty errors surfaced by merge with main - scenario.py: pass ScenarioRunState.CREATED enum (Pydantic ScenarioResult is strict) - add_image_to_video_converter.py: use cv2.VideoWriter.fourcc (modern API, properly stubbed) - hugging_face_chat_target.py: type:ignore[ty:possibly-missing-import] for AutoModelForCausalLM/AutoTokenizer; type:ignore[ty:unresolved-attribute] for tokenizer.decode - test_scorer_contract.py: import from pyrit.models.identifiers instead of deprecated pyrit.identifiers shim (test_no_internal_callers_of_deprecated_pyrit_identifiers_path) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyrit/prompt_converter/add_image_to_video_converter.py | 2 +- .../prompt_target/hugging_face/hugging_face_chat_target.py | 6 +++--- pyrit/scenario/core/scenario.py | 4 ++-- .../azure_ai_evaluation/test_scorer_contract.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyrit/prompt_converter/add_image_to_video_converter.py b/pyrit/prompt_converter/add_image_to_video_converter.py index 0c786e05cd..8f03088a94 100644 --- a/pyrit/prompt_converter/add_image_to_video_converter.py +++ b/pyrit/prompt_converter/add_image_to_video_converter.py @@ -168,7 +168,7 @@ def _add_image_to_video_sync( height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) file_extension = video_path.split(".")[-1].lower() if file_extension in video_encoding_map: - video_char_code = cv2.VideoWriter_fourcc(*video_encoding_map[file_extension]) + video_char_code = cv2.VideoWriter.fourcc(*video_encoding_map[file_extension]) output_video = cv2.VideoWriter(output_path, video_char_code, fps, (width, height)) else: raise ValueError(f"Unsupported video format: {file_extension}") diff --git a/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py b/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py index ee2cc507cc..3d5a08e2d0 100644 --- a/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py +++ b/pyrit/prompt_target/hugging_face/hugging_face_chat_target.py @@ -9,8 +9,8 @@ from typing import Any, cast from transformers import ( - AutoModelForCausalLM, - AutoTokenizer, + AutoModelForCausalLM, # type: ignore[ty:possibly-missing-import] + AutoTokenizer, # type: ignore[ty:possibly-missing-import] BatchEncoding, PretrainedConfig, ) @@ -383,7 +383,7 @@ async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Me assistant_response = cast( "str", - self.tokenizer.decode(generated_tokens, skip_special_tokens=self.skip_special_tokens), + self.tokenizer.decode(generated_tokens, skip_special_tokens=self.skip_special_tokens), # type: ignore[ty:unresolved-attribute] ).strip() if not assistant_response: diff --git a/pyrit/scenario/core/scenario.py b/pyrit/scenario/core/scenario.py index 924b0751d5..cc735481e7 100644 --- a/pyrit/scenario/core/scenario.py +++ b/pyrit/scenario/core/scenario.py @@ -38,7 +38,7 @@ from pyrit.memory import CentralMemory from pyrit.memory.memory_models import ScenarioResultEntry from pyrit.models import AttackOutcome, AttackResult, SeedAttackGroup -from pyrit.models.scenario_result import ScenarioIdentifier, ScenarioResult +from pyrit.models.scenario_result import ScenarioIdentifier, ScenarioResult, ScenarioRunState from pyrit.prompt_target import PromptTarget from pyrit.prompt_target.common.target_requirements import TargetRequirements from pyrit.registry import ScorerRegistry @@ -716,7 +716,7 @@ async def initialize_async( objective_scorer_identifier=self._objective_scorer_identifier, labels=self._memory_labels, attack_results=attack_results, - scenario_run_state="CREATED", + scenario_run_state=ScenarioRunState.CREATED, display_group_map=self._display_group_map, metadata=self._build_initial_scenario_metadata(), ) diff --git a/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py b/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py index 524e93a90e..2bc9f5bee3 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py @@ -48,7 +48,7 @@ class TestScorerUtilities: def test_scorer_identifier_importable(self): """RAIServiceScorer uses ScorerEvaluationIdentifier for identity tracking.""" - from pyrit.identifiers import ScorerEvaluationIdentifier + from pyrit.models.identifiers import ScorerEvaluationIdentifier assert ScorerEvaluationIdentifier is not None From 4c81e25717748aa10dafd4c149b515f1ca61a4c2 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 07:39:43 -0700 Subject: [PATCH 10/13] Merge branch 'main' into pr/1931/romanlutz/audit-unused-dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolved conflicts in 7 files where my PR's # type: ignore[ty:unresolved-import] annotations clashed with PR #1939's noqa audit (which stripped the noqa parts). Took main's cleaner version since CI's ty config reports the type-ignore as unused anyway. Fix: pyrit/models/results/strategy_result.py:12 — add type:ignore for typing.Self import (Python 3.10 doesn't expose typing.Self; the import is TYPE_CHECKING-guarded but ty's stricter unresolved-import=error config still flags it). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/instructions/attacks.instructions.md | 58 +++++++ .github/instructions/scorers.instructions.md | 60 +++++++ .github/instructions/targets.instructions.md | 111 ++++++++++++ pyrit/common/brick_contract.py | 99 +++++++++++ pyrit/models/results/strategy_result.py | 4 +- tests/unit/common/test_brick_contract.py | 168 +++++++++++++++++++ 6 files changed, 498 insertions(+), 2 deletions(-) create mode 100644 .github/instructions/attacks.instructions.md create mode 100644 .github/instructions/scorers.instructions.md create mode 100644 .github/instructions/targets.instructions.md create mode 100644 pyrit/common/brick_contract.py create mode 100644 tests/unit/common/test_brick_contract.py diff --git a/.github/instructions/attacks.instructions.md b/.github/instructions/attacks.instructions.md new file mode 100644 index 0000000000..2d5c4d7c96 --- /dev/null +++ b/.github/instructions/attacks.instructions.md @@ -0,0 +1,58 @@ +--- +applyTo: "pyrit/executor/attack/**" +--- + +# PyRIT AttackStrategy Development Guidelines + +`AttackStrategy` subclasses (single-turn attacks like `PromptSendingAttack`, multi-turn attacks like `RedTeamingAttack`, etc.) are pluggable bricks orchestrated by `AttackExecutor` and the `Scenario` framework. Style rules from `style-guide.instructions.md` (async `_async` suffix, keyword-only args, type hints, enums-over-Literals) still apply and are not repeated here. + +## Constructor contract + +`AttackStrategy` subclasses MUST follow the keyword-only constructor shape: + +```python +class MyAttack(AttackStrategy[MyContext, MyResult]): + def __init__( + self, + *, + objective_target: PromptTarget, + custom_param: str = "", + **kwargs: Any, + ) -> None: + super().__init__( + objective_target=objective_target, + context_type=MyContext, + **kwargs, + ) +``` + +Requirements: + +- All parameters after ``self`` are keyword-only (insert ``*`` immediately + after ``self``). This is **enforced at class-definition time** by + `AttackStrategy.__init_subclass__` calling `enforce_keyword_only_init` + (see `pyrit/common/brick_contract.py`). Non-conforming subclasses + raise `TypeError` at import time. +- ``super().__init__(...)`` must be invoked with at minimum + ``objective_target`` and ``context_type``. +- Existing subclasses that cannot adopt the contract immediately may set + the class attribute ``_brick_legacy_init = True`` to opt into a + one-release grace period that downgrades the error to a + ``DeprecationWarning(removed_in="0.16.0")``. The opt-out is removed in + 0.16.0; classes that still violate the contract at that point will hard + fail. +- ``AttackTechniqueFactory`` already rejects ``**kwargs`` in attack + ``__init__`` at factory-registration time + (`pyrit/scenario/core/attack_technique_factory.py`); the new + ``__init_subclass__`` check is complementary — the factory check catches + scenarios-side wiring mistakes, the subclass check catches the + ``__init__`` shape at class-definition time. + +## Common pitfalls + +- Forgetting ``*`` after ``self`` — the new check will surface this at + import time with the exact list of positional parameters that need to be + made keyword-only. +- Calling ``super().__init__`` with positional arguments — the base + ``AttackStrategy.__init__`` is already keyword-only, so positional calls + raise ``TypeError`` at runtime. Always forward via kwargs. diff --git a/.github/instructions/scorers.instructions.md b/.github/instructions/scorers.instructions.md new file mode 100644 index 0000000000..b4200704e0 --- /dev/null +++ b/.github/instructions/scorers.instructions.md @@ -0,0 +1,60 @@ +--- +applyTo: "pyrit/score/**" +--- + +# PyRIT Scorer Development Guidelines + +Scorers evaluate model responses against an objective and live under `pyrit/score/`. Style rules from `style-guide.instructions.md` (async `_async` suffix, keyword-only args, type hints, enums-over-Literals) still apply and are not repeated here. + +## Constructor contract + +`Scorer` subclasses MUST use the keyword-only constructor shape: + +```python +class MyScorer(Scorer): + def __init__( + self, + *, + chat_target: PromptTarget | None = None, + threshold: float = 0.5, + validator: ScorerPromptValidator | None = None, + ) -> None: + super().__init__( + validator=validator or self._DEFAULT_VALIDATOR, + chat_target=chat_target, + ) +``` + +Requirements: + +- All parameters after ``self`` are keyword-only (insert ``*`` immediately + after ``self``). This is **enforced at class-definition time** by + `Scorer.__init_subclass__` calling `enforce_keyword_only_init` + (see `pyrit/common/brick_contract.py`). Non-conforming subclasses + raise `TypeError` at import time. +- ``super().__init__(validator=..., chat_target=...)`` is required so the + base class wires the validator and validates ``TARGET_REQUIREMENTS`` + against any provided ``chat_target``. +- Existing subclasses that cannot adopt the contract immediately may set + the class attribute ``_brick_legacy_init = True`` to opt into a + one-release grace period that downgrades the error to a + ``DeprecationWarning(removed_in="0.16.0")``. The opt-out is removed in + 0.16.0; classes that still violate the contract at that point will hard + fail. + +### Currently grandfathered + +- ``PlagiarismScorer`` (``pyrit/score/float_scale/plagiarism_scorer.py``) — + accepts ``reference_text`` positionally as part of its public API. The + positional shape is preserved through one release cycle via + ``_brick_legacy_init = True`` and is scheduled to become + keyword-only in 0.16.0 (``BREAKING CHANGE``). + +## Common pitfalls + +- Forgetting ``*`` after ``self`` — the new check will surface this at + import time with the exact list of positional parameters that need to be + made keyword-only. +- Calling ``super().__init__`` with positional args — the base + ``Scorer.__init__`` is already keyword-only, so positional calls raise + ``TypeError`` at runtime. Always forward via kwargs. diff --git a/.github/instructions/targets.instructions.md b/.github/instructions/targets.instructions.md new file mode 100644 index 0000000000..19040be72b --- /dev/null +++ b/.github/instructions/targets.instructions.md @@ -0,0 +1,111 @@ +--- +applyTo: "pyrit/prompt_target/**" +--- + +# Prompt Target Development Guidelines + +## Base Class Contract + +All targets MUST inherit from ``PromptTarget`` (or one of its public +subclasses such as ``OpenAITarget`` / ``HTTPTarget``) and implement +``_send_prompt_to_target_async``: + +```python +from pyrit.prompt_target.common.prompt_target import PromptTarget + + +class MyTarget(PromptTarget): + def __init__( + self, + *, + endpoint: str, + api_key: str, + max_requests_per_minute: int | None = None, + custom_configuration: TargetConfiguration | None = None, + ) -> None: + super().__init__( + endpoint=endpoint, + max_requests_per_minute=max_requests_per_minute, + custom_configuration=custom_configuration, + ) + self._api_key = api_key + + async def _send_prompt_to_target_async( + self, *, normalized_conversation: list[Message] + ) -> list[Message]: + ... +``` + +``send_prompt_async`` (the public entry point) is ``@final`` and MUST NOT +be overridden. Override ``_send_prompt_to_target_async`` instead. + +## Keyword-only ``__init__`` is enforced + +Every ``PromptTarget`` subclass MUST make all ``__init__`` parameters +keyword-only (i.e., place ``*`` as the first parameter after ``self``). +``PromptTarget.__init_subclass__`` validates this at class-definition time +via ``enforce_keyword_only_init`` and raises ``TypeError`` on violations. + +The check is satisfied by either of: + +```python +def __init__(self, *, endpoint: str, api_key: str) -> None: ... + +def __init__(self, *args: Any, **kwargs: Any) -> None: ... # *args after self +``` + +It rejects: + +```python +def __init__(self, endpoint: str, api_key: str) -> None: ... # missing * +``` + +> [!NOTE] +> ``PromptTarget.__init__`` *itself* still accepts positional parameters and +> is not currently keyword-only. The ``__init_subclass__`` hook only runs for +> subclasses, so the base class non-compliance is tolerated during the warn- +> first phase. The base ``__init__`` will be reshaped to be keyword-only in +> 0.16.0 as a BREAKING CHANGE. + +## Temporary opt-out: ``_brick_legacy_init`` + +A handful of legacy targets whose positional ``__init__`` is part of the +public API are grandfathered with ``_brick_legacy_init = True``. They +emit a ``DeprecationWarning`` at import time and the opt-out is scheduled +for removal in **0.16.0**. Do not set this flag on new targets; new +targets MUST follow the keyword-only contract. + +Currently grandfathered (slated for cleanup in 0.16.0): +``HTTPTarget``, ``OpenAICompletionTarget``, ``OpenAIImageTarget``, +``PromptShieldTarget``. + +## Configuration and Capabilities + +- Set ``_DEFAULT_CONFIGURATION`` at the class level when your target's + capabilities differ from the base defaults (multi-turn support, non-text + modalities, JSON-mode responses, etc.). +- Accept ``custom_configuration: TargetConfiguration | None = None`` in + ``__init__`` and forward it to ``super().__init__`` so callers can + override capabilities per-instance (this is required for HTTP / Playwright + targets whose capabilities depend on deployment configuration). + +## Identifiable Pattern + +All targets inherit ``Identifiable``. Override ``_build_identifier()`` to +include parameters that affect target behaviour: + +```python +def _build_identifier(self) -> ComponentIdentifier: + return self._create_identifier( + params={"endpoint": self._endpoint, "model_name": self._model_name}, + ) +``` + +Include: endpoint, model_name, deployment identifiers, custom headers that +affect routing. +Exclude: API keys, retry counts, logging config, timeouts. + +## Exports + +New targets MUST be added to ``pyrit/prompt_target/__init__.py`` — both +the import and the ``__all__`` list. diff --git a/pyrit/common/brick_contract.py b/pyrit/common/brick_contract.py new file mode 100644 index 0000000000..fe2a0bd8e0 --- /dev/null +++ b/pyrit/common/brick_contract.py @@ -0,0 +1,99 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Constructor contract enforcement for PyRIT's pluggable brick base classes. + +Several PyRIT base classes (``PromptConverter``, ``Scorer``, ``PromptTarget``, +``Scenario``, ``AttackStrategy``, ``SeedDatasetProvider``) are extension +points that users routinely swap in and out. To make those swaps predictable, +every subclass must use the keyword-only constructor shape mandated by the +style guide: ``def __init__(self, *, ...)``. + +This module provides one shared helper, ``enforce_keyword_only_init``, +that bases invoke from their own ``__init_subclass__`` hook. The helper +inspects the subclass's directly-defined ``__init__`` (not inherited) and +classifies it as compliant or non-compliant. Non-compliant subclasses either +raise ``TypeError`` at class definition time, or, if they opt in via the +``_brick_legacy_init`` class attribute, emit a ``DeprecationWarning`` +via ``print_deprecation_message`` and continue. +The opt-out is intended to be removed in ``0.16.0``. +""" + +from __future__ import annotations + +import inspect +from inspect import Parameter + +from pyrit.common.deprecation import print_deprecation_message + +#: Class attribute name that opts a subclass into the legacy-init grace period. +#: When ``True`` on a class, ``enforce_keyword_only_init`` downgrades the +#: ``TypeError`` to a ``DeprecationWarning`` until ``_LEGACY_REMOVED_IN``. +LEGACY_INIT_OPT_OUT_ATTR = "_brick_legacy_init" + +#: Version in which the legacy-init opt-out is removed; non-conforming +#: subclasses will hard-fail at that point. +_LEGACY_REMOVED_IN = "0.16.0" + + +def enforce_keyword_only_init(cls: type, *, base_name: str) -> None: + """ + Validate that ``cls.__init__`` only accepts keyword-only parameters. + + Intended to be called from a base class's ``__init_subclass__`` hook to + enforce the brick constructor contract on subclasses. + + The helper only inspects ``__init__`` defined directly on ``cls`` (i.e. + ``"__init__" in cls.__dict__``). Subclasses that inherit ``__init__`` + from their parent are not re-checked — the parent will already have been + checked at its own definition time. + + Args: + cls: The subclass being defined. Pass through from + ``__init_subclass__``. + base_name: Display name of the base class (e.g. ``"Scenario"``). + Used in error messages so the user knows which contract was + violated. + + Raises: + TypeError: If ``cls.__init__`` accepts any positional or + positional-or-keyword parameters after ``self``, and ``cls`` does + not opt into the legacy-init grace period via the + ``_brick_legacy_init`` class attribute. The opt-out is only + honored when set directly on ``cls`` (it is not inherited from a + base class), so new subclasses always get the hard check by + default. + """ + if "__init__" not in cls.__dict__: + # Subclass inherits __init__ from its parent; the parent has already + # been validated. Nothing to check here. + return + + sig = inspect.signature(cls.__init__) + # Skip ``self`` (always the first parameter on an unbound method). + params = list(sig.parameters.values())[1:] + + offenders = [p.name for p in params if p.kind in (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD)] + if not offenders: + return + + if cls.__dict__.get(LEGACY_INIT_OPT_OUT_ATTR, False): + # Opt-in legacy period: warn rather than break, so existing users + # whose code calls these constructors positionally have one release + # cycle to migrate. + print_deprecation_message( + old_item=(f"{cls.__module__}.{cls.__qualname__}.__init__ with positional parameters {offenders!r}"), + new_item=(f"keyword-only parameters per the {base_name} contract (insert ``*`` after ``self``)"), + removed_in=_LEGACY_REMOVED_IN, + ) + return + + raise TypeError( + f"{cls.__name__}.__init__ violates the {base_name} contract: " + f"all parameters after ``self`` must be keyword-only, but the " + f"following are positional: {offenders!r}. Insert ``*,`` after " + f"``self`` to fix, or set ``{LEGACY_INIT_OPT_OUT_ATTR} = True`` on " + f"the class to opt into a temporary deprecation period (removed in " + f"{_LEGACY_REMOVED_IN})." + ) diff --git a/pyrit/models/results/strategy_result.py b/pyrit/models/results/strategy_result.py index 2c30c4509b..486c3c82e8 100644 --- a/pyrit/models/results/strategy_result.py +++ b/pyrit/models/results/strategy_result.py @@ -9,12 +9,12 @@ from pydantic import BaseModel, ConfigDict if TYPE_CHECKING: - from typing import Self + from typing import Self # type: ignore[ty:unresolved-import] # noqa: F401 StrategyResultT = TypeVar("StrategyResultT", bound="StrategyResult") -class StrategyResult(BaseModel, ABC): # noqa: B024 +class StrategyResult(BaseModel, ABC): """Base class for all strategy results.""" model_config = ConfigDict( diff --git a/tests/unit/common/test_brick_contract.py b/tests/unit/common/test_brick_contract.py new file mode 100644 index 0000000000..a845eab4fa --- /dev/null +++ b/tests/unit/common/test_brick_contract.py @@ -0,0 +1,168 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import warnings + +import pytest + +from pyrit.common.brick_contract import enforce_keyword_only_init + + +class _FakeBase: + """Standalone base class used to drive the helper in isolation.""" + + def __init_subclass__(cls, **kwargs: object) -> None: + super().__init_subclass__(**kwargs) + enforce_keyword_only_init(cls, base_name="_FakeBase") + + +def test_compliant_keyword_only_init_passes() -> None: + class Compliant(_FakeBase): + def __init__(self, *, foo: str, bar: int = 0) -> None: + self.foo = foo + self.bar = bar + + instance = Compliant(foo="hello", bar=3) + assert instance.foo == "hello" + assert instance.bar == 3 + + +def test_self_only_init_passes() -> None: + class SelfOnly(_FakeBase): + def __init__(self) -> None: + pass + + assert SelfOnly() is not None + + +def test_inherited_init_is_not_double_checked() -> None: + """Subclasses without their own __init__ inherit the (compliant) parent.""" + + class Parent(_FakeBase): + def __init__(self, *, foo: str = "") -> None: + self.foo = foo + + class Child(Parent): + pass + + assert Child(foo="x").foo == "x" + + +def test_positional_init_raises_typeerror() -> None: + with pytest.raises(TypeError) as excinfo: + + class Violator(_FakeBase): + def __init__(self, foo: str, bar: int = 0) -> None: + self.foo = foo + self.bar = bar + + message = str(excinfo.value) + assert "_FakeBase contract" in message + assert "foo" in message + assert "bar" in message + assert "_brick_legacy_init" in message + + +def test_positional_or_keyword_default_still_raises() -> None: + """A param with a default is still positional-or-keyword by default.""" + with pytest.raises(TypeError): + + class StillPositional(_FakeBase): + def __init__(self, foo: str = "x") -> None: + self.foo = foo + + +def test_starargs_without_star_marker_raises() -> None: + """``*args`` after positional params doesn't fix the positional params.""" + with pytest.raises(TypeError) as excinfo: + + class StarArgsSandwich(_FakeBase): + def __init__(self, foo: str = "", *args: object) -> None: + self.foo = foo + + assert "foo" in str(excinfo.value) + + +def test_starargs_first_passes() -> None: + """``*args`` immediately after ``self`` makes subsequent params kw-only.""" + + class StarArgsFirst(_FakeBase): + def __init__(self, *args: object, bar: int = 0) -> None: + self.args = args + self.bar = bar + + assert StarArgsFirst(bar=1).bar == 1 + + +def test_legacy_opt_out_downgrades_to_warning() -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + + class Grandfathered(_FakeBase): + _brick_legacy_init = True + + def __init__(self, foo: str, bar: int = 0) -> None: + self.foo = foo + self.bar = bar + + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert len(deprecations) == 1 + message = str(deprecations[0].message) + assert "Grandfathered" in message + assert "0.16.0" in message + assert "foo" in message + # Class still works after the warning. + instance = Grandfathered("hi", 2) + assert instance.foo == "hi" + assert instance.bar == 2 + + +def test_legacy_opt_out_false_still_raises() -> None: + with pytest.raises(TypeError): + + class NotGrandfathered(_FakeBase): + _brick_legacy_init = False + + def __init__(self, foo: str) -> None: + self.foo = foo + + +def test_legacy_opt_out_is_not_inherited_by_subclass() -> None: + """The opt-out only applies to the class that sets it; subclasses still hard-fail.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + + class Grandfathered(_FakeBase): + _brick_legacy_init = True + + def __init__(self, foo: str) -> None: + self.foo = foo + + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert len(deprecations) == 1 + + with pytest.raises(TypeError) as excinfo: + + class NewSubclass(Grandfathered): + def __init__(self, foo: str, bar: int = 0) -> None: + self.foo = foo + self.bar = bar + + message = str(excinfo.value) + assert "_FakeBase contract" in message + assert "foo" in message + assert "bar" in message + + +def test_error_message_lists_only_positional_offenders() -> None: + """The error message should only list positional offenders, not kw-only ones.""" + with pytest.raises(TypeError) as excinfo: + + class Mixed(_FakeBase): + def __init__(self, positional_one: str, *, keyword_only: int = 0) -> None: + self.positional_one = positional_one + self.keyword_only = keyword_only + + message = str(excinfo.value) + assert "positional_one" in message + assert "keyword_only" not in message From ec9d501eac1dd4eef8105161ae629b2c7d53671f Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 13:18:56 -0700 Subject: [PATCH 11/13] Address review comments from hannahwestra25 - strategy_result.py: use typing_extensions.Self instead of typing.Self (works on 3.10+ without a type:ignore; typing_extensions is a transitive dep already used in 40+ pyrit files) - test_add_image_video_converter.py: VideoWriter_fourcc -> VideoWriter.fourcc (matches the production fix); drop unused type:ignore[ty:unresolved-import] on lazy cv2 imports - style-guide.instructions.md: document typing_extensions convention for Self/override/TypeAlias/etc. that aren't on every supported Python Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/instructions/style-guide.instructions.md | 16 ++++++++++++++++ pyrit/models/results/strategy_result.py | 2 +- .../test_add_image_video_converter.py | 8 ++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/instructions/style-guide.instructions.md b/.github/instructions/style-guide.instructions.md index edee2e0b59..643a7c0662 100644 --- a/.github/instructions/style-guide.instructions.md +++ b/.github/instructions/style-guide.instructions.md @@ -192,6 +192,22 @@ from pyrit.common.net_utility import get_httpx_client Within the same package, import from the specific file to avoid circular imports. +### Typing Backports (`typing_extensions`) + +For typing features that don't exist on every supported Python (`Self`, +`override`, `TypeAlias`, `Unpack`, `NotRequired`, etc.), import from +``typing_extensions`` rather than ``typing``. `typing_extensions` is already a +transitive dependency (pulled in by ``pydantic``) and works across all supported +Python versions, so this avoids per-version branching and ``# type: ignore`` noise. + +```python +# CORRECT — works on 3.10+ +from typing_extensions import Self, override + +# INCORRECT — `Self` is 3.11+, `override` is 3.12+, breaks on older runtimes +from typing import Self, override +``` + ## Documentation Standards ### Docstring Format diff --git a/pyrit/models/results/strategy_result.py b/pyrit/models/results/strategy_result.py index 486c3c82e8..a644f504d5 100644 --- a/pyrit/models/results/strategy_result.py +++ b/pyrit/models/results/strategy_result.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, ConfigDict if TYPE_CHECKING: - from typing import Self # type: ignore[ty:unresolved-import] # noqa: F401 + from typing_extensions import Self StrategyResultT = TypeVar("StrategyResultT", bound="StrategyResult") diff --git a/tests/unit/prompt_converter/test_add_image_video_converter.py b/tests/unit/prompt_converter/test_add_image_video_converter.py index d8f28df3d8..3b47b97dd4 100644 --- a/tests/unit/prompt_converter/test_add_image_video_converter.py +++ b/tests/unit/prompt_converter/test_add_image_video_converter.py @@ -9,7 +9,7 @@ def is_opencv_installed(): try: - import cv2 # type: ignore[ty:unresolved-import] # noqa: F401 + import cv2 # noqa: F401 return True except ModuleNotFoundError: @@ -21,9 +21,9 @@ def video_converter_sample_video(tmp_path, patch_central_database): video_path = str(tmp_path / "test_video.mp4") width, height = 640, 480 if is_opencv_installed(): - import cv2 # type: ignore[ty:unresolved-import] # noqa: F401 + import cv2 - video_encoding = cv2.VideoWriter_fourcc(*"mp4v") + video_encoding = cv2.VideoWriter.fourcc(*"mp4v") output_video = cv2.VideoWriter(video_path, video_encoding, 1, (width, height)) for _i in range(10): frame = np.zeros((height, width, 3), dtype=np.uint8) @@ -37,7 +37,7 @@ def video_converter_sample_image(tmp_path): image_path = str(tmp_path / "test_image.png") image = np.zeros((100, 100, 3), dtype=np.uint8) if is_opencv_installed(): - import cv2 # type: ignore[ty:unresolved-import] + import cv2 cv2.imwrite(image_path, image) return image_path From 10490387253b36e56999d2c6e8f845223f2675b4 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 13:37:30 -0700 Subject: [PATCH 12/13] Make scenario short-path aliases safe for submodules Walk each canonical package with pkgutil.walk_packages and register every submodule under the short name too, so the second import returns the same module object. Verified module identity for deep nesting (e.g. pyrit.scenario.adaptive.selectors.epsilon_greedy). Reverts the docs in doc/scanner/garak.{py,ipynb} back to the intended short path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/scanner/garak.ipynb | 2 +- doc/scanner/garak.py | 2 +- pyrit/scenario/__init__.py | 33 ++++++++++++++++++++++++++++----- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/doc/scanner/garak.ipynb b/doc/scanner/garak.ipynb index 22ce389806..6b9d68b91b 100644 --- a/doc/scanner/garak.ipynb +++ b/doc/scanner/garak.ipynb @@ -44,7 +44,7 @@ "from pyrit.output import output_scenario_async\n", "from pyrit.registry import TargetRegistry\n", "from pyrit.scenario.garak import Encoding, EncodingStrategy\n", - "from pyrit.scenario.scenarios.garak.encoding import EncodingDatasetConfiguration\n", + "from pyrit.scenario.garak.encoding import EncodingDatasetConfiguration\n", "from pyrit.setup import initialize_from_config_async\n", "\n", "await initialize_from_config_async(config_path=Path(\"pyrit_conf.yaml\")) # type: ignore\n", diff --git a/doc/scanner/garak.py b/doc/scanner/garak.py index 5e9f2d044b..e86c03146f 100644 --- a/doc/scanner/garak.py +++ b/doc/scanner/garak.py @@ -24,7 +24,7 @@ from pyrit.output import output_scenario_async from pyrit.registry import TargetRegistry from pyrit.scenario.garak import Encoding, EncodingStrategy -from pyrit.scenario.scenarios.garak.encoding import EncodingDatasetConfiguration +from pyrit.scenario.garak.encoding import EncodingDatasetConfiguration from pyrit.setup import initialize_from_config_async await initialize_from_config_async(config_path=Path("pyrit_conf.yaml")) # type: ignore diff --git a/pyrit/scenario/__init__.py b/pyrit/scenario/__init__.py index 1be2caea53..f87468dd3e 100644 --- a/pyrit/scenario/__init__.py +++ b/pyrit/scenario/__init__.py @@ -37,11 +37,34 @@ from pyrit.scenario.scenarios import foundry as _foundry_module from pyrit.scenario.scenarios import garak as _garak_module -sys.modules["pyrit.scenario.adaptive"] = _adaptive_module -sys.modules["pyrit.scenario.airt"] = _airt_module -sys.modules["pyrit.scenario.benchmark"] = _benchmark_module -sys.modules["pyrit.scenario.garak"] = _garak_module -sys.modules["pyrit.scenario.foundry"] = _foundry_module + +def _register_scenario_alias(short_name: str, canonical_module) -> None: + """Alias ``pyrit.scenario.`` (and every submodule) to ``canonical_module``. + + A bare ``sys.modules[short] = canonical`` only fixes ``import + pyrit.scenario.`` itself. Accessing a submodule via the alias path + (``pyrit.scenario..``) re-runs the submodule's file under the + aliased fully-qualified name and produces a duplicate class object — which + silently breaks ``isinstance`` against the canonical class. To prevent that, + we walk the canonical package's submodules eagerly and register every one + under both names so the second import returns the same module object. + """ + import importlib + import pkgutil + + sys.modules[f"pyrit.scenario.{short_name}"] = canonical_module + canonical_prefix = canonical_module.__name__ + "." + short_prefix = f"pyrit.scenario.{short_name}." + for module_info in pkgutil.walk_packages(canonical_module.__path__, canonical_prefix): + submodule = importlib.import_module(module_info.name) + sys.modules[short_prefix + module_info.name[len(canonical_prefix) :]] = submodule + + +_register_scenario_alias("adaptive", _adaptive_module) +_register_scenario_alias("airt", _airt_module) +_register_scenario_alias("benchmark", _benchmark_module) +_register_scenario_alias("foundry", _foundry_module) +_register_scenario_alias("garak", _garak_module) # Also expose as attributes for IDE support adaptive = _adaptive_module From 0cb28d5ff0160e1a2d570146a5aea4e6534657d3 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:31:36 -0700 Subject: [PATCH 13/13] Move scenario alias imports to top of file (review feedback) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hannah called out that 'import importlib' and 'import pkgutil' were inside the helper function — moved them to the top-level imports block per the style guide. Also added the missing 'canonical_module: ModuleType' annotation that pre-commit ruff (ANN001) flagged in CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyrit/scenario/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyrit/scenario/__init__.py b/pyrit/scenario/__init__.py index f87468dd3e..bbc849e6fc 100644 --- a/pyrit/scenario/__init__.py +++ b/pyrit/scenario/__init__.py @@ -13,7 +13,10 @@ from pyrit.scenario.foundry import RedTeamAgent """ +import importlib +import pkgutil import sys +from types import ModuleType from pyrit.common.parameter import Parameter from pyrit.models.scenario_result import ScenarioIdentifier, ScenarioResult @@ -38,8 +41,9 @@ from pyrit.scenario.scenarios import garak as _garak_module -def _register_scenario_alias(short_name: str, canonical_module) -> None: - """Alias ``pyrit.scenario.`` (and every submodule) to ``canonical_module``. +def _register_scenario_alias(short_name: str, canonical_module: ModuleType) -> None: + """ + Alias ``pyrit.scenario.`` (and every submodule) to ``canonical_module``. A bare ``sys.modules[short] = canonical`` only fixes ``import pyrit.scenario.`` itself. Accessing a submodule via the alias path @@ -49,9 +53,6 @@ def _register_scenario_alias(short_name: str, canonical_module) -> None: we walk the canonical package's submodules eagerly and register every one under both names so the second import returns the same module object. """ - import importlib - import pkgutil - sys.modules[f"pyrit.scenario.{short_name}"] = canonical_module canonical_prefix = canonical_module.__name__ + "." short_prefix = f"pyrit.scenario.{short_name}."