MAINT Scenario Namespace & Directory Changes

doc/api.rst

@@ -531,10 +531,10 @@ API Reference
     TrueFalseScoreAggregator
     TrueFalseScorer
 
-:py:mod:`pyrit.scenarios`
+:py:mod:`pyrit.scenario`
 =========================
 
-.. automodule:: pyrit.scenarios
+.. automodule:: pyrit.scenario
     :no-members:
     :no-inherited-members:
 

doc/code/front_end/1_pyrit_scan.ipynb

@@ -247,7 +247,7 @@
     "\n",
     "You need a single scenario to run, you need two things:\n",
     "\n",
-    "1. A Scenario. Many are defined in `pyrit.scenarios.scenarios`. But you can also define your own in initialization_scripts.\n",
+    "1. A Scenario. Many are defined in `pyrit.scenario.scenarios`. But you can also define your own in initialization_scripts.\n",
     "2. Initializers (which can be supplied via `--initializers` or `--initialization-scripts`). Scenarios often don't need many arguments, but they can be configured in different ways. And at the very least, most need an `objective_target` (the thing you're running a scan against).\n",
     "3. Scenario Strategies (optional). These are supplied by the `--scenario-strategies` flag and tell the scenario what to test, but they are always optional. Also note you can obtain these by running `--list-scenarios`\n",
     "\n",
@@ -413,8 +413,8 @@
    "source": [
     "# my_custom_scenarios.py\n",
     "from pyrit.common.apply_defaults import apply_defaults\n",
-    "from pyrit.scenarios import Scenario\n",
-    "from pyrit.scenarios.scenario_strategy import ScenarioStrategy\n",
+    "from pyrit.scenario import Scenario\n",
+    "from pyrit.scenario.scenario_strategy import ScenarioStrategy\n",
     "\n",
     "\n",
     "class MyCustomStrategy(ScenarioStrategy):\n",

doc/code/front_end/1_pyrit_scan.py

@@ -59,7 +59,7 @@
 #
 # You need a single scenario to run, you need two things:
 #
-# 1. A Scenario. Many are defined in `pyrit.scenarios.scenarios`. But you can also define your own in initialization_scripts.
+# 1. A Scenario. Many are defined in `pyrit.scenario.scenarios`. But you can also define your own in initialization_scripts.
 # 2. Initializers (which can be supplied via `--initializers` or `--initialization-scripts`). Scenarios often don't need many arguments, but they can be configured in different ways. And at the very least, most need an `objective_target` (the thing you're running a scan against).
 # 3. Scenario Strategies (optional). These are supplied by the `--scenario-strategies` flag and tell the scenario what to test, but they are always optional. Also note you can obtain these by running `--list-scenarios`
 #
@@ -124,8 +124,8 @@
 
 # %%
 # my_custom_scenarios.py
-from pyrit.scenarios import Scenario
-from pyrit.scenarios.scenario_strategy import ScenarioStrategy
+from pyrit.scenario import Scenario
+from pyrit.scenario.scenario_strategy import ScenarioStrategy
 
 
 class MyCustomStrategy(ScenarioStrategy):

doc/code/scenarios/scenarios.ipynb

@@ -41,7 +41,7 @@
     "\n",
     "## Creating Custom Scenarios\n",
     "\n",
-    "To create a custom scenario, extend the `Scenario` base class. See [`FoundryScenario`](../../../pyrit/scenarios/scenarios/foundry_scenario.py) for an example.\n",
+    "To create a custom scenario, extend the `Scenario` base class. See [`FoundryScenario`](../../../pyrit/scenario/scenarios/foundry_scenario.py) for an example.\n",
     "\n",
     "## Using Scenarios\n",
     "\n",
@@ -64,7 +64,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a1dd9e091cb741e59cd9408027ba6e2e",
+       "model_id": "a99d236b595c4934ad564e9562190ff0",
        "version_major": 2,
        "version_minor": 0
       },
@@ -101,7 +101,7 @@
       "\u001b[1m  🎯 Target Information\u001b[0m\n",
       "\u001b[36m    • Target Type: OpenAIChatTarget\u001b[0m\n",
       "\u001b[36m    • Target Model: gpt4o\u001b[0m\n",
-      "\u001b[36m    • Target Endpoint: https://pyrit-github-pipeline.openai.azure.com/openai/deployments/gpt-40/chat/completions?api-version=2024-10-21\u001b[0m\n",
+      "\u001b[36m    • Target Endpoint: https://pyrit-japan-test.openai.azure.com/openai/deployments/gpt-4o-japan-nilfilter/chat/completions?api-version=2024-10-21\u001b[0m\n",
       "\n",
       "\u001b[1m  📊 Scorer Information\u001b[0m\n",
       "\u001b[36m    • Scorer Type: TrueFalseCompositeScorer\u001b[0m\n",
@@ -118,7 +118,7 @@
       "\u001b[1m  📈 Summary\u001b[0m\n",
       "\u001b[32m    • Total Strategies: 4\u001b[0m\n",
       "\u001b[32m    • Total Attack Results: 8\u001b[0m\n",
-      "\u001b[32m    • Overall Success Rate: 12%\u001b[0m\n",
+      "\u001b[32m    • Overall Success Rate: 0%\u001b[0m\n",
       "\u001b[32m    • Unique Objectives: 2\u001b[0m\n",
       "\n",
       "\u001b[1m\u001b[36m▼ Per-Strategy Breakdown\u001b[0m\n",
@@ -130,7 +130,7 @@
       "\n",
       "\u001b[1m  🔸 Strategy: base64\u001b[0m\n",
       "\u001b[33m    • Number of Results: 2\u001b[0m\n",
-      "\u001b[33m    • Success Rate: 50%\u001b[0m\n",
+      "\u001b[32m    • Success Rate: 0%\u001b[0m\n",
       "\n",
       "\u001b[1m  🔸 Strategy: binary\u001b[0m\n",
       "\u001b[33m    • Number of Results: 2\u001b[0m\n",
@@ -147,8 +147,9 @@
    ],
    "source": [
     "from pyrit.prompt_target import OpenAIChatTarget\n",
-    "from pyrit.scenarios import FoundryScenario, FoundryStrategy, ScenarioCompositeStrategy\n",
-    "from pyrit.scenarios.printer.console_printer import ConsoleScenarioResultPrinter\n",
+    "from pyrit.scenario.core import ScenarioCompositeStrategy\n",
+    "from pyrit.scenario.printer.console_printer import ConsoleScenarioResultPrinter\n",
+    "from pyrit.scenario.scenarios import FoundryScenario, FoundryStrategy\n",
     "from pyrit.setup import IN_MEMORY, initialize_pyrit\n",
     "\n",
     "initialize_pyrit(\n",
@@ -174,10 +175,8 @@
     "\n",
     "\n",
     "# Create a scenario from the pre-configured Foundry scenario\n",
-    "foundry_scenario = FoundryScenario(\n",
-    "    objectives=objectives\n",
-    ")\n",
-    "await foundry_scenario.initialize_async( # type: ignore\n",
+    "foundry_scenario = FoundryScenario(objectives=objectives)\n",
+    "await foundry_scenario.initialize_async(  # type: ignore\n",
     "    objective_target=objective_target,\n",
     "    scenario_strategies=scenario_strategies,\n",
     "    max_concurrency=10,\n",
@@ -186,8 +185,8 @@
     "print(f\"Created scenario: {foundry_scenario.name}\")\n",
     "\n",
     "# Execute the entire scenario\n",
-    "foundry_results = await foundry_scenario.run_async() # type: ignore\n",
-    "await printer.print_summary_async(foundry_results) # type: ignore"
+    "foundry_results = await foundry_scenario.run_async()  # type: ignore\n",
+    "await printer.print_summary_async(foundry_results)  # type: ignore"
    ]
   },
   {
@@ -226,7 +225,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.11"
+   "version": "3.11.14"
   }
  },
  "nbformat": 4,

doc/code/scenarios/scenarios.py

@@ -45,7 +45,7 @@
 #
 # ## Creating Custom Scenarios
 #
-# To create a custom scenario, extend the `Scenario` base class. See [`FoundryScenario`](../../../pyrit/scenarios/scenarios/foundry_scenario.py) for an example.
+# To create a custom scenario, extend the `Scenario` base class. See [`FoundryScenario`](../../../pyrit/scenario/scenarios/foundry_scenario.py) for an example.
 #
 # ## Using Scenarios
 #
@@ -54,8 +54,9 @@
 
 # %%
 from pyrit.prompt_target import OpenAIChatTarget
-from pyrit.scenarios import FoundryScenario, FoundryStrategy, ScenarioCompositeStrategy
-from pyrit.scenarios.printer.console_printer import ConsoleScenarioResultPrinter
+from pyrit.scenario.core import ScenarioCompositeStrategy
+from pyrit.scenario.printer.console_printer import ConsoleScenarioResultPrinter
+from pyrit.scenario.scenarios import FoundryScenario, FoundryStrategy
 from pyrit.setup import IN_MEMORY, initialize_pyrit
 
 initialize_pyrit(

doc/code/setup/2_resiliency.ipynb

@@ -156,7 +156,7 @@
    ],
    "source": [
     "from pyrit.prompt_target import OpenAIChatTarget\n",
-    "from pyrit.scenarios import FoundryScenario, FoundryStrategy\n",
+    "from pyrit.scenario import FoundryScenario, FoundryStrategy\n",
     "from pyrit.setup import IN_MEMORY, initialize_pyrit\n",
     "\n",
     "initialize_pyrit(memory_db_type=IN_MEMORY)\n",

doc/code/setup/2_resiliency.py

@@ -124,7 +124,7 @@
 
 # %%
 from pyrit.prompt_target import OpenAIChatTarget
-from pyrit.scenarios import FoundryScenario, FoundryStrategy
+from pyrit.scenario import FoundryScenario, FoundryStrategy
 from pyrit.setup import IN_MEMORY, initialize_pyrit
 
 initialize_pyrit(memory_db_type=IN_MEMORY)

pyrit/cli/frontend_core.py

@@ -214,7 +214,7 @@ async def run_scenario_async(
     Note:
         Initializers from PyRITContext will be run before the scenario executes.
     """
-    from pyrit.scenarios.printer.console_printer import ConsoleScenarioResultPrinter
+    from pyrit.scenario.printer.console_printer import ConsoleScenarioResultPrinter
     from pyrit.setup import initialize_pyrit
 
     # Ensure context is initialized first (loads registries)

pyrit/cli/pyrit_shell.py

@@ -296,7 +296,7 @@ def do_print_scenario(self, arg):
                 print(f"\n{'#' * 80}")
                 print(f"Scenario Run #{idx}: {command}")
                 print(f"{'#' * 80}")
-                from pyrit.scenarios.printer.console_printer import (
+                from pyrit.scenario.printer.console_printer import (
                     ConsoleScenarioResultPrinter,
                 )
 
@@ -313,7 +313,7 @@ def do_print_scenario(self, arg):
                 command, result = self._scenario_history[scenario_num - 1]
                 print(f"\nScenario Run #{scenario_num}: {command}")
                 print("=" * 80)
-                from pyrit.scenarios.printer.console_printer import (
+                from pyrit.scenario.printer.console_printer import (
                     ConsoleScenarioResultPrinter,
                 )
 

pyrit/cli/scenario_registry.py

@@ -7,7 +7,7 @@
 Scenario registry for discovering and instantiating PyRIT scenarios.
 
 This module provides functionality to discover all available Scenario subclasses
-from the pyrit.scenarios.scenarios module and from user-defined initialization scripts.
+from the pyrit.scenario.scenarios module and from user-defined initialization scripts.
 
 PERFORMANCE OPTIMIZATION:
 This module uses lazy imports to minimize overhead during CLI operations:
@@ -35,7 +35,7 @@
 # Lazy import to avoid loading heavy scenario modules when just listing scenarios
 if TYPE_CHECKING:
     from pyrit.cli.frontend_core import ScenarioInfo
-    from pyrit.scenarios.scenario import Scenario
+    from pyrit.scenario.core import Scenario
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +45,7 @@ class ScenarioRegistry:
     Registry for discovering and managing available scenarios.
 
     This class discovers all Scenario subclasses from:
-    1. Built-in scenarios in pyrit.scenarios.scenarios module
+    1. Built-in scenarios in pyrit.scenario.scenarios module
     2. User-defined scenarios from initialization scripts (set via globals)
 
     Scenarios are identified by their simple name (e.g., "encoding_scenario", "foundry_scenario").
@@ -65,15 +65,15 @@ def _ensure_discovered(self) -> None:
 
     def _discover_builtin_scenarios(self) -> None:
         """
-        Discover all built-in scenarios from pyrit.scenarios.scenarios module.
+        Discover all built-in scenarios from pyrit.scenario.scenarios module.
 
         This method dynamically imports all modules in the scenarios package
         and registers any Scenario subclasses found.
         """
-        from pyrit.scenarios.scenario import Scenario
+        from pyrit.scenario.core import Scenario
 
         try:
-            import pyrit.scenarios.scenarios as scenarios_package
+            import pyrit.scenario.scenarios as scenarios_package
 
             # Get the path to the scenarios package
             package_file = scenarios_package.__file__
@@ -94,7 +94,7 @@ def _discover_builtin_scenarios(self) -> None:
 
                 try:
                     # Import the module
-                    full_module_name = f"pyrit.scenarios.scenarios.{module_name}"
+                    full_module_name = f"pyrit.scenario.scenarios.{module_name}"
                     module = importlib.import_module(full_module_name)
 
                     # Find all Scenario subclasses in the module
@@ -121,7 +121,7 @@ def discover_user_scenarios(self) -> None:
 
         User scenarios will override built-in scenarios with the same name.
         """
-        from pyrit.scenarios.scenario import Scenario
+        from pyrit.scenario.core import Scenario
 
         try:
             # Check the global namespace for Scenario subclasses
@@ -142,8 +142,8 @@ def discover_user_scenarios(self) -> None:
                 # Look for Scenario subclasses in the module
                 for name, obj in inspect.getmembers(module, inspect.isclass):
                     if issubclass(obj, Scenario) and obj is not Scenario:
-                        # Check if this is a user-defined class (not from pyrit.scenarios.scenarios)
-                        if not obj.__module__.startswith("pyrit.scenarios.scenarios"):
+                        # Check if this is a user-defined class (not from pyrit.scenario.scenarios)
+                        if not obj.__module__.startswith("pyrit.scenario.scenarios"):
                             # Convert class name to snake_case for scenario name
                             scenario_name = self._class_name_to_scenario_name(obj.__name__)
                             self._scenarios[scenario_name] = obj

pyrit/scenarios/__init__.py → pyrit/scenario/__init__.py

@@ -3,14 +3,17 @@
 
 """High-level scenario classes for running attack configurations."""
 
-from pyrit.scenarios.atomic_attack import AtomicAttack
-from pyrit.scenarios.scenario import Scenario
+from pyrit.scenario.core import AtomicAttack, Scenario, ScenarioCompositeStrategy, ScenarioStrategy
 from pyrit.models.scenario_result import ScenarioIdentifier, ScenarioResult
 
-from pyrit.scenarios.scenarios.airt.cyber_scenario import CyberScenario, CyberStrategy
-from pyrit.scenarios.scenarios.encoding_scenario import EncodingScenario, EncodingStrategy
-from pyrit.scenarios.scenarios.foundry_scenario import FoundryStrategy, FoundryScenario
-from pyrit.scenarios.scenario_strategy import ScenarioCompositeStrategy, ScenarioStrategy
+from pyrit.scenario.scenarios import (
+    CyberScenario,
+    CyberStrategy,
+    EncodingScenario,
+    EncodingStrategy,
+    FoundryStrategy,
+    FoundryScenario,
+)
 
 __all__ = [
     "AtomicAttack",

pyrit/scenario/core/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Core scenario classes for running attack configurations."""
+
+from pyrit.scenario.core.atomic_attack import AtomicAttack
+from pyrit.scenario.core.scenario import Scenario
+from pyrit.scenario.core.scenario_strategy import ScenarioStrategy, ScenarioCompositeStrategy
+
+
+__all__ = [
+    "AtomicAttack",
+    "Scenario",
+    "ScenarioCompositeStrategy",
+    "ScenarioStrategy",
+]

pyrit/scenarios/atomic_attack.py → pyrit/scenario/core/atomic_attack.py

@@ -41,7 +41,7 @@ class AtomicAttack:
     seed_groups. For multi-turn attacks, you can provide custom_prompts.
 
     Example:
-        >>> from pyrit.scenarios import AtomicAttack
+        >>> from pyrit.scenario import AtomicAttack
         >>> from pyrit.attacks import PromptAttack
         >>> from pyrit.prompt_target import OpenAIChatTarget
         >>>

pyrit/scenarios/scenario.py → pyrit/scenario/core/scenario.py

@@ -23,8 +23,8 @@
 from pyrit.models import AttackResult
 from pyrit.models.scenario_result import ScenarioIdentifier, ScenarioResult
 from pyrit.prompt_target import PromptTarget
-from pyrit.scenarios.atomic_attack import AtomicAttack
-from pyrit.scenarios.scenario_strategy import (
+from pyrit.scenario.core.atomic_attack import AtomicAttack
+from pyrit.scenario.core.scenario_strategy import (
     ScenarioCompositeStrategy,
     ScenarioStrategy,
 )
@@ -41,7 +41,7 @@ class Scenario(ABC):
     aggregates the results into a ScenarioResult.
 
     Example:
-        >>> from pyrit.scenarios import Scenario, AtomicAttack
+        >>> from pyrit.scenario import Scenario, AtomicAttack
         >>> from pyrit.executor.attack.single_turn.prompt_sending import PromptSendingAttack
         >>> from pyrit.prompt_target import OpenAIChatTarget
         >>> from pyrit.prompt_converter import Base64Converter
@@ -236,8 +236,8 @@ async def initialize_async(
             memory_labels (Optional[Dict[str, str]]): Additional labels to apply to all
                 attack runs in the scenario. These help track and categorize the scenario.
 
-        Returns:
-            None
+        Raises:
+            ValueError: If no objective_target is provided.
 
         Example:
             >>> # New scenario
@@ -552,6 +552,7 @@ async def run_async(self) -> ScenarioResult:
             ValueError: If the scenario has no atomic attacks configured. If your scenario
                 requires initialization, call await scenario.initialize() first.
             ValueError: If the scenario raises an exception after exhausting all retry attempts.
+            RuntimeError: If the scenario fails for any other reason while executing.
 
         Example:
             >>> result = await scenario.run_async()
@@ -610,7 +611,7 @@ async def run_async(self) -> ScenarioResult:
 
     async def _execute_scenario_async(self) -> ScenarioResult:
         """
-        Internal method that performs a single execution attempt of the scenario.
+        Perform a single execution attempt of the scenario.
 
         This method contains the core execution logic and can be called multiple times
         for retry attempts. It increments the try counter, executes remaining atomic attacks,
@@ -621,6 +622,8 @@ async def _execute_scenario_async(self) -> ScenarioResult:
 
         Raises:
             Exception: Any exception that occurs during scenario execution.
+            ValueError: If a lookup for a scenario for a given ID fails.
+            ValueError: If atomic attack execution fails.
         """
         logger.info(f"Starting scenario '{self._name}' execution with {len(self._atomic_attacks)} atomic attacks")