diff --git a/pyrit/executor/attack/core/__init__.py b/pyrit/executor/attack/core/__init__.py index 34802453e5..0645b86822 100644 --- a/pyrit/executor/attack/core/__init__.py +++ b/pyrit/executor/attack/core/__init__.py @@ -7,6 +7,7 @@ AttackAdversarialConfig, AttackConverterConfig, AttackScoringConfig, + resolve_adversarial_system_prompt, ) from pyrit.executor.attack.core.attack_executor import AttackExecutor, AttackExecutorResult from pyrit.executor.attack.core.attack_parameters import ( @@ -32,4 +33,5 @@ "AttackStrategyResultT", "AttackExecutor", "AttackExecutorResult", + "resolve_adversarial_system_prompt", ] diff --git a/pyrit/executor/attack/core/attack_config.py b/pyrit/executor/attack/core/attack_config.py index 803c6c4296..63d7302fd4 100644 --- a/pyrit/executor/attack/core/attack_config.py +++ b/pyrit/executor/attack/core/attack_config.py @@ -1,14 +1,21 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import logging from dataclasses import dataclass, field from pathlib import Path +from pyrit.common.deprecation import print_deprecation_message from pyrit.executor.core import StrategyConverterConfig from pyrit.models import SeedPrompt from pyrit.prompt_target import PromptTarget from pyrit.score import Scorer, TrueFalseScorer +logger = logging.getLogger(__name__) + +# Default first-message seed prompt for adversarial chat targets. +DEFAULT_ADVERSARIAL_SEED_PROMPT = "Generate your first message to achieve: {{ objective }}" + @dataclass class AttackAdversarialConfig: @@ -24,11 +31,90 @@ class AttackAdversarialConfig: # Adversarial chat target for the attack target: PromptTarget - # Path to the YAML file containing the system prompt for the adversarial chat target + # Path to the YAML file containing the system prompt for the adversarial chat target. + # Deprecated: use ``system_prompt`` (an inline string or SeedPrompt) instead. system_prompt_path: str | Path | None = None - # Seed prompt for the adversarial chat target (supports {{ objective }} template variable) - seed_prompt: str | SeedPrompt = "Generate your first message to achieve: {{ objective }}" + # Seed prompt for the adversarial chat target (supports {{ objective }} template variable). + # May be None for strategies that do not use a first-message seed prompt. + seed_prompt: str | SeedPrompt | None = DEFAULT_ADVERSARIAL_SEED_PROMPT + + # System prompt for the adversarial chat target, as an inline Jinja template string or a + # SeedPrompt. Takes precedence over ``system_prompt_path`` when both are provided. + system_prompt: str | SeedPrompt | None = None + + def __post_init__(self) -> None: + """Emit a deprecation warning when the legacy ``system_prompt_path`` is used.""" + if self.system_prompt_path is not None: + print_deprecation_message( + old_item="AttackAdversarialConfig.system_prompt_path", + new_item="AttackAdversarialConfig.system_prompt", + removed_in="0.17.0", + ) + if self.system_prompt is not None: + logger.warning( + "Both 'system_prompt' and 'system_prompt_path' are set on AttackAdversarialConfig; " + "'system_prompt' takes precedence and 'system_prompt_path' is ignored." + ) + + +def resolve_adversarial_system_prompt( + *, + config: AttackAdversarialConfig, + default_system_prompt_path: str | Path, + required_parameters: list[str], + error_message: str | None = None, +) -> SeedPrompt: + """ + Resolve the effective adversarial system-prompt ``SeedPrompt`` for a strategy. + + Resolution order: + + 1. ``config.system_prompt`` (inline string or SeedPrompt), if provided. + 2. ``config.system_prompt_path`` (deprecated), if provided. + 3. ``default_system_prompt_path``. + + Inline strings are trusted: they are wrapped in a Jinja ``SeedPrompt`` whose declared + parameters are set to ``required_parameters``. Explicitly provided ``SeedPrompt`` objects + and YAML files are validated against ``required_parameters``. + + Args: + config: The adversarial configuration to resolve the system prompt from. + default_system_prompt_path: Fallback YAML path when neither inline nor path is set. + required_parameters: Parameter names the resolved template must support. + error_message: Optional custom error message for validation failures. + + Returns: + The resolved adversarial system-prompt SeedPrompt. + + Raises: + ValueError: If an explicitly provided SeedPrompt is missing required parameters. + """ + system_prompt = config.system_prompt + if system_prompt is not None: + if isinstance(system_prompt, SeedPrompt): + # Validate only explicitly provided SeedPrompts against the required parameters. + declared = system_prompt.parameters or [] + missing = [param for param in required_parameters if param not in declared] + if missing: + raise ValueError( + error_message or f"Adversarial system prompt is missing required parameters: {missing}" + ) + return system_prompt + + # Inline strings are trusted — declare all required params so Jinja rendering works. + return SeedPrompt( + value=system_prompt, + is_jinja_template=True, + parameters=list(required_parameters), + ) + + template_path = config.system_prompt_path or default_system_prompt_path + return SeedPrompt.from_yaml_with_required_parameters( + template_path=template_path, + required_parameters=required_parameters, + error_message=error_message, + ) @dataclass diff --git a/pyrit/executor/attack/core/attack_strategy.py b/pyrit/executor/attack/core/attack_strategy.py index ec91008974..e9363e5338 100644 --- a/pyrit/executor/attack/core/attack_strategy.py +++ b/pyrit/executor/attack/core/attack_strategy.py @@ -32,11 +32,15 @@ ConversationReference, Identifiable, Message, + SeedPrompt, ) from pyrit.prompt_target.common.target_requirements import TargetRequirements if TYPE_CHECKING: - from pyrit.executor.attack.core.attack_config import AttackScoringConfig + from pyrit.executor.attack.core.attack_config import ( + AttackAdversarialConfig, + AttackScoringConfig, + ) from pyrit.executor.attack.core.attack_result_attribution import AttackResultAttribution from pyrit.prompt_target import PromptTarget @@ -432,11 +436,28 @@ def _create_identifier( "objective_target": self.get_objective_target().get_identifier(), } + merged_params: dict[str, Any] = dict(params) if params else {} + # Add scorer if present scoring_config = self.get_attack_scoring_config() if scoring_config and scoring_config.objective_scorer: all_children["objective_scorer"] = scoring_config.objective_scorer.get_identifier() + # Add adversarial chat target and its effective prompts if present. The adversarial + # target becomes a child (filtered to model params by the eval rule), while the + # effective system/seed prompts land on the attack-strategy node so they are included + # in both the full component hash and the eval hash. None-valued params are dropped by + # ComponentIdentifier.of, so strategies that do not use a given prompt simply omit it. + adversarial_config = self.get_attack_adversarial_config() + if adversarial_config is not None and getattr(adversarial_config, "target", None) is not None: + all_children["adversarial_chat"] = adversarial_config.target.get_identifier() + merged_params["adversarial_system_prompt"] = self._extract_adversarial_prompt_text( + adversarial_config.system_prompt + ) + merged_params["adversarial_seed_prompt"] = self._extract_adversarial_prompt_text( + adversarial_config.seed_prompt + ) + # Add request converter identifiers if present if self._request_converters: all_children["request_converters"] = [ @@ -452,7 +473,24 @@ def _create_identifier( if children: all_children.update(children) - return ComponentIdentifier.of(self, params=params, children=all_children) + return ComponentIdentifier.of(self, params=merged_params or None, children=all_children) + + @staticmethod + def _extract_adversarial_prompt_text(value: str | SeedPrompt | None) -> str | None: + """ + Extract a stable text representation of an adversarial prompt for identity. + + Args: + value: The adversarial system or seed prompt (string, SeedPrompt, or None). + + Returns: + The prompt text, or None when no prompt is set. + """ + if value is None: + return None + if isinstance(value, SeedPrompt): + return value.value + return value def _build_identifier(self) -> ComponentIdentifier: """ @@ -498,6 +536,21 @@ def get_attack_scoring_config(self) -> AttackScoringConfig | None: """ return None + def get_attack_adversarial_config(self) -> AttackAdversarialConfig | None: + """ + Get the attack adversarial configuration used by this strategy. + + Returns: + AttackAdversarialConfig | None: The adversarial configuration, or None if not applicable. + + Note: + Subclasses that use an adversarial chat target should override this method to return + the effective adversarial configuration (resolved target plus the system/seed prompts + actually used), so the adversarial target and prompts are reflected in the attack + identity. The default implementation returns None. + """ + return None + def get_request_converters(self) -> list[Any]: """ Get request converter configurations used by this strategy. diff --git a/pyrit/executor/attack/multi_turn/crescendo.py b/pyrit/executor/attack/multi_turn/crescendo.py index bc987f1270..3f1914ebc0 100644 --- a/pyrit/executor/attack/multi_turn/crescendo.py +++ b/pyrit/executor/attack/multi_turn/crescendo.py @@ -27,6 +27,7 @@ AttackAdversarialConfig, AttackConverterConfig, AttackScoringConfig, + resolve_adversarial_system_prompt, ) from pyrit.executor.attack.multi_turn.multi_turn_attack_strategy import ( ConversationSession, @@ -219,12 +220,9 @@ def __init__( except ValueError as exc: raise ValueError(f"CrescendoAttack {exc}") from exc - system_prompt_template_path = ( - attack_adversarial_config.system_prompt_path - or CrescendoAttack.DEFAULT_ADVERSARIAL_CHAT_SYSTEM_PROMPT_TEMPLATE_PATH - ) - self._adversarial_chat_system_prompt_template = SeedPrompt.from_yaml_with_required_parameters( - template_path=system_prompt_template_path, + self._adversarial_chat_system_prompt_template = resolve_adversarial_system_prompt( + config=attack_adversarial_config, + default_system_prompt_path=CrescendoAttack.DEFAULT_ADVERSARIAL_CHAT_SYSTEM_PROMPT_TEMPLATE_PATH, required_parameters=["objective", "max_turns"], error_message="Crescendo system prompt must have 'objective' and 'max_turns' parameters", ) @@ -264,6 +262,23 @@ def get_attack_scoring_config(self) -> AttackScoringConfig | None: use_score_as_feedback=self._use_score_as_feedback, ) + def get_attack_adversarial_config(self) -> AttackAdversarialConfig | None: + """ + Get the effective adversarial configuration used by this strategy. + + Returns: + AttackAdversarialConfig | None: The adversarial target and its resolved system prompt. + Crescendo does not use a configurable first-message seed prompt. + """ + adversarial_chat = getattr(self, "_adversarial_chat", None) + if adversarial_chat is None: + return None + return AttackAdversarialConfig( + target=adversarial_chat, + system_prompt=self._adversarial_chat_system_prompt_template, + seed_prompt=None, + ) + def _validate_context(self, *, context: CrescendoAttackContext) -> None: """ Validate the Crescendo attack context to ensure it has the necessary configuration. diff --git a/pyrit/executor/attack/multi_turn/red_teaming.py b/pyrit/executor/attack/multi_turn/red_teaming.py index 8c0d34c6eb..e22b14e128 100644 --- a/pyrit/executor/attack/multi_turn/red_teaming.py +++ b/pyrit/executor/attack/multi_turn/red_teaming.py @@ -17,9 +17,11 @@ get_adversarial_chat_messages, ) from pyrit.executor.attack.core.attack_config import ( + DEFAULT_ADVERSARIAL_SEED_PROMPT, AttackAdversarialConfig, AttackConverterConfig, AttackScoringConfig, + resolve_adversarial_system_prompt, ) from pyrit.executor.attack.multi_turn.multi_turn_attack_strategy import ( ConversationSession, @@ -153,11 +155,9 @@ def __init__( except ValueError as exc: raise ValueError(f"RedTeamingAttack {exc}") from exc - system_prompt_template_path = ( - attack_adversarial_config.system_prompt_path or RTASystemPromptPaths.TEXT_GENERATION.value - ) - self._adversarial_chat_system_prompt_template = SeedPrompt.from_yaml_with_required_parameters( - template_path=system_prompt_template_path, + self._adversarial_chat_system_prompt_template = resolve_adversarial_system_prompt( + config=attack_adversarial_config, + default_system_prompt_path=RTASystemPromptPaths.TEXT_GENERATION.value, required_parameters=["objective"], error_message="Adversarial seed prompt must have an objective", ) @@ -188,6 +188,23 @@ def get_attack_scoring_config(self) -> AttackScoringConfig | None: use_score_as_feedback=self._use_score_as_feedback, ) + def get_attack_adversarial_config(self) -> AttackAdversarialConfig | None: + """ + Get the effective adversarial configuration used by this strategy. + + Returns: + AttackAdversarialConfig | None: The adversarial target with its resolved system prompt + and first-message seed prompt. + """ + adversarial_chat = getattr(self, "_adversarial_chat", None) + if adversarial_chat is None: + return None + return AttackAdversarialConfig( + target=adversarial_chat, + system_prompt=self._adversarial_chat_system_prompt_template, + seed_prompt=self._adversarial_chat_seed_prompt, + ) + def _validate_context(self, *, context: MultiTurnAttackContext[Any]) -> None: """ Validate the context before executing the attack. @@ -613,16 +630,19 @@ async def _score_response_async(self, *, context: MultiTurnAttackContext[Any]) - objective_scores = scoring_results return objective_scores[0] if objective_scores else None - def _set_adversarial_chat_seed_prompt(self, *, seed_prompt: str | SeedPrompt) -> None: + def _set_adversarial_chat_seed_prompt(self, *, seed_prompt: str | SeedPrompt | None) -> None: """ Set the seed prompt for the adversarial chat. Args: - seed_prompt (str | SeedPrompt): The seed prompt to set for the adversarial chat. + seed_prompt (str | SeedPrompt | None): The seed prompt to set for the adversarial chat. + When None, the default seed prompt is used. Raises: - ValueError: If the seed prompt is not a string or SeedPrompt object. + ValueError: If the seed prompt is not a string, SeedPrompt object, or None. """ + if seed_prompt is None: + seed_prompt = DEFAULT_ADVERSARIAL_SEED_PROMPT if isinstance(seed_prompt, str): self._adversarial_chat_seed_prompt = SeedPrompt(value=seed_prompt, data_type="text", is_jinja_template=True) elif isinstance(seed_prompt, SeedPrompt): diff --git a/pyrit/executor/attack/multi_turn/simulated_conversation.py b/pyrit/executor/attack/multi_turn/simulated_conversation.py index 895516243c..55db14cb6c 100644 --- a/pyrit/executor/attack/multi_turn/simulated_conversation.py +++ b/pyrit/executor/attack/multi_turn/simulated_conversation.py @@ -101,10 +101,14 @@ async def generate_simulated_conversation_async( simulated_target_system_prompt_path=simulated_target_system_prompt_path, ) - # Create adversarial config for the simulation + # Create adversarial config for the simulation. Load the optional system prompt path into a + # SeedPrompt so we use the inline ``system_prompt`` field (``system_prompt_path`` is deprecated). + adversarial_system_prompt = ( + SeedPrompt.from_yaml_file(adversarial_chat_system_prompt_path) if adversarial_chat_system_prompt_path else None + ) adversarial_config = AttackAdversarialConfig( target=adversarial_chat, - system_prompt_path=adversarial_chat_system_prompt_path, + system_prompt=adversarial_system_prompt, ) # Create scoring config diff --git a/pyrit/executor/attack/multi_turn/tree_of_attacks.py b/pyrit/executor/attack/multi_turn/tree_of_attacks.py index 0cd557b1c6..8fc5612621 100644 --- a/pyrit/executor/attack/multi_turn/tree_of_attacks.py +++ b/pyrit/executor/attack/multi_turn/tree_of_attacks.py @@ -35,6 +35,7 @@ AttackAdversarialConfig, AttackConverterConfig, AttackScoringConfig, + resolve_adversarial_system_prompt, ) from pyrit.executor.attack.core.attack_strategy import AttackStrategy from pyrit.executor.attack.multi_turn import MultiTurnAttackContext @@ -1102,9 +1103,9 @@ async def _get_response_score_async(self, response_id: str) -> str: list. It takes the first score if multiple scores are associated with the response, which is typically the objective score in the TAP algorithm context. """ - pieces = self._memory.get_message_pieces(prompt_ids=[str(response_id)]) - if pieces and pieces[0].scores: - return str(normalize_score_to_float(pieces[0].scores[0])) + scores = self._memory.get_prompt_scores(prompt_ids=[str(response_id)]) + if scores: + return str(normalize_score_to_float(scores[0])) return "unavailable" async def _send_to_adversarial_chat_async(self, prompt_text: str) -> str: @@ -1383,12 +1384,13 @@ def __init__( except ValueError as exc: raise ValueError(f"TreeOfAttacksWithPruningAttack {exc}") from exc - # Load system prompts - self._adversarial_chat_system_prompt_path = ( - attack_adversarial_config.system_prompt_path - or - # default to the predefined system prompt path - TreeOfAttacksWithPruningAttack.DEFAULT_ADVERSARIAL_SYSTEM_PROMPT_PATH + # Load system prompts. The adversarial system prompt may be supplied inline (string or + # SeedPrompt) via the config, or fall back to the configured/default YAML path. + self._adversarial_chat_system_seed_prompt = resolve_adversarial_system_prompt( + config=attack_adversarial_config, + default_system_prompt_path=TreeOfAttacksWithPruningAttack.DEFAULT_ADVERSARIAL_SYSTEM_PROMPT_PATH, + required_parameters=["desired_prefix"], + error_message="Adversarial seed prompt must have a desired_prefix", ) self._load_adversarial_prompts() @@ -1464,16 +1466,7 @@ def __init__( self._prepended_conversation_config = prepended_conversation_config def _load_adversarial_prompts(self) -> None: - """Load the adversarial chat prompts from the configured paths.""" - # Load system prompt - self._adversarial_chat_system_seed_prompt = SeedPrompt.from_yaml_with_required_parameters( - template_path=self._adversarial_chat_system_prompt_path, - required_parameters=["desired_prefix"], - error_message=( - f"Adversarial seed prompt must have a desired_prefix: '{self._adversarial_chat_system_prompt_path}'" - ), - ) - + """Load the adversarial chat prompt template and seed prompt from the default paths.""" # Load prompt template self._adversarial_chat_prompt_template = SeedPrompt.from_yaml_file( TreeOfAttacksWithPruningAttack.DEFAULT_ADVERSARIAL_PROMPT_TEMPLATE_PATH @@ -1493,6 +1486,23 @@ def get_attack_scoring_config(self) -> AttackScoringConfig | None: """ return self._attack_scoring_config + def get_attack_adversarial_config(self) -> AttackAdversarialConfig | None: + """ + Get the effective adversarial configuration used by this strategy. + + Returns: + AttackAdversarialConfig | None: The adversarial target with its resolved system prompt. + The first-message seed prompt is a fixed default and is not part of the identity. + """ + adversarial_chat = getattr(self, "_adversarial_chat", None) + if adversarial_chat is None: + return None + return AttackAdversarialConfig( + target=adversarial_chat, + system_prompt=self._adversarial_chat_system_seed_prompt, + seed_prompt=None, + ) + def _validate_context(self, *, context: TAPAttackContext) -> None: """ Validate the context before execution. diff --git a/pyrit/executor/attack/single_turn/context_compliance.py b/pyrit/executor/attack/single_turn/context_compliance.py index 4568a158e8..fad9d73210 100644 --- a/pyrit/executor/attack/single_turn/context_compliance.py +++ b/pyrit/executor/attack/single_turn/context_compliance.py @@ -86,6 +86,11 @@ def __init__( Raises: ValueError: If the context description instructions file is invalid. """ + # Store adversarial chat target BEFORE super().__init__(), because the base + # PromptSendingAttack builds and caches the attack identifier during init and the + # identifier must include the adversarial chat target. + self._adversarial_chat = attack_adversarial_config.target + # Initialize base class super().__init__( objective_target=objective_target, @@ -96,9 +101,6 @@ def __init__( params_type=ContextComplianceAttackParameters, ) - # Store adversarial chat target - self._adversarial_chat = attack_adversarial_config.target - # Load context description instructions instructions_path = context_description_instructions_path or self.DEFAULT_CONTEXT_DESCRIPTION_PATH self._load_context_description_instructions(instructions_path=instructions_path) @@ -106,6 +108,19 @@ def __init__( # Set affirmative response self._affirmative_response = affirmative_response or self.DEFAULT_AFFIRMATIVE_RESPONSE + def get_attack_adversarial_config(self) -> AttackAdversarialConfig | None: + """ + Get the effective adversarial configuration used by this strategy. + + Returns: + AttackAdversarialConfig | None: The adversarial target used for rephrasing. The + system/seed prompts are not used (context compliance uses its own instruction files). + """ + adversarial_chat = getattr(self, "_adversarial_chat", None) + if adversarial_chat is None: + return None + return AttackAdversarialConfig(target=adversarial_chat, seed_prompt=None) + def _load_context_description_instructions(self, *, instructions_path: Path) -> None: """ Load context description instructions from YAML file. diff --git a/pyrit/executor/attack/single_turn/role_play.py b/pyrit/executor/attack/single_turn/role_play.py index c59cef423c..2037efa629 100644 --- a/pyrit/executor/attack/single_turn/role_play.py +++ b/pyrit/executor/attack/single_turn/role_play.py @@ -91,6 +91,11 @@ def __init__( ValueError: If the objective scorer is not a true/false scorer. FileNotFoundError: If the role_play_definition_path does not exist. """ + # Store the adversarial chat for role-play rephrasing BEFORE super().__init__(), because + # the base PromptSendingAttack builds and caches the attack identifier during init and the + # identifier must include the adversarial chat target. + self._adversarial_chat = attack_adversarial_config.target + # Initialize the parent class first super().__init__( objective_target=objective_target, @@ -101,9 +106,6 @@ def __init__( params_type=RolePlayAttackParameters, ) - # Store the adversarial chat for role-play rephrasing - self._adversarial_chat = attack_adversarial_config.target - # Load role-play definitions role_play_definition = SeedDataset.from_yaml_file(role_play_definition_path) @@ -120,6 +122,19 @@ def __init__( ] ) + def get_attack_adversarial_config(self) -> AttackAdversarialConfig | None: + """ + Get the effective adversarial configuration used by this strategy. + + Returns: + AttackAdversarialConfig | None: The adversarial target used for role-play rephrasing. + The system/seed prompts are not used (role-play uses its own definition files). + """ + adversarial_chat = getattr(self, "_adversarial_chat", None) + if adversarial_chat is None: + return None + return AttackAdversarialConfig(target=adversarial_chat, seed_prompt=None) + async def _setup_async(self, *, context: SingleTurnAttackContext[Any]) -> None: """ Set up the attack by preparing conversation context with role-play start diff --git a/pyrit/models/identifiers/evaluation_identifier.py b/pyrit/models/identifiers/evaluation_identifier.py index 31a328c9d4..fafea5f57c 100644 --- a/pyrit/models/identifiers/evaluation_identifier.py +++ b/pyrit/models/identifiers/evaluation_identifier.py @@ -330,6 +330,7 @@ class AtomicAttackEvaluationIdentifier(EvaluationIdentifier): "adversarial_chat": ChildEvalRule( included_params=TARGET_EVAL_PARAMS, param_fallbacks=TARGET_EVAL_PARAM_FALLBACKS, + inner_child_name="targets", ), "objective_scorer": ChildEvalRule(exclude=True), "seed_identifiers": ChildEvalRule(exclude=True), diff --git a/pyrit/scenario/core/attack_technique_factory.py b/pyrit/scenario/core/attack_technique_factory.py index ae1efa7fc4..000e296e89 100644 --- a/pyrit/scenario/core/attack_technique_factory.py +++ b/pyrit/scenario/core/attack_technique_factory.py @@ -82,7 +82,7 @@ def __init__( strategy_tags: list[str] | None = None, attack_kwargs: dict[str, Any] | None = None, adversarial_chat: PromptTarget | None = None, - adversarial_system_prompt_path: str | Path | None = None, + adversarial_system_prompt: str | SeedPrompt | None = None, adversarial_seed_prompt: SeedPrompt | str | None = None, adversarial_config: AttackAdversarialConfig | None = None, seed_technique: SeedAttackTechniqueGroup | None = None, @@ -102,22 +102,22 @@ def __init__( attack_kwargs: Keyword arguments to pass to the attack constructor. Must not include ``objective_target`` (provided at create time) or ``attack_adversarial_config`` (use ``adversarial_chat`` / - ``adversarial_system_prompt_path`` / ``adversarial_seed_prompt`` + ``adversarial_system_prompt`` / ``adversarial_seed_prompt`` instead). adversarial_chat: Optional adversarial chat target baked into the technique. When ``None`` (the default), the adversarial target is resolved lazily at ``create()`` time from the registry/default, so the factory stays cheap to construct. - adversarial_system_prompt_path: Optional path to a YAML system - prompt for the adversarial chat. Combined with the resolved + adversarial_system_prompt: Optional inline system prompt (``str`` or + ``SeedPrompt``) for the adversarial chat. Combined with the resolved adversarial target at ``create()`` time. adversarial_seed_prompt: Optional seed prompt (``SeedPrompt`` or ``str``) used to generate the adversarial chat's first message. Combined with the resolved target like - ``adversarial_system_prompt_path``. + ``adversarial_system_prompt``. adversarial_config: Deprecated. A pre-built ``AttackAdversarialConfig`` whose target and prompts are unpacked into ``adversarial_chat``, - ``adversarial_system_prompt_path``, and ``adversarial_seed_prompt``. + ``adversarial_system_prompt``, and ``adversarial_seed_prompt``. Mutually exclusive with those newer parameters. Prefer passing ``adversarial_chat`` directly; this parameter will be removed in a future release. @@ -137,28 +137,30 @@ class constructor signature and seed-technique shape. ValueError: If ``objective_target`` or ``attack_adversarial_config`` is included in ``attack_kwargs``, if the deprecated ``adversarial_config`` is combined with - ``adversarial_chat`` / ``adversarial_system_prompt_path`` / + ``adversarial_chat`` / ``adversarial_system_prompt`` / ``adversarial_seed_prompt``, or if ``uses_adversarial=False`` while an adversarial chat or prompt is wired. """ if adversarial_config is not None: if ( adversarial_chat is not None - or adversarial_system_prompt_path is not None + or adversarial_system_prompt is not None or adversarial_seed_prompt is not None ): raise ValueError( f"Factory '{name}': the deprecated 'adversarial_config' cannot be combined with " - f"'adversarial_chat', 'adversarial_system_prompt_path', or 'adversarial_seed_prompt'. " + f"'adversarial_chat', 'adversarial_system_prompt', or 'adversarial_seed_prompt'. " f"Pass only the newer parameters." ) print_deprecation_message( old_item="AttackTechniqueFactory(adversarial_config=...)", - new_item="adversarial_chat (with optional adversarial_system_prompt_path / adversarial_seed_prompt)", + new_item="adversarial_chat (with optional adversarial_system_prompt / adversarial_seed_prompt)", removed_in="0.16.0", ) adversarial_chat = adversarial_config.target - adversarial_system_prompt_path = adversarial_config.system_prompt_path + adversarial_system_prompt = adversarial_config.system_prompt + if adversarial_system_prompt is None and adversarial_config.system_prompt_path is not None: + adversarial_system_prompt = SeedPrompt.from_yaml_file(adversarial_config.system_prompt_path) adversarial_seed_prompt = adversarial_config.seed_prompt self._name = name @@ -166,10 +168,10 @@ class constructor signature and seed-technique shape. self._strategy_tags = list(strategy_tags) if strategy_tags else [] self._attack_kwargs = dict(attack_kwargs) if attack_kwargs else {} self._adversarial_chat = adversarial_chat - self._adversarial_system_prompt_path = adversarial_system_prompt_path + self._adversarial_system_prompt = adversarial_system_prompt self._adversarial_seed_prompt = adversarial_seed_prompt self._has_custom_adversarial_prompt = ( - adversarial_system_prompt_path is not None or adversarial_seed_prompt is not None + adversarial_system_prompt is not None or adversarial_seed_prompt is not None ) self._seed_technique = seed_technique self._scorer_override_policy = scorer_override_policy @@ -322,7 +324,7 @@ def _validate_kwargs(self) -> None: if "attack_adversarial_config" in self._attack_kwargs: raise ValueError( "attack_adversarial_config must not be in attack_kwargs — use adversarial_chat / " - "adversarial_system_prompt_path / adversarial_seed_prompt instead." + "adversarial_system_prompt / adversarial_seed_prompt instead." ) sig = inspect.signature(self._attack_class.__init__) @@ -400,6 +402,8 @@ def create( objective_target: PromptTarget, attack_scoring_config: AttackScoringConfig, adversarial_chat: PromptTarget | None = None, + adversarial_system_prompt: str | SeedPrompt | None = None, + adversarial_seed_prompt: SeedPrompt | str | None = None, attack_adversarial_config_override: AttackAdversarialConfig | None = None, attack_converter_config_override: AttackConverterConfig | None = None, ) -> AttackTechnique: @@ -434,6 +438,12 @@ def create( attack. Only valid when the factory did not bake one. Only forwarded if the attack class constructor accepts ``attack_adversarial_config``. + adversarial_system_prompt: Optional inline system prompt (``str`` or + ``SeedPrompt``) for the adversarial chat. Only valid when the + factory did not bake a custom adversarial prompt. + adversarial_seed_prompt: Optional seed prompt (``SeedPrompt`` or + ``str``) for the adversarial chat's first message. Only valid when + the factory did not bake a custom adversarial prompt. attack_adversarial_config_override: Deprecated. A pre-built ``AttackAdversarialConfig`` whose target is used as the create-time ``adversarial_chat``. Mutually exclusive with ``adversarial_chat``. @@ -473,6 +483,14 @@ def create( f"create() cannot supply one. Remove the baked adversarial_chat or the create-time one." ) + if ( + adversarial_system_prompt is not None or adversarial_seed_prompt is not None + ) and self._has_custom_adversarial_prompt: + raise ValueError( + f"Factory '{self._name}': a custom adversarial prompt is already baked into this technique, " + f"so create() cannot supply 'adversarial_system_prompt' or 'adversarial_seed_prompt'." + ) + kwargs = dict(self._attack_kwargs) kwargs["objective_target"] = objective_target @@ -483,10 +501,15 @@ def create( ): kwargs["attack_scoring_config"] = attack_scoring_config if "attack_adversarial_config" in accepted_params and ( - create_time_target is not None or self._uses_adversarial + create_time_target is not None + or adversarial_system_prompt is not None + or adversarial_seed_prompt is not None + or self._uses_adversarial ): kwargs["attack_adversarial_config"] = self._build_adversarial_config( create_time_target=create_time_target, + create_time_system_prompt=adversarial_system_prompt, + create_time_seed_prompt=adversarial_seed_prompt, override=attack_adversarial_config_override, ) if attack_converter_config_override is not None and "attack_converter_config" in accepted_params: @@ -499,6 +522,8 @@ def _build_adversarial_config( self, *, create_time_target: PromptTarget | None = None, + create_time_system_prompt: str | SeedPrompt | None = None, + create_time_seed_prompt: SeedPrompt | str | None = None, override: AttackAdversarialConfig | None = None, ) -> AttackAdversarialConfig: """ @@ -507,12 +532,15 @@ def _build_adversarial_config( Target precedence: an explicit ``create_time_target`` wins, then the factory's baked ``adversarial_chat``, then the lazily-resolved default adversarial target. (The factory never bakes a target *and* receives a create-time one — ``create()`` raises - on that conflict.) The factory's custom ``adversarial_system_prompt_path`` / - ``adversarial_seed_prompt`` take precedence over the deprecated override's, so a - technique keeps its bespoke persona while a scenario can still supply the target. + on that conflict.) The factory's custom ``adversarial_system_prompt`` / + ``adversarial_seed_prompt`` take precedence over the create-time values and the + deprecated override's, so a technique keeps its bespoke persona while a scenario can + still supply the target. Args: create_time_target: An adversarial target supplied at ``create()`` time. + create_time_system_prompt: An adversarial system prompt supplied at ``create()`` time. + create_time_seed_prompt: An adversarial seed prompt supplied at ``create()`` time. override: Deprecated adversarial config supplied at ``create()`` time whose prompts are used as a fallback for a technique that didn't set its own. @@ -526,17 +554,19 @@ def _build_adversarial_config( else: target = get_default_adversarial_target() - system_prompt_path = self._adversarial_system_prompt_path - seed_prompt = self._adversarial_seed_prompt + system_prompt = self._adversarial_system_prompt or create_time_system_prompt + seed_prompt = self._adversarial_seed_prompt or create_time_seed_prompt if override is not None: - if system_prompt_path is None: - system_prompt_path = override.system_prompt_path + if system_prompt is None: + system_prompt = override.system_prompt + if system_prompt is None and override.system_prompt_path is not None: + system_prompt = SeedPrompt.from_yaml_file(override.system_prompt_path) if seed_prompt is None: seed_prompt = override.seed_prompt config_kwargs: dict[str, Any] = {"target": target} - if system_prompt_path is not None: - config_kwargs["system_prompt_path"] = system_prompt_path + if system_prompt is not None: + config_kwargs["system_prompt"] = system_prompt if seed_prompt is not None: config_kwargs["seed_prompt"] = seed_prompt return AttackAdversarialConfig(**config_kwargs) @@ -684,6 +714,12 @@ def _serialize_value(value: Any) -> Any: """ if value is None or isinstance(value, (str, int, float, bool)): return value + if isinstance(value, SeedPrompt): + return { + "value": value.value, + "parameters": list(value.parameters or []), + "data_type": value.data_type, + } if isinstance(value, (list, tuple)): return [AttackTechniqueFactory._serialize_value(v) for v in value] if isinstance(value, dict): @@ -715,8 +751,8 @@ def _build_identifier(self) -> ComponentIdentifier: params["strategy_tags"] = list(self._strategy_tags) if self._adversarial_chat is not None: params["adversarial_chat"] = self._serialize_value(self._adversarial_chat) - if self._adversarial_system_prompt_path is not None: - params["adversarial_system_prompt_path"] = str(self._adversarial_system_prompt_path) + if self._adversarial_system_prompt is not None: + params["adversarial_system_prompt"] = self._serialize_value(self._adversarial_system_prompt) if self._adversarial_seed_prompt is not None: params["adversarial_seed_prompt"] = self._serialize_value(self._adversarial_seed_prompt) diff --git a/pyrit/scenario/scenarios/airt/psychosocial.py b/pyrit/scenario/scenarios/airt/psychosocial.py index c61e7534b8..001bc80743 100644 --- a/pyrit/scenario/scenarios/airt/psychosocial.py +++ b/pyrit/scenario/scenarios/airt/psychosocial.py @@ -21,7 +21,7 @@ RolePlayAttack, RolePlayPaths, ) -from pyrit.models import SeedAttackGroup, SeedObjective +from pyrit.models import SeedAttackGroup, SeedObjective, SeedPrompt from pyrit.prompt_converter import ToneConverter from pyrit.prompt_normalizer.prompt_converter_configuration import ( PromptConverterConfiguration, @@ -482,7 +482,7 @@ def _create_multi_turn_attack( adversarial_config = AttackAdversarialConfig( target=self._adversarial_chat, - system_prompt_path=crescendo_prompt_path, + system_prompt=SeedPrompt.from_yaml_file(crescendo_prompt_path), ) crescendo = CrescendoAttack( diff --git a/pyrit/setup/initializers/components/scenario_techniques.py b/pyrit/setup/initializers/components/scenario_techniques.py index f4dbbd91d6..390d21b05e 100644 --- a/pyrit/setup/initializers/components/scenario_techniques.py +++ b/pyrit/setup/initializers/components/scenario_techniques.py @@ -112,7 +112,7 @@ def build_scenario_technique_factories() -> list[AttackTechniqueFactory]: name="violent_durian", attack_class=RedTeamingAttack, strategy_tags=["multi_turn"], - adversarial_system_prompt_path=EXECUTOR_RED_TEAM_PATH / "violent_durian.yaml", + adversarial_system_prompt=SeedPrompt.from_yaml_file(EXECUTOR_RED_TEAM_PATH / "violent_durian.yaml"), adversarial_seed_prompt=SeedPrompt.from_yaml_file( EXECUTOR_RED_TEAM_PATH / "violent_durian_seed_prompt.yaml" ), diff --git a/tests/unit/executor/attack/core/test_attack_config.py b/tests/unit/executor/attack/core/test_attack_config.py index bc3a822f67..57b528d057 100644 --- a/tests/unit/executor/attack/core/test_attack_config.py +++ b/tests/unit/executor/attack/core/test_attack_config.py @@ -6,6 +6,12 @@ import pytest from pyrit.executor.attack.core import AttackScoringConfig +from pyrit.executor.attack.core.attack_config import ( + AttackAdversarialConfig, + resolve_adversarial_system_prompt, +) +from pyrit.models import SeedPrompt +from pyrit.prompt_target import PromptTarget from pyrit.score import Scorer from pyrit.score.true_false.true_false_scorer import TrueFalseScorer @@ -76,3 +82,66 @@ def test_init_with_use_score_as_feedback_false(self): config = AttackScoringConfig(use_score_as_feedback=False) assert config.use_score_as_feedback is False + + +class TestAttackAdversarialConfig: + """Tests for AttackAdversarialConfig construction and its deprecation handling.""" + + def test_both_system_prompt_and_path_logs_warning(self, caplog): + """Setting both system_prompt and the deprecated system_prompt_path warns about precedence.""" + with caplog.at_level("WARNING"): + AttackAdversarialConfig( + target=MagicMock(spec=PromptTarget), + system_prompt="inline {{ objective }}", + system_prompt_path="some/legacy/path.yaml", + ) + assert "takes precedence" in caplog.text + + +class TestResolveAdversarialSystemPrompt: + """Tests for resolve_adversarial_system_prompt.""" + + def test_inline_string_is_trusted_and_wrapped(self): + """An inline string is wrapped in a Jinja SeedPrompt declaring the required parameters.""" + config = AttackAdversarialConfig(target=MagicMock(spec=PromptTarget), system_prompt="persona {{ objective }}") + seed = resolve_adversarial_system_prompt( + config=config, + default_system_prompt_path="unused.yaml", + required_parameters=["objective"], + ) + assert seed.value == "persona {{ objective }}" + assert "objective" in (seed.parameters or []) + + def test_explicit_seedprompt_with_required_params_returned_as_is(self): + """An explicitly provided SeedPrompt declaring the required params is returned unchanged.""" + provided = SeedPrompt(value="persona {{ objective }}", data_type="text", parameters=["objective"]) + config = AttackAdversarialConfig(target=MagicMock(spec=PromptTarget), system_prompt=provided) + seed = resolve_adversarial_system_prompt( + config=config, + default_system_prompt_path="unused.yaml", + required_parameters=["objective"], + ) + assert seed is provided + + def test_explicit_seedprompt_missing_required_params_raises(self): + """An explicit SeedPrompt missing a required parameter raises ValueError.""" + provided = SeedPrompt(value="persona", data_type="text", parameters=[]) + config = AttackAdversarialConfig(target=MagicMock(spec=PromptTarget), system_prompt=provided) + with pytest.raises(ValueError, match="missing required parameters"): + resolve_adversarial_system_prompt( + config=config, + default_system_prompt_path="unused.yaml", + required_parameters=["objective"], + ) + + def test_explicit_seedprompt_missing_params_uses_custom_error_message(self): + """A custom error_message overrides the default missing-parameters message.""" + provided = SeedPrompt(value="persona", data_type="text", parameters=[]) + config = AttackAdversarialConfig(target=MagicMock(spec=PromptTarget), system_prompt=provided) + with pytest.raises(ValueError, match="must declare objective"): + resolve_adversarial_system_prompt( + config=config, + default_system_prompt_path="unused.yaml", + required_parameters=["objective"], + error_message="must declare objective", + ) diff --git a/tests/unit/executor/attack/core/test_attack_strategy.py b/tests/unit/executor/attack/core/test_attack_strategy.py index f455871f05..7940ccf668 100644 --- a/tests/unit/executor/attack/core/test_attack_strategy.py +++ b/tests/unit/executor/attack/core/test_attack_strategy.py @@ -7,6 +7,7 @@ import pytest from pyrit.exceptions.retry_collector import RetryCollector +from pyrit.executor.attack.core.attack_config import AttackAdversarialConfig from pyrit.executor.attack.core.attack_parameters import AttackParameters from pyrit.executor.attack.core.attack_strategy import ( AttackContext, @@ -20,6 +21,11 @@ AttackResult, ComponentIdentifier, Message, + SeedPrompt, +) +from pyrit.models.identifiers import ( + AtomicAttackEvaluationIdentifier, + build_atomic_attack_identifier, ) from pyrit.models.retry_event import RetryEvent from pyrit.prompt_target import PromptTarget @@ -773,3 +779,161 @@ async def _teardown_async(self, *, context): # The default handler should still be present assert len(strategy._event_handlers) == 1 assert "_DefaultAttackStrategyEventHandler" in strategy._event_handlers + + +def _adv_target(*, model_name: str = "gpt-adv", extra_params: dict | None = None) -> PromptTarget: + """Build a mock adversarial chat target whose identifier carries the given params.""" + target = MagicMock(spec=PromptTarget) + params: dict = {"model_name": model_name} + if extra_params: + params.update(extra_params) + target.get_identifier.return_value = ComponentIdentifier(class_name="AdvChat", class_module="test", params=params) + return target + + +class _IdentityTestStrategy(AttackStrategy): + """Minimal concrete strategy that exposes a settable adversarial config for identity tests.""" + + def __init__(self, *, objective_target, adversarial_config=None): + super().__init__(context_type=AttackContext, objective_target=objective_target) + self._test_adversarial_config = adversarial_config + + def _validate_context(self, *, context): + pass + + async def _setup_async(self, *, context): + pass + + async def _perform_async(self, *, context): + return AttackResult( + conversation_id="c", + objective="o", + outcome=AttackOutcome.SUCCESS, + outcome_reason="ok", + execution_time_ms=0, + executed_turns=1, + ) + + async def _teardown_async(self, *, context): + pass + + def get_attack_adversarial_config(self): + return self._test_adversarial_config + + +def _eval_hash(attack_identifier: ComponentIdentifier) -> str: + composite = build_atomic_attack_identifier(attack_identifier=attack_identifier) + return AtomicAttackEvaluationIdentifier(composite).eval_hash + + +@pytest.mark.usefixtures("patch_central_database") +class TestCreateIdentifierAdversarial: + """Tests for adversarial config wiring into the attack identifier (component + eval hash).""" + + def test_base_returns_none_omits_adversarial_child_and_params(self, mock_objective_target): + """When get_attack_adversarial_config() returns None, no adversarial child/params appear.""" + strategy = _IdentityTestStrategy(objective_target=mock_objective_target, adversarial_config=None) + identifier = strategy.get_identifier() + assert "adversarial_chat" not in identifier.children + assert "adversarial_system_prompt" not in identifier.params + assert "adversarial_seed_prompt" not in identifier.params + + def test_adversarial_target_added_as_child(self, mock_objective_target): + adv = _adv_target() + config = AttackAdversarialConfig(target=adv, system_prompt=None, seed_prompt=None) + strategy = _IdentityTestStrategy(objective_target=mock_objective_target, adversarial_config=config) + identifier = strategy.get_identifier() + assert identifier.children["adversarial_chat"] == adv.get_identifier.return_value + + def test_target_only_config_omits_prompt_params(self, mock_objective_target): + """A target-only config (no prompts) emits the child but no prompt params.""" + config = AttackAdversarialConfig(target=_adv_target(), system_prompt=None, seed_prompt=None) + strategy = _IdentityTestStrategy(objective_target=mock_objective_target, adversarial_config=config) + identifier = strategy.get_identifier() + assert "adversarial_chat" in identifier.children + assert "adversarial_system_prompt" not in identifier.params + assert "adversarial_seed_prompt" not in identifier.params + + def test_system_prompt_string_stored_in_params(self, mock_objective_target): + config = AttackAdversarialConfig( + target=_adv_target(), system_prompt="persona {{ objective }}", seed_prompt=None + ) + strategy = _IdentityTestStrategy(objective_target=mock_objective_target, adversarial_config=config) + identifier = strategy.get_identifier() + assert identifier.params["adversarial_system_prompt"] == "persona {{ objective }}" + + def test_seed_prompt_seedprompt_value_stored_in_params(self, mock_objective_target): + seed = SeedPrompt(value="seed {{ objective }}", data_type="text", parameters=["objective"]) + config = AttackAdversarialConfig(target=_adv_target(), system_prompt=None, seed_prompt=seed) + strategy = _IdentityTestStrategy(objective_target=mock_objective_target, adversarial_config=config) + identifier = strategy.get_identifier() + assert identifier.params["adversarial_seed_prompt"] == "seed {{ objective }}" + + def test_different_system_prompt_changes_full_and_eval_hash(self, mock_objective_target): + adv = _adv_target() + s1 = _IdentityTestStrategy( + objective_target=mock_objective_target, + adversarial_config=AttackAdversarialConfig(target=adv, system_prompt="persona A", seed_prompt=None), + ) + s2 = _IdentityTestStrategy( + objective_target=mock_objective_target, + adversarial_config=AttackAdversarialConfig(target=adv, system_prompt="persona B", seed_prompt=None), + ) + id1, id2 = s1.get_identifier(), s2.get_identifier() + assert id1.hash != id2.hash + assert _eval_hash(id1) != _eval_hash(id2) + + def test_different_seed_prompt_changes_full_and_eval_hash(self, mock_objective_target): + adv = _adv_target() + s1 = _IdentityTestStrategy( + objective_target=mock_objective_target, + adversarial_config=AttackAdversarialConfig(target=adv, system_prompt=None, seed_prompt="first A"), + ) + s2 = _IdentityTestStrategy( + objective_target=mock_objective_target, + adversarial_config=AttackAdversarialConfig(target=adv, system_prompt=None, seed_prompt="first B"), + ) + id1, id2 = s1.get_identifier(), s2.get_identifier() + assert id1.hash != id2.hash + assert _eval_hash(id1) != _eval_hash(id2) + + def test_different_adversarial_model_changes_eval_hash(self, mock_objective_target): + """model_name is in the adversarial_chat eval allowlist -> different eval hash.""" + s1 = _IdentityTestStrategy( + objective_target=mock_objective_target, + adversarial_config=AttackAdversarialConfig( + target=_adv_target(model_name="gpt-4o"), system_prompt=None, seed_prompt=None + ), + ) + s2 = _IdentityTestStrategy( + objective_target=mock_objective_target, + adversarial_config=AttackAdversarialConfig( + target=_adv_target(model_name="gpt-3.5"), system_prompt=None, seed_prompt=None + ), + ) + assert _eval_hash(s1.get_identifier()) != _eval_hash(s2.get_identifier()) + + def test_adversarial_endpoint_does_not_change_eval_hash(self, mock_objective_target): + """endpoint is NOT in the adversarial_chat eval allowlist -> same eval hash.""" + s1 = _IdentityTestStrategy( + objective_target=mock_objective_target, + adversarial_config=AttackAdversarialConfig( + target=_adv_target(extra_params={"endpoint": "https://a.com"}), system_prompt=None, seed_prompt=None + ), + ) + s2 = _IdentityTestStrategy( + objective_target=mock_objective_target, + adversarial_config=AttackAdversarialConfig( + target=_adv_target(extra_params={"endpoint": "https://b.com"}), system_prompt=None, seed_prompt=None + ), + ) + assert _eval_hash(s1.get_identifier()) == _eval_hash(s2.get_identifier()) + + def test_adversarial_presence_changes_hash_vs_none(self, mock_objective_target): + """An attack with an adversarial child must not collide with one that has none.""" + plain = _IdentityTestStrategy(objective_target=mock_objective_target, adversarial_config=None) + adversarial = _IdentityTestStrategy( + objective_target=mock_objective_target, + adversarial_config=AttackAdversarialConfig(target=_adv_target(), system_prompt=None, seed_prompt=None), + ) + assert plain.get_identifier().hash != adversarial.get_identifier().hash diff --git a/tests/unit/executor/attack/multi_turn/test_crescendo.py b/tests/unit/executor/attack/multi_turn/test_crescendo.py index e8e295ba88..6533c75bcf 100644 --- a/tests/unit/executor/attack/multi_turn/test_crescendo.py +++ b/tests/unit/executor/attack/multi_turn/test_crescendo.py @@ -32,6 +32,7 @@ MessagePiece, Score, ScoreType, + SeedPrompt, ) from pyrit.prompt_normalizer import PromptNormalizer from pyrit.prompt_target import PromptTarget @@ -2370,3 +2371,73 @@ async def test_setup_tracks_adversarial_chat_conversation_id( and ref.conversation_type == ConversationType.ADVERSARIAL for ref in basic_context.related_conversations ) + + +@pytest.mark.usefixtures("patch_central_database") +class TestCrescendoAdversarialIdentity: + """Tests for adversarial config in the Crescendo attack identity and inline system prompt.""" + + def test_get_attack_adversarial_config_returns_target_and_system_prompt( + self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer + ): + attack = CrescendoTestHelper.create_attack( + objective_target=mock_objective_target, + adversarial_chat=mock_adversarial_chat, + objective_scorer=mock_objective_scorer, + ) + config = attack.get_attack_adversarial_config() + assert config is not None + assert config.target is mock_adversarial_chat + assert config.system_prompt is attack._adversarial_chat_system_prompt_template + assert config.seed_prompt is None + + def test_get_attack_adversarial_config_returns_none_without_target( + self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer + ): + attack = CrescendoTestHelper.create_attack( + objective_target=mock_objective_target, + adversarial_chat=mock_adversarial_chat, + objective_scorer=mock_objective_scorer, + ) + attack._adversarial_chat = None + assert attack.get_attack_adversarial_config() is None + + def test_identifier_includes_adversarial_chat_child( + self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer + ): + attack = CrescendoTestHelper.create_attack( + objective_target=mock_objective_target, + adversarial_chat=mock_adversarial_chat, + objective_scorer=mock_objective_scorer, + ) + identifier = attack.get_identifier() + assert "adversarial_chat" in identifier.children + assert identifier.children["adversarial_chat"] == mock_adversarial_chat.get_identifier.return_value + + def test_inline_system_prompt_string_resolved_and_in_identity( + self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer + ): + attack = CrescendoAttack( + objective_target=mock_objective_target, + attack_adversarial_config=AttackAdversarialConfig( + target=mock_adversarial_chat, system_prompt="custom crescendo persona" + ), + attack_scoring_config=AttackScoringConfig(objective_scorer=mock_objective_scorer), + ) + assert attack._adversarial_chat_system_prompt_template.value == "custom crescendo persona" + assert attack.get_identifier().params["adversarial_system_prompt"] == "custom crescendo persona" + + def test_inline_system_prompt_seedprompt_resolved( + self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer + ): + seed = SeedPrompt( + value="persona {{ objective }} {{ max_turns }}", + data_type="text", + parameters=["objective", "max_turns"], + ) + attack = CrescendoAttack( + objective_target=mock_objective_target, + attack_adversarial_config=AttackAdversarialConfig(target=mock_adversarial_chat, system_prompt=seed), + attack_scoring_config=AttackScoringConfig(objective_scorer=mock_objective_scorer), + ) + assert attack._adversarial_chat_system_prompt_template is seed diff --git a/tests/unit/executor/attack/multi_turn/test_pair.py b/tests/unit/executor/attack/multi_turn/test_pair.py index e7c4cadaec..54b85454c4 100644 --- a/tests/unit/executor/attack/multi_turn/test_pair.py +++ b/tests/unit/executor/attack/multi_turn/test_pair.py @@ -185,3 +185,28 @@ def test_pair_accepts_tap_scoring_config(self, objective_target, adversarial_con result = attack.get_attack_scoring_config() assert isinstance(result, TAPAttackScoringConfig) assert result.threshold == 0.85 + + +@pytest.mark.usefixtures("patch_central_database") +class TestPAIRAdversarialIdentity: + """PAIR inherits TAP's adversarial identity wiring.""" + + def test_get_attack_adversarial_config_includes_target(self, objective_target, adversarial_config): + attack = PAIRAttack( + objective_target=objective_target, + attack_adversarial_config=adversarial_config, + ) + config = attack.get_attack_adversarial_config() + assert config is not None + assert config.target is adversarial_config.target + assert config.system_prompt is attack._adversarial_chat_system_seed_prompt + assert config.seed_prompt is None + + def test_identifier_includes_adversarial_chat_child(self, objective_target, adversarial_config): + attack = PAIRAttack( + objective_target=objective_target, + attack_adversarial_config=adversarial_config, + ) + identifier = attack.get_identifier() + assert "adversarial_chat" in identifier.children + assert identifier.children["adversarial_chat"] == adversarial_config.target.get_identifier.return_value diff --git a/tests/unit/executor/attack/multi_turn/test_red_teaming.py b/tests/unit/executor/attack/multi_turn/test_red_teaming.py index e5e14f1eff..b180500250 100644 --- a/tests/unit/executor/attack/multi_turn/test_red_teaming.py +++ b/tests/unit/executor/attack/multi_turn/test_red_teaming.py @@ -18,6 +18,7 @@ RedTeamingAttack, RTASystemPromptPaths, ) +from pyrit.executor.attack.core.attack_config import DEFAULT_ADVERSARIAL_SEED_PROMPT from pyrit.models import ( AttackOutcome, AttackResult, @@ -1964,3 +1965,79 @@ async def test_score_last_turn_only_can_still_succeed_on_last_turn( # Should succeed based on final score assert result.outcome == AttackOutcome.SUCCESS assert result.last_score == success_score + + +@pytest.mark.usefixtures("patch_central_database") +class TestRedTeamingAdversarialIdentity: + """Tests for adversarial config in the RedTeaming attack identity and inline system prompt.""" + + def test_get_attack_adversarial_config_includes_target_system_and_seed( + self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer + ): + attack = RedTeamingAttack( + objective_target=mock_objective_target, + attack_adversarial_config=AttackAdversarialConfig(target=mock_adversarial_chat), + attack_scoring_config=AttackScoringConfig(objective_scorer=mock_objective_scorer), + ) + config = attack.get_attack_adversarial_config() + assert config is not None + assert config.target is mock_adversarial_chat + assert config.system_prompt is attack._adversarial_chat_system_prompt_template + assert config.seed_prompt is attack._adversarial_chat_seed_prompt + + def test_identifier_includes_adversarial_chat_child( + self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer + ): + attack = RedTeamingAttack( + objective_target=mock_objective_target, + attack_adversarial_config=AttackAdversarialConfig(target=mock_adversarial_chat), + attack_scoring_config=AttackScoringConfig(objective_scorer=mock_objective_scorer), + ) + identifier = attack.get_identifier() + assert "adversarial_chat" in identifier.children + assert identifier.children["adversarial_chat"] == mock_adversarial_chat.get_identifier.return_value + + def test_inline_system_prompt_string_resolved_and_in_identity( + self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer + ): + attack = RedTeamingAttack( + objective_target=mock_objective_target, + attack_adversarial_config=AttackAdversarialConfig( + target=mock_adversarial_chat, system_prompt="custom red team persona" + ), + attack_scoring_config=AttackScoringConfig(objective_scorer=mock_objective_scorer), + ) + assert attack._adversarial_chat_system_prompt_template.value == "custom red team persona" + assert attack.get_identifier().params["adversarial_system_prompt"] == "custom red team persona" + + def test_inline_seed_prompt_string_used(self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer): + attack = RedTeamingAttack( + objective_target=mock_objective_target, + attack_adversarial_config=AttackAdversarialConfig( + target=mock_adversarial_chat, seed_prompt="kick off {{ objective }}" + ), + attack_scoring_config=AttackScoringConfig(objective_scorer=mock_objective_scorer), + ) + assert attack._adversarial_chat_seed_prompt.value == "kick off {{ objective }}" + assert attack.get_identifier().params["adversarial_seed_prompt"] == "kick off {{ objective }}" + + def test_seed_prompt_none_falls_back_to_default( + self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer + ): + attack = RedTeamingAttack( + objective_target=mock_objective_target, + attack_adversarial_config=AttackAdversarialConfig(target=mock_adversarial_chat, seed_prompt=None), + attack_scoring_config=AttackScoringConfig(objective_scorer=mock_objective_scorer), + ) + assert attack._adversarial_chat_seed_prompt.value == DEFAULT_ADVERSARIAL_SEED_PROMPT + + def test_get_attack_adversarial_config_returns_none_without_target( + self, mock_objective_target, mock_adversarial_chat, mock_objective_scorer + ): + attack = RedTeamingAttack( + objective_target=mock_objective_target, + attack_adversarial_config=AttackAdversarialConfig(target=mock_adversarial_chat), + attack_scoring_config=AttackScoringConfig(objective_scorer=mock_objective_scorer), + ) + attack._adversarial_chat = None + assert attack.get_attack_adversarial_config() is None diff --git a/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py b/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py index aebd00702b..49b3fe0c3b 100644 --- a/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py +++ b/tests/unit/executor/attack/multi_turn/test_tree_of_attacks.py @@ -1813,17 +1813,16 @@ def test_load_adversarial_prompts_default(self, attack_builder): """Test loading prompts with default paths.""" attack = attack_builder.with_default_mocks().build() - # Mock SeedPrompt loading - mock_system = MagicMock(spec=SeedPrompt) + # Mock SeedPrompt loading. The system seed prompt is resolved in __init__ + # (via resolve_adversarial_system_prompt); _load_adversarial_prompts only + # loads the prompt template and the first-message seed prompt. mock_template = MagicMock(spec=SeedPrompt) mock_seed = MagicMock(spec=SeedPrompt) - with patch.object(SeedPrompt, "from_yaml_with_required_parameters", return_value=mock_system): - with patch.object(SeedPrompt, "from_yaml_file", side_effect=[mock_template, mock_seed]): - attack._load_adversarial_prompts() + with patch.object(SeedPrompt, "from_yaml_file", side_effect=[mock_template, mock_seed]): + attack._load_adversarial_prompts() # Verify prompts were loaded and stored - assert attack._adversarial_chat_system_seed_prompt == mock_system assert attack._adversarial_chat_prompt_template == mock_template assert attack._adversarial_chat_seed_prompt == mock_seed @@ -2125,6 +2124,7 @@ def test_tap_init_raises_when_objective_scorer_is_none(): objective_target=MagicMock(spec=PromptTarget), attack_adversarial_config=MagicMock( target=MagicMock(spec=PromptTarget), + system_prompt=None, system_prompt_path=None, ), attack_scoring_config=scoring_config, @@ -2480,3 +2480,43 @@ def _create_node_side_effect(**kwargs): assert abs(context.best_objective_score.get_value() - expected_best_score) < 0.01, ( f"Expected best score ~{expected_best_score}, got {context.best_objective_score.get_value()}" ) + + +@pytest.mark.usefixtures("patch_central_database") +class TestTAPAdversarialIdentity: + """Tests for adversarial config in the TAP attack identity and inline system prompt.""" + + def test_get_attack_adversarial_config_includes_target_and_system_seed_only(self): + builder = AttackBuilder().with_default_mocks() + attack = builder.build() + config = attack.get_attack_adversarial_config() + assert config is not None + assert config.target is builder.adversarial_chat + assert config.system_prompt is attack._adversarial_chat_system_seed_prompt + # TAP's first-message seed prompt is a fixed default and is excluded from identity. + assert config.seed_prompt is None + + def test_get_attack_adversarial_config_returns_none_without_target(self): + builder = AttackBuilder().with_default_mocks() + attack = builder.build() + attack._adversarial_chat = None + assert attack.get_attack_adversarial_config() is None + + def test_identifier_includes_adversarial_chat_child(self): + builder = AttackBuilder().with_default_mocks() + attack = builder.build() + identifier = attack.get_identifier() + assert "adversarial_chat" in identifier.children + assert identifier.children["adversarial_chat"] == builder.adversarial_chat.get_identifier.return_value + + def test_inline_system_prompt_string_resolved_and_in_identity(self): + objective_target = AttackBuilder._create_mock_target() + adversarial_chat = AttackBuilder._create_mock_chat() + attack = TreeOfAttacksWithPruningAttack( + objective_target=objective_target, + attack_adversarial_config=AttackAdversarialConfig( + target=adversarial_chat, system_prompt="tap persona {{ desired_prefix }}" + ), + ) + assert attack._adversarial_chat_system_seed_prompt.value == "tap persona {{ desired_prefix }}" + assert attack.get_identifier().params["adversarial_system_prompt"] == "tap persona {{ desired_prefix }}" diff --git a/tests/unit/executor/attack/single_turn/test_context_compliance.py b/tests/unit/executor/attack/single_turn/test_context_compliance.py index b10ff8e640..4109d5204c 100644 --- a/tests/unit/executor/attack/single_turn/test_context_compliance.py +++ b/tests/unit/executor/attack/single_turn/test_context_compliance.py @@ -875,3 +875,40 @@ def test_params_type_includes_objective(self, mock_objective_target, mock_attack fields = {f.name for f in dataclasses.fields(attack.params_type)} assert "objective" in fields + + +@pytest.mark.usefixtures("patch_central_database") +class TestContextComplianceAttackAdversarialIdentity: + """Tests that the adversarial chat target is included in the attack identity.""" + + def test_get_attack_adversarial_config_returns_target_only( + self, mock_objective_target, mock_attack_adversarial_config, mock_adversarial_chat + ): + attack = ContextComplianceAttack( + objective_target=mock_objective_target, attack_adversarial_config=mock_attack_adversarial_config + ) + config = attack.get_attack_adversarial_config() + assert config is not None + assert config.target is mock_adversarial_chat + assert config.seed_prompt is None + + def test_get_attack_adversarial_config_returns_none_without_target( + self, mock_objective_target, mock_attack_adversarial_config + ): + attack = ContextComplianceAttack( + objective_target=mock_objective_target, attack_adversarial_config=mock_attack_adversarial_config + ) + attack._adversarial_chat = None + assert attack.get_attack_adversarial_config() is None + + def test_identifier_includes_adversarial_chat_child( + self, mock_objective_target, mock_attack_adversarial_config, mock_adversarial_chat + ): + """Regression: PromptSendingAttack caches the identifier in __init__, so the adversarial + target must be set BEFORE super().__init__() for the child to appear.""" + attack = ContextComplianceAttack( + objective_target=mock_objective_target, attack_adversarial_config=mock_attack_adversarial_config + ) + identifier = attack.get_identifier() + assert "adversarial_chat" in identifier.children + assert identifier.children["adversarial_chat"] == mock_adversarial_chat.get_identifier.return_value diff --git a/tests/unit/executor/attack/single_turn/test_role_play.py b/tests/unit/executor/attack/single_turn/test_role_play.py index eda4782e1f..39c04501de 100644 --- a/tests/unit/executor/attack/single_turn/test_role_play.py +++ b/tests/unit/executor/attack/single_turn/test_role_play.py @@ -443,3 +443,26 @@ async def test_rephrase_objective_returns_string(self, role_play_attack): assert isinstance(result, str) assert result == "Rephrased text" + + +@pytest.mark.usefixtures("patch_central_database") +class TestRolePlayAttackAdversarialIdentity: + """Tests that the adversarial chat target is included in the attack identity.""" + + def test_get_attack_adversarial_config_returns_target_only(self, role_play_attack, mock_adversarial_chat_target): + config = role_play_attack.get_attack_adversarial_config() + assert config is not None + assert config.target is mock_adversarial_chat_target + # RolePlay uses its own definition files, not the adversarial seed prompt. + assert config.seed_prompt is None + + def test_get_attack_adversarial_config_returns_none_without_target(self, role_play_attack): + role_play_attack._adversarial_chat = None + assert role_play_attack.get_attack_adversarial_config() is None + + def test_identifier_includes_adversarial_chat_child(self, role_play_attack, mock_adversarial_chat_target): + """Regression: PromptSendingAttack caches the identifier in __init__, so the adversarial + target must be set BEFORE super().__init__() for the child to appear.""" + identifier = role_play_attack.get_identifier() + assert "adversarial_chat" in identifier.children + assert identifier.children["adversarial_chat"] == mock_adversarial_chat_target.get_identifier.return_value diff --git a/tests/unit/models/identifiers/test_atomic_attack_identifier.py b/tests/unit/models/identifiers/test_atomic_attack_identifier.py index 10005da8d5..933c233207 100644 --- a/tests/unit/models/identifiers/test_atomic_attack_identifier.py +++ b/tests/unit/models/identifiers/test_atomic_attack_identifier.py @@ -209,6 +209,7 @@ def test_adversarial_chat_rule(self): rule = AtomicAttackEvaluationIdentifier.CHILD_EVAL_RULES["adversarial_chat"] assert rule.included_params == frozenset({"underlying_model_name", "temperature", "top_p"}) assert rule.param_fallbacks == {"underlying_model_name": "model_name"} + assert rule.inner_child_name == "targets" assert not rule.exclude def test_scorer_only_keys_absent(self): @@ -296,6 +297,25 @@ def test_adversarial_chat_endpoint_ignored(self): c2 = build_atomic_attack_identifier(attack_identifier=a2) assert AtomicAttackEvaluationIdentifier(c1).eval_hash == AtomicAttackEvaluationIdentifier(c2).eval_hash + def test_adversarial_chat_wrapper_unwrapped_via_inner_child_name(self): + """A wrapper adversarial target is looked through to its inner 'targets' child, + so a bare target and the same target wrapped collapse to the same eval hash.""" + inner = ComponentIdentifier(class_name="OpenAIChatTarget", class_module="m", params={"model_name": "gpt-4o"}) + bare = ComponentIdentifier(class_name="OpenAIChatTarget", class_module="m", params={"model_name": "gpt-4o"}) + wrapper = ComponentIdentifier( + class_name="RoundRobinTarget", + class_module="m", + params={"strategy": "round_robin"}, + children={"targets": [inner]}, + ) + a_bare = _make_attack(children={"adversarial_chat": bare}) + a_wrapped = _make_attack(children={"adversarial_chat": wrapper}) + c_bare = build_atomic_attack_identifier(attack_identifier=a_bare) + c_wrapped = build_atomic_attack_identifier(attack_identifier=a_wrapped) + assert ( + AtomicAttackEvaluationIdentifier(c_bare).eval_hash == AtomicAttackEvaluationIdentifier(c_wrapped).eval_hash + ) + # -- objective_scorer exclusion ---------------------------------------- def test_objective_scorer_excluded_from_eval_hash(self): diff --git a/tests/unit/scenario/core/test_attack_technique_factory.py b/tests/unit/scenario/core/test_attack_technique_factory.py index da7a44790b..c76a7724c8 100644 --- a/tests/unit/scenario/core/test_attack_technique_factory.py +++ b/tests/unit/scenario/core/test_attack_technique_factory.py @@ -572,7 +572,7 @@ def test_apply_scorer_policy_raise_raises(self): class TestCustomAdversarialPrompt: - """Tests for the adversarial_system_prompt_path / adversarial_seed_prompt params.""" + """Tests for the adversarial_system_prompt / adversarial_seed_prompt params.""" class _AdversarialAttack: def __init__(self, *, objective_target, attack_scoring_config=None, attack_adversarial_config=None): @@ -591,7 +591,7 @@ def test_custom_prompt_implies_uses_adversarial(self): factory = AttackTechniqueFactory( name="durian", attack_class=_StubAttack, - adversarial_system_prompt_path="some/path.yaml", + adversarial_system_prompt="custom {{ objective }}", ) assert factory.uses_adversarial is True @@ -611,13 +611,13 @@ def test_custom_prompt_with_baked_chat_coexist(self): name="durian", attack_class=self._AdversarialAttack, adversarial_chat=target, - adversarial_system_prompt_path="some/path.yaml", + adversarial_system_prompt="sys {{ objective }}", adversarial_seed_prompt=seed, ) technique = factory.create(objective_target=MagicMock(spec=PromptTarget), attack_scoring_config=self._scoring()) config = technique.attack.attack_adversarial_config assert config.target is target - assert config.system_prompt_path == "some/path.yaml" + assert config.system_prompt == "sys {{ objective }}" assert config.seed_prompt is seed def test_adversarial_chat_implies_uses_adversarial(self): @@ -676,7 +676,7 @@ def test_custom_prompt_with_uses_adversarial_false_raises(self): AttackTechniqueFactory( name="durian", attack_class=_StubAttack, - adversarial_system_prompt_path="some/path.yaml", + adversarial_system_prompt="custom {{ objective }}", uses_adversarial=False, ) @@ -685,7 +685,7 @@ def test_lazy_resolution_attaches_custom_prompts(self): factory = AttackTechniqueFactory( name="durian", attack_class=self._AdversarialAttack, - adversarial_system_prompt_path="durian/system.yaml", + adversarial_system_prompt="durian sys {{ objective }}", adversarial_seed_prompt=seed, ) fallback = MagicMock(spec=PromptTarget) @@ -699,7 +699,7 @@ def test_lazy_resolution_attaches_custom_prompts(self): config = technique.attack.attack_adversarial_config assert config.target is fallback - assert config.system_prompt_path == "durian/system.yaml" + assert config.system_prompt == "durian sys {{ objective }}" assert config.seed_prompt is seed def test_create_adversarial_chat_is_combined_with_custom_prompts(self): @@ -707,7 +707,7 @@ def test_create_adversarial_chat_is_combined_with_custom_prompts(self): factory = AttackTechniqueFactory( name="durian", attack_class=self._AdversarialAttack, - adversarial_system_prompt_path="durian/system.yaml", + adversarial_system_prompt="durian sys {{ objective }}", adversarial_seed_prompt=seed, ) create_target = MagicMock(spec=PromptTarget) @@ -721,7 +721,7 @@ def test_create_adversarial_chat_is_combined_with_custom_prompts(self): config = technique.attack.attack_adversarial_config # The create-time target is used; the technique keeps its custom prompts. assert config.target is create_target - assert config.system_prompt_path == "durian/system.yaml" + assert config.system_prompt == "durian sys {{ objective }}" assert config.seed_prompt is seed def test_create_adversarial_chat_used_as_target(self): @@ -771,13 +771,66 @@ def test_create_adversarial_chat_with_deprecated_override_raises(self): def test_identifier_distinguishes_custom_system_prompt(self): f1 = AttackTechniqueFactory( - name="durian", attack_class=self._AdversarialAttack, adversarial_system_prompt_path="a.yaml" + name="durian", attack_class=self._AdversarialAttack, adversarial_system_prompt="a {{ objective }}" ) f2 = AttackTechniqueFactory( - name="durian", attack_class=self._AdversarialAttack, adversarial_system_prompt_path="b.yaml" + name="durian", attack_class=self._AdversarialAttack, adversarial_system_prompt="b {{ objective }}" ) assert f1.get_identifier().hash != f2.get_identifier().hash + def test_identifier_distinguishes_custom_seed_prompt_object(self): + """A SeedPrompt adversarial_seed_prompt is serialized by value, so different prompts differ.""" + f1 = AttackTechniqueFactory( + name="durian", + attack_class=self._AdversarialAttack, + adversarial_seed_prompt=SeedPrompt(value="a {{ objective }}", data_type="text", parameters=["objective"]), + ) + f2 = AttackTechniqueFactory( + name="durian", + attack_class=self._AdversarialAttack, + adversarial_seed_prompt=SeedPrompt(value="b {{ objective }}", data_type="text", parameters=["objective"]), + ) + assert f1.get_identifier().hash != f2.get_identifier().hash + + def test_create_custom_prompt_conflicts_with_baked_raises(self): + """create() must not supply adversarial prompts when the factory baked a custom one.""" + factory = AttackTechniqueFactory( + name="durian", + attack_class=self._AdversarialAttack, + adversarial_system_prompt="baked {{ objective }}", + ) + with pytest.raises(ValueError, match="custom adversarial prompt is already baked"): + factory.create( + objective_target=MagicMock(spec=PromptTarget), + attack_scoring_config=self._scoring(), + adversarial_system_prompt="create-time {{ objective }}", + ) + + def test_create_override_with_system_prompt_path_loads_yaml(self): + """A deprecated override carrying system_prompt_path is resolved via SeedPrompt.from_yaml_file.""" + factory = AttackTechniqueFactory( + name="durian", + attack_class=self._AdversarialAttack, + ) + loaded = SeedPrompt(value="from yaml {{ objective }}", data_type="text", parameters=["objective"]) + with ( + patch( + "pyrit.scenario.core.attack_technique_factory.SeedPrompt.from_yaml_file", + return_value=loaded, + ) as mock_from_yaml, + pytest.warns(DeprecationWarning), + ): + override = AttackAdversarialConfig( + target=MagicMock(spec=PromptTarget), system_prompt_path="legacy/persona.yaml" + ) + technique = factory.create( + objective_target=MagicMock(spec=PromptTarget), + attack_scoring_config=self._scoring(), + attack_adversarial_config_override=override, + ) + mock_from_yaml.assert_called_once_with("legacy/persona.yaml") + assert technique.attack.attack_adversarial_config.system_prompt is loaded + class TestDeprecatedAdversarialConfig: """Tests for the deprecated ``adversarial_config`` parameter.""" @@ -814,15 +867,35 @@ def test_adversarial_config_unpacked_into_create(self): name="durian", attack_class=self._AdversarialAttack, adversarial_config=AttackAdversarialConfig( - target=target, system_prompt_path="some/path.yaml", seed_prompt=seed + target=target, system_prompt="sys {{ objective }}", seed_prompt=seed ), ) technique = factory.create(objective_target=MagicMock(spec=PromptTarget), attack_scoring_config=self._scoring()) config = technique.attack.attack_adversarial_config assert config.target is target - assert config.system_prompt_path == "some/path.yaml" + assert config.system_prompt == "sys {{ objective }}" assert config.seed_prompt is seed + def test_adversarial_config_with_system_prompt_path_loads_yaml(self): + """A deprecated adversarial_config carrying system_prompt_path is resolved via from_yaml_file.""" + target = MagicMock(spec=PromptTarget) + loaded = SeedPrompt(value="from yaml {{ objective }}", data_type="text", parameters=["objective"]) + with ( + patch( + "pyrit.scenario.core.attack_technique_factory.SeedPrompt.from_yaml_file", + return_value=loaded, + ) as mock_from_yaml, + pytest.warns(DeprecationWarning), + ): + factory = AttackTechniqueFactory( + name="durian", + attack_class=self._AdversarialAttack, + adversarial_config=AttackAdversarialConfig(target=target, system_prompt_path="legacy/persona.yaml"), + ) + mock_from_yaml.assert_called_once_with("legacy/persona.yaml") + technique = factory.create(objective_target=MagicMock(spec=PromptTarget), attack_scoring_config=self._scoring()) + assert technique.attack.attack_adversarial_config.system_prompt is loaded + def test_adversarial_config_with_adversarial_chat_raises(self): target = MagicMock(spec=PromptTarget) with pytest.raises(ValueError, match="cannot be combined"):