Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions .pyrit_conf_example
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,27 @@ memory_db_type: sqlite
#
# Each initializer can be specified as:
# - A simple string (name only)
# - A dictionary with 'name' and optional 'args' for constructor arguments
# - A dictionary with 'name' and optional 'args' for parameters
#
# Parameters are lists of strings. Use the CLI command
# `pyrit_scan --list-initializers` to see available parameters.
#
# Example:
# initializers:
# - simple
# - name: airt
# - name: target
# args:
# some_param: value
# tags:
# - default
# - scorer
initializers:
- simple
- name: simple
- name: scorer
- name: target
args:
tags:
- default
- scorer

# Operator and Operation Labels
# ------------------------------
Expand Down
4 changes: 3 additions & 1 deletion build_scripts/evaluate_scorers.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,11 @@ async def evaluate_scorers() -> None:
5. Save results to scorer_evals directory
"""
print("Initializing PyRIT...")
target_init = TargetInitializer()
target_init.params = {"tags": ["default", "scorer"]}
await initialize_pyrit_async(
memory_db_type=IN_MEMORY,
initializers=[TargetInitializer(tags=["default", "scorer"]), ScorerInitializer()],
initializers=[target_init, ScorerInitializer()],
)

registry = ScorerRegistry.get_registry_singleton()
Expand Down
21 changes: 17 additions & 4 deletions doc/code/setup/pyrit_initializer.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
{
"data": {
"text/plain": [
"<__main__.CustomInitializer at 0x230d9e527b0>"
"<__main__.CustomInitializer at 0x187393c96a0>"
]
},
"execution_count": null,
Expand Down Expand Up @@ -92,7 +92,17 @@
"execution_count": null,
"id": "4",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found default environment files: ['./.pyrit/.env', './.pyrit/.env.local']\n",
"Loaded environment file: ./.pyrit/.env\n",
"Loaded environment file: ./.pyrit/.env.local\n"
]
}
],
"source": [
"from pyrit.setup import initialize_pyrit_async\n",
"from pyrit.setup.initializers import SimpleInitializer\n",
Expand Down Expand Up @@ -130,7 +140,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Created: ./AppData/Local/Temp/tmpa2k36reo/custom_init.py\n"
"Created: ./AppData/Local/Temp/tmp7zzfts8u/custom_init.py\n",
"Found default environment files: ['./.pyrit/.env', './.pyrit/.env.local']\n",
"Loaded environment file: ./.pyrit/.env\n",
"Loaded environment file: ./.pyrit/.env.local\n"
]
}
],
Expand Down Expand Up @@ -213,7 +226,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.12"
"version": "3.13.5"
}
},
"nbformat": 4,
Expand Down
11 changes: 6 additions & 5 deletions doc/code/setup/pyrit_initializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# extension: .py
# format_name: percent
# format_version: '1.3'
# jupytext_version: 1.17.3
# jupytext_version: 1.18.1
# ---

# %% [markdown]
Expand All @@ -17,7 +17,7 @@
#
# ## Execution Order
#
# When `initialize_pyrit` is called:
# When `initialize_pyrit_async` is called:
# 1. Environment files are loaded (`.env`, `.env.local`)
# 2. Memory database is configured
# 3. All initializers are sorted by `execution_order` and executed
Expand All @@ -27,10 +27,9 @@
# %% [markdown]
# The following is a minimal `PyRITInitializer` class. It doesn't need much! In this case, it sets the default value for temperature for all OpenAIChatTargets to .9.

# %%
from pyrit.common.apply_defaults import set_default_value
from pyrit.prompt_target import OpenAIChatTarget

# %%
from pyrit.setup.initializers.pyrit_initializer import PyRITInitializer


Expand Down Expand Up @@ -68,7 +67,9 @@ def description(self) -> str:
from pyrit.setup.initializers import SimpleInitializer

# Using built-in initializer
await initialize_pyrit_async(memory_db_type="InMemory", initializers=[SimpleInitializer()]) # type: ignore
await initialize_pyrit_async( # type: ignore
memory_db_type="InMemory", initializers=[SimpleInitializer()]
)

# %% [markdown]
# ## External Scripts
Expand Down
8 changes: 5 additions & 3 deletions doc/setup/pyrit_conf.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,18 @@ A list of built-in initializers to run during PyRIT initialization. Initializers
Each entry can be:

- **A simple string** — just the initializer name
- **A dictionary** — with `name` and optional `args` for constructor arguments
- **A dictionary** — with `name` and optional `args` (each arg is a list of strings passed to `initialize_async`)

Example:

```yaml
initializers:
- simple
- name: airt
- name: target
args:
some_param: value
tags:
- default
- scorer
```

Use `pyrit list initializers` in the CLI to see all registered initializers. See the [initializer documentation notebook](../code/setup/pyrit_initializer.ipynb) for reference.
Expand Down
2 changes: 1 addition & 1 deletion docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -183,5 +183,5 @@ The JupyterLab instance is configured to run without authentication by default f

- 📖 **[Docker Installation Guide](./../doc/setup/1b_install_docker.md)** - Complete user-friendly installation instructions
- 🚀 **[PyRIT Documentation](https://azure.github.io/PyRIT/)** - Full documentation site
- 🔧 **[Contributing Guide](https://azure.github.io/PyRIT/contributing/README.html)** - For developers and contributors
- 🔧 **[Contributing Guide](https://azure.github.io/PyRIT/contributing/readme/)** - For developers and contributors
- 🐛 **[Issues](https://github.com/Azure/PyRIT/issues)** - Report bugs or request features
85 changes: 72 additions & 13 deletions pyrit/cli/frontend_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def __init__(
config_file: Optional[Path] = None,
database: Optional[str] = None,
initialization_scripts: Optional[list[Path]] = None,
initializer_names: Optional[list[str]] = None,
initializer_names: Optional[list[Any]] = None,
env_files: Optional[list[Path]] = None,
log_level: Optional[int] = None,
):
Expand All @@ -94,7 +94,9 @@ def __init__(
The file uses .pyrit_conf extension but is YAML format.
database: Database type (InMemory, SQLite, or AzureSQL).
initialization_scripts: Optional list of initialization script paths.
initializer_names: Optional list of built-in initializer names to run.
initializer_names: Optional list of initializer entries. Each entry can be
a string name (e.g., "simple") or a dict with 'name' and optional 'args'
(e.g., {"name": "target", "args": {"tags": "default,scorer"}}).
env_files: Optional list of environment file paths to load in order.
log_level: Logging level constant (e.g., logging.WARNING). Defaults to logging.WARNING.

Expand Down Expand Up @@ -130,9 +132,7 @@ def __init__(
# Use canonical mapping from configuration_loader
self._database = _MEMORY_DB_TYPE_MAP[config.memory_db_type]
self._initialization_scripts = config._resolve_initialization_scripts()
self._initializer_names = (
[ic.name for ic in config._initializer_configs] if config._initializer_configs else None
)
self._initializer_configs = config._initializer_configs if config._initializer_configs else None
self._env_files = config._resolve_env_files()
self._operator = config.operator
self._operation = config.operation
Expand Down Expand Up @@ -289,15 +289,18 @@ async def run_scenario_async(

# Run initializers before scenario
initializer_instances = None
if context._initializer_names:
print(f"Running {len(context._initializer_names)} initializer(s)...")
if context._initializer_configs:
print(f"Running {len(context._initializer_configs)} initializer(s)...")
sys.stdout.flush()

initializer_instances = []

for name in context._initializer_names:
initializer_class = context.initializer_registry.get_class(name)
initializer_instances.append(initializer_class())
for config in context._initializer_configs:
initializer_class = context.initializer_registry.get_class(config.name)
instance = initializer_class()
if config.args:
instance.set_params_from_args(args=config.args)
initializer_instances.append(instance)

# Re-initialize PyRIT with the scenario-specific initializers
# This resets memory and applies initializer defaults
Expand Down Expand Up @@ -479,6 +482,13 @@ def format_initializer_metadata(*, initializer_metadata: InitializerMetadata) ->
else:
print(" Required Environment Variables: None")

if initializer_metadata.supported_parameters:
print(" Supported Parameters:")
for param_name, param_desc, param_required, param_default in initializer_metadata.supported_parameters:
req_str = " (required)" if param_required else ""
default_str = f" [default: {param_default}]" if param_default else ""
print(f" - {param_name}{req_str}{default_str}: {param_desc}")

if initializer_metadata.class_description:
print(" Description:")
print(_format_wrapped_text(text=initializer_metadata.class_description, indent=" "))
Expand Down Expand Up @@ -775,7 +785,11 @@ async def print_initializers_list_async(*, context: FrontendCore, discovery_path
"initialization scripts, and env files. CLI arguments override config file values. "
"If not specified, ~/.pyrit/.pyrit_conf is loaded if it exists."
),
"initializers": "Built-in initializer names to run before the scenario (e.g., openai_objective_target)",
"initializers": (
"Built-in initializer names to run before the scenario. "
"Supports optional params with name:key=val syntax "
"(e.g., target:tags=default,scorer dataset:mode=strict)"
),
"initialization_scripts": "Paths to custom Python initialization scripts to run before the scenario",
"env_files": "Paths to environment files to load in order (e.g., .env.production .env.local). Later files "
"override earlier ones.",
Expand All @@ -792,6 +806,51 @@ async def print_initializers_list_async(*, context: FrontendCore, discovery_path
}


def _parse_initializer_arg(arg: str) -> str | dict[str, Any]:
"""
Parse an initializer CLI argument into a string or dict for ConfigurationLoader.

Supports two formats:
- Simple name: "simple" → "simple"
- Name with params: "target:tags=default,scorer" → {"name": "target", "args": {"tags": ["default", "scorer"]}}

For multiple params on one initializer, separate with semicolons: "name:key1=val1;key2=val2"
For multiple initializers with params, space-separate them: "target:tags=a,b dataset:mode=strict"

Args:
arg: The CLI argument string.

Returns:
str | dict[str, Any]: A plain name string, or a dict with 'name' and 'args' keys.

Raises:
ValueError: If the argument format is invalid.
"""
if ":" not in arg:
return arg

name, params_str = arg.split(":", 1)
if not name:
raise ValueError(f"Invalid initializer argument '{arg}': missing name before ':'")

args: dict[str, list[str]] = {}
for pair in params_str.split(";"):
pair = pair.strip()
if not pair:
continue
if "=" not in pair:
raise ValueError(f"Invalid initializer parameter '{pair}' in '{arg}': expected key=value format")
key, value = pair.split("=", 1)
key = key.strip()
if not key:
raise ValueError(f"Invalid initializer parameter in '{arg}': empty key")
args[key] = [v.strip() for v in value.split(",")]

if args:
return {"name": name, "args": args}
return name


def parse_run_arguments(*, args_string: str) -> dict[str, Any]:
"""
Parse run command arguments from a string (for shell mode).
Expand Down Expand Up @@ -839,11 +898,11 @@ def parse_run_arguments(*, args_string: str) -> dict[str, Any]:
i = 1
while i < len(parts):
if parts[i] == "--initializers":
# Collect initializers until next flag
# Collect initializers until next flag, parsing name:key=val syntax
result["initializers"] = []
i += 1
while i < len(parts) and not parts[i].startswith("--"):
result["initializers"].append(parts[i])
result["initializers"].append(_parse_initializer_arg(parts[i]))
i += 1
elif parts[i] == "--initialization-scripts":
# Collect script paths until next flag
Expand Down
19 changes: 11 additions & 8 deletions pyrit/cli/pyrit_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def parse_args(*, args: Optional[list[str]] = None) -> Namespace:

parser.add_argument(
"--initializers",
type=str,
type=frontend_core._parse_initializer_arg,
nargs="+",
help=frontend_core.ARG_HELP["initializers"],
)
Expand Down Expand Up @@ -165,12 +165,15 @@ async def initialize_and_run_async(*, parsed_args: Namespace) -> int:

# Run initializers up-front (backend runs them once at startup, not per-scenario)
initializer_instances = None
if context._initializer_names:
print(f"Running {len(context._initializer_names)} initializer(s)...")
if context._initializer_configs:
print(f"Running {len(context._initializer_configs)} initializer(s)...")
initializer_instances = []
for name in context._initializer_names:
initializer_class = context.initializer_registry.get_class(name)
initializer_instances.append(initializer_class())
for config in context._initializer_configs:
initializer_class = context.initializer_registry.get_class(config.name)
instance = initializer_class()
if config.args:
instance.set_params_from_args(args=config.args)
initializer_instances.append(instance)

# Re-initialize with initializers applied
await initialize_pyrit_async(
Expand All @@ -196,14 +199,14 @@ async def initialize_and_run_async(*, parsed_args: Namespace) -> int:
print(f"🚀 Starting PyRIT backend on http://{parsed_args.host}:{parsed_args.port}")
print(f" API Docs: http://{parsed_args.host}:{parsed_args.port}/docs")

config = uvicorn.Config(
uvicorn_config = uvicorn.Config(
"pyrit.backend.main:app",
host=parsed_args.host,
port=parsed_args.port,
log_level=parsed_args.log_level,
reload=parsed_args.reload,
)
server = uvicorn.Server(config)
server = uvicorn.Server(uvicorn_config)
await server.serve()

return 0
Expand Down
2 changes: 1 addition & 1 deletion pyrit/cli/pyrit_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def parse_args(args: Optional[list[str]] = None) -> Namespace:

parser.add_argument(
"--initializers",
type=str,
type=frontend_core._parse_initializer_arg,
nargs="+",
help=frontend_core.ARG_HELP["initializers"],
)
Expand Down
Loading
Loading