Feat: Evaluation preview features - Batch evaluation and config bundles

pyproject.toml

@@ -26,8 +26,8 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [
-    "boto3>=1.42.86",
-    "botocore>=1.42.86",
+    "boto3>=1.43.0",
+    "botocore>=1.43.0",
     "pydantic>=2.0.0,<2.41.3",
     "urllib3>=1.26.0",
     "starlette>=0.46.2",
@@ -167,3 +167,7 @@ strands-agents = [
 strands-agents-evals = [
     "strands-agents-evals>=0.1.0"
 ]
+simulation = [
+    "jinja2>=3.1.0",
+    "strands-agents-evals>=0.1.0",
+]

src/bedrock_agentcore/_utils/endpoints.py

@@ -7,7 +7,7 @@
 # Environment-configurable constants with fallback defaults
 DP_ENDPOINT_OVERRIDE = os.getenv("BEDROCK_AGENTCORE_DP_ENDPOINT")
 CP_ENDPOINT_OVERRIDE = os.getenv("BEDROCK_AGENTCORE_CP_ENDPOINT")
-DEFAULT_REGION = os.getenv("AWS_REGION", "us-west-2")
+DEFAULT_REGION = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") or "us-west-2"
 
 # Regex for valid AWS region names (e.g., us-east-1, eu-west-2, cn-north-1, us-gov-west-1).
 # Uses \A and \Z anchors to prevent newline injection bypass that $ allows.

src/bedrock_agentcore/config_bundle/__init__.py

@@ -0,0 +1,9 @@
+"""Configuration bundle support for BedrockAgentCore."""
+
+from .bundle import ConfigBundleRef
+from .client import ConfigBundleClient
+
+__all__ = [
+    "ConfigBundleRef",
+    "ConfigBundleClient",
+]

src/bedrock_agentcore/config_bundle/baggage.py

@@ -0,0 +1,97 @@
+"""Parse W3C baggage headers for configuration bundle references."""
+
+import logging
+from typing import Any, Dict, List, Optional
+from urllib.parse import unquote
+
+from .bundle import ConfigBundleRef
+
+BAGGAGE_HEADER = "baggage"
+
+logger = logging.getLogger(__name__)
+
+
+def _extract_baggage(headers: Any) -> Dict[str, List[str]]:
+    """Extract all W3C baggage entries from request headers into a multi-value dict.
+
+    Args:
+        headers: A Starlette ``Headers`` object or a list of ``(name, value)``
+            tuples.  Must preserve duplicate header names as separate entries so
+            that multiple ``baggage`` headers are each processed independently.
+            A plain ``dict`` is not suitable — it can only hold one ``baggage``
+            entry and will silently drop the rest.
+
+    Returns:
+        A dict mapping each baggage key to a list of its decoded values in the
+        order they were encountered.  A key that appears in more than one
+        ``baggage`` header, or more than once within a single header value,
+        accumulates one entry per occurrence.
+
+    Notes:
+        - Header name matching is case-insensitive (``Baggage`` == ``baggage``).
+        - Per-entry properties (the ``;property=value`` suffix) are stripped
+          before the value is returned.
+        - Values are percent-decoded (``%XX`` → character).
+        - Entries with no ``=`` sign, an empty key, or an empty value are skipped.
+    """
+    result: Dict[str, List[str]] = {}
+    items = headers.items() if hasattr(headers, "items") else headers
+    for key, value in items:
+        if key.lower() != BAGGAGE_HEADER:
+            continue
+        for item in value.split(","):
+            item = item.strip()
+            if not item:
+                continue
+            if "=" not in item:
+                logger.warning("Skipping malformed baggage entry (no '='): %r", item)
+                continue
+            entry_key, _, entry_value = item.partition("=")
+            entry_key = entry_key.strip()
+            if not entry_key:
+                logger.warning("Skipping baggage entry with empty key in: %r", item)
+                continue
+            decoded_value = unquote(entry_value.split(";")[0].strip())
+            if not decoded_value:
+                logger.warning("Skipping baggage entry with empty value for key %r", entry_key)
+                continue
+            result.setdefault(entry_key, []).append(decoded_value)
+    return result
+
+
+def _parse_config_bundle_baggage(all_baggage: Dict[str, List[str]]) -> Optional[ConfigBundleRef]:
+    """Build a ``ConfigBundleRef`` from extracted baggage entries, or ``None`` if absent.
+
+    Expects ``all_baggage`` to have been produced by :func:`_extract_baggage`.
+    The two keys used are:
+
+    - ``aws.agentcore.configbundle_arn`` — full ARN of the configuration bundle
+    - ``aws.agentcore.configbundle_version`` — version ID of the bundle
+
+    Only a single bundle is supported::
+
+        baggage: aws.agentcore.configbundle_arn=<arn>,aws.agentcore.configbundle_version=<version>
+
+    If multiple values are present for either key, only the first is used and a
+    warning is logged.
+
+    Args:
+        all_baggage: Multi-value baggage dict from :func:`_extract_baggage`.
+
+    Returns:
+        A ``ConfigBundleRef`` when both keys are present and valid, otherwise ``None``.
+    """
+    arns = all_baggage.get("aws.agentcore.configbundle_arn", [])
+    versions = all_baggage.get("aws.agentcore.configbundle_version", [])
+
+    if not arns or not versions:
+        return None
+
+    if len(arns) > 1 or len(versions) > 1:
+        logger.warning("Multiple config bundle ARNs/versions found in baggage — only the first will be used")
+
+    try:
+        return ConfigBundleRef(bundle_arn=arns[0], bundle_version=versions[0])
+    except ValueError as e:
+        logger.warning("Skipping invalid config bundle ref (arn=%r, version=%r): %s", arns[0], versions[0], e)
+        return None

src/bedrock_agentcore/config_bundle/bundle.py

@@ -0,0 +1,34 @@
+"""Configuration bundle reference model."""
+
+from dataclasses import dataclass
+from typing import Any, Dict
+
+# ComponentConfigurationMap value: {componentId: {"configuration": <Document>}}
+ConfigBundleComponents = Dict[str, Dict[str, Any]]
+
+
+@dataclass(frozen=True)
+class ConfigBundleRef:
+    """Lightweight reference to a configuration bundle version, parsed from OTEL baggage.
+
+    .. warning::
+        This feature is in preview and may change in future releases.
+    """
+
+    bundle_arn: str
+    bundle_version: str
+
+    def __post_init__(self) -> None:
+        """Validate bundle ARN and version."""
+        if not self.bundle_arn:
+            raise ValueError("bundle_arn must not be empty")
+        if not self.bundle_version:
+            raise ValueError("bundle_version must not be empty")
+        parts = self.bundle_arn.rsplit("/", 1)
+        if len(parts) != 2 or not parts[1]:
+            raise ValueError(f"bundle_arn does not contain a valid bundle ID segment: {self.bundle_arn!r}")
+
+    @property
+    def bundle_id(self) -> str:
+        """Extract bundle ID from ARN (last path segment after '/')."""
+        return self.bundle_arn.rsplit("/", 1)[-1]

src/bedrock_agentcore/config_bundle/client.py

@@ -0,0 +1,75 @@
+"""Client for fetching configuration bundle versions from the AgentCore control plane."""
+
+import logging
+import threading
+from typing import Optional
+
+import boto3
+
+from .._utils.endpoints import DEFAULT_REGION, get_control_plane_endpoint
+
+logger = logging.getLogger(__name__)
+
+_ALLOWED_OPERATIONS = frozenset(
+    {
+        "create_configuration_bundle",
+        "delete_configuration_bundle",
+        "get_configuration_bundle",
+        "get_configuration_bundle_version",
+        "list_configuration_bundle_versions",
+        "list_configuration_bundles",
+        "update_configuration_bundle",
+    }
+)
+
+
+class ConfigBundleClient:
+    """Client for AgentCore configuration bundle operations.
+
+    .. warning::
+        This feature is in preview and may change in future releases.
+
+    Wraps the ``bedrock-agentcore-control`` boto3 client and forwards all method
+    calls to it via ``__getattr__``, so any boto3 operation (e.g.
+    ``get_configuration_bundle_version``, ``list_configuration_bundles``) is
+    available without explicit definitions.
+
+    Intended to be created once at application startup and reused across requests.
+    The underlying boto3 client is created lazily on first use so that agents
+    which never receive config bundle baggage incur no startup overhead.
+    """
+
+    def __init__(self, region_name: Optional[str] = None, boto3_session: Optional[boto3.Session] = None):
+        """Initialise the client with an optional region and boto3 session."""
+        self._region = region_name or DEFAULT_REGION
+        self._boto3_session = boto3_session
+        self._client = None
+        self._client_lock = threading.Lock()
+
+    def _get_client(self):
+        # Use __dict__ directly to avoid triggering __getattr__ if _client is
+        # not yet set (e.g. during unpickling before __init__ completes).
+        if self.__dict__.get("_client") is None:
+            with self._client_lock:
+                if self.__dict__.get("_client") is None:
+                    session = self._boto3_session or boto3.Session()
+                    self._client = session.client(
+                        "bedrock-agentcore-control",
+                        region_name=self._region,
+                        endpoint_url=get_control_plane_endpoint(self._region),
+                    )
+        return self._client
+
+    def __getattr__(self, name: str):
+        """Forward configuration bundle method calls to the underlying boto3 client.
+
+        Only operations in ``_ALLOWED_OPERATIONS`` are exposed. Attempts to call
+        any other operation raise ``AttributeError``.
+
+        Uses ``object.__getattribute__`` to access ``_get_client`` so that if Python
+        looks up dunder attributes during unpickling or deepcopy before instance
+        attributes are initialised, this method does not recurse into itself.
+        """
+        if name not in _ALLOWED_OPERATIONS:
+            raise AttributeError(f"'{type(self).__name__}' does not expose operation '{name}'")
+        return getattr(object.__getattribute__(self, "_get_client")(), name)

src/bedrock_agentcore/evaluation/__init__.py

@@ -6,15 +6,31 @@
     EvaluatorOutput,
     custom_code_based_evaluator,
 )
+from bedrock_agentcore.evaluation.runner.batch.batch_evaluation_models import (
+    BatchEvaluationResult,
+    BatchEvaluationRunConfig,
+    BatchEvaluationSummary,
+    BatchEvaluatorConfig,
+    CloudWatchDataSourceConfig,
+    CloudWatchOutputDataConfig,
+    EvaluatorStatistics,
+    EvaluatorSummary,
+    FailedScenario,
+)
+from bedrock_agentcore.evaluation.runner.batch.batch_evaluation_runner import (
+    BatchEvaluationRunner,
+)
 from bedrock_agentcore.evaluation.runner.dataset_providers import (
     DatasetProvider,
     FileDatasetProvider,
 )
 from bedrock_agentcore.evaluation.runner.dataset_types import (
+    ActorProfile,
     Dataset,
     Input,
     PredefinedScenario,
     Scenario,
+    SimulatedScenario,
     Turn,
 )
 from bedrock_agentcore.evaluation.runner.invoker_types import (
@@ -34,6 +50,8 @@
     ScenarioExecutionResult,
     ScenarioExecutor,
     ScenarioResult,
+    SimulatedScenarioExecutor,
+    SimulationConfig,
 )
 from bedrock_agentcore.evaluation.span_to_adot_serializer import (
     convert_strands_to_adot,
@@ -43,7 +61,18 @@
 )
 
 __all__ = [
+    "ActorProfile",
     "AgentInvokerFn",
+    "BatchEvaluationRunner",
+    "BatchEvaluationResult",
+    "BatchEvaluationRunConfig",
+    "CloudWatchOutputDataConfig",
+    "CloudWatchDataSourceConfig",
+    "BatchEvaluatorConfig",
+    "BatchEvaluationSummary",
+    "EvaluatorStatistics",
+    "EvaluatorSummary",
+    "FailedScenario",
     "AgentInvokerInput",
     "AgentInvokerOutput",
     "CloudWatchAgentSpanCollector",
@@ -65,10 +94,13 @@
     "ScenarioExecutor",
     "ScenarioResult",
     "AgentSpanCollector",
+    "SimulationConfig",
     "StrandsEvalsAgentCoreEvaluator",
     "Turn",
     "PredefinedScenario",
     "PredefinedScenarioExecutor",
+    "SimulatedScenario",
+    "SimulatedScenarioExecutor",
     "custom_code_based_evaluator",
     "convert_strands_to_adot",
     "create_strands_evaluator",

src/bedrock_agentcore/evaluation/runner/__init__.py

@@ -2,10 +2,12 @@
 
 from .dataset_providers import DatasetProvider, FileDatasetProvider
 from .dataset_types import (
+    ActorProfile,
     Dataset,
     Input,
     PredefinedScenario,
     Scenario,
+    SimulatedScenario,
     Turn,
 )
 from .invoker_types import (
@@ -22,33 +24,43 @@
     EvaluatorResult,
     OnDemandEvaluationDatasetRunner,
     ScenarioResult,
+    SimulationConfig,
 )
 from .scenario_executor import (
+    AgentCoreActorSimulator,
     PredefinedScenarioExecutor,
     ScenarioExecutionResult,
     ScenarioExecutor,
+    SimulatedScenarioExecutor,
+    SimulatorResult,
 )
 
 __all__ = [
+    "ActorProfile",
     "AgentInvokerFn",
     "AgentInvokerInput",
     "AgentInvokerOutput",
+    "AgentSpanCollector",
     "CloudWatchAgentSpanCollector",
     "Dataset",
     "DatasetProvider",
     "EvaluationResult",
     "EvaluationRunConfig",
-    "OnDemandEvaluationDatasetRunner",
     "EvaluatorConfig",
     "EvaluatorResult",
     "FileDatasetProvider",
     "Input",
+    "OnDemandEvaluationDatasetRunner",
+    "PredefinedScenario",
+    "AgentCoreActorSimulator",
+    "PredefinedScenarioExecutor",
     "Scenario",
     "ScenarioExecutionResult",
     "ScenarioExecutor",
+    "SimulatorResult",
     "ScenarioResult",
-    "AgentSpanCollector",
+    "SimulatedScenario",
+    "SimulatedScenarioExecutor",
+    "SimulationConfig",
     "Turn",
-    "PredefinedScenario",
-    "PredefinedScenarioExecutor",
 ]