Support native JSON output and strict tool calls for Anthropic

docs/output.md

@@ -308,7 +308,7 @@ _(This example is complete, it can be run "as is")_
 
 #### Native Output
 
-Native Output mode uses a model's native "Structured Outputs" feature (aka "JSON Schema response format"), where the model is forced to only output text matching the provided JSON schema. Note that this is not supported by all models, and sometimes comes with restrictions. For example, Anthropic does not support this at all, and Gemini cannot use tools at the same time as structured output, and attempting to do so will result in an error.
+Native Output mode uses a model's native "Structured Outputs" feature (aka "JSON Schema response format"), where the model is forced to only output text matching the provided JSON schema. Note that this is not supported by all models, and sometimes comes with restrictions. For example, Gemini cannot use tools at the same time as structured output, and attempting to do so will result in an error.
 
 To use this mode, you can wrap the output type(s) in the [`NativeOutput`][pydantic_ai.output.NativeOutput] marker class that also lets you specify a `name` and `description` if the name and docstring of the type or function are not sufficient.
 

pydantic_ai_slim/pydantic_ai/_json_schema.py

@@ -15,6 +15,9 @@
 class JsonSchemaTransformer(ABC):
     """Walks a JSON schema, applying transformations to it at each level.
 
+    The transformer is called during a model's prepare_request() step to build the JSON schema
+    before it is sent to the model provider.
+
     Note: We may eventually want to rework tools to build the JSON schema from the type directly, using a subclass of
     pydantic.json_schema.GenerateJsonSchema, rather than making use of this machinery.
     """
@@ -30,8 +33,15 @@ def __init__(
         self.schema = schema
 
         self.strict = strict
-        self.is_strict_compatible = True  # Can be set to False by subclasses to set `strict` on `ToolDefinition` when set not set by user explicitly
+        """The `strict` parameter forces the conversion of the original JSON schema (`self.schema`) of a `ToolDefinition` or `OutputObjectDefinition` to a format supported by the model provider.
+
+        The "strict mode" offered by model providers ensures that the model's output adheres closely to the defined schema. However, not all model providers offer it, and their support for various schema features may differ. For example, a model provider's required schema may not support certain validation constraints like `minLength` or `pattern`.
+        """
+        self.is_strict_compatible = True
+        """Whether the schema is compatible with strict mode.
 
+        This value is used to set `ToolDefinition.strict` or `OutputObjectDefinition.strict` when their values are `None`.
+        """
         self.prefer_inlined_defs = prefer_inlined_defs
         self.simplify_nullable_unions = simplify_nullable_unions
 

pydantic_ai_slim/pydantic_ai/models/__init__.py

@@ -57,9 +57,6 @@
     Literal[
         'anthropic:claude-3-5-haiku-20241022',
         'anthropic:claude-3-5-haiku-latest',
-        'anthropic:claude-3-5-sonnet-20240620',
-        'anthropic:claude-3-5-sonnet-20241022',
-        'anthropic:claude-3-5-sonnet-latest',
         'anthropic:claude-3-7-sonnet-20250219',
         'anthropic:claude-3-7-sonnet-latest',
         'anthropic:claude-3-haiku-20240307',
@@ -380,7 +377,10 @@ async def request(
         model_settings: ModelSettings | None,
         model_request_parameters: ModelRequestParameters,
     ) -> ModelResponse:
-        """Make a request to the model."""
+        """Make a request to the model.
+
+        This is ultimately called by `pydantic_ai._agent_graph.ModelRequestNode._make_request(...)`.
+        """
         raise NotImplementedError()
 
     async def count_tokens(
@@ -987,23 +987,27 @@ def get_user_agent() -> str:
     return f'pydantic-ai/{__version__}'
 
 
-def _customize_tool_def(transformer: type[JsonSchemaTransformer], t: ToolDefinition):
-    schema_transformer = transformer(t.parameters_json_schema, strict=t.strict)
+def _customize_tool_def(transformer: type[JsonSchemaTransformer], tool_def: ToolDefinition):
+    """Customize the tool definition using the given transformer.
+
+    If the tool definition has `strict` set to None, the strictness will be inferred from the transformer.
+    """
+    schema_transformer = transformer(tool_def.parameters_json_schema, strict=tool_def.strict)
     parameters_json_schema = schema_transformer.walk()
     return replace(
-        t,
+        tool_def,
         parameters_json_schema=parameters_json_schema,
-        strict=schema_transformer.is_strict_compatible if t.strict is None else t.strict,
+        strict=schema_transformer.is_strict_compatible if tool_def.strict is None else tool_def.strict,
     )
 
 
-def _customize_output_object(transformer: type[JsonSchemaTransformer], o: OutputObjectDefinition):
-    schema_transformer = transformer(o.json_schema, strict=o.strict)
+def _customize_output_object(transformer: type[JsonSchemaTransformer], output_object: OutputObjectDefinition):
+    schema_transformer = transformer(output_object.json_schema, strict=output_object.strict)
     json_schema = schema_transformer.walk()
     return replace(
-        o,
+        output_object,
         json_schema=json_schema,
-        strict=schema_transformer.is_strict_compatible if o.strict is None else o.strict,
+        strict=schema_transformer.is_strict_compatible if output_object.strict is None else output_object.strict,
     )
 
 

pydantic_ai_slim/pydantic_ai/models/anthropic.py

@@ -77,6 +77,7 @@
         BetaContentBlockParam,
         BetaImageBlockParam,
         BetaInputJSONDelta,
+        BetaJSONOutputFormatParam,
         BetaMCPToolResultBlock,
         BetaMCPToolUseBlock,
         BetaMCPToolUseBlockParam,
@@ -205,8 +206,9 @@ def __init__(
             model_name: The name of the Anthropic model to use. List of model names available
                 [here](https://docs.anthropic.com/en/docs/about-claude/models).
             provider: The provider to use for the Anthropic API. Can be either the string 'anthropic' or an
-                instance of `Provider[AsyncAnthropicClient]`. If not provided, the other parameters will be used.
+                instance of `Provider[AsyncAnthropicClient]`. Defaults to 'anthropic'.
             profile: The model profile to use. Defaults to a profile picked by the provider based on the model name.
+                The default 'anthropic' provider will use the default `..profiles.anthropic_model_profile`.
             settings: Default model settings for this model instance.
         """
         self._model_name = model_name
@@ -296,14 +298,29 @@ def prepare_request(
             and thinking.get('type') == 'enabled'
         ):
             if model_request_parameters.output_mode == 'auto':
-                model_request_parameters = replace(model_request_parameters, output_mode='prompted')
+                output_mode = 'native' if self.profile.supports_json_schema_output else 'prompted'
+                model_request_parameters = replace(model_request_parameters, output_mode=output_mode)
             elif (
                 model_request_parameters.output_mode == 'tool' and not model_request_parameters.allow_text_output
             ):  # pragma: no branch
                 # This would result in `tool_choice=required`, which Anthropic does not support with thinking.
+                output_mode = 'NativeOutput' if self.profile.supports_json_schema_output else 'PromptedOutput'
                 raise UserError(
-                    'Anthropic does not support thinking and output tools at the same time. Use `output_type=PromptedOutput(...)` instead.'
+                    f'Anthropic does not support thinking and output tools at the same time. Use `output_type={output_mode}(...)` instead.'
                 )
+
+        # NOTE forcing `strict=True` here is a bit eager, because the transformer may still determine that the transformation is lossy.
+        # so we're relying on anthropic's strict mode being better than prompting the model with pydantic's schema
+        if model_request_parameters.output_mode == 'native' and model_request_parameters.output_object is not None:
+            # force strict=True for native output
+            # this needs to be done here because `super().prepare_request` calls
+            # -> Model.customize_request_parameters(model_request_parameters) which calls
+            # -> -> _customize_output_object(transformer: type[JsonSchemaTransformer], output_object: OutputObjectDefinition)
+            # which finally instantiates the transformer (default AnthropicJsonSchemaTransformer)
+            # `schema_transformer = transformer(output_object.json_schema, strict=output_object.strict)`
+            model_request_parameters = replace(
+                model_request_parameters, output_object=replace(model_request_parameters.output_object, strict=True)
+            )
         return super().prepare_request(model_settings, model_request_parameters)
 
     @overload
@@ -333,16 +350,24 @@ async def _messages_create(
         model_settings: AnthropicModelSettings,
         model_request_parameters: ModelRequestParameters,
     ) -> BetaMessage | AsyncStream[BetaRawMessageStreamEvent]:
-        # standalone function to make it easier to override
+        """Calls the Anthropic API to create a message.
+
+        This is the last step before sending the request to the API.
+        Most preprocessing has happened in `prepare_request()`.
+        """
         tools = self._get_tools(model_request_parameters, model_settings)
-        tools, mcp_servers, beta_features = self._add_builtin_tools(tools, model_request_parameters)
+        tools, mcp_servers, builtin_tool_betas = self._add_builtin_tools(tools, model_request_parameters)
+        output_format = self._native_output_format(model_request_parameters)
 
         tool_choice = self._infer_tool_choice(tools, model_settings, model_request_parameters)
 
         system_prompt, anthropic_messages = await self._map_message(messages, model_request_parameters, model_settings)
 
+        betas_set = self._get_required_betas(tools, model_request_parameters)
+        betas_set.update(builtin_tool_betas)
+
         try:
-            extra_headers = self._map_extra_headers(beta_features, model_settings)
+            betas, extra_headers = self._prepare_betas_and_headers(betas_set, model_settings)
 
             return await self.client.beta.messages.create(
                 max_tokens=model_settings.get('max_tokens', 4096),
@@ -352,6 +377,8 @@ async def _messages_create(
                 tools=tools or OMIT,
                 tool_choice=tool_choice or OMIT,
                 mcp_servers=mcp_servers or OMIT,
+                output_format=output_format or OMIT,
+                betas=betas or OMIT,
                 stream=stream,
                 thinking=model_settings.get('anthropic_thinking', OMIT),
                 stop_sequences=model_settings.get('stop_sequences', OMIT),
@@ -380,14 +407,18 @@ async def _messages_count_tokens(
 
         # standalone function to make it easier to override
         tools = self._get_tools(model_request_parameters, model_settings)
-        tools, mcp_servers, beta_features = self._add_builtin_tools(tools, model_request_parameters)
+        tools, mcp_servers, builtin_tool_betas = self._add_builtin_tools(tools, model_request_parameters)
+        output_format = self._native_output_format(model_request_parameters)
 
         tool_choice = self._infer_tool_choice(tools, model_settings, model_request_parameters)
 
         system_prompt, anthropic_messages = await self._map_message(messages, model_request_parameters, model_settings)
 
+        betas = self._get_required_betas(tools, model_request_parameters)
+        betas.update(builtin_tool_betas)
+
         try:
-            extra_headers = self._map_extra_headers(beta_features, model_settings)
+            betas_list, extra_headers = self._prepare_betas_and_headers(betas, model_settings)
 
             return await self.client.beta.messages.count_tokens(
                 system=system_prompt or OMIT,
@@ -396,6 +427,8 @@ async def _messages_count_tokens(
                 tools=tools or OMIT,
                 tool_choice=tool_choice or OMIT,
                 mcp_servers=mcp_servers or OMIT,
+                betas=betas_list or OMIT,
+                output_format=output_format or OMIT,
                 thinking=model_settings.get('anthropic_thinking', OMIT),
                 timeout=model_settings.get('timeout', NOT_GIVEN),
                 extra_headers=extra_headers,
@@ -497,10 +530,31 @@ def _get_tools(
 
         return tools
 
+    def _get_required_betas(
+        self, tools: list[BetaToolUnionParam], model_request_parameters: ModelRequestParameters
+    ) -> set[str]:
+        """Determine which beta features are needed based on tools and output format.
+
+        Args:
+            tools: The transformed tool dictionaries that will be sent to the API
+            model_request_parameters: Model request parameters containing output settings
+
+        Returns:
+            Set of beta feature strings (naturally deduplicated)
+        """
+        betas: set[str] = set()
+
+        has_strict_tools = any(tool.get('strict') for tool in tools)
+
+        if has_strict_tools or model_request_parameters.output_mode == 'native':
+            betas.add('structured-outputs-2025-11-13')
+
+        return betas
+
     def _add_builtin_tools(
         self, tools: list[BetaToolUnionParam], model_request_parameters: ModelRequestParameters
-    ) -> tuple[list[BetaToolUnionParam], list[BetaRequestMCPServerURLDefinitionParam], list[str]]:
-        beta_features: list[str] = []
+    ) -> tuple[list[BetaToolUnionParam], list[BetaRequestMCPServerURLDefinitionParam], set[str]]:
+        beta_features: set[str] = set()
         mcp_servers: list[BetaRequestMCPServerURLDefinitionParam] = []
         for tool in model_request_parameters.builtin_tools:
             if isinstance(tool, WebSearchTool):
@@ -517,14 +571,14 @@ def _add_builtin_tools(
                 )
             elif isinstance(tool, CodeExecutionTool):  # pragma: no branch
                 tools.append(BetaCodeExecutionTool20250522Param(name='code_execution', type='code_execution_20250522'))
-                beta_features.append('code-execution-2025-05-22')
+                beta_features.add('code-execution-2025-05-22')
             elif isinstance(tool, MemoryTool):  # pragma: no branch
                 if 'memory' not in model_request_parameters.tool_defs:
                     raise UserError("Built-in `MemoryTool` requires a 'memory' tool to be defined.")
                 # Replace the memory tool definition with the built-in memory tool
                 tools = [tool for tool in tools if tool['name'] != 'memory']
                 tools.append(BetaMemoryTool20250818Param(name='memory', type='memory_20250818'))
-                beta_features.append('context-management-2025-06-27')
+                beta_features.add('context-management-2025-06-27')
             elif isinstance(tool, MCPServerTool) and tool.url:
                 mcp_server_url_definition_param = BetaRequestMCPServerURLDefinitionParam(
                     type='url',
@@ -539,7 +593,7 @@ def _add_builtin_tools(
                 if tool.authorization_token:  # pragma: no cover
                     mcp_server_url_definition_param['authorization_token'] = tool.authorization_token
                 mcp_servers.append(mcp_server_url_definition_param)
-                beta_features.append('mcp-client-2025-04-04')
+                beta_features.add('mcp-client-2025-04-04')
             else:  # pragma: no cover
                 raise UserError(
                     f'`{tool.__class__.__name__}` is not supported by `AnthropicModel`. If it should be, please file an issue.'
@@ -567,15 +621,28 @@ def _infer_tool_choice(
 
             return tool_choice
 
-    def _map_extra_headers(self, beta_features: list[str], model_settings: AnthropicModelSettings) -> dict[str, str]:
-        """Apply beta_features to extra_headers in model_settings."""
+    def _prepare_betas_and_headers(
+        self, betas: set[str], model_settings: AnthropicModelSettings
+    ) -> tuple[list[str], dict[str, str]]:
+        """Prepare beta features list and extra headers for API request.
+
+        Handles merging custom anthropic-beta header from extra_headers into betas set
+        and ensuring User-Agent is set.
+
+        Args:
+            betas: Set of beta feature strings (naturally deduplicated)
+            model_settings: Model settings containing extra_headers
+
+        Returns:
+            Tuple of (betas list, extra_headers dict)
+        """
         extra_headers = model_settings.get('extra_headers', {})
         extra_headers.setdefault('User-Agent', get_user_agent())
-        if beta_features:
-            if 'anthropic-beta' in extra_headers:
-                beta_features.insert(0, extra_headers['anthropic-beta'])
-            extra_headers['anthropic-beta'] = ','.join(beta_features)
-        return extra_headers
+
+        if beta_header := extra_headers.pop('anthropic-beta', None):
+            betas.update({stripped_beta for beta in beta_header.split(',') if (stripped_beta := beta.strip())})
+
+        return sorted(betas), extra_headers
 
     async def _map_message(  # noqa: C901
         self,
@@ -846,13 +913,23 @@ async def _map_user_prompt(
                 else:
                     raise RuntimeError(f'Unsupported content type: {type(item)}')  # pragma: no cover
 
-    @staticmethod
-    def _map_tool_definition(f: ToolDefinition) -> BetaToolParam:
-        return {
+    def _map_tool_definition(self, f: ToolDefinition) -> BetaToolParam:
+        """Maps a `ToolDefinition` dataclass to an Anthropic `BetaToolParam` dictionary."""
+        tool_param: BetaToolParam = {
             'name': f.name,
             'description': f.description or '',
             'input_schema': f.parameters_json_schema,
         }
+        if f.strict and self.profile.supports_json_schema_output:  # pragma: no branch
+            tool_param['strict'] = f.strict
+        return tool_param
+
+    @staticmethod
+    def _native_output_format(model_request_parameters: ModelRequestParameters) -> BetaJSONOutputFormatParam | None:
+        if model_request_parameters.output_mode != 'native':
+            return None
+        assert model_request_parameters.output_object is not None
+        return {'type': 'json_schema', 'schema': model_request_parameters.output_object.json_schema}
 
 
 def _map_usage(

pydantic_ai_slim/pydantic_ai/models/openai.py

@@ -1569,7 +1569,7 @@ async def _map_messages(  # noqa: C901
                             param['id'] = id
                         openai_messages.append(param)
                     elif isinstance(item, BuiltinToolCallPart):
-                        if item.provider_name == self.system and send_item_ids:
+                        if item.provider_name == self.system and send_item_ids:  # pragma: no branch
                             if (
                                 item.tool_name == CodeExecutionTool.kind
                                 and item.tool_call_id
@@ -1639,7 +1639,7 @@ async def _map_messages(  # noqa: C901
                                     openai_messages.append(mcp_call_item)
 
                     elif isinstance(item, BuiltinToolReturnPart):
-                        if item.provider_name == self.system and send_item_ids:
+                        if item.provider_name == self.system and send_item_ids:  # pragma: no branch
                             if (
                                 item.tool_name == CodeExecutionTool.kind
                                 and code_interpreter_item is not None