From b2d6f2c5bba82927b6756d3fb45a962f6892fa28 Mon Sep 17 00:00:00 2001 From: Antoine Griffon Date: Tue, 21 Oct 2025 12:10:26 +0200 Subject: [PATCH 1/2] Fix structured output capability check for unsupported models --- src/agent/src/Agent.php | 8 ++++ src/agent/tests/AgentTest.php | 41 +++++++++++++++++++ .../src/Bridge/Gemini/ModelCatalog.php | 5 --- .../src/Bridge/VertexAi/ModelCatalog.php | 2 - 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/agent/src/Agent.php b/src/agent/src/Agent.php index e50eb8677..2819bd19b 100644 --- a/src/agent/src/Agent.php +++ b/src/agent/src/Agent.php @@ -14,6 +14,7 @@ use Symfony\AI\Agent\Exception\InvalidArgumentException; use Symfony\AI\Agent\Exception\MissingModelSupportException; use Symfony\AI\Agent\Exception\RuntimeException; +use Symfony\AI\Platform\Capability; use Symfony\AI\Platform\Exception\ExceptionInterface; use Symfony\AI\Platform\Message\MessageBag; use Symfony\AI\Platform\PlatformInterface; @@ -77,6 +78,13 @@ public function call(MessageBag $messages, array $options = []): ResultInterface $messages = $input->getMessageBag(); $options = $input->getOptions(); + if (isset($options['output_structure'])) { + $modelObject = $this->platform->getModelCatalog()->getModel($model); + if (!\in_array(Capability::OUTPUT_STRUCTURED, $modelObject->getCapabilities(), true)) { + throw MissingModelSupportException::forStructuredOutput($modelObject->getName()); + } + } + $result = $this->platform->invoke($model, $messages, $options)->getResult(); $output = new Output($model, $result, $messages, $options); diff --git a/src/agent/tests/AgentTest.php b/src/agent/tests/AgentTest.php index a3e3636de..e95459746 100644 --- a/src/agent/tests/AgentTest.php +++ b/src/agent/tests/AgentTest.php @@ -16,15 +16,19 @@ use Symfony\AI\Agent\AgentAwareInterface; use Symfony\AI\Agent\AgentInterface; use Symfony\AI\Agent\Exception\InvalidArgumentException; +use Symfony\AI\Agent\Exception\MissingModelSupportException; use Symfony\AI\Agent\Input; use Symfony\AI\Agent\InputProcessorInterface; use Symfony\AI\Agent\Output; use Symfony\AI\Agent\OutputProcessorInterface; +use Symfony\AI\Platform\Capability; use Symfony\AI\Platform\Message\Content\Audio; use Symfony\AI\Platform\Message\Content\Image; use Symfony\AI\Platform\Message\Content\Text; use Symfony\AI\Platform\Message\MessageBag; use Symfony\AI\Platform\Message\UserMessage; +use Symfony\AI\Platform\Model; +use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface; use Symfony\AI\Platform\PlatformInterface; use Symfony\AI\Platform\Result\DeferredResult; use Symfony\AI\Platform\Result\RawResultInterface; @@ -221,6 +225,43 @@ public function testCallPassesOptionsToInvoke() $this->assertSame($result, $actualResult); } + public function testCallThrowsExceptionForMissingStructuredOutputSupport() + { + $modelName = 'model-without-structured-output'; + + $modelMock = $this->createMock(Model::class); + $modelMock->method('getCapabilities')->willReturn([ + Capability::INPUT_MESSAGES, + Capability::OUTPUT_TEXT, + ]); + $modelMock->method('getName')->willReturn($modelName); + + $modelCatalogMock = $this->createMock(ModelCatalogInterface::class); + $modelCatalogMock + ->expects($this->once()) + ->method('getModel') + ->with($modelName) + ->willReturn($modelMock); + + $platformMock = $this->createMock(PlatformInterface::class); + $platformMock + ->expects($this->once()) + ->method('getModelCatalog') + ->willReturn($modelCatalogMock); + $platformMock + ->expects($this->never()) + ->method('invoke'); + + $agent = new Agent($platformMock, $modelName); + $messages = new MessageBag(new UserMessage(new Text('Hello'))); + $options = ['output_structure' => 'App\Dto\MyStructure']; + + $this->expectException(MissingModelSupportException::class); + $this->expectExceptionMessage(\sprintf('Model "%s" does not support "structured output".', $modelName)); + + $agent->call($messages, $options); + } + public function testConstructorAcceptsTraversableProcessors() { $platform = $this->createMock(PlatformInterface::class); diff --git a/src/platform/src/Bridge/Gemini/ModelCatalog.php b/src/platform/src/Bridge/Gemini/ModelCatalog.php index b78fa3320..cad961b7a 100644 --- a/src/platform/src/Bridge/Gemini/ModelCatalog.php +++ b/src/platform/src/Bridge/Gemini/ModelCatalog.php @@ -69,7 +69,6 @@ public function __construct(array $additionalModels = []) Capability::INPUT_AUDIO, Capability::INPUT_PDF, Capability::OUTPUT_STREAMING, - Capability::OUTPUT_STRUCTURED, Capability::TOOL_CALLING, ], ], @@ -81,7 +80,6 @@ public function __construct(array $additionalModels = []) Capability::INPUT_AUDIO, Capability::INPUT_PDF, Capability::OUTPUT_STREAMING, - Capability::OUTPUT_STRUCTURED, Capability::TOOL_CALLING, ], ], @@ -93,7 +91,6 @@ public function __construct(array $additionalModels = []) Capability::INPUT_AUDIO, Capability::INPUT_PDF, Capability::OUTPUT_STREAMING, - Capability::OUTPUT_STRUCTURED, Capability::TOOL_CALLING, ], ], @@ -105,7 +102,6 @@ public function __construct(array $additionalModels = []) Capability::INPUT_AUDIO, Capability::INPUT_PDF, Capability::OUTPUT_STREAMING, - Capability::OUTPUT_STRUCTURED, Capability::TOOL_CALLING, ], ], @@ -117,7 +113,6 @@ public function __construct(array $additionalModels = []) Capability::INPUT_AUDIO, Capability::INPUT_PDF, Capability::OUTPUT_STREAMING, - Capability::OUTPUT_STRUCTURED, Capability::TOOL_CALLING, ], ], diff --git a/src/platform/src/Bridge/VertexAi/ModelCatalog.php b/src/platform/src/Bridge/VertexAi/ModelCatalog.php index f6bc3074c..0dc21178f 100644 --- a/src/platform/src/Bridge/VertexAi/ModelCatalog.php +++ b/src/platform/src/Bridge/VertexAi/ModelCatalog.php @@ -66,7 +66,6 @@ public function __construct(array $additionalModels = []) Capability::INPUT_PDF, Capability::OUTPUT_TEXT, Capability::OUTPUT_STREAMING, - Capability::OUTPUT_STRUCTURED, Capability::TOOL_CALLING, ], ], @@ -92,7 +91,6 @@ public function __construct(array $additionalModels = []) Capability::INPUT_PDF, Capability::OUTPUT_TEXT, Capability::OUTPUT_STREAMING, - Capability::OUTPUT_STRUCTURED, Capability::TOOL_CALLING, ], ], From 0d976ce3e0bb82602d9d7b0a581df201a5b2a982 Mon Sep 17 00:00:00 2001 From: Antoine Griffon Date: Tue, 21 Oct 2025 18:49:02 +0200 Subject: [PATCH 2/2] Fix structured output capability check via AgentProcessor --- src/agent/src/Agent.php | 8 - .../src/StructuredOutput/AgentProcessor.php | 8 + src/agent/tests/AgentTest.php | 41 ----- .../StructuredOutput/AgentProcessorTest.php | 150 ++++++++++++------ src/ai-bundle/config/services.php | 3 +- 5 files changed, 109 insertions(+), 101 deletions(-) diff --git a/src/agent/src/Agent.php b/src/agent/src/Agent.php index 2819bd19b..e50eb8677 100644 --- a/src/agent/src/Agent.php +++ b/src/agent/src/Agent.php @@ -14,7 +14,6 @@ use Symfony\AI\Agent\Exception\InvalidArgumentException; use Symfony\AI\Agent\Exception\MissingModelSupportException; use Symfony\AI\Agent\Exception\RuntimeException; -use Symfony\AI\Platform\Capability; use Symfony\AI\Platform\Exception\ExceptionInterface; use Symfony\AI\Platform\Message\MessageBag; use Symfony\AI\Platform\PlatformInterface; @@ -78,13 +77,6 @@ public function call(MessageBag $messages, array $options = []): ResultInterface $messages = $input->getMessageBag(); $options = $input->getOptions(); - if (isset($options['output_structure'])) { - $modelObject = $this->platform->getModelCatalog()->getModel($model); - if (!\in_array(Capability::OUTPUT_STRUCTURED, $modelObject->getCapabilities(), true)) { - throw MissingModelSupportException::forStructuredOutput($modelObject->getName()); - } - } - $result = $this->platform->invoke($model, $messages, $options)->getResult(); $output = new Output($model, $result, $messages, $options); diff --git a/src/agent/src/StructuredOutput/AgentProcessor.php b/src/agent/src/StructuredOutput/AgentProcessor.php index 7fb1dd01d..1ab96102b 100644 --- a/src/agent/src/StructuredOutput/AgentProcessor.php +++ b/src/agent/src/StructuredOutput/AgentProcessor.php @@ -17,6 +17,8 @@ use Symfony\AI\Agent\InputProcessorInterface; use Symfony\AI\Agent\Output; use Symfony\AI\Agent\OutputProcessorInterface; +use Symfony\AI\Platform\Capability; +use Symfony\AI\Platform\PlatformInterface; use Symfony\AI\Platform\Result\ObjectResult; use Symfony\Component\PropertyInfo\Extractor\PhpDocExtractor; use Symfony\Component\PropertyInfo\Extractor\ReflectionExtractor; @@ -39,6 +41,7 @@ final class AgentProcessor implements InputProcessorInterface, OutputProcessorIn private string $outputStructure; public function __construct( + private PlatformInterface $platform, private readonly ResponseFormatFactoryInterface $responseFormatFactory = new ResponseFormatFactory(), private ?SerializerInterface $serializer = null, ) { @@ -77,6 +80,11 @@ public function processInput(Input $input): void throw new InvalidArgumentException('Streamed responses are not supported for structured output.'); } + $modelObject = $this->platform->getModelCatalog()->getModel($input->getModel()); + if (!\in_array(Capability::OUTPUT_STRUCTURED, $modelObject->getCapabilities(), true)) { + throw MissingModelSupportException::forStructuredOutput($modelObject->getName()); + } + $options['response_format'] = $this->responseFormatFactory->create($options['output_structure']); $this->outputStructure = $options['output_structure']; diff --git a/src/agent/tests/AgentTest.php b/src/agent/tests/AgentTest.php index e95459746..a3e3636de 100644 --- a/src/agent/tests/AgentTest.php +++ b/src/agent/tests/AgentTest.php @@ -16,19 +16,15 @@ use Symfony\AI\Agent\AgentAwareInterface; use Symfony\AI\Agent\AgentInterface; use Symfony\AI\Agent\Exception\InvalidArgumentException; -use Symfony\AI\Agent\Exception\MissingModelSupportException; use Symfony\AI\Agent\Input; use Symfony\AI\Agent\InputProcessorInterface; use Symfony\AI\Agent\Output; use Symfony\AI\Agent\OutputProcessorInterface; -use Symfony\AI\Platform\Capability; use Symfony\AI\Platform\Message\Content\Audio; use Symfony\AI\Platform\Message\Content\Image; use Symfony\AI\Platform\Message\Content\Text; use Symfony\AI\Platform\Message\MessageBag; use Symfony\AI\Platform\Message\UserMessage; -use Symfony\AI\Platform\Model; -use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface; use Symfony\AI\Platform\PlatformInterface; use Symfony\AI\Platform\Result\DeferredResult; use Symfony\AI\Platform\Result\RawResultInterface; @@ -225,43 +221,6 @@ public function testCallPassesOptionsToInvoke() $this->assertSame($result, $actualResult); } - public function testCallThrowsExceptionForMissingStructuredOutputSupport() - { - $modelName = 'model-without-structured-output'; - - $modelMock = $this->createMock(Model::class); - $modelMock->method('getCapabilities')->willReturn([ - Capability::INPUT_MESSAGES, - Capability::OUTPUT_TEXT, - ]); - $modelMock->method('getName')->willReturn($modelName); - - $modelCatalogMock = $this->createMock(ModelCatalogInterface::class); - $modelCatalogMock - ->expects($this->once()) - ->method('getModel') - ->with($modelName) - ->willReturn($modelMock); - - $platformMock = $this->createMock(PlatformInterface::class); - $platformMock - ->expects($this->once()) - ->method('getModelCatalog') - ->willReturn($modelCatalogMock); - $platformMock - ->expects($this->never()) - ->method('invoke'); - - $agent = new Agent($platformMock, $modelName); - $messages = new MessageBag(new UserMessage(new Text('Hello'))); - $options = ['output_structure' => 'App\Dto\MyStructure']; - - $this->expectException(MissingModelSupportException::class); - $this->expectExceptionMessage(\sprintf('Model "%s" does not support "structured output".', $modelName)); - - $agent->call($messages, $options); - } - public function testConstructorAcceptsTraversableProcessors() { $platform = $this->createMock(PlatformInterface::class); diff --git a/src/agent/tests/StructuredOutput/AgentProcessorTest.php b/src/agent/tests/StructuredOutput/AgentProcessorTest.php index 74391e817..b55358e57 100644 --- a/src/agent/tests/StructuredOutput/AgentProcessorTest.php +++ b/src/agent/tests/StructuredOutput/AgentProcessorTest.php @@ -13,9 +13,11 @@ use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +use Symfony\AI\Agent\Exception\MissingModelSupportException; use Symfony\AI\Agent\Input; use Symfony\AI\Agent\Output; use Symfony\AI\Agent\StructuredOutput\AgentProcessor; +use Symfony\AI\Agent\StructuredOutput\ResponseFormatFactoryInterface; use Symfony\AI\Fixtures\SomeStructure; use Symfony\AI\Fixtures\StructuredOutput\MathReasoning; use Symfony\AI\Fixtures\StructuredOutput\PolymorphicType\ListItemAge; @@ -25,18 +27,31 @@ use Symfony\AI\Fixtures\StructuredOutput\UnionType\HumanReadableTimeUnion; use Symfony\AI\Fixtures\StructuredOutput\UnionType\UnionTypeDto; use Symfony\AI\Fixtures\StructuredOutput\UnionType\UnixTimestampUnion; +use Symfony\AI\Platform\Capability; use Symfony\AI\Platform\Message\MessageBag; +use Symfony\AI\Platform\Message\UserMessage; use Symfony\AI\Platform\Metadata\Metadata; +use Symfony\AI\Platform\Model; +use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface; +use Symfony\AI\Platform\PlatformInterface; use Symfony\AI\Platform\Result\ObjectResult; use Symfony\AI\Platform\Result\TextResult; use Symfony\Component\Serializer\SerializerInterface; +if (!class_exists(__NAMESPACE__.'\ConfigurableResponseFormatFactory')) { + class ConfigurableResponseFormatFactory implements ResponseFormatFactoryInterface { + public function __construct(private array $format = []) {} + public function create(string $structure): array { return $this->format; } + } +} + final class AgentProcessorTest extends TestCase { public function testProcessInputWithOutputStructure() { - $processor = new AgentProcessor(new ConfigurableResponseFormatFactory(['some' => 'format'])); - $input = new Input('gpt-4', new MessageBag(), ['output_structure' => 'SomeStructure']); + $platformMock = $this->createPlatformMock('gpt-4'); + $processor = new AgentProcessor($platformMock, new ConfigurableResponseFormatFactory(['some' => 'format'])); + $input = new Input('gpt-4', new MessageBag(), ['output_structure' => SomeStructure::class]); $processor->processInput($input); @@ -45,7 +60,8 @@ public function testProcessInputWithOutputStructure() public function testProcessInputWithoutOutputStructure() { - $processor = new AgentProcessor(new ConfigurableResponseFormatFactory()); + $platformMock = $this->createMock(PlatformInterface::class); + $processor = new AgentProcessor($platformMock, new ConfigurableResponseFormatFactory()); $input = new Input('gpt-4', new MessageBag()); $processor->processInput($input); @@ -53,30 +69,66 @@ public function testProcessInputWithoutOutputStructure() $this->assertSame([], $input->getOptions()); } + public function testProcessInputThrowsExceptionForMissingSupport() + { + $modelName = 'model-without-structured-output'; + + $modelMock = $this->createMock(Model::class); + $modelMock->method('getCapabilities')->willReturn([ + Capability::INPUT_MESSAGES, + Capability::OUTPUT_TEXT, + ]); + $modelMock->method('getName')->willReturn($modelName); + + $modelCatalogMock = $this->createMock(ModelCatalogInterface::class); + $modelCatalogMock + ->expects($this->once()) + ->method('getModel') + ->with($modelName) + ->willReturn($modelMock); + + $platformMock = $this->createMock(PlatformInterface::class); + $platformMock + ->expects($this->once()) + ->method('getModelCatalog') + ->willReturn($modelCatalogMock); + + $processor = new AgentProcessor($platformMock, new ConfigurableResponseFormatFactory()); + $messages = new MessageBag(new UserMessage(new \Symfony\AI\Platform\Message\Content\Text('Hello'))); + $options = ['output_structure' => 'App\Dto\MyStructure']; + $input = new Input($modelName, $messages, $options); + + $this->expectException(MissingModelSupportException::class); + $this->expectExceptionMessage(\sprintf('Model "%s" does not support "structured output".', $modelName)); + + $processor->processInput($input); + } + public function testProcessOutputWithResponseFormat() { - $processor = new AgentProcessor(new ConfigurableResponseFormatFactory(['some' => 'format'])); + $platformMock = $this->createPlatformMock('gpt-4'); + $processor = new AgentProcessor($platformMock, new ConfigurableResponseFormatFactory(['some' => 'format'])); $options = ['output_structure' => SomeStructure::class]; $input = new Input('gpt-4', new MessageBag(), $options); $processor->processInput($input); $result = new TextResult('{"some": "data"}'); - $output = new Output('gpt-4', $result, new MessageBag(), $input->getOptions()); - $processor->processOutput($output); $this->assertInstanceOf(ObjectResult::class, $output->getResult()); - $this->assertInstanceOf(SomeStructure::class, $output->getResult()->getContent()); + $resultContent = $output->getResult()->getContent(); + $this->assertInstanceOf(SomeStructure::class, $resultContent); $this->assertInstanceOf(Metadata::class, $output->getResult()->getMetadata()); $this->assertNull($output->getResult()->getRawResult()); - $this->assertSame('data', $output->getResult()->getContent()->some); + $this->assertSame('data', $resultContent->some); } public function testProcessOutputWithComplexResponseFormat() { - $processor = new AgentProcessor(new ConfigurableResponseFormatFactory(['some' => 'format'])); + $platformMock = $this->createPlatformMock('gpt-4'); + $processor = new AgentProcessor($platformMock, new ConfigurableResponseFormatFactory(['some' => 'format'])); $options = ['output_structure' => MathReasoning::class]; $input = new Input('gpt-4', new MessageBag(), $options); @@ -112,15 +164,17 @@ public function testProcessOutputWithComplexResponseFormat() JSON); $output = new Output('gpt-4', $result, new MessageBag(), $input->getOptions()); - $processor->processOutput($output); $this->assertInstanceOf(ObjectResult::class, $output->getResult()); - $this->assertInstanceOf(MathReasoning::class, $structure = $output->getResult()->getContent()); + $structure = $output->getResult()->getContent(); + $this->assertInstanceOf(MathReasoning::class, $structure); $this->assertInstanceOf(Metadata::class, $output->getResult()->getMetadata()); $this->assertNull($output->getResult()->getRawResult()); $this->assertCount(5, $structure->steps); $this->assertInstanceOf(Step::class, $structure->steps[0]); + $this->assertSame("We want to isolate the term with x. First, let's subtract 7 from both sides of the equation.", $structure->steps[0]->explanation); + $this->assertSame("8x + 7 - 7 = -23 - 7", $structure->steps[0]->output); $this->assertInstanceOf(Step::class, $structure->steps[1]); $this->assertInstanceOf(Step::class, $structure->steps[2]); $this->assertInstanceOf(Step::class, $structure->steps[3]); @@ -129,13 +183,11 @@ public function testProcessOutputWithComplexResponseFormat() $this->assertSame('x = -3.75', $structure->finalAnswer); } - /** - * @param class-string $expectedTimeStructure - */ #[DataProvider('unionTimeTypeProvider')] public function testProcessOutputWithUnionTypeResponseFormat(TextResult $result, string $expectedTimeStructure) { - $processor = new AgentProcessor(new ConfigurableResponseFormatFactory(['some' => 'format'])); + $platformMock = $this->createPlatformMock('gpt-4'); + $processor = new AgentProcessor($platformMock, new ConfigurableResponseFormatFactory(['some' => 'format'])); $options = ['output_structure' => UnionTypeDto::class]; $input = new Input('gpt-4', new MessageBag(), $options); @@ -154,21 +206,8 @@ public function testProcessOutputWithUnionTypeResponseFormat(TextResult $result, public static function unionTimeTypeProvider(): array { - $unixTimestampResult = new TextResult(<< 'format'])); + $platformMock = $this->createPlatformMock('gpt-4'); + $processor = new AgentProcessor($platformMock, new ConfigurableResponseFormatFactory(['some' => 'format'])); $options = ['output_structure' => ListOfPolymorphicTypesDto::class]; $input = new Input('gpt-4', new MessageBag(), $options); $processor->processInput($input); $result = new TextResult(<<getOptions()); - $processor->processOutput($output); $this->assertInstanceOf(ObjectResult::class, $output->getResult()); - /** @var ListOfPolymorphicTypesDto $structure */ $structure = $output->getResult()->getContent(); $this->assertInstanceOf(ListOfPolymorphicTypesDto::class, $structure); $this->assertCount(2, $structure->items); - $nameItem = $structure->items[0]; $ageItem = $structure->items[1]; $this->assertInstanceOf(ListItemName::class, $nameItem); $this->assertInstanceOf(ListItemAge::class, $ageItem); - $this->assertSame('John Doe', $nameItem->name); $this->assertSame(24, $ageItem->age); - $this->assertSame('name', $nameItem->type); $this->assertSame('age', $ageItem->type); } public function testProcessOutputWithoutResponseFormat() { + $platformMock = $this->createMock(PlatformInterface::class); $resultFormatFactory = new ConfigurableResponseFormatFactory(); - $serializer = self::createMock(SerializerInterface::class); - $processor = new AgentProcessor($resultFormatFactory, $serializer); + $serializer = $this->createMock(SerializerInterface::class); + $processor = new AgentProcessor($platformMock, $resultFormatFactory, $serializer); $result = new TextResult(''); $output = new Output('gpt4', $result, new MessageBag()); - $processor->processOutput($output); $this->assertSame($result, $output->getResult()); } + + private function createPlatformMock(string $modelName): PlatformInterface + { + $modelMock = $this->createMock(Model::class); + $modelMock->method('getCapabilities')->willReturn([ + Capability::INPUT_MESSAGES, + Capability::OUTPUT_TEXT, + Capability::OUTPUT_STRUCTURED, + ]); + $modelMock->method('getName')->willReturn($modelName); + + $modelCatalogMock = $this->createMock(ModelCatalogInterface::class); + $modelCatalogMock + ->method('getModel') + ->with($modelName) + ->willReturn($modelMock); + + $platformMock = $this->createMock(PlatformInterface::class); + $platformMock + ->method('getModelCatalog') + ->willReturn($modelCatalogMock); + + return $platformMock; + } } diff --git a/src/ai-bundle/config/services.php b/src/ai-bundle/config/services.php index b135a090b..4271bfcc3 100644 --- a/src/ai-bundle/config/services.php +++ b/src/ai-bundle/config/services.php @@ -118,8 +118,9 @@ ->alias(ResponseFormatFactoryInterface::class, 'ai.agent.response_format_factory') ->set('ai.agent.structured_output_processor', StructureOutputProcessor::class) ->args([ + service('ai.platform'), service('ai.agent.response_format_factory'), - service('serializer'), + service('serializer')->nullOnInvalid(), ]) // tools