diff --git a/src/ai-bundle/config/services.php b/src/ai-bundle/config/services.php index b135a090b..b70b86a39 100644 --- a/src/ai-bundle/config/services.php +++ b/src/ai-bundle/config/services.php @@ -11,6 +11,7 @@ namespace Symfony\Component\DependencyInjection\Loader\Configurator; +use Symfony\AI\Agent\AgentInterface; use Symfony\AI\Agent\StructuredOutput\AgentProcessor as StructureOutputProcessor; use Symfony\AI\Agent\StructuredOutput\ResponseFormatFactory; use Symfony\AI\Agent\StructuredOutput\ResponseFormatFactoryInterface; @@ -23,6 +24,8 @@ use Symfony\AI\AiBundle\Command\AgentCallCommand; use Symfony\AI\AiBundle\Command\PlatformInvokeCommand; use Symfony\AI\AiBundle\Profiler\DataCollector; +use Symfony\AI\AiBundle\Profiler\TraceableAgent; +use Symfony\AI\AiBundle\Profiler\TraceableToolbox; use Symfony\AI\AiBundle\Security\EventListener\IsGrantedToolAttributeListener; use Symfony\AI\Chat\Command\DropStoreCommand as DropMessageStoreCommand; use Symfony\AI\Chat\Command\SetupStoreCommand as SetupMessageStoreCommand; @@ -165,10 +168,18 @@ ->tag('kernel.event_listener') // profiler + ->set('ai.traceable_agent', TraceableAgent::class) + ->decorate(AgentInterface::class, priority: 5) + ->args([ + service('.inner'), + service('ai.data_collector'), + service('request_stack'), + ]) ->set('ai.data_collector', DataCollector::class) ->args([ - tagged_iterator('ai.traceable_platform'), - tagged_iterator('ai.traceable_toolbox'), + tagged_iterator('ai.platform'), + service('ai.toolbox.default'), + tagged_iterator('ai.toolbox'), ]) ->tag('data_collector') diff --git a/src/ai-bundle/src/Profiler/DataCollector.php b/src/ai-bundle/src/Profiler/DataCollector.php index 88e8ac56d..c5b5fd4b7 100644 --- a/src/ai-bundle/src/Profiler/DataCollector.php +++ b/src/ai-bundle/src/Profiler/DataCollector.php @@ -11,18 +11,20 @@ namespace Symfony\AI\AiBundle\Profiler; -use Symfony\AI\Agent\Toolbox\ToolResult; -use Symfony\AI\Platform\Metadata\Metadata; +use Symfony\AI\Agent\Toolbox\ToolboxInterface; +use Symfony\AI\Platform\Model; use Symfony\AI\Platform\Tool\Tool; use Symfony\Bundle\FrameworkBundle\DataCollector\AbstractDataCollector; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpKernel\DataCollector\LateDataCollectorInterface; +use Symfony\Component\VarDumper\Cloner\Data; /** * @author Christopher Hertel * * @phpstan-import-type PlatformCallData from TraceablePlatform + * @phpstan-import-type ToolCallData from TraceableToolbox */ final class DataCollector extends AbstractDataCollector implements LateDataCollectorInterface { @@ -37,11 +39,17 @@ final class DataCollector extends AbstractDataCollector implements LateDataColle private readonly array $toolboxes; /** - * @param TraceablePlatform[] $platforms - * @param TraceableToolbox[] $toolboxes + * @var list + */ + private array $collectedChatCalls = []; + + /** + * @param iterable $platforms + * @param iterable $toolboxes */ public function __construct( iterable $platforms, + private readonly ToolboxInterface $defaultToolBox, iterable $toolboxes, ) { $this->platforms = $platforms instanceof \Traversable ? iterator_to_array($platforms) : $platforms; @@ -50,15 +58,26 @@ public function __construct( public function collect(Request $request, Response $response, ?\Throwable $exception = null): void { - $this->lateCollect(); } public function lateCollect(): void { $this->data = [ - 'tools' => $this->getAllTools(), + 'tools' => $this->defaultToolBox->getTools(), 'platform_calls' => array_merge(...array_map($this->awaitCallResults(...), $this->platforms)), 'tool_calls' => array_merge(...array_map(fn (TraceableToolbox $toolbox) => $toolbox->calls, $this->toolboxes)), + 'chat_calls' => $this->cloneVar($this->collectedChatCalls), + ]; + } + + public function collectChatCall(string $method, float $duration, mixed $input, mixed $result, ?\Throwable $error): void + { + $this->collectedChatCalls[] = [ + 'method' => $method, + 'duration' => $duration, + 'input' => $input, + 'result' => $result, + 'error' => $error, ]; } @@ -84,7 +103,7 @@ public function getTools(): array } /** - * @return ToolResult[] + * @return ToolCallData[] */ public function getToolCalls(): array { @@ -92,36 +111,46 @@ public function getToolCalls(): array } /** - * @return Tool[] + * @return list */ - private function getAllTools(): array + public function getChatCalls(): array { - return array_merge(...array_map(fn (TraceableToolbox $toolbox) => $toolbox->getTools(), $this->toolboxes)); + if (!isset($this->data['chat_calls'])) { + return []; + } + + $chatCalls = $this->data['chat_calls']->getValue(true); + + /** @var list $chatCalls */ + return $chatCalls; + } + + public function reset(): void + { + $this->data = []; + $this->collectedChatCalls = []; } /** * @return array{ - * model: string, - * input: array|string|object, - * options: array, - * result: string|iterable|object|null, - * metadata: Metadata, + * model: Model, + * input: array|string|object, + * options: array, + * result: string|iterable|object|null * }[] */ private function awaitCallResults(TraceablePlatform $platform): array { $calls = $platform->calls; foreach ($calls as $key => $call) { - $result = $call['result']->getResult(); + $result = $call['result']; if (isset($platform->resultCache[$result])) { $call['result'] = $platform->resultCache[$result]; } else { - $call['result'] = $result->getContent(); + $call['result'] = $result->asText(); } - $call['metadata'] = $result->getMetadata(); - $calls[$key] = $call; } diff --git a/src/ai-bundle/src/Profiler/TraceableAgent.php b/src/ai-bundle/src/Profiler/TraceableAgent.php new file mode 100644 index 000000000..05720f1ed --- /dev/null +++ b/src/ai-bundle/src/Profiler/TraceableAgent.php @@ -0,0 +1,55 @@ +decorated->call($messages, $options); + } catch (\Throwable $e) { + $error = $e; + throw $e; + } finally { + if ($this->requestStack->getMainRequest() === $this->requestStack->getCurrentRequest()) { + $this->collector->collectChatCall( + 'call', + microtime(true) - $startTime, + $messages, + $response, + $error + ); + } + } + } + + public function getName(): string + { + return $this->decorated->getName(); + } + + public function reset(): void + { + if ($this->decorated instanceof ResetInterface) { + $this->decorated->reset(); + } + } +} diff --git a/src/ai-bundle/templates/data_collector.html.twig b/src/ai-bundle/templates/data_collector.html.twig index b8cbc9345..d650643d0 100644 --- a/src/ai-bundle/templates/data_collector.html.twig +++ b/src/ai-bundle/templates/data_collector.html.twig @@ -1,10 +1,10 @@ {% extends '@WebProfiler/Profiler/layout.html.twig' %} {% block toolbar %} - {% if collector.platformCalls|length > 0 %} + {% if collector.platformCalls|length > 0 or collector.chatCalls|length > 0 %} {% set icon %} {{ include('@Ai/icon.svg', { y: 18 }) }} - {{ collector.platformCalls|length }} + {{ collector.platformCalls|length + collector.chatCalls|length }} calls @@ -12,6 +12,10 @@ {% set text %}
+
+ Chat Calls + {{ collector.chatCalls|length }} +
Configured Platforms 1 @@ -39,7 +43,7 @@ {{ include('@Ai/icon.svg', { y: 16 }) }} Symfony AI - {{ collector.platformCalls|length }} + {{ collector.platformCalls|length + collector.chatCalls|length }} {% endblock %} @@ -57,7 +61,51 @@ {% block panel %}

Symfony AI

+ +

Chat Calls

+ {% if collector.chatCalls|length %} + + + + + + + + + + + + {% for call in collector.chatCalls %} + + + + + + + + {% endfor %} + +
MethodDurationInputResultError
{{ call.method }}{{ (call.duration * 1000)|round(2) }} ms{{ dump(call.input) }}{{ dump(call.result) }} + {% if call.error %} + {{ dump(call.error) }} + {% else %} + None + {% endif %} +
+ {% else %} +
+

No chat calls were made.

+
+ {% endif %} +
+
+
+ {{ collector.chatCalls|length }} + Chat Calls +
+
+
1 @@ -89,92 +137,83 @@ {% for call in collector.platformCalls %} - - - + + + - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - + + {% else %} + {{ call.result }} + {% endif %} + +
Call {{ loop.index }}
Call {{ loop.index }}
Model{{ call.model }}
Input - {% if call.input.messages is defined %}{# expect MessageBag #} -
    - {% for message in call.input.messages %} -
  1. - {{ message.role.value|title }}: - {% if 'assistant' == message.role.value and message.hasToolCalls%} - {{ _self.tool_calls(message.toolCalls) }} - {% elseif 'tool' == message.role.value %} - Result of tool call with ID {{ message.toolCall.id }}
    - {{ message.content|nl2br }} - {% elseif 'user' == message.role.value %} - {% for item in message.content %} - {% if item.text is defined %} - {{ item.text|nl2br }} - {% else %} - - {% endif %} - {% endfor %} - {% else %} - {{ message.content|nl2br }} - {% endif %} -
    2. - {% endfor %} -
- {% else %} - {{ dump(call.input) }} - {% endif %} -
Options -
    - {% for key, value in call.options %} - {% if key == 'tools' %} -
  • {{ key }}: -
      - {% for tool in value %} -
    • {{ tool.name }}
      • - {% endfor %} -
    -
    • - {% else %} -
  • {{ key }}: {{ dump(value) }}
    • - {% endif %} +
Model{{ call.model }}
Input + {% if call.input.messages is defined %}{# expect MessageBag #} +
    + {% for message in call.input.messages %} +
  1. + {{ message.role.value|title }}: + {% if 'assistant' == message.role.value and message.hasToolCalls%} + {{ _self.tool_calls(message.toolCalls) }} + {% elseif 'tool' == message.role.value %} + Result of tool call with ID {{ message.toolCall.id }}
    + {{ message.content|nl2br }} + {% elseif 'user' == message.role.value %} + {% for item in message.content %} + {% if item.text is defined %} + {{ item.text|nl2br }} + {% else %} + + {% endif %} + {% endfor %} + {% else %} + {{ message.content|nl2br }} + {% endif %} +
    2. + {% endfor %} +
+ {% else %} + {{ dump(call.input) }} + {% endif %} +
Options +
    + {% for key, value in call.options %} + {% if key == 'tools' %} +
  • {{ key }}: +
      + {% for tool in value %} +
    • {{ tool.name }}
      • + {% endfor %} +
    +
    • + {% else %} +
  • {{ key }}: {{ dump(value) }}
    • + {% endif %} + {% endfor %} +
+
Result + {% if call.input.messages is defined and call.result is iterable %}{# expect array of ToolCall #} + {{ _self.tool_calls(call.result) }} + {% elseif call.result is iterable %}{# expect array of Vectors #} +
    + {% for vector in call.result %} +
  1. Vector with {{ vector.dimensions }} dimensions
    2. {% endfor %} - -
Result - {% if call.input.messages is defined and call.result is iterable %}{# expect array of ToolCall #} - {{ _self.tool_calls(call.result) }} - {% elseif call.result is iterable %}{# expect array of Vectors #} -
    - {% for vector in call.result %} -
  1. Vector with {{ vector.dimensions }} dimensions
    2. - {% endfor %} -
- {% else %} - {{ call.result }} - {% endif %} - {% if call.metadata|length > 0 %} -
- Metadata -
    - {% for key, value in call.metadata.all %} -
  • {{ key }}:
    {{ dump(value) }}
    • - {% endfor %} -
- {% endif %} -
{% endfor %} @@ -191,35 +230,35 @@ {% if collector.tools|length %} - - - - - - + + + + + + - {% for tool in collector.tools %} - - - - - - - {% endfor %} + {% for tool in collector.tools %} + + + + + + + {% endfor %}
NameDescriptionClass & MethodParameters
NameDescriptionClass & MethodParameters
{{ tool.name }}{{ tool.description }}{{ tool.reference.class }}::{{ tool.reference.method }} - {% if tool.parameters %} -
    - {% for name, parameter in tool.parameters.properties %} -
  • - {{ name }} ({{ parameter.type is iterable ? parameter.type|join(', ') : parameter.type }})
    - {{ parameter.description|default() }} -
    • - {% endfor %} -
- {% else %} - none - {% endif %} -
{{ tool.name }}{{ tool.description }}{{ tool.reference.class }}::{{ tool.reference.method }} + {% if tool.parameters %} +
    + {% for name, parameter in tool.parameters.properties %} +
  • + {{ name }} ({{ parameter.type }})
    + {{ parameter.description|default() }} +
    • + {% endfor %} +
+ {% else %} + none + {% endif %} +
{% else %} @@ -230,26 +269,26 @@

Tool Calls

{% if collector.toolCalls|length %} - {% for toolResult in collector.toolCalls %} + {% for call in collector.toolCalls %} - - - + + + - - - - - - - - - - - - + + + + + + + + + + + +
{{ toolResult.toolCall.name }}
{{ call.call.name }}
ID{{ toolResult.toolCall.id }}
Arguments{{ dump(toolResult.toolCall.arguments) }}
Result{{ dump(toolResult.result) }}
ID{{ call.call.id }}
Arguments{{ dump(call.call.arguments) }}
Result{{ dump(call.result) }}
{% endfor %} diff --git a/src/ai-bundle/tests/Profiler/DataCollectorTest.php b/src/ai-bundle/tests/Profiler/DataCollectorTest.php index e3f886133..7ee2c7abf 100644 --- a/src/ai-bundle/tests/Profiler/DataCollectorTest.php +++ b/src/ai-bundle/tests/Profiler/DataCollectorTest.php @@ -42,7 +42,7 @@ public function testCollectsDataForNonStreamingResponse() $dataCollector->lateCollect(); $this->assertCount(1, $dataCollector->getPlatformCalls()); - $this->assertSame('Assistant response', $dataCollector->getPlatformCalls()[0]['result']); + $this->assertSame('Assistant response', $dataCollector->getPlatformCalls()[0]['result']->getValue(true)); } public function testCollectsDataForStreamingResponse() @@ -66,6 +66,6 @@ public function testCollectsDataForStreamingResponse() $dataCollector->lateCollect(); $this->assertCount(1, $dataCollector->getPlatformCalls()); - $this->assertSame('Assistant response', $dataCollector->getPlatformCalls()[0]['result']); + $this->assertSame('Assistant response', $dataCollector->getPlatformCalls()[0]['result']->getValue(true)); } }