[Platform][LiteLLM] Add Bridge

examples/.env

@@ -112,6 +112,9 @@ MEILISEARCH_API_KEY=changeMe
 # For using LMStudio
 LMSTUDIO_HOST_URL=http://127.0.0.1:1234
 
+# For using LiteLLM
+LITELLM_HOST_URL=http://127.0.0.1:4000
+
 # Qdrant (store)
 QDRANT_HOST=http://127.0.0.1:6333
 QDRANT_SERVICE_API_KEY=changeMe

examples/compose.yaml

@@ -188,6 +188,16 @@ services:
             - '8080:8080'
             - '50051:50051'
 
+    litellm:
+        image: ghcr.io/berriai/litellm:v1.79.1-stable
+        ports:
+            - "4000:4000"
+        volumes:
+            - ./litellm/config.yaml:/app/config.yaml
+        env_file:
+            - .env
+        command: [ "--config", "/app/config.yaml", "--port", "4000", "--num_workers", "8" ]
+
 volumes:
     typesense_data:
     etcd_vlm:

examples/litellm/chat.php

@@ -0,0 +1,28 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Platform\Bridge\LiteLlm\PlatformFactory;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+$platform = PlatformFactory::create(env('LITELLM_HOST_URL'), http_client());
+
+$messages = new MessageBag(
+    Message::forSystem('You are a pirate and you write funny.'),
+    Message::ofUser('What is the Symfony framework?'),
+);
+$result = $platform->invoke('mistral-small-latest', $messages, [
+    'max_tokens' => 500, // specific options just for this call
+]);
+
+echo $result->asText().\PHP_EOL;

examples/litellm/config.yaml

@@ -0,0 +1,5 @@
+model_list:
+    - model_name: mistral-small-latest
+      litellm_params:
+          model: mistral/mistral-small-latest
+          api_key: "os.environ/MISTRAL_API_KEY"

src/platform/composer.json

@@ -16,6 +16,7 @@
         "gemini",
         "huggingface",
         "inference",
+        "litellm",
         "llama",
         "lmstudio",
         "meta",

src/platform/src/Bridge/LiteLlm/ModelCatalog.php

@@ -0,0 +1,23 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Platform\Bridge\LiteLlm;
+
+use Symfony\AI\Platform\ModelCatalog\FallbackModelCatalog;
+
+/**
+ * @author Mathieu Santostefano <msantostefano@proton.me>
+ */
+final class ModelCatalog extends FallbackModelCatalog
+{
+    // LiteLLM can use any model that is loaded locally
+    // Models are dynamically available based on what's configured in LiteLLM
+}

src/platform/src/Bridge/LiteLlm/ModelClient.php

@@ -0,0 +1,45 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Platform\Bridge\LiteLlm;
+
+use Symfony\AI\Platform\Model;
+use Symfony\AI\Platform\ModelClientInterface;
+use Symfony\AI\Platform\Result\RawHttpResult;
+use Symfony\Component\HttpClient\EventSourceHttpClient;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+
+/**
+ * @author Mathieu Santostefano <msantostefano@proton.me>
+ */
+final class ModelClient implements ModelClientInterface
+{
+    private readonly EventSourceHttpClient $httpClient;
+
+    public function __construct(
+        HttpClientInterface $httpClient,
+        private readonly string $hostUrl,
+    ) {
+        $this->httpClient = $httpClient instanceof EventSourceHttpClient ? $httpClient : new EventSourceHttpClient($httpClient);
+    }
+
+    public function supports(Model $model): bool
+    {
+        return true;
+    }
+
+    public function request(Model $model, array|string $payload, array $options = []): RawHttpResult
+    {
+        return new RawHttpResult($this->httpClient->request('POST', \sprintf('%s/v1/chat/completions', $this->hostUrl), [
+            'json' => array_merge($options, $payload),
+        ]));
+    }
+}

src/platform/src/Bridge/LiteLlm/PlatformFactory.php

@@ -0,0 +1,47 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Platform\Bridge\LiteLlm;
+
+use Psr\EventDispatcher\EventDispatcherInterface;
+use Symfony\AI\Platform\Contract;
+use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface;
+use Symfony\AI\Platform\Platform;
+use Symfony\Component\HttpClient\EventSourceHttpClient;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+
+/**
+ * @author Mathieu Santostefano <msantostefano@proton.me>
+ */
+class PlatformFactory
+{
+    public static function create(
+        string $hostUrl = 'http://localhost:4000',
+        ?HttpClientInterface $httpClient = null,
+        ModelCatalogInterface $modelCatalog = new ModelCatalog(),
+        ?Contract $contract = null,
+        ?EventDispatcherInterface $eventDispatcher = null,
+    ): Platform {
+        $httpClient = $httpClient instanceof EventSourceHttpClient ? $httpClient : new EventSourceHttpClient($httpClient);
+
+        return new Platform(
+            [
+                new ModelClient($httpClient, $hostUrl),
+            ],
+            [
+                new ResultConverter(),
+            ],
+            $modelCatalog,
+            $contract,
+            $eventDispatcher,
+        );
+    }
+}

src/platform/src/Bridge/LiteLlm/ResultConverter.php

@@ -0,0 +1,39 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Platform\Bridge\LiteLlm;
+
+use Symfony\AI\Platform\Bridge\OpenAi\Gpt\ResultConverter as OpenAiResponseConverter;
+use Symfony\AI\Platform\Model;
+use Symfony\AI\Platform\Result\RawResultInterface;
+use Symfony\AI\Platform\Result\ResultInterface;
+use Symfony\AI\Platform\ResultConverterInterface;
+
+/**
+ * @author Mathieu Santostefano <msantostefano@proton.me>
+ */
+final class ResultConverter implements ResultConverterInterface
+{
+    public function __construct(
+        private readonly OpenAiResponseConverter $gptResponseConverter = new OpenAiResponseConverter(),
+    ) {
+    }
+
+    public function supports(Model $model): bool
+    {
+        return true;
+    }
+
+    public function convert(RawResultInterface $result, array $options = []): ResultInterface
+    {
+        return $this->gptResponseConverter->convert($result, $options);
+    }
+}

src/platform/tests/Bridge/LiteLlm/ModelClientTest.php

@@ -0,0 +1,101 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Platform\Tests\Bridge\LiteLlm;
+
+use PHPUnit\Framework\TestCase;
+use Symfony\AI\Platform\Bridge\LiteLlm\ModelClient;
+use Symfony\AI\Platform\Model;
+use Symfony\Component\HttpClient\EventSourceHttpClient;
+use Symfony\Component\HttpClient\MockHttpClient;
+use Symfony\Component\HttpClient\Response\MockResponse;
+
+class ModelClientTest extends TestCase
+{
+    public function testItIsSupportingTheCorrectModel()
+    {
+        $client = new ModelClient(new MockHttpClient(), 'http://localhost:4000');
+
+        $this->assertTrue($client->supports(new Model('test-model')));
+    }
+
+    public function testItIsExecutingTheCorrectRequest()
+    {
+        $resultCallback = static function (string $method, string $url, array $options): MockResponse {
+            self::assertSame('POST', $method);
+            self::assertSame('http://localhost:4000/v1/chat/completions', $url);
+            self::assertSame(
+                '{"model":"test-model","messages":[{"role":"user","content":"Hello, world!"}]}',
+                $options['body']
+            );
+
+            return new MockResponse();
+        };
+
+        $httpClient = new MockHttpClient([$resultCallback]);
+        $client = new ModelClient($httpClient, 'http://localhost:4000');
+
+        $payload = [
+            'model' => 'test-model',
+            'messages' => [
+                ['role' => 'user', 'content' => 'Hello, world!'],
+            ],
+        ];
+
+        $client->request(new Model('test-model'), $payload);
+    }
+
+    public function testItMergesOptionsWithPayload()
+    {
+        $resultCallback = static function (string $method, string $url, array $options): MockResponse {
+            self::assertSame('POST', $method);
+            self::assertSame('http://localhost:4000/v1/chat/completions', $url);
+            self::assertSame(
+                '{"temperature":0.7,"model":"test-model","messages":[{"role":"user","content":"Hello, world!"}]}',
+                $options['body']
+            );
+
+            return new MockResponse();
+        };
+
+        $httpClient = new MockHttpClient([$resultCallback]);
+        $client = new ModelClient($httpClient, 'http://localhost:4000');
+
+        $payload = [
+            'model' => 'test-model',
+            'messages' => [
+                ['role' => 'user', 'content' => 'Hello, world!'],
+            ],
+        ];
+
+        $client->request(new Model('test-model'), $payload, ['temperature' => 0.7]);
+    }
+
+    public function testItUsesEventSourceHttpClient()
+    {
+        $httpClient = new MockHttpClient();
+        $client = new ModelClient($httpClient, 'http://localhost:4000');
+
+        $reflection = new \ReflectionProperty($client, 'httpClient');
+
+        $this->assertInstanceOf(EventSourceHttpClient::class, $reflection->getValue($client));
+    }
+
+    public function testItKeepsExistingEventSourceHttpClient()
+    {
+        $eventSourceHttpClient = new EventSourceHttpClient(new MockHttpClient());
+        $client = new ModelClient($eventSourceHttpClient, 'http://localhost:4000');
+
+        $reflection = new \ReflectionProperty($client, 'httpClient');
+
+        $this->assertSame($eventSourceHttpClient, $reflection->getValue($client));
+    }
+}