diff --git a/examples/.env b/examples/.env index 3f3652496..4c5dcde42 100644 --- a/examples/.env +++ b/examples/.env @@ -112,6 +112,9 @@ MEILISEARCH_API_KEY=changeMe # For using LMStudio LMSTUDIO_HOST_URL=http://127.0.0.1:1234 +# For using LiteLLM +LITELLM_HOST_URL=http://127.0.0.1:4000 + # Qdrant (store) QDRANT_HOST=http://127.0.0.1:6333 QDRANT_SERVICE_API_KEY=changeMe diff --git a/examples/compose.yaml b/examples/compose.yaml index 3dccd929f..91cd8a012 100644 --- a/examples/compose.yaml +++ b/examples/compose.yaml @@ -188,6 +188,16 @@ services: - '8080:8080' - '50051:50051' + litellm: + image: ghcr.io/berriai/litellm:v1.79.1-stable + ports: + - "4000:4000" + volumes: + - ./litellm/config.yaml:/app/config.yaml + env_file: + - .env + command: [ "--config", "/app/config.yaml", "--port", "4000", "--num_workers", "8" ] + volumes: typesense_data: etcd_vlm: diff --git a/examples/litellm/chat.php b/examples/litellm/chat.php new file mode 100644 index 000000000..743db4d7b --- /dev/null +++ b/examples/litellm/chat.php @@ -0,0 +1,28 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +use Symfony\AI\Platform\Bridge\LiteLlm\PlatformFactory; +use Symfony\AI\Platform\Message\Message; +use Symfony\AI\Platform\Message\MessageBag; + +require_once dirname(__DIR__).'/bootstrap.php'; + +$platform = PlatformFactory::create(env('LITELLM_HOST_URL'), http_client()); + +$messages = new MessageBag( + Message::forSystem('You are a pirate and you write funny.'), + Message::ofUser('What is the Symfony framework?'), +); +$result = $platform->invoke('mistral-small-latest', $messages, [ + 'max_tokens' => 500, // specific options just for this call +]); + +echo $result->asText().\PHP_EOL; diff --git a/examples/litellm/config.yaml b/examples/litellm/config.yaml new file mode 100644 index 000000000..b116f87b4 --- /dev/null +++ b/examples/litellm/config.yaml @@ -0,0 +1,5 @@ +model_list: + - model_name: mistral-small-latest + litellm_params: + model: mistral/mistral-small-latest + api_key: "os.environ/MISTRAL_API_KEY" diff --git a/src/platform/composer.json b/src/platform/composer.json index d4f13b86e..f9dbf124e 100644 --- a/src/platform/composer.json +++ b/src/platform/composer.json @@ -16,6 +16,7 @@ "gemini", "huggingface", "inference", + "litellm", "llama", "lmstudio", "meta", diff --git a/src/platform/src/Bridge/LiteLlm/ModelCatalog.php b/src/platform/src/Bridge/LiteLlm/ModelCatalog.php new file mode 100644 index 000000000..8c839d528 --- /dev/null +++ b/src/platform/src/Bridge/LiteLlm/ModelCatalog.php @@ -0,0 +1,23 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\AI\Platform\Bridge\LiteLlm; + +use Symfony\AI\Platform\ModelCatalog\FallbackModelCatalog; + +/** + * @author Mathieu Santostefano + */ +final class ModelCatalog extends FallbackModelCatalog +{ + // LiteLLM can use any model that is loaded locally + // Models are dynamically available based on what's configured in LiteLLM +} diff --git a/src/platform/src/Bridge/LiteLlm/ModelClient.php b/src/platform/src/Bridge/LiteLlm/ModelClient.php new file mode 100644 index 000000000..635c7abe3 --- /dev/null +++ b/src/platform/src/Bridge/LiteLlm/ModelClient.php @@ -0,0 +1,45 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\AI\Platform\Bridge\LiteLlm; + +use Symfony\AI\Platform\Model; +use Symfony\AI\Platform\ModelClientInterface; +use Symfony\AI\Platform\Result\RawHttpResult; +use Symfony\Component\HttpClient\EventSourceHttpClient; +use Symfony\Contracts\HttpClient\HttpClientInterface; + +/** + * @author Mathieu Santostefano + */ +final class ModelClient implements ModelClientInterface +{ + private readonly EventSourceHttpClient $httpClient; + + public function __construct( + HttpClientInterface $httpClient, + private readonly string $hostUrl, + ) { + $this->httpClient = $httpClient instanceof EventSourceHttpClient ? $httpClient : new EventSourceHttpClient($httpClient); + } + + public function supports(Model $model): bool + { + return true; + } + + public function request(Model $model, array|string $payload, array $options = []): RawHttpResult + { + return new RawHttpResult($this->httpClient->request('POST', \sprintf('%s/v1/chat/completions', $this->hostUrl), [ + 'json' => array_merge($options, $payload), + ])); + } +} diff --git a/src/platform/src/Bridge/LiteLlm/PlatformFactory.php b/src/platform/src/Bridge/LiteLlm/PlatformFactory.php new file mode 100644 index 000000000..7969eb1a9 --- /dev/null +++ b/src/platform/src/Bridge/LiteLlm/PlatformFactory.php @@ -0,0 +1,47 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\AI\Platform\Bridge\LiteLlm; + +use Psr\EventDispatcher\EventDispatcherInterface; +use Symfony\AI\Platform\Contract; +use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface; +use Symfony\AI\Platform\Platform; +use Symfony\Component\HttpClient\EventSourceHttpClient; +use Symfony\Contracts\HttpClient\HttpClientInterface; + +/** + * @author Mathieu Santostefano + */ +class PlatformFactory +{ + public static function create( + string $hostUrl = 'http://localhost:4000', + ?HttpClientInterface $httpClient = null, + ModelCatalogInterface $modelCatalog = new ModelCatalog(), + ?Contract $contract = null, + ?EventDispatcherInterface $eventDispatcher = null, + ): Platform { + $httpClient = $httpClient instanceof EventSourceHttpClient ? $httpClient : new EventSourceHttpClient($httpClient); + + return new Platform( + [ + new ModelClient($httpClient, $hostUrl), + ], + [ + new ResultConverter(), + ], + $modelCatalog, + $contract, + $eventDispatcher, + ); + } +} diff --git a/src/platform/src/Bridge/LiteLlm/ResultConverter.php b/src/platform/src/Bridge/LiteLlm/ResultConverter.php new file mode 100644 index 000000000..452af93cc --- /dev/null +++ b/src/platform/src/Bridge/LiteLlm/ResultConverter.php @@ -0,0 +1,39 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\AI\Platform\Bridge\LiteLlm; + +use Symfony\AI\Platform\Bridge\OpenAi\Gpt\ResultConverter as OpenAiResponseConverter; +use Symfony\AI\Platform\Model; +use Symfony\AI\Platform\Result\RawResultInterface; +use Symfony\AI\Platform\Result\ResultInterface; +use Symfony\AI\Platform\ResultConverterInterface; + +/** + * @author Mathieu Santostefano + */ +final class ResultConverter implements ResultConverterInterface +{ + public function __construct( + private readonly OpenAiResponseConverter $gptResponseConverter = new OpenAiResponseConverter(), + ) { + } + + public function supports(Model $model): bool + { + return true; + } + + public function convert(RawResultInterface $result, array $options = []): ResultInterface + { + return $this->gptResponseConverter->convert($result, $options); + } +} diff --git a/src/platform/tests/Bridge/LiteLlm/ModelClientTest.php b/src/platform/tests/Bridge/LiteLlm/ModelClientTest.php new file mode 100644 index 000000000..f45430dea --- /dev/null +++ b/src/platform/tests/Bridge/LiteLlm/ModelClientTest.php @@ -0,0 +1,101 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\AI\Platform\Tests\Bridge\LiteLlm; + +use PHPUnit\Framework\TestCase; +use Symfony\AI\Platform\Bridge\LiteLlm\ModelClient; +use Symfony\AI\Platform\Model; +use Symfony\Component\HttpClient\EventSourceHttpClient; +use Symfony\Component\HttpClient\MockHttpClient; +use Symfony\Component\HttpClient\Response\MockResponse; + +class ModelClientTest extends TestCase +{ + public function testItIsSupportingTheCorrectModel() + { + $client = new ModelClient(new MockHttpClient(), 'http://localhost:4000'); + + $this->assertTrue($client->supports(new Model('test-model'))); + } + + public function testItIsExecutingTheCorrectRequest() + { + $resultCallback = static function (string $method, string $url, array $options): MockResponse { + self::assertSame('POST', $method); + self::assertSame('http://localhost:4000/v1/chat/completions', $url); + self::assertSame( + '{"model":"test-model","messages":[{"role":"user","content":"Hello, world!"}]}', + $options['body'] + ); + + return new MockResponse(); + }; + + $httpClient = new MockHttpClient([$resultCallback]); + $client = new ModelClient($httpClient, 'http://localhost:4000'); + + $payload = [ + 'model' => 'test-model', + 'messages' => [ + ['role' => 'user', 'content' => 'Hello, world!'], + ], + ]; + + $client->request(new Model('test-model'), $payload); + } + + public function testItMergesOptionsWithPayload() + { + $resultCallback = static function (string $method, string $url, array $options): MockResponse { + self::assertSame('POST', $method); + self::assertSame('http://localhost:4000/v1/chat/completions', $url); + self::assertSame( + '{"temperature":0.7,"model":"test-model","messages":[{"role":"user","content":"Hello, world!"}]}', + $options['body'] + ); + + return new MockResponse(); + }; + + $httpClient = new MockHttpClient([$resultCallback]); + $client = new ModelClient($httpClient, 'http://localhost:4000'); + + $payload = [ + 'model' => 'test-model', + 'messages' => [ + ['role' => 'user', 'content' => 'Hello, world!'], + ], + ]; + + $client->request(new Model('test-model'), $payload, ['temperature' => 0.7]); + } + + public function testItUsesEventSourceHttpClient() + { + $httpClient = new MockHttpClient(); + $client = new ModelClient($httpClient, 'http://localhost:4000'); + + $reflection = new \ReflectionProperty($client, 'httpClient'); + + $this->assertInstanceOf(EventSourceHttpClient::class, $reflection->getValue($client)); + } + + public function testItKeepsExistingEventSourceHttpClient() + { + $eventSourceHttpClient = new EventSourceHttpClient(new MockHttpClient()); + $client = new ModelClient($eventSourceHttpClient, 'http://localhost:4000'); + + $reflection = new \ReflectionProperty($client, 'httpClient'); + + $this->assertSame($eventSourceHttpClient, $reflection->getValue($client)); + } +}