[Store] Manticore support

docs/components/store.rst

@@ -36,6 +36,7 @@ used vector store::
 You can find more advanced usage in combination with an Agent using the store for RAG in the examples folder:
 
 * `Similarity Search with Cloudflare (RAG)`_
+* `Similarity Search with Manticore (RAG)`_
 * `Similarity Search with MariaDB (RAG)`_
 * `Similarity Search with Meilisearch (RAG)`_
 * `Similarity Search with memory storage (RAG)`_
@@ -63,6 +64,7 @@ Supported Stores
 * `Chroma`_ (requires `codewithkyrian/chromadb-php` as additional dependency)
 * `Cloudflare`_
 * `InMemory`_
+* `Manticore`_
 * `MariaDB`_ (requires `ext-pdo`)
 * `Meilisearch`_
 * `Milvus`_
@@ -128,6 +130,7 @@ This leads to a store implementing two methods::
 
 .. _`Retrieval Augmented Generation`: https://en.wikipedia.org/wiki/Retrieval-augmented_generation
 .. _`Similarity Search with Cloudflare (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/cloudflare.php
+.. _`Similarity Search with Manticore (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/manticore.php
 .. _`Similarity Search with MariaDB (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/mariadb-gemini.php
 .. _`Similarity Search with Meilisearch (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/meilisearch.php
 .. _`Similarity Search with memory storage (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/in-memory.php
@@ -144,6 +147,7 @@ This leads to a store implementing two methods::
 .. _`Azure AI Search`: https://azure.microsoft.com/products/ai-services/ai-search
 .. _`Chroma`: https://www.trychroma.com/
 .. _`Cloudflare`: https://developers.cloudflare.com/vectorize/
+.. _`Manticore`: https://manticoresearch.com/
 .. _`MariaDB`: https://mariadb.org/projects/mariadb-vector/
 .. _`Pinecone`: https://www.pinecone.io/
 .. _`Postgres`: https://www.postgresql.org/about/news/pgvector-070-released-2852/

examples/.env

@@ -171,3 +171,6 @@ POGOCACHE_PASSWORD=symfony
 
 # Redis (both store and message store)
 REDIS_HOST=localhost
+
+# Manticore (store)
+MANTICORE_HOST=http://127.0.0.1:9308

examples/commands/stores.php

@@ -17,6 +17,7 @@
 use Symfony\AI\Store\Bridge\ClickHouse\Store as ClickHouseStore;
 use Symfony\AI\Store\Bridge\Local\CacheStore;
 use Symfony\AI\Store\Bridge\Local\InMemoryStore;
+use Symfony\AI\Store\Bridge\Manticore\Store as ManticoreStore;
 use Symfony\AI\Store\Bridge\MariaDb\Store as MariaDbStore;
 use Symfony\AI\Store\Bridge\Meilisearch\Store as MeilisearchStore;
 use Symfony\AI\Store\Bridge\Milvus\Store as MilvusStore;
@@ -44,6 +45,12 @@
         env('CLICKHOUSE_DATABASE'),
         env('CLICKHOUSE_TABLE'),
     ),
+    'manticore' => static fn (): ManticoreStore => new ManticoreStore(
+        http_client(),
+        env('MANTICORE_HOST'),
+        'symfony',
+        '_vectors',
+    ),
     'mariadb' => static fn (): MariaDbStore => MariaDbStore::fromDbal(
         DriverManager::getConnection((new DsnParser())->parse(env('MARIADB_URI'))),
         'my_table_for_commands',

examples/compose.yaml

@@ -19,6 +19,20 @@ services:
         ports:
             - '8123:8123'
 
+    manticore:
+        image: manticoresearch/manticore
+        ulimits:
+            nproc: 65535
+            nofile:
+                soft: 65535
+                hard: 65535
+            memlock:
+                soft: -1
+                hard: -1
+        ports:
+            - '9306:9306'
+            - '9308:9308'
+
     mariadb:
         image: mariadb:11.7
         environment:

examples/rag/manticore.php

@@ -0,0 +1,68 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Agent\Agent;
+use Symfony\AI\Agent\Toolbox\AgentProcessor;
+use Symfony\AI\Agent\Toolbox\Tool\SimilaritySearch;
+use Symfony\AI\Agent\Toolbox\Toolbox;
+use Symfony\AI\Fixtures\Movies;
+use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+use Symfony\AI\Store\Bridge\Manticore\Store;
+use Symfony\AI\Store\Document\Loader\InMemoryLoader;
+use Symfony\AI\Store\Document\Metadata;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\Vectorizer;
+use Symfony\AI\Store\Indexer;
+use Symfony\Component\Uid\Uuid;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+// initialize the store
+$store = new Store(
+    httpClient: http_client(),
+    host: 'http://127.0.0.1:9308',
+    table: 'movies',
+    field: '_movie_vectors',
+);
+
+// Create the table
+$store->setup();
+
+// create embeddings and documents
+$documents = [];
+foreach (Movies::all() as $i => $movie) {
+    $documents[] = new TextDocument(
+        id: Uuid::v4(),
+        content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'],
+        metadata: new Metadata($movie),
+    );
+}
+
+// create embeddings for documents
+$platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
+$vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
+$indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
+$indexer->index($documents);
+
+$similaritySearch = new SimilaritySearch($vectorizer, $store);
+$toolbox = new Toolbox([$similaritySearch], logger: logger());
+$processor = new AgentProcessor($toolbox);
+$agent = new Agent($platform, 'gpt-4o-mini', [$processor], [$processor]);
+
+$messages = new MessageBag(
+    Message::forSystem('Please answer all user questions only using SimilaritySearch function.'),
+    Message::ofUser('Which movie fits the theme of technology?')
+);
+$result = $agent->call($messages);
+
+echo $result->getContent().\PHP_EOL;

src/ai-bundle/config/options.php

@@ -514,6 +514,20 @@
                             ->end()
                         ->end()
                     ->end()
+                    ->arrayNode('manticore')
+                        ->useAttributeAsKey('name')
+                        ->arrayPrototype()
+                            ->children()
+                                ->stringNode('endpoint')->cannotBeEmpty()->end()
+                                ->stringNode('table')->cannotBeEmpty()->end()
+                                ->stringNode('field')->end()
+                                ->stringNode('type')->end()
+                                ->stringNode('similarity')->end()
+                                ->integerNode('dimensions')->end()
+                                ->stringNode('quantization')->end()
+                            ->end()
+                        ->end()
+                    ->end()
                     ->arrayNode('meilisearch')
                         ->useAttributeAsKey('name')
                         ->arrayPrototype()

src/ai-bundle/src/AiBundle.php

@@ -69,6 +69,7 @@
 use Symfony\AI\Store\Bridge\Local\DistanceCalculator;
 use Symfony\AI\Store\Bridge\Local\DistanceStrategy;
 use Symfony\AI\Store\Bridge\Local\InMemoryStore;
+use Symfony\AI\Store\Bridge\Manticore\Store as ManticoreStore;
 use Symfony\AI\Store\Bridge\Meilisearch\Store as MeilisearchStore;
 use Symfony\AI\Store\Bridge\Milvus\Store as MilvusStore;
 use Symfony\AI\Store\Bridge\MongoDb\Store as MongoDbStore;
@@ -912,6 +913,45 @@ private function processStoreConfig(string $type, array $stores, ContainerBuilde
             }
         }
 
+        if ('manticore' === $type) {
+            foreach ($stores as $name => $store) {
+                $arguments = [
+                    new Reference('http_client'),
+                    $store['endpoint'],
+                    $store['table'],
+                ];
+
+                if (\array_key_exists('field', $store)) {
+                    $arguments[3] = $store['field'];
+                }
+
+                if (\array_key_exists('type', $store)) {
+                    $arguments[4] = $store['type'];
+                }
+
+                if (\array_key_exists('similarity', $store)) {
+                    $arguments[5] = $store['similarity'];
+                }
+
+                if (\array_key_exists('dimensions', $store)) {
+                    $arguments[6] = $store['dimensions'];
+                }
+
+                if (\array_key_exists('quantization', $store)) {
+                    $arguments[7] = $store['quantization'];
+                }
+
+                $definition = new Definition(ManticoreStore::class);
+                $definition
+                    ->addTag('ai.store')
+                    ->setArguments($arguments);
+
+                $container->setDefinition('ai.store.'.$type.'.'.$name, $definition);
+                $container->registerAliasForArgument('ai.store.'.$type.'.'.$name, StoreInterface::class, $name);
+                $container->registerAliasForArgument('ai.store.'.$type.'.'.$name, StoreInterface::class, $type.'_'.$name);
+            }
+        }
+
         if ('meilisearch' === $type) {
             foreach ($stores as $name => $store) {
                 $arguments = [

src/ai-bundle/tests/DependencyInjection/AiBundleTest.php

@@ -2911,6 +2911,17 @@ private function getFullConfig(): array
                             'endpoint_url' => 'https://api.cloudflare.com/client/v5/accounts',
                         ],
                     ],
+                    'manticore' => [
+                        'my_manticore_store' => [
+                            'endpoint' => 'http://127.0.0.1:9306',
+                            'table' => 'test',
+                            'field' => 'foo_vector',
+                            'type' => 'hnsw',
+                            'similarity' => 'cosine',
+                            'dimensions' => 768,
+                            'quantization' => '1bit',
+                        ],
+                    ],
                     'meilisearch' => [
                         'my_meilisearch_store' => [
                             'endpoint' => 'http://127.0.0.1:7700',

src/store/CHANGELOG.md

@@ -38,6 +38,7 @@ CHANGELOG
    - ChromaDB
    - ClickHouse
    - Cloudflare
+   - Manticore
    - MariaDB
    - Meilisearch
    - MongoDB