[AI Bundle][Store] Add Retriever

.github/workflows/integration-tests.yaml

@@ -104,6 +104,10 @@ jobs:
                   composer-options: "--no-scripts"
                   working-directory: demo
 
+            - name: Link local packages
+              working-directory: demo
+              run: ../link
+
             - run: composer run-script auto-scripts --no-interaction
               working-directory: demo
 

demo/AGENTS.md

@@ -31,8 +31,8 @@ composer install
 echo "OPENAI_API_KEY='sk-...'" > .env.local
 
 # Initialize vector store
-symfony console app:blog:embed -vv
-symfony console app:blog:query
+symfony console ai:store:index blog -vv
+symfony console ai:store:retrieve blog "Week of Symfony"
 
 # Start server
 symfony serve -d

demo/CLAUDE.md

@@ -39,7 +39,7 @@ echo "OPENAI_API_KEY='sk-...'" > .env.local
 symfony console ai:store:index blog -vv
 
 # Test vector store
-symfony console app:blog:query
+symfony console ai:store:retrieve blog "Week of Symfony"
 
 # Start development server
 symfony serve -d

demo/README.md

@@ -77,10 +77,10 @@ To initialize the Chroma DB, you need to run the following command:
 symfony console ai:store:index blog -vv
 ```
 
-Now you should be able to run the test command and get some results:
+Now you should be able to retrieve documents from the store:
 
 ```shell
-symfony console app:blog:query
+symfony console ai:store:retrieve blog "Week of Symfony"
 ```
 
 **Don't forget to set up the project in your favorite IDE or editor.**

demo/config/packages/ai.yaml

@@ -100,6 +100,10 @@ ai:
                 - 'Symfony\AI\Store\Document\Transformer\TextTrimTransformer'
             vectorizer: 'ai.vectorizer.openai'
             store: 'ai.store.chroma_db.symfonycon'
+    retriever:
+        blog:
+            vectorizer: 'ai.vectorizer.openai'
+            store: 'ai.store.chroma_db.symfonycon'
 
 services:
     _defaults:

docs/bundles/ai-bundle.rst

@@ -972,6 +972,69 @@ Benefits of Configured Vectorizers
 * **Consistency**: Ensure all indexers using the same vectorizer have identical embedding configuration
 * **Maintainability**: Change vectorizer settings in one place
 
+Retrievers
+----------
+
+Retrievers are the opposite of indexers. While indexers populate a vector store with documents,
+retrievers allow you to search for documents in a store based on a query string.
+They vectorize the query and retrieve similar documents from the store.
+
+Configuring Retrievers
+~~~~~~~~~~~~~~~~~~~~~~
+
+Retrievers are defined in the ``retriever`` section of your configuration:
+
+.. code-block:: yaml
+
+    ai:
+        retriever:
+            default:
+                vectorizer: 'ai.vectorizer.openai_small'
+                store: 'ai.store.chroma_db.default'
+
+            research:
+                vectorizer: 'ai.vectorizer.mistral_embed'
+                store: 'ai.store.memory.research'
+
+Using Retrievers
+~~~~~~~~~~~~~~~~
+
+The retriever can be injected into your services using the ``RetrieverInterface``::
+
+    use Symfony\AI\Store\RetrieverInterface;
+
+    final readonly class MyService
+    {
+        public function __construct(
+            private RetrieverInterface $retriever,
+        ) {
+        }
+
+        public function search(string $query): array
+        {
+            $documents = [];
+            foreach ($this->retriever->retrieve($query) as $document) {
+                $documents[] = $document;
+            }
+
+            return $documents;
+        }
+    }
+
+When you have multiple retrievers configured, you can use the ``#[Autowire]`` attribute to inject a specific one::
+
+    use Symfony\AI\Store\RetrieverInterface;
+    use Symfony\Component\DependencyInjection\Attribute\Autowire;
+
+    final readonly class ResearchService
+    {
+        public function __construct(
+            #[Autowire(service: 'ai.retriever.research')]
+            private RetrieverInterface $retriever,
+        ) {
+        }
+    }
+
 Profiler
 --------
 

docs/components/store.rst

@@ -33,7 +33,34 @@ used vector store::
     $document = new TextDocument('This is a sample document.');
     $indexer->index($document);
 
-You can find more advanced usage in combination with an Agent using the store for RAG in the examples folder:
+You can find more advanced usage in combination with an Agent using the store for RAG in the examples folder.
+
+Retrieving
+----------
+
+The opposite of indexing is retrieving. The :class:`Symfony\\AI\\Store\\Retriever` is a higher level feature that allows you to
+search for documents in a store based on a query string. It vectorizes the query and retrieves similar documents from the store::
+
+    use Symfony\AI\Store\Retriever;
+
+    $retriever = new Retriever($vectorizer, $store);
+    $documents = $retriever->retrieve('What is the capital of France?');
+
+    foreach ($documents as $document) {
+        echo $document->metadata->get('source');
+    }
+
+The retriever accepts optional parameters to customize the retrieval:
+
+* ``$options``: An array of options to pass to the underlying store query (e.g., limit, filters)
+
+Example Usage
+~~~~~~~~~~~~~
+
+* `Basic Retriever Example`_
+
+Similarity Search Examples
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 * `Similarity Search with Cloudflare (RAG)`_
 * `Similarity Search with Manticore (RAG)`_
@@ -129,6 +156,7 @@ This leads to a store implementing two methods::
     }
 
 .. _`Retrieval Augmented Generation`: https://en.wikipedia.org/wiki/Retrieval-augmented_generation
+.. _`Basic Retriever Example`: https://github.com/symfony/ai/blob/main/examples/retriever/basic.php
 .. _`Similarity Search with Cloudflare (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/cloudflare.php
 .. _`Similarity Search with Manticore (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/manticore.php
 .. _`Similarity Search with MariaDB (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/mariadb-gemini.php

examples/retriever/basic.php

@@ -0,0 +1,54 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
+use Symfony\AI\Store\Bridge\Local\InMemoryStore;
+use Symfony\AI\Store\Document\Loader\TextFileLoader;
+use Symfony\AI\Store\Document\Transformer\TextSplitTransformer;
+use Symfony\AI\Store\Document\Vectorizer;
+use Symfony\AI\Store\Indexer;
+use Symfony\AI\Store\Retriever;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+$store = new InMemoryStore();
+
+$platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
+$vectorizer = new Vectorizer($platform, 'text-embedding-3-small');
+
+$indexer = new Indexer(
+    loader: new TextFileLoader(),
+    vectorizer: $vectorizer,
+    store: $store,
+    source: [
+        dirname(__DIR__, 2).'/fixtures/movies/gladiator.md',
+        dirname(__DIR__, 2).'/fixtures/movies/inception.md',
+        dirname(__DIR__, 2).'/fixtures/movies/jurassic-park.md',
+    ],
+    transformers: [
+        new TextSplitTransformer(chunkSize: 500, overlap: 100),
+    ],
+);
+$indexer->index();
+
+$retriever = new Retriever(
+    vectorizer: $vectorizer,
+    store: $store,
+);
+
+echo "Searching for: 'Roman gladiator revenge'\n\n";
+$results = $retriever->retrieve('Roman gladiator revenge', ['maxItems' => 1]);
+
+foreach ($results as $i => $document) {
+    echo sprintf("%d. Document ID: %s\n", $i + 1, $document->id);
+    echo sprintf("   Score: %s\n", $document->score ?? 'n/a');
+    echo sprintf("   Source: %s\n\n", $document->metadata->getSource() ?? 'unknown');
+}

examples/retriever/movies.php

@@ -0,0 +1,56 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Fixtures\Movies;
+use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
+use Symfony\AI\Store\Bridge\Local\InMemoryStore;
+use Symfony\AI\Store\Document\Loader\InMemoryLoader;
+use Symfony\AI\Store\Document\Metadata;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\Vectorizer;
+use Symfony\AI\Store\Indexer;
+use Symfony\AI\Store\Retriever;
+use Symfony\Component\Uid\Uuid;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+$store = new InMemoryStore();
+
+$documents = [];
+foreach (Movies::all() as $movie) {
+    $documents[] = new TextDocument(
+        id: Uuid::v4(),
+        content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'],
+        metadata: new Metadata($movie),
+    );
+}
+
+$platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
+$vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
+
+$indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
+$indexer->index();
+
+$retriever = new Retriever($vectorizer, $store, logger());
+
+echo "Searching for movies about 'crime family mafia'\n";
+echo "================================================\n\n";
+
+$results = $retriever->retrieve('crime family mafia');
+
+foreach ($results as $i => $document) {
+    $title = $document->metadata['title'];
+    $director = $document->metadata['director'];
+    $score = $document->score;
+
+    echo sprintf("%d. %s (Director: %s)\n", $i + 1, $title, $director);
+    echo sprintf("   Score: %.4f\n\n", $score ?? 0);
+}

src/ai-bundle/config/options.php

@@ -1089,6 +1089,22 @@
                     ->end()
                 ->end()
             ->end()
+            ->arrayNode('retriever')
+                ->info('Retrievers for fetching documents from a vector store based on a query')
+                ->useAttributeAsKey('name')
+                ->arrayPrototype()
+                    ->children()
+                        ->scalarNode('vectorizer')
+                            ->info('Service name of vectorizer')
+                            ->defaultValue(VectorizerInterface::class)
+                        ->end()
+                        ->stringNode('store')
+                            ->info('Service name of store')
+                            ->defaultValue(StoreInterface::class)
+                        ->end()
+                    ->end()
+                ->end()
+            ->end()
         ->end()
         ->validate()
             ->ifTrue(function ($v) {

src/ai-bundle/config/services.php

@@ -68,6 +68,7 @@
 use Symfony\AI\Platform\StructuredOutput\ResponseFormatFactoryInterface;
 use Symfony\AI\Store\Command\DropStoreCommand;
 use Symfony\AI\Store\Command\IndexCommand;
+use Symfony\AI\Store\Command\RetrieveCommand;
 use Symfony\AI\Store\Command\SetupStoreCommand;
 
 return static function (ContainerConfigurator $container): void {
@@ -220,6 +221,11 @@
                 tagged_locator('ai.indexer', 'name'),
             ])
             ->tag('console.command')
+        ->set('ai.command.retrieve', RetrieveCommand::class)
+            ->args([
+                tagged_locator('ai.retriever', 'name'),
+            ])
+            ->tag('console.command')
         ->set('ai.command.platform_invoke', PlatformInvokeCommand::class)
             ->args([
                 tagged_locator('ai.platform', 'name'),