[Store] Do not allow to store state in Indexer type

demo/config/packages/ai.yaml

@@ -91,8 +91,7 @@ ai:
             model: 'text-embedding-ada-002'
     indexer:
         blog:
-            loader: 'Symfony\AI\Store\Document\Loader\RssFeedLoader'
-            source: 'https://feeds.feedburner.com/symfony/blog'
+            loader: app.loader.blog
             filters:
                 - 'app.filter.week_of_symfony'
             transformers:
@@ -123,3 +122,8 @@ services:
         arguments:
             $needle: 'Week of Symfony'
             $caseSensitive: false
+
+    app.loader.blog:
+        class: 'Symfony\AI\Store\Document\Loader\RssFeedLoader'
+        arguments:
+            $defaultSource: 'https://feeds.feedburner.com/symfony/blog'

docs/components/store.rst

@@ -31,7 +31,7 @@ used vector store::
 
     $indexer = new Indexer($platform, $model, $store);
     $document = new TextDocument('This is a sample document.');
-    $indexer->index($document);
+    $indexer->loadAndIndex($document);
 
 You can find more advanced usage in combination with an Agent using the store for RAG in the examples folder.
 

docs/cookbook/rag-implementation.rst

@@ -97,7 +97,7 @@ Use a vectorizer to convert documents into embeddings and store them::
         $vectorizer,
         $store
     );
-    $indexer->index($documents);
+    $indexer->loadAndIndex();
 
 The indexer handles:
 
@@ -324,7 +324,7 @@ Index documents in batches for better performance::
 
     $batchSize = 100;
     foreach (array_chunk($documents, $batchSize) as $batch) {
-        $indexer->index($batch);
+        $indexer->loadAndIndex(options: $batch);
     }
 
 Caching Embeddings

examples/indexer/index-file-loader.php

@@ -26,18 +26,17 @@
     loader: new TextFileLoader(),
     vectorizer: $vectorizer,
     store: $store,
-    source: [
-        dirname(__DIR__, 2).'/fixtures/movies/gladiator.md',
-        dirname(__DIR__, 2).'/fixtures/movies/inception.md',
-        dirname(__DIR__, 2).'/fixtures/movies/jurassic-park.md',
-    ],
     transformers: [
         new TextReplaceTransformer(search: '## Plot', replace: '## Synopsis'),
         new TextSplitTransformer(chunkSize: 500, overlap: 100),
     ],
 );
 
-$indexer->index();
+$indexer->loadAndIndex([
+    dirname(__DIR__, 2).'/fixtures/movies/gladiator.md',
+    dirname(__DIR__, 2).'/fixtures/movies/inception.md',
+    dirname(__DIR__, 2).'/fixtures/movies/jurassic-park.md',
+]);
 
 $vector = $vectorizer->vectorize('Roman gladiator revenge');
 $results = $store->query($vector);

examples/indexer/index-inmemory-loader.php

@@ -42,13 +42,12 @@
     loader: new InMemoryLoader($documents),
     vectorizer: $vectorizer,
     store: $store,
-    source: null,
     transformers: [
         new TextSplitTransformer(chunkSize: 100, overlap: 20),
     ],
 );
 
-$indexer->index();
+$indexer->loadAndIndex();
 
 $vector = $vectorizer->vectorize('machine learning artificial intelligence');
 $results = $store->query($vector);

examples/indexer/index-rss-loader.php

@@ -26,16 +26,15 @@
     loader: new RssFeedLoader(HttpClient::create()),
     vectorizer: $vectorizer,
     store: $store,
-    source: [
-        'https://feeds.feedburner.com/symfony/blog',
-        'https://www.tagesschau.de/index~rss2.xml',
-    ],
     transformers: [
         new TextSplitTransformer(chunkSize: 500, overlap: 100),
     ],
 );
 
-$indexer->index();
+$indexer->loadAndIndex([
+    'https://feeds.feedburner.com/symfony/blog',
+    'https://www.tagesschau.de/index~rss2.xml',
+]);
 
 $vector = $vectorizer->vectorize('Week of Symfony');
 $results = $store->query($vector);

examples/indexer/index-with-filters.php

@@ -60,14 +60,13 @@
     loader: new InMemoryLoader($documents),
     vectorizer: $vectorizer,
     store: $store,
-    source: null,
     filters: $filters,
     transformers: [
         new TextTrimTransformer(),
     ],
 );
 
-$indexer->index();
+$indexer->loadAndIndex();
 
 $vector = $vectorizer->vectorize('technology artificial intelligence');
 $results = $store->query($vector);

examples/memory/mariadb.php

@@ -58,7 +58,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, $embeddings = 'text-embedding-3-small');
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 // Execute a chat call that is utilizing the memory
 $embeddingsModel = $platform->getModelCatalog()->getModel($embeddings);

examples/ollama/indexer.php

@@ -26,18 +26,17 @@
     loader: new TextFileLoader(),
     vectorizer: $vectorizer,
     store: $store,
-    source: [
-        dirname(__DIR__, 2).'/fixtures/movies/gladiator.md',
-        dirname(__DIR__, 2).'/fixtures/movies/inception.md',
-        dirname(__DIR__, 2).'/fixtures/movies/jurassic-park.md',
-    ],
     transformers: [
         new TextReplaceTransformer(search: '## Plot', replace: '## Synopsis'),
         new TextSplitTransformer(chunkSize: 500, overlap: 100),
     ],
 );
 
-$indexer->index();
+$indexer->loadAndIndex([
+    dirname(__DIR__, 2).'/fixtures/movies/gladiator.md',
+    dirname(__DIR__, 2).'/fixtures/movies/inception.md',
+    dirname(__DIR__, 2).'/fixtures/movies/jurassic-park.md',
+]);
 
 $vector = $vectorizer->vectorize('Roman gladiator revenge');
 $results = $store->query($vector);

examples/ollama/rag.php

@@ -44,7 +44,7 @@
 $platform = PlatformFactory::create(env('OLLAMA_HOST_URL'), http_client());
 $vectorizer = new Vectorizer($platform, env('OLLAMA_EMBEDDINGS'), logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/cache.php

@@ -45,7 +45,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/chromadb.php

@@ -52,7 +52,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/clickhouse.php

@@ -52,7 +52,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/cloudflare.php

@@ -52,7 +52,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/in-memory.php

@@ -44,7 +44,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/manticore.php

@@ -52,7 +52,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/mariadb-gemini.php

@@ -54,7 +54,7 @@
 $model = 'gemini-embedding-exp-03-07?dimensions=768&task_type=SEMANTIC_SIMILARITY';
 $vectorizer = new Vectorizer($platform, $model, logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/mariadb-openai.php

@@ -53,7 +53,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/meilisearch-hybrid.php

@@ -51,7 +51,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 // Create a query embedding
 $queryText = 'futuristic technology and artificial intelligence';

examples/rag/meilisearch.php

@@ -52,7 +52,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/milvus.php

@@ -53,7 +53,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/mongodb.php

@@ -51,7 +51,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'));
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 // initialize the index
 $store->setup();

examples/rag/neo4j.php

@@ -55,7 +55,7 @@
 $platform = PlatformFactory::create($_SERVER['OPENAI_API_KEY']);
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/pinecone.php

@@ -45,7 +45,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/postgres.php

@@ -52,7 +52,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/qdrant.php

@@ -52,7 +52,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/redis.php

@@ -54,7 +54,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/supabase.php

@@ -52,7 +52,7 @@ functionName: env('SUPABASE_MATCH_FUNCTION'),
 $vectorizer = new Vectorizer($platform, env('OLLAMA_EMBEDDINGS'));
 $loader = new InMemoryLoader($documents);
 $indexer = new Indexer($loader, $vectorizer, $store, logger: logger());
-$indexer->index();
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/surrealdb.php

@@ -55,7 +55,7 @@
 $platform = PlatformFactory::create($_SERVER['OPENAI_API_KEY']);
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/typesense.php

@@ -52,7 +52,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/rag/weaviate.php

@@ -53,7 +53,7 @@
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index($documents);
+$indexer->loadAndIndex();
 
 $similaritySearch = new SimilaritySearch($vectorizer, $store);
 $toolbox = new Toolbox([$similaritySearch], logger: logger());

examples/retriever/basic.php

@@ -28,16 +28,15 @@
     loader: new TextFileLoader(),
     vectorizer: $vectorizer,
     store: $store,
-    source: [
-        dirname(__DIR__, 2).'/fixtures/movies/gladiator.md',
-        dirname(__DIR__, 2).'/fixtures/movies/inception.md',
-        dirname(__DIR__, 2).'/fixtures/movies/jurassic-park.md',
-    ],
     transformers: [
         new TextSplitTransformer(chunkSize: 500, overlap: 100),
     ],
 );
-$indexer->index();
+$indexer->loadAndIndex([
+    dirname(__DIR__, 2).'/fixtures/movies/gladiator.md',
+    dirname(__DIR__, 2).'/fixtures/movies/inception.md',
+    dirname(__DIR__, 2).'/fixtures/movies/jurassic-park.md',
+]);
 
 $retriever = new Retriever(
     vectorizer: $vectorizer,

examples/retriever/movies.php

@@ -37,7 +37,7 @@
 $vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
 
 $indexer = new Indexer(new InMemoryLoader($documents), $vectorizer, $store, logger: logger());
-$indexer->index();
+$indexer->loadAndIndex();
 
 $retriever = new Retriever($vectorizer, $store, logger());
 

src/ai-bundle/config/options.php

@@ -1064,10 +1064,6 @@
                             ->info('Service name of loader')
                             ->isRequired()
                         ->end()
-                        ->variableNode('source')
-                            ->info('Source identifier (file path, URL, etc.) or array of sources')
-                            ->defaultNull()
-                        ->end()
                         ->arrayNode('transformers')
                             ->info('Array of transformer service names')
                             ->scalarPrototype()->end()