Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/store/src/Document/Loader/TextFileLoader.php
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
*/
final readonly class TextFileLoader implements LoaderInterface
{
public function __invoke(string $source, array $options = []): iterable
public function load(string $source, array $options = []): iterable
{
if (!is_file($source)) {
throw new RuntimeException(\sprintf('File "%s" does not exist.', $source));
Expand Down
2 changes: 1 addition & 1 deletion src/store/src/Document/LoaderInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,5 @@ interface LoaderInterface
*
* @return iterable<TextDocument> iterable of TextDocuments loaded from the source
*/
public function __invoke(string $source, array $options = []): iterable;
public function load(string $source, array $options = []): iterable;
}
4 changes: 2 additions & 2 deletions src/store/src/Document/Transformer/ChainTransformer.php
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ public function __construct(iterable $transformers)
$this->transformers = $transformers instanceof \Traversable ? iterator_to_array($transformers) : $transformers;
}

public function __invoke(iterable $documents, array $options = []): iterable
public function transform(iterable $documents, array $options = []): iterable
{
foreach ($this->transformers as $transformer) {
$documents = $transformer($documents, $options);
$documents = $transformer->transform($documents, $options);
}

return $documents;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public function __construct(
/**
* @param array{chunk_size?: int, delay?: int} $options
*/
public function __invoke(iterable $documents, array $options = []): iterable
public function transform(iterable $documents, array $options = []): iterable
{
$chunkSize = $options[self::OPTION_CHUNK_SIZE] ?? 50;
$delay = $options[self::OPTION_DELAY] ?? 10;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
/**
* @param array{chunk_size?: int, overlap?: int} $options
*/
public function __invoke(iterable $documents, array $options = []): iterable
public function transform(iterable $documents, array $options = []): iterable
{
$chunkSize = $options[self::OPTION_CHUNK_SIZE] ?? 1000;
$overlap = $options[self::OPTION_OVERLAP] ?? 200;
Expand Down
2 changes: 1 addition & 1 deletion src/store/src/Document/TransformerInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,5 @@ interface TransformerInterface
*
* @return iterable<TextDocument>
*/
public function __invoke(iterable $documents, array $options = []): iterable;
public function transform(iterable $documents, array $options = []): iterable;
}
2 changes: 1 addition & 1 deletion src/store/src/Document/Vectorizer.php
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public function __construct(
) {
}

public function __invoke(array $documents): array
public function vectorize(array $documents): array
{
$documentCount = \count($documents);
$this->logger->info('Starting vectorization process', ['document_count' => $documentCount]);
Expand Down
2 changes: 1 addition & 1 deletion src/store/src/Document/VectorizerInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,5 @@ interface VectorizerInterface
*
* @return VectorDocument[]
*/
public function __invoke(array $documents): array;
public function vectorize(array $documents): array;
}
4 changes: 2 additions & 2 deletions src/store/src/Indexer.php
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,13 @@ public function index(TextDocument|iterable $documents, int $chunkSize = 50): vo
++$counter;

if ($chunkSize === \count($chunk)) {
$this->store->add(...($this->vectorizer)($chunk));
$this->store->add(...$this->vectorizer->vectorize($chunk));
$chunk = [];
}
}

if (\count($chunk) > 0) {
$this->store->add(...($this->vectorizer)($chunk));
$this->store->add(...$this->vectorizer->vectorize($chunk));
}

$this->logger->debug(0 === $counter ? 'No documents to index' : \sprintf('Indexed %d documents', $counter));
Expand Down
6 changes: 3 additions & 3 deletions src/store/tests/Document/Loader/TextFileLoaderTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@ public function testLoadWithInvalidSource()
$this->expectException(RuntimeException::class);
$this->expectExceptionMessage('File "/invalid/source.txt" does not exist.');

iterator_to_array($loader('/invalid/source.txt'));
iterator_to_array($loader->load('/invalid/source.txt'));
}

public function testLoadWithValidSource()
{
$loader = new TextFileLoader();

$documents = iterator_to_array($loader(\dirname(__DIR__, 5).'/fixtures/lorem.txt'));
$documents = iterator_to_array($loader->load(\dirname(__DIR__, 5).'/fixtures/lorem.txt'));

$this->assertCount(1, $documents);
$this->assertInstanceOf(TextDocument::class, $document = $documents[0]);
Expand All @@ -48,7 +48,7 @@ public function testSourceIsPresentInMetadata()
$loader = new TextFileLoader();

$source = \dirname(__DIR__, 5).'/fixtures/lorem.txt';
$documents = iterator_to_array($loader($source));
$documents = iterator_to_array($loader->load($source));

$this->assertCount(1, $documents);
$this->assertInstanceOf(TextDocument::class, $document = $documents[0]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ final class ChainTransformerTest extends TestCase
public function testChainTransformerAppliesAllTransformersInOrder()
{
$transformerA = new class implements TransformerInterface {
public function __invoke(iterable $documents, array $options = []): iterable
public function transform(iterable $documents, array $options = []): iterable
{
foreach ($documents as $document) {
yield new TextDocument($document->id, $document->content.'-A');
Expand All @@ -33,7 +33,7 @@ public function __invoke(iterable $documents, array $options = []): iterable
};

$transformerB = new class implements TransformerInterface {
public function __invoke(iterable $documents, array $options = []): iterable
public function transform(iterable $documents, array $options = []): iterable
{
foreach ($documents as $document) {
yield new TextDocument($document->id, $document->content.'-B');
Expand All @@ -47,7 +47,7 @@ public function __invoke(iterable $documents, array $options = []): iterable
new TextDocument(Uuid::v4(), 'bar'),
];

$result = iterator_to_array($chain->__invoke($documents));
$result = iterator_to_array($chain->transform($documents));

$this->assertSame('foo-A-B', $result[0]->content);
$this->assertSame('bar-A-B', $result[1]->content);
Expand All @@ -58,7 +58,7 @@ public function testChainTransformerWithNoTransformersReturnsInput()
$chain = new ChainTransformer([]);
$documents = [new TextDocument(Uuid::v4(), 'baz')];

$result = iterator_to_array($chain->__invoke($documents));
$result = iterator_to_array($chain->transform($documents));

$this->assertSame('baz', $result[0]->content);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public function testDefaultChunkSizeAndDelay()
$documents[] = new TextDocument(Uuid::v4(), 'content-'.$i);
}

$result = iterator_to_array($transformer($documents));
$result = iterator_to_array($transformer->transform($documents));

$this->assertCount(30, $result);
for ($i = 0; $i < 30; ++$i) {
Expand All @@ -57,7 +57,7 @@ public function testSleepsAfterChunkSize()
$documents[] = new TextDocument(Uuid::v4(), 'content-'.$i);
}

$result = iterator_to_array($transformer($documents, [
$result = iterator_to_array($transformer->transform($documents, [
ChunkDelayTransformer::OPTION_CHUNK_SIZE => 50,
ChunkDelayTransformer::OPTION_DELAY => 5,
]));
Expand All @@ -79,7 +79,7 @@ public function testCustomChunkSizeAndDelay()
$documents[] = new TextDocument(Uuid::v4(), 'content-'.$i);
}

$result = iterator_to_array($transformer($documents, [
$result = iterator_to_array($transformer->transform($documents, [
ChunkDelayTransformer::OPTION_CHUNK_SIZE => 10,
ChunkDelayTransformer::OPTION_DELAY => 2,
]));
Expand All @@ -100,7 +100,7 @@ public function testNoSleepWhenDelayIsZero()
$documents[] = new TextDocument(Uuid::v4(), 'content-'.$i);
}

$result = iterator_to_array($transformer($documents, [
$result = iterator_to_array($transformer->transform($documents, [
ChunkDelayTransformer::OPTION_CHUNK_SIZE => 5,
ChunkDelayTransformer::OPTION_DELAY => 0,
]));
Expand All @@ -119,7 +119,7 @@ public function testYieldsDocumentsInCorrectOrder()
new TextDocument(Uuid::v4(), 'third'),
];

$result = iterator_to_array($transformer($documents, [
$result = iterator_to_array($transformer->transform($documents, [
ChunkDelayTransformer::OPTION_CHUNK_SIZE => 2,
ChunkDelayTransformer::OPTION_DELAY => 1,
]));
Expand All @@ -137,7 +137,7 @@ public function testHandlesEmptyIterable()

$transformer = new ChunkDelayTransformer($clock);

$result = iterator_to_array($transformer([]));
$result = iterator_to_array($transformer->transform([]));

$this->assertCount(0, $result);
}
Expand All @@ -153,7 +153,7 @@ public function testSingleDocument()

$documents = [new TextDocument(Uuid::v4(), 'single')];

$result = iterator_to_array($transformer($documents, [
$result = iterator_to_array($transformer->transform($documents, [
ChunkDelayTransformer::OPTION_CHUNK_SIZE => 1,
ChunkDelayTransformer::OPTION_DELAY => 5,
]));
Expand All @@ -176,7 +176,7 @@ public function testExactlyChunkSizeDocuments()
$documents[] = new TextDocument(Uuid::v4(), 'content-'.$i);
}

$result = iterator_to_array($transformer($documents, [
$result = iterator_to_array($transformer->transform($documents, [
ChunkDelayTransformer::OPTION_CHUNK_SIZE => 10,
ChunkDelayTransformer::OPTION_DELAY => 3,
]));
Expand All @@ -198,7 +198,7 @@ public function testMultipleExactChunks()
$documents[] = new TextDocument(Uuid::v4(), 'content-'.$i);
}

$result = iterator_to_array($transformer($documents, [
$result = iterator_to_array($transformer->transform($documents, [
ChunkDelayTransformer::OPTION_CHUNK_SIZE => 5,
ChunkDelayTransformer::OPTION_DELAY => 1,
]));
Expand All @@ -220,7 +220,7 @@ public function testLazyEvaluation()
$documents[] = new TextDocument(Uuid::v4(), 'content-'.$i);
}

$generator = $transformer($documents, [
$generator = $transformer->transform($documents, [
ChunkDelayTransformer::OPTION_CHUNK_SIZE => 3,
ChunkDelayTransformer::OPTION_DELAY => 1,
]);
Expand Down Expand Up @@ -248,7 +248,7 @@ public function testWithMockClock()

$startTime = $clock->now();

iterator_to_array($transformer($documents, [
iterator_to_array($transformer->transform($documents, [
ChunkDelayTransformer::OPTION_CHUNK_SIZE => 5,
ChunkDelayTransformer::OPTION_DELAY => 30,
]));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public function testSplitReturnsSingleChunkForShortText()
{
$document = new TextDocument(Uuid::v4(), 'short text');

$chunks = iterator_to_array(($this->transformer)([$document]));
$chunks = iterator_to_array($this->transformer->transform([$document]));

$this->assertCount(1, $chunks);
$this->assertSame('short text', $chunks[0]->content);
Expand All @@ -48,7 +48,7 @@ public function testSplitSplitsLongTextWithOverlap()
{
$document = new TextDocument(Uuid::v4(), $this->getLongText());

$chunks = iterator_to_array(($this->transformer)([$document]));
$chunks = iterator_to_array($this->transformer->transform([$document]));

$this->assertCount(2, $chunks);

Expand All @@ -63,7 +63,7 @@ public function testSplitWithCustomChunkSizeAndOverlap()
{
$document = new TextDocument(Uuid::v4(), $this->getLongText());

$chunks = iterator_to_array(($this->transformer)([$document], [
$chunks = iterator_to_array($this->transformer->transform([$document], [
TextSplitTransformer::OPTION_CHUNK_SIZE => 150,
TextSplitTransformer::OPTION_OVERLAP => 25,
]));
Expand Down Expand Up @@ -111,7 +111,7 @@ public function testSplitWithZeroOverlap()
{
$document = new TextDocument(Uuid::v4(), $this->getLongText());

$chunks = iterator_to_array(($this->transformer)([$document], [
$chunks = iterator_to_array($this->transformer->transform([$document], [
TextSplitTransformer::OPTION_OVERLAP => 0,
]));

Expand All @@ -124,7 +124,7 @@ public function testParentIdIsSetInMetadata()
{
$document = new TextDocument(Uuid::v4(), $this->getLongText());

$chunks = iterator_to_array(($this->transformer)([$document], [
$chunks = iterator_to_array($this->transformer->transform([$document], [
TextSplitTransformer::OPTION_CHUNK_SIZE => 1000,
TextSplitTransformer::OPTION_OVERLAP => 200,
]));
Expand All @@ -141,7 +141,7 @@ public function testMetadataIsInherited()
'foo' => 'bar',
]));

$chunks = iterator_to_array(($this->transformer)([$document]));
$chunks = iterator_to_array($this->transformer->transform([$document]));

$this->assertCount(2, $chunks);
$this->assertSame('value', $chunks[0]->metadata['key']);
Expand All @@ -154,7 +154,7 @@ public function testSplitWithChunkSizeLargerThanText()
{
$document = new TextDocument(Uuid::v4(), 'tiny');

$chunks = iterator_to_array(($this->transformer)([$document]));
$chunks = iterator_to_array($this->transformer->transform([$document]));

$this->assertCount(1, $chunks);
$this->assertSame('tiny', $chunks[0]->content);
Expand All @@ -166,7 +166,7 @@ public function testSplitWithOverlapGreaterThanChunkSize()
$this->expectException(InvalidArgumentException::class);
$this->expectExceptionMessage('Overlap must be non-negative and less than chunk size.');

iterator_to_array(($this->transformer)([$document], [
iterator_to_array($this->transformer->transform([$document], [
TextSplitTransformer::OPTION_CHUNK_SIZE => 10,
TextSplitTransformer::OPTION_OVERLAP => 20,
]));
Expand All @@ -178,7 +178,7 @@ public function testSplitWithNegativeOverlap()
$this->expectException(InvalidArgumentException::class);
$this->expectExceptionMessage('Overlap must be non-negative and less than chunk size.');

iterator_to_array(($this->transformer)([$document], [
iterator_to_array($this->transformer->transform([$document], [
TextSplitTransformer::OPTION_CHUNK_SIZE => 10,
TextSplitTransformer::OPTION_OVERLAP => -1,
]));
Expand Down
Loading