add document chunks

alnutile · May 8, 2023 · d54c9e5 · d54c9e5
1 parent 5967f14
commit d54c9e5
Show file tree

Hide file tree

Showing 8 changed files with 15 additions and 35 deletions.
diff --git a/app/Models/Document.php b/app/Models/Document.php
@@ -6,8 +6,6 @@
 use App\Ingress\StatusEnum;
 use Illuminate\Database\Eloquent\Factories\HasFactory;
 use Illuminate\Database\Eloquent\Model;
-use Illuminate\Database\Eloquent\Relations\BelongsTo;
-use Illuminate\Database\Eloquent\Relations\HasOneThrough;
 use Pgvector\Laravel\Vector;
 
 /**
@@ -49,7 +47,8 @@ public function project()
         );
     }
 
-    public function document_chunks() {
+    public function document_chunks()
+    {
         return $this->hasMany(DocumentChunk::class);
     }
 }
diff --git a/app/Models/DocumentChunk.php b/app/Models/DocumentChunk.php
@@ -11,9 +11,8 @@ class DocumentChunk extends Model
 
     protected $guarded = [];
 
-    public function document() {
+    public function document()
+    {
         return $this->belongsTo(Document::class);
     }
-
-
 }
diff --git a/app/Transformers/BaseTransformer.php b/app/Transformers/BaseTransformer.php
@@ -2,18 +2,13 @@
 
 namespace App\Transformers;
 
-use App\LLMModels\OpenAi\EmbeddingsResponseDto;
 use App\Models\Document;
-use Facades\App\LLMModels\OpenAi\ClientWrapper;
 
 abstract class BaseTransformer
 {
-
     public function __construct(public Document $document)
     {
     }
 
-
-    abstract  public function handle() : Document;
-
+    abstract public function handle(): Document;
 }
diff --git a/app/Transformers/Types/PdfTransformer.php b/app/Transformers/Types/PdfTransformer.php
@@ -2,14 +2,11 @@
 
 namespace App\Transformers\Types;
 
-use App\LLMModels\OpenAi\EmbeddingsResponseDto;
 use App\Models\Document;
 use App\Transformers\BaseTransformer;
-use Facades\App\LLMModels\OpenAi\ClientWrapper;
 
 class PdfTransformer extends BaseTransformer
 {
-
     public function handle(): Document
     {
         //find the source

diff --git a/database/factories/DocumentChunkFactory.php b/database/factories/DocumentChunkFactory.php
@@ -2,9 +2,7 @@
 
 namespace Database\Factories;
 
-use App\Ingress\StatusEnum;
 use App\Models\Document;
-use App\Models\Source;
 use Illuminate\Database\Eloquent\Factories\Factory;
 
 /**

diff --git a/database/migrations/2023_05_08_130543_create_document_chunks_table.php b/database/migrations/2023_05_08_130543_create_document_chunks_table.php
@@ -15,8 +15,8 @@ public function up(): void
         Schema::create('document_chunks', function (Blueprint $table) {
             $table->id();
             $table->foreignIdFor(Document::class);
-            $table->string("guid"); //page number or other
-            $table->longText("content")->nullable();
+            $table->string('guid'); //page number or other
+            $table->longText('content')->nullable();
             $table->vector('embedding', 1536)->nullable();
             $table->timestamps();
         });

diff --git a/tests/Feature/Models/DocumentChunkTest.php b/tests/Feature/Models/DocumentChunkTest.php
@@ -3,14 +3,10 @@
 namespace Tests\Feature\Models;
 
 use App\Models\DocumentChunk;
-use Illuminate\Foundation\Testing\RefreshDatabase;
-use Illuminate\Foundation\Testing\WithFaker;
 use Tests\TestCase;
 
 class DocumentChunkTest extends TestCase
 {
-
-
     public function test_dc_factory()
     {
         $model = DocumentChunk::factory()->create();

diff --git a/tests/Feature/PdfTransformerTest.php b/tests/Feature/PdfTransformerTest.php
@@ -5,36 +5,32 @@
 use App\Models\Document;
 use App\Models\Source;
 use App\Transformers\Types\PdfTransformer;
-use Illuminate\Foundation\Testing\RefreshDatabase;
-use Illuminate\Foundation\Testing\WithFaker;
 use Illuminate\Support\Facades\Storage;
 use Tests\TestCase;
 
 class PdfTransformerTest extends TestCase
 {
+    public function test_gets_data_from_pdf()
+    {
 
-    public function test_gets_data_from_pdf() {
-
-        $this->markTestSkipped("Need to do document chunks first");
+        $this->markTestSkipped('Need to do document chunks first');
 
         $source = Source::factory()
             ->webFileMetaData()
             ->create();
 
-        Storage::disk("projects")->copy(
-            base_path("tests/fixtures/example.pdf"),
-            sprintf(storage_path("app/projects/%d/sources/%d/example.pdf",
+        Storage::disk('projects')->copy(
+            base_path('tests/fixtures/example.pdf'),
+            sprintf(storage_path('app/projects/%d/sources/%d/example.pdf',
             $source->project_id, $source->id))
         );
 
         $document = Document::factory()->create([
-            "source_id" => $source->id,
-            "guid" => "example.pdf"
+            'source_id' => $source->id,
+            'guid' => 'example.pdf',
         ]);
 
         $trasformer = new PdfTransformer($document);
 
-
-
     }
 }