fix overflow

app/Console/Commands/RoachSite.php

@@ -0,0 +1,41 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Spiders\GetPageSpider;
+use Illuminate\Console\Command;
+use Illuminate\Support\Arr;
+use RoachPHP\ItemPipeline\Item;
+use RoachPHP\Roach;
+use RoachPHP\Spider\Configuration\Overrides;
+
+class RoachSite extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'roach:example {url}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Pass url to see it work';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $items = Roach::collectSpider(
+            GetPageSpider::class,
+            new Overrides(startUrls: [$this->argument('url')])
+        );
+
+        $item = Arr::first($items);
+        dd($item->get(0));
+    }
+}

app/Events/MessageStatusEvent.php

@@ -3,9 +3,7 @@
 namespace App\Events;
 
 use App\Models\Message;
-use Illuminate\Broadcasting\Channel;
 use Illuminate\Broadcasting\InteractsWithSockets;
-use Illuminate\Broadcasting\PresenceChannel;
 use Illuminate\Broadcasting\PrivateChannel;
 use Illuminate\Contracts\Broadcasting\ShouldBroadcast;
 use Illuminate\Foundation\Events\Dispatchable;
@@ -35,7 +33,6 @@ public function broadcastOn(): array
         ];
     }
 
-
     /**
      * The event's broadcast name.
      */

app/Jobs/MailBoxParserJob.php

@@ -6,6 +6,8 @@
 use App\Models\Message;
 use App\Models\Tag;
 use App\Models\User;
+use Facades\App\OpenAi\ChatClient;
+use Facades\App\Tools\GetSiteWrapper;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
@@ -31,21 +33,57 @@ public function handle(): void
     {
         try {
 
-            $tag = Tag::firstOrCreate([
+            $tag1 = Tag::firstOrCreate([
                 'label' => 'email',
             ], [
                 'active' => 1,
             ]);
 
+            $tag2 = Tag::firstOrCreate([
+                'label' => 'tldr',
+            ], [
+                'active' => 1,
+            ]);
+
+            $content = $this->mailDto->body;
+
+            $hasUrl = get_url_from_body($content);
+
+            if ($hasUrl) {
+                $body = GetSiteWrapper::handle($hasUrl);
+                $messages = [];
+                $messages[] = [
+                    'role' => 'system',
+                    'content' => 'Please TLDR the following content the user provides',
+                ];
+                $messages[] = [
+                    'role' => 'user',
+                    'content' => $body,
+                ];
+
+                $results = ChatClient::chat($messages);
+
+                $content = $results->content;
+
+                $content = sprintf("
+                URL: %s\n
+                Content: %s",
+                    $hasUrl,
+                    $content
+                );
+            }
+
             $message = Message::create([
                 'role' => 'user',
                 'user_id' => User::first()->id,
                 'content' => sprintf("subject: %s \n body: %s",
                     $this->mailDto->subject,
-                    $this->mailDto->body),
+                    $content),
             ]);
 
-            $message->tags()->attach($tag->id);
+            $message->tags()->attach([
+                $tag1->id, $tag2->id,
+            ]);
 
             /**
              * If tons of text we tldr it

app/Jobs/MessageCreatedJob.php

@@ -30,7 +30,7 @@ public function __construct(public Message $message)
     public function handle(): void
     {
         try {
-            logger("Sending llm request");
+            logger('Sending llm request');
             MessageStatusEvent::dispatch($this->message);
             /** @var Response $results */
             $results = MessageRepository::handle($this->message);

app/Spiders/GetPageSpider.php

@@ -0,0 +1,50 @@
+<?php
+
+namespace App\Spiders;
+
+use Generator;
+use RoachPHP\Downloader\Middleware\RequestDeduplicationMiddleware;
+use RoachPHP\Extensions\LoggerExtension;
+use RoachPHP\Extensions\StatsCollectorExtension;
+use RoachPHP\Http\Response;
+use RoachPHP\Spider\BasicSpider;
+use RoachPHP\Spider\ParseResult;
+
+class GetPageSpider extends BasicSpider
+{
+    public array $startUrls = [
+        //
+    ];
+
+    public array $downloaderMiddleware = [
+        RequestDeduplicationMiddleware::class,
+    ];
+
+    public array $spiderMiddleware = [
+        //
+    ];
+
+    public array $itemProcessors = [
+        //
+    ];
+
+    public array $extensions = [
+        LoggerExtension::class,
+        StatsCollectorExtension::class,
+    ];
+
+    public int $concurrency = 2;
+
+    public int $requestDelay = 1;
+
+    /**
+     * @return Generator<ParseResult>
+     */
+    public function parse(Response $response): Generator
+    {
+        $body = $response->filter('body')->text();
+        yield $this->item([
+            $body,
+        ]);
+    }
+}

app/Spiders/ResponseDto.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace App\Spiders;
+
+class ResponseDto extends \Spatie\LaravelData\Data
+{
+    public function __construct(
+        public string $content)
+    {
+    }
+}

app/Tools/GetSiteWrapper.php

@@ -0,0 +1,30 @@
+<?php
+
+namespace App\Tools;
+
+use App\Spiders\GetPageSpider;
+use Illuminate\Support\Arr;
+use RoachPHP\ItemPipeline\Item;
+use RoachPHP\Roach;
+use RoachPHP\Spider\Configuration\Overrides;
+
+class GetSiteWrapper
+{
+    public function handle(string $url): string|null
+    {
+
+        /** @var Item[] $items */
+        $items = Roach::collectSpider(
+            GetPageSpider::class,
+            new Overrides(startUrls: [$url])
+        );
+
+        $item = Arr::first($items);
+
+        if ($item) {
+            return $item->get(0);
+        }
+
+        return null;
+    }
+}

app/helpers.php

@@ -1,12 +1,26 @@
 <?php
 
+use Illuminate\Support\Arr;
+
 if (! function_exists('test_helper')) {
     function test_helper()
     {
         return 'foo';
     }
 }
 
+if (! function_exists('get_url_from_body')) {
+    function get_url_from_body(string $body): string|null
+    {
+        $pattern = '#\bhttps?://[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/))#';
+        if (preg_match($pattern, $body, $match)) {
+            return Arr::first($match);
+        }
+
+        return null;
+    }
+}
+
 if (! function_exists('get_current_weather')) {
     function get_current_weather($location, $unit = 'fahrenheit'): string
     {

composer.json

@@ -21,6 +21,8 @@
         "opcodesio/log-viewer": "^2.4",
         "openai-php/laravel": "^0.6.0",
         "pusher/pusher-php-server": "^7.2",
+        "roach-php/core": "^2.0",
+        "roach-php/laravel": "^2.0",
         "spatie/laravel-backup": "^8.1",
         "sundance-solutions/larachain-token-count": "^1.0",
         "sundance-solutions/larachain-trim-text": "^1.0",