Get the url's content on the Robots class in order to reduce the numb…

…er of requests to the remote server.

src/Robots.php

@@ -39,17 +39,29 @@ public function mayIndex(string $url, string $userAgent = null): bool
 
         $robotsTxt = $this->robotsTxt ?? RobotsTxt::create($this->createRobotsUrl($url));
 
+        $content = @file_get_contents($url);
+
+        if ($content === false) {
+            throw new InvalidArgumentException("Could not read url `{$url}`");
+        }
+
         return
             $robotsTxt->allows($url, $userAgent)
-            && RobotsMeta::readFrom($url)->mayIndex()
-            && RobotsHeaders::readFrom($url)->mayIndex();
+            && RobotsMeta::create($content)->mayIndex()
+            && RobotsHeaders::create($http_response_header ?? [])->mayIndex();
     }
 
     public function mayFollowOn(string $url): bool
     {
+        $content = @file_get_contents($url);
+
+        if ($content === false) {
+            throw new InvalidArgumentException("Could not read url `{$url}`");
+        }
+
         return
-            RobotsMeta::readFrom($url)->mayFollow()
-            && RobotsHeaders::readFrom($url)->mayFollow();
+            RobotsMeta::create($content)->mayFollow()
+            && RobotsHeaders::create($http_response_header ?? [])->mayFollow();
     }
 
     protected function createRobotsUrl(string $url): string

tests/RobotsTest.php

@@ -3,6 +3,7 @@
 namespace Spatie\Robots\Tests;
 
 use Spatie\Robots\Robots;
+use Spatie\Robots\RobotsTxt;
 
 class RobotsTest extends TestCase
 {
@@ -15,6 +16,16 @@ public function test()
         $this->assertTrue($robots->mayIndex('/'));
     }
 
+    /** @test */
+    public function it_can_be_created_with_a_robots_txt_object()
+    {
+        $robotsTxt = RobotsTxt::create(__DIR__.'/data/robots.txt');
+        $robots = Robots::create()
+            ->withTxt($robotsTxt);
+
+        $this->assertTrue($robots->mayIndex('/'));
+    }
+
     /** @test */
     public function it_return_true_on_source_string()
     {