Add ability to process prefetched content

fetchContent() now accepts an optional parameter, prefetchedContent, which can contain the content of a page that was fetched before calling Graby. If we take the example of Wallabag it gives the ability of sending the content of a page (through a browser extension for example) without making network calls to fetch the page. Signed-off-by: Kevin Decherf <kevin@kdecherf.com>
j0k3r · Feb 3, 2022 · 31135a7 · 31135a7
1 parent fda6724
commit 31135a7
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 5 deletions.
diff --git a/README.md b/README.md
@@ -131,6 +131,22 @@ array(
 
 The `date` result is the same as displayed in the content. If `date` is not `null` in the result, we recommend you to parse it using [`date_parse`](http://php.net/date_parse) (this is what we are using to validate that the date is correct).
 
+### Retrieve content from a prefetched page
+
+If you want to extract content from a page you fetched outside of Graby, you can call `setContentAsPrefetched()` before calling `fetchContent()`, e.g.:
+
+``` php
+use Graby\Graby;
+
+$article = 'http://www.bbc.com/news/entertainment-arts-32547474';
+
+$input = '<html>[...]</html>';
+
+$graby = new Graby();
+$graby->setContentAsPrefetched($input);
+$result = $graby->fetchContent($article);
+```
+
 ### Cleanup content
 
 Since the 1.9.0 version, you can also send html content to be cleanup in the same way graby clean content retrieved from an url. The url is still needed to convert links to absolute, etc.

diff --git a/src/Graby.php b/src/Graby.php
@@ -34,6 +34,8 @@ class Graby
     private Punycode $punycode;
     private bool $imgNoReferrer = false;
 
+    private ?string $prefetchedContent = null;
+
     public function __construct(array $config = [], Client $client = null, ConfigBuilder $configBuilder = null)
     {
         $this->config = new GrabyConfig($config);
@@ -103,6 +105,11 @@ public function reloadConfigFiles(): void
         $this->configBuilder->loadConfigFiles();
     }
 
+    public function setContentAsPrefetched(string $content): void
+    {
+        $this->prefetchedContent = $content;
+    }
+
     /**
      * Fetch content from the given url and return a readable content.
      *
@@ -208,6 +215,18 @@ public function cleanupHtml($contentBlock, string $url): string
         return trim($this->cleanupXss((string) $html));
     }
 
+    private function getResponseForPrefetchedContent(string $url): array
+    {
+        return [
+            'body' => $this->prefetchedContent,
+            'effective_url' => $url,
+            'headers' => [
+                'content-type' => 'text/html',
+            ],
+            'status' => 200,
+        ];
+    }
+
     /**
      * Do fetch content from an url.
      *
@@ -218,9 +237,13 @@ private function doFetchContent(string $url): array
         $url = $this->validateUrl($url);
         $siteConfig = $this->configBuilder->buildFromUrl($url);
 
-        $this->logger->info('Fetching url: {url}', ['url' => $url]);
-
-        $response = $this->httpClient->fetch($url, false, $siteConfig->http_header);
+        if (null === $this->prefetchedContent) {
+            $this->logger->info('Fetching url: {url}', ['url' => $url]);
+            $response = $this->httpClient->fetch($url, false, $siteConfig->http_header);
+        } else {
+            $this->logger->info('Content provided as prefetched for url: {url}', ['url' => $url]);
+            $response = $this->getResponseForPrefetchedContent($url);
+        }
 
         $effectiveUrl = $response['effective_url'];
         $effectiveUrl = str_replace(' ', '%20', $effectiveUrl);
@@ -264,7 +287,7 @@ private function doFetchContent(string $url): array
 
         // check site config for single page URL - fetch it if found
         $isSinglePage = false;
-        if ($this->config->getSinglepage() && ($singlePageResponse = $this->getSinglePage($html, $effectiveUrl))) {
+        if ($this->config->getSinglepage() && null === $this->prefetchedContent && ($singlePageResponse = $this->getSinglePage($html, $effectiveUrl))) {
             $isSinglePage = true;
             $effectiveUrl = $singlePageResponse['effective_url'];
 
@@ -305,7 +328,7 @@ private function doFetchContent(string $url): array
 
         // Deal with multi-page articles
         $isMultiPage = (!$isSinglePage && $extractResult && null !== $this->extractor->getNextPageUrl());
-        if ($this->config->getMultipage() && $isMultiPage) {
+        if ($this->config->getMultipage() && null === $this->prefetchedContent && $isMultiPage) {
             $this->logger->info('Attempting to process multi-page article');
             // store first page to avoid parsing it again (previous url content is in `$contentBlock`)
             $multiPageUrls = [$effectiveUrl];

diff --git a/tests/GrabyTest.php b/tests/GrabyTest.php
@@ -1342,6 +1342,23 @@ public function testWithTooLongHtmlJitFail(): void
         $this->assertNotSame('No title found', $res['title']);
     }
 
+    public function testPrefetchedContent(): void
+    {
+        $httpMockClient = new HttpMockClient();
+        $graby = new Graby([
+            'debug' => true,
+        ], $httpMockClient);
+
+        $input = '<html><body><h1>This is my awesome article</h1><article><p>' . str_repeat('This is an awesome text with some links, here there are the awesome', 7) . '</p></article></body></html>';
+
+        $graby->setContentAsPrefetched($input);
+        $res = $graby->fetchContent('https://example.com/prefetched-content');
+
+        $this->assertSame('This is my awesome article', $res['title']);
+        $this->assertSame('https://example.com/prefetched-content', $res['url']);
+        $this->assertStringContainsString('here there are the awesome', $res['html']);
+    }
+
     /**
      * Return an instance of graby with a mocked Guzzle client returning data from a predefined file.
      */