Implement a broken link checker

composer.json

@@ -111,7 +111,7 @@
         "symfony/var-dumper": "4.4.*",
         "symfony/web-profiler-bundle": "4.4.*",
         "symfony/yaml": "4.4.*",
-        "terminal42/escargot": "^0.1",
+        "terminal42/escargot": "^0.3",
         "terminal42/service-annotation-bundle": "^1.0",
         "toflar/psr6-symfony-http-cache-store": "^2.1",
         "true/punycode": "^2.1",

core-bundle/composer.json

@@ -96,7 +96,7 @@
         "symfony/twig-bundle": "4.4.*",
         "symfony/var-dumper": "4.4.*",
         "symfony/yaml": "4.4.*",
-        "terminal42/escargot": "^0.1",
+        "terminal42/escargot": "^0.3",
         "terminal42/service-annotation-bundle": "^1.0",
         "true/punycode": "^2.1",
         "twig/twig": "^2.7",

core-bundle/src/Resources/config/services.yml

@@ -513,6 +513,11 @@ services:
             - '@contao.framework'
         public: true
 
+    contao.search.escargot_subscriber.broken_link_checker:
+        class: Contao\CoreBundle\Search\Escargot\Subscriber\BrokenLinkCheckerSubscriber
+        tags:
+            - { name: 'contao.escargot_subscriber' }
+
     contao.search.escargot_subscriber.search_index:
         class: Contao\CoreBundle\Search\Escargot\Subscriber\SearchIndexSubscriber
         arguments:

core-bundle/src/Search/Escargot/Factory.php

@@ -185,9 +185,21 @@ public function createFromJobId(string $jobId, QueueInterface $queue, array $sel
         return $escargot;
     }
 
+    /**
+     * Creates an HttpClientInterface instance that behaves like a regular
+     * browser by default.
+     */
     private function createDefaultHttpClient(): HttpClientInterface
     {
-        return HttpClient::create($this->getDefaultHttpClientOptions());
+        return HttpClient::create(
+            array_merge_recursive(
+                [
+                    'headers' => ['accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'],
+                    'max_duration' => 20, // Ignore requests that take longer than 20 seconds
+                ],
+                $this->getDefaultHttpClientOptions()
+            )
+        );
     }
 
     private function registerDefaultSubscribers(Escargot $escargot): void

core-bundle/src/Search/Escargot/Subscriber/BrokenLinkCheckerSubscriber.php

@@ -0,0 +1,137 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of Contao.
+ *
+ * (c) Leo Feyer
+ *
+ * @license LGPL-3.0-or-later
+ */
+
+namespace Contao\CoreBundle\Search\Escargot\Subscriber;
+
+use Psr\Log\LogLevel;
+use Symfony\Contracts\HttpClient\ChunkInterface;
+use Symfony\Contracts\HttpClient\Exception\ExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\HttpExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
+use Symfony\Contracts\HttpClient\ResponseInterface;
+use Terminal42\Escargot\CrawlUri;
+use Terminal42\Escargot\EscargotAwareInterface;
+use Terminal42\Escargot\EscargotAwareTrait;
+use Terminal42\Escargot\Subscriber\ExceptionSubscriberInterface;
+use Terminal42\Escargot\Subscriber\SubscriberInterface;
+
+class BrokenLinkCheckerSubscriber implements EscargotSubscriberInterface, EscargotAwareInterface, ExceptionSubscriberInterface
+{
+    use EscargotAwareTrait;
+
+    /**
+     * @var array
+     */
+    private $stats = ['ok' => 0, 'error' => 0];
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getName(): string
+    {
+        return 'broken-link-checker';
+    }
+
+    public function shouldRequest(CrawlUri $crawlUri): string
+    {
+        // Only check URIs that are part of our base collection or were found on one
+        $fromBaseUriCollection = $this->escargot->getBaseUris()->containsHost($crawlUri->getUri()->getHost());
+
+        $foundOnBaseUriCollection = null !== $crawlUri->getFoundOn()
+            && ($originalCrawlUri = $this->escargot->getCrawlUri($crawlUri->getFoundOn()))
+            && $this->escargot->getBaseUris()->containsHost($originalCrawlUri->getUri()->getHost());
+
+        if (!$fromBaseUriCollection && !$foundOnBaseUriCollection) {
+            $this->escargot->log(
+                LogLevel::DEBUG,
+                $crawlUri->createLogMessage('Did not check because it is not part of the base URI collection or was not found on one of that is.'),
+                ['source' => static::class]
+            );
+
+            return SubscriberInterface::DECISION_NEGATIVE;
+        }
+
+        // Let's go otherwise!
+        return SubscriberInterface::DECISION_POSITIVE;
+    }
+
+    public function needsContent(CrawlUri $crawlUri, ResponseInterface $response, ChunkInterface $chunk): string
+    {
+        if ($response->getStatusCode() < 200 || $response->getStatusCode() >= 400) {
+            $this->logError($crawlUri, 'HTTP Status Code: '.$response->getStatusCode());
+        } else {
+            ++$this->stats['ok'];
+        }
+
+        return SubscriberInterface::DECISION_NEGATIVE;
+    }
+
+    public function onLastChunk(CrawlUri $crawlUri, ResponseInterface $response, ChunkInterface $chunk): void
+    {
+        // noop
+    }
+
+    public function getResult(SubscriberResult $previousResult = null): SubscriberResult
+    {
+        $stats = $this->stats;
+
+        if (null !== $previousResult) {
+            $stats['ok'] += $previousResult->getInfo('stats')['ok'];
+            $stats['error'] += $previousResult->getInfo('stats')['error'];
+        }
+
+        $result = new SubscriberResult(
+            0 === $stats['error'],
+            sprintf('Checked %d link(s) successfully. %d were broken!', $stats['ok'], $stats['error'])
+        );
+
+        $result->addInfo('stats', $stats);
+
+        return $result;
+    }
+
+    public function onException(CrawlUri $crawlUri, ExceptionInterface $exception, ResponseInterface $response, ChunkInterface $chunk = null): void
+    {
+        if ($exception instanceof TransportExceptionInterface) {
+            $this->logError($crawlUri, 'Could not request properly: '.$exception->getMessage());
+
+            return;
+        }
+
+        try {
+            $isLastChunk = null !== $chunk && $chunk->isLast();
+            $statusCode = $response->getStatusCode();
+        } catch (TransportExceptionInterface $exception) {
+            $this->logError($crawlUri, 'Could not request properly: '.$exception->getMessage());
+
+            return;
+        }
+
+        // Only log on last chunk for HttpExceptions, otherwise we have a log entry on every chunk that arrives.
+        if ($exception instanceof HttpExceptionInterface && null !== $chunk && !$isLastChunk) {
+            return;
+        }
+
+        $this->logError($crawlUri, 'HTTP Status Code: '.$statusCode);
+    }
+
+    private function logError(CrawlUri $crawlUri, string $message): void
+    {
+        ++$this->stats['error'];
+
+        $this->escargot->log(
+            LogLevel::ERROR,
+            $crawlUri->createLogMessage(sprintf('Broken link! %s.', $message)),
+            ['source' => static::class]
+        );
+    }
+}

core-bundle/tests/DependencyInjection/ContaoCoreExtensionTest.php

@@ -99,6 +99,7 @@
 use Contao\CoreBundle\Routing\ScopeMatcher;
 use Contao\CoreBundle\Routing\UrlGenerator;
 use Contao\CoreBundle\Search\Escargot\Factory;
+use Contao\CoreBundle\Search\Escargot\Subscriber\BrokenLinkCheckerSubscriber;
 use Contao\CoreBundle\Search\Escargot\Subscriber\SearchIndexSubscriber;
 use Contao\CoreBundle\Search\Indexer\IndexerInterface;
 use Contao\CoreBundle\Security\Authentication\AuthenticationEntryPoint;
@@ -2316,7 +2317,27 @@ public function testRegistersTheSearchEscargotFactory(): void
         );
     }
 
-    public function testRegistersTheSearchEscargotSubscriber(): void
+    public function testRegistersTheSearchEscargotBrokenLinkCheckerSubscriber(): void
+    {
+        $this->assertTrue($this->container->has('contao.search.escargot_subscriber.broken_link_checker'));
+
+        $definition = $this->container->getDefinition('contao.search.escargot_subscriber.broken_link_checker');
+
+        $this->assertSame(BrokenLinkCheckerSubscriber::class, $definition->getClass());
+        $this->assertTrue($definition->isPrivate());
+        $this->assertSame([], $definition->getArguments());
+
+        $this->assertSame(
+            [
+                'contao.escargot_subscriber' => [
+                    [],
+                ],
+            ],
+            $definition->getTags()
+        );
+    }
+
+    public function testRegistersTheSearchEscargotSearchIndexSubscriber(): void
     {
         $this->assertTrue($this->container->has('contao.search.escargot_subscriber.search_index'));