Optimized for increased performance + better render compression

src/Client/Cache/MySQL/Manager.php

@@ -290,17 +290,17 @@ public function cron($timeLimit, $workerID)
 SELECT base
 FROM robotstxt__cache1
 WHERE worker = :workerID
+ORDER BY nextUpdate DESC
 LIMIT 10;
 SQL
             );
             $query->bindParam(':workerID', $worker, PDO::PARAM_INT);
             $query->execute();
             if ($query->rowCount() > 0) {
                 while ($row = $query->fetch(PDO::FETCH_ASSOC)) {
-                    if (!$this->push(new UriClient($row['base'], $this->curlOptions, $this->byteLimit))) {
-                        throw new ClientException('Unable to update `' . $row['base'] . '`');
+                    if ($this->push(new UriClient($row['base'], $this->curlOptions, $this->byteLimit))) {
+                        $log[(string)microtime(true)] = $row['base'];
                     }
-                    $log[(string)microtime(true)] = $row['base'];
                 }
             }
         }
@@ -338,7 +338,7 @@ public function clean($delay)
         $delay = self::CACHE_TIME + $delay;
         $query = $this->pdo->prepare(<<<SQL
 DELETE FROM robotstxt__cache1
-WHERE (worker = 0 OR worker IS NULL) AND nextUpdate < (UNIX_TIMESTAMP() - :delay);
+WHERE nextUpdate < (UNIX_TIMESTAMP() - :delay);
 SQL
         );
         $query->bindParam(':delay', $delay, PDO::PARAM_INT);

src/Client/Directives/DelayCore.php

@@ -77,10 +77,10 @@ public function getUserAgent()
      */
     public function handle(PDO $pdo)
     {
-        if ($this->handler === null) {
-            $handler = new DatabaseHandler($pdo);
-            $this->handler = $handler->delayClient($this->base, $this->userAgent, $this->getValue());
+        if (isset($this->handler)) {
+            return $this->handler;
         }
-        return $this->handler;
+        $handler = new DatabaseHandler($pdo);
+        return $this->handler = $handler->delayClient($this->base, $this->userAgent, $this->getValue());
     }
 }

src/Client/Directives/UserAgentTools.php

@@ -133,7 +133,7 @@ private function checkPath($directive, $uri)
                      self::DIRECTIVE_NO_INDEX => $this->handler->noIndex(),
                      self::DIRECTIVE_DISALLOW => $this->handler->disallow(),
                      self::DIRECTIVE_ALLOW => $this->handler->allow(),
-                 ] as $currentDirective => $handler) {
+                 ] as $currentDirective => &$handler) {
             if ($handler->client()->isListed($uri)) {
                 if ($currentDirective === self::DIRECTIVE_NO_INDEX) {
                     return $directive === self::DIRECTIVE_DISALLOW;

src/Parser/Directives/AllowParser.php

@@ -54,6 +54,18 @@ class AllowParser implements ParserInterface, RobotsTxtInterface
      */
     private $host;
 
+    /**
+     * Optimized for performance
+     * @var bool
+     */
+    private $optimized = false;
+
+    /**
+     * Client cache
+     * @var AllowClient
+     */
+    private $client;
+
     /**
      * AllowParser constructor
      *
@@ -94,39 +106,37 @@ public function add($line)
      */
     private function addPath($path)
     {
-        foreach ([
-                     $path,
-                     '/',
-                     '*',
-                 ] as $testPath) {
-            if (in_array($testPath, $this->path)) {
-                return false;
-            }
-        }
-        if ($this->isPath($path)) {
+        $path = rtrim($path, '*');
+        if (in_array(mb_substr($path, 0, 1), [
+            '/',
+            '*',
+        ])) {
             $this->path[] = $path;
-            $this->removeOverlapping();
+            $this->optimized = false;
         }
         return in_array($path, $this->path);
     }
 
     /**
-     * Check if path is valid
+     * Render
      *
-     * @param string $path
+     * @param RenderHandler $handler
      * @return bool
      */
-    private function isPath($path)
+    public function render(RenderHandler $handler)
     {
-        if (mb_strpos($path, '/') !== 0) {
-            foreach ([
-                         '*',
-                         '?',
-                     ] as $char) {
-                $path = str_replace($char, '/', $path);
-            }
+        if (!$this->optimized) {
+            $this->removeOverlapping();
+        }
+        sort($this->path);
+        $inline = new RenderHandler($handler->getLevel());
+        $this->host->render($inline);
+        $this->cleanParam->render($inline);
+        $handler->addInline($this->directive, $inline);
+        foreach ($this->path as $path) {
+            $handler->add($this->directive, $path);
         }
-        return mb_strpos($path, '/') === 0;
+        return true;
     }
 
     /**
@@ -136,35 +146,18 @@ private function isPath($path)
      */
     private function removeOverlapping()
     {
-        foreach ($this->path as $key1 => $path1) {
-            foreach ($this->path as $key2 => $path2) {
+        foreach ($this->path as $key1 => &$path1) {
+            foreach ($this->path as $key2 => &$path2) {
                 if ($key1 !== $key2 &&
-                    mb_strpos($path1, $path2) === 0
+                    (mb_strpos($path1, $path2) === 0 ||
+                        mb_strpos(str_replace('*', '/', $path1), $path2) === 0
+                    )
                 ) {
                     unset($this->path[$key1]);
-                    return $this->removeOverlapping();
                 }
             }
         }
-        return true;
-    }
-
-    /**
-     * Render
-     *
-     * @param RenderHandler $handler
-     * @return bool
-     */
-    public function render(RenderHandler $handler)
-    {
-        sort($this->path);
-        $inline = new RenderHandler($handler->getLevel());
-        $this->host->render($inline);
-        $this->cleanParam->render($inline);
-        $handler->addInline($this->directive, $inline);
-        foreach ($this->path as $path) {
-            $handler->add($this->directive, $path);
-        }
+        $this->optimized = true;
         return true;
     }
 
@@ -175,6 +168,11 @@ public function render(RenderHandler $handler)
      */
     public function client()
     {
-        return new AllowClient($this->path, $this->host->client(), $this->cleanParam->client());
+        if (isset($this->client)) {
+            return $this->client;
+        } elseif (!$this->optimized) {
+            $this->removeOverlapping();
+        }
+        return $this->client = new AllowClient($this->path, $this->host->client(), $this->cleanParam->client());
     }
 }

src/Parser/Directives/CleanParamParser.php

@@ -17,6 +17,12 @@
  */
 class CleanParamParser extends CleanParamParserCore
 {
+    /**
+     * Client cache
+     * @var CleanParamClient
+     */
+    private $client;
+
     /**
      * CleanParamParser constructor.
      */
@@ -32,6 +38,9 @@ public function __construct()
      */
     public function client()
     {
-        return new CleanParamClient($this->cleanParam);
+        if (isset($this->client)) {
+            return $this->client;
+        }
+        return $this->client = new CleanParamClient($this->cleanParam);
     }
 }

src/Parser/Directives/CleanParamParserCore.php

@@ -42,12 +42,12 @@ public function add($line)
     {
         // split into parameter and path
         $array = array_map('trim', mb_split('\s+', $line, 2));
-        // strip any invalid characters from path prefix
-        $path = '/';
         if (isset($array[1])) {
+            // strip any invalid characters from path prefix
             $uriParser = new UriParser(preg_replace('/[^A-Za-z0-9\.-\/\*\_]/', '', $array[1]));
-            $path = $uriParser->encode();
+            $path = rtrim($uriParser->encode(), '*');
         }
+        $path = empty($path) ? '/' : $path;
         $param = array_map('trim', explode('&', $array[0]));
         foreach ($param as $key) {
             $this->cleanParam[$key][] = $path;

src/Parser/Directives/DelayParser.php

@@ -37,6 +37,12 @@ class DelayParser implements ParserInterface, RobotsTxtInterface
      */
     private $delay;
 
+    /**
+     * Client cache
+     * @var DelayClient
+     */
+    private $client;
+
     /**
      * DelayParser constructor.
      *
@@ -80,7 +86,10 @@ public function add($line)
      */
     public function client($userAgent = self::USER_AGENT, $fallbackValue = 0)
     {
-        return new DelayClient($this->base, $userAgent, $this->delay, $fallbackValue);
+        if (isset($this->client)) {
+            return $this->client;
+        }
+        return $this->client = new DelayClient($this->base, $userAgent, $this->delay, $fallbackValue);
     }
 
     /**

src/Parser/Directives/HostParser.php

@@ -18,6 +18,12 @@
  */
 class HostParser extends HostParserCore
 {
+    /**
+     * Client cache
+     * @var HostClient
+     */
+    private $client;
+
     /**
      * HostParser constructor.
      *
@@ -36,7 +42,10 @@ public function __construct($base, $effective)
      */
     public function client()
     {
-        return new HostClient($this->base, $this->effective, isset($this->host[0]) ? [$this->host[0]] : []);
+        if (isset($this->client)) {
+            return $this->client;
+        }
+        return $this->client = new HostClient($this->base, $this->effective, isset($this->host[0]) ? [$this->host[0]] : []);
     }
 
     /**

src/Parser/Directives/InlineCleanParamParser.php

@@ -18,6 +18,12 @@
  */
 class InlineCleanParamParser extends CleanParamParserCore
 {
+    /**
+     * Client cache
+     * @var InlineCleanParamClient
+     */
+    private $client;
+
     /**
      * InlineCleanParamParser constructor.
      */
@@ -33,6 +39,9 @@ public function __construct()
      */
     public function client()
     {
-        return new InlineCleanParamClient($this->cleanParam);
+        if (isset($this->client)) {
+            return $this->client;
+        }
+        return $this->client = new InlineCleanParamClient($this->cleanParam);
     }
 }

src/Parser/Directives/InlineHostParser.php

@@ -18,6 +18,12 @@
  */
 class InlineHostParser extends HostParserCore
 {
+    /**
+     * Client cache
+     * @var InlineHostClient
+     */
+    private $client;
+
     /**
      * InlineHostParser constructor.
      *
@@ -36,7 +42,10 @@ public function __construct($base, $effective)
      */
     public function client()
     {
-        return new InlineHostClient($this->base, $this->effective, $this->host);
+        if (isset($this->client)) {
+            return $this->client;
+        }
+        return $this->client = new InlineHostClient($this->base, $this->effective, $this->host);
     }
 
     /**

src/Parser/Directives/RequestRateParser.php

@@ -33,6 +33,12 @@ class RequestRateParser implements ParserInterface, RobotsTxtInterface
      */
     private $requestRates = [];
 
+    /**
+     * Client cache
+     * @var RequestRateClient
+     */
+    private $client;
+
     /**
      * RequestRate constructor.
      *
@@ -107,8 +113,11 @@ private function draftParseRate($string)
      */
     public function client($userAgent = self::USER_AGENT, $fallbackValue = 0)
     {
+        if (isset($this->client)) {
+            return $this->client;
+        }
         $this->sort();
-        return new RequestRateClient($this->base, $userAgent, $this->requestRates, $fallbackValue);
+        return $this->client = new RequestRateClient($this->base, $userAgent, $this->requestRates, $fallbackValue);
     }
 
     /**

src/Parser/Directives/RobotVersionParser.php

@@ -25,6 +25,12 @@ class RobotVersionParser implements ParserInterface, RobotsTxtInterface
      */
     private $version;
 
+    /**
+     * Client cache
+     * @var RobotVersionClient
+     */
+    private $client;
+
     /**
      * RobotVersionParser constructor.
      */
@@ -54,7 +60,10 @@ public function add($line)
      */
     public function client()
     {
-        return new RobotVersionClient($this->version);
+        if (isset($this->client)) {
+            return $this->client;
+        }
+        return $this->client = new RobotVersionClient($this->version);
     }
 
     /**