chore: Move calculation of "retry after" into Response

Author: marienfressinaud

lib/SpiderBits/src/Response.php

@@ -230,4 +230,112 @@ public static function extractCharsetFromContentType(string $content_type): ?str
         $charset = trim($charset, '"');
         return $charset;
     }
+
+    /**
+     * Returns the date after which the response is considered as stale.
+     *
+     * The expiration duration is calculated based on HTTP headers of the
+     * response.
+     *
+     * @see https://httpwg.org/specs/rfc9111.html
+     */
+    public function getRetryAfter(
+        int $default_duration = 1 * 60 * 60,
+        int $min_duration = 1 * 60 * 15,
+        int $max_duration = 1 * 60 * 60 * 24 * 7,
+    ): \DateTimeImmutable {
+        $age = $this->header('Age', '0');
+        $expires = $this->header('Expires', '');
+        $retry_after = $this->header('Retry-After', '0');
+
+        $cache_control_directives = $this->getCacheControlDirectives();
+
+        $duration = $default_duration;
+
+        if (isset($cache_control_directives['max-age'])) {
+            $max_age = (int) $cache_control_directives['max-age'];
+            $age = (int) $age;
+            $duration = $max_age - $age;
+        } elseif ($expires) {
+            $expired_at = self::parseHttpDate($expires);
+
+            if ($expired_at === null) {
+                $expired_at = \Minz\Time::now();
+            }
+
+            $expires_timestamp = $expired_at->getTimestamp();
+            $now_timestamp = \Minz\Time::now()->getTimestamp();
+
+            $duration = $expires_timestamp - $now_timestamp;
+        } elseif ($this->status === 429) {
+            $retry_at = self::parseHttpDate($retry_after);
+
+            if ($retry_at === null) {
+                $duration = (int) $retry_after;
+            } else {
+                $retry_at_timestamp = $retry_at->getTimestamp();
+                $now_timestamp = \Minz\Time::now()->getTimestamp();
+
+                $duration = $retry_at_timestamp - $now_timestamp;
+            }
+        }
+
+        $duration = max($min_duration, $duration);
+        $duration = min($max_duration, $duration);
+        return \Minz\Time::fromNow($duration, 'seconds');
+    }
+
+    /**
+     * Parses the "Cache-Control" HTTP header and returns an array with the
+     * different cache directives.
+     *
+     * @return array<string, string|true>
+     */
+    public function getCacheControlDirectives(): array
+    {
+        $directives = [];
+
+        $cache_control = $this->header('Cache-Control', '');
+        $cache_control_parts = explode(',', $cache_control);
+
+        foreach ($cache_control_parts as $part) {
+            $part = trim($part);
+
+            if (str_contains($part, '=')) {
+                list($directive, $value) = explode('=', $part, 2);
+            } else {
+                $directive = $part;
+                $value = true;
+            }
+
+            $directive = strtolower($directive);
+
+            $directives[$directive] = $value;
+        }
+
+        return $directives;
+    }
+
+    /**
+     * Parses an HTTP date header.
+     */
+    public static function parseHttpDate(string $expires): ?\DateTimeImmutable
+    {
+        $formats = [
+            \DateTimeInterface::RFC7231,
+            \DateTimeInterface::RFC850,
+            // Ignore the ANSI C's asctime() format as obsolete and more
+            // difficult to parse.
+        ];
+
+        foreach ($formats as $format) {
+            $expired_at = \DateTimeImmutable::createFromFormat($format, $expires);
+
+            if ($expired_at) {
+                return $expired_at;
+            }
+        }
+
+        return null;
+    }
 }

src/http/Cache.php

@@ -57,70 +57,27 @@ public function getResponse(string $url): ?Response
     /**
      * Saves an HTTP response in the cache.
      *
-     * The expiration duration is calculated based on HTTP headers of the
-     * response.
-     *
      * @see https://httpwg.org/specs/rfc9111.html
      */
     public function saveResponse(string $url, Response $response): void
     {
         $code = $response->status;
 
-        if ($code < 200 || $code === 206 || $code === 304 || $code >= 500) {
+        if ($code < 200 || $code === 206 || $code === 304) {
             // Don't cache the response if the response is not finale, or if
-            // status is 206 or 304 as we don't handle these special codes, or
-            // if the server is in error.
+            // status is 206 or 304 as we don't handle these special codes.
             return;
         }
 
-        $cache_control = $response->header('Cache-Control', '');
-        $age = $response->header('Age', '0');
-        $expires = $response->header('Expires', '');
-        $retry_after = $response->header('Retry-After', '0');
-
-        $directives = $this->parseCacheControl($cache_control);
+        $cache_control_directives = $response->getCacheControlDirectives();
 
-        if (isset($directives['no-store']) || isset($directives['no-cache'])) {
+        if (isset($cache_control_directives['no-store']) || isset($cache_control_directives['no-cache'])) {
             // These directives tell us to not cache the response.
             // We may consider to use the min_duration though, but let's try
             // for a bit with this configuration.
             return;
         }
 
-        $duration = $this->default_duration;
-
-        if (isset($directives['max-age'])) {
-            $max_age = (int) $directives['max-age'];
-            $age = (int) $age;
-            $duration = $max_age - $age;
-        } elseif ($expires) {
-            $expired_at = $this->parseHttpDate($expires);
-
-            if ($expired_at === null) {
-                $expired_at = \Minz\Time::now();
-            }
-
-            $expires_timestamp = $expired_at->getTimestamp();
-            $now_timestamp = \Minz\Time::now()->getTimestamp();
-
-            $duration = $expires_timestamp - $now_timestamp;
-        } elseif ($response->status === 429) {
-            $retry_at = $this->parseHttpDate($retry_after);
-
-            if ($retry_at === null) {
-                $duration = (int) $retry_after;
-            } else {
-                $retry_at_timestamp = $retry_at->getTimestamp();
-                $now_timestamp = \Minz\Time::now()->getTimestamp();
-
-                $duration = $retry_at_timestamp - $now_timestamp;
-            }
-        }
-
-        $duration = max($this->min_duration, $duration);
-        $duration = min($this->max_duration, $duration);
-        $expiration = \Minz\Time::fromNow($duration, 'seconds');
-
         $response_text = (string) $response;
         $response_compressed = @gzencode($response_text);
 
@@ -130,64 +87,11 @@ public function saveResponse(string $url, Response $response): void
 
         $cache_item = $this->getItem($url);
         $cache_item->set($response_compressed);
-        $cache_item->expiresAt($expiration);
+        $cache_item->expiresAt($response->getRetryAfter());
         $result = $this->save($cache_item);
 
         if ($result === false) {
             throw new CacheError("Response to {$url} cannot be saved in cache.");
         }
     }
-
-    /**
-     * Parses a "Cache-Control" HTTP header and returns an array with the
-     * different cache directives.
-     *
-     * @return array<string, string|true>
-     */
-    private function parseCacheControl(string $cache_control): array
-    {
-        $directives = [];
-
-        $cache_control_parts = explode(',', $cache_control);
-
-        foreach ($cache_control_parts as $part) {
-            $part = trim($part);
-
-            if (str_contains($part, '=')) {
-                list($directive, $value) = explode('=', $part, 2);
-            } else {
-                $directive = $part;
-                $value = true;
-            }
-
-            $directive = strtolower($directive);
-
-            $directives[$directive] = $value;
-        }
-
-        return $directives;
-    }
-
-    /**
-     * Parses an HTTP date header.
-     */
-    private function parseHttpDate(string $expires): ?\DateTimeImmutable
-    {
-        $formats = [
-            \DateTimeInterface::RFC7231,
-            \DateTimeInterface::RFC850,
-            // Ignore the ANSI C's asctime() format as obsolete and more
-            // difficult to parse.
-        ];
-
-        foreach ($formats as $format) {
-            $expired_at = \DateTimeImmutable::createFromFormat($format, $expires);
-
-            if ($expired_at) {
-                return $expired_at;
-            }
-        }
-
-        return null;
-    }
 }

tests/http/CacheTest.php

@@ -327,11 +327,6 @@ public static function unsupportedStatusCodesProvider(): array
             [103],
             [206],
             [304],
-            [500],
-            [501],
-            [502],
-            [503],
-            [504],
         ];
     }
 }