LibWeb: Start adding infrastructure for an HTTP Cache

Userland/Libraries/LibWeb/Fetch/Fetching/Fetching.cpp

@@ -1252,6 +1252,126 @@ WebIDL::ExceptionOr<JS::GCPtr<PendingResponse>> http_redirect_fetch(JS::Realm& r
     return main_fetch(realm, fetch_params, recursive);
 }
 
+// https://fetch.spec.whatwg.org/#network-partition-key
+struct NetworkPartitionKey {
+    HTML::Origin top_level_origin;
+    // FIXME: See https://github.com/whatwg/fetch/issues/1035
+    //     This is the document origin in other browsers
+    void* second_key = nullptr;
+
+    bool operator==(NetworkPartitionKey const&) const = default;
+};
+
+}
+
+// FIXME: Take this with us to the eventual header these structs end up in to avoid closing and re-opening the namespace.
+template<>
+class AK::Traits<Web::Fetch::Fetching::NetworkPartitionKey> : public DefaultTraits<Web::Fetch::Fetching::NetworkPartitionKey> {
+public:
+    static unsigned hash(Web::Fetch::Fetching::NetworkPartitionKey const& partition_key)
+    {
+        return ::AK::Traits<Web::HTML::Origin>::hash(partition_key.top_level_origin);
+    }
+};
+
+namespace Web::Fetch::Fetching {
+
+struct CachedResponse {
+    Vector<Infrastructure::Header> headers;
+    ByteBuffer body;
+    ByteBuffer method;
+    URL::URL url;
+    UnixDateTime current_age;
+};
+
+class CachePartition {
+public:
+    // FIXME: Copy the headers... less
+    Optional<CachedResponse> select_response(URL::URL const& url, ReadonlyBytes method, Vector<Infrastructure::Header> const& headers) const
+    {
+        auto it = m_cache.find(url);
+        if (it == m_cache.end())
+            return {};
+
+        auto const& cached_response = it->value;
+
+        // FIXME: Validate headers and method
+        (void)method;
+        (void)headers;
+        return cached_response;
+    }
+
+private:
+    HashMap<URL::URL, CachedResponse> m_cache;
+};
+
+class HTTPCache {
+public:
+    CachePartition& get(NetworkPartitionKey const& key)
+    {
+        return *m_cache.ensure(key, [] {
+            return make<CachePartition>();
+        });
+    }
+
+    static HTTPCache& the()
+    {
+        static HTTPCache s_cache;
+        return s_cache;
+    }
+
+private:
+    HashMap<NetworkPartitionKey, NonnullOwnPtr<CachePartition>> m_cache;
+};
+
+// https://fetch.spec.whatwg.org/#determine-the-network-partition-key
+static NetworkPartitionKey determine_the_network_partition_key(HTML::Environment const& environment)
+{
+    // 1. Let topLevelOrigin be environment’s top-level origin.
+    auto top_level_origin = environment.top_level_origin;
+
+    // FIXME: 2. If topLevelOrigin is null, then set topLevelOrigin to environment’s top-level creation URL’s origin
+    // This field is supposed to be nullable
+
+    // 3. Assert: topLevelOrigin is an origin.
+
+    // FIXME: 4. Let topLevelSite be the result of obtaining a site, given topLevelOrigin.
+
+    // 5. Let secondKey be null or an implementation-defined value.
+    void* second_key = nullptr;
+
+    // 6. Return (topLevelSite, secondKey).
+    return { top_level_origin, second_key };
+}
+
+// https://fetch.spec.whatwg.org/#request-determine-the-network-partition-key
+static Optional<NetworkPartitionKey> determine_the_network_partition_key(Infrastructure::Request const& request)
+{
+    // 1. If request’s reserved client is non-null, then return the result of determining the network partition key given request’s reserved client.
+    if (auto reserved_client = request.reserved_client())
+        return determine_the_network_partition_key(*reserved_client);
+
+    // 2. If request’s client is non-null, then return the result of determining the network partition key given request’s client.
+    if (auto client = request.client())
+        return determine_the_network_partition_key(*client);
+
+    return {};
+}
+
+// https://fetch.spec.whatwg.org/#determine-the-http-cache-partition
+static Optional<CachePartition> determine_the_http_cache_partition(Infrastructure::Request const& request)
+{
+    // 1. Let key be the result of determining the network partition key given request.
+    auto key = determine_the_network_partition_key(request);
+
+    // 2. If key is null, then return null.
+    if (!key.has_value())
+        return OptionalNone {};
+
+    // 3. Return the unique HTTP cache associated with key. [HTTP-CACHING]
+    return HTTPCache::the().get(key.value());
+}
+
 // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch
 WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fetch(JS::Realm& realm, Infrastructure::FetchParams const& fetch_params, IsAuthenticationFetch is_authentication_fetch, IsNewConnectionFetch is_new_connection_fetch)
 {
@@ -1277,7 +1397,7 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fet
 
     // 6. Let httpCache be null.
     // (Typeless until we actually implement it, needed for checks below)
-    void* http_cache = nullptr;
+    Optional<CachePartition> http_cache;
 
     // 7. Let the revalidatingFlag be unset.
     auto revalidating_flag = RefCountedFlag::create(false);
@@ -1394,7 +1514,10 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fet
         // 13. Append the Fetch metadata headers for httpRequest.
         append_fetch_metadata_headers_for_request(*http_request);
 
-        // 14. If httpRequest’s header list does not contain `User-Agent`, then user agents should append
+        // 14. FIXME If httpRequest’s initiator is "prefetch", then set a structured field value
+        //     given (`Sec-Purpose`, the token prefetch) in httpRequest’s header list.
+
+        // 15. If httpRequest’s header list does not contain `User-Agent`, then user agents should append
         //     (`User-Agent`, default `User-Agent` value) to httpRequest’s header list.
         if (!http_request->header_list()->contains("User-Agent"sv.bytes())) {
             auto header = Infrastructure::Header {
@@ -1404,7 +1527,7 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fet
             http_request->header_list()->append(move(header));
         }
 
-        // 15. If httpRequest’s cache mode is "default" and httpRequest’s header list contains `If-Modified-Since`,
+        // 16. If httpRequest’s cache mode is "default" and httpRequest’s header list contains `If-Modified-Since`,
         //     `If-None-Match`, `If-Unmodified-Since`, `If-Match`, or `If-Range`, then set httpRequest’s cache mode to
         //     "no-store".
         if (http_request->cache_mode() == Infrastructure::Request::CacheMode::Default
@@ -1416,7 +1539,7 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fet
             http_request->set_cache_mode(Infrastructure::Request::CacheMode::NoStore);
         }
 
-        // 16. If httpRequest’s cache mode is "no-cache", httpRequest’s prevent no-cache cache-control header
+        // 17. If httpRequest’s cache mode is "no-cache", httpRequest’s prevent no-cache cache-control header
         //     modification flag is unset, and httpRequest’s header list does not contain `Cache-Control`, then append
         //     (`Cache-Control`, `max-age=0`) to httpRequest’s header list.
         if (http_request->cache_mode() == Infrastructure::Request::CacheMode::NoCache
@@ -1426,7 +1549,7 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fet
             http_request->header_list()->append(move(header));
         }
 
-        // 17. If httpRequest’s cache mode is "no-store" or "reload", then:
+        // 18. If httpRequest’s cache mode is "no-store" or "reload", then:
         if (http_request->cache_mode() == Infrastructure::Request::CacheMode::NoStore
             || http_request->cache_mode() == Infrastructure::Request::CacheMode::Reload) {
             // 1. If httpRequest’s header list does not contain `Pragma`, then append (`Pragma`, `no-cache`) to
@@ -1444,7 +1567,7 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fet
             }
         }
 
-        // 18. If httpRequest’s header list contains `Range`, then append (`Accept-Encoding`, `identity`) to
+        // 19. If httpRequest’s header list contains `Range`, then append (`Accept-Encoding`, `identity`) to
         //     httpRequest’s header list.
         // NOTE: This avoids a failure when handling content codings with a part of an encoded response.
         //       Additionally, many servers mistakenly ignore `Range` headers if a non-identity encoding is accepted.
@@ -1453,7 +1576,7 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fet
             http_request->header_list()->append(move(header));
         }
 
-        // 19. Modify httpRequest’s header list per HTTP. Do not append a given header if httpRequest’s header list
+        // 20. Modify httpRequest’s header list per HTTP. Do not append a given header if httpRequest’s header list
         //     contains that header’s name.
         // NOTE: It would be great if we could make this more normative somehow. At this point headers such as
         //       `Accept-Encoding`, `Connection`, `DNT`, and `Host`, are to be appended if necessary.
@@ -1462,7 +1585,7 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fet
         //       the latter by default), and `Accept-Charset` is a waste of bytes. See HTTP header layer division for
         //       more details.
 
-        // 20. If includeCredentials is true, then:
+        // 21. If includeCredentials is true, then:
         if (include_credentials == IncludeCredentials::Yes) {
             // 1. If the user agent is not configured to block cookies for httpRequest (see section 7 of [COOKIES]),
             //    then:
@@ -1513,28 +1636,88 @@ WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fet
             }
         }
 
-        // FIXME: 21. If there’s a proxy-authentication entry, use it as appropriate.
+        // FIXME: 22. If there’s a proxy-authentication entry, use it as appropriate.
         // NOTE: This intentionally does not depend on httpRequest’s credentials mode.
 
-        // FIXME: 22. Set httpCache to the result of determining the HTTP cache partition, given httpRequest.
+        // 23. Set httpCache to the result of determining the HTTP cache partition, given httpRequest.
+        http_cache = determine_the_http_cache_partition(*http_request);
 
-        // 23. If httpCache is null, then set httpRequest’s cache mode to "no-store".
-        if (!http_cache)
+        // 24. If httpCache is null, then set httpRequest’s cache mode to "no-store".
+        if (!http_cache.has_value())
             http_request->set_cache_mode(Infrastructure::Request::CacheMode::NoStore);
 
-        // 24. If httpRequest’s cache mode is neither "no-store" nor "reload", then:
+        // 25. If httpRequest’s cache mode is neither "no-store" nor "reload", then:
         if (http_request->cache_mode() != Infrastructure::Request::CacheMode::NoStore
             && http_request->cache_mode() != Infrastructure::Request::CacheMode::Reload) {
             // 1. Set storedResponse to the result of selecting a response from the httpCache, possibly needing
             //    validation, as per the "Constructing Responses from Caches" chapter of HTTP Caching [HTTP-CACHING],
             //    if any.
             // NOTE: As mandated by HTTP, this still takes the `Vary` header into account.
-            stored_response = nullptr;
-
+            auto raw_response = http_cache->select_response(http_request->url(), http_request->method(), *http_request->header_list());
             // 2. If storedResponse is non-null, then:
-            if (stored_response) {
-                // FIXME: Caching is not implemented yet.
-                VERIFY_NOT_REACHED();
+            if (raw_response.has_value()) {
+
+                // FIXME: Set more properties from the cached response
+                auto [body, _] = TRY(extract_body(realm, ReadonlyBytes(raw_response->body)));
+                stored_response = Infrastructure::Response::create(vm);
+                stored_response->set_body(body);
+
+                // 1. If cache mode is "default", storedResponse is a stale-while-revalidate response,
+                //    and httpRequest’s client is non-null, then:
+                if (http_request->cache_mode() == Infrastructure::Request::CacheMode::Default
+                    && stored_response->is_stale_while_revalidate()
+                    && http_request->client() != nullptr) {
+
+                    // 1. Set response to storedResponse.
+                    response = stored_response;
+
+                    // 2. Set response’s cache state to "local".
+                    response->set_cache_state(Infrastructure::Response::CacheState::Local);
+
+                    // 3. Let revalidateRequest be a clone of request.
+                    auto revalidate_request = request->clone(realm);
+
+                    // 4. Set revalidateRequest’s cache mode set to "no-cache".
+                    revalidate_request->set_cache_mode(Infrastructure::Request::CacheMode::NoCache);
+
+                    // 5. Set revalidateRequest’s prevent no-cache cache-control header modification flag.
+                    revalidate_request->set_prevent_no_cache_cache_control_header_modification(true);
+
+                    // 6. Set revalidateRequest’s service-workers mode set to "none".
+                    revalidate_request->set_service_workers_mode(Infrastructure::Request::ServiceWorkersMode::None);
+
+                    // 7. In parallel, run main fetch given a new fetch params whose request is revalidateRequest.
+                    Platform::EventLoopPlugin::the().deferred_invoke([&vm, &realm, revalidate_request, fetch_params = JS::NonnullGCPtr(fetch_params)] {
+                        (void)main_fetch(realm, Infrastructure::FetchParams::create(vm, revalidate_request, fetch_params->timing_info()));
+                    });
+                }
+                // 2. Otherwise:
+                else {
+                    // 1. If storedResponse is a stale response, then set the revalidatingFlag.
+                    if (stored_response->is_stale())
+                        revalidating_flag->set_value(true);
+
+                    // 2. If the revalidatingFlag is set and httpRequest’s cache mode is neither "force-cache" nor "only-if-cached", then:
+                    if (revalidating_flag->value()
+                        && http_request->cache_mode() != Infrastructure::Request::CacheMode::ForceCache
+                        && http_request->cache_mode() != Infrastructure::Request::CacheMode::OnlyIfCached) {
+
+                        // 1. If storedResponse’s header list contains `ETag`, then append (`If-None-Match`, `ETag`'s value) to httpRequest’s header list.
+                        if (auto etag = stored_response->header_list()->get("ETag"sv.bytes()); etag.has_value()) {
+                            stored_response->header_list()->append(Infrastructure::Header::from_string_pair("If-None-Match"sv, *etag));
+                        }
+
+                        // 2. If storedResponse’s header list contains `Last-Modified`, then append (`If-Modified-Since`, `Last-Modified`'s value) to httpRequest’s header list.
+                        if (auto last_modified = stored_response->header_list()->get("Last-Modified"sv.bytes()); last_modified.has_value()) {
+                            stored_response->header_list()->append(Infrastructure::Header::from_string_pair("If-Modified-Since"sv, *last_modified));
+                        }
+                    }
+                    // 3. Otherwise, set response to storedResponse and set response’s cache state to "local".
+                    else {
+                        response = stored_response;
+                        response->set_cache_state(Infrastructure::Response::CacheState::Local);
+                    }
+                }
             }
         }
     }

Userland/Libraries/LibWeb/Fetch/Infrastructure/HTTP/Headers.h

@@ -33,7 +33,7 @@ struct Header {
 // A header list is a list of zero or more headers. It is initially the empty list.
 class HeaderList final
     : public JS::Cell
-    , Vector<Header> {
+    , public Vector<Header> {
     JS_CELL(HeaderList, JS::Cell);
     JS_DECLARE_ALLOCATOR(HeaderList);