LibWebView+RequestServer: Add a simple test mode for the HTTP disk cache

This mode allows us to test the HTTP disk cache with two mechanisms:

1. If RequestServer is launched with --http-disk-cache-mode=testing, it
   will cache requests with a X-Ladybird-Enable-Disk-Cache header.

2. In test mode, RS will include a X-Ladybird-Disk-Cache-Status response
   header indicating how the response was handled by the cache. There is
   no standard way for a web request to know what happened with respect
   to the disk cache, so this fills that hole for testing.

This mode is not exposed to users.

Author: trflynn89

Libraries/LibWebView/Application.cpp

@@ -262,7 +262,7 @@ ErrorOr<void> Application::initialize(Main::Arguments const& arguments)
 
     m_request_server_options = {
         .certificates = move(certificates),
-        .enable_http_disk_cache = enable_http_disk_cache ? EnableHTTPDiskCache::Yes : EnableHTTPDiskCache::No,
+        .http_disk_cache_mode = enable_http_disk_cache ? HTTPDiskCacheMode::Enabled : HTTPDiskCacheMode::Disabled,
     };
 
     m_web_content_options = {

Libraries/LibWebView/HelperProcess.cpp

@@ -213,8 +213,19 @@ ErrorOr<NonnullRefPtr<Requests::RequestClient>> launch_request_server_process()
     for (auto const& certificate : request_server_options.certificates)
         arguments.append(ByteString::formatted("--certificate={}", certificate));
 
-    if (request_server_options.enable_http_disk_cache == EnableHTTPDiskCache::Yes)
-        arguments.append("--enable-http-disk-cache"sv);
+    arguments.append("--http-disk-cache-mode"sv);
+
+    switch (request_server_options.http_disk_cache_mode) {
+    case HTTPDiskCacheMode::Disabled:
+        arguments.append("disabled"sv);
+        break;
+    case HTTPDiskCacheMode::Enabled:
+        arguments.append("enabled"sv);
+        break;
+    case HTTPDiskCacheMode::Testing:
+        arguments.append("testing"sv);
+        break;
+    }
 
     if (auto server = mach_server_name(); server.has_value()) {
         arguments.append("--mach-server-name"sv);

Libraries/LibWebView/Options.h

@@ -93,14 +93,15 @@ struct BrowserOptions {
     EnableContentFilter enable_content_filter { EnableContentFilter::Yes };
 };
 
-enum class EnableHTTPDiskCache {
-    No,
-    Yes,
+enum class HTTPDiskCacheMode {
+    Disabled,
+    Enabled,
+    Testing,
 };
 
 struct RequestServerOptions {
     Vector<ByteString> certificates;
-    EnableHTTPDiskCache enable_http_disk_cache { EnableHTTPDiskCache::No };
+    HTTPDiskCacheMode http_disk_cache_mode { HTTPDiskCacheMode::Disabled };
 };
 
 enum class IsLayoutTestMode {

Services/RequestServer/Cache/DiskCache.cpp

@@ -15,28 +15,38 @@ namespace RequestServer {
 
 static constexpr auto INDEX_DATABASE = "INDEX"sv;
 
-ErrorOr<DiskCache> DiskCache::create()
+ErrorOr<DiskCache> DiskCache::create(Mode mode)
 {
-    auto cache_directory = LexicalPath::join(Core::StandardPaths::cache_directory(), "Ladybird"sv, "Cache"sv);
+    auto cache_name = mode == Mode::Normal ? "Cache"sv : "TestCache"sv;
+    auto cache_directory = LexicalPath::join(Core::StandardPaths::cache_directory(), "Ladybird"sv, cache_name);
 
     auto database = TRY(Database::Database::create(cache_directory.string(), INDEX_DATABASE));
     auto index = TRY(CacheIndex::create(database));
 
-    return DiskCache { move(database), move(cache_directory), move(index) };
+    return DiskCache { mode, move(database), move(cache_directory), move(index) };
 }
 
-DiskCache::DiskCache(NonnullRefPtr<Database::Database> database, LexicalPath cache_directory, CacheIndex index)
-    : m_database(move(database))
+DiskCache::DiskCache(Mode mode, NonnullRefPtr<Database::Database> database, LexicalPath cache_directory, CacheIndex index)
+    : m_mode(mode)
+    , m_database(move(database))
     , m_cache_directory(move(cache_directory))
     , m_index(move(index))
 {
+    // Start with a clean slate in test mode.
+    if (m_mode == Mode::Testing)
+        remove_entries_accessed_since(UnixDateTime::earliest());
 }
 
 Variant<Optional<CacheEntryWriter&>, DiskCache::CacheHasOpenEntry> DiskCache::create_entry(Request& request)
 {
     if (!is_cacheable(request.method()))
         return Optional<CacheEntryWriter&> {};
 
+    if (m_mode == Mode::Testing) {
+        if (!request.request_headers().contains(TEST_CACHE_ENABLED_HEADER))
+            return Optional<CacheEntryWriter&> {};
+    }
+
     auto serialized_url = serialize_url_for_cache_storage(request.url());
     auto cache_key = create_cache_key(serialized_url, request.method());
 

Services/RequestServer/Cache/DiskCache.h

@@ -21,7 +21,16 @@ namespace RequestServer {
 
 class DiskCache {
 public:
-    static ErrorOr<DiskCache> create();
+    enum class Mode {
+        Normal,
+
+        // In test mode, we only enable caching of responses on a per-request basis, signified by a request header. The
+        // response headers will include some status on how the request was handled.
+        Testing,
+    };
+    static ErrorOr<DiskCache> create(Mode);
+
+    Mode mode() const { return m_mode; }
 
     struct CacheHasOpenEntry { };
     Variant<Optional<CacheEntryWriter&>, CacheHasOpenEntry> create_entry(Request&);
@@ -35,14 +44,16 @@ class DiskCache {
     void cache_entry_closed(Badge<CacheEntry>, CacheEntry const&);
 
 private:
-    DiskCache(NonnullRefPtr<Database::Database>, LexicalPath cache_directory, CacheIndex);
+    DiskCache(Mode, NonnullRefPtr<Database::Database>, LexicalPath cache_directory, CacheIndex);
 
     enum class CheckReaderEntries {
         No,
         Yes,
     };
     bool check_if_cache_has_open_entry(Request&, u64 cache_key, CheckReaderEntries);
 
+    Mode m_mode;
+
     NonnullRefPtr<Database::Database> m_database;
 
     HashMap<u64, Vector<NonnullOwnPtr<CacheEntry>, 1>> m_open_cache_entries;

Services/RequestServer/Cache/Utilities.cpp

@@ -192,7 +192,7 @@ bool is_header_exempted_from_storage(StringView name)
         "Proxy-Connection"sv,
         "TE"sv,
         "Transfer-Encoding"sv,
-        "Upgrade"sv
+        "Upgrade"sv,
 
         // * Likewise, some fields' semantics require them to be removed before forwarding the message, and this MAY be
         //   implemented by doing so before storage; see Section 7.6.1 of [HTTP] for some examples.
@@ -204,7 +204,10 @@ bool is_header_exempted_from_storage(StringView name)
         //   unless the cache incorporates the identity of the proxy into the cache key. Effectively, this is limited to
         //   Proxy-Authenticate (Section 11.7.1 of [HTTP]), Proxy-Authentication-Info (Section 11.7.3 of [HTTP]), and
         //   Proxy-Authorization (Section 11.7.2 of [HTTP]).
-    );
+
+        // AD-HOC: Exclude headers used only for testing.
+        TEST_CACHE_ENABLED_HEADER,
+        TEST_CACHE_STATUS_HEADER);
 }
 
 // https://httpwg.org/specs/rfc9111.html#heuristic.freshness

Services/RequestServer/Cache/Utilities.h

@@ -15,6 +15,9 @@
 
 namespace RequestServer {
 
+constexpr inline auto TEST_CACHE_ENABLED_HEADER = "X-Ladybird-Enable-Disk-Cache"sv;
+constexpr inline auto TEST_CACHE_STATUS_HEADER = "X-Ladybird-Disk-Cache-Status"sv;
+
 String serialize_url_for_cache_storage(URL::URL const&);
 u64 create_cache_key(StringView url, StringView method);
 LexicalPath path_for_cache_key(LexicalPath const& cache_directory, u64 cache_key);

Services/RequestServer/Request.cpp

@@ -211,6 +211,9 @@ void Request::handle_initial_state()
         m_disk_cache->create_entry(*this).visit(
             [&](Optional<CacheEntryWriter&> cache_entry_writer) {
                 m_cache_entry_writer = cache_entry_writer;
+
+                if (!m_cache_entry_writer.has_value())
+                    m_cache_status = CacheStatus::NotCached;
             },
             [&](DiskCache::CacheHasOpenEntry) {
                 // If an existing entry is open for reading or writing, we must wait for it to complete. An entry being
@@ -228,15 +231,13 @@ void Request::handle_initial_state()
 
 void Request::handle_read_cache_state()
 {
-    m_status_code = m_cache_entry_reader->status_code();
     m_reason_phrase = m_cache_entry_reader->reason_phrase();
     m_response_headers = m_cache_entry_reader->response_headers();
+    m_cache_status = CacheStatus::ReadFromCache;
 
     if (inform_client_request_started().is_error())
         return;
-
-    m_client.async_headers_became_available(m_request_id, m_response_headers, m_status_code, m_reason_phrase);
-    m_sent_response_headers_to_client = true;
+    transfer_headers_to_client_if_needed();
 
     m_cache_entry_reader->pipe_to(
         m_client_request_pipe->writer_fd(),
@@ -556,13 +557,37 @@ void Request::transfer_headers_to_client_if_needed()
     if (exchange(m_sent_response_headers_to_client, true))
         return;
 
-    m_status_code = acquire_status_code();
-    m_client.async_headers_became_available(m_request_id, m_response_headers, m_status_code, m_reason_phrase);
+    if (m_cache_entry_reader.has_value())
+        m_status_code = m_cache_entry_reader->status_code();
+    else
+        m_status_code = acquire_status_code();
 
     if (m_cache_entry_writer.has_value()) {
-        if (m_cache_entry_writer->write_status_and_reason(m_status_code, m_reason_phrase, m_response_headers).is_error())
+        if (m_cache_entry_writer->write_status_and_reason(m_status_code, m_reason_phrase, m_response_headers).is_error()) {
+            m_cache_status = CacheStatus::NotCached;
             m_cache_entry_writer.clear();
+        } else {
+            m_cache_status = CacheStatus::WrittenToCache;
+        }
+    }
+
+    if (m_disk_cache.has_value() && m_disk_cache->mode() == DiskCache::Mode::Testing) {
+        switch (m_cache_status) {
+        case CacheStatus::Unknown:
+            break;
+        case CacheStatus::NotCached:
+            m_response_headers.set(TEST_CACHE_STATUS_HEADER, "not-cached"sv);
+            break;
+        case CacheStatus::WrittenToCache:
+            m_response_headers.set(TEST_CACHE_STATUS_HEADER, "written-to-cache"sv);
+            break;
+        case CacheStatus::ReadFromCache:
+            m_response_headers.set(TEST_CACHE_STATUS_HEADER, "read-from-cache"sv);
+            break;
+        }
     }
+
+    m_client.async_headers_became_available(m_request_id, m_response_headers, m_status_code, m_reason_phrase);
 }
 
 ErrorOr<void> Request::write_queued_bytes_without_blocking()

Services/RequestServer/Request.h

@@ -53,6 +53,7 @@ class Request : public Weakable<Request> {
 
     URL::URL const& url() const { return m_url; }
     ByteString const& method() const { return m_method; }
+    HTTP::HeaderMap const& request_headers() const { return m_request_headers; }
     UnixDateTime request_start_time() const { return m_request_start_time; }
 
     void notify_request_unblocked(Badge<DiskCache>);
@@ -75,6 +76,13 @@ class Request : public Weakable<Request> {
         Error,        // Any error occured during the request's lifetime.
     };
 
+    enum class CacheStatus : u8 {
+        Unknown,
+        NotCached,
+        WrittenToCache,
+        ReadFromCache,
+    };
+
     Request(
         i32 request_id,
         Optional<DiskCache&> disk_cache,
@@ -159,6 +167,7 @@ class Request : public Weakable<Request> {
 
     Optional<CacheEntryReader&> m_cache_entry_reader;
     Optional<CacheEntryWriter&> m_cache_entry_writer;
+    CacheStatus m_cache_status { CacheStatus::Unknown };
 
     Optional<Requests::NetworkError> m_network_error;
 };

Services/RequestServer/main.cpp

@@ -34,13 +34,13 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
 
     Vector<ByteString> certificates;
     StringView mach_server_name;
-    bool enable_http_disk_cache = false;
+    StringView http_disk_cache_mode;
     bool wait_for_debugger = false;
 
     Core::ArgsParser args_parser;
     args_parser.add_option(certificates, "Path to a certificate file", "certificate", 'C', "certificate");
     args_parser.add_option(mach_server_name, "Mach server name", "mach-server-name", 0, "mach_server_name");
-    args_parser.add_option(enable_http_disk_cache, "Enable HTTP disk cache", "enable-http-disk-cache");
+    args_parser.add_option(http_disk_cache_mode, "HTTP disk cache mode", "http-disk-cache-mode", 0, "mode");
     args_parser.add_option(wait_for_debugger, "Wait for debugger", "wait-for-debugger");
     args_parser.parse(arguments);
 
@@ -58,8 +58,12 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
         Core::Platform::register_with_mach_server(mach_server_name);
 #endif
 
-    if (enable_http_disk_cache) {
-        if (auto cache = RequestServer::DiskCache::create(); cache.is_error())
+    if (http_disk_cache_mode.is_one_of("enabled"sv, "testing"sv)) {
+        auto mode = http_disk_cache_mode == "enabled"sv
+            ? RequestServer::DiskCache::Mode::Normal
+            : RequestServer::DiskCache::Mode::Testing;
+
+        if (auto cache = RequestServer::DiskCache::create(mode); cache.is_error())
             warnln("Unable to create disk cache: {}", cache.error());
         else
             RequestServer::g_disk_cache = cache.release_value();