LibHTTP+RequestServer: Move the HTTP cache implementation to LibHTTP

We currently have two ongoing implementations of RFC 9111, HTTP caching.
In order to consolidate these, this patch moves the implementation from
RequestServer to LibHTTP for re-use within LibWeb.

Author: trflynn89

Libraries/LibHTTP/CMakeLists.txt

@@ -1,4 +1,8 @@
 set(SOURCES
+    Cache/CacheEntry.cpp
+    Cache/CacheIndex.cpp
+    Cache/DiskCache.cpp
+    Cache/Utilities.cpp
     Header.cpp
     HeaderList.cpp
     HTTP.cpp
@@ -7,4 +11,4 @@ set(SOURCES
 )
 
 ladybird_lib(LibHTTP http)
-target_link_libraries(LibHTTP PRIVATE LibCompress LibCore LibIPC LibRegex LibTextCodec LibTLS LibURL)
+target_link_libraries(LibHTTP PRIVATE LibCompress LibCore LibCrypto LibDatabase LibFileSystem LibIPC LibRegex LibTextCodec LibTLS LibURL)

Services/RequestServer/Cache/CacheEntry.cpp renamed to Libraries/LibHTTP/Cache/CacheEntry.cpp

@@ -6,15 +6,14 @@
 
 #include <AK/HashFunctions.h>
 #include <AK/ScopeGuard.h>
-#include <LibCore/Notifier.h>
 #include <LibCore/System.h>
 #include <LibFileSystem/FileSystem.h>
-#include <RequestServer/Cache/CacheEntry.h>
-#include <RequestServer/Cache/CacheIndex.h>
-#include <RequestServer/Cache/DiskCache.h>
-#include <RequestServer/Cache/Utilities.h>
+#include <LibHTTP/Cache/CacheEntry.h>
+#include <LibHTTP/Cache/CacheIndex.h>
+#include <LibHTTP/Cache/DiskCache.h>
+#include <LibHTTP/Cache/Utilities.h>
 
-namespace RequestServer {
+namespace HTTP {
 
 ErrorOr<CacheHeader> CacheHeader::read_from_stream(Stream& stream)
 {
@@ -117,7 +116,7 @@ CacheEntryWriter::CacheEntryWriter(DiskCache& disk_cache, CacheIndex& index, u64
 {
 }
 
-ErrorOr<void> CacheEntryWriter::write_status_and_reason(u32 status_code, Optional<String> reason_phrase, HTTP::HeaderList const& response_headers)
+ErrorOr<void> CacheEntryWriter::write_status_and_reason(u32 status_code, Optional<String> reason_phrase, HeaderList const& response_headers)
 {
     if (m_marked_for_deletion) {
         close_and_destroy_cache_entry();
@@ -183,7 +182,7 @@ ErrorOr<void> CacheEntryWriter::write_data(ReadonlyBytes data)
     return {};
 }
 
-ErrorOr<void> CacheEntryWriter::flush(NonnullRefPtr<HTTP::HeaderList> response_headers)
+ErrorOr<void> CacheEntryWriter::flush(NonnullRefPtr<HeaderList> response_headers)
 {
     ScopeGuard guard { [&]() { close_and_destroy_cache_entry(); } };
 
@@ -205,7 +204,7 @@ ErrorOr<void> CacheEntryWriter::flush(NonnullRefPtr<HTTP::HeaderList> response_h
     return {};
 }
 
-ErrorOr<NonnullOwnPtr<CacheEntryReader>> CacheEntryReader::create(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, NonnullRefPtr<HTTP::HeaderList> response_headers, u64 data_size)
+ErrorOr<NonnullOwnPtr<CacheEntryReader>> CacheEntryReader::create(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, NonnullRefPtr<HeaderList> response_headers, u64 data_size)
 {
     auto path = path_for_cache_key(disk_cache.cache_directory(), cache_key);
 
@@ -253,7 +252,7 @@ ErrorOr<NonnullOwnPtr<CacheEntryReader>> CacheEntryReader::create(DiskCache& dis
     return adopt_own(*new CacheEntryReader { disk_cache, index, cache_key, move(url), move(path), move(file), fd, cache_header, move(reason_phrase), move(response_headers), data_offset, data_size });
 }
 
-CacheEntryReader::CacheEntryReader(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, String url, LexicalPath path, NonnullOwnPtr<Core::File> file, int fd, CacheHeader cache_header, Optional<String> reason_phrase, NonnullRefPtr<HTTP::HeaderList> response_headers, u64 data_offset, u64 data_size)
+CacheEntryReader::CacheEntryReader(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, String url, LexicalPath path, NonnullOwnPtr<Core::File> file, int fd, CacheHeader cache_header, Optional<String> reason_phrase, NonnullRefPtr<HeaderList> response_headers, u64 data_offset, u64 data_size)
     : CacheEntry(disk_cache, index, cache_key, move(url), move(path), cache_header)
     , m_file(move(file))
     , m_fd(fd)
@@ -264,7 +263,7 @@ CacheEntryReader::CacheEntryReader(DiskCache& disk_cache, CacheIndex& index, u64
 {
 }
 
-void CacheEntryReader::revalidation_succeeded(HTTP::HeaderList const& response_headers)
+void CacheEntryReader::revalidation_succeeded(HeaderList const& response_headers)
 {
     dbgln("\033[34;1mCache revalidation succeeded for\033[0m {}", m_url);
 

Services/RequestServer/Cache/CacheEntry.h renamed to Libraries/LibHTTP/Cache/CacheEntry.h

@@ -13,11 +13,12 @@
 #include <AK/Time.h>
 #include <AK/Types.h>
 #include <LibCore/File.h>
+#include <LibCore/Notifier.h>
+#include <LibHTTP/Cache/Version.h>
+#include <LibHTTP/Forward.h>
 #include <LibHTTP/HeaderList.h>
-#include <RequestServer/Cache/Version.h>
-#include <RequestServer/Forward.h>
 
-namespace RequestServer {
+namespace HTTP {
 
 struct CacheHeader {
     static ErrorOr<CacheHeader> read_from_stream(Stream&);
@@ -87,9 +88,9 @@ class CacheEntryWriter : public CacheEntry {
     static ErrorOr<NonnullOwnPtr<CacheEntryWriter>> create(DiskCache&, CacheIndex&, u64 cache_key, String url, UnixDateTime request_time, AK::Duration current_time_offset_for_testing);
     virtual ~CacheEntryWriter() override = default;
 
-    ErrorOr<void> write_status_and_reason(u32 status_code, Optional<String> reason_phrase, HTTP::HeaderList const&);
+    ErrorOr<void> write_status_and_reason(u32 status_code, Optional<String> reason_phrase, HeaderList const&);
     ErrorOr<void> write_data(ReadonlyBytes);
-    ErrorOr<void> flush(NonnullRefPtr<HTTP::HeaderList>);
+    ErrorOr<void> flush(NonnullRefPtr<HeaderList>);
 
 private:
     CacheEntryWriter(DiskCache&, CacheIndex&, u64 cache_key, String url, LexicalPath, NonnullOwnPtr<Core::OutputBufferedFile>, CacheHeader, UnixDateTime request_time, AK::Duration current_time_offset_for_testing);
@@ -104,24 +105,24 @@ class CacheEntryWriter : public CacheEntry {
 
 class CacheEntryReader : public CacheEntry {
 public:
-    static ErrorOr<NonnullOwnPtr<CacheEntryReader>> create(DiskCache&, CacheIndex&, u64 cache_key, NonnullRefPtr<HTTP::HeaderList>, u64 data_size);
+    static ErrorOr<NonnullOwnPtr<CacheEntryReader>> create(DiskCache&, CacheIndex&, u64 cache_key, NonnullRefPtr<HeaderList>, u64 data_size);
     virtual ~CacheEntryReader() override = default;
 
     bool must_revalidate() const { return m_must_revalidate; }
     void set_must_revalidate() { m_must_revalidate = true; }
 
-    void revalidation_succeeded(HTTP::HeaderList const&);
+    void revalidation_succeeded(HeaderList const&);
     void revalidation_failed();
 
     void pipe_to(int pipe_fd, Function<void(u64 bytes_piped)> on_complete, Function<void(u64 bytes_piped)> on_error);
 
     u32 status_code() const { return m_cache_header.status_code; }
     Optional<String> const& reason_phrase() const { return m_reason_phrase; }
-    HTTP::HeaderList& response_headers() { return m_response_headers; }
-    HTTP::HeaderList const& response_headers() const { return m_response_headers; }
+    HeaderList& response_headers() { return m_response_headers; }
+    HeaderList const& response_headers() const { return m_response_headers; }
 
 private:
-    CacheEntryReader(DiskCache&, CacheIndex&, u64 cache_key, String url, LexicalPath, NonnullOwnPtr<Core::File>, int fd, CacheHeader, Optional<String> reason_phrase, NonnullRefPtr<HTTP::HeaderList>, u64 data_offset, u64 data_size);
+    CacheEntryReader(DiskCache&, CacheIndex&, u64 cache_key, String url, LexicalPath, NonnullOwnPtr<Core::File>, int fd, CacheHeader, Optional<String> reason_phrase, NonnullRefPtr<HeaderList>, u64 data_offset, u64 data_size);
 
     void pipe_without_blocking();
     void pipe_complete();
@@ -140,7 +141,7 @@ class CacheEntryReader : public CacheEntry {
     u64 m_bytes_piped { 0 };
 
     Optional<String> m_reason_phrase;
-    NonnullRefPtr<HTTP::HeaderList> m_response_headers;
+    NonnullRefPtr<HeaderList> m_response_headers;
 
     bool m_must_revalidate { false };
 

Services/RequestServer/Cache/CacheIndex.cpp renamed to Libraries/LibHTTP/Cache/CacheIndex.cpp

@@ -5,15 +5,15 @@
  */
 
 #include <AK/StringBuilder.h>
-#include <RequestServer/Cache/CacheIndex.h>
-#include <RequestServer/Cache/Utilities.h>
-#include <RequestServer/Cache/Version.h>
+#include <LibHTTP/Cache/CacheIndex.h>
+#include <LibHTTP/Cache/Utilities.h>
+#include <LibHTTP/Cache/Version.h>
 
-namespace RequestServer {
+namespace HTTP {
 
 static constexpr u32 CACHE_METADATA_KEY = 12389u;
 
-static ByteString serialize_headers(HTTP::HeaderList const& headers)
+static ByteString serialize_headers(HeaderList const& headers)
 {
     StringBuilder builder;
 
@@ -27,9 +27,9 @@ static ByteString serialize_headers(HTTP::HeaderList const& headers)
     return builder.to_byte_string();
 }
 
-static NonnullRefPtr<HTTP::HeaderList> deserialize_headers(StringView serialized_headers)
+static NonnullRefPtr<HeaderList> deserialize_headers(StringView serialized_headers)
 {
-    auto headers = HTTP::HeaderList::create();
+    auto headers = HeaderList::create();
 
     serialized_headers.for_each_split_view('\n', SplitBehavior::Nothing, [&](StringView serialized_header) {
         auto index = serialized_header.find(':');
@@ -110,7 +110,7 @@ CacheIndex::CacheIndex(Database::Database& database, Statements statements)
 {
 }
 
-void CacheIndex::create_entry(u64 cache_key, String url, NonnullRefPtr<HTTP::HeaderList> response_headers, u64 data_size, UnixDateTime request_time, UnixDateTime response_time)
+void CacheIndex::create_entry(u64 cache_key, String url, NonnullRefPtr<HeaderList> response_headers, u64 data_size, UnixDateTime request_time, UnixDateTime response_time)
 {
     auto now = UnixDateTime::now();
 
@@ -133,36 +133,36 @@ void CacheIndex::create_entry(u64 cache_key, String url, NonnullRefPtr<HTTP::Hea
         .last_access_time = now,
     };
 
-    m_database.execute_statement(m_statements.insert_entry, {}, entry.cache_key, entry.url, serialize_headers(entry.response_headers), entry.data_size, entry.request_time, entry.response_time, entry.last_access_time);
+    m_database->execute_statement(m_statements.insert_entry, {}, entry.cache_key, entry.url, serialize_headers(entry.response_headers), entry.data_size, entry.request_time, entry.response_time, entry.last_access_time);
     m_entries.set(cache_key, move(entry));
 }
 
 void CacheIndex::remove_entry(u64 cache_key)
 {
-    m_database.execute_statement(m_statements.remove_entry, {}, cache_key);
+    m_database->execute_statement(m_statements.remove_entry, {}, cache_key);
     m_entries.remove(cache_key);
 }
 
 void CacheIndex::remove_entries_accessed_since(UnixDateTime since, Function<void(u64 cache_key)> on_entry_removed)
 {
-    m_database.execute_statement(
+    m_database->execute_statement(
         m_statements.remove_entries_accessed_since,
         [&](auto statement_id) {
-            auto cache_key = m_database.result_column<u64>(statement_id, 0);
+            auto cache_key = m_database->result_column<u64>(statement_id, 0);
             m_entries.remove(cache_key);
 
             on_entry_removed(cache_key);
         },
         since);
 }
 
-void CacheIndex::update_response_headers(u64 cache_key, NonnullRefPtr<HTTP::HeaderList> response_headers)
+void CacheIndex::update_response_headers(u64 cache_key, NonnullRefPtr<HeaderList> response_headers)
 {
     auto entry = m_entries.get(cache_key);
     if (!entry.has_value())
         return;
 
-    m_database.execute_statement(m_statements.update_response_headers, {}, serialize_headers(response_headers), cache_key);
+    m_database->execute_statement(m_statements.update_response_headers, {}, serialize_headers(response_headers), cache_key);
     entry->response_headers = move(response_headers);
 }
 
@@ -174,7 +174,7 @@ void CacheIndex::update_last_access_time(u64 cache_key)
 
     auto now = UnixDateTime::now();
 
-    m_database.execute_statement(m_statements.update_last_access_time, {}, now, cache_key);
+    m_database->execute_statement(m_statements.update_last_access_time, {}, now, cache_key);
     entry->last_access_time = now;
 }
 
@@ -183,17 +183,17 @@ Optional<CacheIndex::Entry&> CacheIndex::find_entry(u64 cache_key)
     if (auto entry = m_entries.get(cache_key); entry.has_value())
         return entry;
 
-    m_database.execute_statement(
+    m_database->execute_statement(
         m_statements.select_entry, [&](auto statement_id) {
             int column = 0;
 
-            auto cache_key = m_database.result_column<u64>(statement_id, column++);
-            auto url = m_database.result_column<String>(statement_id, column++);
-            auto response_headers = m_database.result_column<ByteString>(statement_id, column++);
-            auto data_size = m_database.result_column<u64>(statement_id, column++);
-            auto request_time = m_database.result_column<UnixDateTime>(statement_id, column++);
-            auto response_time = m_database.result_column<UnixDateTime>(statement_id, column++);
-            auto last_access_time = m_database.result_column<UnixDateTime>(statement_id, column++);
+            auto cache_key = m_database->result_column<u64>(statement_id, column++);
+            auto url = m_database->result_column<String>(statement_id, column++);
+            auto response_headers = m_database->result_column<ByteString>(statement_id, column++);
+            auto data_size = m_database->result_column<u64>(statement_id, column++);
+            auto request_time = m_database->result_column<UnixDateTime>(statement_id, column++);
+            auto response_time = m_database->result_column<UnixDateTime>(statement_id, column++);
+            auto last_access_time = m_database->result_column<UnixDateTime>(statement_id, column++);
 
             Entry entry { cache_key, move(url), deserialize_headers(response_headers), data_size, request_time, response_time, last_access_time };
             m_entries.set(cache_key, move(entry));
@@ -203,18 +203,18 @@ Optional<CacheIndex::Entry&> CacheIndex::find_entry(u64 cache_key)
     return m_entries.get(cache_key);
 }
 
-Requests::CacheSizes CacheIndex::estimate_cache_size_accessed_since(UnixDateTime since) const
+Requests::CacheSizes CacheIndex::estimate_cache_size_accessed_since(UnixDateTime since)
 {
     Requests::CacheSizes sizes;
 
-    m_database.execute_statement(
+    m_database->execute_statement(
         m_statements.estimate_cache_size_accessed_since,
-        [&](auto statement_id) { sizes.since_requested_time = m_database.result_column<u64>(statement_id, 0); },
+        [&](auto statement_id) { sizes.since_requested_time = m_database->result_column<u64>(statement_id, 0); },
         since);
 
-    m_database.execute_statement(
+    m_database->execute_statement(
         m_statements.estimate_cache_size_accessed_since,
-        [&](auto statement_id) { sizes.total = m_database.result_column<u64>(statement_id, 0); },
+        [&](auto statement_id) { sizes.total = m_database->result_column<u64>(statement_id, 0); },
         UnixDateTime::earliest());
 
     return sizes;

Services/RequestServer/Cache/CacheIndex.h renamed to Libraries/LibHTTP/Cache/CacheIndex.h

@@ -8,13 +8,14 @@
 
 #include <AK/Error.h>
 #include <AK/HashMap.h>
+#include <AK/NonnullRawPtr.h>
 #include <AK/Time.h>
 #include <AK/Types.h>
 #include <LibDatabase/Database.h>
 #include <LibHTTP/HeaderList.h>
 #include <LibRequests/CacheSizes.h>
 
-namespace RequestServer {
+namespace HTTP {
 
 // The cache index is a SQL database containing metadata about each cache entry. An entry in the index is created once
 // the entire cache entry has been successfully written to disk.
@@ -23,7 +24,7 @@ class CacheIndex {
         u64 cache_key { 0 };
 
         String url;
-        NonnullRefPtr<HTTP::HeaderList> response_headers;
+        NonnullRefPtr<HeaderList> response_headers;
         u64 data_size { 0 };
 
         UnixDateTime request_time;
@@ -34,16 +35,16 @@ class CacheIndex {
 public:
     static ErrorOr<CacheIndex> create(Database::Database&);
 
-    void create_entry(u64 cache_key, String url, NonnullRefPtr<HTTP::HeaderList>, u64 data_size, UnixDateTime request_time, UnixDateTime response_time);
+    void create_entry(u64 cache_key, String url, NonnullRefPtr<HeaderList>, u64 data_size, UnixDateTime request_time, UnixDateTime response_time);
     void remove_entry(u64 cache_key);
     void remove_entries_accessed_since(UnixDateTime, Function<void(u64 cache_key)> on_entry_removed);
 
     Optional<Entry&> find_entry(u64 cache_key);
 
-    void update_response_headers(u64 cache_key, NonnullRefPtr<HTTP::HeaderList>);
+    void update_response_headers(u64 cache_key, NonnullRefPtr<HeaderList>);
     void update_last_access_time(u64 cache_key);
 
-    Requests::CacheSizes estimate_cache_size_accessed_since(UnixDateTime since) const;
+    Requests::CacheSizes estimate_cache_size_accessed_since(UnixDateTime since);
 
 private:
     struct Statements {
@@ -58,7 +59,7 @@ class CacheIndex {
 
     CacheIndex(Database::Database&, Statements);
 
-    Database::Database& m_database;
+    NonnullRawPtr<Database::Database> m_database;
     Statements m_statements;
 
     HashMap<u32, Entry> m_entries;

Libraries/LibHTTP/Cache/CacheRequest.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2025, Tim Flynn <trflynn89@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Badge.h>
+#include <AK/Optional.h>
+#include <AK/Weakable.h>
+#include <LibHTTP/Forward.h>
+
+namespace HTTP {
+
+class CacheRequest : public Weakable<CacheRequest> {
+public:
+    virtual ~CacheRequest() = default;
+
+    virtual void notify_request_unblocked(Badge<DiskCache>) = 0;
+
+protected:
+    enum class CacheStatus : u8 {
+        Unknown,
+        NotCached,
+        WrittenToCache,
+        ReadFromCache,
+    };
+
+    Optional<CacheEntryReader&> m_cache_entry_reader;
+    Optional<CacheEntryWriter&> m_cache_entry_writer;
+    CacheStatus m_cache_status { CacheStatus::Unknown };
+};
+
+}