RequestServer: Add a hook to advance a request's clock time for testing

For example, we will want to be able to test that a cached object was
expired after N seconds. Rather than waiting that time during testing,
this adds a testing-only request header to internally advance the clock
for a single HTTP request.

Author: trflynn89

Services/RequestServer/Cache/CacheEntry.cpp

@@ -76,7 +76,7 @@ void CacheEntry::close_and_destroy_cache_entry()
     m_disk_cache.cache_entry_closed({}, *this);
 }
 
-ErrorOr<NonnullOwnPtr<CacheEntryWriter>> CacheEntryWriter::create(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, String url, UnixDateTime request_time)
+ErrorOr<NonnullOwnPtr<CacheEntryWriter>> CacheEntryWriter::create(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, String url, UnixDateTime request_time, AK::Duration current_time_offset_for_testing)
 {
     auto path = path_for_cache_key(disk_cache.cache_directory(), cache_key);
 
@@ -87,14 +87,15 @@ ErrorOr<NonnullOwnPtr<CacheEntryWriter>> CacheEntryWriter::create(DiskCache& dis
     cache_header.url_size = url.byte_count();
     cache_header.url_hash = url.hash();
 
-    return adopt_own(*new CacheEntryWriter { disk_cache, index, cache_key, move(url), move(path), move(file), cache_header, request_time });
+    return adopt_own(*new CacheEntryWriter { disk_cache, index, cache_key, move(url), move(path), move(file), cache_header, request_time, current_time_offset_for_testing });
 }
 
-CacheEntryWriter::CacheEntryWriter(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, String url, LexicalPath path, NonnullOwnPtr<Core::OutputBufferedFile> file, CacheHeader cache_header, UnixDateTime request_time)
+CacheEntryWriter::CacheEntryWriter(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, String url, LexicalPath path, NonnullOwnPtr<Core::OutputBufferedFile> file, CacheHeader cache_header, UnixDateTime request_time, AK::Duration current_time_offset_for_testing)
     : CacheEntry(disk_cache, index, cache_key, move(url), move(path), cache_header)
     , m_file(move(file))
     , m_request_time(request_time)
-    , m_response_time(UnixDateTime::now())
+    , m_response_time(UnixDateTime::now() + current_time_offset_for_testing)
+    , m_current_time_offset_for_testing(current_time_offset_for_testing)
 {
 }
 
@@ -116,8 +117,8 @@ ErrorOr<void> CacheEntryWriter::write_status_and_reason(u32 status_code, Optiona
         if (!is_cacheable(status_code, response_headers))
             return Error::from_string_literal("Response is not cacheable");
 
-        auto freshness_lifetime = calculate_freshness_lifetime(status_code, response_headers);
-        auto current_age = calculate_age(response_headers, m_request_time, m_response_time);
+        auto freshness_lifetime = calculate_freshness_lifetime(status_code, response_headers, m_current_time_offset_for_testing);
+        auto current_age = calculate_age(response_headers, m_request_time, m_response_time, m_current_time_offset_for_testing);
 
         // We can cache already-expired responses if there are other cache directives that allow us to revalidate the
         // response on subsequent requests. For example, `Cache-Control: max-age=0, must-revalidate`.

Services/RequestServer/Cache/CacheEntry.h

@@ -79,20 +79,22 @@ class CacheEntry {
 
 class CacheEntryWriter : public CacheEntry {
 public:
-    static ErrorOr<NonnullOwnPtr<CacheEntryWriter>> create(DiskCache&, CacheIndex&, u64 cache_key, String url, UnixDateTime request_time);
+    static ErrorOr<NonnullOwnPtr<CacheEntryWriter>> create(DiskCache&, CacheIndex&, u64 cache_key, String url, UnixDateTime request_time, AK::Duration current_time_offset_for_testing);
     virtual ~CacheEntryWriter() override = default;
 
     ErrorOr<void> write_status_and_reason(u32 status_code, Optional<String> reason_phrase, HTTP::HeaderMap const&);
     ErrorOr<void> write_data(ReadonlyBytes);
     ErrorOr<void> flush(HTTP::HeaderMap);
 
 private:
-    CacheEntryWriter(DiskCache&, CacheIndex&, u64 cache_key, String url, LexicalPath, NonnullOwnPtr<Core::OutputBufferedFile>, CacheHeader, UnixDateTime request_time);
+    CacheEntryWriter(DiskCache&, CacheIndex&, u64 cache_key, String url, LexicalPath, NonnullOwnPtr<Core::OutputBufferedFile>, CacheHeader, UnixDateTime request_time, AK::Duration current_time_offset_for_testing);
 
     NonnullOwnPtr<Core::OutputBufferedFile> m_file;
 
     UnixDateTime m_request_time;
     UnixDateTime m_response_time;
+
+    AK::Duration m_current_time_offset_for_testing;
 };
 
 class CacheEntryReader : public CacheEntry {

Services/RequestServer/Cache/DiskCache.cpp

@@ -53,7 +53,7 @@ Variant<Optional<CacheEntryWriter&>, DiskCache::CacheHasOpenEntry> DiskCache::cr
     if (check_if_cache_has_open_entry(request, cache_key, CheckReaderEntries::Yes))
         return CacheHasOpenEntry {};
 
-    auto cache_entry = CacheEntryWriter::create(*this, m_index, cache_key, move(serialized_url), request.request_start_time());
+    auto cache_entry = CacheEntryWriter::create(*this, m_index, cache_key, move(serialized_url), request.request_start_time(), request.current_time_offset_for_testing());
     if (cache_entry.is_error()) {
         dbgln("\033[31;1mUnable to create cache entry for\033[0m {}: {}", request.url(), cache_entry.error());
         return Optional<CacheEntryWriter&> {};
@@ -93,8 +93,8 @@ Variant<Optional<CacheEntryReader&>, DiskCache::CacheHasOpenEntry> DiskCache::op
     }
 
     auto const& response_headers = cache_entry.value()->response_headers();
-    auto freshness_lifetime = calculate_freshness_lifetime(cache_entry.value()->status_code(), response_headers);
-    auto current_age = calculate_age(response_headers, index_entry->request_time, index_entry->response_time);
+    auto freshness_lifetime = calculate_freshness_lifetime(cache_entry.value()->status_code(), response_headers, request.current_time_offset_for_testing());
+    auto current_age = calculate_age(response_headers, index_entry->request_time, index_entry->response_time, request.current_time_offset_for_testing());
 
     switch (cache_lifetime_status(response_headers, freshness_lifetime, current_age)) {
     case CacheLifetimeStatus::Fresh:

Services/RequestServer/Cache/Utilities.cpp

@@ -6,6 +6,7 @@
 
 #include <LibCrypto/Hash/SHA1.h>
 #include <LibURL/URL.h>
+#include <RequestServer/Cache/DiskCache.h>
 #include <RequestServer/Cache/Utilities.h>
 
 namespace RequestServer {
@@ -207,11 +208,12 @@ bool is_header_exempted_from_storage(StringView name)
 
         // AD-HOC: Exclude headers used only for testing.
         TEST_CACHE_ENABLED_HEADER,
-        TEST_CACHE_STATUS_HEADER);
+        TEST_CACHE_STATUS_HEADER,
+        TEST_CACHE_REQUEST_TIME_OFFSET);
 }
 
 // https://httpwg.org/specs/rfc9111.html#heuristic.freshness
-static AK::Duration calculate_heuristic_freshness_lifetime(HTTP::HeaderMap const& headers)
+static AK::Duration calculate_heuristic_freshness_lifetime(HTTP::HeaderMap const& headers, AK::Duration current_time_offset_for_testing)
 {
     // Since origin servers do not always provide explicit expiration times, a cache MAY assign a heuristic expiration
     // time when an explicit time is not specified, employing algorithms that use other field values (such as the
@@ -230,7 +232,7 @@ static AK::Duration calculate_heuristic_freshness_lifetime(HTTP::HeaderMap const
     if (!last_modified.has_value())
         return {};
 
-    auto now = UnixDateTime::now();
+    auto now = UnixDateTime::now() + current_time_offset_for_testing;
     auto since_last_modified = now - *last_modified;
     auto seconds = since_last_modified.to_seconds();
 
@@ -243,7 +245,7 @@ static AK::Duration calculate_heuristic_freshness_lifetime(HTTP::HeaderMap const
 }
 
 // https://httpwg.org/specs/rfc9111.html#calculating.freshness.lifetime
-AK::Duration calculate_freshness_lifetime(u32 status_code, HTTP::HeaderMap const& headers)
+AK::Duration calculate_freshness_lifetime(u32 status_code, HTTP::HeaderMap const& headers, AK::Duration current_time_offset_for_testing)
 {
     // A cache can calculate the freshness lifetime (denoted as freshness_lifetime) of a response by evaluating the
     // following rules and using the first match:
@@ -265,8 +267,8 @@ AK::Duration calculate_freshness_lifetime(u32 status_code, HTTP::HeaderMap const
     // * If the Expires response header field (Section 5.3) is present, use its value minus the value of the Date response
     //   header field (using the time the message was received if it is not present, as per Section 6.6.1 of [HTTP]), or
     if (auto expires = parse_http_date(headers.get("Expires"sv)); expires.has_value()) {
-        auto date = parse_http_date(headers.get("Date"sv)).value_or_lazy_evaluated([]() {
-            return UnixDateTime::now();
+        auto date = parse_http_date(headers.get("Date"sv)).value_or_lazy_evaluated([&]() {
+            return UnixDateTime::now() + current_time_offset_for_testing;
         });
 
         return *expires - date;
@@ -287,14 +289,14 @@ AK::Duration calculate_freshness_lifetime(u32 status_code, HTTP::HeaderMap const
     }
 
     if (heuristics_allowed)
-        return calculate_heuristic_freshness_lifetime(headers);
+        return calculate_heuristic_freshness_lifetime(headers, current_time_offset_for_testing);
 
     // No explicit expiration time, and heuristics not allowed or not applicable.
     return {};
 }
 
 // https://httpwg.org/specs/rfc9111.html#age.calculations
-AK::Duration calculate_age(HTTP::HeaderMap const& headers, UnixDateTime request_time, UnixDateTime response_time)
+AK::Duration calculate_age(HTTP::HeaderMap const& headers, UnixDateTime request_time, UnixDateTime response_time, AK::Duration current_time_offset_for_testing)
 {
     // The term "age_value" denotes the value of the Age header field (Section 5.1), in a form appropriate for arithmetic
     // operation; or 0, if not available.
@@ -306,7 +308,7 @@ AK::Duration calculate_age(HTTP::HeaderMap const& headers, UnixDateTime request_
     }
 
     // The term "now" means the current value of this implementation's clock (Section 5.6.7 of [HTTP]).
-    auto now = UnixDateTime::now();
+    auto now = UnixDateTime::now() + current_time_offset_for_testing;
 
     // The term "date_value" denotes the value of the Date header field, in a form appropriate for arithmetic operations.
     // See Section 6.6.1 of [HTTP] for the definition of the Date header field and for requirements regarding responses
@@ -406,4 +408,16 @@ void update_header_fields(HTTP::HeaderMap& stored_headers, HTTP::HeaderMap const
     }
 }
 
+AK::Duration compute_current_time_offset_for_testing(Optional<DiskCache&> disk_cache, HTTP::HeaderMap const& request_headers)
+{
+    if (disk_cache.has_value() && disk_cache->mode() == DiskCache::Mode::Testing) {
+        if (auto header = request_headers.get(TEST_CACHE_REQUEST_TIME_OFFSET); header.has_value()) {
+            if (auto offset = header->to_number<i64>(); offset.has_value())
+                return AK::Duration::from_seconds(*offset);
+        }
+    }
+
+    return {};
+}
+
 }

Services/RequestServer/Cache/Utilities.h

@@ -12,11 +12,13 @@
 #include <AK/Types.h>
 #include <LibHTTP/HeaderMap.h>
 #include <LibURL/Forward.h>
+#include <RequestServer/Forward.h>
 
 namespace RequestServer {
 
 constexpr inline auto TEST_CACHE_ENABLED_HEADER = "X-Ladybird-Enable-Disk-Cache"sv;
 constexpr inline auto TEST_CACHE_STATUS_HEADER = "X-Ladybird-Disk-Cache-Status"sv;
+constexpr inline auto TEST_CACHE_REQUEST_TIME_OFFSET = "X-Ladybird-Request-Time-Offset"sv;
 
 String serialize_url_for_cache_storage(URL::URL const&);
 u64 create_cache_key(StringView url, StringView method);
@@ -26,8 +28,8 @@ bool is_cacheable(StringView method);
 bool is_cacheable(u32 status_code, HTTP::HeaderMap const&);
 bool is_header_exempted_from_storage(StringView name);
 
-AK::Duration calculate_freshness_lifetime(u32 status_code, HTTP::HeaderMap const&);
-AK::Duration calculate_age(HTTP::HeaderMap const&, UnixDateTime request_time, UnixDateTime response_time);
+AK::Duration calculate_freshness_lifetime(u32 status_code, HTTP::HeaderMap const&, AK::Duration current_time_offset_for_testing);
+AK::Duration calculate_age(HTTP::HeaderMap const&, UnixDateTime request_time, UnixDateTime response_time, AK::Duration current_time_offset_for_testing);
 
 enum class CacheLifetimeStatus {
     Fresh,
@@ -45,4 +47,6 @@ struct RevalidationAttributes {
 
 void update_header_fields(HTTP::HeaderMap&, HTTP::HeaderMap const&);
 
+AK::Duration compute_current_time_offset_for_testing(Optional<DiskCache&>, HTTP::HeaderMap const& request_headers);
+
 }

Services/RequestServer/Request.cpp

@@ -85,7 +85,9 @@ Request::Request(
     , m_request_body(move(request_body))
     , m_alt_svc_cache_path(move(alt_svc_cache_path))
     , m_proxy_data(proxy_data)
+    , m_current_time_offset_for_testing(compute_current_time_offset_for_testing(m_disk_cache, m_request_headers))
 {
+    m_request_start_time += m_current_time_offset_for_testing;
 }
 
 Request::Request(
@@ -100,7 +102,9 @@ Request::Request(
     , m_curl_multi_handle(curl_multi)
     , m_resolver(resolver)
     , m_url(move(url))
+    , m_current_time_offset_for_testing(compute_current_time_offset_for_testing(m_disk_cache, m_request_headers))
 {
+    m_request_start_time += m_current_time_offset_for_testing;
 }
 
 Request::~Request()

Services/RequestServer/Request.h

@@ -55,6 +55,7 @@ class Request : public Weakable<Request> {
     ByteString const& method() const { return m_method; }
     HTTP::HeaderMap const& request_headers() const { return m_request_headers; }
     UnixDateTime request_start_time() const { return m_request_start_time; }
+    AK::Duration current_time_offset_for_testing() const { return m_current_time_offset_for_testing; }
 
     void notify_request_unblocked(Badge<DiskCache>);
     void notify_fetch_complete(Badge<ConnectionFromClient>, int result_code);
@@ -170,6 +171,8 @@ class Request : public Weakable<Request> {
     CacheStatus m_cache_status { CacheStatus::Unknown };
 
     Optional<Requests::NetworkError> m_network_error;
+
+    AK::Duration m_current_time_offset_for_testing;
 };
 
 }