LibWeb: Implement most of the 'Fetching' AOs

This implements the following operations from section 4 of the Fetch
spec (https://fetch.spec.whatwg.org/#fetching):

- Fetch
- Main fetch
- Fetch response handover
- Scheme fetch
- HTTP fetch
- HTTP-redirect fetch
- HTTP-network-or-cache fetch (without caching)

It does *not* implement:

- HTTP-network fetch
- CORS-preflight fetch

Instead, we let ResourceLoader handle the actual networking for now,
which isn't ideal, but certainly enough to get enough functionality up
and running for most websites to not complain.

Author: linusg

Userland/Libraries/LibWeb/CMakeLists.txt

@@ -123,6 +123,9 @@ set(SOURCES
     Fetch/BodyInit.cpp
     Fetch/Enums.cpp
     Fetch/Fetching/Checks.cpp
+    Fetch/Fetching/Fetching.cpp
+    Fetch/Fetching/PendingResponse.cpp
+    Fetch/Fetching/RefCountedFlag.cpp
     Fetch/Headers.cpp
     Fetch/HeadersIterator.cpp
     Fetch/Infrastructure/ConnectionTimingInfo.cpp

Userland/Libraries/LibWeb/Fetch/Fetching/Fetching.cpp

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2022, Linus Groh <linusg@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Forward.h>
+#include <LibJS/Forward.h>
+#include <LibWeb/Forward.h>
+
+namespace Web::Fetch::Fetching {
+
+#define ENUMERATE_BOOL_PARAMS                     \
+    __ENUMERATE_BOOL_PARAM(IncludeCredentials)    \
+    __ENUMERATE_BOOL_PARAM(IsAuthenticationFetch) \
+    __ENUMERATE_BOOL_PARAM(IsNewConnectionFetch)  \
+    __ENUMERATE_BOOL_PARAM(MakeCORSPreflight)     \
+    __ENUMERATE_BOOL_PARAM(Recursive)             \
+    __ENUMERATE_BOOL_PARAM(UseParallelQueue)
+
+#define __ENUMERATE_BOOL_PARAM(Name) \
+    enum class Name {                \
+        Yes,                         \
+        No,                          \
+    };
+ENUMERATE_BOOL_PARAMS
+#undef __ENUMERATE_BOOL_PARAM
+
+WebIDL::ExceptionOr<JS::NonnullGCPtr<Infrastructure::FetchController>> fetch(JS::Realm&, Infrastructure::Request&, Infrastructure::FetchAlgorithms const&, UseParallelQueue use_parallel_queue = UseParallelQueue::No);
+WebIDL::ExceptionOr<Optional<JS::NonnullGCPtr<PendingResponse>>> main_fetch(JS::Realm&, Infrastructure::FetchParams const&, Recursive recursive = Recursive::No);
+WebIDL::ExceptionOr<void> fetch_response_handover(JS::Realm&, Infrastructure::FetchParams const&, Infrastructure::Response const&);
+WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> scheme_fetch(JS::Realm&, Infrastructure::FetchParams const&);
+WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_fetch(JS::Realm&, Infrastructure::FetchParams const&, MakeCORSPreflight make_cors_preflight = MakeCORSPreflight::No);
+WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_redirect_fetch(JS::Realm&, Infrastructure::FetchParams const&, Infrastructure::Response const&);
+WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> http_network_or_cache_fetch(JS::Realm&, Infrastructure::FetchParams const&, IsAuthenticationFetch is_authentication_fetch = IsAuthenticationFetch::No, IsNewConnectionFetch is_new_connection_fetch = IsNewConnectionFetch::No);
+WebIDL::ExceptionOr<JS::NonnullGCPtr<PendingResponse>> nonstandard_resource_loader_http_network_fetch(JS::Realm&, Infrastructure::FetchParams const&, IncludeCredentials include_credentials = IncludeCredentials::No, IsNewConnectionFetch is_new_connection_fetch = IsNewConnectionFetch::No);
+
+}

Userland/Libraries/LibWeb/Fetch/Fetching/Fetching.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2022, Linus Groh <linusg@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibJS/Heap/Heap.h>
+#include <LibJS/Runtime/VM.h>
+#include <LibWeb/Fetch/Fetching/PendingResponse.h>
+#include <LibWeb/Platform/EventLoopPlugin.h>
+
+namespace Web::Fetch::Fetching {
+
+JS::NonnullGCPtr<PendingResponse> PendingResponse::create(JS::VM& vm)
+{
+    return { *vm.heap().allocate_without_realm<PendingResponse>() };
+}
+
+JS::NonnullGCPtr<PendingResponse> PendingResponse::create(JS::VM& vm, JS::NonnullGCPtr<Infrastructure::Response> response)
+{
+    return { *vm.heap().allocate_without_realm<PendingResponse>(response) };
+}
+
+PendingResponse::PendingResponse(JS::NonnullGCPtr<Infrastructure::Response> response)
+    : m_response(response)
+{
+}
+
+void PendingResponse::visit_edges(JS::Cell::Visitor& visitor)
+{
+    Base::visit_edges(visitor);
+    visitor.visit(m_response);
+}
+
+void PendingResponse::when_loaded(Callback callback)
+{
+    VERIFY(!m_callback);
+    m_callback = move(callback);
+    if (m_response)
+        run_callback();
+}
+
+void PendingResponse::resolve(JS::NonnullGCPtr<Infrastructure::Response> response)
+{
+    VERIFY(!m_response);
+    m_response = response;
+    if (m_callback)
+        run_callback();
+}
+
+void PendingResponse::run_callback() const
+{
+    VERIFY(m_callback);
+    VERIFY(m_response);
+    Platform::EventLoopPlugin::the().deferred_invoke([strong_this = JS::make_handle(const_cast<PendingResponse&>(*this))]() {
+        strong_this->m_callback(*strong_this->m_response);
+    });
+}
+
+}

Userland/Libraries/LibWeb/Fetch/Fetching/PendingResponse.cpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2022, Linus Groh <linusg@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibJS/Forward.h>
+#include <LibJS/Heap/Cell.h>
+#include <LibJS/SafeFunction.h>
+#include <LibWeb/Fetch/Infrastructure/HTTP/Responses.h>
+
+namespace Web::Fetch::Fetching {
+
+// Non-standard wrapper around a possibly pending Infrastructure::Response.
+// This is needed to fit the asynchronous nature of ResourceLoader into the synchronous expectations
+// of the Fetch spec - we run 'in parallel' as a deferred_invoke(), which is still on the main thread;
+// therefore we use callbacks to run portions of the spec that require waiting for an HTTP load.
+class PendingResponse : public JS::Cell {
+    JS_CELL(PendingResponse, JS::Cell);
+
+public:
+    using Callback = JS::SafeFunction<void(JS::NonnullGCPtr<Infrastructure::Response>)>;
+
+    [[nodiscard]] static JS::NonnullGCPtr<PendingResponse> create(JS::VM&);
+    [[nodiscard]] static JS::NonnullGCPtr<PendingResponse> create(JS::VM&, JS::NonnullGCPtr<Infrastructure::Response>);
+
+    void when_loaded(Callback);
+    void resolve(JS::NonnullGCPtr<Infrastructure::Response>);
+
+private:
+    PendingResponse() = default;
+    explicit PendingResponse(JS::NonnullGCPtr<Infrastructure::Response>);
+
+    virtual void visit_edges(JS::Cell::Visitor&) override;
+
+    void run_callback() const;
+
+    Callback m_callback;
+    JS::GCPtr<Infrastructure::Response> m_response;
+};
+
+}

Userland/Libraries/LibWeb/Fetch/Fetching/PendingResponse.h

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2022, Linus Groh <linusg@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/NonnullRefPtr.h>
+#include <LibWeb/Fetch/Fetching/RefCountedFlag.h>
+
+namespace Web::Fetch::Fetching {
+
+NonnullRefPtr<RefCountedFlag> RefCountedFlag::create(bool value)
+{
+    return adopt_ref(*new RefCountedFlag(value));
+}
+
+RefCountedFlag::RefCountedFlag(bool value)
+    : m_value(value)
+{
+}
+
+}

Userland/Libraries/LibWeb/Fetch/Fetching/RefCountedFlag.cpp

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2022, Linus Groh <linusg@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/RefCounted.h>
+
+namespace Web::Fetch::Fetching {
+
+/// A ref-counted boolean flag.
+/// This is used to share flags between multiple callback closures.
+class RefCountedFlag : public RefCounted<RefCountedFlag> {
+public:
+    static NonnullRefPtr<RefCountedFlag> create(bool);
+
+    [[nodiscard]] bool value() const { return m_value; }
+    void set_value(bool value) { m_value = value; }
+
+private:
+    explicit RefCountedFlag(bool);
+
+    bool m_value { false };
+};
+
+}

Userland/Libraries/LibWeb/Fetch/Fetching/RefCountedFlag.h

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
+#include <AK/Debug.h>
 #include <AK/URLParser.h>
 #include <LibJS/Heap/Heap.h>
 #include <LibJS/Runtime/VM.h>
@@ -43,6 +44,7 @@ JS::NonnullGCPtr<Response> Response::aborted_network_error(JS::VM& vm)
 
 JS::NonnullGCPtr<Response> Response::network_error(JS::VM& vm, String message)
 {
+    dbgln_if(WEB_FETCH_DEBUG, "Fetch: Creating network error response with message: {}", message);
     auto response = Response::create(vm);
     response->set_status(0);
     response->set_type(Type::Error);

Userland/Libraries/LibWeb/Fetch/Infrastructure/HTTP/Responses.cpp

@@ -189,6 +189,11 @@ class Request;
 class Response;
 }
 
+namespace Web::Fetch::Fetching {
+class PendingResponse;
+class RefCountedFlag;
+}
+
 namespace Web::Fetch::Infrastructure {
 class Body;
 struct BodyWithType;