Async: Add AsyncLoopWork to execute callbacks in background threads o…

…n the EventLoop
Pagghiu · Apr 13, 2024 · 4ab30f5 · 4ab30f5
1 parent 79fdec0
commit 4ab30f5
Show file tree

Hide file tree

Showing 8 changed files with 135 additions and 10 deletions.
diff --git a/Documentation/Libraries/Async.md b/Documentation/Libraries/Async.md
@@ -24,7 +24,8 @@ This is the list of supported async operations:
 | [AsyncFileClose](@ref SC::AsyncFileClose)         | @copybrief SC::AsyncFileClose     |
 | [AsyncLoopTimeout](@ref SC::AsyncLoopTimeout)     | @copybrief SC::AsyncLoopTimeout   |
 | [AsyncLoopWakeUp](@ref SC::AsyncLoopWakeUp)       | @copybrief SC::AsyncLoopWakeUp    |
-| [AsyncProcessExit](@ref SC::AsyncProcessExit)     | @copybrief SC::AsyncProcessExit    |
+| [AsyncLoopWork](@ref SC::AsyncLoopWork)           | @copybrief SC::AsyncLoopWork      |
+| [AsyncProcessExit](@ref SC::AsyncProcessExit)     | @copybrief SC::AsyncProcessExit   |
 | [AsyncFilePoll](@ref SC::AsyncFilePoll)           | @copybrief SC::AsyncFilePoll      |
 
 # Status
@@ -53,6 +54,9 @@ Event loop can be run in different ways to allow integrated it in multiple ways
 ## AsyncLoopWakeUp
 @copydoc SC::AsyncLoopWakeUp
 
+## AsyncLoopWork
+@copydoc SC::AsyncLoopWork
+
 ## AsyncProcessExit
 @copydoc SC::AsyncProcessExit
 

diff --git a/Libraries/Async/Async.cpp b/Libraries/Async/Async.cpp
@@ -36,6 +36,7 @@ const char* SC::AsyncRequest::TypeToString(Type type)
     {
     case Type::LoopTimeout: return "LoopTimeout";
     case Type::LoopWakeUp: return "LoopWakeUp";
+    case Type::LoopWork: return "LoopWork";
     case Type::ProcessExit: return "ProcessExit";
     case Type::SocketAccept: return "SocketAccept";
     case Type::SocketConnect: return "SocketConnect";
@@ -120,6 +121,12 @@ SC::Result SC::AsyncLoopWakeUp::start(AsyncEventLoop& loop, EventObject* eo)
 
 SC::Result SC::AsyncLoopWakeUp::wakeUp() { return getEventLoop()->wakeUpFromExternalThread(*this); }
 
+SC::Result SC::AsyncLoopWork::start(AsyncEventLoop& loop, ThreadPool& threadPool)
+{
+    SC_TRY_MSG(work.isValid(), "AsyncLoopWork::start - Invalid work callback");
+    return queueSubmission(loop, threadPool, task);
+}
+
 SC::Result SC::AsyncProcessExit::start(AsyncEventLoop& loop, ProcessDescriptor::Handle process)
 {
     handle = process;
@@ -189,8 +196,14 @@ SC::Result SC::AsyncFileRead::start(AsyncEventLoop& loop, ThreadPool& threadPool
     SC_TRY_MSG(buffer.sizeInBytes() > 0, "AsyncFileRead::start - Zero sized read buffer");
     SC_TRY_MSG(fileDescriptor != FileDescriptor::Invalid, "AsyncFileRead::start - Invalid file descriptor");
     SC_TRY(validateAsync());
-    SC_TRY(queueSubmission(loop, threadPool, task));
-    return SC::Result(true);
+    if (loop.internalSelf.makesSenseToRunInThreadPool(*this))
+    {
+        return queueSubmission(loop, threadPool, task);
+    }
+    else
+    {
+        return queueSubmission(loop);
+    }
 }
 
 SC::Result SC::AsyncFileWrite::start(AsyncEventLoop& loop)
@@ -207,8 +220,14 @@ SC::Result SC::AsyncFileWrite::start(AsyncEventLoop& loop, ThreadPool& threadPoo
     SC_TRY_MSG(buffer.sizeInBytes() > 0, "AsyncFileWrite::start - Zero sized write buffer");
     SC_TRY_MSG(fileDescriptor != FileDescriptor::Invalid, "AsyncFileWrite::start - Invalid file descriptor");
     SC_TRY(validateAsync());
-    SC_TRY(queueSubmission(loop, threadPool, task));
-    return SC::Result(true);
+    if (loop.internalSelf.makesSenseToRunInThreadPool(*this))
+    {
+        return queueSubmission(loop, threadPool, task);
+    }
+    else
+    {
+        return queueSubmission(loop);
+    }
 }
 
 SC::Result SC::AsyncFileClose::start(AsyncEventLoop& loop, FileDescriptor::Handle fd)
@@ -336,7 +355,7 @@ SC::Result SC::AsyncEventLoop::Private::queueSubmission(AsyncRequest& async, Asy
     async.state     = AsyncRequest::State::Setup;
 
     // Only set the async tasks for operations and backends that are not io_uring
-    if (task and eventLoop->internalSelf.makesSenseToRunInThreadPool(async))
+    if (task)
     {
         async.asyncTask = task;
         task->async     = &async;
@@ -934,6 +953,7 @@ void SC::AsyncEventLoop::Private::removeActiveHandle(AsyncRequest& async)
     {
         case AsyncRequest::Type::LoopTimeout:   activeLoopTimeouts.remove(*static_cast<AsyncLoopTimeout*>(&async));     break;
         case AsyncRequest::Type::LoopWakeUp:    activeLoopWakeUps.remove(*static_cast<AsyncLoopWakeUp*>(&async));       break;
+        case AsyncRequest::Type::LoopWork:      activeLoopWork.remove(*static_cast<AsyncLoopWork*>(&async));            break;
         case AsyncRequest::Type::ProcessExit:   activeProcessExits.remove(*static_cast<AsyncProcessExit*>(&async));     break;
         case AsyncRequest::Type::SocketAccept:  activeSocketAccepts.remove(*static_cast<AsyncSocketAccept*>(&async));   break;
         case AsyncRequest::Type::SocketConnect: activeSocketConnects.remove(*static_cast<AsyncSocketConnect*>(&async)); break;
@@ -970,6 +990,7 @@ void SC::AsyncEventLoop::Private::addActiveHandle(AsyncRequest& async)
     {
         case AsyncRequest::Type::LoopTimeout:   activeLoopTimeouts.queueBack(*static_cast<AsyncLoopTimeout*>(&async));      break;
         case AsyncRequest::Type::LoopWakeUp:    activeLoopWakeUps.queueBack(*static_cast<AsyncLoopWakeUp*>(&async));        break;
+        case AsyncRequest::Type::LoopWork:      activeLoopWork.queueBack(*static_cast<AsyncLoopWork*>(&async));             break;
         case AsyncRequest::Type::ProcessExit:   activeProcessExits.queueBack(*static_cast<AsyncProcessExit*>(&async));      break;
         case AsyncRequest::Type::SocketAccept:  activeSocketAccepts.queueBack(*static_cast<AsyncSocketAccept*>(&async));    break;
         case AsyncRequest::Type::SocketConnect: activeSocketConnects.queueBack(*static_cast<AsyncSocketConnect*>(&async));  break;
@@ -997,6 +1018,7 @@ SC::Result SC::AsyncEventLoop::Private::applyOnAsync(AsyncRequest& async, Lambda
     {
     case AsyncRequest::Type::LoopTimeout: SC_TRY(lambda(*static_cast<AsyncLoopTimeout*>(&async))); break;
     case AsyncRequest::Type::LoopWakeUp: SC_TRY(lambda(*static_cast<AsyncLoopWakeUp*>(&async))); break;
+    case AsyncRequest::Type::LoopWork: SC_TRY(lambda(*static_cast<AsyncLoopWork*>(&async))); break;
     case AsyncRequest::Type::ProcessExit: SC_TRY(lambda(*static_cast<AsyncProcessExit*>(&async))); break;
     case AsyncRequest::Type::SocketAccept: SC_TRY(lambda(*static_cast<AsyncSocketAccept*>(&async))); break;
     case AsyncRequest::Type::SocketConnect: SC_TRY(lambda(*static_cast<AsyncSocketConnect*>(&async))); break;

diff --git a/Libraries/Async/Async.h b/Libraries/Async/Async.h
@@ -137,6 +137,7 @@ struct SC::AsyncRequest
     {
         LoopTimeout,   ///< Request is an AsyncLoopTimeout object
         LoopWakeUp,    ///< Request is an AsyncLoopWakeUp object
+        LoopWork,      ///< Request is an AsyncLoopWork object
         ProcessExit,   ///< Request is an AsyncProcessExit object
         SocketAccept,  ///< Request is an AsyncSocketAccept object
         SocketConnect, ///< Request is an AsyncSocketConnect object
@@ -349,6 +350,33 @@ struct AsyncLoopWakeUp : public AsyncRequest
     Atomic<bool> pending     = false;
 };
 
+/// @brief Executes work in a thread pool and then invokes a callback on the event loop thread. @n
+/// AsyncLoopWork::work is invoked on one of the thread supplied by the ThreadPool passed during AsyncLoopWork::start.
+/// AsyncLoopWork::callback will be called as a completion, on the event loop thread AFTER work callback is finished.
+///
+/// \snippet Libraries/Async/Tests/AsyncTest.cpp AsyncLoopWorkSnippet1
+struct AsyncLoopWork : public AsyncRequest
+{
+    AsyncLoopWork() : AsyncRequest(Type::LoopWork) {}
+
+    /// @brief Completion data for AsyncLoopWakeUp
+    using CompletionData = AsyncCompletionData;
+
+    /// @brief Callback result for AsyncLoopWakeUp
+    using Result = AsyncResultOf<AsyncLoopWork, CompletionData>;
+
+    /// @brief Schedule work to be executed on a background thread, notifying the event loop when it's finished.
+    /// @param eventLoop The AsyncEventLoop where to schedule this work on
+    /// @param threadPool The ThreadPool that will supply the background thread
+    [[nodiscard]] SC::Result start(AsyncEventLoop& eventLoop, ThreadPool& threadPool);
+
+    Function<SC::Result()>  work;     /// Called to execute the work in a background threadpool thread
+    Function<void(Result&)> callback; /// Called after work is done, on the thread calling EventLoop::run()
+
+  private:
+    AsyncTaskOf<AsyncLoopWork> task;
+};
+
 /// @brief Starts monitoring a process, notifying about its termination.
 /// @ref library_process library can be used to start a process and obtain the native process handle.
 ///
@@ -390,7 +418,7 @@ struct AsyncProcessExit : public AsyncRequest
     detail::WinOverlappedOpaque overlapped;
     detail::WinWaitHandle       waitHandle;
 #elif SC_PLATFORM_LINUX
-    FileDescriptor pidFd;
+    FileDescriptor     pidFd;
 #endif
 };
 
@@ -897,9 +925,9 @@ struct SC::AsyncEventLoop
 
     struct PrivateDefinition
     {
-        static constexpr int Windows = 320;
-        static constexpr int Apple   = 344;
-        static constexpr int Default = 328;
+        static constexpr int Windows = 336;
+        static constexpr int Apple   = 360;
+        static constexpr int Default = 344;
 
         static constexpr size_t Alignment = 8;
 
@@ -950,6 +978,8 @@ struct SC::AsyncEventLoop
     InternalOpaque internal;
     Internal&      internalSelf;
     friend struct AsyncRequest;
+    friend struct AsyncFileWrite;
+    friend struct AsyncFileRead;
 };
 
 //! @}
diff --git a/Libraries/Async/Internal/AsyncLinux.inl b/Libraries/Async/Internal/AsyncLinux.inl
@@ -342,6 +342,11 @@ struct SC::AsyncEventLoop::KernelQueueIoURing
     //-------------------------------------------------------------------------------------------------------
     // Nothing to do :)
 
+    //-------------------------------------------------------------------------------------------------------
+    // WORK
+    //-------------------------------------------------------------------------------------------------------
+    static Result executeOperation(AsyncLoopWork& loopWork, AsyncLoopWork::CompletionData&) { return loopWork.work(); }
+
     //-------------------------------------------------------------------------------------------------------
     // Socket ACCEPT
     //-------------------------------------------------------------------------------------------------------

diff --git a/Libraries/Async/Internal/AsyncPosix.inl b/Libraries/Async/Internal/AsyncPosix.inl
@@ -474,6 +474,11 @@ struct SC::AsyncEventLoop::KernelQueuePosix
     //-------------------------------------------------------------------------------------------------------
     // Nothing to do :)
 
+    //-------------------------------------------------------------------------------------------------------
+    // WORK
+    //-------------------------------------------------------------------------------------------------------
+    static Result executeOperation(AsyncLoopWork& loopWork, AsyncLoopWork::CompletionData&) { return loopWork.work(); }
+
     //-------------------------------------------------------------------------------------------------------
     // Socket ACCEPT
     //-------------------------------------------------------------------------------------------------------

diff --git a/Libraries/Async/Internal/AsyncPrivate.h b/Libraries/Async/Internal/AsyncPrivate.h
@@ -20,6 +20,7 @@ struct SC::AsyncEventLoop::Private
     // Active phase
     IntrusiveDoubleLinkedList<AsyncLoopTimeout>   activeLoopTimeouts;
     IntrusiveDoubleLinkedList<AsyncLoopWakeUp>    activeLoopWakeUps;
+    IntrusiveDoubleLinkedList<AsyncLoopWork>      activeLoopWork;
     IntrusiveDoubleLinkedList<AsyncProcessExit>   activeProcessExits;
     IntrusiveDoubleLinkedList<AsyncSocketAccept>  activeSocketAccepts;
     IntrusiveDoubleLinkedList<AsyncSocketConnect> activeSocketConnects;

diff --git a/Libraries/Async/Internal/AsyncWindows.inl b/Libraries/Async/Internal/AsyncWindows.inl
@@ -239,6 +239,12 @@ struct SC::AsyncEventLoop::KernelQueue
     //-------------------------------------------------------------------------------------------------------
     [[nodiscard]] static bool setupAsync(AsyncLoopWakeUp&) { return true; }
 
+    //-------------------------------------------------------------------------------------------------------
+    // WORK
+    //-------------------------------------------------------------------------------------------------------
+    static bool   setupAsync(AsyncLoopWork&) { return true; }
+    static Result executeOperation(AsyncLoopWork& loopWork, AsyncLoopWork::CompletionData&) { return loopWork.work(); }
+
     //-------------------------------------------------------------------------------------------------------
     // Socket ACCEPT
     //-------------------------------------------------------------------------------------------------------

diff --git a/Libraries/Async/Tests/AsyncTest.cpp b/Libraries/Async/Tests/AsyncTest.cpp
@@ -27,6 +27,10 @@ struct SC::AsyncTest : public SC::TestCase
         }
         for (int i = 0; i < numTestsToRun; ++i)
         {
+            if (test_section("loop work"))
+            {
+                loopWork();
+            }
             loopTimeout();
             loopWakeUpFromExternalThread();
             loopWakeUp();
@@ -50,6 +54,8 @@ struct SC::AsyncTest : public SC::TestCase
         }
     }
 
+    void loopWork();
+
     void loopFreeSubmittingOnClose()
     {
         // This test checks that on close asyncs being submitted are being removed for submission queue and set as Free.
@@ -819,6 +825,52 @@ struct SC::AsyncTest : public SC::TestCase
     }
 };
 
+void SC::AsyncTest::loopWork()
+{
+    //! [AsyncLoopWorkSnippet1]
+    // This test creates a thread pool with 4 thread and 16 AsyncLoopWork.
+    // All the 16 AsyncLoopWork are scheduled to do some work on a background thread.
+    // After work is done, their respective after-work callback is invoked on the event loop thread.
+
+    static constexpr int NUM_THREADS = 4;
+    static constexpr int NUM_WORKS   = NUM_THREADS * NUM_THREADS;
+
+    ThreadPool threadPool;
+    SC_TEST_EXPECT(threadPool.create(NUM_THREADS));
+
+    AsyncEventLoop eventLoop;
+    SC_TEST_EXPECT(eventLoop.create());
+
+    AsyncLoopWork works[NUM_WORKS];
+
+    int         numAfterWorkCallbackCalls = 0;
+    Atomic<int> numWorkCallbackCalls      = 0;
+
+    for (int idx = 0; idx < NUM_WORKS; ++idx)
+    {
+        works[idx].work = [&]
+        {
+            // This work callback is called on some random threadPool thread
+            Thread::Sleep(50);                 // Execute some work on the thread
+            numWorkCallbackCalls.fetch_add(1); // Atomically increment this counter
+            return Result(true);
+        };
+        works[idx].callback = [&](AsyncLoopWork::Result&)
+        {
+            // This after-work callback is invoked on the event loop thread.
+            // More precisely this runs on the thread calling eventLoop.run().
+            numAfterWorkCallbackCalls++; // No need for atomics here, callback is run inside loop thread
+        };
+        SC_TEST_EXPECT(works[idx].start(eventLoop, threadPool));
+    }
+    SC_TEST_EXPECT(eventLoop.run());
+
+    // Check that callbacks have been actually called
+    SC_TEST_EXPECT(numWorkCallbackCalls.load() == NUM_WORKS);
+    SC_TEST_EXPECT(numAfterWorkCallbackCalls == NUM_WORKS);
+    //! [AsyncLoopWorkSnippet1]
+}
+
 namespace SC
 {
 void runAsyncTest(SC::TestReport& report) { AsyncTest test(report); }