perf(timer): embed the waiter node in the awaitable frame

include/boost/corosio/detail/timer.hpp

@@ -13,6 +13,7 @@
 
 #include <boost/corosio/detail/config.hpp>
 #include <boost/corosio/detail/intrusive.hpp>
+#include <boost/corosio/detail/scheduler_op.hpp>
 #include <boost/corosio/io/io_object.hpp>
 #include <boost/capy/continuation.hpp>
 #include <boost/capy/io_result.hpp>
@@ -28,19 +29,22 @@
 #include <coroutine>
 #include <cstddef>
 #include <limits>
+#include <new>
 #include <stop_token>
 #include <system_error>
 #include <type_traits>
 
 namespace boost::corosio::detail {
 
-// Defined in timer_service.hpp, which includes this header. The
-// implementation::wait() body needs waiter_node and timer_service to
-// be complete, so it is declared here and defined there; intrusive_list
-// only stores waiter_node pointers, so the forward declaration suffices
-// for implementation's data layout.
+// timer_service is defined in timer_service.hpp, which includes this
+// header. waiter_node and wait_awaitable are defined below the timer
+// class: waiter_node stores a timer::implementation*, which cannot be
+// forward-declared as a nested type. intrusive_list only stores
+// waiter_node pointers, so this forward declaration suffices for
+// implementation's data layout.
 class timer_service;
 struct waiter_node;
+struct wait_awaitable;
 
 /** An asynchronous timer for coroutine I/O.
 
@@ -68,46 +72,7 @@ struct waiter_node;
 */
 class BOOST_COROSIO_DECL timer : public io_object
 {
-    struct wait_awaitable
-    {
-        timer& t_;
-        std::error_code ec_;
-        capy::continuation cont_;
-
-        explicit wait_awaitable(timer& t) noexcept : t_(t) {}
-
-        bool await_ready() const noexcept
-        {
-            return false;
-        }
-
-        // Cancellation surfaces through ec_: the stop_token path in
-        // wait() completes the waiter with error::canceled written to
-        // ec_, so there is no separate token to consult here.
-        capy::io_result<> await_resume() const noexcept
-        {
-            return {ec_};
-        }
-
-        auto await_suspend(std::coroutine_handle<> h, capy::io_env const* env)
-            -> std::coroutine_handle<>
-        {
-            cont_.h = h;
-            auto& impl = t_.get();
-            // Inline fast path: already expired and not in the heap
-            if (impl.heap_index_.load(std::memory_order_relaxed) ==
-                    implementation::npos &&
-                (impl.expiry_ == (time_point::min)() ||
-                 impl.expiry_ <= clock_type::now()))
-            {
-                ec_ = {};
-                auto d = env->executor;
-                d.post(cont_);
-                return std::noop_coroutine();
-            }
-            return impl.wait(h, env->executor, env->stop_token, &ec_, &cont_);
-        }
-    };
+    friend struct wait_awaitable;
 
 public:
     /** Backend state and wait entry point for a timer.
@@ -158,18 +123,40 @@ class BOOST_COROSIO_DECL timer : public io_object
         /// Construct bound to the given timer service.
         explicit implementation(timer_service& svc) noexcept : svc_(&svc) {}
 
-        /// Initiate an asynchronous wait for the timer to expire.
+        /** Check whether the timer is expired and absent from the heap.
+
+            The single definition of the already-expired fast-path
+            predicate: `await_suspend` tests it inline and `wait()`
+            re-tests it because the expiry can elapse between the two
+            reads.
+        */
+        bool already_expired() const noexcept
+        {
+            return heap_index_.load(std::memory_order_relaxed) == npos &&
+                (expiry_ ==
+                     (std::chrono::steady_clock::time_point::min)() ||
+                 expiry_ <= std::chrono::steady_clock::now());
+        }
+
+        /** Asynchronously wait for the timer to expire.
+
+            Publishes the waiter into the service's heap and waiter
+            list, after which it may complete on any thread. If the
+            timer is already expired and not in the heap, completes
+            by posting the continuation without publishing.
+
+            @par Preconditions
+            @p w is fully initialized, and its storage (the awaitable
+            on the suspended coroutine's frame) outlives the wait.
+
+            @param w The waiter to publish.
+        */
         // Exported at member level: dllexport on the enclosing timer
         // class does not extend to nested classes, and header-inline
         // callers (wait_awaitable::await_suspend) reference this
         // symbol from outside the corosio DLL.
         BOOST_COROSIO_DECL
-        std::coroutine_handle<> wait(
-            std::coroutine_handle<>,
-            capy::executor_ref,
-            std::stop_token,
-            std::error_code*,
-            capy::continuation*);
+        std::coroutine_handle<> wait(waiter_node& w);
     };
 
     /// The clock type used for time operations.
@@ -419,10 +406,8 @@ class BOOST_COROSIO_DECL timer : public io_object
 
         @return An awaitable that completes with `io_result<>`.
     */
-    auto wait()
-    {
-        return wait_awaitable(*this);
-    }
+    // Defined below wait_awaitable, which needs timer complete.
+    wait_awaitable wait();
 
 protected:
     explicit timer(handle h) noexcept : io_object(std::move(h)) {}
@@ -442,6 +427,181 @@ class BOOST_COROSIO_DECL timer : public io_object
     }
 };
 
+/** Frame-resident per-wait state for a timer wait.
+
+    One node exists per `co_await` on a timer, embedded in the
+    awaitable on the suspended coroutine's frame — never allocated.
+    Once published by `implementation::wait()` the node may be
+    completed from any thread; every completion path finishes
+    touching the node before resuming or destroying the coroutine,
+    because either act may end the node's storage.
+
+    The node owns no resources: the stop token is borrowed from the
+    awaiting chain's `io_env` (which outlives the suspension) and
+    the stop callback is managed manually in `cb_buf_`, destroyed on
+    every completion path before the frame can die.
+*/
+struct BOOST_COROSIO_SYMBOL_VISIBLE waiter_node
+    : intrusive_list<waiter_node>::node
+{
+    // Embedded completion op — avoids heap allocation per fire/cancel.
+    // Members are exported and defined non-inline in timer.cpp: the
+    // inline waiter_node constructor references do_complete and the
+    // vtable from translation units that reach this header through
+    // delay.hpp without ever including timer_service.hpp, so the one
+    // strong definition must live in a TU that is always linked.
+    struct BOOST_COROSIO_SYMBOL_VISIBLE completion_op final : scheduler_op
+    {
+        waiter_node* waiter_ = nullptr;
+
+        BOOST_COROSIO_DECL
+        static void do_complete(
+            void* owner, scheduler_op* base, std::uint32_t, std::uint32_t);
+
+        completion_op() noexcept : scheduler_op(&do_complete) {}
+
+        BOOST_COROSIO_DECL void operator()() override;
+        BOOST_COROSIO_DECL void destroy() override;
+    };
+
+    // Per-waiter stop_token cancellation
+    struct canceller
+    {
+        waiter_node* waiter_;
+        BOOST_COROSIO_DECL void operator()() const;
+    };
+
+    using stop_cb_type = std::stop_callback<canceller>;
+
+    // nullptr once removed from timer's waiter list (concurrency marker)
+    /// The timer this waiter is published on, or `nullptr`.
+    timer::implementation* impl_ = nullptr;
+
+    /// The timer service that completes this waiter.
+    timer_service* svc_ = nullptr;
+
+    /// The suspended coroutine, destroyed by the shutdown drains.
+    std::coroutine_handle<> h_;
+
+    /// The continuation posted to resume the coroutine.
+    capy::continuation cont_;
+
+    /// The executor the continuation is posted through.
+    capy::executor_ref d_;
+
+    // Borrowed from the awaiting chain's io_env, which outlives the
+    // suspension; the node holds no owning state.
+    /// The stop token observed for cancellation.
+    std::stop_token const* token_ = nullptr;
+
+    /// The completion result read by `await_resume`.
+    std::error_code ec_;
+
+    /// The embedded completion op posted to the scheduler.
+    completion_op op_;
+
+    // stop_callback is neither movable nor assignable; construct it
+    // in place once the node is pinned on the coroutine frame, and
+    // destroy it manually on every completion path.
+    /// Storage for the armed stop callback.
+    alignas(stop_cb_type) unsigned char cb_buf_[sizeof(stop_cb_type)];
+
+    /// True while `cb_buf_` holds a live stop callback.
+    bool cb_active_ = false;
+
+    waiter_node() noexcept
+    {
+        op_.waiter_ = this;
+    }
+
+    // The embedded op self-points and the list hooks are published
+    // to other threads; the node never moves.
+    waiter_node(waiter_node const&)            = delete;
+    waiter_node& operator=(waiter_node const&) = delete;
+
+    /** Arm the stop callback.
+
+        @par Preconditions
+        `token_` is set.
+    */
+    void arm_stop_cb()
+    {
+        new (cb_buf_) stop_cb_type(*token_, canceller{this});
+        cb_active_ = true;
+    }
+
+    /// Destroy the stop callback if armed.
+    void reset_stop_cb() noexcept
+    {
+        if (cb_active_)
+        {
+            std::launder(reinterpret_cast<stop_cb_type*>(cb_buf_))
+                ->~stop_cb_type();
+            cb_active_ = false;
+        }
+    }
+};
+
+/** Awaitable returned by `timer::wait()`.
+
+    Carries the waiter node so a wait performs no allocation. The
+    awaitable is movable only before `await_suspend` publishes the
+    node (a move builds a fresh, quiescent node); afterwards it is
+    pinned on the coroutine frame until the wait completes.
+*/
+struct wait_awaitable
+{
+    timer& t_;
+    waiter_node w_;
+
+    explicit wait_awaitable(timer& t) noexcept : t_(t) {}
+
+    wait_awaitable(wait_awaitable&& o) noexcept : t_(o.t_) {}
+
+    wait_awaitable(wait_awaitable const&)            = delete;
+    wait_awaitable& operator=(wait_awaitable const&) = delete;
+    wait_awaitable& operator=(wait_awaitable&&)      = delete;
+
+    bool await_ready() const noexcept
+    {
+        return false;
+    }
+
+    // Cancellation surfaces through w_.ec_: the stop_token path in
+    // wait() completes the waiter with error::canceled written to
+    // it, so there is no separate token to consult here.
+    capy::io_result<> await_resume() const noexcept
+    {
+        return {w_.ec_};
+    }
+
+    auto await_suspend(std::coroutine_handle<> h, capy::io_env const* env)
+        -> std::coroutine_handle<>
+    {
+        auto& impl = t_.get();
+        w_.h_      = h;
+        w_.cont_.h = h;
+        w_.d_      = env->executor;
+
+        // Inline fast path: already expired and not in the heap
+        if (impl.already_expired())
+        {
+            w_.ec_ = {};
+            w_.d_.post(w_.cont_);
+            return std::noop_coroutine();
+        }
+
+        w_.token_ = &env->stop_token;
+        return impl.wait(w_);
+    }
+};
+
+inline wait_awaitable
+timer::wait()
+{
+    return wait_awaitable(*this);
+}
+
 } // namespace boost::corosio::detail
 
 #endif