[V4] Small object optimizations

benchmark/lib/CMakeLists.txt

@@ -9,7 +9,6 @@ target_sources(benchmark_common
     FILE_SET HEADERS
     BASE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}
     FILES
-      common.hpp
       fib.hpp
       uts.hpp
       macros.hpp

benchmark/lib/macros.hpp

@@ -2,12 +2,17 @@
 
 #include <benchmark/benchmark.h>
 
-#include "common.hpp"
-
+// Use `import std;` by default. Textually `#include <thread>` drags in
+// `<stop_token>`, which triggers a libc++ 22 link-time bug (undefined
+// `__atomic_unique_lock::__set_locked_bit`) in TUs that later instantiate
+// anything touching std::stop_*. Targets that can't use modules (e.g. the
+// openmp benchmarks, see benchmark/src/openmp/CMakeLists.txt) define
+// LF_BENCH_NO_IMPORT_STD and get textual includes instead.
 #ifdef LF_BENCH_NO_IMPORT_STD
   #include <algorithm>
   #include <cstdint>
   #include <string>
+  #include <thread>
 #else
 import std;
 #endif
@@ -16,6 +21,16 @@ import std;
 
 namespace lf_bench {
 
+inline void bench_thread_args(benchmark::Benchmark *bench, auto make_args) {
+  unsigned hw = std::max(1U, std::thread::hardware_concurrency());
+  for (unsigned t : {1U, 2U, 4U, 6U, 8U, 12U, 16U, 24U, 32U, 48U, 64U, 96U}) {
+    if (t > hw) {
+      return;
+    }
+    make_args(bench, t);
+  }
+}
+
 inline auto sanitize(std::string s) -> std::string {
   s.erase(std::remove(s.begin(), s.end(), ' '), s.end());
   return s;

benchmark/src/baremetal/fib.cpp

@@ -1,6 +1,5 @@
 #include <benchmark/benchmark.h>
 
-#include "common.hpp"
 #include "fib.hpp"
 #include "macros.hpp"
 

benchmark/src/libfork/fib.cpp

@@ -1,6 +1,5 @@
 #include <benchmark/benchmark.h>
 
-#include "common.hpp"
 #include "fib.hpp"
 
 #include "helpers.hpp"

benchmark/src/libfork/uts.cpp

@@ -1,6 +1,5 @@
 #include <benchmark/benchmark.h>
 
-#include "common.hpp"
 #include "uts.hpp"
 
 #include "helpers.hpp"

benchmark/src/serial/fib.cpp

@@ -1,6 +1,5 @@
 #include <benchmark/benchmark.h>
 
-#include "common.hpp"
 #include "fib.hpp"
 #include "macros.hpp"
 

benchmark/src/serial/uts.cpp

@@ -1,6 +1,5 @@
 #include <benchmark/benchmark.h>
 
-#include "common.hpp"
 #include "macros.hpp"
 #include "uts.hpp"
 

src/core/execute.cxx

@@ -35,7 +35,7 @@ constexpr void execute(Context &context, sched_handle<Context> handle) {
 
   thread_local_context<Context> = std::addressof(context);
 
-  defer _ = [] noexcept -> void {
+  defer _ = [] static noexcept -> void {
     thread_local_context<Context> = nullptr;
   };
 
@@ -60,7 +60,7 @@ constexpr void execute(Context &context, steal_handle<Context> handle) {
 
   thread_local_context<Context> = std::addressof(context);
 
-  defer _ = [] noexcept -> void {
+  defer _ = [] static noexcept -> void {
     thread_local_context<Context> = nullptr;
   };
 

src/core/ops.cxx

@@ -16,6 +16,25 @@ namespace lf {
 struct no_stop_t {};
 struct no_ret_t {};
 
+// =============== Value-or-reference storage policy =============== //
+
+// For rvalue-reference arguments that are trivially copyable and fit in two
+// pointer-sized words, store by value inside pkg instead of keeping a reference.
+// This lets [[no_unique_address]] collapse empty functors to zero bytes and
+// allows the compiler to treat the stored values as local data (no aliasing).
+template <typename T>
+concept small_trivially_copyable = !std::is_reference_v<T>                     //
+                                   && std::is_trivially_copyable_v<T>          //
+                                   && sizeof(T) <= 2 * sizeof(void *)          //
+                                   && alignof(T) <= alignof(std::max_align_t); //
+
+// Only collapses rvalue refs; lvalue refs are kept as-is to preserve reference semantics.
+template <typename T>
+using store_as_t =
+    std::conditional_t<std::is_rvalue_reference_v<T> && small_trivially_copyable<std::remove_cvref_t<T>>,
+                       std::remove_cvref_t<T>,
+                       T>;
+
 // clang-format off
 
 template <category Cat, bool StopToken, typename Context, typename R, typename Fn, typename... Args>
@@ -29,20 +48,21 @@ struct [[nodiscard("You should immediately co_await this!")]] pkg {
 // clang-format on
 
 /**
- * @brief Forward the function member of a pkg correctly
+ * @brief Forward the function member of a pkg correctly.
  *
- * The Fn member should be an l/r value reference, r-value reference need an
- * explicit move to be forwarded correctly.
+ * Handles three cases:
+ *  - rvalue reference Fn: move it.
+ *  - lvalue reference Fn: return by reference.
+ *  - value type Fn (small trivially-copyable stored directly): return by value.
  */
 template <typename Fn>
 constexpr auto fwd_fn(auto &&fn) noexcept -> Fn {
-
-  static_assert(std::is_reference_v<Fn>);
-
   if constexpr (std::is_rvalue_reference_v<Fn>) {
     return std::move(fn);
-  } else {
+  } else if constexpr (std::is_lvalue_reference_v<Fn> || small_trivially_copyable<Fn>) {
     return fn;
+  } else {
+    static_assert(false, "Invalid Fn type in fwd_fn");
   }
 }
 
@@ -68,10 +88,10 @@ template <typename Context>
 struct scope_ops : scope_base {
  private:
   template <typename R, typename Fn, typename... Args>
-  using call_pkg = pkg<category::call, false, Context, R, Fn &&, Args &&...>;
+  using call_pkg = pkg<category::call, false, Context, R, store_as_t<Fn &&>, store_as_t<Args &&>...>;
 
   template <typename R, typename Fn, typename... Args>
-  using fork_pkg = pkg<category::fork, false, Context, R, Fn &&, Args &&...>;
+  using fork_pkg = pkg<category::fork, false, Context, R, store_as_t<Fn &&>, store_as_t<Args &&>...>;
 
  public:
   // Default constructible
@@ -137,10 +157,10 @@ template <typename Context>
 struct child_scope_ops : scope_base, stop_source {
  private:
   template <typename R, typename Fn, typename... Args>
-  using call_pkg = pkg<category::call, true, Context, R, Fn &&, Args &&...>;
+  using call_pkg = pkg<category::call, true, Context, R, store_as_t<Fn &&>, store_as_t<Args &&>...>;
 
   template <typename R, typename Fn, typename... Args>
-  using fork_pkg = pkg<category::fork, true, Context, R, Fn &&, Args &&...>;
+  using fork_pkg = pkg<category::fork, true, Context, R, store_as_t<Fn &&>, store_as_t<Args &&>...>;
 
  public:
   /**

src/core/promise.cxx

@@ -504,9 +504,6 @@ struct mixin_frame {
   await_transform_pkg(this auto const &self, pkg<Cat, StopToken, Context, R, Fn, Args...> &&pkg) noexcept(
       async_nothrow_invocable<Fn, Context, Args...>) -> awaitable<Cat, Context> {
 
-    // Required for noexcept specifier to be correct
-    static_assert(std::is_reference_v<Fn> && (... && std::is_reference_v<Args>));
-
     using U = async_result_t<Fn, Context, Args...>;
 
     // clang-format off

src/core/thread_locals.cxx

@@ -12,7 +12,7 @@ namespace lf {
 template <worker_context Context>
 constinit inline thread_local Context *thread_local_context = nullptr;
 
-// TODO: implictaions of thread local on constexpr
+// TODO: implications of thread local on constexpr
 
 /**
  * @brief A getter for the current worker context, checks for null in debug.
@@ -22,6 +22,9 @@ constexpr auto get_tls_context() noexcept -> Context & {
   return *not_null(thread_local_context<Context>);
 }
 
+/**
+ * @brief A getter for the current worker context's stack, checks for null in debug.
+ */
 template <worker_context Context>
 constexpr auto get_tls_stack() noexcept -> stack_t<Context> & {
   return get_tls_context<Context>().stack();