Merge "tests: perf: measure coroutines performance" from Benny

"

This series improves perf measurements by
counting also the number of allocations, tasks,
and instructions per test iteration.

Also, it makes use of an existing mechanism
to normalize the results for inner-loops
by returning the number of inner loops from
parallel_for_each test cases that iterate over
a vector of elements.

Then tests are added for basic coroutines operations,
as well as the integration of coroutines and
parallel_for_each.

Test: coroutine_perf(release), future_util_perf(release)
"

* tag 'perf-coroutine-v5' of github.com:bhalevy/seastar:
  perf: add coroutine parallel_for_each test cases
  tests: perf: future_util: consider range size
  tests: perf: add coroutine perf test
  tests: perf: measure normalized allocs, tasks, and instructions

CMakeLists.txt

@@ -1051,7 +1051,8 @@ if (Seastar_INSTALL OR Seastar_TESTING)
   add_library(seastar_perf_testing
     src/testing/random.cc
     include/seastar/testing/perf_tests.hh
-    tests/perf/perf_tests.cc)
+    tests/perf/perf_tests.cc
+    tests/perf/linux_perf_event.cc)
   add_library (Seastar::seastar_perf_testing ALIAS seastar_perf_testing)
   target_compile_definitions (seastar_perf_testing
     PRIVATE ${Seastar_PRIVATE_COMPILE_DEFINITIONS})

include/seastar/testing/linux_perf_event.hh

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2021-present ScyllaDB
+ */
+
+/*
+ * This file is open source software, licensed to you under the terms
+ * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
+ * distributed with this work for additional information regarding copyright
+ * ownership.  You may not use this file except in compliance with the License.
+ *
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * This file was copied from Scylla (https://github.com/scylladb/scylla)
+ */
+
+#pragma once
+
+
+#include <cstdint>
+#include <utility>
+#include <unistd.h>
+
+struct perf_event_attr; // from <linux/perf_event.h>
+
+class linux_perf_event {
+    int _fd = -1;
+public:
+    linux_perf_event(const struct ::perf_event_attr& attr, pid_t pid, int cpu, int group_fd, unsigned long flags);
+    linux_perf_event(linux_perf_event&& x) noexcept : _fd(std::exchange(x._fd, -1)) {}
+    linux_perf_event& operator=(linux_perf_event&& x) noexcept;
+    ~linux_perf_event();
+    uint64_t read();
+    void enable();
+    void disable();
+public:
+    static linux_perf_event user_instructions_retired();
+};
+

include/seastar/testing/perf_tests.hh

@@ -28,6 +28,11 @@
 
 #include <seastar/core/future.hh>
 #include <seastar/core/loop.hh>
+#include <seastar/testing/linux_perf_event.hh>
+
+#ifdef SEASTAR_COROUTINES_ENABLED
+#include <seastar/core/coroutine.hh>
+#endif
 
 using namespace seastar;
 
@@ -38,14 +43,86 @@ struct config;
 
 using clock_type = std::chrono::steady_clock;
 
+class perf_stats {
+public:
+    uint64_t allocations = 0;
+    uint64_t tasks_executed = 0;
+    uint64_t instructions_retired = 0;
+
+private:
+    static uint64_t perf_mallocs();
+    static uint64_t perf_tasks_processed();
+
+public:
+    perf_stats() = default;
+    perf_stats(uint64_t allocations_, uint64_t tasks_executed_, uint64_t instructions_retired_ = 0)
+        : allocations(allocations_)
+        , tasks_executed(tasks_executed_)
+        , instructions_retired(instructions_retired_)
+    {}
+    perf_stats(perf_stats&& o) noexcept
+        : allocations(std::exchange(o.allocations, 0))
+        , tasks_executed(std::exchange(o.tasks_executed, 0))
+        , instructions_retired(std::exchange(o.instructions_retired, 0))
+    {}
+    perf_stats(const perf_stats& o) = default;
+
+    perf_stats& operator=(perf_stats&& o) = default;
+    perf_stats& operator=(const perf_stats& o) = default;
+
+    perf_stats& operator+=(perf_stats b);
+    perf_stats& operator-=(perf_stats b);
+
+    static perf_stats snapshot(linux_perf_event* instructions_retired_counter = nullptr);
+};
+
+inline
+perf_stats
+operator+(perf_stats a, perf_stats b) {
+    a.allocations += b.allocations;
+    a.tasks_executed += b.tasks_executed;
+    a.instructions_retired += b.instructions_retired;
+    return a;
+}
+
+inline
+perf_stats
+operator-(perf_stats a, perf_stats b) {
+    a.allocations -= b.allocations;
+    a.tasks_executed -= b.tasks_executed;
+    a.instructions_retired -= b.instructions_retired;
+    return a;
+}
+
+inline perf_stats& perf_stats::operator+=(perf_stats b) {
+    allocations += b.allocations;
+    tasks_executed += b.tasks_executed;
+    instructions_retired += b.instructions_retired;
+    return *this;
+}
+
+inline perf_stats& perf_stats::operator-=(perf_stats b) {
+    allocations -= b.allocations;
+    tasks_executed -= b.tasks_executed;
+    instructions_retired -= b.instructions_retired;
+    return *this;
+}
+
 class performance_test {
     std::string _test_case;
     std::string _test_group;
 
     uint64_t _single_run_iterations = 0;
     std::atomic<uint64_t> _max_single_run_iterations;
+protected:
+    linux_perf_event _instructions_retired_counter = linux_perf_event::user_instructions_retired();
 private:
     void do_run(const config&);
+public:
+    struct run_result {
+        clock_type::duration duration;
+        perf_stats stats;
+    };
 protected:
     [[gnu::always_inline]] [[gnu::hot]]
     bool stop_iteration() const {
@@ -59,7 +136,7 @@ protected:
 
     virtual void set_up() = 0;
     virtual void tear_down() noexcept = 0;
-    virtual future<clock_type::duration> do_single_run() = 0;
+    virtual future<run_result> do_single_run() = 0;
 public:
     performance_test(const std::string& test_case, const std::string& test_group)
         : _test_case(test_case)
@@ -86,33 +163,47 @@ class time_measurement {
     clock_type::time_point _run_start_time;
     clock_type::time_point _start_time;
     clock_type::duration _total_time;
+
+    perf_stats _start_stats;
+    perf_stats _total_stats;
+
 public:
     [[gnu::always_inline]] [[gnu::hot]]
-    void start_run() {
+    void start_run(linux_perf_event* instructions_retired_counter = nullptr) {
         _total_time = { };
         auto t = clock_type::now();
         _run_start_time = t;
         _start_time = t;
+        _start_stats = perf_stats::snapshot(instructions_retired_counter);
     }
 
     [[gnu::always_inline]] [[gnu::hot]]
-    clock_type::duration stop_run() {
+    performance_test::run_result stop_run(linux_perf_event* instructions_retired_counter = nullptr) {
         auto t = clock_type::now();
+        performance_test::run_result ret;
         if (_start_time == _run_start_time) {
-            return t - _start_time;
+            ret.duration = t - _start_time;
+            auto stats = perf_stats::snapshot(instructions_retired_counter);
+            ret.stats = stats - _start_stats;
+        } else {
+            ret.duration = _total_time;
+            ret.stats = _total_stats;
         }
-        return _total_time;
+        return ret;
     }
 
     [[gnu::always_inline]] [[gnu::hot]]
-    void start_iteration() {
+    void start_iteration(linux_perf_event* instructions_retired_counter = nullptr) {
         _start_time = clock_type::now();
+        _start_stats = perf_stats::snapshot(instructions_retired_counter);
     }
 
     [[gnu::always_inline]] [[gnu::hot]]
-    void stop_iteration() {
+    void stop_iteration(linux_perf_event* instructions_retired_counter = nullptr) {
         auto t = clock_type::now();
         _total_time += t - _start_time;
+        auto stats = perf_stats::snapshot(instructions_retired_counter);
+        _total_stats += stats - _start_stats;
     }
 };
 
@@ -162,10 +253,11 @@ protected:
     }
 
     [[gnu::hot]]
-    virtual future<clock_type::duration> do_single_run() override {
+    virtual future<run_result> do_single_run() override {
         // Redundant 'this->'s courtesy of https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61636
+        _instructions_retired_counter.enable();
         return if_constexpr_<is_future<decltype(_test->run())>::value>([&] (auto&&...) {
-            measure_time.start_run();
+            measure_time.start_run(&_instructions_retired_counter);
             return do_until([this] { return this->stop_iteration(); }, [this] {
                 return if_constexpr_<std::is_same<decltype(_test->run()), future<>>::value>([&] (auto&&...) {
                     this->next_iteration(1);
@@ -177,11 +269,13 @@ protected:
                         this->next_iteration(n);
                     });
                 })();
-            }).then([] {
-                return measure_time.stop_run();
+            }).then([this] {
+                return measure_time.stop_run(&_instructions_retired_counter);
+            }).finally([this] {
+                _instructions_retired_counter.disable();
             });
         }, [&] (auto&&...) {
-            measure_time.start_run();
+            measure_time.start_run(&_instructions_retired_counter);
             while (!stop_iteration()) {
                 if_constexpr_<std::is_void<decltype(_test->run())>::value>([&] (auto&&...) {
                     (void)_test->run();
@@ -192,7 +286,9 @@ protected:
                     this->next_iteration(run_test(dependency...));
                 })();
             }
-            return make_ready_future<clock_type::duration>(measure_time.stop_run());
+            auto ret = measure_time.stop_run(&_instructions_retired_counter);
+            _instructions_retired_counter.disable();
+            return make_ready_future<run_result>(std::move(ret));
         })();
     }
 public:
@@ -234,6 +330,8 @@ void do_not_optimize(const T& v)
 
 // PERF_TEST and PERF_TEST_F support both synchronous and asynchronous functions.
 // The former should return `void`, the latter `future<>`.
+// PERF_TEST_C executes a coroutine function, if enabled.
+// PERF_TEST_CN executes a coroutine function, if enabled, returning the number of inner-loops.
 //
 // Test cases may perform multiple operations in a single run, this may be desirable
 // if the cost of an individual operation is very small. This allows measuring either
@@ -257,3 +355,23 @@ void do_not_optimize(const T& v)
     static ::perf_tests::internal::test_registrar<test_##test_group##_##test_case> \
     test_##test_group##_##test_case##_registrar(#test_group, #test_case); \
     [[gnu::always_inline]] auto test_##test_group##_##test_case::run()
+
+#ifdef SEASTAR_COROUTINES_ENABLED
+
+#define PERF_TEST_C(test_group, test_case) \
+    struct test_##test_group##_##test_case : test_group { \
+        [[gnu::always_inline]] inline future<> run(); \
+    }; \
+    static ::perf_tests::internal::test_registrar<test_##test_group##_##test_case> \
+    test_##test_group##_##test_case##_registrar(#test_group, #test_case); \
+    [[gnu::always_inline]] future<> test_##test_group##_##test_case::run()
+
+#define PERF_TEST_CN(test_group, test_case) \
+    struct test_##test_group##_##test_case : test_group { \
+        [[gnu::always_inline]] inline future<size_t> run(); \
+    }; \
+    static ::perf_tests::internal::test_registrar<test_##test_group##_##test_case> \
+    test_##test_group##_##test_case##_registrar(#test_group, #test_case); \
+    [[gnu::always_inline]] future<size_t> test_##test_group##_##test_case::run()
+
+#endif // SEASTAR_COROUTINES_ENABLED

tests/perf/CMakeLists.txt

@@ -76,3 +76,6 @@ seastar_add_test (rpc
 seastar_add_test (smp_submit_to
   SOURCES smp_submit_to_perf.cc
   NO_SEASTAR_PERF_TESTING_LIBRARY)
+
+seastar_add_test (coroutine
+  SOURCES coroutine_perf.cc)

tests/perf/coroutine_perf.cc

@@ -0,0 +1,52 @@
+/*
+ * This file is open source software, licensed to you under the terms
+ * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
+ * distributed with this work for additional information regarding copyright
+ * ownership.  You may not use this file except in compliance with the License.
+ *
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Copyright (C) 2022-present ScyllaDB
+ */
+
+#include <seastar/testing/perf_tests.hh>
+
+#ifdef SEASTAR_COROUTINES_ENABLED
+
+#include <seastar/core/coroutine.hh>
+#include <seastar/coroutine/maybe_yield.hh>
+
+struct coroutine_test {
+};
+
+PERF_TEST_C(coroutine_test, empty)
+{
+    co_return;
+}
+
+PERF_TEST_C(coroutine_test, without_preemption_check)
+{
+    co_await coroutine::without_preemption_check(make_ready_future<>());
+}
+
+PERF_TEST_C(coroutine_test, ready)
+{
+    co_await make_ready_future<>();
+}
+
+PERF_TEST_C(coroutine_test, maybe_yield)
+{
+    co_await coroutine::maybe_yield();
+}
+
+#endif // SEASTAR_COROUTINES_ENABLED