reactor: avoid issuing systemwide memory barriers in parallel

Before going to sleep, a reactor must force a memory barrier across
the entire system, so that other reactors can tell it is asleep and
wake it up when necessary.

This operation, however, is slow on large machines, and very unscalable,
so that if many reactors fall asleep concurrently, the result is quite
long latencies, in the tens of milliseconds.

Fix by noticing that another reactor is falling asleep, and aborting the
sleep attempt in that case.

Message-Id: <1460657435-22990-3-git-send-email-avi@scylladb.com>

core/reactor.cc

@@ -1667,14 +1667,31 @@ class reactor::lowres_timer_pollfn final : public reactor::pollfn {
 
 class reactor::smp_pollfn final : public reactor::pollfn {
     reactor& _r;
+    struct aligned_flag {
+        std::atomic<bool> flag;
+        char pad[63];
+        bool try_lock() {
+            return !flag.exchange(true, std::memory_order_relaxed);
+        }
+        void unlock() {
+            flag.store(false, std::memory_order_relaxed);
+        }
+    };
+    static aligned_flag _membarrier_lock;
 public:
     smp_pollfn(reactor& r) : _r(r) {}
     virtual bool poll() final override {
         return smp::poll_queues();
     }
     virtual bool try_enter_interrupt_mode() override {
+        // systemwide_memory_barrier() is very slow if run concurrently,
+        // so don't go to sleep if it is running now.
+        if (!_membarrier_lock.try_lock()) {
+            return false;
+        }
         _r._sleeping.store(true, std::memory_order_relaxed);
         systemwide_memory_barrier();
+        _membarrier_lock.unlock();
         if (poll()) {
             // raced
             _r._sleeping.store(false, std::memory_order_relaxed);
@@ -1687,6 +1704,8 @@ class reactor::smp_pollfn final : public reactor::pollfn {
     }
 };
 
+alignas(64) reactor::smp_pollfn::aligned_flag reactor::smp_pollfn::_membarrier_lock;
+
 class reactor::epoll_pollfn final : public reactor::pollfn {
     reactor& _r;
 public: