Make sure the wakePend flag does not incur undefined behavior.

src/XrdSys/XrdSysAtomics.hh

@@ -59,3 +59,24 @@
 #define AtomicZAP(x)        x = 0
 #endif
 #endif
+
+/*
+ * The following definitions give a mechanism for using C++ atomics
+ * (when using at least C++11).  *Note* that these can't be relied
+ * on for correct behavior as they are non-atomic for C++03 compilers.
+ *
+ * Only use them for standards correctness (eliminating C++11 undefined
+ * behavior).
+ */
+#if __cplusplus >= 201103L
+#include <atomic>
+#define CPP_ATOMIC_LOAD(x, order)       x.load(order)
+#define CPP_ATOMIC_STORE(x, val, order) x.store(val, order)
+#define CPP_ATOMIC_TYPE(kind)           std::atomic<kind>
+#else
+#define CPP_ATOMIC_LOAD(x, order)       x
+#define CPP_ATOMIC_STORE(x, val, order) x = val
+#define CPP_ATOMIC_TYPE(kind)           kind
+#endif
+
+

src/XrdSys/XrdSysIOEvents.cc

@@ -434,7 +434,8 @@ bool XrdSys::IOEvents::Channel::Enable(int events, int timeout,
 // that we cannot hold the channel mutex for this call because it may wait.
 //
    if (isLocked) chMutex.UnLock();
-   if (retval && !(chPollXQ->wakePend) && setTO && isLocked) chPollXQ->WakeUp();
+   bool isWakePend = CPP_ATOMIC_LOAD(chPollXQ->wakePend, std::memory_order_consume);
+   if (retval && !isWakePend && setTO && isLocked) chPollXQ->WakeUp();
 
 // All done
 //
@@ -1142,7 +1143,7 @@ int XrdSys::IOEvents::Poller::TmoGet()
 
 // Return the value
 //
-   wakePend = false;
+   CPP_ATOMIC_STORE(wakePend, false, std::memory_order_release);
    toMutex.UnLock();
    return wtval;
 }
@@ -1158,9 +1159,19 @@ void XrdSys::IOEvents::Poller::WakeUp()
 // Send it off to wakeup the poller thread, but only if here is no wakeup in
 // progress.
 //
+// We use a mutex here because we want to produce a synchronization point - all
+// threads that might be interested timeouts and wakeups are going to incur a
+// cache bounce for the page where wakePend resides; they will see a consistent
+// view of the wakePend flag.  For those threads, this is equivalent to
+// an atomic with memory_order std::memory_order_seq_cst (the strongest ordering).
+// However, the threads that are not interested in timeouts will not get a flush
+// for their copy of the wakePend page.  They will still have the weaker memory
+// ordering of consume/release (which is guaranteed anyway on all current architectures
+// except for DEC Alpha).
    toMutex.Lock();
-   if (wakePend) toMutex.UnLock();
-      else {wakePend = true;
+   bool isWakePend = CPP_ATOMIC_LOAD(wakePend, std::memory_order_consume);
+   if (isWakePend) {toMutex.UnLock();}
+      else {CPP_ATOMIC_STORE(wakePend, true, std::memory_order_release);
             toMutex.UnLock();
             SendCmd(cmdbuff);
            }

src/XrdSys/XrdSysIOEvents.hh

@@ -35,6 +35,7 @@
 #include <sys/types.h>
 
 #include "XrdSys/XrdSysPthread.hh"
+#include "XrdSys/XrdSysAtomics.hh"
 
 //-----------------------------------------------------------------------------
 //! IOEvents
@@ -506,7 +507,7 @@ PipeData        reqBuff;    // Buffer used by poller thread to recv data
 char           *pipeBuff;   // Read resumption point in buffer
 int             pipeBlen;   // Number of outstanding bytes
 char            tmoMask;    // Timeout mask
-bool            wakePend;   // Wakeup is effectively pending (don't send)
+CPP_ATOMIC_TYPE(bool) wakePend;   // Wakeup is effectively pending (don't send)
 bool            chDead;     // True if channel deleted by callback
 
 static time_t   maxTime;    // Maximum time allowed

src/XrdSys/XrdSysIOEventsPollE.icc

@@ -207,8 +207,9 @@ void XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore *syncsem,
 //
    do {do {numpolled = epoll_wait(pollDfd, pollTab, pollMax, TmoGet());}
           while (numpolled < 0 && errno == EINTR);
-       wakePend = true; numPoll = numpolled;
-            if (numpolled == 0) CbkTMO();
+       CPP_ATOMIC_STORE(wakePend, true, std::memory_order_release);
+       numPoll = numpolled;
+       if (numpolled == 0) CbkTMO();
        else if (numpolled <  0)
                {int rc = errno;
                 cerr <<"EPoll: " <<strerror(rc) <<" polling for events" <<endl;