diff --git a/src/Xrd/XrdPollE.icc b/src/Xrd/XrdPollE.icc
index 03fbb522329..fecc48ff190 100644
--- a/src/Xrd/XrdPollE.icc
+++ b/src/Xrd/XrdPollE.icc
@@ -199,9 +199,12 @@ void XrdPollE::remFD(XrdPollInfo &pInfo, unsigned int events)
 // It works out that ONESHOT mode or even CTL_MOD requests do not necessarily
 // prevent epoll_wait() from returning on an error event. So, we must manually
 // remove the fd from the set and assume the logic was actually correct. If it
-// wasn't then the client will eventually timeout and retry the request.
+// wasn't then the client will eventually timeout and retry the request. This
+// may cause a double remove; so we don't consider these an arror.
 //
-   if (epoll_ctl(PollDfd, EPOLL_CTL_DEL, pInfo.FD, &myEvents))
+   TRACEI(POLL, "Poller " <<PID <<" removing FD " <<pInfo.FD);
+   if (epoll_ctl(PollDfd, EPOLL_CTL_DEL, pInfo.FD, &myEvents)
+   && (errno != ENOENT || events != 0))
       {const char *why;
        char buff[96];
        if (events & (EPOLLHUP | EPOLLRDHUP)) why = "sever";