diff --git a/fs/vfs/fs_epoll.c b/fs/vfs/fs_epoll.c
index dc78f935d28c0..f61cf9f631e61 100644
--- a/fs/vfs/fs_epoll.c
+++ b/fs/vfs/fs_epoll.c
@@ -56,6 +56,7 @@ struct epoll_node_s
   epoll_data_t             data;
   bool                     notified;
   struct pollfd            pfd;
+  FAR struct file         *filep;
   FAR struct epoll_head_s *eph;
 };
 
@@ -198,7 +199,8 @@ static int epoll_do_close(FAR struct file *filep)
       nxmutex_destroy(&eph->lock);
       list_for_every_entry(&eph->setup, epn, epoll_node_t, node)
         {
-          poll_fdsetup(epn->pfd.fd, &epn->pfd, false);
+          file_poll(epn->filep, &epn->pfd, false);
+          fs_putfilep(epn->filep);
         }
 
       list_for_every_entry_safe(&eph->extend, epn, tmp, epoll_node_t, node)
@@ -302,11 +304,11 @@ static int epoll_setup(FAR epoll_head_t *eph)
 
       epn->notified    = false;
       epn->pfd.revents = 0;
-      ret = poll_fdsetup(epn->pfd.fd, &epn->pfd, true);
+      ret = file_poll(epn->filep, &epn->pfd, true);
       if (ret < 0)
         {
-          ferr("epoll setup failed, fd=%d, events=%08" PRIx32 ", ret=%d\n",
-               epn->pfd.fd, epn->pfd.events, ret);
+          ferr("epoll setup failed, filep=%p, events=%08" PRIx32 ", "
+               "ret=%d\n", epn->filep, epn->pfd.events, ret);
           break;
         }
 
@@ -356,7 +358,7 @@ static int epoll_teardown(FAR epoll_head_t *eph, FAR struct epoll_event *evs,
 
       /* Teradown all the notified fd */
 
-      poll_fdsetup(epn->pfd.fd, &epn->pfd, false);
+      file_poll(epn->filep, &epn->pfd, false);
       list_delete(&epn->node);
 
       if (epn->pfd.revents != 0 && i < maxevents)
@@ -488,6 +490,7 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
   eph = epoll_head_from_fd(epfd, &filep);
   if (eph == NULL)
     {
+      set_errno(EBADF);
       return ERROR;
     }
 
@@ -565,9 +568,17 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
         epn->pfd.cb      = epoll_default_cb;
         epn->pfd.revents = 0;
 
-        ret = poll_fdsetup(fd, &epn->pfd, true);
+        ret = fs_getfilep(fd, &epn->filep);
+        if (ret < 0)
+          {
+            list_add_tail(&eph->free, &epn->node);
+            goto err;
+          }
+
+        ret = file_poll(epn->filep, &epn->pfd, true);
         if (ret < 0)
           {
+            fs_putfilep(epn->filep);
             list_add_tail(&eph->free, &epn->node);
             goto err;
           }
@@ -581,7 +592,8 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
           {
             if (epn->pfd.fd == fd)
               {
-                poll_fdsetup(fd, &epn->pfd, false);
+                file_poll(epn->filep, &epn->pfd, false);
+                fs_putfilep(epn->filep);
                 list_delete(&epn->node);
                 list_add_tail(&eph->free, &epn->node);
                 goto out;
@@ -592,6 +604,7 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
           {
             if (epn->pfd.fd == fd)
               {
+                fs_putfilep(epn->filep);
                 list_delete(&epn->node);
                 list_add_tail(&eph->free, &epn->node);
                 goto out;
@@ -602,6 +615,7 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
           {
             if (epn->pfd.fd == fd)
               {
+                fs_putfilep(epn->filep);
                 list_delete(&epn->node);
                 list_add_tail(&eph->free, &epn->node);
                 goto out;
@@ -618,15 +632,14 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
               {
                 if (epn->pfd.events != (ev->events | POLLALWAYS))
                   {
-                    poll_fdsetup(fd, &epn->pfd, false);
+                    file_poll(epn->filep, &epn->pfd, false);
 
                     epn->notified    = false;
                     epn->data        = ev->data;
                     epn->pfd.events  = ev->events | POLLALWAYS;
-                    epn->pfd.fd      = fd;
                     epn->pfd.revents = 0;
 
-                    ret = poll_fdsetup(fd, &epn->pfd, true);
+                    ret = file_poll(epn->filep, &epn->pfd, true);
                     if (ret < 0)
                       {
                         goto err;
@@ -646,10 +659,9 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
                     epn->notified    = false;
                     epn->data        = ev->data;
                     epn->pfd.events  = ev->events | POLLALWAYS;
-                    epn->pfd.fd      = fd;
                     epn->pfd.revents = 0;
 
-                    ret = poll_fdsetup(fd, &epn->pfd, true);
+                    ret = file_poll(epn->filep, &epn->pfd, true);
                     if (ret < 0)
                       {
                         goto err;
@@ -670,10 +682,9 @@ int epoll_ctl(int epfd, int op, int fd, FAR struct epoll_event *ev)
                 epn->notified    = false;
                 epn->data        = ev->data;
                 epn->pfd.events  = ev->events | POLLALWAYS;
-                epn->pfd.fd      = fd;
                 epn->pfd.revents = 0;
 
-                ret = poll_fdsetup(fd, &epn->pfd, true);
+                ret = file_poll(epn->filep, &epn->pfd, true);
                 if (ret < 0)
                   {
                     goto err;
diff --git a/fs/vfs/fs_poll.c b/fs/vfs/fs_poll.c
index f309c655e9a75..f53f5609815b9 100644
--- a/fs/vfs/fs_poll.c
+++ b/fs/vfs/fs_poll.c
@@ -79,8 +79,24 @@ static inline void poll_teardown(FAR struct pollfd *fds, nfds_t nfds,
     {
       if (fds[i].fd >= 0)
         {
-          int status = poll_fdsetup(fds[i].fd, &fds[i], false);
-          if (status < 0)
+          FAR struct file *filep;
+          int ret;
+
+          ret = fs_getfilep(fds[i].fd, &filep);
+          if (ret >= 0)
+            {
+              ret = file_poll(filep, &fds[i], false);
+
+              /* Calling putfilep twice to ensure reference counting
+               * for filep remains consistent with its state for
+               * before the poll.
+               */
+
+              fs_putfilep(filep);
+              fs_putfilep(filep);
+            }
+
+          if (ret < 0)
             {
               fds[i].revents |= POLLERR;
             }
@@ -144,10 +160,26 @@ static inline int poll_setup(FAR struct pollfd *fds, nfds_t nfds,
 
       if (fds[i].fd >= 0)
         {
-          ret = poll_fdsetup(fds[i].fd, &fds[i], true);
+          FAR struct file *filep;
+          int num = i;
+
+          ret = fs_getfilep(fds[i].fd, &filep);
+          if (ret < 0)
+            {
+              num -= 1;
+            }
+          else
+            {
+              ret = file_poll(filep, &fds[i], true);
+            }
+
           if (ret < 0)
             {
-              poll_teardown(fds, i, &count);
+              if (num >= 0)
+                {
+                  poll_teardown(fds, num, &count);
+                }
+
               fds[i].revents |= POLLERR;
               fds[i].arg = NULL;
               fds[i].cb = NULL;
@@ -192,38 +224,6 @@ static void poll_cleanup(FAR void *arg)
  * Public Functions
  ****************************************************************************/
 
-/****************************************************************************
- * Name: poll_fdsetup
- *
- * Description:
- *   Configure (or unconfigure) one file/socket descriptor for the poll
- *   operation.  If fds and sem are non-null, then the poll is being setup.
- *   if fds and sem are NULL, then the poll is being torn down.
- *
- ****************************************************************************/
-
-int poll_fdsetup(int fd, FAR struct pollfd *fds, bool setup)
-{
-  FAR struct file *filep;
-  int ret;
-
-  /* Get the file pointer corresponding to this file descriptor */
-
-  ret = fs_getfilep(fd, &filep);
-  if (ret < 0)
-    {
-      return ret;
-    }
-
-  DEBUGASSERT(filep != NULL);
-
-  /* Let file_poll() do the rest */
-
-  ret = file_poll(filep, fds, setup);
-  fs_putfilep(filep);
-  return ret;
-}
-
 /****************************************************************************
  * Name: poll_default_cb
  *
@@ -309,9 +309,8 @@ void poll_notify(FAR struct pollfd **afds, int nfds, pollevent_t eventset)
  * Name: file_poll
  *
  * Description:
- *   Low-level poll operation based on struct file.  This is used both to (1)
- *   support detached file, and also (2) by poll_fdsetup() to perform all
- *   normal operations on file descriptors.
+ *   Low-level poll operation based on struct file.  This is used to
+ *   support detached file.
  *
  * Input Parameters:
  *   file     File structure instance
diff --git a/include/nuttx/fs/fs.h b/include/nuttx/fs/fs.h
index 156d67279a146..4e73f8b2c9741 100644
--- a/include/nuttx/fs/fs.h
+++ b/include/nuttx/fs/fs.h
@@ -1676,9 +1676,8 @@ int file_fcntl(FAR struct file *filep, int cmd, ...);
  * Name: file_poll
  *
  * Description:
- *   Low-level poll operation based on struct file.  This is used both to (1)
- *   support detached file, and also (2) by poll_fdsetup() to perform all
- *   normal operations on file descriptors.
+ *   Low-level poll operation based on struct file.  This is used to
+ *   support detached file.
  *
  * Input Parameters:
  *   file     File structure instance
diff --git a/include/sys/poll.h b/include/sys/poll.h
index 23690e8c13b62..4188ead31a900 100644
--- a/include/sys/poll.h
+++ b/include/sys/poll.h
@@ -154,7 +154,6 @@ int ppoll(FAR struct pollfd *fds, nfds_t nfds,
           FAR const struct timespec *timeout_ts,
           FAR const sigset_t *sigmask);
 
-int poll_fdsetup(int fd, FAR struct pollfd *fds, bool setup);
 void poll_default_cb(FAR struct pollfd *fds);
 void poll_notify(FAR struct pollfd **afds, int nfds, pollevent_t eventset);