diff --git a/src/frontend/mosh-server.cc b/src/frontend/mosh-server.cc
index 97e652164..b118daa8e 100644
--- a/src/frontend/mosh-server.cc
+++ b/src/frontend/mosh-server.cc
@@ -510,8 +510,6 @@ void serve( int host_fd, Terminal::Complete &terminal, ServerConnection &network
 {
   /* prepare to poll for events */
   Select &sel = Select::get_instance();
-  sel.add_fd( network.fd() );
-  sel.add_fd( host_fd );
   sel.add_signal( SIGTERM );
   sel.add_signal( SIGINT );
 
@@ -534,6 +532,11 @@ void serve( int host_fd, Terminal::Complete &terminal, ServerConnection &network
         timeout = min( timeout, timeout_if_no_client );
       }
 
+      /* poll for events */
+      sel.clear_fds();
+      sel.add_fd( network.fd() );
+      sel.add_fd( host_fd );
+
       int active_fds = sel.select( timeout );
       if ( active_fds < 0 ) {
 	perror( "select" );
diff --git a/src/frontend/stmclient.cc b/src/frontend/stmclient.cc
index 2b69f7b84..18c9dbe9a 100644
--- a/src/frontend/stmclient.cc
+++ b/src/frontend/stmclient.cc
@@ -309,8 +309,6 @@ void STMClient::main( void )
 
   /* prepare to poll for events */
   Select &sel = Select::get_instance();
-  sel.add_fd( network->fd() );
-  sel.add_fd( STDIN_FILENO );
 
   while ( 1 ) {
     try {
@@ -323,6 +321,12 @@ void STMClient::main( void )
 	wait_time = min( 250, wait_time );
       }
 
+      /* poll for events */
+      /* network->fd() can in theory change over time */
+      sel.clear_fds();
+      sel.add_fd( network->fd() );
+      sel.add_fd( STDIN_FILENO );
+
       int active_fds = sel.select( wait_time );
       if ( active_fds < 0 ) {
 	perror( "select" );
diff --git a/src/util/select.h b/src/util/select.h
index a0168e3bb..dc0671650 100644
--- a/src/util/select.h
+++ b/src/util/select.h
@@ -93,6 +93,11 @@ class Select {
     FD_SET( fd, &all_fds );
   }
 
+  void clear_fds( void )
+  {
+    FD_ZERO( &all_fds );
+  }
+
   void add_signal( int signum )
   {
     fatal_assert( signum >= 0 );