Don't set O_NONBLOCK on sockets used for tests.

This fixes a problem where every thread would essentially burn
a CPU core busy-waiting, when it didn't need to. It's believed
that this excess CPU usage might contribute to packet loss and
poor performance.

Non-blocking sockets were a necessity with the original single-
thread process model of iperf3, in order to allow for
concurrency between different test streams. With multiple
threads, this is no longer necesary (it's perfectly fine for a
thread to block on I/O, as it only services a single test stream
and won't affect any others).

Problem pointed out by @bltierney. Code review by @swlars.

Fixes IPERF-177.

src/iperf_client_api.c

@@ -662,12 +662,6 @@ iperf_run_client(struct iperf_test * test)
                     goto cleanup_and_fail;
                 }
 
-		// Set non-blocking for non-UDP tests
-		if (test->protocol->id != Pudp) {
-		    SLIST_FOREACH(sp, &test->streams, streams) {
-			setnonblocking(sp->socket, 1);
-		    }
-		}
 	    }
 
             /* Run the timers. */
@@ -706,13 +700,6 @@ iperf_run_client(struct iperf_test * test)
                     iperf_printf(test, "Sender threads stopped\n");
                 }
 
-		// Unset non-blocking for non-UDP tests
-		if (test->protocol->id != Pudp) {
-		    SLIST_FOREACH(sp, &test->streams, streams) {
-			setnonblocking(sp->socket, 0);
-		    }
-		}
-
 		/* Yes, done!  Send TEST_END. */
 		test->done = 1;
 		cpu_util(test->cpu_util);

src/iperf_server_api.c

@@ -757,17 +757,6 @@ iperf_run_server(struct iperf_test *test)
 
                             if (s > test->max_fd) test->max_fd = s;
 
-                            /*
-                             * If the protocol isn't UDP, or even if it is but
-                             * we're the receiver, set nonblocking sockets.
-                             * We need this to allow a server receiver to
-                             * maintain interactivity with the control channel.
-                             */
-                            if (test->protocol->id != Pudp ||
-                                !sp->sender) {
-                                setnonblocking(s, 1);
-                            }
-
                             if (test->on_new_stream)
                                 test->on_new_stream(sp);
 

src/net.c

@@ -1,5 +1,5 @@
 /*
- * iperf, Copyright (c) 2014-2019, The Regents of the University of
+ * iperf, Copyright (c) 2014-2023, The Regents of the University of
  * California, through Lawrence Berkeley National Laboratory (subject
  * to receipt of any required approvals from the U.S. Dept. of
  * Energy).  All rights reserved.
@@ -405,6 +405,7 @@ Nread(int fd, char *buf, size_t count, int prot)
     while (nleft > 0) {
         r = read(fd, buf, nleft);
         if (r < 0) {
+            /* XXX EWOULDBLOCK can't happen without non-blocking sockets */
             if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
                 break;
             else
@@ -469,6 +470,7 @@ Nwrite(int fd, const char *buf, size_t count, int prot)
 		case EINTR:
 		case EAGAIN:
 #if (EAGAIN != EWOULDBLOCK)
+                    /* XXX EWOULDBLOCK can't happen without non-blocking sockets */
 		case EWOULDBLOCK:
 #endif
 		return count - nleft;
@@ -539,6 +541,7 @@ Nsendfile(int fromfd, int tofd, const char *buf, size_t count)
 		case EINTR:
 		case EAGAIN:
 #if (EAGAIN != EWOULDBLOCK)
+                    /* XXX EWOULDBLOCK can't happen without non-blocking sockets */
 		case EWOULDBLOCK:
 #endif
 		if (count == nleft)