SO_REUSEADDR causes system-allocated ports to overlap

[tamird]

On Linux, the behavior of SO_REUSEADDR is ignored when the system is allocating a port. That is:

struct sockaddr_in addr = {
    .sin_family = AF_INET,
    .sin_addr.s_addr = htonl(INADDR_LOOPBACK),
    .sin_port = htons(31337),
};

int fd1 = socket(AF_INET, SOCK_STREAM, 0);
setsockopt(fd1, SOL_SOCKET, SO_REUSEADDR, 1, sizeof(int));
bind(fd1, (const struct sockaddr*)(&addr), sizeof(addr));

int fd2 = socket(AF_INET, SOCK_STREAM, 0);
setsockopt(fd2, SOL_SOCKET, SO_REUSEADDR, 1, sizeof(int));
addr.sin_port = 0;
bind(fd2, (const struct sockaddr*)(&addr), sizeof(addr));

socklen_t addrlen = sizeof(addr);
getsockname(fd2, (struct sockaddr*)(&addr), &addrlen);
ASSERT(ntohs(addr.sin_port) != 31337);

In other words, Linux will never allocate a port that is held by another socket, regardless of SO_REUSEADDR. The same is not true on gvisor, and can cause concurrent bind calls (with SO_REUSEADDR) to receive the same system-allocated port, causing one of them to later file on a call to listen with EADDRINUSE.

[tamird]

Ready-made unit test from https://fuchsia-review.googlesource.com/c/fuchsia/+/545286:

TEST(LocalhostTest, BindZeroPortIgnoresReuseAddr) {
  const struct sockaddr_in addr = {
      .sin_family = AF_INET,
      .sin_addr.s_addr = htonl(INADDR_LOOPBACK),
  };

  // This set is not strictly necessary for the correctness of this test, but it greatly reduces the
  // running time in the failure case.
  std::unordered_set<in_port_t> ports;
  for (in_port_t i = 0; i <= std::numeric_limits<in_port_t>::max(); i++) {
    fbl::unique_fd fd;
    ASSERT_TRUE(fd = fbl::unique_fd(socket(AF_INET, SOCK_STREAM, 0))) << strerror(errno);

    constexpr int kOne = 1;
    ASSERT_EQ(setsockopt(fd.get(), SOL_SOCKET, SO_REUSEADDR, &kOne, sizeof(kOne)), 0)
        << strerror(errno);

    switch (bind(fd.get(), reinterpret_cast<const struct sockaddr*>(&addr), sizeof(addr))) {
      default:
        // All ephemeral ports are in TIME_WAIT.
        ASSERT_EQ(errno, EADDRINUSE) << strerror(errno);
        return;
      case 0: {
        struct sockaddr_in bound_addr;
        socklen_t bound_addrlen = sizeof(bound_addr);
        ASSERT_EQ(
            getsockname(fd.get(), reinterpret_cast<struct sockaddr*>(&bound_addr), &bound_addrlen),
            0)
            << strerror(errno);
        ASSERT_EQ(bound_addrlen, sizeof(bound_addr));

        auto [it, inserted] = ports.insert(bound_addr.sin_port);
        ASSERT_TRUE(inserted) << "port " << *it << " should already be in use";

        // Accept a connection on the bound port so that we can force it to end up in TIME_WAIT.
        ASSERT_EQ(listen(fd.get(), 0), 0) << strerror(errno);

        fbl::unique_fd client;
        ASSERT_TRUE(client = fbl::unique_fd(socket(AF_INET, SOCK_STREAM, 0))) << strerror(errno);
        ASSERT_EQ(
            connect(client.get(), reinterpret_cast<struct sockaddr*>(&bound_addr), bound_addrlen),
            0)
            << strerror(errno);

        // accepted will end up in TIME_WAIT, eating up an ephemeral port.
        fbl::unique_fd accepted;
        ASSERT_TRUE(accepted = fbl::unique_fd(accept(fd.get(), nullptr, nullptr)))
            << strerror(errno);
      } break;
    }
  }
  FAIL() << "ephemeral ports should be depleted";
}

[hbhasker]

   fbl::unique_fd client;
    ASSERT_TRUE(client = fbl::unique_fd(socket(AF_INET, SOCK_STREAM, 0))) << strerror(errno);
    ASSERT_EQ(
        connect(client.get(), reinterpret_cast<struct sockaddr*>(&bound_addr), bound_addrlen),
        0)
        << strerror(errno);

    // accepted will end up in TIME_WAIT, eating up an ephemeral port.
    fbl::unique_fd accepted;
    ASSERT_TRUE(accepted = fbl::unique_fd(accept(fd.get(), nullptr, nullptr)))
        << strerror(errno);

This is not guaranteed to cause TIME-WAIT on accepted on gVisor. Even though close(accepted) may happen first its possible due to goroutine scheduling etc the client socket ends up sending its FIN first. The best way to ensure TIME-WAIT ends up on accepted is to do an explicit close on accepted first, then poll on client to ensure it sees the FIN. Then close the client socket.