socket: evaluate buffer size in recv / recvfrom

When size parameter is not passed to socket:recv() or socket:recvfrom()
it will call a) or b) on the socket to evaluate size of the buffer to
store the receiving datagram. Before this commit a datagram will be
truncated to 512 bytes in the case.

a) Linux: recv(fd, NULL, 0 , MSG_TRUNC | MSG_PEEK)
b) Mac OS: getsockopt(fd, SOL_SOCKET, SO_NREAD, &val, &len)

It is recommended to set 'size' parameter (size of the input buffer)
explicitly based on known message format and known network conditions
(say, set it less then MTU to prevent IP fragmentation, which can be
inefficient) or pass it from a configuration option of a library / an
application. The reason is that explicit buffer size provided allows to
avoid extra syscall to evaluate necessary buffer size.

When 'size' parameter is set explicitly for recv / recvfrom on a UDP
socket and the next datagram length is larger then the size, the
returned message will be truncated to the size provided and the rest of
the datagram will be discarded. Of course, the tail will not be
discarded in case of a TCP socket and will be available to read by the
next recv / recvfrom call.

Fixes #3619.

src/lua/socket.c

@@ -268,6 +268,9 @@ static const struct lbox_sockopt_reg so_opts[] = {
 #ifdef SO_PROTOCOL
 	{"SO_PROTOCOL",		SO_PROTOCOL,		1,	0, },
 #endif
+#ifdef SO_NREAD /* available on Darwin */
+	{"SO_NREAD",		SO_NREAD,		1,	0, },
+#endif
 
 	{"SO_TYPE",		SO_TYPE,		1,	0, },
 

src/lua/socket.lua

@@ -86,8 +86,12 @@ local function check_socket(socket)
     end
 end
 
-local function make_socket(fd)
-    local socket = { _gc_socket = ffi.new(gc_socket_t, { fd = fd }) }
+local function make_socket(fd, itype)
+    assert(itype ~= nil)
+    local socket = {
+        _gc_socket = ffi.new(gc_socket_t, { fd = fd }),
+        itype = itype,
+    }
     return setmetatable(socket, socket_mt)
 end
 
@@ -590,7 +594,7 @@ local function socket_accept(self)
         self._errno = boxerrno()
         return nil
     end
-    local client = make_socket(client_fd)
+    local client = make_socket(client_fd, self.itype)
     if not client:nonblock(true) then
         client:close()
         return
@@ -770,6 +774,56 @@ local function socket_send(self, octets, flags)
     return tonumber(res)
 end
 
+-- Returns nil and sets EAGAIN when tries to determine the next
+-- datagram length and there are no datagrams in the receive
+-- queue.
+local function get_recv_size(self, size)
+    if size ~= nil then
+        return size
+    end
+
+    if self.itype ~= get_ivalue(internal.SO_TYPE, 'SOCK_DGRAM') then
+        return 512
+    end
+
+    -- Determine the next datagram length.
+    local fd = check_socket(self)
+    self._errno = nil
+    if jit.os == 'OSX' then
+        size = self:getsockopt('SOL_SOCKET', 'SO_NREAD')
+        -- recv() with zero length buffer is always successful on
+        -- Mac OS (at least for valid UDP sockets), so we need
+        -- extra calls below to distinguish a zero length datagram
+        -- from no datagram situation to set EAGAIN correctly.
+        if size == 0 then
+            -- No datagram or zero length datagram: distinguish
+            -- them using message peek.
+            local iflags = get_iflags(internal.SEND_FLAGS, {'MSG_PEEK'})
+            assert(iflags ~= nil)
+            local buf = ffi.new('char[?]', 1)
+            size = tonumber(ffi.C.recv(fd, buf, 1, iflags))
+            -- Prevent race condition: proceed with the case when
+            -- a datagram of length > 0 has been arrived after the
+            -- getsockopt call above.
+            if size > 0 then
+                size = self:getsockopt('SOL_SOCKET', 'SO_NREAD')
+                assert(size > 0)
+            end
+        end
+    else
+        local iflags = get_iflags(internal.SEND_FLAGS, {'MSG_TRUNC', 'MSG_PEEK'})
+        assert(iflags ~= nil)
+        size = tonumber(ffi.C.recv(fd, nil, 0, iflags))
+    end
+
+    if size == -1 then
+        self._errno = boxerrno()
+        return nil
+    end
+
+    return size
+end
+
 local function socket_recv(self, size, flags)
     local fd = check_socket(self)
     local iflags = get_iflags(internal.SEND_FLAGS, flags)
@@ -778,7 +832,11 @@ local function socket_recv(self, size, flags)
         return nil
     end
 
-    size = size or 512
+    size = get_recv_size(self, size)
+    if size == nil then
+        return nil
+    end
+
     self._errno = nil
     local buf = ffi.new("char[?]", size)
 
@@ -799,7 +857,11 @@ local function socket_recvfrom(self, size, flags)
         return nil
     end
 
-    size = size or 512
+    size = get_recv_size(self, size)
+    if size == nil then
+        return nil
+    end
+
     self._errno = nil
     local res, from = internal.recvfrom(fd, size, iflags)
     if res == nil then
@@ -860,7 +922,7 @@ local function socket_new(domain, stype, proto)
 
     local fd = ffi.C.socket(idomain, itype, iproto)
     if fd >= 0 then
-        local socket = make_socket(fd)
+        local socket = make_socket(fd, itype)
         if not socket:nonblock(true) then
             socket:close()
         else