ksmbd-tools: set netlink socket's receive buffer size to 1 MiB

This commit fixes the intermittent `Recv() error Out of memory [-5]`
crashes of worker process by raising the netlink socket's receive
buffer size to 1 MiB.

A netlink socket's default receive buffer size is 208 KiB (taken from
net.core.rmem_default). When incoming messages fill up the receive
buffer, the recvmsg(2) call returns -ENOBUFS, which is then translated
to the Out of memory error above by libnl.

Both NetworkManager and systemd-networkd raise their netlink socket's
buffer size to work around the same issue. More details on
systemd-networkd's decision can be found at
systemd/systemd#14417 and
systemd/systemd#14434.

Fixes cifsd-team#235.

Signed-off-by: database64128 <free122448@hotmail.com>
Signed-off-by: Namjae Jeon <linkinjeon@kernel.org>

include/ipc.h

@@ -14,6 +14,12 @@
  */
 #define KSMBD_IPC_MAX_MESSAGE_SIZE	(16 * 1024)
 
+/*
+ * The netlink socket's receive buffer size needs to be increased
+ * to avoid -ENOBUFS errors when receiving.
+ */
+#define KSMBD_IPC_SO_RCVBUF_SIZE	(1 * 1024 * 1024)
+
 struct ksmbd_ipc_msg {
 	unsigned int	type;
 	unsigned int	sz;

mountd/ipc.c

@@ -495,6 +495,12 @@ int ipc_init(void)
 		goto out_error;
 	}
 
+	ret = nl_socket_set_buffer_size(sk, KSMBD_IPC_SO_RCVBUF_SIZE, 0);
+	if (ret) {
+		pr_err("Cannot set netlink socket buffer size: %s [%d]\n", nl_geterror(ret), ret);
+		goto out_error;
+	}
+
 	if (genl_register_family(&ksmbd_family_ops)) {
 		pr_err("Cannot register netlink family\n");
 		goto out_error;