[LibOS] Use static buffer for first thread executing sendfile()

This performance optimization uses a statically allocated buffer instead
of allocating on the heap in the `sendfile()` syscall. To prevent data
races on a single static buffer, this optimization is applied only to
the first thread. This optimization is useful for sendfile-heavy
workloads, e.g. Nginx in HTTP mode (and with config `sendfile=on`).

Co-authored-by: TejaswineeL <tejaswinee.ramdas.langhe@intel.com>
Signed-off-by: TejaswineeL <tejaswinee.ramdas.langhe@intel.com>
Signed-off-by: Dmitrii Kuvaiskii <dmitrii.kuvaiskii@intel.com>

libos/src/sys/libos_file.c

@@ -17,7 +17,17 @@
 #include "perm.h"
 #include "stat.h"
 
-#define BUF_SIZE (64 * 1024) /* read/write in 64KB chunks for sendfile() */
+/*
+ * Read/write in 64KB chunks in the sendfile() syscall. This syscall also has an optimization of
+ * using a statically allocated buffer instead of allocating on the heap (as our internal malloc()
+ * has subpar performance). To prevent data races of multiple threads executing sendfile() at the
+ * same time and thus potentially corrupting a single static buffer, we optimize for a common case:
+ * only the first thread uses the static buffer whereas other threads fall back to a slower heap
+ * allocation.
+ */
+#define BUF_SIZE (64 * 1024)
+static char g_sendfile_buf[BUF_SIZE];
+static bool g_sendfile_buf_in_use = false;
 
 /* The kernel would look up the parent directory, and remove the child from the inode. But we are
  * working with the PAL, so we open the file, truncate and close it. */
@@ -442,10 +452,17 @@ long libos_syscall_sendfile(int out_fd, int in_fd, off_t* offset, size_t count)
     /* FIXME: This sendfile() emulation is very simple and not particularly efficient: it reads from
      *        input FD in BUF_SIZE chunks and writes into output FD. Mmap-based emulation may be
      *        more efficient but adds complexity (not all handle types provide mmap callback). */
-    buf = malloc(BUF_SIZE);
-    if (!buf) {
-        ret = -ENOMEM;
-        goto out;
+
+    bool buf_in_use = __atomic_exchange_n(&g_sendfile_buf_in_use, true, __ATOMIC_ACQUIRE);
+    if (!buf_in_use) {
+        /* no other thread was using the static buffer */
+        buf = g_sendfile_buf;
+    } else {
+        buf = malloc(BUF_SIZE);
+        if (!buf) {
+            ret = -ENOMEM;
+            goto out;
+        }
     }
 
     if (!count) {
@@ -533,7 +550,10 @@ long libos_syscall_sendfile(int out_fd, int in_fd, off_t* offset, size_t count)
     }
 
 out:
-    free(buf);
+    if (buf == g_sendfile_buf)
+        __atomic_store_n(&g_sendfile_buf_in_use, 0, __ATOMIC_RELEASE);
+    else
+        free(buf);
     put_handle(in_hdl);
     put_handle(out_hdl);
     return copied_to_out ? (long)copied_to_out : ret;