[LibOS,PAL/Linux-SGX] Add shared untrusted memory support

Add mount point type `untrusted_shm` that allows Gramine to mount
shared memory files or directories that contain them (sub-directories
are disallowed for security reasons), aka "shared memory objects". They
are accessible by both the application running inside Gramine and by
other host software/hardware (host OS, other host processes, devices
connected to the host).

It is the responsibility of the app developer to correctly use shared
memory, with security implications in mind. In most cases, data in
shared memory should be preliminarily encrypted or integrity-protected
by the user app with a key pre-shared between all participating
processes (and possibly devices that use this shared memory).

This commit adds a LibOS regression test `shm` to test shared memory.

Signed-off-by: Li, Xun <xun.li@intel.com>

Documentation/manifest-syntax.rst

@@ -427,6 +427,9 @@ Gramine currently supports the following types of mount points:
 * ``encrypted``: Host-backed encrypted files. See :ref:`encrypted-files` for
   more information.
 
+* ``untrusted_shm``: Untrusted shared memory files. See
+  :ref:`untrusted-shared-memory` for more information.
+
 * ``tmpfs``: Temporary in-memory-only files. These files are *not* backed by
   host-level files. The tmpfs files are created under ``[PATH]`` (this path is
   empty on Gramine instance startup) and are destroyed when a Gramine instance
@@ -1010,6 +1013,56 @@ Gramine:
    in the application is insecure. If you need to derive encryption keys from
    such a "doubly-used" key, you must apply a KDF.
 
+.. _untrusted-shared-memory:
+
+Untrusted shared memory
+^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
+    fs.mounts = [
+      { type = "untrusted_shm", path = "[PATH]", uri = "[URI]" },
+    ]
+
+This syntax allows mounting shared memory objects that are accessible by both
+the application running inside Gramine and by other host software/hardware (host
+OS, other host processes, devices connected to the host). In Gramine, untrusted
+shared memory applies to files which must be mapped into application address
+space with the ``MAP_SHARED`` flag.
+
+URI can be a file or a directory (with a ``dev:`` prefix). If a directory is
+mounted, all files under this directory are treated as shared memory objects
+(but sub-directories are inaccessible for security reasons). New files created
+in a shared memory mount are also automatically treated as shared memory
+objects. Creating sub-directories in a shared memory mount is not allowed, for
+security reasons. Files in a shared memory mount (or the mounted directory
+itself) need to be explicitly listed as ``allowed_files`` to be accessed. See
+:ref:`allowed_files` for more information.
+
+Typically, you should mount the directory ``/dev/shm/`` (it is used for sharing
+data between processes and devices) and allow specific files in it. When this
+directory is mounted, the Gramine application may create the files -- called
+"shared memory objects" in POSIX -- under this directory (for example, this is
+how ``shm_open()`` Glibc function works). It is not recommended to allow a
+directory unless the application creates shared memory objects with
+unpredictable names. Allowing a directory creates a risk of exposing unexpected
+data to the host.
+
+.. note ::
+   Adding shared memory mounts is insecure by itself in SGX environments:
+
+       - All data put in shared memory reside outside of the SGX enclave.
+       - Typically applications do not encrypt the data put in shared memory,
+         which may lead to leaks of enclave data.
+       - Untrusted host can modify data in shared memory as it wishes, so
+         applications could see or operate on maliciously modified data.
+
+   It is the responsibility of the app developer to correctly use shared memory,
+   with security implications in mind. In most cases, data in shared memory
+   should be preliminarily encrypted or integrity-protected by the user app
+   with a key pre-shared between all participating processes (and possibly
+   devices that use this shared memory).
+
 File check policy
 ^^^^^^^^^^^^^^^^^
 

libos/include/libos_fs.h

@@ -926,6 +926,7 @@ extern struct libos_fs epoll_builtin_fs;
 extern struct libos_fs eventfd_builtin_fs;
 extern struct libos_fs synthetic_builtin_fs;
 extern struct libos_fs path_builtin_fs;
+extern struct libos_fs shm_builtin_fs;
 
 struct libos_fs* find_fs(const char* name);
 

libos/include/libos_handle.h

@@ -37,6 +37,7 @@ enum libos_handle_type {
     TYPE_TMPFS,      /* string-based files (with data inside dentry), used by `tmpfs` filesystem */
     TYPE_SYNTHETIC,  /* synthetic files, used by `synthetic` filesystem */
     TYPE_PATH,       /* path to a file (the file is not actually opened) */
+    TYPE_SHM,        /* shared files, used by `shm` filesystem */
 
     /* Pipes and sockets: */
     TYPE_PIPE,       /* pipes, used by `pipe` filesystem */

libos/src/bookkeep/libos_vma.c

@@ -999,8 +999,10 @@ int bkeep_mmap_any_in_range(void* _bottom_addr, void* _top_addr, size_t length,
                             struct libos_handle* file, uint64_t offset, const char* comment,
                             void** ret_val_ptr) {
     assert(_bottom_addr < _top_addr);
-    assert(g_pal_public_state->memory_address_start <= _bottom_addr
-           && _top_addr <= g_pal_public_state->memory_address_end);
+    assert((g_pal_public_state->memory_address_start <= _bottom_addr
+               && _top_addr <= g_pal_public_state->memory_address_end)
+           || (g_pal_public_state->shared_address_start <= _bottom_addr
+               && _top_addr <= g_pal_public_state->shared_address_end));
 
     if (!length || !IS_ALLOC_ALIGNED(length)) {
         return -EINVAL;

libos/src/fs/libos_fs.c

@@ -32,6 +32,7 @@ static struct libos_fs* g_builtin_fs[] = {
     &pseudo_builtin_fs,
     &synthetic_builtin_fs,
     &path_builtin_fs,
+    &shm_builtin_fs,
 };
 
 static struct libos_lock g_mount_mgr_lock;

libos/src/fs/proc/thread.c

@@ -286,6 +286,7 @@ static char* describe_handle(struct libos_handle* hdl) {
         case TYPE_SOCK:    str = "sock:[?]";    break;
         case TYPE_EPOLL:   str = "epoll:[?]";   break;
         case TYPE_EVENTFD: str = "eventfd:[?]"; break;
+        case TYPE_SHM:     str = "shm:[?]";     break;
         default:           str = "unknown:[?]"; break;
     }
     return strdup(str);

libos/src/fs/shm/fs.c

@@ -0,0 +1,188 @@
+/* SPDX-License-Identifier: LGPL-3.0-or-later */
+/* Copyright (C) 2023 Intel Corporation
+ *                    Li Xun <xun.li@intel.com>
+ */
+
+/*
+ * This file contains code for implementation of 'shm' filesystem.
+ * If mounted in manifest, files of this type are shared with the host OS when mapped.
+ */
+
+#include "libos_flags_conv.h"
+#include "libos_fs.h"
+#include "libos_handle.h"
+#include "libos_lock.h"
+#include "linux_abi/errors.h"
+#include "perm.h"
+#include "stat.h"
+
+#define HOST_PERM(perm) ((perm) | PERM_r________)
+
+static int shm_mount(struct libos_mount_params* params, void** mount_data) {
+    __UNUSED(mount_data);
+
+    if (!params->uri) {
+        log_error("Missing shared memory URI");
+        return -EINVAL;
+    }
+    if (!strstartswith(params->uri, URI_PREFIX_DEV)) {
+        log_error("'%s' is invalid shared memory URI (expected prefix %s)", params->uri,
+                  URI_PREFIX_DEV);
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int shm_mmap(struct libos_handle* hdl, void* addr, size_t size, int prot, int flags,
+                    uint64_t offset) {
+    assert(hdl->type == TYPE_SHM);
+    assert(addr);
+
+    pal_prot_flags_t pal_prot = LINUX_PROT_TO_PAL(prot, flags);
+
+    if (flags & MAP_ANONYMOUS)
+        return -EINVAL;
+
+    int ret = PalStreamMap(hdl->pal_handle, addr, pal_prot, offset, size);
+    if (ret < 0)
+        return pal_to_unix_errno(ret);
+
+    return 0;
+}
+
+/* Open a PAL handle, and associate it with a LibOS handle. */
+static int shm_do_open(struct libos_handle* hdl, struct libos_dentry* dent, mode_t type,
+                       int flags, mode_t perm) {
+    assert(locked(&g_dcache_lock));
+
+    char* uri;
+    int ret = chroot_dentry_uri(dent, type, &uri);
+    if (ret < 0)
+        return ret;
+
+    PAL_HANDLE palhdl;
+    enum pal_access access = LINUX_OPEN_FLAGS_TO_PAL_ACCESS(flags);
+    enum pal_create_mode create = LINUX_OPEN_FLAGS_TO_PAL_CREATE(flags);
+    pal_stream_options_t options = LINUX_OPEN_FLAGS_TO_PAL_OPTIONS(flags);
+    mode_t host_perm = HOST_PERM(perm);
+    ret = PalStreamOpen(uri, access, host_perm, create, options, &palhdl);
+    if (ret < 0) {
+        ret = pal_to_unix_errno(ret);
+        goto out;
+    }
+
+    hdl->uri = uri;
+    uri = NULL;
+
+    hdl->type = TYPE_SHM;
+    hdl->pal_handle = palhdl;
+    ret = 0;
+
+out:
+    free(uri);
+    return ret;
+}
+
+static int shm_setup_dentry(struct libos_dentry* dent, mode_t type, mode_t perm,
+                            file_off_t size) {
+    assert(locked(&g_dcache_lock));
+    assert(!dent->inode);
+
+    struct libos_inode* inode = get_new_inode(dent->mount, type, perm);
+    if (!inode)
+        return -ENOMEM;
+    inode->size = size;
+    dent->inode = inode;
+    return 0;
+}
+
+static int shm_lookup(struct libos_dentry* dent) {
+    assert(locked(&g_dcache_lock));
+
+    char* uri = NULL;
+    /*
+     * We don't know the file type yet, so we can't construct a PAL URI with the right prefix.
+     * However, "file:" prefix is good enough here: `PalStreamAttributesQuery` will access the file
+     * and report the right file type.
+     */
+    int ret = chroot_dentry_uri(dent, S_IFREG, &uri);
+    if (ret < 0)
+        goto out;
+
+    PAL_STREAM_ATTR pal_attr;
+    ret = PalStreamAttributesQuery(uri, &pal_attr);
+    if (ret < 0) {
+        ret = pal_to_unix_errno(ret);
+        goto out;
+    }
+
+    mode_t type;
+    switch (pal_attr.handle_type) {
+        case PAL_TYPE_FILE: /* Regular files in shm file system are device files. */
+        case PAL_TYPE_DEV:
+            type = S_IFCHR;
+            break;
+        case PAL_TYPE_DIR:
+            /* Subdirectories (e.g. /dev/shm/subdir/) are not allowed in shm file system. */
+            if (dent != dent->mount->root) {
+                log_warning("trying to access '%s' which is a subdirectory of shared memory mount",
+                            uri);
+                ret = -EACCES;
+                goto out;
+            }
+            type = S_IFDIR;
+            break;
+        default:
+            log_error("unexpected handle type returned by PAL: %d", pal_attr.handle_type);
+            BUG();
+    }
+
+    file_off_t size = (type == S_IFCHR ? pal_attr.pending_size : 0);
+
+    ret = shm_setup_dentry(dent, type, pal_attr.share_flags, size);
+out:
+    free(uri);
+    return ret;
+}
+
+static int shm_open(struct libos_handle* hdl, struct libos_dentry* dent, int flags) {
+    assert(locked(&g_dcache_lock));
+    assert(dent->inode);
+
+    return shm_do_open(hdl, dent, dent->inode->type, flags, /*perm=*/0);
+}
+
+static int shm_creat(struct libos_handle* hdl, struct libos_dentry* dent, int flags, mode_t perm) {
+    assert(locked(&g_dcache_lock));
+    assert(!dent->inode);
+
+    mode_t type = S_IFCHR;
+    int ret = shm_do_open(hdl, dent, type, flags | O_CREAT | O_EXCL, perm);
+    if (ret < 0)
+        return ret;
+
+    return shm_setup_dentry(dent, type, perm, /*size=*/0);
+}
+
+struct libos_fs_ops shm_fs_ops = {
+    .mount      = shm_mount,
+    /* .read and .write are intentionally not supported according to POSIX shared memory API. */
+    .mmap       = shm_mmap,
+    .hstat      = generic_inode_hstat,
+    .truncate   = generic_truncate,
+};
+
+struct libos_d_ops shm_d_ops = {
+    .open    = shm_open,
+    .lookup  = shm_lookup,
+    .creat   = shm_creat,
+    .stat    = generic_inode_stat,
+    .unlink  = chroot_unlink,
+};
+
+struct libos_fs shm_builtin_fs = {
+    .name   = "untrusted_shm",
+    .fs_ops = &shm_fs_ops,
+    .d_ops  = &shm_d_ops,
+};

libos/src/meson.build

@@ -37,6 +37,7 @@ libos_sources = files(
     'fs/proc/info.c',
     'fs/proc/ipc_thread.c',
     'fs/proc/thread.c',
+    'fs/shm/fs.c',
     'fs/socket/fs.c',
     'fs/sys/cache_info.c',
     'fs/sys/cpu_info.c',

libos/src/sys/libos_mmap.c

@@ -152,10 +152,21 @@ void* libos_syscall_mmap(void* addr, size_t length, int prot, int flags, int fd,
         flags &= ~MAP_32BIT;
 #endif
 
+    void* memory_range_start = NULL;
+    void* memory_range_end   = NULL;
+
+    /* Shared mappings of files of "untrusted_shm" type use a different memory range.
+     * See "libos/src/fs/shm/fs.c" for more details. */
+    if ((flags & MAP_SHARED) && hdl && hdl->fs && !strcmp(hdl->fs->name, "untrusted_shm")) {
+        memory_range_start = g_pal_public_state->shared_address_start;
+        memory_range_end = g_pal_public_state->shared_address_end;
+    } else {
+        memory_range_start = g_pal_public_state->memory_address_start;
+        memory_range_end = g_pal_public_state->memory_address_end;
+    }
     if (flags & (MAP_FIXED | MAP_FIXED_NOREPLACE)) {
         /* We know that `addr + length` does not overflow (`access_ok` above). */
-        if (addr < g_pal_public_state->memory_address_start
-                || (uintptr_t)g_pal_public_state->memory_address_end < (uintptr_t)addr + length) {
+        if (addr < memory_range_start || (uintptr_t)memory_range_end < (uintptr_t)addr + length) {
             ret = -EINVAL;
             goto out_handle;
         }
@@ -208,18 +219,23 @@ void* libos_syscall_mmap(void* addr, size_t length, int prot, int flags, int fd,
         }
     } else {
         /* We know that `addr + length` does not overflow (`access_ok` above). */
-        if (addr && (uintptr_t)g_pal_public_state->memory_address_start <= (uintptr_t)addr
-                && (uintptr_t)addr + length <= (uintptr_t)g_pal_public_state->memory_address_end) {
-            ret = bkeep_mmap_any_in_range(g_pal_public_state->memory_address_start,
-                                          (char*)addr + length, length, prot, flags, hdl, offset,
-                                          NULL, &addr);
+        if (addr && (uintptr_t)memory_range_start <= (uintptr_t)addr
+                && (uintptr_t)addr + length <= (uintptr_t)memory_range_end) {
+            ret = bkeep_mmap_any_in_range(memory_range_start, (char*)addr + length, length, prot,
+                                          flags, hdl, offset, NULL, &addr);
         } else {
             /* Hacky way to mark we had no hit and need to search below. */
             ret = -1;
         }
         if (ret < 0) {
             /* We either had no hinted address or could not allocate memory at it. */
-            ret = bkeep_mmap_any_aslr(length, prot, flags, hdl, offset, NULL, &addr);
+            if (memory_range_start == g_pal_public_state->memory_address_start) {
+                ret = bkeep_mmap_any_aslr(length, prot, flags, hdl, offset, NULL, &addr);
+            } else {
+                /* Shared memory range does not have ASLR. */
+                ret = bkeep_mmap_any_in_range(memory_range_start, memory_range_end, length, prot,
+                                              flags, hdl, offset, NULL, &addr);
+            }
         }
         if (ret < 0) {
             ret = -ENOMEM;

libos/test/regression/meson.build

@@ -117,6 +117,9 @@ tests = {
     'shared_object': {
         'pie': true,
     },
+    'shm': {
+        'link_args': '-lrt',
+    },
     'sid': {},
     'sigaction_per_process': {},
     'sigaltstack': {},