Add proper tracing options, remove getenv calls from enclave code.

src/enclave/enclave_init.c

@@ -12,7 +12,7 @@
 #include "enclave/lthread_int.h"
 #include "enclave/wireguard.h"
 #include "enclave/wireguard_util.h"
-#include "shared/env.h"
+#include "shared/util.h"
 
 #include "../../user/userargs.h"
 
@@ -30,6 +30,53 @@ extern struct mpmcq __scheduler_queue;
 _Noreturn void __dls3(elf64_stack_t* conf, void* tos);
 extern void init_sysconf(long nproc_conf, long nproc_onln);
 
+static void get_disk_keys()
+{
+    /* Here or earlier: get keys from remote key provider or auxv. For now we
+     * only support plaintext keys in sgxlkl_enclave_state.config, which we copy
+     * into the right places here. */
+    const sgxlkl_enclave_config_t* cfg = sgxlkl_enclave_state.config;
+    sgxlkl_enclave_disk_state_t* disk_states = sgxlkl_enclave_state.disk_state;
+    if (cfg->root.key)
+    {
+        uint8_t* key = cfg->root.key;
+        size_t len = cfg->root.key_len;
+        disk_states[0].key = oe_malloc(sizeof(uint8_t) * len);
+        memcpy(disk_states[0].key, key, len);
+    }
+    for (size_t i = 0; i < cfg->num_mounts; i++)
+    {
+        if (cfg->mounts[i].key)
+        {
+            uint8_t* key = cfg->mounts[i].key;
+            size_t len = cfg->mounts[i].key_len;
+            disk_states[i + 1].key = oe_malloc(sizeof(uint8_t) * len);
+            memcpy(disk_states[i + 1].key, key, len);
+        }
+    }
+}
+
+/* Wipe copies of (currently local, in future remote) keys after we don't need
+ * them anymore, i.e. after mounting the respective file systems. */
+static void wipe_disk_keys()
+{
+    sgxlkl_enclave_disk_state_t* disk_state = sgxlkl_enclave_state.disk_state;
+    for (size_t i = 0; i < sgxlkl_enclave_state.num_disk_state; i++)
+    {
+        if (disk_state[i].key)
+        {
+            oe_memset_s(
+                disk_state[i].key,
+                disk_state[i].key_len,
+                0,
+                disk_state[i].key_len);
+            oe_free(disk_state[i].key);
+        }
+        disk_state[i].key = NULL;
+        disk_state[i].key_len = 0;
+    }
+}
+
 static void find_and_mount_disks()
 {
     const sgxlkl_enclave_config_t* cfg = sgxlkl_enclave_state.config;
@@ -41,7 +88,7 @@ static void find_and_mount_disks()
     estate->disk_state = oe_calloc(n, sizeof(sgxlkl_enclave_disk_state_t));
     estate->num_disk_state = n;
 
-    // root disk index
+    // root disk index 0
     estate->disk_state[0].host_disk_index = 0;
 
     for (int i = 0; i < cfg->num_mounts; i++)
@@ -68,7 +115,9 @@ static void find_and_mount_disks()
                 cfg_disk->destination);
     }
 
+    get_disk_keys();
     lkl_mount_disks(&cfg->root, cfg->mounts, cfg->num_mounts, cfg->cwd);
+    wipe_disk_keys();
 }
 
 static void init_wireguard_peers()
@@ -179,7 +228,8 @@ static int app_main_thread(void* args_)
 
 static int kernel_main_thread(void* args)
 {
-    __init_libc(sgxlkl_enclave_state.elf64_stack.envp,
+    __init_libc(
+        sgxlkl_enclave_state.elf64_stack.envp,
         sgxlkl_enclave_state.elf64_stack.argv[0]);
     __libc_start_init();
     a_barrier();
@@ -252,7 +302,7 @@ int __libc_init_enclave(int argc, char** argv)
     max_lthreads = next_power_of_2(max_lthreads);
 
     newmpmcq(&__scheduler_queue, max_lthreads, 0);
-    
+
     init_ethread_tp();
 
     size_t espins = cfg->espins;

src/enclave/enclave_mem.c

@@ -36,7 +36,6 @@ static size_t used_pages =
     0; // Tracks the number of used pages for the mmap tracing
 
 #if DEBUG
-extern int sgxlkl_trace_mmap;
 static size_t mmap_max_allocated = 0; // Maximum amount of memory used thus far
 #endif
 
@@ -139,7 +138,7 @@ void enclave_mman_init(const void* base, size_t num_pages, int _mmap_files)
 
     // Base address for range of pages available to mmap calls
     mmap_base = (char*)mmap_bitmap + (2 * bitmap_req_pages * PAGE_SIZE);
-    // Set mmap_end to one less page than we normally would to address 
+    // Set mmap_end to one less page than we normally would to address
     // https://github.com/lsds/sgx-lkl/issues/742
     mmap_end = (char*)mmap_base + (mmap_num_pages - 2) * PAGE_SIZE;
 
@@ -194,7 +193,7 @@ void* enclave_mmap(
             index_top = addr_to_index(addr) - (pages - 1);
 
 #if DEBUG
-            if (sgxlkl_trace_mmap)
+            if (sgxlkl_enclave_state.config->trace.mmap)
                 replaced_pages = bitmap_count_set_bits(
                     mmap_bitmap, mmap_num_pages, index_top, pages);
 #endif
@@ -296,7 +295,7 @@ void* enclave_mmap(
     }
 
 #if DEBUG
-    if (sgxlkl_trace_mmap)
+    if (sgxlkl_enclave_state.config->trace.mmap)
     {
         size_t requested = pages * PAGESIZE;
         size_t total = mmap_num_pages * PAGESIZE;
@@ -354,7 +353,7 @@ long enclave_munmap(void* addr, size_t length)
     ticket_unlock(&mmaplock);
 
 #if DEBUG
-    if (sgxlkl_trace_mmap)
+    if (sgxlkl_enclave_state.config->trace.mmap)
     {
         size_t requested = pages * PAGESIZE;
         size_t total = mmap_num_pages * PAGESIZE;

src/enclave/enclave_oe.c

@@ -12,8 +12,8 @@
 #include "enclave/enclave_util.h"
 #include "enclave/lthread.h"
 #include "enclave/lthread_int.h"
-#include "shared/env.h"
 #include "shared/timer_dev.h"
+#include "shared/util.h"
 
 #define AUXV_ENTRIES 13
 
@@ -299,11 +299,25 @@ static void _read_eeid_config()
     const oe_eeid_t* eeid = (oe_eeid_t*)__oe_get_eeid();
     const char* config_json = (const char*)eeid->data;
 
-    sgxlkl_enclave_config_t* cfg = oe_malloc(sizeof(sgxlkl_enclave_config_t));
-    if (!cfg)
-        sgxlkl_fail("out of memory, cannot allocate enclave config.\n");
-    sgxlkl_read_enclave_config(config_json, cfg, true);
-    sgxlkl_enclave_state.config = cfg;
+    size_t num_pages = 0;
+    const sgxlkl_enclave_config_page_t* cfg_pages =
+        sgxlkl_read_enclave_config(config_json, true, &num_pages);
+
+    /* Make the config pages read-only */
+    for (size_t i = 0; i < num_pages; i++)
+    {
+        int mprotect_ret;
+        sgxlkl_host_syscall_mprotect(
+            &mprotect_ret,
+            (void*)&cfg_pages[i],
+            sizeof(sgxlkl_enclave_config_t),
+            PROT_READ);
+
+        if (mprotect_ret != 0)
+            sgxlkl_warn("Could not protect memory pages for config\n");
+    }
+
+    sgxlkl_enclave_state.config = &cfg_pages->config;
 }
 
 static void _copy_shared_memory(const sgxlkl_shared_memory_t* host)
@@ -397,7 +411,7 @@ int sgxlkl_enclave_init(const sgxlkl_shared_memory_t* shared_memory)
 #ifdef DEBUG
     /* Make sure verbosity is off before loading the config (we don't know
      * whether it's enabled yet).*/
-    sgxlkl_enclave_state.verbose = false;
+    sgxlkl_enclave_state.trace_enabled.verbose = false;
 #endif
 
     _read_eeid_config();
@@ -406,7 +420,12 @@ int sgxlkl_enclave_init(const sgxlkl_shared_memory_t* shared_memory)
 #ifdef DEBUG
     // Initialise verbosity setting, so SGXLKL_VERBOSE can be used from this
     // point onwards
-    sgxlkl_enclave_state.verbose = sgxlkl_enclave_state.config->verbose;
+    sgxlkl_enclave_state.trace_enabled.verbose =
+        sgxlkl_enclave_state.config->verbose;
+    sgxlkl_enclave_state.trace_enabled.internal_syscall =
+        sgxlkl_enclave_state.config->trace.internal_syscall;
+    sgxlkl_enclave_state.trace_enabled.lkl_syscall =
+        sgxlkl_enclave_state.config->trace.lkl_syscall;
 #endif
 
     SGXLKL_VERBOSE("enter\n");

src/enclave/enclave_signal.c

@@ -5,8 +5,8 @@
 
 #include <asm-generic/ucontext.h>
 
-#include <lkl_host.h>
 #include <lkl/setup.h>
+#include <lkl_host.h>
 #include <string.h>
 
 #include <openenclave/enclave.h>
@@ -16,7 +16,6 @@
 #include "enclave/enclave_util.h"
 #include "enclave/lthread.h"
 #include "enclave/sgxlkl_t.h"
-#include "shared/env.h"
 
 #define RDTSC_OPCODE 0x310F
 
@@ -151,7 +150,7 @@ static uint64_t sgxlkl_enclave_signal_handler(
             opcode);
 
 #ifdef DEBUG
-        if (sgxlkl_trace_signal)
+        if (sgxlkl_enclave_state.config->trace.signal)
         {
             sgxlkl_print_backtrace((void*)oe_ctx->rbp);
         }

src/host_interface/virtio_blkdev.c

@@ -6,7 +6,7 @@
 #include <host/vio_host_event_channel.h>
 #include <host/virtio_blkdev.h>
 #include <host/virtio_debug.h>
-#include <shared/env.h>
+#include <shared/util.h>
 #include <string.h>
 #include <sys/mman.h>
 #include <sys/syscall.h>

src/host_interface/virtio_console.c

@@ -8,7 +8,7 @@
 #include <host/vio_host_event_channel.h>
 #include <host/virtio_console.h>
 #include <poll.h>
-#include <shared/env.h>
+#include <shared/util.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

src/host_interface/virtio_debug.c

@@ -1,7 +1,6 @@
 #if DEBUG && VIRTIO_TEST_HOOK
 
 #include <host/virtio_debug.h>
-#include <shared/env.h>
 #include <string.h>
 
 /* Virtio debug module enables different debug options for virtio device

src/host_interface/virtio_netdev.c

@@ -24,7 +24,7 @@
 #include <host/virtio_debug.h>
 #include <host/virtio_netdev.h>
 #include <poll.h>
-#include <shared/env.h>
+#include <shared/util.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

src/include/enclave/enclave_state.h

@@ -20,6 +20,8 @@ typedef struct sgxlkl_enclave_disk_state
     int fd;                 /* File descriptor of the disk */
     size_t capacity;        /* Capacity of the disk */
     bool mounted;           /* Tracks whether the disk has been mounted */
+    uint8_t* key;           /* Encryption key */
+    size_t key_len;         /* Length of encryption key */
 } sgxlkl_enclave_disk_state_t;
 
 typedef struct
@@ -51,8 +53,15 @@ typedef struct sgxlkl_enclave_state
     /* Memory shared with the host */
     sgxlkl_shared_memory_t shared_memory;
 
-    /* This flag is used by the tracing macros */
-    bool verbose;
+    /* Flags to track whether tracing macros are currently enabled (initialized
+     * according to the settings in `config`; the respective traces may be
+     * disabled temporarily and this flags track the current state) */
+    struct
+    {
+        bool verbose;
+        bool lkl_syscall;
+        bool internal_syscall;
+    } trace_enabled;
 } sgxlkl_enclave_state_t;
 
 extern sgxlkl_enclave_state_t sgxlkl_enclave_state;