[Common,Pal,LibOS] Enable compiling with ASan

As with UBSan, this adds our own handlers to Graphene's common library.
The current version of ASan integration supports heap allocation only.

Signed-off-by: Paweł Marczewski <pawel@invisiblethingslab.com>

.ci/lib/config-asan.jenkinsfile

@@ -0,0 +1 @@
+env.ASAN = '1'

.ci/lib/stage-build-nosgx.jenkinsfile

@@ -6,7 +6,10 @@ stage('build') {
 
     env.MESON_OPTIONS = ''
     if (env.UBSAN == '1') {
-        env.MESON_OPTIONS += '-Dubsan=enabled'
+        env.MESON_OPTIONS += ' -Dubsan=enabled'
+    }
+    if (env.ASAN == '1') {
+        env.MESON_OPTIONS += ' -Dasan=enabled'
     }
 
     try {

.ci/lib/stage-build-sgx.jenkinsfile

@@ -29,6 +29,9 @@ stage('build') {
     if (env.UBSAN == '1') {
         env.MESON_OPTIONS += ' -Dubsan=enabled'
     }
+    if (env.ASAN == '1') {
+        env.MESON_OPTIONS += ' -Dasan=enabled'
+    }
 
     try {
         sh '''

.ci/linux-direct-sanitizers.jenkinsfile

@@ -11,6 +11,7 @@ node('nonsgx_slave') {
         load '.ci/lib/config-clang.jenkinsfile'
         load '.ci/lib/config-debug.jenkinsfile'
         load '.ci/lib/config-ubsan.jenkinsfile'
+        load '.ci/lib/config-asan.jenkinsfile'
 
         load '.ci/lib/stage-lint.jenkinsfile'
         load '.ci/lib/stage-clean-check-prepare.jenkinsfile'

.ci/linux-sgx-sanitizers.jenkinsfile

@@ -13,6 +13,7 @@ node('sgx_slave_2.6') {
         load '.ci/lib/config-clang.jenkinsfile'
         load '.ci/lib/config-debug.jenkinsfile'
         load '.ci/lib/config-ubsan.jenkinsfile'
+        load '.ci/lib/config-asan.jenkinsfile'
 
         load '.ci/lib/stage-lint.jenkinsfile'
         load '.ci/lib/stage-clean-check-prepare.jenkinsfile'

Documentation/building.rst

@@ -209,6 +209,18 @@ Additional build options
   .. warning::
      UBSan builds (even non-debug) are not suitable for production.
 
+- To compile with address sanitization (ASan), run :command:`make ASAN=1` and
+  :command:`meson -Dasan=enabled`. In this mode, Graphene will attempt to detect
+  invalid memory accesses. ASan can be enabled for both debug and non-debug
+  builds.
+
+  ASan is supported only when compiling with Clang (before building, set the
+  appropriate environment variables with :command:`export CC=clang CXX=clang++
+  AS=clang`).
+
+  .. warning::
+     ASan builds (even non-debug) are not suitable for production.
+
 - To build with ``-Werror``, run :command:`make WERROR=1` and
   :command:`meson --werror`.
 

LibOS/shim/src/shim_call.c

@@ -9,34 +9,62 @@
 #include <limits.h>
 
 #include "api.h"
+#include "asan.h"
 #include "shim_entry.h"
 #include "shim_entry_api.h"
+#include "shim_thread.h"
 #include "shim_utils.h"
 
 /* Test: do nothing, return success */
 static int run_test_pass(void) {
     return 0;
 }
 
-/* Test: invoke undefined behavior; enabled only when Gramine is compiled with UBSan */
-static int run_test_undefined(void) {
+/* Test: invoke undefined behavior (UBSan only) */
 #ifdef UBSAN
+static int run_test_ubsan_int_overflow(void) {
     volatile int x = INT_MAX;
     x++;
     return 0;
-#else
-    return -EINVAL;
+}
 #endif
+
+/* Test: allocate a buffer on heap, write past the end of buffer (ASan only) */
+#ifdef ASAN
+static int run_test_asan_buffer_overflow(void) {
+    uint8_t* buf = malloc(30);
+    buf[30] = 1;
+    free(buf);
+    return 0;
 }
+#endif
+
+static const struct shim_test {
+    const char* name;
+    int (*func)(void);
+} tests[] = {
+    { "pass", &run_test_pass },
+#ifdef UBSAN
+    { "ubsan_int_overflow", &run_test_ubsan_int_overflow },
+#endif
+#ifdef ASAN
+    { "asan_buffer_overflow", &run_test_asan_buffer_overflow },
+#endif
+    { NULL, NULL },
+};
 
 static int run_test(const char* test_name) {
     int ret;
 
     log_always("run_test(\"%s\") ...", test_name);
-    if (strcmp(test_name, "pass") == 0) {
-        ret = run_test_pass();
-    } else if (strcmp(test_name, "undefined") == 0) {
-        ret = run_test_undefined();
+
+    const struct shim_test* test;
+    for (test = &tests[0]; test->name; test++) {
+        if (strcmp(test_name, test->name) == 0)
+            break;
+    }
+    if (test->name) {
+        ret = test->func();
     } else {
         log_warning("run_test: invalid test name: \"%s\"", test_name);
         ret = -EINVAL;

LibOS/shim/src/shim_malloc.c

@@ -11,6 +11,7 @@
 
 #include <asm/mman.h>
 
+#include "asan.h"
 #include "pal.h"
 #include "shim_checkpoint.h"
 #include "shim_internal.h"
@@ -53,6 +54,9 @@ void* __system_malloc(size_t size) {
         return NULL;
     }
 
+#ifdef ASAN
+    asan_poison_region((uintptr_t)addr, alloc_size, ASAN_POISON_HEAP_LEFT_REDZONE);
+#endif
     return addr;
 }
 
@@ -64,6 +68,9 @@ void __system_free(void* addr, size_t size) {
     if (DkVirtualMemoryFree(addr, ALLOC_ALIGN_UP(size)) < 0) {
         BUG();
     }
+#ifdef ASAN
+    asan_unpoison_region((uintptr_t)addr, ALLOC_ALIGN_UP(size));
+#endif
     bkeep_remove_tmp_vma(tmp_vma);
 }
 

LibOS/shim/test/regression/test_libos.py

@@ -25,15 +25,22 @@ def test_010_shim_run_test(self):
 
     @unittest.skipUnless(os.environ.get('UBSAN') == '1', 'test only enabled with UBSAN=1')
     def test_020_ubsan(self):
+        self._test_abort('ubsan_int_overflow', 'ubsan: overflow')
+
+    @unittest.skipUnless(os.environ.get('ASAN') == '1', 'test only enabled with ASAN=1')
+    def test_021_asan(self):
+        self._test_abort('asan_buffer_overflow', 'asan: heap-buffer-overflow')
+
+    def _test_abort(self, test_name, expected):
         try:
-            self.run_binary(['run_test', 'undefined'])
+            self.run_binary(['run_test', test_name])
             self.fail('run_test unexpectedly succeeded')
         except subprocess.CalledProcessError as e:
             stderr = e.stderr.decode()
-            self.assertIn('run_test("undefined") ...', stderr,
+            self.assertIn('run_test("{}") ...'.format(test_name), stderr,
                           'Gramine should not abort before attempting to run test')
-            self.assertIn('ubsan: overflow', stderr)
-            self.assertNotIn('run_test("undefined") =', stderr,
+            self.assertIn(expected, stderr)
+            self.assertNotIn('run_test("{}") ='.format(test_name), stderr,
                              'Gramine should abort before returning to application')
 
 class TC_01_Bootstrap(RegressionTestCase):

Pal/src/host/Linux-SGX/Makefile

@@ -133,7 +133,11 @@ common_syscall.o: ../Linux-common/arch/$(ARCH)/syscall.S
 %.s: %.S
 	$(call cmd,cpp_s_S)
 
-CFLAGS-pal-sgx = -Wl,-z,relro,-z,now -pie
+# Add `-Wunused-command-line-argument`, because with ASan, Clang complains that LLVM flags
+# (`-mllvm ...`) are unused.
+# TODO: `cmulti` probably should use LDFLAGS, not CFLAGS here, because we're not compiling any C
+# code.
+CFLAGS-pal-sgx = -Wl,-z,relro,-z,now -pie -Wno-unused-command-line-argument
 LDLIBS-pal-sgx += -lprotobuf-c
 pal-sgx: $(urts-objs) $(urts-asm-objs) $(gramine_lib)
 	$(call cmd,cmulti)

Pal/src/host/Linux-SGX/enclave_untrusted.c

@@ -2,6 +2,7 @@
 /* Copyright (C) 2014 Stony Brook University */
 
 #include "api.h"
+#include "asan.h"
 #include "enclave_ocalls.h"
 #include "pal_error.h"
 #include "pal_internal.h"
@@ -21,12 +22,21 @@ static inline void* __malloc(size_t size) {
     void* addr = NULL;
     int ret = ocall_mmap_untrusted(&addr, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE,
                                    /*fd=*/-1, /*offset=*/0);
-    return ret < 0 ? NULL : addr;
+    if (ret < 0)
+        return NULL;
+
+#ifdef ASAN
+    asan_poison_region((uintptr_t)addr, size, ASAN_POISON_HEAP_LEFT_REDZONE);
+#endif
+    return addr;
 }
 
 #define system_malloc(size) __malloc(size)
 
 static inline void __free(void* addr, size_t size) {
+#ifdef ASAN
+    asan_unpoison_region((uintptr_t)addr, size);
+#endif
     ocall_munmap_untrusted(addr, size);
 }
 

Pal/src/host/Linux-SGX/sgx_main.c

@@ -19,6 +19,7 @@
 #include "toml.h"
 #include "topo_info.h"
 
+#include "asan.h"
 #include "debug_map.h"
 #include "gdb_integration/sgx_gdb.h"
 #include "linux_utils.h"
@@ -1073,12 +1074,33 @@ noreturn static void print_usage_and_exit(const char* argv_0) {
     die_or_inf_loop();
 }
 
+#ifdef ASAN
+__attribute_no_sanitize_address
+static void setup_asan(void) {
+    int prot = PROT_READ | PROT_WRITE;
+    int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED;
+    void* addr = (void*)DO_SYSCALL(mmap, (void*)ASAN_SHADOW_START, ASAN_SHADOW_LENGTH, prot, flags,
+                                   /*fd=*/-1, /*offset=*/0);
+    if (IS_PTR_ERR(addr)) {
+        int err = PTR_TO_ERR(addr);
+        log_error("asan: error setting up shadow memory: %d", err);
+        DO_SYSCALL(exit_group, unix_to_pal_error(err));
+        die_or_inf_loop();
+    }
+}
+#endif
+
+__attribute_no_sanitize_address
 int main(int argc, char* argv[], char* envp[]) {
     char* manifest_path = NULL;
     int ret = 0;
     bool need_gsgx = true;
     char* manifest = NULL;
 
+#ifdef ASAN
+    setup_asan();
+#endif
+
     force_linux_to_grow_stack();
 
     if (argc < 4)

Pal/src/host/Linux/db_main.c

@@ -12,6 +12,7 @@
 #include <linux/personality.h>
 
 #include "api.h"
+#include "asan.h"
 #include "elf/elf.h"
 #include "linux_utils.h"
 #include "pal.h"
@@ -142,13 +143,31 @@ noreturn static void print_usage_and_exit(const char* argv_0) {
     _DkProcessExit(1);
 }
 
+#ifdef ASAN
+__attribute_no_stack_protector
+__attribute_no_sanitize_address
+static void setup_asan(void) {
+    int prot = PROT_READ | PROT_WRITE;
+    int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED;
+    void* addr = (void*)DO_SYSCALL(mmap, (void*)ASAN_SHADOW_START, ASAN_SHADOW_LENGTH, prot, flags,
+                                   /*fd=*/-1, /*offset=*/0);
+    if (IS_PTR_ERR(addr))
+        die_or_inf_loop();
+}
+#endif
+
 /* Gramine uses GCC's stack protector that looks for a canary at gs:[0x8], but this function starts
  * with no TCB in the GS register, so we disable stack protector here */
 __attribute_no_stack_protector
+__attribute_no_sanitize_address
 noreturn void pal_linux_main(void* initial_rsp, void* fini_callback) {
     __UNUSED(fini_callback);  // TODO: We should call `fini_callback` at the end.
     int ret;
 
+#ifdef ASAN
+    setup_asan();
+#endif
+
     /* we don't yet have a TCB in the GS register, but GCC's stack protector will look for a canary
      * at gs:[0x8] in functions called below, so let's install a dummy TCB with a default canary */
     PAL_TCB_LINUX dummy_tcb_for_stack_protector = { 0 };

Pal/src/slab.c

@@ -6,6 +6,7 @@
  */
 
 #include "api.h"
+#include "asan.h"
 #include "pal.h"
 #include "pal_error.h"
 #include "pal_internal.h"
@@ -38,7 +39,7 @@ static inline void __free(void* addr, size_t size);
 static inline void* __malloc(size_t size) {
     void* addr = NULL;
 
-    size = ALIGN_UP(size, MIN_MALLOC_ALIGNMENT);;
+    size = ALIGN_UP(size, MIN_MALLOC_ALIGNMENT);
 
     SYSTEM_LOCK();
     if (g_low + size <= g_high) {
@@ -68,6 +69,10 @@ static inline void* __malloc(size_t size) {
         log_error("*** Out-of-memory in PAL (try increasing `loader.pal_internal_mem_size`) ***");
         _DkProcessExit(ENOMEM);
     }
+#ifdef ASAN
+    asan_poison_region((uintptr_t)addr, ALLOC_ALIGN_UP(size), ASAN_POISON_HEAP_LEFT_REDZONE);
+#endif
+
     return addr;
 }
 
@@ -76,6 +81,9 @@ static inline void* __malloc(size_t size) {
 static inline void __free(void* addr, size_t size) {
     if (!addr)
         return;
+
+    size = ALIGN_UP(size, MIN_MALLOC_ALIGNMENT);
+
     if (addr >= (void*)g_mem_pool && addr < g_mem_pool_end) {
         SYSTEM_LOCK();
         if (addr == g_high) {
@@ -86,10 +94,20 @@ static inline void __free(void* addr, size_t size) {
             g_low = addr;
         }
         /* not a last object from low/high addresses, can't do anything about this case */
+#ifdef ASAN
+        /* Keep the now-unused part of `g_mem_pool` poisoned, because we know it won't be used by
+         * anything other than our allocator */
+        asan_poison_region((uintptr_t)addr, size, ASAN_POISON_HEAP_LEFT_REDZONE);
+#endif
         SYSTEM_UNLOCK();
         return;
     }
 
+#ifdef ASAN
+    /* Unpoison the memory before unmapping it */
+    asan_unpoison_region((uintptr_t)addr, ALLOC_ALIGN_UP(size));
+#endif
+
     _DkVirtualMemoryFree(addr, ALLOC_ALIGN_UP(size));
 }
 
@@ -101,6 +119,10 @@ void init_slab_mgr(void) {
     g_slab_mgr = create_slab_mgr();
     if (!g_slab_mgr)
         INIT_FAIL(PAL_ERROR_NOMEM, "cannot initialize slab manager");
+#ifdef ASAN
+    /* Poison all of `g_mem_pool` initially */
+    asan_poison_region((uintptr_t)&g_mem_pool, sizeof(g_mem_pool), ASAN_POISON_HEAP_LEFT_REDZONE);
+#endif
 }
 
 void* malloc(size_t size) {