Lots of changes around memory managment in the Runtime. Added memory …

…regions and fixed race caused by calling rust_srv::malloc() from multiple threads when sending messages.

src/Makefile

@@ -266,7 +266,8 @@ RUNTIME_CS := rt/sync/timer.cpp \
               rt/rust_timer.cpp \
               rt/circular_buffer.cpp \
               rt/isaac/randport.cpp \
-              rt/rust_srv.cpp
+              rt/rust_srv.cpp \
+              rt/memory_region.cpp
 
 RUNTIME_HDR := rt/globals.h \
                rt/rust.h \
@@ -285,7 +286,9 @@ RUNTIME_HDR := rt/globals.h \
                rt/util/hash_map.h \
                rt/sync/sync.h \
                rt/sync/timer.h \
-               rt/rust_srv.h
+               rt/rust_srv.h \
+               rt/memory_region.h \
+               rt/memory.h
 
 RUNTIME_INCS := -Irt/isaac -Irt/uthash
 RUNTIME_OBJS := $(RUNTIME_CS:.cpp=$(CFG_OBJ_SUFFIX))

src/rt/memory.h

@@ -0,0 +1,60 @@
+/*
+ *
+ */
+
+#ifndef MEMORY_H
+#define MEMORY_H
+
+
+inline void *operator new(size_t size, void *mem) {
+    return mem;
+}
+
+inline void *operator new(size_t size, rust_dom *dom) {
+    return dom->malloc(size, memory_region::LOCAL);
+}
+
+inline void *operator new[](size_t size, rust_dom *dom) {
+    return dom->malloc(size, memory_region::LOCAL);
+}
+
+inline void *operator new(size_t size, rust_dom &dom) {
+    return dom.malloc(size, memory_region::LOCAL);
+}
+
+inline void *operator new[](size_t size, rust_dom &dom) {
+    return dom.malloc(size, memory_region::LOCAL);
+}
+
+inline void *operator new(size_t size, rust_dom *dom,
+    memory_region::memory_region_type type) {
+    return dom->malloc(size, type);
+}
+
+inline void *operator new[](size_t size, rust_dom *dom,
+    memory_region::memory_region_type type) {
+    return dom->malloc(size, type);
+}
+
+inline void *operator new(size_t size, rust_dom &dom,
+    memory_region::memory_region_type type) {
+    return dom.malloc(size, type);
+}
+
+inline void *operator new[](size_t size, rust_dom &dom,
+    memory_region::memory_region_type type) {
+    return dom.malloc(size, type);
+}
+
+inline void operator delete(void *mem, rust_dom *dom) {
+    dom->free(mem, memory_region::LOCAL);
+    return;
+}
+
+inline void operator delete(void *mem, rust_dom *dom,
+    memory_region::memory_region_type type) {
+    dom->free(mem, type);
+    return;
+}
+
+#endif /* MEMORY_H */

src/rt/memory_region.cpp

@@ -0,0 +1,100 @@
+/*
+ *
+ */
+
+#include "rust_internal.h"
+#include "memory_region.h"
+
+#define TRACK_ALLOCATIONS
+
+memory_region::memory_region(rust_srv *srv, bool synchronized) :
+    _srv(srv), _parent(NULL), _live_allocations(0),
+    _synchronized(synchronized) {
+    // Nop.
+}
+
+memory_region::memory_region(memory_region *parent) :
+    _srv(parent->_srv), _parent(parent), _live_allocations(0),
+    _synchronized(parent->_synchronized) {
+    // Nop.
+}
+
+void memory_region::free(void *mem) {
+    if (_synchronized) { _lock.lock(); }
+#ifdef TRACK_ALLOCATIONS
+    if (_allocation_list.replace(mem, NULL) == false) {
+        printf("free: ptr 0x%" PRIxPTR " is not in allocation_list\n",
+            (uintptr_t) mem);
+        _srv->fatal("not in allocation_list", __FILE__, __LINE__, "");
+    }
+#endif
+    if (_live_allocations < 1) {
+        _srv->fatal("live_allocs < 1", __FILE__, __LINE__, "");
+    }
+    _live_allocations--;
+    _srv->free(mem);
+    if (_synchronized) { _lock.unlock(); }
+
+}
+
+void *
+memory_region::realloc(void *mem, size_t size) {
+    if (_synchronized) { _lock.lock(); }
+    if (!mem) {
+        _live_allocations++;
+    }
+    void *newMem = _srv->realloc(mem, size);
+#ifdef TRACK_ALLOCATIONS
+    if (_allocation_list.replace(mem, newMem) == false) {
+        printf("realloc: ptr 0x%" PRIxPTR " is not in allocation_list\n",
+            (uintptr_t) mem);
+        _srv->fatal("not in allocation_list", __FILE__, __LINE__, "");
+    }
+#endif
+    if (_synchronized) { _lock.unlock(); }
+    return newMem;
+}
+
+void *
+memory_region::malloc(size_t size) {
+    if (_synchronized) { _lock.lock(); }
+    _live_allocations++;
+    void *mem = _srv->malloc(size);
+#ifdef TRACK_ALLOCATIONS
+    _allocation_list.append(mem);
+#endif
+    if (_synchronized) { _lock.unlock(); }
+    return mem;
+}
+
+void *
+memory_region::calloc(size_t size) {
+    if (_synchronized) { _lock.lock(); }
+    _live_allocations++;
+    void *mem = _srv->malloc(size);
+    memset(mem, 0, size);
+#ifdef TRACK_ALLOCATIONS
+    _allocation_list.append(mem);
+#endif
+    if (_synchronized) { _lock.unlock(); }
+    return mem;
+}
+
+memory_region::~memory_region() {
+    if (_live_allocations == 0) {
+        return;
+    }
+    char msg[128];
+    snprintf(msg, sizeof(msg),
+        "leaked memory in rust main loop (%" PRIuPTR " objects)",
+        _live_allocations);
+#ifdef TRACK_ALLOCATIONS
+    for (size_t i = 0; i < _allocation_list.size(); i++) {
+        if (_allocation_list[i] != NULL) {
+            printf("allocation 0x%" PRIxPTR " was not freed\n",
+                (uintptr_t) _allocation_list[i]);
+        }
+    }
+#endif
+    _srv->fatal(msg, __FILE__, __LINE__, "%d objects", _live_allocations);
+}

src/rt/memory_region.h

@@ -0,0 +1,37 @@
+/*
+ * The Rust runtime uses memory regions to provide a primitive level of
+ * memory management and isolation between tasks, and domains.
+ *
+ * TODO: Implement a custom lock-free malloc / free instead of relying solely
+ *       on the standard malloc / free.
+ */
+
+#ifndef MEMORY_REGION_H
+#define MEMORY_REGION_H
+
+#include "sync/spin_lock.h"
+
+class rust_srv;
+
+class memory_region {
+private:
+    rust_srv *_srv;
+    memory_region *_parent;
+    size_t _live_allocations;
+    array_list<void *> _allocation_list;
+    const bool _synchronized;
+    spin_lock _lock;
+public:
+    enum memory_region_type {
+        LOCAL = 0x1, SYNCHRONIZED = 0x2
+    };
+    memory_region(rust_srv *srv, bool synchronized);
+    memory_region(memory_region *parent);
+    void *malloc(size_t size);
+    void *calloc(size_t size);
+    void *realloc(void *mem, size_t size);
+    void free(void *mem);
+    virtual ~memory_region();
+};
+
+#endif /* MEMORY_REGION_H */

src/rt/rust.h

@@ -21,11 +21,6 @@
 
 #include "rust_srv.h"
 
-inline void *operator new(size_t size, rust_srv *srv)
-{
-    return srv->malloc(size);
-}
-
 /*
  * Local Variables:
  * fill-column: 78;

src/rt/rust_builtin.cpp

@@ -38,7 +38,7 @@ last_os_error(rust_task *task) {
 #endif
     size_t fill = strlen(buf) + 1;
     size_t alloc = next_power_of_two(sizeof(rust_str) + fill);
-    void *mem = dom->malloc(alloc);
+    void *mem = dom->malloc(alloc, memory_region::LOCAL);
     if (!mem) {
         task->fail(1);
         return NULL;
@@ -134,7 +134,7 @@ str_alloc_with_data(rust_task *task,
 {
     rust_dom *dom = task->dom;
     size_t alloc = next_power_of_two(sizeof(rust_str) + n_bytes);
-    void *mem = dom->malloc(alloc);
+    void *mem = dom->malloc(alloc, memory_region::LOCAL);
     if (!mem)
         return NULL;
     rust_str *st = new (mem) rust_str(dom, alloc, fill, d);

src/rt/rust_dom.cpp

@@ -13,6 +13,8 @@ rust_dom::rust_dom(rust_srv *srv, rust_crate const *root_crate,
     root_crate(root_crate),
     _log(srv, this),
     srv(srv),
+    local_region(&srv->local_region),
+    synchronized_region(&srv->synchronized_region),
     name(name),
     running_tasks(this),
     blocked_tasks(this),
@@ -144,36 +146,65 @@ rust_dom::fail() {
 }
 
 void *
-rust_dom::malloc(size_t sz) {
-    void *p = srv->malloc(sz);
-    I(this, p);
-    log(rust_log::MEM,
-        "%s @0x%" PRIxPTR " rust_dom::malloc(%d) -> 0x%" PRIxPTR,
-        name, (uintptr_t) this, sz, p);
-    return p;
+rust_dom::malloc(size_t size) {
+    return malloc(size, memory_region::LOCAL);
 }
 
 void *
-rust_dom::calloc(size_t sz) {
-    void *p = this->malloc(sz);
-    memset(p, 0, sz);
-    return p;
+rust_dom::malloc(size_t size, memory_region::memory_region_type type) {
+    if (type == memory_region::LOCAL) {
+        return local_region.malloc(size);
+    } else if (type == memory_region::SYNCHRONIZED) {
+        return synchronized_region.malloc(size);
+    }
+    return NULL;
+}
+
+void *
+rust_dom::calloc(size_t size) {
+    return calloc(size, memory_region::LOCAL);
+}
+
+void *
+rust_dom::calloc(size_t size, memory_region::memory_region_type type) {
+    if (type == memory_region::LOCAL) {
+        return local_region.calloc(size);
+    } else if (type == memory_region::SYNCHRONIZED) {
+        return synchronized_region.calloc(size);
+    }
+    return NULL;
 }
 
 void *
-rust_dom::realloc(void *p, size_t sz) {
-    void *p1 = srv->realloc(p, sz);
-    I(this, p1);
-    log(rust_log::MEM, "rust_dom::realloc(0x%" PRIxPTR ", %d) -> 0x%" PRIxPTR,
-        p, sz, p1);
-    return p1;
+rust_dom::realloc(void *mem, size_t size) {
+    return realloc(mem, size, memory_region::LOCAL);
+}
+
+void *
+rust_dom::realloc(void *mem, size_t size,
+    memory_region::memory_region_type type) {
+    if (type == memory_region::LOCAL) {
+        return local_region.realloc(mem, size);
+    } else if (type == memory_region::SYNCHRONIZED) {
+        return synchronized_region.realloc(mem, size);
+    }
+    return NULL;
 }
 
 void
-rust_dom::free(void *p) {
-    log(rust_log::MEM, "rust_dom::free(0x%" PRIxPTR ")", p);
-    I(this, p);
-    srv->free(p);
+rust_dom::free(void *mem) {
+    free(mem, memory_region::LOCAL);
+}
+
+void
+rust_dom::free(void *mem, memory_region::memory_region_type type) {
+    log(rust_log::MEM, "rust_dom::free(0x%" PRIxPTR ")", mem);
+    if (type == memory_region::LOCAL) {
+        local_region.free(mem);
+    } else if (type == memory_region::SYNCHRONIZED) {
+        synchronized_region.free(mem);
+    }
+    return;
 }
 
 #ifdef __WIN32__
@@ -264,7 +295,6 @@ void rust_dom::send_message(rust_message *message) {
                         message,
                         &_incoming_message_queue,
                         this);
-    A(this, message->dom == this, "Message owned by non-local domain.");
     _incoming_message_queue.enqueue(message);
 }
 
@@ -277,7 +307,8 @@ void rust_dom::drain_incoming_message_queue() {
         log(rust_log::COMM, "<== processing incoming message \"%s\" 0x%"
             PRIxPTR, message->label, message);
         message->process();
-        delete message;
+        message->~rust_message();
+        this->synchronized_region.free(message);
     }
 }
 
@@ -322,8 +353,7 @@ rust_dom::get_port_proxy_synchronized(rust_port *port) {
  * Returns NULL if no tasks can be scheduled.
  */
 rust_task *
-rust_dom::schedule_task()
-{
+rust_dom::schedule_task() {
     I(this, this);
     // FIXME: in the face of failing tasks, this is not always right.
     // I(this, n_live_tasks() > 0);