Implement serialization of NativeCall reprs with dyncall

When a NativeCall using module is loaded both by the current computation unit and one of its dependencies, native subs may become repossessed. Repossession will clear the MVMNativeCallBody structure as this is supposed to be repopulated on deserialization. This did not happen causing segfaults when the generated function body tried to invoke the JIT compiled calling code. Fixed this by implementing serialization/deserialization of NativeCalls. On deserialization we will use the stored lib_name and sym_name to load the native library and also try to JIT compile the function body.
MoarVM · Oct 13, 2019 · 95f69be · 95f69be
1 parent 1b0950e
commit 95f69be
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 24 deletions.
diff --git a/src/6model/reprs/NativeCall.c b/src/6model/reprs/NativeCall.c
@@ -66,11 +66,50 @@ static const MVMStorageSpec * get_storage_spec(MVMThreadContext *tc, MVMSTable *
  * we just do nothing here since it may well have never been opened. Various
  * more involved approaches are possible. */
 static void serialize(MVMThreadContext *tc, MVMSTable *st, void *data, MVMSerializationWriter *writer) {
+#ifndef HAVE_LIBFFI
+    MVMNativeCallBody *body = (MVMNativeCallBody *)data;
+    MVMint16 i = 0;
+    MVM_serialization_write_cstr(tc, writer, body->lib_name);
+    MVM_serialization_write_cstr(tc, writer, body->sym_name);
+    MVM_serialization_write_int(tc, writer, body->convention);
+    MVM_serialization_write_int(tc, writer, body->num_args);
+    MVM_serialization_write_int(tc, writer, body->ret_type);
+    /* TODO ffi support */
+    for (i = 0; i < body->num_args; i++) {
+        MVM_serialization_write_int(tc, writer, body->arg_types[i]);
+    }
+    for (i = 0; i < body->num_args; i++) {
+        MVM_serialization_write_ref(tc, writer, body->arg_info[i]);
+    }
+#endif
 }
 static void deserialize_stable_size(MVMThreadContext *tc, MVMSTable *st, MVMSerializationReader *reader) {
     st->size = sizeof(MVMNativeCall);
 }
 static void deserialize(MVMThreadContext *tc, MVMSTable *st, MVMObject *root, void *data, MVMSerializationReader *reader) {
+#ifndef HAVE_LIBFFI
+    MVMNativeCallBody *body = (MVMNativeCallBody *)data;
+    MVMint16 i = 0;
+    if (reader->root.version >= 22) {
+        body->lib_name = MVM_serialization_read_cstr(tc, reader);
+        body->sym_name = MVM_serialization_read_cstr(tc, reader);
+        body->convention = MVM_serialization_read_int(tc, reader);
+        body->num_args = MVM_serialization_read_int(tc, reader);
+        body->ret_type = MVM_serialization_read_int(tc, reader);
+        body->arg_types = MVM_malloc(body->num_args * sizeof(MVMint16));
+        body->arg_info  = MVM_malloc(body->num_args * sizeof(MVMObject*));
+        /* TODO ffi support */
+        for (i = 0; i < body->num_args; i++) {
+            body->arg_types[i] = MVM_serialization_read_int(tc, reader);
+        }
+        for (i = 0; i < body->num_args; i++) {
+            body->arg_info[i] = MVM_serialization_read_ref(tc, reader);
+        }
+        if (body->lib_name && body->sym_name) {
+            MVM_nativecall_setup(tc, body, 0);
+        }
+    }
+#endif
 }
 
 static void gc_mark(MVMThreadContext *tc, MVMSTable *st, void *data, MVMGCWorklist *worklist) {

diff --git a/src/6model/serialization.c b/src/6model/serialization.c
@@ -10,7 +10,7 @@
 
 /* Version of the serialization format that we are currently at and lowest
  * version we support. */
-#define CURRENT_VERSION 21
+#define CURRENT_VERSION 22
 #define MIN_VERSION     16
 
 /* Various sizes (in bytes). */

diff --git a/src/6model/serialization.h b/src/6model/serialization.h
@@ -190,11 +190,13 @@ MVMint64 MVM_serialization_read_int64(MVMThreadContext *tc, MVMSerializationRead
 MVMint64 MVM_serialization_read_int(MVMThreadContext *tc, MVMSerializationReader *reader);
 MVMnum64 MVM_serialization_read_num(MVMThreadContext *tc, MVMSerializationReader *reader);
 MVMString * MVM_serialization_read_str(MVMThreadContext *tc, MVMSerializationReader *reader);
+char *MVM_serialization_read_cstr(MVMThreadContext *tc, MVMSerializationReader *reader);
 MVM_PUBLIC MVMObject * MVM_serialization_read_ref(MVMThreadContext *tc, MVMSerializationReader *reader);
 MVMSTable * MVM_serialization_read_stable_ref(MVMThreadContext *tc, MVMSerializationReader *reader);
 void MVM_serialization_force_stable(MVMThreadContext *tc, MVMSerializationReader *reader, MVMSTable *st);
 void MVM_serialization_write_int(MVMThreadContext *tc, MVMSerializationWriter *writer, MVMint64 value);
 void MVM_serialization_write_num(MVMThreadContext *tc, MVMSerializationWriter *writer, MVMnum64 value);
 void MVM_serialization_write_str(MVMThreadContext *tc, MVMSerializationWriter *writer, MVMString *value);
+void MVM_serialization_write_cstr(MVMThreadContext *tc, MVMSerializationWriter *writer, char *string);
 MVM_PUBLIC void MVM_serialization_write_ref(MVMThreadContext *tc, MVMSerializationWriter *writer, MVMObject *ref);
 void MVM_serialization_write_stable_ref(MVMThreadContext *tc, MVMSerializationWriter *writer, MVMSTable *st);
diff --git a/src/core/nativecall.c b/src/core/nativecall.c
@@ -633,6 +633,40 @@ MVMJitCode *create_caller_code(MVMThreadContext *tc, MVMNativeCallBody *body) {
     return jitcode;
 }
 
+void MVM_nativecall_setup(MVMThreadContext *tc, MVMNativeCallBody *body, unsigned int interval_id) {
+    /* Try to load the library. */
+    body->lib_handle = MVM_nativecall_load_lib(body->lib_name[0] ? body->lib_name : NULL);
+
+    if (!body->lib_handle) {
+        char *waste[] = { body->lib_name, NULL };
+        MVM_free(body->sym_name);
+        body->lib_name = NULL;
+        body->sym_name = NULL;
+        if (interval_id)
+            MVM_telemetry_interval_stop(tc, interval_id, "error building native call");
+        MVM_exception_throw_adhoc_free(tc, waste, "Cannot locate native library '%s': %s", body->lib_name, dlerror());
+    }
+
+    if (!body->entry_point) {
+        body->entry_point = MVM_nativecall_find_sym(body->lib_handle, body->sym_name);
+        if (!body->entry_point) {
+            char *waste[] = { body->sym_name, body->lib_name, NULL };
+            body->lib_name = NULL;
+            body->sym_name = NULL;
+            if (interval_id)
+                MVM_telemetry_interval_stop(tc, interval_id, "error building native call");
+            MVM_exception_throw_adhoc_free(tc, waste, "Cannot locate symbol '%s' in native library '%s'",
+                body->sym_name, body->lib_name);
+        }
+    }
+
+    if (tc->instance->jit_enabled) {
+        body->jitcode = create_caller_code(tc, body);
+    }
+    else
+        body->jitcode = NULL;
+}
+
 /* Builds up a native call site out of the supplied arguments. */
 MVMint8 MVM_nativecall_build(MVMThreadContext *tc, MVMObject *site, MVMString *lib,
         MVMString *sym, MVMString *conv, MVMObject *arg_info, MVMObject *ret_info) {
@@ -649,33 +683,17 @@ MVMint8 MVM_nativecall_build(MVMThreadContext *tc, MVMObject *site, MVMString *l
     /* Initialize the object; grab native call part of its body. */
     MVMNativeCallBody *body = MVM_nativecall_get_nc_body(tc, site);
 
-    /* Try to load the library. */
     body->lib_name = lib_name;
-    body->lib_handle = MVM_nativecall_load_lib(lib_name[0] ? lib_name : NULL);
-
-    if (!body->lib_handle) {
-        char *waste[] = { lib_name, NULL };
-        MVM_free(sym_name);
-        MVM_telemetry_interval_stop(tc, interval_id, "error building native call");
-        MVM_exception_throw_adhoc_free(tc, waste, "Cannot locate native library '%s': %s", lib_name, dlerror());
-    }
 
     /* Try to locate the symbol. */
-    if (entry_point_o) {
+    if (entry_point_o && !MVM_is_null(tc, entry_point_o)) {
         body->entry_point = MVM_nativecall_unmarshal_cpointer(tc, entry_point_o, MVM_NATIVECALL_UNMARSHAL_KIND_GENERIC);
         body->sym_name    = sym_name;
         keep_sym_name     = 1;
     }
 
     if (!body->entry_point) {
-        body->entry_point = MVM_nativecall_find_sym(body->lib_handle, sym_name);
-        if (!body->entry_point) {
-            char *waste[] = { sym_name, lib_name, NULL };
-            MVM_telemetry_interval_stop(tc, interval_id, "error building native call");
-            MVM_exception_throw_adhoc_free(tc, waste, "Cannot locate symbol '%s' in native library '%s'",
-                sym_name, lib_name);
-        }
-        body->sym_name = sym_name;
+        body->sym_name    = sym_name;
         keep_sym_name     = 1;
     }
 
@@ -715,11 +733,8 @@ MVMint8 MVM_nativecall_build(MVMThreadContext *tc, MVMObject *site, MVMString *l
 #ifdef HAVE_LIBFFI
     body->ffi_ret_type = MVM_nativecall_get_ffi_type(tc, body->ret_type);
 #endif
-    if (tc->instance->jit_enabled) {
-        body->jitcode = create_caller_code(tc, body);
-    }
-    else
-        body->jitcode = NULL;
+
+    MVM_nativecall_setup(tc, body, interval_id);
 
     MVM_telemetry_interval_stop(tc, interval_id, "nativecall built");
 

diff --git a/src/core/nativecall.h b/src/core/nativecall.h
@@ -94,6 +94,7 @@ MVMNativeCallBody * MVM_nativecall_get_nc_body(MVMThreadContext *tc, MVMObject *
 MVMint16 MVM_nativecall_get_arg_type(MVMThreadContext *tc, MVMObject *info, MVMint16 is_return);
 MVMint8 MVM_nativecall_build(MVMThreadContext *tc, MVMObject *site, MVMString *lib,
     MVMString *sym, MVMString *conv, MVMObject *arg_spec, MVMObject *ret_spec);
+void MVM_nativecall_setup(MVMThreadContext *tc, MVMNativeCallBody *body, unsigned int interval_id);
 MVMObject * MVM_nativecall_invoke(MVMThreadContext *tc, MVMObject *res_type,
     MVMObject *site, MVMObject *args);
 void MVM_nativecall_invoke_jit(MVMThreadContext *tc, MVMObject *site);