Modified PyMem_malloc/realloc/free to provide prepended memory for a …

…JyObject header. This is needed for extensions such as numpy, which use PyMem_malloc and friends for PyObject allocation (rather than PyObject_malloc and friends). Fixes e.g. issue #9 more or less.

JyNI-C/include/JyNI.h

@@ -513,15 +513,15 @@ extern const char* JyAttributeWeakRefCount;
 #define JY_ATTR_VAR_SIZE_FLAG_MASK 2
 #define JY_ATTR_JWEAK_VALUE_FLAG_MASK 4
 typedef struct JyAttribute JyAttribute; /* Forward declaration */
-struct JyAttribute { const char* name; void* value; char flags; JyAttribute* next;};
+struct JyAttribute { void* value; JyAttribute* next; const char* name;  char flags;};
 typedef struct JyAttributeElement JyAttributeElement; /* Forward declaration */
 struct JyAttributeElement {void* value; JyAttributeElement* next;};
-typedef struct { jweak jy; unsigned short flags; JyAttribute* attr;} JyObject;
+typedef struct { jweak jy; JyAttribute* attr; unsigned short flags;} JyObject;
 typedef struct { JyObject jy; PyIntObject pyInt;} JyIntObject; /* only used for pre-allocated blocks */
 typedef struct { JyObject jy; PyFloatObject pyFloat;} JyFloatObject;  /* only used for pre-allocated blocks */
 /* type_name is optional and defaults to py_type->tp_name */
-typedef struct { PyTypeObject* py_type; jclass jy_class; jclass jy_subclass; unsigned short flags;
-		SyncFunctions* sync; size_t truncate_trailing; char* type_name;} TypeMapEntry;
+typedef struct { PyTypeObject* py_type; jclass jy_class; jclass jy_subclass;
+		SyncFunctions* sync; size_t truncate_trailing; unsigned short flags; char* type_name;} TypeMapEntry;
 typedef struct { PyTypeObject* exc_type; jyFactoryMethod exc_factory;} ExceptionMapEntry;
 
 #define JyObject_HasJyGCHead(pyObject, jyObject) \
@@ -566,7 +566,7 @@ typedef struct { PyTypeObject* exc_type; jyFactoryMethod exc_factory;} Exception
 #define FROM_JY_WITH_GC(o) (JyNI_FROM_GC((((JyObject *)(o))+1)))
 #define FROM_JY_NO_GC(o) ((PyObject *)(((JyObject *)(o))+1))
 #define AS_JY_WITH_GC(o) ((JyObject *)(_Py_AS_GC(o))-1)
-#define AS_JY_NO_GC(o) ((JyObject *)(o)-1)
+#define AS_JY_NO_GC(o) (((JyObject *)(o))-1)
 
 #define JySYNC_ON_INIT_FLAGS (SYNC_ON_PY_INIT_FLAG_MASK | SYNC_ON_JY_INIT_FLAG_MASK)
 #define Jy_InitImmutable(jyObj) \
@@ -896,7 +896,10 @@ inline PyObject* JyNI_NewPyObject_FromJythonPyObject(jobject jythonPyObject);
  */
 
 /* JyAttribute management: */
-inline void JyNI_ClearJyAttributes(JyObject* obj);
+#define JyNI_ClearJyAttributes(obj) \
+		_JyNI_ClearJyAttributes(obj)
+		//(jputs(__FUNCTION__), jputsLong(__LINE__), _JyNI_ClearJyAttributes(obj))
+inline void _JyNI_ClearJyAttributes(JyObject* obj);
 inline void JyNI_ClearJyAttribute(JyObject* obj, const char* name);
 inline void JyNI_ClearJyAttributeValue(JyAttribute* att);
 #define JyNI_GetJyAttribute(obj, name) \

JyNI-C/include/Python_JyNI/pymem_JyNI.h

@@ -83,6 +83,7 @@ extern "C" {
 PyAPI_FUNC(void *) PyMem_Malloc(size_t);
 PyAPI_FUNC(void *) PyMem_Realloc(void *, size_t);
 PyAPI_FUNC(void) PyMem_Free(void *);
+//PyAPI_FUNC(void) _PyMem_Free(void *);
 
 /* Starting from Python 1.6, the wrappers Py_{Malloc,Realloc,Free} are
    no longer supported. They used to call PyErr_NoMemory() on failure. */

JyNI-C/src/JyAttributes.c

@@ -50,7 +50,7 @@ const char* JyAttributeWeakRefCount   = "wrct";
 //defaults to 0; note that on alloc this value is added to the anyway allocated size sizeof(PyObjectHead)
 //const char* JyAttributeTruncateSize = "trSi";
 
-inline void JyNI_ClearJyAttributes(JyObject* obj)
+inline void _JyNI_ClearJyAttributes(JyObject* obj)
 {
 	JyAttribute* nxt = obj->attr;
 	while (nxt != NULL)

JyNI-C/src/JyNI.c

@@ -2260,6 +2260,7 @@ inline jobject JyNI_JythonPyObject_FromPyObject(PyObject* op)
 	 * Heap-types are treated like ordinary objects.
 	 * Note: Don't confuse the following line with checking op->ob_type rather than op itself.
 	 */
+//	jboolean dbg = strcmp("numpy.ufunc", Py_TYPE(op)->tp_name) == 0;
 	env(NULL);
 	if (PyType_Check(op) && !PyType_HasFeature((PyTypeObject*) op, Py_TPFLAGS_HEAPTYPE))
 	{
@@ -2306,13 +2307,18 @@ inline jobject JyNI_JythonPyObject_FromPyObject(PyObject* op)
 					JyNI_SyncPy2Jy(op, jy);
 				return result;
 			} else {
+//				if (!(*env)->IsSameObject(env, result, NULL)) jputs("False positive!");
 				/* This is actually okay. It can happen if the Java-counterpart
 				 * of a mirrored object was collected by Java-gc.
 				 */
-				//jputs("JyNI-Warning: Deleted object was not cleared!");
+//				jputs("JyNI-Warning: Deleted object was not cleared!");
+//				if (dbg)
+//				{
+//					jputs(Py_TYPE(op)->tp_name);
+//					jputsLong(op);
+//				}
 				JyNI_CleanUp_JyObject(jy);
 			}
-			//jputsLong(__LINE__);
 		}
 		//jputsLong(__LINE__);
 //		if ((*env)->IsSameObject(env, jy->jy, NULL)) {
@@ -2333,13 +2339,16 @@ inline jobject JyNI_JythonPyObject_FromPyObject(PyObject* op)
 		//if (bl) jputsLong(__LINE__);
 		//printf("%d_______%s\n", __LINE__, __FUNCTION__);
 		tme = (TypeMapEntry*) jy->jy;
+		//if (dbg && tme) {jputs("ufunc tme:"); jputsLong(__LINE__); jputsLong(op);}
 	} else {
 		//if (bl) jputsLong(__LINE__);
 		//printf("%d_______%s\n", __LINE__, __FUNCTION__);
 		tme = JyNI_JythonTypeEntry_FromPyType(Py_TYPE(op));
+		//if (dbg && tme) {jputs("ufunc tme:"); jputsLong(__LINE__); jputsLong(op);}
 		if (!tme) {
 			//printf("%d_______%s\n", __LINE__, __FUNCTION__);
 			tme = JyNI_JythonTypeEntry_FromSubTypeWithPeer(Py_TYPE(op));
+			//if (dbg && tme) {jputs("ufunc tme:"); jputsLong(__LINE__); jputsLong(op);}
 		}
 	}
 	//jputsLong(tme);
@@ -2394,7 +2403,7 @@ inline jobject JyNI_JythonPyObject_FromPyObject(PyObject* op)
 //				}
 			//jputs("opType-address:");
 			//printf("%u\n", (jlong) opType);
-//			jputs("create PyCPeer for ");
+			//if (dbg) {jputs("create PyCPeer for ufunc"); jputsLong(op);}
 //			jputs(Py_TYPE(op)->tp_name);
 			if (PyType_Check(op)) {
 				return _JyNI_JythonPyTypeObject_FromPyTypeObject((PyTypeObject*) op, NULL);
@@ -2405,6 +2414,7 @@ inline jobject JyNI_JythonPyObject_FromPyObject(PyObject* op)
 					(*env)->NewObject(env, pyCPeerGCClass, pyCPeerGCConstructor, (jlong) op, opType) :
 					(*env)->NewObject(env, pyCPeerClass, pyCPeerConstructor, (jlong) op, opType);
 			//jobject er = (*env)->NewObject(env, pyCPeerClass, pyCPeerConstructor, (jlong) op, opType);
+			//if (dbg) {jputsLong(__LINE__); jputsLong(op);}
 			jy->flags |= JY_INITIALIZED_FLAG_MASK;
 			jy->flags |= JY_CPEER_FLAG_MASK;
 			jy->jy = (*env)->NewWeakGlobalRef(env, er);
@@ -2719,6 +2729,7 @@ inline jint JyNI_GetDLOpenFlags()
 inline void JyNI_CleanUp_JyObject(JyObject* obj)
 {
 //	jputs(__FUNCTION__);
+//	jputs(Py_TYPE(FROM_JY(obj))->tp_name);
 //	jputsLong(FROM_JY(obj));
 	//if (FROM_JY(obj) == Py_None) return;
 	//if (obj == NULL) return;

JyNI-C/src/Objects/dictobject.c

@@ -2731,8 +2731,8 @@ PyDict_GetItemString(PyObject *v, const char *key)
 int
 PyDict_SetItemString(PyObject *v, const char *key, PyObject *item)
 {
-	//puts("PyDict_SetItemString");
-	//puts(key);
+//	jputs("PyDict_SetItemString");
+//	jputs(key);
 	if (!PyDict_Check(v)) {
 		return -1;
 	}

JyNI-C/src/Objects/object.c

@@ -2965,24 +2965,45 @@ PyTypeObject *_Py_cobject_hack = &PyCObject_Type;
 Py_ssize_t (*_Py_abstract_hack)(PyObject *) = PyObject_Size;
 */
 
-// Python's malloc wrappers (see pymem.h)
+/* Python's malloc wrappers (see pymem.h) */
+
+/* JyNI-Note:
+ * Some extensions actually use PyMem_Malloc/Realloc/Free
+ * for PyObject subtypes, e.g. for numpy.core.ufunc.
+ * So we have to allocate extra space for JyObject header,
+ * although this often might waste memory. (At least the
+ * potentially memory-heavy numpy-extension uses its own
+ * malloc-family wrappers for array data
+ * (PyDataMem_NEW/FREE/RENEW in multiarray/alloc.c).
+ */
 
 void *
 PyMem_Malloc(size_t nbytes)
 {
-	return PyMem_MALLOC(nbytes);
+	JyObject* er = (JyObject*) PyMem_MALLOC(nbytes+sizeof(JyObject));
+	er->attr = NULL;
+	er->flags = 0;
+	er->jy = NULL;
+	return FROM_JY_NO_GC(er);
 }
 
 void *
 PyMem_Realloc(void *p, size_t nbytes)
 {
-	return PyMem_REALLOC(p, nbytes);
+	JyObject* er = (JyObject*) PyMem_REALLOC(AS_JY_NO_GC(p), nbytes+sizeof(JyObject));
+	er->attr = NULL;
+	er->flags = 0;
+	er->jy = NULL;
+	return FROM_JY_NO_GC(er);
 }
 
 void
 PyMem_Free(void *p)
 {
-	PyMem_FREE(p);
+	if (p)
+	{
+		PyMem_FREE(AS_JY_NO_GC(p));
+	}
 }
 
 /*