Merge pull request #3143 from gacholio/alloc

Prevent allocation of replaced classes

runtime/gc_base/JavaObjectAllocationModel.hpp

@@ -1,6 +1,6 @@
 
 /*******************************************************************************
- * Copyright (c) 1991, 2014 IBM Corp. and others
+ * Copyright (c) 1991, 2018 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -64,7 +64,7 @@ class MM_JavaObjectAllocationModel : public MM_AllocateInitialization
 protected:
 
 public:
-	MMINLINE J9Class *getClass() { return _class; }
+	MMINLINE J9Class *getClass() { return J9_CURRENT_CLASS(_class); }
 
 	/**
 	 * Initializer.
@@ -77,7 +77,7 @@ class MM_JavaObjectAllocationModel : public MM_AllocateInitialization
 		if (NULL != objectPtr) {
 			/* Initialize class pointer in object header -- preserve flags set by base class */
 			MM_GCExtensions *extensions = MM_GCExtensions::getExtensions(env);
-			extensions->objectModel.setObjectClass(objectPtr, _class);
+			extensions->objectModel.setObjectClass(objectPtr, getClass());
 
 			/* This might set the remembered bit in the header flags ... */
 			J9VMThread *vmThread = (J9VMThread *)env->getOmrVMThread()->_language_vmthread;

runtime/gc_modron_startup/mgcalloc.cpp

@@ -344,6 +344,11 @@ J9AllocateObject(J9VMThread *vmThread, J9Class *clazz, uintptr_t allocateFlags)
 	Assert_MM_false(allocateFlags & OMR_GC_ALLOCATE_OBJECT_NO_GC);
 
 	J9Object *objectPtr = NULL;
+	/* Replaced classes have poisoned the totalInstanceSize such that they are not allocatable,
+	 * so inline allocate and NoGC allocate have already failed. If this allocator is reached
+	 * with a replaced class, update to the current version and allocate that.
+	 */
+	clazz = J9_CURRENT_CLASS(clazz);
 	MM_MixedObjectAllocationModel mixedOAM(env, clazz, allocateFlags);
 	if (mixedOAM.initializeAllocateDescription(env)) {
 		objectPtr = OMR_GC_AllocateObject(vmThread->omrVMThread, &mixedOAM);

runtime/util/hshelp.c

@@ -2026,6 +2026,11 @@ copyPreservedValues(J9VMThread * currentThread, J9HashTable * classPairs, UDATA
 			replacementRAMClass->arrayClass = arrayClass;
 			originalRAMClass->arrayClass = replacementRAMClass;
 			originalRAMClass->classDepthAndFlags |= J9AccClassHotSwappedOut;
+			/* Set the totalInstanceSize in the replaced class to a value so large that it
+			 * can never be allocated (but not so large as to overflow the arithmetic when
+			 * the header size addition and rounding are done).
+			 */
+			originalRAMClass->totalInstanceSize = (UDATA)-256;
 		}
 		classPair = hashTableNextDo(&hashTableState);
 	}