Merge pull request #8324 from gacholio/compress

Runtime compressed refs work

runtime/compiler/env/VMJ9.cpp

@@ -1016,7 +1016,7 @@ UDATA TR_J9VMBase::thisThreadGetReturnValueOffset()                 {return offs
 UDATA TR_J9VMBase::getThreadDebugEventDataOffset(int32_t index) {J9VMThread *dummy=0; return offsetof(J9VMThread, debugEventData1) + (index-1)*sizeof(dummy->debugEventData1);} // index counts from 1
 UDATA TR_J9VMBase::getThreadLowTenureAddressPointerOffset()         {return offsetof(J9VMThread, lowTenureAddress);}
 UDATA TR_J9VMBase::getThreadHighTenureAddressPointerOffset()        {return offsetof(J9VMThread, highTenureAddress);}
-UDATA TR_J9VMBase::getObjectHeaderSizeInBytes()                     {return sizeof(J9Object);}
+UDATA TR_J9VMBase::getObjectHeaderSizeInBytes()                     {return TR::Compiler->om.objectHeaderSizeInBytes();}
 
 UDATA TR_J9VMBase::getOffsetOfSuperclassesInClassObject()           {return offsetof(J9Class, superclasses);}
 UDATA TR_J9VMBase::getOffsetOfBackfillOffsetField()                 {return offsetof(J9Class, backfillOffset);}
@@ -1130,12 +1130,12 @@ TR_J9VMBase::getReferenceFieldAtAddress(uintptrj_t fieldAddress)
       vmThread()->javaVM->javaVM->memoryManagerFunctions->J9ReadBarrier(vmThread(), (fj9object_t *)fieldAddress);
 #endif
 
-#if defined(J9VM_GC_COMPRESSED_POINTERS)
-   uintptrj_t compressedResult = *(uint32_t*)fieldAddress;
-   return (compressedResult << TR::Compiler->om.compressedReferenceShift()) + TR::Compiler->vm.heapBaseAddress();
-#else
+   if (TR::Compiler->om.compressObjectReferences())
+      {
+      uintptrj_t compressedResult = *(uint32_t*)fieldAddress;
+      return (compressedResult << TR::Compiler->om.compressedReferenceShift()) + TR::Compiler->vm.heapBaseAddress();
+      }
    return *(uintptrj_t*)fieldAddress;
-#endif
    }
 
 uintptrj_t
@@ -1157,21 +1157,21 @@ int32_t
 TR_J9VMBase::getInt32FieldAt(uintptrj_t objectPointer, uintptrj_t fieldOffset)
    {
    TR_ASSERT(haveAccess(), "Must haveAccess in getInt32Field");
-   return *(int32_t*)(objectPointer + sizeof(J9Object) + fieldOffset);
+   return *(int32_t*)(objectPointer + getObjectHeaderSizeInBytes() + fieldOffset);
    }
 
 int64_t
 TR_J9VMBase::getInt64FieldAt(uintptrj_t objectPointer, uintptrj_t fieldOffset)
    {
    TR_ASSERT(haveAccess(), "Must haveAccess in getInt64Field");
-   return *(int64_t*)(objectPointer + sizeof(J9Object) + fieldOffset);
+   return *(int64_t*)(objectPointer + getObjectHeaderSizeInBytes() + fieldOffset);
    }
 
 void
 TR_J9VMBase::setInt64FieldAt(uintptrj_t objectPointer, uintptrj_t fieldOffset, int64_t newValue)
    {
    TR_ASSERT(haveAccess(), "Must haveAccess in setInt64Field");
-   *(int64_t*)(objectPointer + sizeof(J9Object) + fieldOffset) = newValue;
+   *(int64_t*)(objectPointer + getObjectHeaderSizeInBytes() + fieldOffset) = newValue;
    }
 
 bool
@@ -1792,11 +1792,11 @@ UDATA TR_J9VMBase::thisThreadGetConcurrentScavengeActiveByteAddressOffset()
 UDATA TR_J9VMBase::thisThreadGetEvacuateBaseAddressOffset()
    {
 #if defined(OMR_GC_CONCURRENT_SCAVENGER)
-#if defined(J9VM_GC_COMPRESSED_POINTERS)
-   return offsetof(J9VMThread, readBarrierRangeCheckBaseCompressed);
-#else
+#if defined(OMR_GC_COMPRESSED_POINTERS)
+  if (TR::Compiler->om.compressObjectReferences())
+      return offsetof(J9VMThread, readBarrierRangeCheckBaseCompressed);
+#endif /* defined(OMR_GC_COMPRESSED_POINTERS) */
    return offsetof(J9VMThread, readBarrierRangeCheckBase);
-#endif /* defined(J9VM_GC_COMPRESSED_POINTERS) */
 #else /* defined(OMR_GC_CONCURRENT_SCAVENGER) */
    TR_ASSERT(0,"Field readBarrierRangeCheckBase does not exists in J9VMThread.");
    return 0;
@@ -1809,11 +1809,11 @@ UDATA TR_J9VMBase::thisThreadGetEvacuateBaseAddressOffset()
 UDATA TR_J9VMBase::thisThreadGetEvacuateTopAddressOffset()
    {
 #if defined(OMR_GC_CONCURRENT_SCAVENGER)
-#if defined(J9VM_GC_COMPRESSED_POINTERS)
-   return offsetof(J9VMThread, readBarrierRangeCheckTopCompressed);
-#else
+#if defined(OMR_GC_COMPRESSED_POINTERS)
+   if (TR::Compiler->om.compressObjectReferences())
+      return offsetof(J9VMThread, readBarrierRangeCheckTopCompressed);
+#endif /* defined(OMR_GC_COMPRESSED_POINTERS) */
    return offsetof(J9VMThread, readBarrierRangeCheckTop);
-#endif /* defined(J9VM_GC_COMPRESSED_POINTERS) */
 #else /* defined(OMR_GC_CONCURRENT_SCAVENGER) */
    TR_ASSERT(0,"Field readBarrierRangeCheckTop does not exists in J9VMThread.");
    return 0;
@@ -1923,7 +1923,7 @@ int32_t TR_J9VMBase::getFirstArrayletPointerOffset(TR::Compilation *comp)
    TR_ASSERT(TR::Compiler->om.canGenerateArraylets(), "not supposed to be generating arraylets!");
 
    int32_t headerSize = TR::Compiler->om.useHybridArraylets() ?
-      sizeof(J9IndexableObjectDiscontiguous) : sizeof(J9IndexableObjectContiguous);
+		   TR::Compiler->om.discontiguousArrayHeaderSizeInBytes() : TR::Compiler->om.contiguousArrayHeaderSizeInBytes();
 
    return (headerSize + TR::Compiler->om.sizeofReferenceField()-1) & (-1)*(intptrj_t)(TR::Compiler->om.sizeofReferenceField());
    }
@@ -1966,11 +1966,11 @@ int32_t TR_J9VMBase::getCAASaveOffset()
 uint32_t
 TR_J9VMBase::getWordOffsetToGCFlags()
    {
-#if defined(J9VM_INTERP_FLAGS_IN_CLASS_SLOT) && defined(TR_TARGET_64BIT) && !defined(J9VM_GC_COMPRESSED_POINTERS)
-      return TR::Compiler->om.offsetOfHeaderFlags() + 4;
-#else
-      return TR::Compiler->om.offsetOfHeaderFlags();
+#if defined(J9VM_INTERP_FLAGS_IN_CLASS_SLOT) && defined(TR_TARGET_64BIT)
+   if (!TR::Compiler->om.compressObjectReferences())
+     return TR::Compiler->om.offsetOfHeaderFlags() + 4;
 #endif
+   return TR::Compiler->om.offsetOfHeaderFlags();
    }
 
 uint32_t
@@ -7774,7 +7774,7 @@ TR_J9VM::inlineNativeCall(TR::Compilation * comp, TR::TreeTop * callNodeTreeTop,
       case TR::com_ibm_oti_vm_ORBVMHelpers_getNumBytesInJ9ObjectHeader:
       case TR::com_ibm_jit_JITHelpers_getNumBytesInJ9ObjectHeader:
          {
-         int32_t intValue = sizeof(J9Object);
+         int32_t intValue = getObjectHeaderSizeInBytes();
          TR::Node::recreate(callNode, TR::iconst);
          callNode->setNumChildren(0);
          callNode->setInt(intValue);
@@ -8271,7 +8271,7 @@ uint32_t
 TR_J9VMBase::getAllocationSize(TR::StaticSymbol *classSym, TR_OpaqueClassBlock * opaqueClazz)
    {
    J9Class * clazz = (J9Class * ) opaqueClazz;
-   int32_t headerSize = sizeof(J9Object);
+   int32_t headerSize = getObjectHeaderSizeInBytes();
    int32_t objectSize = (int32_t)clazz->totalInstanceSize + headerSize;
 
    // gc requires objects to have

runtime/compiler/env/VMJ9Server.cpp

@@ -669,7 +669,7 @@ TR_J9ServerVM::getAllocationSize(TR::StaticSymbol *classSym, TR_OpaqueClassBlock
    JITServer::ServerStream *stream = _compInfoPT->getMethodBeingCompiled()->_stream;
    JITServerHelpers::getAndCacheRAMClassInfo((J9Class *)clazz, _compInfoPT->getClientData(), stream, JITServerHelpers::CLASSINFO_TOTAL_INSTANCE_SIZE, (void *)&totalInstanceSize);
 
-   uint32_t objectSize = sizeof(J9Object) + (uint32_t)totalInstanceSize;
+   uint32_t objectSize = getObjectHeaderSizeInBytes() + (uint32_t)totalInstanceSize;
    return ((objectSize >= J9_GC_MINIMUM_OBJECT_SIZE) ? objectSize : J9_GC_MINIMUM_OBJECT_SIZE);
    }
 

runtime/compiler/runtime/IProfiler.hpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2019 IBM Corp. and others
+ * Copyright (c) 2000, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -65,6 +65,7 @@
 #include "j9.h"
 #include "j9cfg.h"
 #include "env/jittypes.h"
+#include "env/CompilerEnv.hpp"
 #include "il/Node.hpp"
 #include "infra/Link.hpp"
 #include "runtime/ExternalProfiler.hpp"
@@ -397,14 +398,14 @@ class TR_IPBCDataCallGraph : public TR_IPBytecodeHashTableEntry
    void * operator new (size_t size) throw();
    void * operator new (size_t size, void * placement) {return placement;}
 
-#ifdef OMR_GC_COMPRESSED_POINTERS
    // Set the higher 32 bits to zero under compressedref to avoid assertion in
    // CallSiteProfileInfo::setClazz, which is called by setInvalid with IPROFILING_INVALID
    //
-   static const uintptrj_t IPROFILING_INVALID = 0x00000000FFFFFFFF;
-#else
+   // Set the higher 32 bits to zero under compressedref to avoid assertion in
+   // CallSiteProfileInfo::setClazz, which is called by setInvalid with IPROFILING_INVALID
+   //
+   static const uintptrj_t IPROFILING_INVALID_COMPRESSED = 0x00000000FFFFFFFF;
    static const uintptrj_t IPROFILING_INVALID = ~0;
-#endif
 
    virtual uintptrj_t getData(TR::Compilation *comp = NULL);
    virtual CallSiteProfileInfo* getCGData() { return &_csInfo; } // overloaded
@@ -421,8 +422,8 @@ class TR_IPBCDataCallGraph : public TR_IPBytecodeHashTableEntry
    void setWarmCallGraphTooBig(bool set=true) { _csInfo._tooBigToBeInlined = (set) ? 1 : 0; }
    bool isWarmCallGraphTooBig() { return (_csInfo._tooBigToBeInlined == 1); }
 
-   virtual bool isInvalid() { if (_csInfo.getClazz(0) == IPROFILING_INVALID) return true; return false; }
-   virtual void setInvalid() { _csInfo.setClazz(0, IPROFILING_INVALID); }
+   virtual bool isInvalid() { if (_csInfo.getClazz(0) == (TR::Compiler->om.compressObjectReferences() ? IPROFILING_INVALID_COMPRESSED : IPROFILING_INVALID)) return true; return false; }
+   virtual void setInvalid() { _csInfo.setClazz(0, (TR::Compiler->om.compressObjectReferences() ? IPROFILING_INVALID_COMPRESSED : IPROFILING_INVALID)); }
    virtual uint32_t getBytesFootprint() {return sizeof (TR_IPBCDataCallGraphStorage);}
 
 #if defined(JITSERVER_SUPPORT)

runtime/compiler/x/runtime/X86LockReservation.nasm

@@ -1,4 +1,4 @@
-; Copyright (c) 2000, 2019 IBM Corp. and others
+; Copyright (c) 2000, 2020 IBM Corp. and others
 ;
 ; This program and the accompanying materials are made available under
 ; the terms of the Eclipse Public License 2.0 which accompanies this
@@ -77,6 +77,7 @@ eq_J9Monitor_CNTFLCClearMask  equ 0FFFFFFFFFFFFFF05h
     mov _rax, [_rax + J9TR_J9Class_lockOffset]   ; offset of lock word in receiver class
     lea _rcx, [%1 + _rax]                        ; load the address of object lock word
     mov  eax, [_rcx]
+    jmp short %%done
 %%full:
     mov _rax, [%1 + J9TR_J9Object_class]         ; receiver class
     and _rax, eq_ObjectClassMask
@@ -112,6 +113,13 @@ eq_J9Monitor_CNTFLCClearMask  equ 0FFFFFFFFFFFFFF05h
 ; lockword address <= rcx
 ; vmthread         <= rbp
 %macro TryLock 0
+%ifdef ASM_OMR_GC_COMPRESSED_POINTERS
+%ifdef ASM_OMR_GC_FULL_POINTERS
+    ; set flags for compressed now before corrupting EBP value
+    ; NOTE: there must be no instructions which set flags before the test below
+    cmp qword [_rbp + J9TR_VMThreadCompressObjectReferences], 0
+%endif
+%endif
     push _rbp
     lea  _rbp, [_rbp + eq_J9Monitor_RESINCBits]           ; make thread ID + RES + INC_DEC value
 
@@ -132,10 +140,21 @@ eq_J9Monitor_CNTFLCClearMask  equ 0FFFFFFFFFFFFFF05h
 %endif
 
 %ifdef ASM_OMR_GC_COMPRESSED_POINTERS
+%ifdef ASM_OMR_GC_FULL_POINTERS
+    ; rely on flags set above
+    je  short %%full
     lock cmpxchg [_rcx],  ebp                       ; try taking the lock
+    jmp short %%done
+%%full:
+    lock cmpxchg [_rcx], _rbp                       ; try taking the lock
+%%done:
+%else
+    lock cmpxchg [_rcx],  ebp                       ; try taking the lock
+%endif
 %else
     lock cmpxchg [_rcx], _rbp                       ; try taking the lock
 %endif
+
     pop  _rbp
 %endmacro
 

runtime/gc_base/ClassModel.hpp

@@ -1,6 +1,5 @@
-
 /*******************************************************************************
- * Copyright (c) 1991, 2019 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -109,13 +108,14 @@ class GC_ClassModel
 	/**
 	 * Calculate the number of object slots in the class.
 	 * @param[in] clazz Pointer to the class
+	 * @param[in] vm The J9JavaVM
 	 * @return number of object slots.
 	 */
 	MMINLINE static UDATA
-	getNumberOfObjectSlots(J9Class *clazz)
+	getNumberOfObjectSlots(J9Class *clazz, J9JavaVM *vm)
 	{
 		UDATA totalInstanceSize = clazz->totalInstanceSize;
-		IDATA scanLimit = (IDATA) (totalInstanceSize / sizeof(fj9object_t));
+		IDATA scanLimit = (IDATA) (totalInstanceSize / J9JAVAVM_REFERENCE_SIZE(vm));
 		UDATA tempDescription = (UDATA)clazz->instanceDescription;
 
 		UDATA slotCount = 0;

runtime/gc_base/GCExtensions.cpp

@@ -1,6 +1,5 @@
-
 /*******************************************************************************
- * Copyright (c) 1991, 2019 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -91,7 +90,7 @@ MM_GCExtensions::initialize(MM_EnvironmentBase *env)
 
 #if defined(J9VM_GC_REALTIME)
 	/* only ref slots, size in bytes: 2 * minObjectSize - header size */
-	minArraySizeToSetAsScanned = 2 * (1 << J9VMGC_SIZECLASSES_LOG_SMALLEST) - sizeof(J9IndexableObjectContiguous);
+	minArraySizeToSetAsScanned = 2 * (1 << J9VMGC_SIZECLASSES_LOG_SMALLEST) - J9JAVAVM_CONTIGUOUS_HEADER_SIZE(getJavaVM());
 #endif /* J9VM_GC_REALTIME */
 
 #if defined(J9VM_GC_JNI_ARRAY_CACHE)

runtime/gc_base/IndexableObjectAllocationModel.cpp

@@ -1,6 +1,5 @@
-
 /*******************************************************************************
- * Copyright (c) 1991, 2019 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -183,10 +182,11 @@ MM_IndexableObjectAllocationModel::layoutContiguousArraylet(MM_EnvironmentBase *
 {
 	Assert_MM_true(_numberOfArraylets == _allocateDescription.getNumArraylets());
 	MM_GCExtensions *extensions = MM_GCExtensions::getExtensions(env);
+	bool const compressed = env->compressObjectReferences();
 
 	/* set arraylet pointers in the spine. these all point into the data part of the spine */
 	fj9object_t *arrayoidPtr = extensions->indexableObjectModel.getArrayoidPointer(spine);
-	uintptr_t leafOffset = (uintptr_t)(arrayoidPtr + _numberOfArraylets);
+	uintptr_t leafOffset = (uintptr_t)GC_SlotObject::addToSlotAddress(arrayoidPtr, _numberOfArraylets, compressed);
 	if (_alignSpineDataSection) {
 		leafOffset = MM_Math::roundToCeiling(sizeof(uint64_t), leafOffset);
 	}
@@ -195,7 +195,7 @@ MM_IndexableObjectAllocationModel::layoutContiguousArraylet(MM_EnvironmentBase *
 		GC_SlotObject slotObject(env->getOmrVM(), arrayoidPtr);
 		slotObject.writeReferenceToSlot((omrobjectptr_t)leafOffset);
 		leafOffset += arrayletLeafSize;
-		arrayoidPtr += 1;
+		arrayoidPtr = GC_SlotObject::addToSlotAddress(arrayoidPtr, 1, compressed);
 	}
 
 	return spine;
@@ -217,6 +217,7 @@ MM_IndexableObjectAllocationModel::layoutDiscontiguousArraylet(MM_EnvironmentBas
 	Assert_MM_true(_numberOfArraylets == _allocateDescription.getNumArraylets());
 
 	MM_GCExtensions *extensions = MM_GCExtensions::getExtensions(env);
+	bool const compressed = env->compressObjectReferences();
 
 	/* determine how many bytes to allocate outside of the spine (in arraylet leaves) */
 	const uintptr_t arrayletLeafSize = env->getOmrVM()->_arrayletLeafSize;
@@ -247,7 +248,7 @@ MM_IndexableObjectAllocationModel::layoutDiscontiguousArraylet(MM_EnvironmentBas
 		arrayoidPtr = extensions->indexableObjectModel.getArrayoidPointer(spine);
 
 		/* set the arrayoid pointer in the spine to point to the new leaf */
-		GC_SlotObject slotObject(env->getOmrVM(), &arrayoidPtr[arrayoidIndex]);
+		GC_SlotObject slotObject(env->getOmrVM(), GC_SlotObject::addToSlotAddress(arrayoidPtr, arrayoidIndex, compressed));
 		slotObject.writeReferenceToSlot((omrobjectptr_t)leaf);
 
 		bytesRemaining -= OMR_MIN(bytesRemaining, arrayletLeafSize);
@@ -271,7 +272,7 @@ MM_IndexableObjectAllocationModel::layoutDiscontiguousArraylet(MM_EnvironmentBas
 			/* if last arraylet leaf is empty (contains 0 bytes) arrayoid pointer is set to NULL */
 			if (arrayoidIndex == (_numberOfArraylets - 1)) {
 				Assert_MM_true(0 == (_dataSize % arrayletLeafSize));
-				GC_SlotObject slotObject(env->getOmrVM(), &(arrayoidPtr[arrayoidIndex]));
+				GC_SlotObject slotObject(env->getOmrVM(), GC_SlotObject::addToSlotAddress(arrayoidPtr, arrayoidIndex, compressed));
 				slotObject.writeReferenceToSlot(NULL);
 			} else {
 				Assert_MM_true(0 != (_dataSize % arrayletLeafSize));
@@ -291,12 +292,12 @@ MM_IndexableObjectAllocationModel::layoutDiscontiguousArraylet(MM_EnvironmentBas
 			 */
 			Assert_MM_true(arrayoidIndex == (_numberOfArraylets - 1));
 			{
-				uintptr_t leafOffset = (uintptr_t)&(arrayoidPtr[_numberOfArraylets]);
+				uintptr_t leafOffset = (uintptr_t)GC_SlotObject::addToSlotAddress(arrayoidPtr, _numberOfArraylets, compressed);
 				if (_alignSpineDataSection) {
 					leafOffset = MM_Math::roundToCeiling(env->getObjectAlignmentInBytes(), leafOffset);
 				}
 				/* set the last arrayoid pointer to point to remainder data */
-				GC_SlotObject slotObject(env->getOmrVM(), &(arrayoidPtr[arrayoidIndex]));
+				GC_SlotObject slotObject(env->getOmrVM(), GC_SlotObject::addToSlotAddress(arrayoidPtr, arrayoidIndex, compressed));
 				slotObject.writeReferenceToSlot((omrobjectptr_t)leafOffset);
 			}
 			break;
@@ -356,7 +357,7 @@ MM_IndexableObjectAllocationModel::doubleMapArraylets(MM_EnvironmentBase *env, J
 	/* Number of arraylet leaves in the iterator must match the number of leaves calculated */
 	Assert_MM_true(arrayletLeafCount == count);
 
-	GC_SlotObject objectSlot(env->getOmrVM(), &extensions->indexableObjectModel.getArrayoidPointer((J9IndexableObject *)objectPtr)[0]);
+	GC_SlotObject objectSlot(env->getOmrVM(), extensions->indexableObjectModel.getArrayoidPointer((J9IndexableObject *)objectPtr));
 	J9Object *firstLeafSlot = objectSlot.readReferenceFromSlot();
 
 	MM_HeapRegionDescriptorVLHGC *firstLeafRegionDescriptor = (MM_HeapRegionDescriptorVLHGC *)heap->getHeapRegionManager()->tableDescriptorForAddress(firstLeafSlot);