Check the InnerClass attribute of the enclosing class

The change is to check to see whether an valid entry for
the specified class exists in the InnerClass attribute
of its enclosing class given the InnerClass attribute
of both the inner class and its enclosing class must
remain consistent with each other.

Fixes: #12992

Signed-off-by: Cheng Jin <jincheng@ca.ibm.com>

jcl/src/java.base/share/classes/java/lang/Class.java

@@ -278,6 +278,8 @@ private static final class MetadataCache {
 	private transient Class<?> cachedEnclosingClass;
 	private static long cachedEnclosingClassOffset = -1;
 
+	private transient boolean cachedCheckInnerClassAttr;
+
 	private static Annotation[] EMPTY_ANNOTATION_ARRAY = new Annotation[0];
 
 	static MethodHandles.Lookup implLookup;
@@ -1293,6 +1295,13 @@ public Class<?> getDeclaringClass() {
 	 */
 	Class<?> declaringClass = cachedDeclaringClass == ClassReflectNullPlaceHolder.class ? null : cachedDeclaringClass;
 	if (declaringClass == null) {
+		if (!cachedCheckInnerClassAttr) {
+			/* Check whether the enclosing class has an valid inner class entry to the current class.
+			 * Note: the entries are populated with the InnerClass attribute when creating ROM class.
+			 */
+			checkInnerClassAttrOfEnclosingClass();
+			cachedCheckInnerClassAttr = true;
+		}
 		return declaringClass;
 	}
 	if (declaringClass.isClassADeclaredClass(this)) {
@@ -1320,7 +1329,23 @@ public Class<?> getDeclaringClass() {
 
 	/*[MSG "K0555", "incompatible InnerClasses attribute between \"{0}\" and \"{1}\""]*/
 	throw new IncompatibleClassChangeError(
-			com.ibm.oti.util.Msg.getString("K0555", this.getName(),	declaringClass.getName())); //$NON-NLS-1$
+			com.ibm.oti.util.Msg.getString("K0555", this.getName(), declaringClass.getName())); //$NON-NLS-1$
+}
+
+/**
+ * Checks whether the current class exists in the InnerClass attribute of the specified enclosing class
+ * when this class is not defined directly inside the enclosing class (e.g. defined inside a method).
+ *
+ * Note: The direct inner classes of the declaring class is already checked in getDeclaringClass()
+ * when the enclosing class is the declaring class.
+ */
+private void checkInnerClassAttrOfEnclosingClass() {
+	Class<?> enclosingClass = getEnclosingObjectClass();
+	if ((enclosingClass != null) && !enclosingClass.isClassAnEnclosedClass(this)) {
+		/*[MSG "K0555", "incompatible InnerClasses attribute between \"{0}\" and \"{1}\""]*/
+		throw new IncompatibleClassChangeError(
+				com.ibm.oti.util.Msg.getString("K0555", this.getName(), enclosingClass.getName())); //$NON-NLS-1$
+	}
 }
 
 /**
@@ -1342,6 +1367,18 @@ public Class<?> getDeclaringClass() {
  */
 private native boolean isClassADeclaredClass(Class<?> aClass);
 
+/**
+ * Returns true if the class passed in to the method is an enclosed class of
+ * this class, which includes both the declared classes and the classes defined
+ * inside a method of this class.
+ *
+ * @param		aClass		The class to validate
+ * @return		true if aClass an enclosed class of this class
+ * 				false otherwise.
+ *
+ */
+private native boolean isClassAnEnclosedClass(Class<?> aClass);
+
 /**
  * Answers the class which declared the class represented
  * by the receiver. This will return null if the receiver

runtime/bcutil/ClassFileOracle.cpp

@@ -168,6 +168,7 @@ ClassFileOracle::ClassFileOracle(BufferManager *bufferManager, J9CfrClassFile *c
 	_doubleScalarStaticCount(0),
 	_memberAccessFlags(0),
 	_innerClassCount(0),
+	_enclosedInnerClassCount(0),
 #if JAVA_SPEC_VERSION >= 11
 	_nestMembersCount(0),
 	_nestHost(0),
@@ -459,7 +460,7 @@ ClassFileOracle::walkAttributes()
 				if (outerClassUTF8 == thisClassUTF8) {
 					/* Member class - mark the class' name. */
 					markClassNameAsReferenced(entry->innerClassInfoIndex);
-					_innerClassCount++;
+					_innerClassCount += 1;
 				} else if (innerClassUTF8 == thisClassUTF8) {
 					_isInnerClass = true;
 					_memberAccessFlags = entry->innerClassAccessFlags;
@@ -474,6 +475,13 @@ ClassFileOracle::walkAttributes()
 						markConstantUTF8AsReferenced(entry->innerNameIndex);
 						_simpleNameIndex = entry->innerNameIndex;
 					}
+				} else {
+					/* Count all entries in the InnerClass attribute (except the inner class itself) so as
+					 * to check the InnerClass attribute between the inner classes and the enclosing class.
+					 * See getDeclaringClass() for details.
+					 */
+					markClassNameAsReferenced(entry->innerClassInfoIndex);
+					_enclosedInnerClassCount += 1;
 				}
 			}
 			Trc_BCU_Assert_Equals(NULL, _innerClasses);

runtime/bcutil/ClassFileOracle.hpp

@@ -856,6 +856,28 @@ class RecordComponentIterator
 		}
 	}
 
+	/*
+	 * Iterate over the constant pool indices corresponding to enclosed inner class names (UTF8s).
+	 */
+	void enclosedInnerClassesDo(ConstantPoolIndexVisitor *visitor)
+	{
+		if (NULL != _innerClasses) {
+			J9CfrClassesEntry *end = _innerClasses->classes + _innerClasses->numberOfClasses;
+			U_16 thisClassUTF8 = UTF8_INDEX_FROM_CLASS_INDEX(_classFile->constantPool, _classFile->thisClass);
+			for (J9CfrClassesEntry *entry = _innerClasses->classes; entry != end; ++entry) {
+				U_16  outerClassUTF8 = UTF8_INDEX_FROM_CLASS_INDEX(_classFile->constantPool, entry->outerClassInfoIndex);
+				U_16  innerClassUTF8 = UTF8_INDEX_FROM_CLASS_INDEX(_classFile->constantPool, entry->innerClassInfoIndex);
+				/* Count all remaining entries in the InnerClass attribute (except the entries covered by innerClassesDo())
+				 * so as to check the InnerClass attribute between the inner classes and the enclosing class.
+				 * See getDeclaringClass() for details.
+				 */
+				if ((thisClassUTF8 != outerClassUTF8) && (thisClassUTF8 != innerClassUTF8)) {
+					visitor->visitConstantPoolIndex(innerClassUTF8);
+				}
+			}
+		}
+	}
+
 #if JAVA_SPEC_VERSION >= 11
 	void nestMembersDo(ConstantPoolIndexVisitor *visitor)
 	{
@@ -913,6 +935,7 @@ class RecordComponentIterator
 	U_16 getDoubleScalarStaticCount() const { return _doubleScalarStaticCount; }
 	U_16 getMemberAccessFlags() const { return _memberAccessFlags; }
 	U_16 getInnerClassCount() const { return _innerClassCount; }
+	U_16 getEnclosedInnerClassCount() const { return _enclosedInnerClassCount; }
 #if JAVA_SPEC_VERSION >= 11
 	U_16 getNestMembersCount() const { return _nestMembersCount; }
 	U_16 getNestHostNameIndex() const { return _nestHost; }
@@ -1064,6 +1087,7 @@ class RecordComponentIterator
 	U_16 _doubleScalarStaticCount;
 	U_16 _memberAccessFlags;
 	U_16 _innerClassCount;
+	U_16 _enclosedInnerClassCount;
 #if JAVA_SPEC_VERSION >= 11
 	U_16 _nestMembersCount;
 	U_16 _nestHost;

runtime/bcutil/ClassFileWriter.cpp

@@ -123,7 +123,10 @@ ClassFileWriter::analyzeROMClass()
 #endif /* JAVA_SPEC_VERSION >= 11 */
 
 	/* For a local class only InnerClasses.class[i].inner_name_index is preserved as simpleName in its J9ROMClass */
-	if ((0 != _romClass->innerClassCount) || J9_ARE_ANY_BITS_SET(_romClass->extraModifiers, J9AccClassInnerClass)) {
+	if ((0 != _romClass->innerClassCount)
+	|| (0 != _romClass->enclosedInnerClassCount)
+	|| J9_ARE_ANY_BITS_SET(_romClass->extraModifiers, J9AccClassInnerClass)
+	) {
 		addEntry((void *) &INNER_CLASSES, 0, CFR_CONSTANT_Utf8);
 
 		if (NULL != outerClassName) {
@@ -139,6 +142,11 @@ ClassFileWriter::analyzeROMClass()
 			J9UTF8 * className = NNSRP_PTR_GET(innerClasses, J9UTF8 *);
 			addClassEntry(className, 0);
 		}
+		J9SRP * enclosedInnerClasses = (J9SRP *) J9ROMCLASS_ENCLOSEDINNERCLASSES(_romClass);
+		for (UDATA i = 0; i < _romClass->enclosedInnerClassCount; i++, enclosedInnerClasses++) {
+			J9UTF8 * className = NNSRP_PTR_GET(enclosedInnerClasses, J9UTF8 *);
+			addClassEntry(className, 0);
+		}
 	}
 
 #if JAVA_SPEC_VERSION >= 11
@@ -954,7 +962,10 @@ ClassFileWriter::writeAttributes()
 	U_16 nestMemberCount = _romClass->nestMemberCount;
 #endif /* JAVA_SPEC_VERSION >= 11 */
 
-	if ((0 != _romClass->innerClassCount) || J9_ARE_ANY_BITS_SET(_romClass->extraModifiers, J9AccClassInnerClass)) {
+	if ((0 != _romClass->innerClassCount)
+	|| (0 != _romClass->enclosedInnerClassCount)
+	|| J9_ARE_ANY_BITS_SET(_romClass->extraModifiers, J9AccClassInnerClass)
+	) {
 		attributesCount += 1;
 	}
 	if (NULL != enclosingObject) {
@@ -992,8 +1003,11 @@ ClassFileWriter::writeAttributes()
 #endif /* JAVA_SPEC_VERSION >= 11 */
 	writeU16(attributesCount);
 
-	if ((0 != _romClass->innerClassCount) || J9_ARE_ANY_BITS_SET(_romClass->extraModifiers, J9AccClassInnerClass)) {
-		U_16 innerClassesCount(_romClass->innerClassCount);
+	if ((0 != _romClass->innerClassCount)
+	|| (0 != _romClass->enclosedInnerClassCount)
+	|| J9_ARE_ANY_BITS_SET(_romClass->extraModifiers, J9AccClassInnerClass)
+	) {
+		U_16 innerClassesCount(_romClass->innerClassCount + _romClass->enclosedInnerClassCount);
 
 		if (J9_ARE_ANY_BITS_SET(_romClass->extraModifiers, J9AccClassInnerClass)) {
 			/* This is an inner class, so we have one extra inner class attribute to allocate/write */
@@ -1022,6 +1036,22 @@ ClassFileWriter::writeAttributes()
 			writeU16(0); /* innerClassAccessFlags */
 		}
 
+		/* Write enclosed inner classes of this class */
+		J9SRP * enclosedInnerClasses = (J9SRP *) J9ROMCLASS_ENCLOSEDINNERCLASSES(_romClass);
+		/* Write the enclosed inner class entries for inner classes of this class */
+		for (UDATA i = 0; i < _romClass->enclosedInnerClassCount; i++, enclosedInnerClasses++) {
+			J9UTF8 * enclosedInnerClassName = NNSRP_PTR_GET(enclosedInnerClasses, J9UTF8 *);
+
+			writeU16(indexForClass(enclosedInnerClassName));
+			/* NOTE: outerClassInfoIndex (these inner class are not the declared classes of this class),
+			 * innerNameIndex and innerClassAccessFlags are not preserved in the ROM class
+			 * - technically incorrect, but this should only matter to compilers.
+			 */
+			writeU16(0); /* outerClassInfoIndex */
+			writeU16(0); /* innerNameIndex */
+			writeU16(0); /* innerClassAccessFlags */
+		}
+
 		if (J9_ARE_ANY_BITS_SET(_romClass->extraModifiers, J9AccClassInnerClass)) {
 			/* This is an inner class. Write an inner class attribute for itself. */
 			writeU16(thisClassCPIndex);

runtime/bcutil/ROMClassWriter.cpp

@@ -302,6 +302,7 @@ ROMClassWriter::ROMClassWriter(BufferManager *bufferManager, ClassFileOracle *cl
 	_fieldsSRPKey(srpKeyProducer->generateKey()),
 	_cpDescriptionShapeSRPKey(srpKeyProducer->generateKey()),
 	_innerClassesSRPKey(srpKeyProducer->generateKey()),
+	_enclosedInnerClassesSRPKey(srpKeyProducer->generateKey()),
 #if JAVA_SPEC_VERSION >= 11
 	_nestMembersSRPKey(srpKeyProducer->generateKey()),
 #endif /* JAVA_SPEC_VERSION >= 11 */
@@ -399,6 +400,8 @@ ROMClassWriter::writeROMClass(Cursor *cursor,
 		cursor->writeU32(_classFileOracle->getMemberAccessFlags(), Cursor::GENERIC);
 		cursor->writeU32(_classFileOracle->getInnerClassCount(), Cursor::GENERIC);
 		cursor->writeSRP(_innerClassesSRPKey, Cursor::SRP_TO_GENERIC);
+		cursor->writeU32(_classFileOracle->getEnclosedInnerClassCount(), Cursor::GENERIC);
+		cursor->writeSRP(_enclosedInnerClassesSRPKey, Cursor::SRP_TO_GENERIC);
 #if JAVA_SPEC_VERSION >= 11
 		cursor->writeSRP(_srpKeyProducer->mapCfrConstantPoolIndexToKey(_classFileOracle->getNestHostNameIndex()), Cursor::SRP_TO_UTF8);
 		cursor->writeU16(_classFileOracle->getNestMembersCount(), Cursor::GENERIC);
@@ -440,6 +443,7 @@ ROMClassWriter::writeROMClass(Cursor *cursor,
 	writeFields(cursor, markAndCountOnly);
 	writeInterfaces(cursor, markAndCountOnly);
 	writeInnerClasses(cursor, markAndCountOnly);
+	writeEnclosedInnerClasses(cursor, markAndCountOnly);
 #if JAVA_SPEC_VERSION >= 11
 	writeNestMembers(cursor, markAndCountOnly);
 #endif /* JAVA_SPEC_VERSION >= 11 */
@@ -759,6 +763,13 @@ class ROMClassWriter::Helper :
 		}
 	}
 
+	void writeEnclosedInnerClasses()
+	{
+		if (!_markAndCountOnly) {
+			_classFileOracle->enclosedInnerClassesDo(this); /* visitConstantPoolIndex */
+		}
+	}
+
 #if JAVA_SPEC_VERSION >= 11
 	void writeNestMembers()
 	{
@@ -1159,6 +1170,19 @@ ROMClassWriter::writeInnerClasses(Cursor *cursor, bool markAndCountOnly)
 #endif /* J9VM_OPT_VALHALLA_VALUE_TYPES */
 }
 
+void
+ROMClassWriter::writeEnclosedInnerClasses(Cursor *cursor, bool markAndCountOnly)
+{
+	cursor->mark(_enclosedInnerClassesSRPKey);
+	UDATA size = UDATA(_classFileOracle->getEnclosedInnerClassCount()) * sizeof(J9SRP);
+	CheckSize _(cursor, size);
+#if defined(J9VM_OPT_VALHALLA_VALUE_TYPES)
+	Helper(cursor, markAndCountOnly, _classFileOracle, _srpKeyProducer, _srpOffsetTable, _constantPoolMap, size, _interfaceInjectionInfo).writeEnclosedInnerClasses();
+#else /* J9VM_OPT_VALHALLA_VALUE_TYPES */
+	Helper(cursor, markAndCountOnly, _classFileOracle, _srpKeyProducer, _srpOffsetTable, _constantPoolMap, size).writeEnclosedInnerClasses();
+#endif /* J9VM_OPT_VALHALLA_VALUE_TYPES */
+}
+
 #if JAVA_SPEC_VERSION >= 11
 void
 ROMClassWriter::writeNestMembers(Cursor *cursor, bool markAndCountOnly)

runtime/bcutil/ROMClassWriter.hpp

@@ -131,6 +131,7 @@ class ROMClassWriter
 	void writeFields(Cursor *cursor, bool markAndCountOnly);
 	void writeInterfaces(Cursor *cursor, bool markAndCountOnly);
 	void writeInnerClasses(Cursor *cursor, bool markAndCountOnly);
+	void writeEnclosedInnerClasses(Cursor *cursor, bool markAndCountOnly);
 	void writeNestMembers(Cursor *cursor, bool markAndCountOnly);
 	void writeNameAndSignatureBlock(Cursor *cursor);
 	void writeMethods(Cursor *cursor, Cursor *lineNumberCursor, Cursor *variableInfoCursor, bool markAndCountOnly);
@@ -171,6 +172,7 @@ class ROMClassWriter
 	UDATA _fieldsSRPKey;
 	UDATA _cpDescriptionShapeSRPKey;
 	UDATA _innerClassesSRPKey;
+	UDATA _enclosedInnerClassesSRPKey;
 #if JAVA_SPEC_VERSION >= 11
 	UDATA _nestMembersSRPKey;
 #endif /* JAVA_SPEC_VERSION >= 11 */