Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-34607][SQL][3.1] Add Utils.isMemberClass to fix a malformed class name error on jdk8u #31745

Closed
wants to merge 1 commit into from
Closed

[SPARK-34607][SQL][3.1] Add Utils.isMemberClass to fix a malformed class name error on jdk8u #31745

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions core/src/main/scala/org/apache/spark/util/Utils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2860,6 +2860,34 @@ private[spark] object Utils extends Logging {
Hex.encodeHexString(secretBytes)
}

/**
* Returns true if and only if the underlying class is a member class.
*
* Note: jdk8u throws a "Malformed class name" error if a given class is a deeply-nested
* inner class (See SPARK-34607 for details). This issue has already been fixed in jdk9+, so
* we can remove this helper method safely if we drop the support of jdk8u.
*/
def isMemberClass(cls: Class[_]): Boolean = {
try {
cls.isMemberClass
} catch {
case _: InternalError =>
// We emulate jdk8u `Class.isMemberClass` below:
// public boolean isMemberClass() {
// return getSimpleBinaryName() != null && !isLocalOrAnonymousClass();
// }
// `getSimpleBinaryName()` returns null if a given class is a top-level class,
// so we replace it with `cls.getEnclosingClass != null`. The second condition checks
// if a given class is not a local or an anonymous class, so we replace it with
// `cls.getEnclosingMethod == null` because `cls.getEnclosingMethod()` return a value
// only in either case (JVM Spec 4.8.6).
//
// Note: The newer jdk evaluates `!isLocalOrAnonymousClass()` first,
// we reorder the conditions to follow it.
cls.getEnclosingMethod == null && cls.getEnclosingClass != null
}
}

/**
* Safer than Class obj's getSimpleName which may throw Malformed class name error in scala.
* This method mimics scalatest's getSimpleNameOfAnObjectsClass.
Expand Down
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/OuterScopes.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ object OuterScopes {
* useful for inner class defined in REPL.
*/
def getOuterScope(innerCls: Class[_]): () => AnyRef = {
assert(innerCls.isMemberClass)
assert(Utils.isMemberClass(innerCls))
val outerClassName = innerCls.getDeclaringClass.getName
val outer = outerScopes.get(outerClassName)
if (outer == null) {
Expand Down
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -443,7 +443,7 @@ case class NewInstance(
// Note that static inner classes (e.g., inner classes within Scala objects) don't need
// outer pointer registration.
val needOuterPointer =
outerPointer.isEmpty && cls.isMemberClass && !Modifier.isStatic(cls.getModifiers)
outerPointer.isEmpty && Utils.isMemberClass(cls) && !Modifier.isStatic(cls.getModifiers)
childrenResolved && !needOuterPointer
}

Expand Down
70 changes: 70 additions & 0 deletions ...talyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,76 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes

encodeDecodeTest(Array(Option(InnerClass(1))), "array of optional inner class")

object OuterLevelWithVeryVeryVeryLongClassName1 {
object OuterLevelWithVeryVeryVeryLongClassName2 {
object OuterLevelWithVeryVeryVeryLongClassName3 {
object OuterLevelWithVeryVeryVeryLongClassName4 {
object OuterLevelWithVeryVeryVeryLongClassName5 {
object OuterLevelWithVeryVeryVeryLongClassName6 {
object OuterLevelWithVeryVeryVeryLongClassName7 {
object OuterLevelWithVeryVeryVeryLongClassName8 {
object OuterLevelWithVeryVeryVeryLongClassName9 {
object OuterLevelWithVeryVeryVeryLongClassName10 {
object OuterLevelWithVeryVeryVeryLongClassName11 {
object OuterLevelWithVeryVeryVeryLongClassName12 {
object OuterLevelWithVeryVeryVeryLongClassName13 {
object OuterLevelWithVeryVeryVeryLongClassName14 {
object OuterLevelWithVeryVeryVeryLongClassName15 {
object OuterLevelWithVeryVeryVeryLongClassName16 {
object OuterLevelWithVeryVeryVeryLongClassName17 {
object OuterLevelWithVeryVeryVeryLongClassName18 {
object OuterLevelWithVeryVeryVeryLongClassName19 {
object OuterLevelWithVeryVeryVeryLongClassName20 {
case class MalformedNameExample(x: Int)
}}}}}}}}}}}}}}}}}}}}

{
OuterScopes.addOuterScope(
OuterLevelWithVeryVeryVeryLongClassName1
.OuterLevelWithVeryVeryVeryLongClassName2
.OuterLevelWithVeryVeryVeryLongClassName3
.OuterLevelWithVeryVeryVeryLongClassName4
.OuterLevelWithVeryVeryVeryLongClassName5
.OuterLevelWithVeryVeryVeryLongClassName6
.OuterLevelWithVeryVeryVeryLongClassName7
.OuterLevelWithVeryVeryVeryLongClassName8
.OuterLevelWithVeryVeryVeryLongClassName9
.OuterLevelWithVeryVeryVeryLongClassName10
.OuterLevelWithVeryVeryVeryLongClassName11
.OuterLevelWithVeryVeryVeryLongClassName12
.OuterLevelWithVeryVeryVeryLongClassName13
.OuterLevelWithVeryVeryVeryLongClassName14
.OuterLevelWithVeryVeryVeryLongClassName15
.OuterLevelWithVeryVeryVeryLongClassName16
.OuterLevelWithVeryVeryVeryLongClassName17
.OuterLevelWithVeryVeryVeryLongClassName18
.OuterLevelWithVeryVeryVeryLongClassName19
.OuterLevelWithVeryVeryVeryLongClassName20)
encodeDecodeTest(
OuterLevelWithVeryVeryVeryLongClassName1
.OuterLevelWithVeryVeryVeryLongClassName2
.OuterLevelWithVeryVeryVeryLongClassName3
.OuterLevelWithVeryVeryVeryLongClassName4
.OuterLevelWithVeryVeryVeryLongClassName5
.OuterLevelWithVeryVeryVeryLongClassName6
.OuterLevelWithVeryVeryVeryLongClassName7
.OuterLevelWithVeryVeryVeryLongClassName8
.OuterLevelWithVeryVeryVeryLongClassName9
.OuterLevelWithVeryVeryVeryLongClassName10
.OuterLevelWithVeryVeryVeryLongClassName11
.OuterLevelWithVeryVeryVeryLongClassName12
.OuterLevelWithVeryVeryVeryLongClassName13
.OuterLevelWithVeryVeryVeryLongClassName14
.OuterLevelWithVeryVeryVeryLongClassName15
.OuterLevelWithVeryVeryVeryLongClassName16
.OuterLevelWithVeryVeryVeryLongClassName17
.OuterLevelWithVeryVeryVeryLongClassName18
.OuterLevelWithVeryVeryVeryLongClassName19
.OuterLevelWithVeryVeryVeryLongClassName20
.MalformedNameExample(42),
"deeply nested Scala class should work")
}

productTest(PrimitiveData(1, 1, 1, 1, 1, 1, true))

productTest(
Expand Down