Skip to content

Commit

Permalink
[SPARK-22500][SQL] Fix 64KB JVM bytecode limit problem with cast
Browse files Browse the repository at this point in the history
This PR changes `cast` code generation to place generated code for expression for fields of a structure into separated methods if these size could be large.

Added new test cases into `CastSuite`

Author: Kazuaki Ishizaki <ishizaki@jp.ibm.com>

Closes apache#19730 from kiszk/SPARK-22500.

(cherry picked from commit ac10171)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
  • Loading branch information
kiszk authored and MatthewRBruce committed Jul 31, 2018
1 parent 9e40be2 commit 52c695f
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1039,13 +1039,21 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
}
}
"""
}.mkString("\n")
}
val fieldsEvalCodes = if (ctx.INPUT_ROW != null && ctx.currentVars == null) {
ctx.splitExpressions(
expressions = fieldsEvalCode,
funcName = "castStruct",
arguments = ("InternalRow", tmpRow) :: (rowClass, result) :: Nil)
} else {
fieldsEvalCode.mkString("\n")
}

(c, evPrim, evNull) =>
s"""
final $rowClass $result = new $rowClass(${fieldsCasts.length});
final InternalRow $tmpRow = $c;
$fieldsEvalCode
$fieldsEvalCodes
$evPrim = $result.copy();
"""
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -827,4 +827,22 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {

checkEvaluation(cast(Literal.create(input, from), to), input)
}

test("SPARK-22500: cast for struct should not generate codes beyond 64KB") {
val N = 50

val fromInner = new StructType(
(1 to N).map(i => StructField(s"s$i", DoubleType)).toArray)
val toInner = new StructType(
(1 to N).map(i => StructField(s"i$i", IntegerType)).toArray)
val inputInner = Row.fromSeq((1 to N).map(i => i + 0.5))
val outputInner = Row.fromSeq((1 to N))
val fromOuter = new StructType(
(1 to N).map(i => StructField(s"s$i", fromInner)).toArray)
val toOuter = new StructType(
(1 to N).map(i => StructField(s"s$i", toInner)).toArray)
val inputOuter = Row.fromSeq((1 to N).map(_ => inputInner))
val outputOuter = Row.fromSeq((1 to N).map(_ => outputInner))
checkEvaluation(cast(Literal.create(inputOuter, fromOuter), toOuter), outputOuter)
}
}

0 comments on commit 52c695f

Please sign in to comment.