diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 16ab3359db610..5380e6b0e8954 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -1039,13 +1039,21 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String } } """ - }.mkString("\n") + } + val fieldsEvalCodes = if (ctx.INPUT_ROW != null && ctx.currentVars == null) { + ctx.splitExpressions( + expressions = fieldsEvalCode, + funcName = "castStruct", + arguments = ("InternalRow", tmpRow) :: (rowClass, result) :: Nil) + } else { + fieldsEvalCode.mkString("\n") + } (c, evPrim, evNull) => s""" final $rowClass $result = new $rowClass(${fieldsCasts.length}); final InternalRow $tmpRow = $c; - $fieldsEvalCode + $fieldsEvalCodes $evPrim = $result.copy(); """ } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index a7ffa884d2286..7014994cb2600 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -827,4 +827,22 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(cast(Literal.create(input, from), to), input) } + + test("SPARK-22500: cast for struct should not generate codes beyond 64KB") { + val N = 50 + + val fromInner = new StructType( + (1 to N).map(i => StructField(s"s$i", DoubleType)).toArray) + val toInner = new StructType( + (1 to N).map(i => StructField(s"i$i", IntegerType)).toArray) + val inputInner = Row.fromSeq((1 to N).map(i => i + 0.5)) + val outputInner = Row.fromSeq((1 to N)) + val fromOuter = new StructType( + (1 to N).map(i => StructField(s"s$i", fromInner)).toArray) + val toOuter = new StructType( + (1 to N).map(i => StructField(s"s$i", toInner)).toArray) + val inputOuter = Row.fromSeq((1 to N).map(_ => inputInner)) + val outputOuter = Row.fromSeq((1 to N).map(_ => outputInner)) + checkEvaluation(cast(Literal.create(inputOuter, fromOuter), toOuter), outputOuter) + } }