From 0278dea13085bdeaf0689a2c33a0805a0af2593a Mon Sep 17 00:00:00 2001 From: petermaxlee Date: Thu, 7 Jul 2016 20:53:34 -0700 Subject: [PATCH 1/2] [SPARK-16436][SQL] checkEvaluation should support NaN and RuntimeReplaceable --- .../catalyst/expressions/ExpressionEvalHelper.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala index e58a0df317fe9..22aa1adc9c64f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala @@ -41,7 +41,12 @@ trait ExpressionEvalHelper extends GeneratorDrivenPropertyChecks { } protected def checkEvaluation( - expression: => Expression, expected: Any, inputRow: InternalRow = EmptyRow): Unit = { + originalExpr: => Expression, expected: Any, inputRow: InternalRow = EmptyRow): Unit = { + val expression = originalExpr match { + case replaceable: RuntimeReplaceable => replaceable.replaced + case _ => originalExpr + } + val catalystValue = CatalystTypeConverters.convertToCatalyst(expected) checkEvaluationWithoutCodegen(expression, catalystValue, inputRow) checkEvaluationWithGeneratedMutableProjection(expression, catalystValue, inputRow) @@ -63,6 +68,10 @@ trait ExpressionEvalHelper extends GeneratorDrivenPropertyChecks { expected.asInstanceOf[Spread[Double]].isWithin(result) case (result: MapData, expected: MapData) => result.keyArray() == expected.keyArray() && result.valueArray() == expected.valueArray() + case (result: Double, expected: Double) => + if (expected.isNaN) result.isNaN else expected == result + case (result: Float, expected: Float) => + if (expected.isNaN) result.isNaN else expected == result case _ => result == expected } From ea0e91b87202d2a4ae8de997c40039b9fe9bfcd9 Mon Sep 17 00:00:00 2001 From: petermaxlee Date: Thu, 7 Jul 2016 23:23:47 -0700 Subject: [PATCH 2/2] Remove RuntimeReplaceable --- .../sql/catalyst/expressions/ExpressionEvalHelper.scala | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala index 22aa1adc9c64f..58e9d6f8bdf75 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala @@ -41,12 +41,7 @@ trait ExpressionEvalHelper extends GeneratorDrivenPropertyChecks { } protected def checkEvaluation( - originalExpr: => Expression, expected: Any, inputRow: InternalRow = EmptyRow): Unit = { - val expression = originalExpr match { - case replaceable: RuntimeReplaceable => replaceable.replaced - case _ => originalExpr - } - + expression: => Expression, expected: Any, inputRow: InternalRow = EmptyRow): Unit = { val catalystValue = CatalystTypeConverters.convertToCatalyst(expected) checkEvaluationWithoutCodegen(expression, catalystValue, inputRow) checkEvaluationWithGeneratedMutableProjection(expression, catalystValue, inputRow)