From eab17a1dd2d1bae7d520697e9a1eade331b47323 Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Sun, 2 Sep 2018 18:04:15 +0100 Subject: [PATCH 1/4] initial commit --- .../spark/sql/catalyst/expressions/collectionOperations.scala | 3 ++- .../sql/catalyst/expressions/CollectionExpressionsSuite.scala | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index cf9796ef1948f..b9c3b1941a95d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -1623,12 +1623,13 @@ case class ArraysOverlap(left: Expression, right: Expression) val set = ctx.freshName("set") val addToSetFromSmallerCode = nullSafeElementCodegen( smaller, i, s"$set.add($getFromSmaller);", s"${ev.isNull} = true;") + val setIsNullCode = if (nullable) s"${ev.isNull} = false;" else "" val elementIsInSetCode = nullSafeElementCodegen( bigger, i, s""" |if ($set.contains($getFromBigger)) { - | ${ev.isNull} = false; + | $setIsNullCode | ${ev.value} = true; | break; |} diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index 7b345aabd19c8..1a8ca4d5e4e3b 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -439,6 +439,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper val a4 = Literal.create(Seq[String](null, ""), ArrayType(StringType)) val a5 = Literal.create(Seq[String]("", "abc"), ArrayType(StringType)) val a6 = Literal.create(Seq[String]("def", "ghi"), ArrayType(StringType)) + val a7 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType, false)) val emptyIntArray = Literal.create(Seq.empty[Int], ArrayType(IntegerType)) @@ -454,6 +455,8 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(ArraysOverlap(a4, a6), null) checkEvaluation(ArraysOverlap(a5, a6), false) + checkEvaluation(ArraysOverlap(a7, a7), true) + // null handling checkEvaluation(ArraysOverlap(emptyIntArray, a2), false) checkEvaluation(ArraysOverlap( From 7e2f327d21527ac036f78d6ee2e3bc49ca97a143 Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Mon, 3 Sep 2018 02:05:16 +0100 Subject: [PATCH 2/4] address review comment --- .../spark/sql/catalyst/expressions/collectionOperations.scala | 3 ++- .../sql/catalyst/expressions/CollectionExpressionsSuite.scala | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index b9c3b1941a95d..e32e68ee2c602 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -1654,12 +1654,13 @@ case class ArraysOverlap(left: Expression, right: Expression) val j = ctx.freshName("j") val getFromSmaller = CodeGenerator.getValue(smaller, elementType, j) val getFromBigger = CodeGenerator.getValue(bigger, elementType, i) + val setIsNullCode = if (nullable) s"${ev.isNull} = false;" else "" val compareValues = nullSafeElementCodegen( smaller, j, s""" |if (${ctx.genEqual(elementType, getFromSmaller, getFromBigger)}) { - | ${ev.isNull} = false; + | $setIsNullCode | ${ev.value} = true; |} """.stripMargin, diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index 1a8ca4d5e4e3b..8939b79a49c10 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -474,9 +474,12 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper ArrayType(BinaryType)) val b2 = Literal.create(Seq[Array[Byte]](Array[Byte](2, 1), Array[Byte](4, 3)), ArrayType(BinaryType)) + val b3 = Literal.create(Seq[Array[Byte]](Array[Byte](1, 2), Array[Byte](3, 4)), + ArrayType(BinaryType, false)) checkEvaluation(ArraysOverlap(b0, b1), true) checkEvaluation(ArraysOverlap(b0, b2), false) + checkEvaluation(ArraysOverlap(b3, b3), true) // arrays of complex data types val aa0 = Literal.create(Seq[Array[String]](Array[String]("a", "b"), Array[String]("c", "d")), From caf1bb9cd82ed82ffee3887b4dff189ad9bb75db Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Mon, 3 Sep 2018 09:28:26 +0100 Subject: [PATCH 3/4] address review comment --- .../sql/catalyst/expressions/CollectionExpressionsSuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index 8939b79a49c10..f2cfa76fae098 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -454,7 +454,6 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(ArraysOverlap(a4, a5), true) checkEvaluation(ArraysOverlap(a4, a6), null) checkEvaluation(ArraysOverlap(a5, a6), false) - checkEvaluation(ArraysOverlap(a7, a7), true) // null handling From 7c5b65657f6e58534ff2ad897f1dfa0618634287 Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Mon, 3 Sep 2018 16:13:17 +0100 Subject: [PATCH 4/4] address review comment --- .../sql/catalyst/expressions/CollectionExpressionsSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index f2cfa76fae098..0072c9196c8dc 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -439,7 +439,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper val a4 = Literal.create(Seq[String](null, ""), ArrayType(StringType)) val a5 = Literal.create(Seq[String]("", "abc"), ArrayType(StringType)) val a6 = Literal.create(Seq[String]("def", "ghi"), ArrayType(StringType)) - val a7 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType, false)) + val a7 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType, containsNull = false)) val emptyIntArray = Literal.create(Seq.empty[Int], ArrayType(IntegerType)) @@ -474,7 +474,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper val b2 = Literal.create(Seq[Array[Byte]](Array[Byte](2, 1), Array[Byte](4, 3)), ArrayType(BinaryType)) val b3 = Literal.create(Seq[Array[Byte]](Array[Byte](1, 2), Array[Byte](3, 4)), - ArrayType(BinaryType, false)) + ArrayType(BinaryType, containsNull = false)) checkEvaluation(ArraysOverlap(b0, b1), true) checkEvaluation(ArraysOverlap(b0, b2), false)