From d96929bd4f9f740be82164430c3cbb9f1d8b4035 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Thu, 18 Jun 2015 16:46:29 -0700 Subject: [PATCH] address comment --- python/pyspark/streaming/dstream.py | 2 +- .../spark/sql/catalyst/InternalRow.scala | 47 ++++++++++--------- .../expressions/ExpressionEvalHelper.scala | 4 +- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/python/pyspark/streaming/dstream.py b/python/pyspark/streaming/dstream.py index ff097985fae3e..8dcb9645cdc6b 100644 --- a/python/pyspark/streaming/dstream.py +++ b/python/pyspark/streaming/dstream.py @@ -176,7 +176,7 @@ def takeAndPrint(time, rdd): print(record) if len(taken) > num: print("...") - print() + print("") self.foreachRDD(takeAndPrint) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala index 60fa6f5740adc..5e176fd8175fe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala @@ -26,35 +26,38 @@ import org.apache.spark.sql.catalyst.expressions._ */ abstract class InternalRow extends Row { // A default implementation to change the return type - override def copy(): InternalRow = {this} + override def copy(): InternalRow = this - // A default version (slow), used for tests - override def equals(o: Any): Boolean = o match { - case other: InternalRow => - if (length != other.length) { + override def equals(o: Any): Boolean = { + if (!o.isInstanceOf[Row]) { + return false + } + + val other = o.asInstanceOf[Row] + if (length != other.length) { + return false + } + + for (i <- 0 until length) { + if (isNullAt(i) != other.isNullAt(i)) { return false } - - for (i <- 0 until length) { - if (isNullAt(i) != other.isNullAt(i)) { - return false - } - if (!isNullAt(i)) { - val o1 = apply(i) - val o2 = other.apply(i) - if (o1.isInstanceOf[Array[Byte]]) { - val b1 = o1.asInstanceOf[Array[Byte]] - if (!o2.isInstanceOf[Array[Byte]] || - !java.util.Arrays.equals(b1, o2.asInstanceOf[Array[Byte]])) { - return false - } - } else if (o1 != o2) { + if (!isNullAt(i)) { + val o1 = apply(i) + val o2 = other.apply(i) + if (o1.isInstanceOf[Array[Byte]]) { + // handle equality of Array[Byte] + val b1 = o1.asInstanceOf[Array[Byte]] + if (!o2.isInstanceOf[Array[Byte]] || + !java.util.Arrays.equals(b1, o2.asInstanceOf[Array[Byte]])) { return false } + } else if (o1 != o2) { + return false } } - true - case _ => false + } + true } // Custom hashCode function that matches the efficient code generated version. diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala index aa1ad43149c86..12d2da8b33986 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala @@ -55,7 +55,7 @@ trait ExpressionEvalHelper { val actual = try evaluate(expression, inputRow) catch { case e: Exception => fail(s"Exception evaluating $expression", e) } - if (actual !== expected) { + if (actual != expected) { val input = if (inputRow == EmptyRow) "" else s", input: $inputRow" fail(s"Incorrect evaluation (codegen off): $expression, " + s"actual: $actual, " + @@ -83,7 +83,7 @@ trait ExpressionEvalHelper { } val actual = plan(inputRow).apply(0) - if (actual !== expected) { + if (actual != expected) { val input = if (inputRow == EmptyRow) "" else s", input: $inputRow" fail(s"Incorrect Evaluation: $expression, actual: $actual, expected: $expected$input") }