From 515519bebbec5a46de0789ea2ec82f448be0e8e8 Mon Sep 17 00:00:00 2001 From: "zhichao.li" Date: Fri, 24 Jul 2015 13:49:19 +0800 Subject: [PATCH] add foldable and remove null checking --- .../spark/sql/catalyst/expressions/stringOperations.scala | 1 + .../main/java/org/apache/spark/unsafe/types/UTF8String.java | 5 +---- .../java/org/apache/spark/unsafe/types/UTF8StringSuite.java | 3 --- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala index 18877769bd212..20252ea8a89cc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala @@ -364,6 +364,7 @@ case class Substring_index(strExpr: Expression, delimExpr: Expression, countExpr extends Expression with ImplicitCastInputTypes { override def dataType: DataType = StringType + override def foldable: Boolean = strExpr.foldable && delimExpr.foldable && countExpr.foldable override def inputTypes: Seq[DataType] = Seq(StringType, StringType, IntegerType) override def nullable: Boolean = strExpr.nullable || delimExpr.nullable || countExpr.nullable override def children: Seq[Expression] = Seq(strExpr, delimExpr, countExpr) diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index e84509f18e146..79f2f25142e82 100644 --- a/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -144,7 +144,7 @@ public byte[] getBytes() { */ public UTF8String substring(final int start, final int until) { if (until <= start || start >= numBytes) { - return fromBytes(new byte[0]); + return UTF8String.EMPTY_UTF8; } int j = firstByteIndex(0, 0, start); int i = firstByteIndex(j, start, until); @@ -502,9 +502,6 @@ private int doOrdinalIndexOf( * right) is returned. substring_index performs a case-sensitive match when searching for delim. */ public UTF8String subStringIndex(UTF8String delim, int count) { - if (delim == null) { - return null; - } if (delim.numBytes == 0 || count == 0) { return UTF8String.EMPTY_UTF8; } diff --git a/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java b/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java index 6386c9902885d..a78bb3c4374f4 100644 --- a/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java +++ b/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java @@ -321,9 +321,6 @@ public void substring_index() { fromString("www||apache||org").subStringIndex(fromString("||"), 2)); assertEquals(fromString("apache||org"), fromString("www||apache||org").subStringIndex(fromString("||"), -2)); - // null - assertEquals(null, - fromString("www.apache.org").subStringIndex(null, -2)); // non ascii chars assertEquals(fromString("大千世界大"), fromString("大千世界大千世界").subStringIndex(fromString("千"), 2));