From fb6f3b5a6c5fb80adc249fbb54a8c2ed884c7dbb Mon Sep 17 00:00:00 2001 From: Hiroshi Inoue Date: Tue, 7 Jun 2016 02:31:04 +0900 Subject: [PATCH 1/2] enable null check elimination based on generated code --- .../spark/sql/catalyst/expressions/Expression.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala index 2ec46216e1cdb..c673625de4879 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala @@ -101,6 +101,16 @@ abstract class Expression extends TreeNode[Expression] { val isNull = ctx.freshName("isNull") val value = ctx.freshName("value") val ve = doGenCode(ctx, ExprCode("", isNull, value)) + + // if we can assure that isNull is always false from the generated code, + // set isNull "false" to propagate this information for further optimizations + // condition: "boolean isNull = false | false | false ...;" is the only one + // assignment to the variable ${isNull} in the generated code + if (s"${isNull} = ".r.findAllIn(ve.code).size == 1 && + ve.code.matches(s"[\\s\\S]* boolean ${isNull} = false(?: \\|\\| false)*;[\\s\\S]*")) { + ve.isNull = "false" + } + if (ve.code.nonEmpty) { // Add `this` in the comment. ve.copy(code = s"${ctx.registerComment(this.toString)}\n" + ve.code.trim) From 60f582dc3e75db6ff5fe642b692f22f5d7bc7ab2 Mon Sep 17 00:00:00 2001 From: Hiroshi Inoue Date: Tue, 7 Jun 2016 10:53:33 +0900 Subject: [PATCH 2/2] make definition of isNull final --- .../apache/spark/sql/catalyst/expressions/Expression.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala index c673625de4879..71b7ff7c3f24a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala @@ -109,6 +109,11 @@ abstract class Expression extends TreeNode[Expression] { if (s"${isNull} = ".r.findAllIn(ve.code).size == 1 && ve.code.matches(s"[\\s\\S]* boolean ${isNull} = false(?: \\|\\| false)*;[\\s\\S]*")) { ve.isNull = "false" + + // make ${isNull} variable final to assure that noone overwrite this later + if (!ve.code.contains(s"final boolean ${isNull} = ")) { + ve.code = ve.code.replace(s"boolean ${isNull} = ", s"final boolean ${isNull} = ") + } } if (ve.code.nonEmpty) {