address comments

prabeesh · Jun 24, 2015 · b558549 · b558549
1 parent 214842b
commit b558549
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 7 deletions.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -39,20 +39,23 @@ object DefaultOptimizer extends Optimizer {
     Batch("Distinct", FixedPoint(100),
       ReplaceDistinctWithAggregate) ::
     Batch("Operator Optimizations", FixedPoint(100),
+      // Operator push down
       UnionPushDown,
-      CombineFilters,
+      LimitPushDown,
+      PushPredicateThroughJoin,
       PushPredicateThroughProject,
       PushPredicateThroughGenerate,
       ColumnPruning,
-      LimitPushDown,
+      // Operator combine
       ProjectCollapsing,
+      CombineFilters,
       CombineLimits,
+      // Constant folding
       NullPropagation,
       OptimizeIn,
       ConstantFolding,
       LikeSimplification,
       BooleanSimplification,
-      PushPredicateThroughJoin,
       RemovePositive,
       SimplifyFilters,
       SimplifyCasts,
@@ -111,12 +114,14 @@ object UnionPushDown extends Rule[LogicalPlan] {
 
 object LimitPushDown extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-    // Push down limit when the child is project on limit
+    // Push down limit when the child is project on limit.
     case Limit(expr, Project(projectList, l: Limit)) =>
       Project(projectList, Limit(expr, l))
 
-    // Push down limit when the child is project on sort
-    case Limit(expr, Project(projectList, s: Sort)) =>
+    // Push down limit when the child is project on sort,
+    // and we cannot push down this project through sort.
+    case Limit(expr, p @ Project(projectList, s: Sort))
+      if !s.references.subsetOf(p.outputSet) =>
       Project(projectList, Limit(expr, s))
   }
 }

diff --git a/...atalyst/optimizer/LimitPushDownSuit.scala → ...talyst/optimizer/LimitPushDownSuite.scala b/...atalyst/optimizer/LimitPushDownSuit.scala → ...talyst/optimizer/LimitPushDownSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 
-class LimitPushDownSuit extends PlanTest {
+class LimitPushDownSuite extends PlanTest {
 
   object Optimize extends RuleExecutor[LogicalPlan] {
     val batches =