added comment for spark-4937

apache · Jan 17, 2015 · 2d3406e · 2d3406e
1 parent ee1c1f3
commit 2d3406e
Showing 1 changed file with 14 additions and 2 deletions.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -311,13 +311,19 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
           // a && a => a
           case (l, r) if l fastEquals r => l
           case (_, _) =>
+            /* Do optimize for predicates using formula (a || b) && (a || c) => a || (b && c)
+             * 1. Split left and right to get the disjunctive predicates, i.e. lhsSet = (a, b), rhsSet = (a, c)
+             * 2. Find the common predict between lhsSet and rhsSet, i.e. common = (a)
+             * 3. Remove common predict from lhsSet and rhsSet, i.e. ldiff = (b), rdiff = (c)
+             * 4. Apply the formula, get the optimized predict: common || (ldiff && rdiff)
+             */
             val lhsSet = splitDisjunctivePredicates(left).toSet
             val rhsSet = splitDisjunctivePredicates(right).toSet
             val common = lhsSet.intersect(rhsSet)
             val ldiff = lhsSet.diff(common)
             val rdiff = rhsSet.diff(common)
             if (ldiff.size == 0 || rdiff.size == 0) {
-              // a && (a || b)
+              // a && (a || b) => a
               common.reduce(Or)
             } else {
               // (a || b || c || ...) && (a || b || d || ...) && (a || b || e || ...) ... =>
@@ -339,13 +345,19 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
           // a || a => a
           case (l, r) if l fastEquals r => l
           case (_, _) =>
+            /* Do optimize for predicates using formula (a && b) || (a && c) => a && (b || c)
+             * 1. Split left and right to get the conjunctive predicates, i.e. lhsSet = (a, b), rhsSet = (a, c)
+             * 2. Find the common predict between lhsSet and rhsSet, i.e. common = (a)
+             * 3. Remove common predict from lhsSet and rhsSet, i.e. ldiff = (b), rdiff = (c)
+             * 4. Apply the formula, get the optimized predict: common && (ldiff || rdiff)
+             */
             val lhsSet = splitConjunctivePredicates(left).toSet
             val rhsSet = splitConjunctivePredicates(right).toSet
             val common = lhsSet.intersect(rhsSet)
             val ldiff = lhsSet.diff(common)
             val rdiff = rhsSet.diff(common)
             if ( ldiff.size == 0 || rdiff.size == 0) {
-              // a || (b && a)
+              // a || (b && a) => a
               common.reduce(And)
             } else {
               // (a && b && c && ...) || (a && b && d && ...) || (a && b && e && ...) ... =>