Merge cea91a1 into ba760d5

core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java

@@ -65,6 +65,7 @@
 import org.apache.kylin.storage.IStorageQuery;
 import org.apache.kylin.storage.StorageContext;
 import org.apache.kylin.storage.translate.DerivedFilterTranslator;
+import org.apache.kylin.storage.translate.TranslatedFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -333,28 +334,48 @@ public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> gro
         }
     }
 
+    public TupleFilter translateDerived(TupleFilter filter, Set<TblColRef> collector) {
+        TranslatedFilter translatedFilter = translateDerived(filter, collector, false);
+        if (translatedFilter == null) {
+            return null;
+        }
+        return translatedFilter.getFilter();
+    }
+
     @SuppressWarnings("unchecked")
-    protected TupleFilter translateDerived(TupleFilter filter, Set<TblColRef> collector) {
+    protected TranslatedFilter translateDerived(TupleFilter filter, Set<TblColRef> collector, boolean needLoosen) {
         if (filter == null)
-            return filter;
+            return null;
 
         if (filter instanceof CompareTupleFilter) {
-            return translateDerivedInCompare((CompareTupleFilter) filter, collector);
+            return translateDerivedInCompare((CompareTupleFilter) filter, collector, needLoosen);
         }
 
         List<TupleFilter> children = (List<TupleFilter>) filter.getChildren();
         List<TupleFilter> newChildren = Lists.newArrayListWithCapacity(children.size());
+        Set<TblColRef> allCol = Sets.newHashSet();
         boolean modified = false;
+        boolean hasLoosened = false;
+        boolean childrenNeedLoosen = false;
         for (TupleFilter child : children) {
-            TupleFilter translated = translateDerived(child, collector);
-            newChildren.add(translated);
-            if (child != translated)
+            TranslatedFilter translated = translateDerived(child, collector, childrenNeedLoosen);
+            newChildren.add(translated.getFilter());
+            hasLoosened |= translated.isLoosened();
+            if (translated.isLoosened() && filter.getOperator() == FilterOperatorEnum.OR) {
+                childrenNeedLoosen = true;
+                collector.addAll(allCol);
+            }
+            if (translated.getReferCol() != null) {
+                allCol.addAll(translated.getReferCol());
+            }
+            if (child != translated.getFilter()) {
                 modified = true;
+            }
         }
         if (modified) {
             filter = replaceChildren(filter, newChildren);
         }
-        return filter;
+        return new TranslatedFilter(filter, allCol, hasLoosened);
     }
 
     private TupleFilter replaceChildren(TupleFilter filter, List<TupleFilter> newChildren) {
@@ -371,36 +392,48 @@ private TupleFilter replaceChildren(TupleFilter filter, List<TupleFilter> newChi
         }
     }
 
-    private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
+    private TranslatedFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector,
+            boolean needLoosen) {
         if (compf.getColumn() == null)
-            return compf;
+            return new TranslatedFilter(compf, null, false);
 
         TblColRef derived = compf.getColumn();
         if (cubeDesc.isExtendedColumn(derived)) {
             throw new CubeDesc.CannotFilterExtendedColumnException(derived);
         }
-        if (!cubeDesc.isDerived(derived))
-            return compf;
-
-        DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
-        ILookupTable lookup = cubeDesc.getHostInfo(derived).type == CubeDesc.DeriveType.PK_FK ? null
-                : getLookupStringTableForDerived(derived, hostInfo);
-        Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
-        try {
-            if (lookup != null) {
-                lookup.close();
+        Set<TblColRef> sourceCollector = Sets.newHashSet();
+        Set<TblColRef> currentFilterColRefCollector = Sets.newHashSet();
+        TupleFilter.collectColumns(compf, sourceCollector);
+        for (TblColRef colRef : sourceCollector) {
+            if (cubeDesc.isDerived(colRef)) {
+                currentFilterColRefCollector.addAll(Arrays.asList(cubeDesc.getHostInfo(colRef).columns));
+            } else {
+                currentFilterColRefCollector.add(colRef);
             }
-        } catch (IOException e) {
-            logger.error("error when close lookup table.", e);
         }
-        TupleFilter translatedFilter = translated.getFirst();
-        boolean loosened = translated.getSecond();
-        if (loosened) {
-            collectColumnsRecursively(translatedFilter, collector);
+        boolean loosened = false;
+        TupleFilter translatedFilter = compf;
+        if (cubeDesc.isDerived(derived)) {
+            DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
+            ILookupTable lookup = cubeDesc.getHostInfo(derived).type == CubeDesc.DeriveType.PK_FK ? null
+                    : getLookupStringTableForDerived(derived, hostInfo);
+            Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
+            try {
+                if (lookup != null) {
+                    lookup.close();
+                }
+            } catch (IOException e) {
+                logger.error("error when close lookup table.", e);
+            }
+            translatedFilter = translated.getFirst();
+            loosened = translated.getSecond();
+        }
+        if (needLoosen || loosened) {
+            collector.addAll(currentFilterColRefCollector);
         }
-        return translatedFilter;
+        return new TranslatedFilter(translatedFilter, currentFilterColRefCollector, loosened);
     }
-
+    
     @SuppressWarnings("unchecked")
     protected ILookupTable getLookupStringTableForDerived(TblColRef derived, DeriveInfo hostInfo) {
         CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());

core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java

@@ -18,6 +18,7 @@
 
 package org.apache.kylin.storage.translate;
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 
@@ -79,6 +80,10 @@ public static Pair<TupleFilter, Boolean> translate(ILookupTable lookup, DeriveIn
             return new Pair<TupleFilter, Boolean>(newComp, false);
         }
 
+        if (!compf.isEvaluable()) {
+            return new Pair<>(ConstantTupleFilter.TRUE, true);
+        }
+
         assert hostInfo.type == DeriveType.LOOKUP;
         assert hostCols.length == pkCols.length;
 
@@ -114,6 +119,8 @@ public static Pair<TupleFilter, Boolean> translate(ILookupTable lookup, DeriveIn
             logger.info("Deciding to loosen filter on derived filter as host candidates number {} exceeds threshold {}", //
                     satisfyingHostRecords.size(), KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
             );
+            logger.debug("loosened hostCol is {}",
+                    Arrays.stream(hostCols).map(TblColRef::getCanonicalName).reduce((x1, x2) -> x1 + "," + x2).get());
             translated = buildRangeFilter(hostCols, satisfyingHostRecords);
             loosened = true;
         } else {

core-storage/src/main/java/org/apache/kylin/storage/translate/TranslatedFilter.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.translate;
+
+import java.util.Set;
+
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class TranslatedFilter {
+    private TupleFilter filter;
+    private Set<TblColRef> referCol;
+    private boolean isLoosened;
+
+    public TranslatedFilter(TupleFilter filter, Set<TblColRef> referCol, boolean isLoosened) {
+        this.filter = filter;
+        this.referCol = referCol;
+        this.isLoosened = isLoosened;
+    }
+
+    public TupleFilter getFilter() {
+        return filter;
+    }
+
+    public Set<TblColRef> getReferCol() {
+        return referCol;
+    }
+
+    public boolean isLoosened() {
+        return isLoosened;
+    }
+}