Change logical plan to apply filter from 2ndary index

- Changes the IntroduceLSMComponentFilterRule to replace the constant filter value from the query to the value carried from 2ndary index search. - Can use 2ndary index filter even the query doens't contain any filter related condition. Change-Id: I0e2fe0208662e5dcd49d1a22bfb58f96533e9497 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1727 Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu> BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu> Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
apache · Jun 7, 2017 · de0ece7 · de0ece7
1 parent 8cbb05c
commit de0ece7
Show file tree

Hide file tree

Showing 85 changed files with 12,367 additions and 124 deletions.
diff --git a/.gitignore b/.gitignore
@@ -27,3 +27,4 @@ dist
 *.swp
 .m2*
 ß
+
diff --git a/...bra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java b/...bra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
@@ -115,6 +115,7 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext
 
         int[] minFilterFieldIndexes = getKeyIndexes(unnestMap.getMinFilterVars(), inputSchemas);
         int[] maxFilterFieldIndexes = getKeyIndexes(unnestMap.getMaxFilterVars(), inputSchemas);
+        boolean propagateFilter = unnestMap.propagateIndexFilter();
 
         MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
         Dataset dataset = metadataProvider.findDataset(jobGenParams.getDataverseName(), jobGenParams.getDatasetName());
@@ -124,7 +125,7 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext
         Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> btreeSearch = metadataProvider.buildBtreeRuntime(
                 builder.getJobSpec(), opSchema, typeEnv, context, jobGenParams.getRetainInput(), retainMissing,
                 dataset, jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes, jobGenParams.isLowKeyInclusive(),
-                jobGenParams.isHighKeyInclusive(), minFilterFieldIndexes, maxFilterFieldIndexes);
+                jobGenParams.isHighKeyInclusive(), propagateFilter, minFilterFieldIndexes, maxFilterFieldIndexes);
 
         builder.contributeHyracksOperator(unnestMap, btreeSearch.first);
         builder.contributeAlgebricksPartitionConstraint(btreeSearch.first, btreeSearch.second);

diff --git a/...bra/src/main/java/org/apache/asterix/algebra/operators/physical/IndexSearchPOperator.java b/...bra/src/main/java/org/apache/asterix/algebra/operators/physical/IndexSearchPOperator.java
@@ -63,7 +63,7 @@ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext
         IDataSource<?> ds = idx.getDataSource();
         IDataSourcePropertiesProvider dspp = ds.getPropertiesProvider();
         AbstractScanOperator as = (AbstractScanOperator) op;
-        deliveredProperties = dspp.computePropertiesVector(as.getVariables());
+        deliveredProperties = dspp.computePropertiesVector(as.getScanVariables());
     }
 
     protected int[] getKeyIndexes(List<LogicalVariable> keyVarList, IOperatorSchema[] inputSchemas) {

diff --git a/...a/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/...a/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -135,13 +135,16 @@ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildInvertedInd
             AbstractUnnestMapOperator unnestMap, IOperatorSchema opSchema, boolean retainInput, boolean retainMissing,
             String datasetName, Dataset dataset, String indexName, ATypeTag searchKeyType, int[] keyFields,
             SearchModifierType searchModifierType, IAlgebricksConstantValue similarityThreshold,
-            int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, boolean isFullTextSearchQuery)
-            throws AlgebricksException {
+            int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes,
+            boolean isFullTextSearchQuery) throws AlgebricksException {
         try {
+
+            boolean propagateIndexFilter = unnestMap.propagateIndexFilter();
             IAObject simThresh = ((AsterixConstantValue) similarityThreshold).getObject();
             int numPrimaryKeys = dataset.getPrimaryKeys().size();
-            Index secondaryIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(),
-                    dataset.getDataverseName(), dataset.getDatasetName(), indexName);
+            Index secondaryIndex = MetadataManager.INSTANCE
+                    .getIndex(metadataProvider.getMetadataTxnContext(), dataset.getDataverseName(),
+                            dataset.getDatasetName(), indexName);
             if (secondaryIndex == null) {
                 throw new AlgebricksException(
                         "Code generation error: no index " + indexName + " for dataset " + datasetName);
@@ -160,13 +163,15 @@ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildInvertedInd
             IIndexDataflowHelperFactory dataflowHelperFactory =
                     new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(),
                             secondarySplitsAndConstraint.first);
-            LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp = new LSMInvertedIndexSearchOperatorDescriptor(
-                    jobSpec, outputRecDesc, queryField, dataflowHelperFactory, queryTokenizerFactory,
-                    searchModifierFactory, retainInput, retainMissing, context.getMissingWriterFactory(),
-                    dataset.getSearchCallbackFactory(metadataProvider.getStorageComponentProvider(), secondaryIndex,
-                            ((JobEventListenerFactory) jobSpec.getJobletEventListenerFactory()).getJobId(),
-                            IndexOperation.SEARCH, null),
-                    minFilterFieldIndexes, maxFilterFieldIndexes, isFullTextSearchQuery, numPrimaryKeys, false);
+            LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp =
+                    new LSMInvertedIndexSearchOperatorDescriptor(jobSpec, outputRecDesc, queryField,
+                            dataflowHelperFactory, queryTokenizerFactory, searchModifierFactory, retainInput,
+                            retainMissing, context.getMissingWriterFactory(),
+                            dataset.getSearchCallbackFactory(metadataProvider.getStorageComponentProvider(),
+                                    secondaryIndex,
+                                    ((JobEventListenerFactory) jobSpec.getJobletEventListenerFactory()).getJobId(),
+                                    IndexOperation.SEARCH, null), minFilterFieldIndexes, maxFilterFieldIndexes,
+                            isFullTextSearchQuery, numPrimaryKeys, propagateIndexFilter);
             return new Pair<>(invIndexSearchOp, secondarySplitsAndConstraint.second);
         } catch (MetadataException e) {
             throw new AlgebricksException(e);

diff --git a/...bra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java b/...bra/src/main/java/org/apache/asterix/algebra/operators/physical/RTreeSearchPOperator.java
@@ -66,7 +66,7 @@ public PhysicalOperatorTag getOperatorTag() {
     @Override
     public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
             IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-                    throws AlgebricksException {
+            throws AlgebricksException {
         AbstractUnnestMapOperator unnestMap = (AbstractUnnestMapOperator) op;
         ILogicalExpression unnestExpr = unnestMap.getExpressionRef().getValue();
         if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
@@ -81,6 +81,7 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext
         jobGenParams.readFromFuncArgs(unnestFuncExpr.getArguments());
         int[] keyIndexes = getKeyIndexes(jobGenParams.getKeyVarList(), inputSchemas);
 
+        boolean propagateIndexFilter = unnestMap.propagateIndexFilter();
         int[] minFilterFieldIndexes = getKeyIndexes(unnestMap.getMinFilterVars(), inputSchemas);
         int[] maxFilterFieldIndexes = getKeyIndexes(unnestMap.getMaxFilterVars(), inputSchemas);
 
@@ -97,9 +98,10 @@ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext
             // By nature, LEFT_OUTER_UNNEST_MAP should generate null values for non-matching tuples.
             retainNull = true;
         }
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> rtreeSearch = mp.buildRtreeRuntime(
-                builder.getJobSpec(), outputVars, opSchema, typeEnv, context, jobGenParams.getRetainInput(), retainNull,
-                dataset, jobGenParams.getIndexName(), keyIndexes, minFilterFieldIndexes, maxFilterFieldIndexes);
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> rtreeSearch =
+                mp.buildRtreeRuntime(builder.getJobSpec(), outputVars, opSchema, typeEnv, context,
+                        jobGenParams.getRetainInput(), retainNull, dataset, jobGenParams.getIndexName(), keyIndexes,
+                        propagateIndexFilter, minFilterFieldIndexes, maxFilterFieldIndexes);
 
         builder.contributeHyracksOperator(unnestMap, rtreeSearch.first);
         builder.contributeAlgebricksPartitionConstraint(rtreeSearch.first, rtreeSearch.second);