From 882f722e40865dafc745d3468903a7dda627b926 Mon Sep 17 00:00:00 2001 From: Mehant Baid Date: Wed, 15 Jul 2015 14:30:16 -0700 Subject: [PATCH] DRILL-3500: Add OptimizerRulesContext which exposes information needed by storage plugin specific optimizer rules Add FunctionLookupContext to enable materializing function calls without having access to the entire function registry --- .../exec/store/hbase/HBaseStoragePlugin.java | 3 +- .../exec/store/hive/HiveStoragePlugin.java | 5 +- .../exec/store/mongo/MongoStoragePlugin.java | 4 +- .../exec/expr/ExpressionTreeMaterializer.java | 108 +++++++++--------- .../fn/FunctionImplementationRegistry.java | 4 +- .../exec/expr/fn/FunctionLookupContext.java | 47 ++++++++ .../drill/exec/ops/OptimizerRulesContext.java | 42 +++++++ .../apache/drill/exec/ops/QueryContext.java | 5 +- .../exec/planner/logical/DrillRuleSets.java | 37 +++--- .../logical/partition/PruneScanRule.java | 32 +++--- .../exec/planner/sql/DrillSqlWorker.java | 2 +- .../exec/store/AbstractStoragePlugin.java | 3 +- .../drill/exec/store/StoragePlugin.java | 3 +- .../exec/store/StoragePluginRegistry.java | 5 +- .../exec/store/dfs/FileSystemPlugin.java | 3 +- .../ischema/InfoSchemaStoragePlugin.java | 3 +- 16 files changed, 206 insertions(+), 100 deletions(-) create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionLookupContext.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/ops/OptimizerRulesContext.java diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java index 7737f69c4c5..32a57f291b1 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java @@ -23,6 +23,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.drill.common.JSONOptions; +import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; import org.apache.drill.exec.store.SchemaConfig; @@ -76,7 +77,7 @@ public HBaseStoragePluginConfig getConfig() { } @Override - public Set getOptimizerRules() { + public Set getOptimizerRules(OptimizerRulesContext optimizerRulesContext) { return ImmutableSet.of(HBasePushFilterIntoScan.INSTANCE); } diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java index fb827cc86e2..4813c5e68cb 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java @@ -28,6 +28,7 @@ import org.apache.drill.common.JSONOptions; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.planner.sql.logical.HivePushPartitionFilterIntoScan; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; @@ -85,7 +86,9 @@ public HiveScan getPhysicalScan(String userName, JSONOptions selection, List getOptimizerRules() { + + @Override + public Set getOptimizerRules(OptimizerRulesContext optimizerRulesContext) { return ImmutableSet.of(HivePushPartitionFilterIntoScan.HIVE_FILTER_ON_PROJECT, HivePushPartitionFilterIntoScan.HIVE_FILTER_ON_SCAN); } diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java index 093df573e8f..00912eadd2f 100644 --- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java +++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java @@ -27,6 +27,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.drill.common.JSONOptions; import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; @@ -96,7 +97,8 @@ public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection) return new MongoGroupScan(userName, this, mongoScanSpec, null); } - public Set getOptimizerRules() { + @Override + public Set getOptimizerRules(OptimizerRulesContext optimizerRulesContext) { return ImmutableSet.of(MongoPushDownFilterForScan.INSTANCE); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index 7413a97fa7d..df315b2e07c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -66,6 +66,7 @@ import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder; import org.apache.drill.exec.expr.fn.DrillFuncHolder; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; +import org.apache.drill.exec.expr.fn.FunctionLookupContext; import org.apache.drill.exec.record.TypedFieldId; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.resolver.FunctionResolver; @@ -77,7 +78,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import org.apache.drill.exec.vector.VarCharVector; public class ExpressionTreeMaterializer { @@ -86,22 +86,22 @@ public class ExpressionTreeMaterializer { private ExpressionTreeMaterializer() { }; - public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionImplementationRegistry registry) { - return ExpressionTreeMaterializer.materialize(expr, batch, errorCollector, registry, false); + public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionLookupContext functionLookupContext) { + return ExpressionTreeMaterializer.materialize(expr, batch, errorCollector, functionLookupContext, false); } - public static LogicalExpression materializeAndCheckErrors(LogicalExpression expr, VectorAccessible batch, FunctionImplementationRegistry registry) throws SchemaChangeException { + public static LogicalExpression materializeAndCheckErrors(LogicalExpression expr, VectorAccessible batch, FunctionLookupContext functionLookupContext) throws SchemaChangeException { ErrorCollector collector = new ErrorCollectorImpl(); - LogicalExpression e = ExpressionTreeMaterializer.materialize(expr, batch, collector, registry, false); + LogicalExpression e = ExpressionTreeMaterializer.materialize(expr, batch, collector, functionLookupContext, false); if (collector.hasErrors()) { throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString())); } return e; } - public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionImplementationRegistry registry, + public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionLookupContext functionLookupContext, boolean allowComplexWriterExpr) { - LogicalExpression out = expr.accept(new MaterializeVisitor(batch, errorCollector, allowComplexWriterExpr), registry); + LogicalExpression out = expr.accept(new MaterializeVisitor(batch, errorCollector, allowComplexWriterExpr), functionLookupContext); if (!errorCollector.hasErrors()) { out = out.accept(ConditionalExprOptimizer.INSTANCE, null); @@ -114,14 +114,14 @@ public static LogicalExpression materialize(LogicalExpression expr, VectorAccess } } - public static LogicalExpression convertToNullableType(LogicalExpression fromExpr, MinorType toType, FunctionImplementationRegistry registry, ErrorCollector errorCollector) { + public static LogicalExpression convertToNullableType(LogicalExpression fromExpr, MinorType toType, FunctionLookupContext functionLookupContext, ErrorCollector errorCollector) { String funcName = "convertToNullable" + toType.toString(); List args = Lists.newArrayList(); args.add(fromExpr); FunctionCall funcCall = new FunctionCall(funcName, args, ExpressionPosition.UNKNOWN); FunctionResolver resolver = FunctionResolverFactory.getResolver(funcCall); - DrillFuncHolder matchedConvertToNullableFuncHolder = registry.findDrillFunction(resolver, funcCall); + DrillFuncHolder matchedConvertToNullableFuncHolder = functionLookupContext.findDrillFunction(resolver, funcCall); if (matchedConvertToNullableFuncHolder == null) { logFunctionResolutionError(errorCollector, funcCall); return NullExpression.INSTANCE; @@ -131,7 +131,7 @@ public static LogicalExpression convertToNullableType(LogicalExpression fromExpr } - public static LogicalExpression addCastExpression(LogicalExpression fromExpr, MajorType toType, FunctionImplementationRegistry registry, ErrorCollector errorCollector) { + public static LogicalExpression addCastExpression(LogicalExpression fromExpr, MajorType toType, FunctionLookupContext functionLookupContext, ErrorCollector errorCollector) { String castFuncName = CastFunctions.getCastFunc(toType.getMinorType()); List castArgs = Lists.newArrayList(); castArgs.add(fromExpr); //input_expr @@ -151,7 +151,7 @@ else if (CoreDecimalUtility.isDecimalType(toType)) { } FunctionCall castCall = new FunctionCall(castFuncName, castArgs, ExpressionPosition.UNKNOWN); FunctionResolver resolver = FunctionResolverFactory.getExactResolver(castCall); - DrillFuncHolder matchedCastFuncHolder = registry.findDrillFunction(resolver, castCall); + DrillFuncHolder matchedCastFuncHolder = functionLookupContext.findDrillFunction(resolver, castCall); if (matchedCastFuncHolder == null) { logFunctionResolutionError(errorCollector, castCall); @@ -185,7 +185,7 @@ private static void logFunctionResolutionError(ErrorCollector errorCollector, Fu errorCollector.addGeneralError(call.getPosition(), sb.toString()); } - private static class MaterializeVisitor extends AbstractExprVisitor { + private static class MaterializeVisitor extends AbstractExprVisitor { private ExpressionValidator validator = new ExpressionValidator(); private final ErrorCollector errorCollector; private final VectorAccessible batch; @@ -203,22 +203,22 @@ private LogicalExpression validateNewExpr(LogicalExpression newExpr) { } @Override - public LogicalExpression visitUnknown(LogicalExpression e, FunctionImplementationRegistry registry) + public LogicalExpression visitUnknown(LogicalExpression e, FunctionLookupContext functionLookupContext) throws RuntimeException { return e; } @Override - public LogicalExpression visitFunctionHolderExpression(FunctionHolderExpression holder, FunctionImplementationRegistry value) throws RuntimeException { + public LogicalExpression visitFunctionHolderExpression(FunctionHolderExpression holder, FunctionLookupContext functionLookupContext) throws RuntimeException { // a function holder is already materialized, no need to rematerialize. generally this won't be used unless we materialize a partial tree and rematerialize the whole tree. return holder; } @Override - public LogicalExpression visitBooleanOperator(BooleanOperator op, FunctionImplementationRegistry registry) { + public LogicalExpression visitBooleanOperator(BooleanOperator op, FunctionLookupContext functionLookupContext) { List args = Lists.newArrayList(); for (int i = 0; i < op.args.size(); ++i) { - LogicalExpression newExpr = op.args.get(i).accept(this, registry); + LogicalExpression newExpr = op.args.get(i).accept(this, functionLookupContext); assert newExpr != null : String.format("Materialization of %s return a null expression.", op.args.get(i)); args.add(newExpr); } @@ -228,10 +228,10 @@ public LogicalExpression visitBooleanOperator(BooleanOperator op, FunctionImplem } @Override - public LogicalExpression visitFunctionCall(FunctionCall call, FunctionImplementationRegistry registry) { + public LogicalExpression visitFunctionCall(FunctionCall call, FunctionLookupContext functionLookupContext) { List args = Lists.newArrayList(); for (int i = 0; i < call.args.size(); ++i) { - LogicalExpression newExpr = call.args.get(i).accept(this, registry); + LogicalExpression newExpr = call.args.get(i).accept(this, functionLookupContext); assert newExpr != null : String.format("Materialization of %s returned a null expression.", call.args.get(i)); args.add(newExpr); } @@ -240,7 +240,7 @@ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionImplementa call = new FunctionCall(call.getName(), args, call.getPosition()); FunctionResolver resolver = FunctionResolverFactory.getResolver(call); - DrillFuncHolder matchedFuncHolder = registry.findDrillFunction(resolver, call); + DrillFuncHolder matchedFuncHolder = functionLookupContext.findDrillFunction(resolver, call); if (matchedFuncHolder instanceof DrillComplexWriterFuncHolder && ! allowComplexWriter) { errorCollector.addGeneralError(call.getPosition(), "Only ProjectRecordBatch could have complex writer function. You are using complex writer function " + call.getName() + " in a non-project operation!"); @@ -275,7 +275,7 @@ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionImplementa parmType = MajorType.newBuilder().setMinorType(parmType.getMinorType()).setMode(parmType.getMode()). setScale(currentArg.getMajorType().getScale()).setPrecision(currentArg.getMajorType().getPrecision()).build(); } - argsWithCast.add(addCastExpression(currentArg, parmType, registry, errorCollector)); + argsWithCast.add(addCastExpression(currentArg, parmType, functionLookupContext, errorCollector)); } } @@ -283,7 +283,7 @@ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionImplementa } // as no drill func is found, search for a non-Drill function. - AbstractFuncHolder matchedNonDrillFuncHolder = registry.findNonDrillFunction(call); + AbstractFuncHolder matchedNonDrillFuncHolder = functionLookupContext.findNonDrillFunction(call); if (matchedNonDrillFuncHolder != null) { // Insert implicit cast function holder expressions if required List extArgsWithCast = Lists.newArrayList(); @@ -301,7 +301,7 @@ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionImplementa parmType = MajorType.newBuilder().setMinorType(parmType.getMinorType()).setMode(parmType.getMode()). setScale(currentArg.getMajorType().getScale()).setPrecision(currentArg.getMajorType().getPrecision()).build(); } - extArgsWithCast.add(addCastExpression(call.args.get(i), parmType, registry, errorCollector)); + extArgsWithCast.add(addCastExpression(call.args.get(i), parmType, functionLookupContext, errorCollector)); } } @@ -313,12 +313,12 @@ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionImplementa } @Override - public LogicalExpression visitIfExpression(IfExpression ifExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitIfExpression(IfExpression ifExpr, FunctionLookupContext functionLookupContext) { IfExpression.IfCondition conditions = ifExpr.ifCondition; - LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, registry); + LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, functionLookupContext); - LogicalExpression newCondition = conditions.condition.accept(this, registry); - LogicalExpression newExpr = conditions.expression.accept(this, registry); + LogicalExpression newCondition = conditions.condition.accept(this, functionLookupContext); + LogicalExpression newExpr = conditions.expression.accept(this, functionLookupContext); conditions = new IfExpression.IfCondition(newCondition, newExpr); MinorType thenType = conditions.expression.getMajorType().getMinorType(); @@ -331,10 +331,10 @@ public LogicalExpression visitIfExpression(IfExpression ifExpr, FunctionImplemen if (leastRestrictive != thenType) { // Implicitly cast the then expression conditions = new IfExpression.IfCondition(newCondition, - addCastExpression(conditions.expression, newElseExpr.getMajorType(), registry, errorCollector)); + addCastExpression(conditions.expression, newElseExpr.getMajorType(), functionLookupContext, errorCollector)); } else if (leastRestrictive != elseType) { // Implicitly cast the else expression - newElseExpr = addCastExpression(newElseExpr, conditions.expression.getMajorType(), registry, errorCollector); + newElseExpr = addCastExpression(newElseExpr, conditions.expression.getMajorType(), functionLookupContext, errorCollector); } else { /* Cannot cast one of the two expressions to make the output type of if and else expression * to be the same. Raise error. @@ -381,12 +381,12 @@ public boolean apply(LogicalExpression input) { IfExpression.IfCondition condition = conditions; if (condition.expression.getMajorType().getMode() != DataMode.OPTIONAL) { conditions = new IfExpression.IfCondition(condition.condition, getConvertToNullableExpr(ImmutableList.of(condition.expression), - condition.expression.getMajorType().getMinorType(), registry)); + condition.expression.getMajorType().getMinorType(), functionLookupContext)); } if (newElseExpr.getMajorType().getMode() != DataMode.OPTIONAL) { newElseExpr = getConvertToNullableExpr(ImmutableList.of(newElseExpr), - newElseExpr.getMajorType().getMinorType(), registry); + newElseExpr.getMajorType().getMinorType(), functionLookupContext); } } @@ -394,12 +394,12 @@ public boolean apply(LogicalExpression input) { } private LogicalExpression getConvertToNullableExpr(List args, MinorType minorType, - FunctionImplementationRegistry registry) { + FunctionLookupContext functionLookupContext) { String funcName = "convertToNullable" + minorType.toString(); FunctionCall funcCall = new FunctionCall(funcName, args, ExpressionPosition.UNKNOWN); FunctionResolver resolver = FunctionResolverFactory.getResolver(funcCall); - DrillFuncHolder matchedConvertToNullableFuncHolder = registry.findDrillFunction(resolver, funcCall); + DrillFuncHolder matchedConvertToNullableFuncHolder = functionLookupContext.findDrillFunction(resolver, funcCall); if (matchedConvertToNullableFuncHolder == null) { logFunctionResolutionError(errorCollector, funcCall); @@ -418,7 +418,7 @@ private LogicalExpression rewriteNullExpression(LogicalExpression expr, MajorTyp } @Override - public LogicalExpression visitSchemaPath(SchemaPath path, FunctionImplementationRegistry value) { + public LogicalExpression visitSchemaPath(SchemaPath path, FunctionLookupContext functionLookupContext) { // logger.debug("Visiting schema path {}", path); TypedFieldId tfId = batch.getValueVectorId(path); if (tfId == null) { @@ -431,101 +431,101 @@ public LogicalExpression visitSchemaPath(SchemaPath path, FunctionImplementation } @Override - public LogicalExpression visitIntConstant(IntExpression intExpr, FunctionImplementationRegistry value) { + public LogicalExpression visitIntConstant(IntExpression intExpr, FunctionLookupContext functionLookupContext) { return intExpr; } @Override - public LogicalExpression visitFloatConstant(FloatExpression fExpr, FunctionImplementationRegistry value) { + public LogicalExpression visitFloatConstant(FloatExpression fExpr, FunctionLookupContext functionLookupContext) { return fExpr; } @Override - public LogicalExpression visitLongConstant(LongExpression intExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitLongConstant(LongExpression intExpr, FunctionLookupContext functionLookupContext) { return intExpr; } @Override - public LogicalExpression visitDateConstant(DateExpression intExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitDateConstant(DateExpression intExpr, FunctionLookupContext functionLookupContext) { return intExpr; } @Override - public LogicalExpression visitTimeConstant(TimeExpression intExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitTimeConstant(TimeExpression intExpr, FunctionLookupContext functionLookupContext) { return intExpr; } @Override - public LogicalExpression visitTimeStampConstant(TimeStampExpression intExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitTimeStampConstant(TimeStampExpression intExpr, FunctionLookupContext functionLookupContext) { return intExpr; } @Override - public LogicalExpression visitNullConstant(TypedNullConstant nullConstant, FunctionImplementationRegistry value) throws RuntimeException { + public LogicalExpression visitNullConstant(TypedNullConstant nullConstant, FunctionLookupContext functionLookupContext) throws RuntimeException { return nullConstant; } @Override - public LogicalExpression visitIntervalYearConstant(IntervalYearExpression intExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitIntervalYearConstant(IntervalYearExpression intExpr, FunctionLookupContext functionLookupContext) { return intExpr; } @Override - public LogicalExpression visitIntervalDayConstant(IntervalDayExpression intExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitIntervalDayConstant(IntervalDayExpression intExpr, FunctionLookupContext functionLookupContext) { return intExpr; } @Override - public LogicalExpression visitDecimal9Constant(Decimal9Expression decExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitDecimal9Constant(Decimal9Expression decExpr, FunctionLookupContext functionLookupContext) { return decExpr; } @Override - public LogicalExpression visitDecimal18Constant(Decimal18Expression decExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitDecimal18Constant(Decimal18Expression decExpr, FunctionLookupContext functionLookupContext) { return decExpr; } @Override - public LogicalExpression visitDecimal28Constant(Decimal28Expression decExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitDecimal28Constant(Decimal28Expression decExpr, FunctionLookupContext functionLookupContext) { return decExpr; } @Override - public LogicalExpression visitDecimal38Constant(Decimal38Expression decExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitDecimal38Constant(Decimal38Expression decExpr, FunctionLookupContext functionLookupContext) { return decExpr; } @Override - public LogicalExpression visitDoubleConstant(DoubleExpression dExpr, FunctionImplementationRegistry registry) { + public LogicalExpression visitDoubleConstant(DoubleExpression dExpr, FunctionLookupContext functionLookupContext) { return dExpr; } @Override - public LogicalExpression visitBooleanConstant(BooleanExpression e, FunctionImplementationRegistry registry) { + public LogicalExpression visitBooleanConstant(BooleanExpression e, FunctionLookupContext functionLookupContext) { return e; } @Override - public LogicalExpression visitQuotedStringConstant(QuotedString e, FunctionImplementationRegistry registry) { + public LogicalExpression visitQuotedStringConstant(QuotedString e, FunctionLookupContext functionLookupContext) { return e; } @Override - public LogicalExpression visitConvertExpression(ConvertExpression e, FunctionImplementationRegistry value) { + public LogicalExpression visitConvertExpression(ConvertExpression e, FunctionLookupContext functionLookupContext) { String convertFunctionName = e.getConvertFunction() + e.getEncodingType(); List newArgs = Lists.newArrayList(); newArgs.add(e.getInput()); //input_expr FunctionCall fc = new FunctionCall(convertFunctionName, newArgs, e.getPosition()); - return fc.accept(this, value); + return fc.accept(this, functionLookupContext); } @Override - public LogicalExpression visitCastExpression(CastExpression e, FunctionImplementationRegistry value) { + public LogicalExpression visitCastExpression(CastExpression e, FunctionLookupContext functionLookupContext) { // if the cast is pointless, remove it. - LogicalExpression input = e.getInput().accept(this, value); + LogicalExpression input = e.getInput().accept(this, functionLookupContext); MajorType newMajor = e.getMajorType(); // Output type MinorType newMinor = input.getMajorType().getMinorType(); // Input type @@ -558,7 +558,7 @@ public LogicalExpression visitCastExpression(CastExpression e, FunctionImplement newArgs.add(new ValueExpressions.LongExpression(type.getScale(), null)); } FunctionCall fc = new FunctionCall(castFuncWithType, newArgs, e.getPosition()); - return fc.accept(this, value); + return fc.accept(this, functionLookupContext); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java index 05d59ff7edd..dee93c10fed 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java @@ -37,7 +37,7 @@ import com.google.common.collect.Lists; import org.apache.drill.exec.server.options.OptionManager; -public class FunctionImplementationRegistry { +public class FunctionImplementationRegistry implements FunctionLookupContext { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class); private DrillFunctionRegistry drillFuncRegistry; @@ -101,6 +101,7 @@ public void register(DrillOperatorTable operatorTable) { * @param functionCall * @return */ + @Override public DrillFuncHolder findDrillFunction(FunctionResolver functionResolver, FunctionCall functionCall) { return functionResolver.getBestMatch(drillFuncRegistry.getMethods(functionReplacement(functionCall)), functionCall); } @@ -147,6 +148,7 @@ public DrillFuncHolder findExactMatchingDrillFunction(String name, ListfunctionCall in non-Drill function registries such as Hive UDF + * registry. + * + * Note: Order of searching is same as order of {@link org.apache.drill.exec.expr.fn.PluggableFunctionRegistry} + * implementations found on classpath. + * + * @param functionCall - Specifies function name and type of arguments + * @return + */ + public AbstractFuncHolder findNonDrillFunction(FunctionCall functionCall); +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OptimizerRulesContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OptimizerRulesContext.java new file mode 100644 index 00000000000..33179f383b4 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OptimizerRulesContext.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.ops; + +import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.planner.physical.PlannerSettings; + +public interface OptimizerRulesContext extends UdfUtilities { + /** + * Method returns the function registry + * @return FunctionImplementationRegistry + */ + public FunctionImplementationRegistry getFunctionRegistry(); + + /** + * Method returns the allocator + * @return BufferAllocator + */ + public BufferAllocator getAllocator(); + + /** + * Method returns the planner options + * @return PlannerSettings + */ + public PlannerSettings getPlannerSettings(); +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java index 60fba0f2d0b..c3cd4e609bc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java @@ -54,7 +54,7 @@ // TODO the many methods that just return drillbitContext.getXxx() should be replaced with getDrillbitContext() // TODO - consider re-name to PlanningContext, as the query execution context actually appears // in fragment contexts -public class QueryContext implements AutoCloseable, UdfUtilities { +public class QueryContext implements AutoCloseable, OptimizerRulesContext { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryContext.class); private static final int INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES = 1024 * 1024; @@ -105,6 +105,7 @@ public QueryContext(final UserSession session, final DrillbitContext drillbitCon schemaTreesToClose = Lists.newArrayList(); } + @Override public PlannerSettings getPlannerSettings() { return plannerSettings; } @@ -113,6 +114,7 @@ public UserSession getSession() { return session; } + @Override public BufferAllocator getAllocator() { return allocator; } @@ -205,6 +207,7 @@ public DrillConfig getConfig() { return drillbitContext.getConfig(); } + @Override public FunctionImplementationRegistry getFunctionRegistry() { return drillbitContext.getFunctionImplementationRegistry(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java index daa72766472..a821a629ff9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java @@ -32,7 +32,7 @@ import org.apache.calcite.tools.RuleSet; import org.apache.calcite.rel.rules.FilterMergeRule; -import org.apache.drill.exec.ops.QueryContext; +import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.planner.logical.partition.PruneScanRule; import org.apache.drill.exec.planner.physical.ConvertCountToDirectScan; import org.apache.drill.exec.planner.physical.FilterPrule; @@ -71,15 +71,15 @@ public class DrillRuleSets { * If a rule is intended to always be included with the logical set, it should be added * to the immutable list created in the getDrillBasicRules() method below. * - * @param queryContext - used to get the list of planner settings, other rules may - * also in the future need to get other query state from this, - * such as the available list of UDFs (as is used by the - * DrillMergeProjectRule created in getDrillBasicRules()) + * @param optimizerRulesContext - used to get the list of planner settings, other rules may + * also in the future need to get other query state from this, + * such as the available list of UDFs (as is used by the + * DrillMergeProjectRule created in getDrillBasicRules()) * @return - a list of rules that have been filtered to leave out * rules that have been turned off by system or session settings */ - public static RuleSet getDrillUserConfigurableLogicalRules(QueryContext queryContext) { - PlannerSettings ps = queryContext.getPlannerSettings(); + public static RuleSet getDrillUserConfigurableLogicalRules(OptimizerRulesContext optimizerRulesContext) { + PlannerSettings ps = optimizerRulesContext.getPlannerSettings(); // This list is used to store rules that can be turned on an off // by user facing planning options @@ -110,15 +110,15 @@ public static RuleSet getDrillUserConfigurableLogicalRules(QueryContext queryCon * it on and off with a system/session option, add it in the * getDrillUserConfigurableLogicalRules() method instead of here. * - * @param context - shared state used during planning, currently used here - * to gain access to the fucntion registry described above. + * @param optimizerRulesContext - shared state used during planning, currently used here + * to gain access to the function registry described above. * @return - a RuleSet containing the logical rules that will always * be used, either by VolcanoPlanner directly, or * used VolcanoPlanner as pre-processing for LOPTPlanner. * * Note : Join permutation rule is excluded here. */ - public static RuleSet getDrillBasicRules(QueryContext context) { + public static RuleSet getDrillBasicRules(OptimizerRulesContext optimizerRulesContext) { if (DRILL_BASIC_RULES == null) { DRILL_BASIC_RULES = new DrillRuleSet(ImmutableSet. builder().add( // @@ -141,7 +141,8 @@ public static RuleSet getDrillBasicRules(QueryContext context) { ProjectRemoveRule.NAME_CALC_INSTANCE, SortRemoveRule.INSTANCE, - DrillMergeProjectRule.getInstance(true, RelFactories.DEFAULT_PROJECT_FACTORY, context.getFunctionRegistry()), + DrillMergeProjectRule.getInstance(true, RelFactories.DEFAULT_PROJECT_FACTORY, + optimizerRulesContext.getFunctionRegistry()), AggregateExpandDistinctAggregatesRule.INSTANCE, DrillReduceAggregatesRule.INSTANCE, @@ -153,10 +154,10 @@ public static RuleSet getDrillBasicRules(QueryContext context) { DrillPushProjIntoScan.INSTANCE, DrillProjectSetOpTransposeRule.INSTANCE, - PruneScanRule.getFilterOnProject(context), - PruneScanRule.getFilterOnScan(context), - PruneScanRule.getFilterOnProjectParquet(context), - PruneScanRule.getFilterOnScanParquet(context), + PruneScanRule.getFilterOnProject(optimizerRulesContext), + PruneScanRule.getFilterOnScan(optimizerRulesContext), + PruneScanRule.getFilterOnProjectParquet(optimizerRulesContext), + PruneScanRule.getFilterOnScanParquet(optimizerRulesContext), /* Convert from Calcite Logical to Drill Logical Rules. @@ -181,7 +182,7 @@ public static RuleSet getDrillBasicRules(QueryContext context) { } // Ruleset for join permutation, used only in VolcanoPlanner. - public static RuleSet getJoinPermRules(QueryContext context) { + public static RuleSet getJoinPermRules(OptimizerRulesContext optimizerRulesContext) { return new DrillRuleSet(ImmutableSet. builder().add( // JoinPushThroughJoinRule.RIGHT, JoinPushThroughJoinRule.LEFT @@ -193,10 +194,10 @@ public static RuleSet getJoinPermRules(QueryContext context) { )); - public static final RuleSet getPhysicalRules(QueryContext qcontext) { + public static final RuleSet getPhysicalRules(OptimizerRulesContext optimizerRulesContext) { List ruleList = new ArrayList(); - PlannerSettings ps = qcontext.getPlannerSettings(); + PlannerSettings ps = optimizerRulesContext.getPlannerSettings(); ruleList.add(ConvertCountToDirectScan.AGG_ON_PROJ_ON_SCAN); ruleList.add(ConvertCountToDirectScan.AGG_ON_SCAN); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java index 5b5e4bc66be..eb0889abd0b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java @@ -37,8 +37,8 @@ import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.expr.fn.interpreter.InterpreterEvaluator; import org.apache.drill.exec.memory.BufferAllocator; - import org.apache.drill.exec.ops.QueryContext; - import org.apache.drill.exec.physical.base.FileGroupScan; +import org.apache.drill.exec.ops.OptimizerRulesContext; +import org.apache.drill.exec.physical.base.FileGroupScan; import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.planner.FileSystemPartitionDescriptor; import org.apache.drill.exec.planner.ParquetPartitionDescriptor; @@ -74,11 +74,11 @@ public abstract class PruneScanRule extends RelOptRule { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PruneScanRule.class); - public static final RelOptRule getFilterOnProject(QueryContext context){ + public static final RelOptRule getFilterOnProject(OptimizerRulesContext optimizerRulesContext){ return new PruneScanRule( RelOptHelper.some(DrillFilterRel.class, RelOptHelper.some(DrillProjectRel.class, RelOptHelper.any(DrillScanRel.class))), "PruneScanRule:Filter_On_Project", - context) { + optimizerRulesContext) { @Override public boolean matches(RelOptRuleCall call) { @@ -136,10 +136,10 @@ protected List getFiles(DrillScanRel scanRel) { }; } - public static final RelOptRule getFilterOnScan(QueryContext context){ + public static final RelOptRule getFilterOnScan(OptimizerRulesContext optimizerRulesContext){ return new PruneScanRule( RelOptHelper.some(DrillFilterRel.class, RelOptHelper.any(DrillScanRel.class)), - "PruneScanRule:Filter_On_Scan", context) { + "PruneScanRule:Filter_On_Scan", optimizerRulesContext) { @Override public boolean matches(RelOptRuleCall call) { @@ -196,11 +196,11 @@ protected List getFiles(DrillScanRel scanRel) { }; } - public static final RelOptRule getFilterOnProjectParquet(QueryContext context){ + public static final RelOptRule getFilterOnProjectParquet(OptimizerRulesContext optimizerRulesContext){ return new PruneScanRule( RelOptHelper.some(DrillFilterRel.class, RelOptHelper.some(DrillProjectRel.class, RelOptHelper.any(DrillScanRel.class))), "PruneScanRule:Filter_On_Project_Parquet", - context) { + optimizerRulesContext) { @Override public boolean matches(RelOptRuleCall call) { @@ -258,10 +258,10 @@ protected List getFiles(DrillScanRel scanRel) { // Using separate rules for Parquet column based partition pruning. In the future, we may want to see if we can combine these into // a single rule which handles both types of pruning - public static final RelOptRule getFilterOnScanParquet(QueryContext context){ + public static final RelOptRule getFilterOnScanParquet(OptimizerRulesContext optimizerRulesContext){ return new PruneScanRule( RelOptHelper.some(DrillFilterRel.class, RelOptHelper.any(DrillScanRel.class)), - "PruneScanRule:Filter_On_Scan_Parquet", context) { + "PruneScanRule:Filter_On_Scan_Parquet", optimizerRulesContext) { @Override public boolean matches(RelOptRuleCall call) { @@ -315,11 +315,11 @@ protected List getFiles(DrillScanRel scanRel) { }; } - final QueryContext context; + final OptimizerRulesContext optimizerContext; - private PruneScanRule(RelOptRuleOperand operand, String id, QueryContext context) { + private PruneScanRule(RelOptRuleOperand operand, String id, OptimizerRulesContext optimizerContext) { super(operand, id); - this.context = context; + this.optimizerContext = optimizerContext; } protected abstract PartitionDescriptor getPartitionDescriptor(PlannerSettings settings, DrillScanRel scanRel); @@ -327,7 +327,7 @@ private PruneScanRule(RelOptRuleOperand operand, String id, QueryContext context protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, DrillProjectRel projectRel, DrillScanRel scanRel) { final PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner()); PartitionDescriptor descriptor = getPartitionDescriptor(settings, scanRel); - final BufferAllocator allocator = context.getAllocator(); + final BufferAllocator allocator = optimizerContext.getAllocator(); RexNode condition = null; @@ -411,13 +411,13 @@ protected void doOnMatch(RelOptRuleCall call, DrillFilterRel filterRel, DrillPro logger.debug("Attempting to prune {}", pruneCondition); LogicalExpression expr = DrillOptiq.toDrill(new DrillParseContext(settings), scanRel, pruneCondition); ErrorCollectorImpl errors = new ErrorCollectorImpl(); - LogicalExpression materializedExpr = ExpressionTreeMaterializer.materialize(expr, container, errors, context.getFunctionRegistry()); + LogicalExpression materializedExpr = ExpressionTreeMaterializer.materialize(expr, container, errors, optimizerContext.getFunctionRegistry()); if (errors.getErrorCount() != 0) { logger.warn("Failure while materializing expression [{}]. Errors: {}", expr, errors); } output.allocateNew(partitions.size()); - InterpreterEvaluator.evaluate(partitions.size(), context, container, output, materializedExpr); + InterpreterEvaluator.evaluate(partitions.size(), optimizerContext, container, output, materializedExpr); int record = 0; List newFiles = Lists.newArrayList(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index 2d1bac20bbf..bc79cffcb61 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -121,7 +121,7 @@ private RuleSet[] getRules(QueryContext context) { DrillRuleSets.getDrillUserConfigurableLogicalRules(context)); RuleSet drillPhysicalMem = DrillRuleSets.mergedRuleSets( DrillRuleSets.getPhysicalRules(context), - storagePluginRegistry.getStoragePluginRuleSet()); + storagePluginRegistry.getStoragePluginRuleSet(context)); // Following is used in LOPT join OPT. RuleSet logicalConvertRules = DrillRuleSets.mergedRuleSets( diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java index 58c86227c59..822a210e21c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java @@ -23,6 +23,7 @@ import org.apache.drill.common.JSONOptions; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.physical.base.AbstractGroupScan; import com.google.common.collect.ImmutableSet; @@ -44,7 +45,7 @@ public boolean supportsWrite() { } @Override - public Set getOptimizerRules() { + public Set getOptimizerRules(OptimizerRulesContext optimizerRulesContext) { return ImmutableSet.of(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java index b60c16fa037..eef63a267e3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java @@ -24,6 +24,7 @@ import org.apache.drill.common.JSONOptions; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.physical.base.AbstractGroupScan; public interface StoragePlugin extends SchemaFactory { @@ -31,7 +32,7 @@ public interface StoragePlugin extends SchemaFactory { public boolean supportsWrite(); - public Set getOptimizerRules(); + public Set getOptimizerRules(OptimizerRulesContext optimizerContext); /** * Get the physical scan operator for the particular GroupScan (read) node. diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java index 80a0876ad06..9ca419d3b2c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java @@ -42,6 +42,7 @@ import org.apache.drill.common.util.PathScanner; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.DrillbitStartupException; +import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.ops.ViewExpansionContext; import org.apache.drill.exec.planner.logical.DrillRuleSets; @@ -283,11 +284,11 @@ public Iterator> iterator() { return plugins.entrySet().iterator(); } - public RuleSet getStoragePluginRuleSet() { + public RuleSet getStoragePluginRuleSet(OptimizerRulesContext optimizerRulesContext) { // query registered engines for optimizer rules and build the storage plugin RuleSet Builder setBuilder = ImmutableSet.builder(); for (StoragePlugin plugin : this.plugins.values()) { - Set rules = plugin.getOptimizerRules(); + Set rules = plugin.getOptimizerRules(optimizerRulesContext); if (rules != null && rules.size() > 0) { setBuilder.addAll(rules); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java index 4ae0cc8d681..6f3ce27eff6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -29,6 +29,7 @@ import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.FormatPluginConfig; import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.server.DrillbitContext; @@ -144,7 +145,7 @@ public FormatPlugin getFormatPlugin(FormatPluginConfig config) { } @Override - public Set getOptimizerRules() { + public Set getOptimizerRules(OptimizerRulesContext optimizerRulesContext) { Builder setBuilder = ImmutableSet.builder(); for(FormatPlugin plugin : this.formatPluginsByName.values()){ Set rules = plugin.getOptimizerRules(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java index 597d24caee3..f939ba8265b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java @@ -30,6 +30,7 @@ import org.apache.drill.common.JSONOptions; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.AbstractStoragePlugin; @@ -107,7 +108,7 @@ public String getTypeName() { } @Override - public Set getOptimizerRules() { + public Set getOptimizerRules(OptimizerRulesContext optimizerRulesContext) { return ImmutableSet.of( InfoSchemaPushFilterIntoRecordGenerator.IS_FILTER_ON_PROJECT, InfoSchemaPushFilterIntoRecordGenerator.IS_FILTER_ON_SCAN);