From 0ea8e1dd87517865d07476850e0fa7aa0da3f954 Mon Sep 17 00:00:00 2001 From: Julien Le Dem Date: Thu, 15 Oct 2015 10:59:18 -0700 Subject: [PATCH] DRILL-3742: Classpath scanning and build improvement Makes the classpath scanning a build time class discovery Makes the fmpp generation incremental Removes some slowness in DrillBit closing Reduces the build time by 30% --- common/pom.xml | 19 + .../org/apache/drill/common/JSONOptions.java | 7 +- .../drill/common/config/CommonConstants.java | 16 - .../drill/common/config/DrillConfig.java | 64 +-- .../common/config/LogicalPlanPersistence.java | 65 +++ .../logical/FormatPluginConfigBase.java | 25 +- .../drill/common/logical/LogicalPlan.java | 9 +- .../logical/StoragePluginConfigBase.java | 24 +- .../logical/data/LogicalOperatorBase.java | 18 +- .../drill/common/scanner/BuildTimeScan.java | 143 ++++++ .../common/scanner/ClassPathScanner.java | 446 ++++++++++++++++++ .../drill/common/scanner/RunTimeScan.java | 78 +++ .../persistence/AnnotatedClassDescriptor.java | 93 ++++ .../persistence/AnnotationDescriptor.java | 156 ++++++ .../persistence/AttributeDescriptor.java | 55 +++ .../persistence/ChildClassDescriptor.java | 58 +++ .../scanner/persistence/FieldDescriptor.java | 112 +++++ .../persistence/ParentClassDescriptor.java | 61 +++ .../scanner/persistence/ScanResult.java | 213 +++++++++ .../scanner/persistence/TypeDescriptor.java | 115 +++++ .../apache/drill/common/util/PathScanner.java | 176 ------- common/src/main/resources/drill-module.conf | 15 +- .../drill/storage/CheckStorageConfig.java | 14 +- common/src/test/resources/drill-module.conf | 2 +- common/src/test/resources/logback.xml | 20 +- .../src/main/resources/drill-module.conf | 9 +- .../drill/hbase/TestHBaseTableProvider.java | 7 +- contrib/storage-hive/core/pom.xml | 33 +- .../exec/expr/fn/HiveFunctionRegistry.java | 9 +- .../core/src/main/resources/drill-module.conf | 14 +- .../core/src/test/resources/drill-module.conf | 2 +- .../core/src/test/resources/logback.xml | 27 +- .../src/main/resources/drill-module.conf | 2 +- .../src/main/resources/drill-module.conf | 3 + .../src/main/resources/drill-module.conf | 6 +- .../exec/store/mongo/MongoTestConstants.java | 1 + exec/java-exec/pom.xml | 154 +++--- .../org/apache/drill/exec/ExecConstants.java | 3 - .../exec/client/PrintingResultsListener.java | 2 + .../drill/exec/compile/CodeCompiler.java | 5 +- .../drill/exec/dotdrill/DotDrillFile.java | 5 +- .../drill/exec/expr/DrillFuncHolderExpr.java | 2 +- .../expr/annotations/FunctionTemplate.java | 2 +- .../exec/expr/fn/DrillAggFuncHolder.java | 60 ++- .../exec/expr/fn/DrillBooleanOPHolder.java | 15 +- .../expr/fn/DrillComplexWriterFuncHolder.java | 12 +- .../expr/fn/DrillDecimalAddFuncHolder.java | 7 +- .../expr/fn/DrillDecimalAggFuncHolder.java | 7 +- .../expr/fn/DrillDecimalCastFuncHolder.java | 13 +- .../fn/DrillDecimalDivScaleFuncHolder.java | 11 +- .../fn/DrillDecimalMaxScaleFuncHolder.java | 12 +- .../fn/DrillDecimalModScaleFuncHolder.java | 7 +- .../fn/DrillDecimalSetScaleFuncHolder.java | 11 +- .../expr/fn/DrillDecimalSumAggFuncHolder.java | 7 +- .../fn/DrillDecimalSumScaleFuncHolder.java | 12 +- .../fn/DrillDecimalZeroScaleFuncHolder.java | 7 +- .../drill/exec/expr/fn/DrillFuncHolder.java | 78 +-- .../exec/expr/fn/DrillFunctionRegistry.java | 33 +- .../exec/expr/fn/DrillSimpleFuncHolder.java | 55 +-- .../exec/expr/fn/FunctionAttributes.java | 98 ++++ .../drill/exec/expr/fn/FunctionConverter.java | 320 +++++-------- .../fn/FunctionImplementationRegistry.java | 22 +- .../exec/expr/fn/FunctionInitializer.java | 152 ++++++ .../drill/exec/ops/FragmentContext.java | 2 +- .../apache/drill/exec/ops/QueryContext.java | 5 + .../physical/base/PhysicalOperatorUtil.java | 23 +- .../impl/OperatorCreatorRegistry.java | 16 +- .../exec/planner/PhysicalPlanReader.java | 24 +- .../exec/planner/logical/StoragePlugins.java | 13 +- .../sql/handlers/DefaultSqlHandler.java | 8 +- .../planner/sql/handlers/ExplainHandler.java | 4 +- .../apache/drill/exec/rpc/BasicServer.java | 16 +- .../exec/rpc/data/DataConnectionCreator.java | 1 + .../drill/exec/rpc/user/UserServer.java | 6 +- .../security/UserAuthenticatorFactory.java | 17 +- .../drill/exec/server/BootStrapContext.java | 17 +- .../apache/drill/exec/server/Drillbit.java | 22 +- .../drill/exec/server/DrillbitContext.java | 50 +- .../server/options/SystemOptionManager.java | 6 +- .../exec/server/rest/DrillRestServer.java | 4 +- .../drill/exec/service/ServiceEngine.java | 38 +- .../exec/store/StoragePluginRegistry.java | 31 +- .../exec/store/dfs/FileSystemPlugin.java | 21 +- .../drill/exec/store/dfs/FormatCreator.java | 19 +- .../store/dfs/WorkspaceSchemaFactory.java | 31 +- .../ischema/InfoSchemaStoragePlugin.java | 2 +- .../exec/store/sys/SystemTablePlugin.java | 2 +- .../apache/drill/exec/work/WorkManager.java | 9 +- .../drill/exec/work/foreman/Foreman.java | 8 +- .../src/main/resources/drill-module.conf | 31 +- .../java/org/apache/drill/BaseTestQuery.java | 7 +- .../java/org/apache/drill/PlanningBase.java | 15 +- .../common/scanner/TestClassPathScanner.java | 175 +++++++ .../org/apache/drill/exec/RunRootExec.java | 3 +- .../drill/exec/TestOpSerialization.java | 10 +- .../drill/exec/cache/TestWriteToDisk.java | 1 + .../apache/drill/exec/client/DumpCatTest.java | 19 +- .../exec/compile/CodeCompilerTestFactory.java | 38 ++ .../compile/bytecode/ReplaceMethodInvoke.java | 4 +- .../drill/exec/fn/impl/TestMathFunctions.java | 16 +- .../exec/fn/impl/TestNewMathFunctions.java | 17 +- .../exec/fn/impl/TestRepeatedFunction.java | 16 +- .../{ => testing}/GeneratorFunctions.java | 10 +- .../drill/exec/memory/TestAllocators.java | 8 +- .../drill/exec/opt/BasicOptimizerTest.java | 8 +- .../config/TestParsePhysicalPlan.java | 16 +- .../exec/physical/impl/TestCastFunctions.java | 59 +-- .../impl/TestComparisonFunctions.java | 16 +- .../impl/TestImplicitCastFunctions.java | 16 +- .../exec/physical/impl/TestOptiqPlans.java | 4 +- .../physical/impl/TestSimpleFunctions.java | 29 +- .../physical/impl/TestStringFunctions.java | 16 +- .../drill/exec/physical/impl/agg/TestAgg.java | 16 +- .../physical/impl/common/TestHashTable.java | 23 +- .../impl/filter/TestSimpleFilter.java | 22 +- .../exec/physical/impl/join/TestHashJoin.java | 22 +- .../physical/impl/join/TestMergeJoin.java | 41 +- .../physical/impl/limit/TestSimpleLimit.java | 26 +- .../TestOrderedPartitionExchange.java | 4 +- .../impl/project/TestSimpleProjection.java | 16 +- .../physical/impl/sort/TestSimpleSort.java | 22 +- .../impl/trace/TestTraceMultiRecordBatch.java | 16 +- .../impl/trace/TestTraceOutputDump.java | 16 +- .../physical/impl/union/TestSimpleUnion.java | 16 +- .../PhysicalPlanReaderTestFactory.java | 63 +++ .../drill/exec/pop/TestFragmentChecker.java | 3 +- .../apache/drill/exec/pop/TestFragmenter.java | 6 +- .../drill/exec/pop/TestInjectionValue.java | 3 +- .../security/TestCustomUserAuthenticator.java | 9 +- .../UserAuthenticatorTestImpl.java | 5 +- .../apache/drill/exec/server/TestBitRpc.java | 20 +- .../exec/server/TestOptionsAuthEnabled.java | 22 +- .../test/resources/drill-external-sort.conf | 9 +- .../src/test/resources/drill-module.conf | 15 +- .../src/test/resources/drill-oom-xsort.conf | 11 +- .../resources/drill-spool-test-module.conf | 17 +- exec/java-exec/src/test/resources/logback.xml | 32 +- exec/jdbc-all/example-conf/drill-module.conf | 17 +- exec/jdbc-all/pom.xml | 11 - .../apache/drill/jdbc/test/JdbcAssert.java | 5 +- pom.xml | 10 +- src/main/resources/checkstyle-config.xml | 1 + tools/fmpp/pom.xml | 82 ++++ .../org/apache/drill/fmpp/mojo/FMPPMojo.java | 227 +++++++++ tools/pom.xml | 34 ++ 145 files changed, 3643 insertions(+), 1357 deletions(-) create mode 100644 common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotatedClassDescriptor.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotationDescriptor.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/persistence/AttributeDescriptor.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/persistence/ChildClassDescriptor.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/persistence/FieldDescriptor.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/persistence/ParentClassDescriptor.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/persistence/ScanResult.java create mode 100644 common/src/main/java/org/apache/drill/common/scanner/persistence/TypeDescriptor.java delete mode 100644 common/src/main/java/org/apache/drill/common/util/PathScanner.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionAttributes.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java create mode 100644 exec/java-exec/src/test/java/org/apache/drill/common/scanner/TestClassPathScanner.java create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java rename exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/{ => testing}/GeneratorFunctions.java (85%) create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/planner/PhysicalPlanReaderTestFactory.java rename exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/{ => testing}/UserAuthenticatorTestImpl.java (91%) create mode 100644 tools/fmpp/pom.xml create mode 100644 tools/fmpp/src/main/java/org/apache/drill/fmpp/mojo/FMPPMojo.java create mode 100644 tools/pom.xml diff --git a/common/pom.xml b/common/pom.xml index cdeb453eb94..5b5697822e8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -110,6 +110,25 @@ + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + process-classes + + java + + + + + org.apache.drill.common.scanner.BuildTimeScan + + ${project.build.outputDirectory} + + + maven-surefire-plugin 2.15 diff --git a/common/src/main/java/org/apache/drill/common/JSONOptions.java b/common/src/main/java/org/apache/drill/common/JSONOptions.java index 9e0514c4995..e4321354db8 100644 --- a/common/src/main/java/org/apache/drill/common/JSONOptions.java +++ b/common/src/main/java/org/apache/drill/common/JSONOptions.java @@ -24,6 +24,7 @@ import org.apache.drill.common.JSONOptions.De; import org.apache.drill.common.JSONOptions.Se; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.exceptions.LogicalPlanParsingException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,7 +67,7 @@ public JSONOptions(JsonNode n, JsonLocation location) { } @SuppressWarnings("unchecked") - public T getWith(DrillConfig config, Class c) { + public T getWith(LogicalPlanPersistence lpPersistance, Class c) { try { if (opaque != null) { final Class opaqueClass = opaque.getClass(); @@ -88,7 +89,7 @@ public T getWith(DrillConfig config, Class c) { } //logger.debug("Read tree {}", root); - return config.getMapper().treeToValue(root, c); + return lpPersistance.getMapper().treeToValue(root, c); } catch (JsonProcessingException e) { throw new LogicalPlanParsingException(String.format("Failure while trying to convert late bound " + "json options to type of %s. Reference was originally located at line %d, column %d.", @@ -96,7 +97,7 @@ public T getWith(DrillConfig config, Class c) { } } - public T getListWith(DrillConfig config, TypeReference t) throws IOException { + public T getListWith(LogicalPlanPersistence config, TypeReference t) throws IOException { return getListWith(config.getMapper(), t); } diff --git a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java b/common/src/main/java/org/apache/drill/common/config/CommonConstants.java index 78c435691ca..d3d2b723a35 100644 --- a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java +++ b/common/src/main/java/org/apache/drill/common/config/CommonConstants.java @@ -28,20 +28,4 @@ public interface CommonConstants { /** Override configuration file name. (Classpath resource pathname.) */ String CONFIG_OVERRIDE_RESOURCE_PATHNAME = "drill-override.conf"; - /** Configuration pathname to list of names of packages to scan for logical - * operator subclasses. */ - String LOGICAL_OPERATOR_SCAN_PACKAGES = "drill.logical.operator.packages"; - - /** Configuration pathname to list of names of packages to scan for physical - * operator subclasses. */ - String PHYSICAL_OPERATOR_SCAN_PACKAGES = "drill.physical.operator.packages"; - - /** Configuration pathname to list of names of packages to scan for function - * subclasses. */ - String LOGICAL_FUNCTION_SCAN_PACKAGES = "drill.logical.function.packages"; - - /** Configuration pathname to list of packages to scan for storage plugin - * configuration subclasses. */ - String STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES = "drill.logical.storage.packages"; - } diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java index d747b67717c..507c4f0246d 100644 --- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java +++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java @@ -22,24 +22,16 @@ import java.net.URL; import java.util.Collection; import java.util.List; +import java.util.Map.Entry; import java.util.Properties; +import java.util.concurrent.TimeUnit; import org.apache.drill.common.exceptions.DrillConfigurationException; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.FormatPluginConfigBase; -import org.apache.drill.common.logical.StoragePluginConfigBase; -import org.apache.drill.common.logical.data.LogicalOperatorBase; -import org.apache.drill.common.util.PathScanner; +import org.apache.drill.common.scanner.ClassPathScanner; import org.reflections.util.ClasspathHelper; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser.Feature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; @@ -48,7 +40,6 @@ public final class DrillConfig extends NestedConfig{ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConfig.class); - private final ObjectMapper mapper; private final ImmutableList startupArguments; public static final boolean ON_OSX = System.getProperty("os.name").contains("OS X"); @@ -62,27 +53,10 @@ public DrillConfig(Config config, boolean enableServerConfigs) { logger.debug("Setting up DrillConfig object."); logger.trace("Given Config object is:\n{}", config.root().render(ConfigRenderOptions.defaults())); - mapper = new ObjectMapper(); - - if (enableServerConfigs) { - SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule") - .addDeserializer(LogicalExpression.class, new LogicalExpression.De(this)) - .addDeserializer(SchemaPath.class, new SchemaPath.De()); - - mapper.registerModule(deserModule); - mapper.enable(SerializationFeature.INDENT_OUTPUT); - mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); - mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true); - mapper.configure(Feature.ALLOW_COMMENTS, true); - mapper.registerSubtypes(LogicalOperatorBase.getSubTypes(this)); - mapper.registerSubtypes(StoragePluginConfigBase.getSubTypes(this)); - mapper.registerSubtypes(FormatPluginConfigBase.getSubTypes(this)); - } - RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean(); this.startupArguments = ImmutableList.copyOf(bean.getInputArguments()); logger.debug("DrillConfig object initialized."); - }; + } public List getStartupArguments() { return startupArguments; @@ -168,6 +142,8 @@ public static DrillConfig create(String overrideFileResourcePathname, boolean en private static DrillConfig create(String overrideFileResourcePathname, final Properties overriderProps, final boolean enableServerConfigs) { + final StringBuilder logString = new StringBuilder(); + final Stopwatch watch = new Stopwatch().start(); overrideFileResourcePathname = overrideFileResourcePathname == null ? CommonConstants.CONFIG_OVERRIDE_RESOURCE_PATHNAME @@ -180,7 +156,7 @@ private static DrillConfig create(String overrideFileResourcePathname, final URL url = classLoader.getResource(CommonConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME); if (null != url) { - logger.info("Loading base configuration file at {}.", url); + logString.append("Base Configuration:\n\t- ").append(url).append("\n"); fallback = ConfigFactory.load(classLoader, CommonConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME); @@ -189,14 +165,13 @@ private static DrillConfig create(String overrideFileResourcePathname, } // 2. Load per-module configuration files. - final Collection urls = PathScanner.getConfigURLs(); - final String lineBrokenList = - urls.size() == 0 ? "" : "\n\t- " + Joiner.on("\n\t- ").join(urls); - logger.info("Loading {} module configuration files at: {}.", - urls.size(), lineBrokenList); + final Collection urls = ClassPathScanner.getConfigURLs(); + logString.append("\nIntermediate Configuration and Plugin files, in order of precedence:\n"); for (URL url : urls) { + logString.append("\t- ").append(url).append("\n"); fallback = ConfigFactory.parseURL(url).withFallback(fallback); } + logString.append("\n"); // 3. Load any specified overrides configuration file along with any // overrides from JVM system properties (e.g., {-Dname=value"). @@ -205,19 +180,26 @@ private static DrillConfig create(String overrideFileResourcePathname, final URL overrideFileUrl = Thread.currentThread().getContextClassLoader().getResource(overrideFileResourcePathname); if (null != overrideFileUrl ) { - logger.info("Loading override config. file at {}.", overrideFileUrl); + logString.append("Override File: ").append(overrideFileUrl).append("\n"); } Config effectiveConfig = ConfigFactory.load(overrideFileResourcePathname).withFallback(fallback); // 4. Apply any overriding properties. if (overriderProps != null) { - logger.info("Loading override Properties parameter {}.", overriderProps); + logString.append("Overridden Properties:\n"); + for(Entry entry : overriderProps.entrySet()){ + logString.append("\t-").append(entry.getKey()).append(" = ").append(entry.getValue()).append("\n"); + } + logString.append("\n"); effectiveConfig = ConfigFactory.parseProperties(overriderProps).withFallback(effectiveConfig); } // 5. Create DrillConfig object from Config object. + logger.info("Configuration and plugin file(s) identified in {}ms.\n{}", + watch.elapsed(TimeUnit.MILLISECONDS), + logString); return new DrillConfig(effectiveConfig.resolve(), enableServerConfigs); } @@ -256,10 +238,6 @@ public T getInstanceOf(String location, Class clazz) throws DrillConfigur } } - public ObjectMapper getMapper() { - return mapper; - } - @Override public String toString() { return this.root().render(); diff --git a/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java b/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java new file mode 100644 index 00000000000..cd7a8d0ced3 --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.config; + +import java.util.Set; + +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.logical.FormatPluginConfigBase; +import org.apache.drill.common.logical.StoragePluginConfigBase; +import org.apache.drill.common.logical.data.LogicalOperatorBase; +import org.apache.drill.common.scanner.persistence.ScanResult; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser.Feature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.module.SimpleModule; + + +public class LogicalPlanPersistence { + private ObjectMapper mapper; + + public ObjectMapper getMapper() { + return mapper; + } + + public LogicalPlanPersistence(DrillConfig conf, ScanResult scanResult) { + mapper = new ObjectMapper(); + + SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule") + .addDeserializer(LogicalExpression.class, new LogicalExpression.De(conf)) + .addDeserializer(SchemaPath.class, new SchemaPath.De()); + + mapper.registerModule(deserModule); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true); + mapper.configure(Feature.ALLOW_COMMENTS, true); + registerSubtypes(LogicalOperatorBase.getSubTypes(scanResult)); + registerSubtypes(StoragePluginConfigBase.getSubTypes(scanResult)); + registerSubtypes(FormatPluginConfigBase.getSubTypes(scanResult)); + } + + private void registerSubtypes(Set> types) { + for (Class type : types) { + mapper.registerSubtypes(type); + } + } +} diff --git a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java index 02f3206c03c..50de963f3ed 100644 --- a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java +++ b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java @@ -17,13 +17,9 @@ */ package org.apache.drill.common.logical; -import java.util.List; +import java.util.Set; -import org.apache.drill.common.config.CommonConstants; -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.util.PathScanner; - -import com.google.common.base.Joiner; +import org.apache.drill.common.scanner.persistence.ScanResult; public abstract class FormatPluginConfigBase implements FormatPluginConfig{ @@ -31,21 +27,14 @@ public abstract class FormatPluginConfigBase implements FormatPluginConfig{ /** - * Use reflection to scan for implementations of {@see FormatPlugin}. + * scan for implementations of {@see FormatPlugin}. * - * @param config - Drill configuration object, used to find the packages to scan + * @param classpathScan - Drill configuration object, used to find the packages to scan * @return - list of classes that implement the interface. */ - public synchronized static Class[] getSubTypes(final DrillConfig config) { - final List packages = - config.getStringList(CommonConstants.STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES); - final Class[] pluginClasses = - PathScanner.scanForImplementationsArr(FormatPluginConfig.class, packages); - final String lineBrokenList = - pluginClasses.length == 0 - ? "" : "\n\t- " + Joiner.on("\n\t- ").join(pluginClasses); - logger.debug("Found {} format plugin configuration classes: {}.", - pluginClasses.length, lineBrokenList); + public static Set> getSubTypes(final ScanResult classpathScan) { + final Set> pluginClasses = classpathScan.getImplementations(FormatPluginConfig.class); + logger.debug("Found {} format plugin configuration classes: {}.", pluginClasses.size(), pluginClasses); return pluginClasses; } diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java index 313ed401fa3..8a4a5ab1978 100644 --- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java +++ b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.graph.Graph; import org.apache.drill.common.graph.GraphAlgos; import org.apache.drill.common.logical.data.LogicalOperator; @@ -80,11 +81,11 @@ public Map getStorageEngines() { return storageEngineMap; } - public String toJsonString(DrillConfig config) throws JsonProcessingException { + public String toJsonString(LogicalPlanPersistence config) throws JsonProcessingException { return config.getMapper().writeValueAsString(this); } - public String toJsonStringSafe(DrillConfig config){ + public String toJsonStringSafe(LogicalPlanPersistence config){ try{ return toJsonString(config); }catch(JsonProcessingException e){ @@ -94,7 +95,7 @@ public String toJsonStringSafe(DrillConfig config){ } /** Parses a logical plan. */ - public static LogicalPlan parse(DrillConfig config, String planString) { + public static LogicalPlan parse(LogicalPlanPersistence config, String planString) { ObjectMapper mapper = config.getMapper(); try { LogicalPlan plan = mapper.readValue(planString, LogicalPlan.class); @@ -106,7 +107,7 @@ public static LogicalPlan parse(DrillConfig config, String planString) { } /** Converts a logical plan to a string. (Opposite of {@link #parse}.) */ - public String unparse(DrillConfig config) { + public String unparse(LogicalPlanPersistence config) { try { return config.getMapper().writeValueAsString(this); } catch (JsonProcessingException e) { diff --git a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java index 3ac858bace2..60754835819 100644 --- a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java +++ b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java @@ -17,30 +17,18 @@ */ package org.apache.drill.common.logical; -import java.util.List; +import java.util.Set; -import org.apache.drill.common.config.CommonConstants; -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.util.PathScanner; - -import com.google.common.base.Joiner; +import org.apache.drill.common.scanner.persistence.ScanResult; public abstract class StoragePluginConfigBase extends StoragePluginConfig { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginConfigBase.class); - - public synchronized static Class[] getSubTypes(final DrillConfig config) { - final List packages = - config.getStringList(CommonConstants.STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES); - final Class[] pluginClasses = - PathScanner.scanForImplementationsArr(StoragePluginConfig.class, packages); - final String lineBrokenList = - pluginClasses.length == 0 - ? "" : "\n\t- " + Joiner.on("\n\t- ").join(pluginClasses); - logger.debug("Found {} storage plugin configuration classes: {}.", - pluginClasses.length, lineBrokenList); - return pluginClasses; + public static Set> getSubTypes(final ScanResult classpathScan) { + final Set> packages = classpathScan.getImplementations(StoragePluginConfig.class); + logger.debug("Found {} logical operator classes: {}.", packages.size(), packages); + return packages; } @Override diff --git a/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java b/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java index e97ef1980af..a213c8d3aa7 100644 --- a/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java +++ b/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java @@ -20,17 +20,15 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Set; -import org.apache.drill.common.config.CommonConstants; -import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.graph.GraphVisitor; import org.apache.drill.common.logical.ValidationError; -import org.apache.drill.common.util.PathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Joiner; public abstract class LogicalOperatorBase implements LogicalOperator{ @@ -86,15 +84,9 @@ public void setMemo(String memo) { this.memo = memo; } - public synchronized static Class[] getSubTypes(final DrillConfig config) { - final List packages = - config.getStringList(CommonConstants.LOGICAL_OPERATOR_SCAN_PACKAGES); - final Class[] ops = - PathScanner.scanForImplementationsArr(LogicalOperator.class, packages); - final String lineBrokenList = - ops.length == 0 ? "" : "\n\t- " + Joiner.on("\n\t- ").join(ops); - logger.debug("Found {} logical operator classes: {}.", ops.length, - lineBrokenList); + public static Set> getSubTypes(final ScanResult classpathScan) { + final Set> ops = classpathScan.getImplementations(LogicalOperator.class); + logger.debug("Found {} logical operator classes: {}.", ops.size(), ops); return ops; } } diff --git a/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java new file mode 100644 index 00000000000..ca4a37d8fbd --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner; + +import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT; +import static java.lang.String.format; +import static java.util.Arrays.asList; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.List; +import java.util.Set; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.scanner.persistence.ScanResult; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.ObjectWriter; + +/** + * main class to integrate classpath scanning in the build. + * @see BuildTimeScan#main(String[]) + */ +public class BuildTimeScan { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BuildTimeScan.class); + private static final String REGISTRY_FILE = "META-INF/drill-module-scan/registry.json"; + + private static final ObjectMapper mapper = new ObjectMapper().enable(INDENT_OUTPUT); + private static final ObjectReader reader = mapper.reader(ScanResult.class); + private static final ObjectWriter writer = mapper.writerWithType(ScanResult.class); + + /** + * @return paths that have the prescanned registry file in them + */ + static Set getPrescannedPaths() { + return ClassPathScanner.forResource(REGISTRY_FILE, true); + } + + /** + * loads all the prescanned resources from classpath + * @return the result of the previous scan + */ + static ScanResult load() { + return loadExcept(null); + } + + /** + * loads all the prescanned resources from classpath + * (except for the target location in case it already exists) + * @return the result of the previous scan + */ + private static ScanResult loadExcept(URL ignored) { + Set preScanned = ClassPathScanner.forResource(REGISTRY_FILE, false); + ScanResult result = null; + for (URL u : preScanned) { + if (ignored!= null && u.toString().startsWith(ignored.toString())) { + continue; + } + try (InputStream reflections = u.openStream()) { + ScanResult ref = reader.readValue(reflections); + if (result == null) { + result = ref; + } else { + result = result.merge(ref); + } + } catch (IOException e) { + throw new DrillRuntimeException("can't read function registry at " + u, e); + } + } + if (result != null) { + logger.info(format("Loaded prescanned packages %s from locations %s", result.getScannedPackages(), preScanned)); + return result; + } else { + return ClassPathScanner.emptyResult(); + } + } + + private static void save(ScanResult scanResult, File file) { + try { + writer.writeValue(file, scanResult); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * to generate the prescan file during build + * @param args the root path for the classes where {@link BuildTimeScan#REGISTRY_FILE} is generated + * @throws Exception + */ + public static void main(String[] args) throws Exception { + if (args.length != 1) { + throw new IllegalArgumentException("Usage: java {cp} " + ClassPathScanner.class.getName() + " path/to/scan"); + } + String basePath = args[0]; + System.out.println("Scanning: " + basePath); + File registryFile = new File(basePath, REGISTRY_FILE); + File dir = registryFile.getParentFile(); + if ((!dir.exists() && !dir.mkdirs()) || !dir.isDirectory()) { + throw new IllegalArgumentException("could not create dir " + dir.getAbsolutePath()); + } + DrillConfig config = DrillConfig.create(); + // normalize + if (!basePath.endsWith("/")) { + basePath = basePath + "/"; + } + URL url = new URL("file:" + basePath); + Set markedPaths = ClassPathScanner.getMarkedPaths(); + if (!markedPaths.contains(url)) { + throw new IllegalArgumentException(url + " not in " + markedPaths); + } + List packagePrefixes = ClassPathScanner.getPackagePrefixes(config); + List baseClasses = ClassPathScanner.getScannedBaseClasses(config); + List scannedAnnotations = ClassPathScanner.getScannedAnnotations(config); + ScanResult preScanned = loadExcept(url); + ScanResult scan = ClassPathScanner.scan( + asList(url), + packagePrefixes, + baseClasses, + scannedAnnotations, + preScanned); + save(scan, registryFile); + } +} diff --git a/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java b/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java new file mode 100644 index 00000000000..92b202734a8 --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java @@ -0,0 +1,446 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner; + +import static java.lang.String.format; +import static java.util.Collections.unmodifiableList; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.drill.common.config.CommonConstants; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.persistence.AnnotationDescriptor; +import org.apache.drill.common.scanner.persistence.AttributeDescriptor; +import org.apache.drill.common.scanner.persistence.ChildClassDescriptor; +import org.apache.drill.common.scanner.persistence.FieldDescriptor; +import org.apache.drill.common.scanner.persistence.AnnotatedClassDescriptor; +import org.apache.drill.common.scanner.persistence.ParentClassDescriptor; +import org.apache.drill.common.scanner.persistence.ScanResult; +import org.reflections.Reflections; +import org.reflections.adapters.JavassistAdapter; +import org.reflections.scanners.AbstractScanner; +import org.reflections.util.ConfigurationBuilder; +import org.reflections.util.FilterBuilder; + +import com.google.common.base.Stopwatch; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; + +import javassist.bytecode.AccessFlag; +import javassist.bytecode.AnnotationsAttribute; +import javassist.bytecode.ClassFile; +import javassist.bytecode.FieldInfo; +import javassist.bytecode.annotation.AnnotationMemberValue; +import javassist.bytecode.annotation.ArrayMemberValue; +import javassist.bytecode.annotation.BooleanMemberValue; +import javassist.bytecode.annotation.ByteMemberValue; +import javassist.bytecode.annotation.CharMemberValue; +import javassist.bytecode.annotation.ClassMemberValue; +import javassist.bytecode.annotation.DoubleMemberValue; +import javassist.bytecode.annotation.EnumMemberValue; +import javassist.bytecode.annotation.FloatMemberValue; +import javassist.bytecode.annotation.IntegerMemberValue; +import javassist.bytecode.annotation.LongMemberValue; +import javassist.bytecode.annotation.MemberValue; +import javassist.bytecode.annotation.MemberValueVisitor; +import javassist.bytecode.annotation.ShortMemberValue; +import javassist.bytecode.annotation.StringMemberValue; + +/** + * Classpath scanning utility. + * The classpath should be scanned once at startup from a DrillConfig instance. {@see ClassPathScanner#fromPrescan(DrillConfig)} + * The DrillConfig provides: + * - the list of packages to scan. (drill.classpath.scanning.packages) {@see CommonConstants#IMPLEMENTATIONS_SCAN_PACKAGES} + * - the list of base classes to scan for implementations. (drill.classpath.scanning.base.classes) {@see CommonConstants#IMPLEMENTATIONS_SCAN_CLASSES} + * - the list of annotations to scan for. (drill.classpath.scanning.annotations) {@see CommonConstants#IMPLEMENTATIONS_SCAN_ANNOTATIONS} + * Only the class directories and jars containing a drill-module.conf will be scanned. + * Drill core packages are scanned at build time and the result is saved in a JSON file. {@see ClassPathScanner#FUNCTION_REGISTRY_FILE} + * At runtime only the locations that have not been scanned yet will be scanned. + */ +public final class ClassPathScanner { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClassPathScanner.class); + private static final JavassistAdapter METADATA_ADAPTER = new JavassistAdapter(); + + /** Configuration pathname to list of names of packages to scan for implementations. */ + private static final String IMPLEMENTATIONS_SCAN_PACKAGES = "drill.classpath.scanning.packages"; + + /** Configuration pathname to list of names of base classes to scan for implementations. */ + private static final String IMPLEMENTATIONS_SCAN_CLASSES = "drill.classpath.scanning.base.classes"; + + /** Configuration pathname to list of names of annotations to scan for. */ + private static final String IMPLEMENTATIONS_SCAN_ANNOTATIONS = "drill.classpath.scanning.annotations"; + + /** Configuration pathname to turn off build time caching. */ + private static final String IMPLEMENTATIONS_SCAN_CACHE = "drill.classpath.scanning.cache.enabled"; + + /** + * scans the inheritance tree + */ + private static class SubTypesScanner extends AbstractScanner { + + private Multimap parentsChildren = HashMultimap.create(); + private Multimap children = HashMultimap.create(); + + public SubTypesScanner(List parentImplementations) { + for (ParentClassDescriptor parentClassDescriptor : parentImplementations) { + parentsChildren.putAll(parentClassDescriptor.getName(), parentClassDescriptor.getChildren()); + } + } + + @Override + public void scan(final Object cls) { + final ClassFile classFile = (ClassFile)cls; + String className = classFile.getName(); + String superclass = classFile.getSuperclass(); + boolean isAbstract = (classFile.getAccessFlags() & (AccessFlag.INTERFACE | AccessFlag.ABSTRACT)) != 0; + ChildClassDescriptor scannedClass = new ChildClassDescriptor(className, isAbstract); + if (!superclass.equals(Object.class.getName())) { + children.put(superclass, scannedClass); + } + for (String anInterface : classFile.getInterfaces()) { + children.put(anInterface, scannedClass); + } + } + + /** + * @param name the class name to get all the children of + * @return the class names of all children direct or indirect + */ + public Set getChildrenOf(String name) { + Set result = new HashSet<>(); + Collection scannedChildren = children.get(name); + // add all scanned children + for (ChildClassDescriptor child : scannedChildren) { + result.add(child); + } + // recursively add children's children + Collection allChildren = new ArrayList<>(); + allChildren.addAll(scannedChildren); + allChildren.addAll(parentsChildren.get(name)); + for (ChildClassDescriptor child : allChildren) { + result.addAll(getChildrenOf(child.getName())); + } + return result; + } + + } + + /** + * converts the annotation attribute value into a list of string to simplify + */ + private static class ListingMemberValueVisitor implements MemberValueVisitor { + private final List values; + + private ListingMemberValueVisitor(List values) { + this.values = values; + } + + @Override + public void visitStringMemberValue(StringMemberValue node) { + values.add(node.getValue()); + } + + @Override + public void visitShortMemberValue(ShortMemberValue node) { + values.add(String.valueOf(node.getValue())); + } + + @Override + public void visitLongMemberValue(LongMemberValue node) { + values.add(String.valueOf(node.getValue())); + } + + @Override + public void visitIntegerMemberValue(IntegerMemberValue node) { + values.add(String.valueOf(node.getValue())); + } + + @Override + public void visitFloatMemberValue(FloatMemberValue node) { + values.add(String.valueOf(node.getValue())); + } + + @Override + public void visitEnumMemberValue(EnumMemberValue node) { + values.add(node.getValue()); + } + + @Override + public void visitDoubleMemberValue(DoubleMemberValue node) { + values.add(String.valueOf(node.getValue())); + } + + @Override + public void visitClassMemberValue(ClassMemberValue node) { + values.add(node.getValue()); + } + + @Override + public void visitCharMemberValue(CharMemberValue node) { + values.add(String.valueOf(node.getValue())); + } + + @Override + public void visitByteMemberValue(ByteMemberValue node) { + values.add(String.valueOf(node.getValue())); + } + + @Override + public void visitBooleanMemberValue(BooleanMemberValue node) { + values.add(String.valueOf(node.getValue())); + } + + @Override + public void visitArrayMemberValue(ArrayMemberValue node) { + MemberValue[] nestedValues = node.getValue(); + for (MemberValue v : nestedValues) { + v.accept(new ListingMemberValueVisitor(values) { + @Override + public void visitArrayMemberValue(ArrayMemberValue node) { + values.add(Arrays.toString(node.getValue())); + } + }); + } + } + + @Override + public void visitAnnotationMemberValue(AnnotationMemberValue node) { + values.add(String.valueOf(node.getValue())); + } + } + + /** + * scans functions annotated with configured annotations + * and keeps track of its annotations and fields + */ + private static final class AnnotationScanner extends AbstractScanner { + + private final List functions = new ArrayList<>(); + + private final Set annotationsToScan; + + AnnotationScanner(Collection annotationsToScan) { + super(); + this.annotationsToScan = Collections.unmodifiableSet(new HashSet<>(annotationsToScan)); + } + + public List getAnnotatedClasses() { + return unmodifiableList(functions); + } + + @Override + public void scan(final Object cls) { + final ClassFile classFile = (ClassFile)cls; + AnnotationsAttribute annotations = ((AnnotationsAttribute)classFile.getAttribute(AnnotationsAttribute.visibleTag)); + if (annotations != null) { + boolean isAnnotated = false; + for (javassist.bytecode.annotation.Annotation a : annotations.getAnnotations()) { + if (annotationsToScan.contains(a.getTypeName())) { + isAnnotated = true; + } + } + if (isAnnotated) { + List classAnnotations = getAnnotationDescriptors(annotations); + @SuppressWarnings("unchecked") + List classFields = classFile.getFields(); + List fieldDescriptors = new ArrayList<>(classFields.size()); + for (FieldInfo field : classFields) { + String fieldName = field.getName(); + AnnotationsAttribute fieldAnnotations = ((AnnotationsAttribute)field.getAttribute(AnnotationsAttribute.visibleTag)); + fieldDescriptors.add(new FieldDescriptor(fieldName, field.getDescriptor(), getAnnotationDescriptors(fieldAnnotations))); + } + functions.add(new AnnotatedClassDescriptor(classFile.getName(), classAnnotations, fieldDescriptors)); + } + } + } + + private List getAnnotationDescriptors(AnnotationsAttribute annotationsAttr) { + List annotationDescriptors = new ArrayList<>(annotationsAttr.numAnnotations()); + for (javassist.bytecode.annotation.Annotation annotation : annotationsAttr.getAnnotations()) { + // Sigh: javassist uses raw collections (is this 2002?) + @SuppressWarnings("unchecked") + Set memberNames = annotation.getMemberNames(); + List attributes = new ArrayList<>(); + if (memberNames != null) { + for (String name : memberNames) { + MemberValue memberValue = annotation.getMemberValue(name); + final List values = new ArrayList<>(); + memberValue.accept(new ListingMemberValueVisitor(values)); + attributes.add(new AttributeDescriptor(name, values)); + } + } + annotationDescriptors.add(new AnnotationDescriptor(annotation.getTypeName(), attributes)); + } + return annotationDescriptors; + } + } + + /** + * @return paths that have a drill config file in them + */ + static Set getMarkedPaths() { + return forResource(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, true); + } + + public static Collection getConfigURLs() { + return forResource(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, false); + } + + /** + * Gets URLs of any classpath resources with given resource pathname. + * + * @param resourcePathname resource pathname of classpath resource instances + * to scan for (relative to specified class loaders' classpath roots) + * @param returnRootPathname whether to collect classpath root portion of + * URL for each resource instead of full URL of each resource + * @param classLoaders set of class loaders in which to look up resource; + * none (empty array) to specify to use current thread's context + * class loader and {@link Reflections}'s class loader + * @returns ...; empty set if none + */ + public static Set forResource(final String resourcePathname, final boolean returnRootPathname) { + logger.debug("Scanning classpath for resources with pathname \"{}\".", + resourcePathname); + final Set resultUrlSet = Sets.newHashSet(); + final ClassLoader classLoader = ClassPathScanner.class.getClassLoader(); + try { + final Enumeration resourceUrls = classLoader.getResources(resourcePathname); + while (resourceUrls.hasMoreElements()) { + final URL resourceUrl = resourceUrls.nextElement(); + logger.trace("- found a(n) {} at {}.", resourcePathname, resourceUrl); + int index = resourceUrl.toExternalForm().lastIndexOf(resourcePathname); + if (index != -1 && returnRootPathname) { + final URL classpathRootUrl = new URL(resourceUrl.toExternalForm().substring(0, index)); + resultUrlSet.add(classpathRootUrl); + logger.debug("- collected resource's classpath root URL {}.", classpathRootUrl); + } else { + resultUrlSet.add(resourceUrl); + logger.debug("- collected resource URL {}.", resourceUrl); + } + } + } catch (IOException e) { + throw new RuntimeException("Error scanning for resources named " + resourcePathname, e); + } + return resultUrlSet; + } + + static List getPackagePrefixes(DrillConfig config) { + return config.getStringList(IMPLEMENTATIONS_SCAN_PACKAGES); + } + + static List getScannedBaseClasses(DrillConfig config) { + return config.getStringList(IMPLEMENTATIONS_SCAN_CLASSES); + } + + static List getScannedAnnotations(DrillConfig config) { + if (config.hasPath(IMPLEMENTATIONS_SCAN_ANNOTATIONS)) { + return config.getStringList(IMPLEMENTATIONS_SCAN_ANNOTATIONS); + } else { + return Collections.emptyList(); + } + } + + static boolean isScanBuildTimeCacheEnabled(DrillConfig config) { + if (config.hasPath(IMPLEMENTATIONS_SCAN_CACHE)) { + return config.getBoolean(IMPLEMENTATIONS_SCAN_CACHE); + } else { + return true; // on by default + } + } + + /** + * + * @param pathsToScan the locations to scan for .class files + * @param packagePrefixes the whitelist of package prefixes to scan + * @param parentResult if there was a prescan, its result + * @return the merged scan + */ + static ScanResult scan(Collection pathsToScan, Collection packagePrefixes, Collection scannedClasses, Collection scannedAnnotations, ScanResult parentResult) { + Stopwatch watch = new Stopwatch().start(); + try { + AnnotationScanner annotationScanner = new AnnotationScanner(scannedAnnotations); + SubTypesScanner subTypesScanner = new SubTypesScanner(parentResult.getImplementations()); + if (packagePrefixes.size() > 0) { + final FilterBuilder filter = new FilterBuilder(); + for (String prefix : packagePrefixes) { + filter.include(FilterBuilder.prefix(prefix)); + } + ConfigurationBuilder conf = new ConfigurationBuilder() + .setUrls(pathsToScan) + .setMetadataAdapter(METADATA_ADAPTER) // Scanners depend on this + .filterInputsBy(filter) + .setScanners(annotationScanner, subTypesScanner); + + // scans stuff, but don't use the funky storage layer + new Reflections(conf); + } + List implementations = new ArrayList<>(); + for (String baseTypeName: scannedClasses) { + implementations.add( + new ParentClassDescriptor( + baseTypeName, + new ArrayList<>(subTypesScanner.getChildrenOf(baseTypeName)))); + } + List annotated = annotationScanner.getAnnotatedClasses(); + verifyClassUnicity(annotated); + return new ScanResult( + packagePrefixes, + scannedClasses, + scannedAnnotations, + annotated, + implementations); + } finally { + logger.info( + format("Scanning packages %s in locations %s took %dms", + packagePrefixes, pathsToScan, watch.elapsed(MILLISECONDS))); + } + } + + private static void verifyClassUnicity(List annotatedClasses) { + Set scanned = new HashSet<>(); + for (AnnotatedClassDescriptor annotated : annotatedClasses) { + if (!scanned.add(annotated.getClassName())) { + throw new RuntimeException("BUG: " + annotated.getClassName() + " scanned twice"); + } + } + } + + static ScanResult emptyResult() { + return new ScanResult( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList()); + } + + public static ScanResult fromPrescan(DrillConfig config) { + return RunTimeScan.fromPrescan(config); + } +} diff --git a/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java b/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java new file mode 100644 index 00000000000..1d95b04e53d --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner; + +import java.net.URL; +import java.util.Collection; +import java.util.List; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.persistence.ScanResult; + +/** + * Utility to scan classpath at runtime + * + */ +public class RunTimeScan { + + /** result of prescan */ + private static final ScanResult PRESCANNED = BuildTimeScan.load(); + + /** urls of the locations (classes directory or jar) to scan that don't have a registry in them */ + private static final Collection NON_PRESCANNED_MARKED_PATHS = getNonPrescannedMarkedPaths(); + + /** + * @return getMarkedPaths() sans getPrescannedPaths() + */ + static Collection getNonPrescannedMarkedPaths() { + Collection markedPaths = ClassPathScanner.getMarkedPaths(); + markedPaths.removeAll(BuildTimeScan.getPrescannedPaths()); + return markedPaths; + } + + /** + * loads prescanned classpath info and scans for extra ones based on configuration. + * (unless prescan is disabled with {@see ClassPathScanner#IMPLEMENTATIONS_SCAN_CACHE}=falses) + * @param config to retrieve the packages to scan + * @return the scan result + */ + public static ScanResult fromPrescan(DrillConfig config) { + List packagePrefixes = ClassPathScanner.getPackagePrefixes(config); + List scannedBaseClasses = ClassPathScanner.getScannedBaseClasses(config); + List scannedAnnotations = ClassPathScanner.getScannedAnnotations(config); + if (ClassPathScanner.isScanBuildTimeCacheEnabled(config)) { + // scan only locations that have not been scanned yet + ScanResult runtimeScan = ClassPathScanner.scan( + NON_PRESCANNED_MARKED_PATHS, + packagePrefixes, + scannedBaseClasses, + scannedAnnotations, + PRESCANNED); + return runtimeScan.merge(PRESCANNED); + } else { + // scan everything + return ClassPathScanner.scan( + ClassPathScanner.getMarkedPaths(), + packagePrefixes, + scannedBaseClasses, + scannedAnnotations, + ClassPathScanner.emptyResult()); + } + } + +} diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotatedClassDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotatedClassDescriptor.java new file mode 100644 index 00000000000..13abbb5ab7a --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotatedClassDescriptor.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner.persistence; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * describe a class that was annotated with one of the configured annotations + */ +public final class AnnotatedClassDescriptor { + + private final String className; + private final List annotations; + private final List fields; + private final Map annotationMap; + + @JsonCreator public AnnotatedClassDescriptor( + @JsonProperty("className") String className, + @JsonProperty("annotations") List annotations, + @JsonProperty("fields") List fields) { + this.className = className; + this.annotations = Collections.unmodifiableList(annotations); + this.fields = Collections.unmodifiableList(fields); + this.annotationMap = AnnotationDescriptor.buildAnnotationsMap(annotations); + } + + /** + * @return the fully qualified name of the annotated class + */ + public String getClassName() { + return className; + } + + /** + * @return the class level annotations + */ + public List getAnnotations() { + return annotations; + } + + /** + * @return the fields declared by the class + */ + public List getFields() { + return fields; + } + + /** + * @param clazz the annotation type + * @return the corresponding annotation descriptor + */ + public AnnotationDescriptor getAnnotation(Class clazz) { + return annotationMap.get(clazz.getName()); + } + + /** + * @param clazz the annotation type + * @return a bytecode based implementation of the annotation + */ + public T getAnnotationProxy(Class clazz) { + final AnnotationDescriptor annotationDescriptor = getAnnotation(clazz); + if (annotationDescriptor == null) { + return null; + } + return annotationDescriptor.getProxy(clazz); + } + + @Override + public String toString() { + return "Function [className=" + className + ", annotations=" + annotations + + ", fields=" + fields + "]"; + } +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotationDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotationDescriptor.java new file mode 100644 index 00000000000..f76ad49e2bc --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotationDescriptor.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner.persistence; + +import static java.lang.String.format; + +import java.lang.reflect.Array; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; + +/** + * a class annotation + */ +public final class AnnotationDescriptor { + private final String annotationType; + private final List attributes; + private final Map attributeMap; + + @JsonCreator public AnnotationDescriptor( + @JsonProperty("annotationType") String annotationType, + @JsonProperty("attributes") List attributes) { + this.annotationType = annotationType; + this.attributes = Collections.unmodifiableList(attributes); + ImmutableMap.Builder mapBuilder = ImmutableMap.builder(); + for (AttributeDescriptor att : attributes) { + mapBuilder.put(att.getName(), att); + } + this.attributeMap = mapBuilder.build(); + } + + /** + * @return the class name of the annotation + */ + public String getAnnotationType() { + return annotationType; + } + + public List getAttributes() { + return attributes; + } + + public boolean hasAttribute(String attributeName) { + return attributeMap.containsKey(attributeName); + } + + public List getValues(String attributeName) { + AttributeDescriptor desc = attributeMap.get(attributeName); + return desc == null ? Collections.emptyList() : desc.getValues(); + } + + public String getSingleValue(String attributeName, String defaultValue) { + List values = getValues(attributeName); + if (values.size() > 1) { + throw new IllegalStateException(String.format( + "Expected a single value for the attribute named %s but found %d (%s)", + attributeName, values.size(), values.toString())); + } + return values.isEmpty() ? defaultValue : values.get(0); + } + + public String getSingleValue(String attributeName) { + String val = getSingleValue(attributeName, null); + if (val == null) { + throw new IllegalStateException(format("Attribute %s not found in %s", attributeName, attributes)); + } + return val; + } + + @Override + public String toString() { + return "Annotation[type=" + annotationType + ", attributes=" + attributes + "]"; + } + + static Map buildAnnotationsMap(List annotations) { + ImmutableMap.Builder annMapBuilder = ImmutableMap.builder(); + for (AnnotationDescriptor ann : annotations) { + annMapBuilder.put(ann.getAnnotationType(), ann); + } + return annMapBuilder.build(); + } + + @SuppressWarnings("unchecked") + private T proxy(Class interfc, InvocationHandler ih) { + if (!interfc.isInterface()) { + throw new IllegalArgumentException("only proxying interfaces: " + interfc); + } + return (T)Proxy.newProxyInstance(this.getClass().getClassLoader(), new Class[]{ interfc }, ih); + } + + public T getProxy(Class clazz) { + return proxy(clazz, new InvocationHandler() { + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + if (args != null && args.length > 0) { + // just property methods + throw new UnsupportedOperationException(method + " " + Arrays.toString(args)); + } + String attributeName = method.getName(); + if (!hasAttribute(attributeName)) { + return method.getDefaultValue(); + } + Class returnType = method.getReturnType(); + if (returnType.isArray()) { + List values = getValues(attributeName); + Class componentType = returnType.getComponentType(); + Object[] result = (Object[])Array.newInstance(componentType, values.size()); + for (int i = 0; i < result.length; i++) { + String value = values.get(i); + result[i] = convertValue(componentType, value); + } + return result; + } else { + String value = getSingleValue(attributeName, null); + return convertValue(returnType, value); + } + } + + private Object convertValue(Class c, String value) { + if (c.equals(String.class)) { + return value; + } else if (c.isEnum()) { + @SuppressWarnings("unchecked") + Enum enumValue = Enum.valueOf(c.asSubclass(Enum.class), value); + return enumValue; + } else if (c.equals(boolean.class)) { + return Boolean.valueOf(value); + } + throw new UnsupportedOperationException(c.toString()); + } + }); + } +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/AttributeDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/AttributeDescriptor.java new file mode 100644 index 00000000000..81a80974768 --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/AttributeDescriptor.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner.persistence; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * a field set in an annotation + * (to simplify we have a list of string representations of the values) + */ +public final class AttributeDescriptor { + private final String name; + private final List values; + + @JsonCreator public AttributeDescriptor( + @JsonProperty("name") String name, + @JsonProperty("values") List values) { + this.name = name; + this.values = values; + } + + /** + * @return field name + */ + public String getName() { + return name; + } + + public List getValues() { + return values; + } + + @Override + public String toString() { + return "Attribute[" + name + "=" + values + "]"; + } +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/ChildClassDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/ChildClassDescriptor.java new file mode 100644 index 00000000000..7e81bd8a118 --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/ChildClassDescriptor.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner.persistence; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * a class that implements a specific type + */ +public final class ChildClassDescriptor { + + private final String name; + private final boolean isAbstract; + + @JsonCreator + public ChildClassDescriptor( + @JsonProperty("name") String name, + @JsonProperty("abstract") boolean isAbstract) { + this.name = name; + this.isAbstract = isAbstract; + } + + /** + * @return the class name + */ + public String getName() { + return name; + } + + /** + * @return whether the class is abstract + */ + public boolean isAbstract() { + return isAbstract; + } + + @Override + public String toString() { + return "ChildClassDescriptor [name=" + name + ", isAbstract=" + isAbstract + "]"; + } + +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/FieldDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/FieldDescriptor.java new file mode 100644 index 00000000000..dae2a74902e --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/FieldDescriptor.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner.persistence; + +import java.lang.reflect.Array; +import java.util.List; +import java.util.Map; + +import org.apache.drill.common.exceptions.DrillRuntimeException; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * a class fields + */ +public final class FieldDescriptor { + private final String name; + private final String descriptor; + private final List annotations; + private final Map annotationMap; + + @JsonCreator public FieldDescriptor( + @JsonProperty("name") String name, + @JsonProperty("descriptor") String descriptor, + @JsonProperty("annotations") List annotations) { + this.name = name; + this.descriptor = descriptor; + this.annotations = annotations; + this.annotationMap = AnnotationDescriptor.buildAnnotationsMap(annotations); + // validate the descriptor + getType(); + } + + public String getName() { + return name; + } + + /** + * @return the descriptor of the type of the field as defined in the bytecode + */ + public String getDescriptor() { + return descriptor; + } + + public List getAnnotations() { + return annotations; + } + + public AnnotationDescriptor getAnnotation(Class clazz) { + return annotationMap.get(clazz.getName()); + } + + public T getAnnotationProxy(Class clazz) { + final AnnotationDescriptor annotationDescriptor = getAnnotation(clazz); + if (annotationDescriptor == null) { + return null; + } + return annotationDescriptor.getProxy(clazz); + } + + @JsonIgnore + public TypeDescriptor getType() { + int arrayDim = 0; + char c; + while ((c = descriptor.charAt(arrayDim)) == '[') { + ++arrayDim; + } + if (c == 'L') { + int lastIndex = descriptor.length() - 1; + if (descriptor.charAt(lastIndex) != ';') { + throw new DrillRuntimeException("Illegal descriptor: " + descriptor); + } + String className = descriptor.substring(arrayDim + 1, lastIndex).replace('/', '.'); + return TypeDescriptor.forClass(className, arrayDim); + } else { + return TypeDescriptor.forPrimitive(c, arrayDim); + } + } + + @JsonIgnore + public Class getFieldClass() { + TypeDescriptor type = getType(); + Class elementClass = type.getType(); + if (type.isArray()) { + return Array.newInstance(elementClass, new int[type.getArrayDim()]).getClass(); + } else { + return elementClass; + } + } + + @Override + public String toString() { + return "Field[name=" + name + ", descriptor=" + descriptor + ", annotations=" + annotations + "]"; + } +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/ParentClassDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/ParentClassDescriptor.java new file mode 100644 index 00000000000..210c4f5cfb0 --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/ParentClassDescriptor.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner.persistence; + +import static java.util.Collections.unmodifiableList; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * a parent class and its implementations that was specifically searched for during scanning + */ +public final class ParentClassDescriptor { + private final String name; + private final List children; + + @JsonCreator + public ParentClassDescriptor( + @JsonProperty("name") String name, + @JsonProperty("children") List children) { + this.name = name; + this.children = unmodifiableList(children); + } + + /** + * @return the class name + */ + public String getName() { + return name; + } + + /** + * @return the implementations + */ + public List getChildren() { + return children; + } + + @Override + public String toString() { + return "ParentClassDescriptor [name=" + name + ", children=" + children + "]"; + } + +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/ScanResult.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/ScanResult.java new file mode 100644 index 00000000000..a85fa15210d --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/ScanResult.java @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner.persistence; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableSet; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.drill.common.exceptions.DrillRuntimeException; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import com.google.common.base.Stopwatch; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; + +/** + * The root doc of the scan result + */ +public final class ScanResult { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScanResult.class); + + private final List scannedPackages; + private final Set scannedClasses; + private final Set scannedAnnotations; + private final List annotatedClasses; + private final List implementations; + + private final Map parentClassByName; + private final Multimap annotationsByName; + + @JsonCreator public ScanResult( + @JsonProperty("scannedPackages") Collection scannedPackages, + @JsonProperty("scannedClasses") Collection scannedClasses, + @JsonProperty("scannedAnnotations") Collection scannedAnnotations, + @JsonProperty("annotatedClasses") Collection annotatedClasses, + @JsonProperty("implementations") Collection implementations) { + this.scannedPackages = unmodifiableList(new ArrayList<>(checkNotNull(scannedPackages))); + this.scannedClasses = unmodifiableSet(new HashSet<>(checkNotNull(scannedClasses))); + this.scannedAnnotations = unmodifiableSet(new HashSet<>(checkNotNull(scannedAnnotations))); + this.annotatedClasses = unmodifiableList(new ArrayList<>(checkNotNull(annotatedClasses))); + this.implementations = unmodifiableList(new ArrayList<>(checkNotNull(implementations))); + this.parentClassByName = new HashMap<>(); + for (ParentClassDescriptor parentClassDescriptor : implementations) { + this.parentClassByName.put(parentClassDescriptor.getName(), parentClassDescriptor); + } + this.annotationsByName = HashMultimap.create(); + for (AnnotatedClassDescriptor annotated : annotatedClasses) { + for (AnnotationDescriptor annotationDescriptor : annotated.getAnnotations()) { + if (scannedAnnotations.contains(annotationDescriptor.getAnnotationType())) { + this.annotationsByName.put(annotationDescriptor.getAnnotationType(), annotated); + } + } + } + } + + /** + * @return packages that were scanned as defined in the drill configs + */ + public List getScannedPackages() { + return scannedPackages; + } + + /** + * @return functions annotated with configured annotations + */ + public List getAnnotatedClasses() { + return annotatedClasses; + } + + /** + * @return list of implementations per parent class + */ + public List getImplementations() { + return implementations; + } + + public Set getScannedClasses() { + return scannedClasses; + } + + public Set getScannedAnnotations() { + return scannedAnnotations; + } + + /** + * @param c the parent class name + * @return the descriptor for the implementations found + */ + public ParentClassDescriptor getImplementations(String c) { + if (!scannedClasses.contains(c)) { + throw new IllegalArgumentException( + c + " is not scanned. " + + "Only implementations for the following classes are scanned: " + + scannedClasses); + } + return parentClassByName.get(c); + } + + /** + * This will load all the scanned classes for this parent as a side effect + * @param c the parent + * @return all the classes found + */ + public Set> getImplementations(Class c) { + ParentClassDescriptor p = getImplementations(c.getName()); + Stopwatch watch = new Stopwatch().start(); + Set> result = new HashSet<>(); + try { + if (p != null) { + for (ChildClassDescriptor child : p.getChildren()) { + if (!child.isAbstract()) { + try { + result.add(Class.forName(child.getName()).asSubclass(c)); + } catch (ClassNotFoundException e) { + throw new DrillRuntimeException("scanned class could not be found: " + child.getName(), e); + } + } + } + } + return result; + } finally { + logger.info( + format("loading %d classes for %s took %dms", + result.size(), c.getName(), watch.elapsed(MILLISECONDS))); + } + } + + + /** + * @param c the annotation class name + * @return the descriptor of the annotated class + */ + public List getAnnotatedClasses(String c) { + if (!scannedAnnotations.contains(c)) { + throw new IllegalArgumentException( + c + " is not scanned. " + + "Only the following Annotations are scanned: " + + scannedAnnotations); + } + return new ArrayList<>(annotationsByName.get(c)); + } + + @Override + public String toString() { + return "ScanResult [" + + "scannedPackages=" + scannedPackages + ", " + + "scannedClasses=" + scannedClasses + ", " + + "scannedAnnotations=" + scannedAnnotations + ", " + + "annotatedClasses=" + annotatedClasses + ", " + + "implementations=" + implementations + "]"; + } + + private List merge(Collection a, Collection b) { + final List newList = new ArrayList<>(a); + newList.addAll(b); + return newList; + } + + /** + * merges this and other together into a new result object + * @param other + * @return the resulting merge + */ + public ScanResult merge(ScanResult other) { + final Multimap newImpls = HashMultimap.create(); + for (Collection impls : asList(implementations, other.implementations)) { + for (ParentClassDescriptor c : impls) { + newImpls.putAll(c.getName(), c.getChildren()); + } + } + List newImplementations = new ArrayList<>(); + for (Entry> entry : newImpls.asMap().entrySet()) { + newImplementations.add(new ParentClassDescriptor(entry.getKey(), new ArrayList<>(entry.getValue()))); + } + + return new ScanResult( + merge(scannedPackages, other.scannedPackages), + merge(scannedClasses, other.scannedClasses), + merge(scannedAnnotations, other.scannedAnnotations), + merge(annotatedClasses, other.annotatedClasses), + newImplementations); + } + +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/TypeDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/TypeDescriptor.java new file mode 100644 index 00000000000..f144ccaab4f --- /dev/null +++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/TypeDescriptor.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner.persistence; + +import static java.lang.String.format; + +import org.apache.drill.common.exceptions.DrillRuntimeException; + +/** + * represents the type of a field + */ +public abstract class TypeDescriptor { + + private final boolean primitive; + private final int arrayDim; + + TypeDescriptor(boolean primitive, int arrayDim) { + this.primitive = primitive; + this.arrayDim = arrayDim; + } + + public boolean isPrimitive() { + return primitive; + } + + public boolean isArray() { + return arrayDim > 0; + } + + /** + * @return the number of dimensions of the array or 0 if it's not an array + */ + public int getArrayDim() { + return arrayDim; + } + + /** + * @return the type (element type if an array) + */ + public abstract Class getType(); + + private static final class PrimitiveTypeDescriptor extends TypeDescriptor { + private final Class type; + + private static Class getPrimitiveType(char descriptor) { + switch (descriptor) { + case 'V': return void.class; + case 'I': return int.class; + case 'B': return byte.class; + case 'J': return long.class; + case 'D': return double.class; + case 'F': return float.class; + case 'C': return char.class; + case 'S': return short.class; + case 'Z': return boolean.class; + default: throw new DrillRuntimeException("bad descriptor: " + descriptor); + } + } + + private PrimitiveTypeDescriptor(char typeDescriptor, int arrayDepth) { + super(true, arrayDepth); + this.type = getPrimitiveType(typeDescriptor); + } + + @Override + public Class getType() { + return this.type; + } + } + + private static final class ClassTypeDescriptor extends TypeDescriptor { + private String className; + + private ClassTypeDescriptor(String className, int arrayDim) { + super(false, arrayDim); + this.className = className; + } + + @Override + public Class getType() { + try { + return Class.forName(className); + } catch (ClassNotFoundException e) { + throw new DrillRuntimeException( + format( + "Unexpected class load failure while attempting to load Function Registry (%s)", + className), + e); + } + } + } + + public static TypeDescriptor forClass(String className, int arrayDim) { + return new ClassTypeDescriptor(className, arrayDim); + } + + public static TypeDescriptor forPrimitive(char c, int arrayDim) { + return new PrimitiveTypeDescriptor(c, arrayDim); + } +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/drill/common/util/PathScanner.java b/common/src/main/java/org/apache/drill/common/util/PathScanner.java deleted file mode 100644 index 65bf9b5d8bd..00000000000 --- a/common/src/main/java/org/apache/drill/common/util/PathScanner.java +++ /dev/null @@ -1,176 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.common.util; - -import java.io.IOException; -import java.lang.annotation.Annotation; -import java.lang.reflect.Modifier; -import java.net.URL; -import java.util.Collection; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.apache.drill.common.config.CommonConstants; -import org.reflections.Reflections; -import org.reflections.scanners.ResourcesScanner; -import org.reflections.scanners.SubTypesScanner; -import org.reflections.scanners.TypeAnnotationsScanner; -import org.reflections.util.ConfigurationBuilder; - -import com.google.common.base.Stopwatch; -import com.google.common.collect.Sets; - -public class PathScanner { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PathScanner.class); - - private static final SubTypesScanner subTypeScanner = new SubTypesScanner(); - private static final TypeAnnotationsScanner annotationsScanner = new TypeAnnotationsScanner(); - private static final ResourcesScanner resourcesScanner = new ResourcesScanner(); - private static final Object SYNC = new Object(); - static volatile Reflections REFLECTIONS = null; - - public static Map> scanForAnnotatedImplementations( - Class annotationClass, Class baseClass, final List scanPackages) { - final Collection> providerClasses = - scanForImplementations(baseClass, scanPackages); - - final Map> map = new HashMap>(); - - for (Class c : providerClasses) { - final A annotation = (A) c.getAnnotation(annotationClass); - if (annotation == null) { - continue; - } - map.put(annotation, c); - } - - return map; - } - - private static Reflections getReflections() { - if (REFLECTIONS == null) { - REFLECTIONS = - new Reflections( - new ConfigurationBuilder() - .setUrls(getMarkedPaths()) - .setScanners(subTypeScanner, annotationsScanner, resourcesScanner)); - } - return REFLECTIONS; - } - - /** - * @param scanPackages note: not currently used - */ - public static Class[] scanForImplementationsArr(final Class baseClass, - final List scanPackages) { - Collection> imps = scanForImplementations(baseClass, scanPackages); - Class[] arr = imps.toArray(new Class[imps.size()]); - return arr; - } - - /** - * @param scanPackages note: not currently used - */ - public static Set> scanForImplementations(final Class baseClass, - final List scanPackages) { - final Stopwatch w = new Stopwatch().start(); - try { - synchronized(SYNC) { - final Set> classes = getReflections().getSubTypesOf(baseClass); - for (Iterator> i = classes.iterator(); i.hasNext();) { - final Class c = i.next(); - assert baseClass.isAssignableFrom(c); - if (Modifier.isAbstract(c.getModifiers())) { - i.remove(); - } - } - return classes; - } - } finally { - logger.debug("Implementations scanning took {} ms for {}.", - w.elapsed(TimeUnit.MILLISECONDS), - baseClass); // no .getName(), so it has "class "/"interface " - } - } - - private static Collection getMarkedPaths() { - Collection urls = - forResource(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, true); - return urls; - } - - public static Collection getConfigURLs() { - return forResource(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, false); - } - - /** - * Gets URLs of any classpath resources with given resource pathname. - * - * @param resourcePathname resource pathname of classpath resource instances - * to scan for (relative to specified class loaders' classpath roots) - * @param returnRootPathname whether to collect classpath root portion of - * URL for each resource instead of full URL of each resource - * @param classLoaders not currently used (was: "set of class loaders in - * which to look up resource; none (empty array) to specify to use - * current thread's context class loader and {@link Reflections}'s - * class loader") - * @returns ...; empty set if none - */ - public static Set forResource(final String resourcePathname, - final boolean returnRootPathname, - final ClassLoader... classLoaders) { - logger.debug("Scanning classpath for resources with pathname \"{}\".", - resourcePathname); - final Set resultUrlSet = Sets.newHashSet(); - - final ClassLoader classLoader = PathScanner.class.getClassLoader(); - try { - final Enumeration resourceUrls = - classLoader.getResources(resourcePathname); - while (resourceUrls.hasMoreElements()) { - final URL resourceUrl = resourceUrls.nextElement(); - logger.trace("- found a(n) {} at {}.", resourcePathname, resourceUrl); - - int index = resourceUrl.toExternalForm().lastIndexOf(resourcePathname); - if (index != -1 && returnRootPathname) { - final URL classpathRootUrl = - new URL(resourceUrl.toExternalForm().substring(0, index)); - resultUrlSet.add(classpathRootUrl); - logger.debug("- collected resource's classpath root URL {}.", - classpathRootUrl); - } else { - resultUrlSet.add(resourceUrl); - logger.debug("- collected resource URL {}.", resourceUrl); - } - } - } catch (IOException e) { - if (Reflections.log != null) { - Reflections.log.error( - "Error scanning for resources named " + resourcePathname, e); - } - } - - return resultUrlSet; - } - -} diff --git a/common/src/main/resources/drill-module.conf b/common/src/main/resources/drill-module.conf index c01939c9f05..f8226f8cb6f 100644 --- a/common/src/main/resources/drill-module.conf +++ b/common/src/main/resources/drill-module.conf @@ -17,4 +17,17 @@ // This file tells Drill to consider this module when class path scanning. // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.exec.storage.packages += org.apache.drill.exec.store.mock +drill { + classpath.scanning { + base.classes : ${?drill.classpath.scanning.base.classes} [ + org.apache.drill.common.logical.data.LogicalOperator, + org.apache.drill.common.logical.FormatPluginConfig, + org.apache.drill.common.logical.StoragePluginConfig + ], + + packages : ${?drill.classpath.scanning.packages} [ + org.apache.drill.exec.store.mock, + org.apache.drill.common.logical + ] + } +} diff --git a/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java b/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java index a3a8257dc2b..090b8f45fae 100644 --- a/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java +++ b/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java @@ -22,26 +22,30 @@ import java.util.Collection; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.logical.LogicalPlan; import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.common.util.FileUtils; -import org.apache.drill.common.util.PathScanner; import org.apache.drill.test.DrillTest; import org.junit.Test; -import com.google.common.collect.Lists; - public class CheckStorageConfig extends DrillTest { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CheckStorageConfig.class); @Test public void ensureStorageEnginePickup() { - Collection engines = PathScanner.scanForImplementations(StoragePluginConfig.class, Lists.newArrayList("org")); + DrillConfig config = DrillConfig.create(); + ScanResult scan = ClassPathScanner.fromPrescan(config); + Collection engines = scan.getImplementations(StoragePluginConfig.class); assertEquals(engines.size(), 1); } @Test public void checkPlanParsing() throws Exception{ - LogicalPlan plan = LogicalPlan.parse(DrillConfig.create(), FileUtils.getResourceAsString("/storage_engine_plan.json")); + DrillConfig config = DrillConfig.create(); + ScanResult scan = ClassPathScanner.fromPrescan(config); + LogicalPlan plan = LogicalPlan.parse(new LogicalPlanPersistence(config, scan), FileUtils.getResourceAsString("/storage_engine_plan.json")); } } diff --git a/common/src/test/resources/drill-module.conf b/common/src/test/resources/drill-module.conf index 5b9605aa0af..4236559c936 100644 --- a/common/src/test/resources/drill-module.conf +++ b/common/src/test/resources/drill-module.conf @@ -14,4 +14,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -drill.logical.storage.packages += "org.apache.drill.storage" +drill.classpath.scanning.packages += "org.apache.drill.storage" diff --git a/common/src/test/resources/logback.xml b/common/src/test/resources/logback.xml index a6a3dbad53a..101b0c631fc 100644 --- a/common/src/test/resources/logback.xml +++ b/common/src/test/resources/logback.xml @@ -10,14 +10,12 @@ OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> - - - true - 10000 - true - ${LILITH_HOSTNAME:-localhost} - - + + + + + + @@ -40,14 +38,14 @@ --> - + - - + + diff --git a/contrib/storage-hbase/src/main/resources/drill-module.conf b/contrib/storage-hbase/src/main/resources/drill-module.conf index 0f0a0c6d097..150a3bb94be 100644 --- a/contrib/storage-hbase/src/main/resources/drill-module.conf +++ b/contrib/storage-hbase/src/main/resources/drill-module.conf @@ -17,7 +17,13 @@ // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.exec: { +drill: { + classpath.scanning: { + packages += "org.apache.drill.exec.store.hbase" + packages += "org.apache.drill.exec.expr.fn.impl.conv" + } + + exec: { sys.store.provider: { # HBasePStoreProvider is disabled by default @@ -35,4 +41,5 @@ drill.exec: { samplerows.count: 100, sizecalculator.enabled: true } + } } diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java index 5f2d6c7ad99..9ff1b089cca 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.util.Map.Entry; +import org.apache.drill.common.config.LogicalPlanPersistence; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.store.hbase.config.HBasePStoreProvider; import org.apache.drill.exec.store.sys.PStore; import org.apache.drill.exec.store.sys.PStoreConfig; @@ -41,7 +43,8 @@ public static void setUpBeforeTestHBaseTableProvider() throws Exception { @Test public void testTableProvider() throws IOException { - PStore hbaseStore = provider.getStore(PStoreConfig.newJacksonBuilder(config.getMapper(), String.class).name("hbase").build()); + LogicalPlanPersistence lp = PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(config); + PStore hbaseStore = provider.getStore(PStoreConfig.newJacksonBuilder(lp.getMapper(), String.class).name("hbase").build()); hbaseStore.put("", "v0"); hbaseStore.put("k1", "v1"); hbaseStore.put("k2", "v2"); @@ -60,7 +63,7 @@ public void testTableProvider() throws IOException { } assertEquals(7, rowCount); - PStore hbaseTestStore = provider.getStore(PStoreConfig.newJacksonBuilder(config.getMapper(), String.class).name("hbase.test").build()); + PStore hbaseTestStore = provider.getStore(PStoreConfig.newJacksonBuilder(lp.getMapper(), String.class).name("hbase.test").build()); hbaseTestStore.put("", "v0"); hbaseTestStore.put("k1", "v1"); hbaseTestStore.put("k2", "v2"); diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index 87384a08277..53614e3ccd3 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -104,9 +104,9 @@ - com.googlecode.fmpp-maven-plugin - fmpp-maven-plugin - 1.0 + org.apache.drill.tools + drill-fmpp-maven-plugin + ${project.version} org.freemarker @@ -114,36 +114,17 @@ 2.3.19 - - src/main/codegen/config.fmpp - target/generated-sources - src/main/codegen/templates - generate-fmpp-sources - initialize + generate-sources generate - - - - - org.codehaus.mojo - build-helper-maven-plugin - 1.5 - - - add-fmpp-sources - validate - - add-source - - - ${project.build.directory}/generated-sources - + src/main/codegen/config.fmpp + ${project.build.directory}/generated-sources + src/main/codegen/templates diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java index 13fb107f71a..728954d8068 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java @@ -22,10 +22,11 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; -import org.apache.drill.common.util.PathScanner; import org.apache.drill.exec.expr.fn.impl.hive.ObjectInspectorHelper; import org.apache.drill.exec.planner.sql.DrillOperatorTable; import org.apache.drill.exec.planner.sql.HiveUDFOperator; @@ -53,12 +54,14 @@ public class HiveFunctionRegistry implements PluggableFunctionRegistry{ * @param config */ public HiveFunctionRegistry(DrillConfig config) { - Set> genericUDFClasses = PathScanner.scanForImplementations(GenericUDF.class, null); + // TODO: see if we can avoid this. We can't change the constructor right now. + ScanResult classpathScan = ClassPathScanner.fromPrescan(config); + Set> genericUDFClasses = classpathScan.getImplementations(GenericUDF.class); for (Class clazz : genericUDFClasses) { register(clazz, methodsGenericUDF); } - Set> udfClasses = PathScanner.scanForImplementations(UDF.class, null); + Set> udfClasses = classpathScan.getImplementations(UDF.class); for (Class clazz : udfClasses) { register(clazz, methodsUDF); } diff --git a/contrib/storage-hive/core/src/main/resources/drill-module.conf b/contrib/storage-hive/core/src/main/resources/drill-module.conf index 271395e5f27..bef453b577e 100644 --- a/contrib/storage-hive/core/src/main/resources/drill-module.conf +++ b/contrib/storage-hive/core/src/main/resources/drill-module.conf @@ -2,4 +2,16 @@ // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.logical.function.packages += "org.apache.hadoop.hive" +drill { + classpath.scanning { + base.classes : ${?drill.classpath.scanning.base.classes} [ + org.apache.hadoop.hive.ql.udf.generic.GenericUDF, + org.apache.hadoop.hive.ql.exec.UDF + ], + + packages : ${?drill.classpath.scanning.packages} [ + org.apache.hadoop.hive, + org.apache.drill.exec.fn.hive + ] + } +} \ No newline at end of file diff --git a/contrib/storage-hive/core/src/test/resources/drill-module.conf b/contrib/storage-hive/core/src/test/resources/drill-module.conf index 271395e5f27..7451eae4bef 100644 --- a/contrib/storage-hive/core/src/test/resources/drill-module.conf +++ b/contrib/storage-hive/core/src/test/resources/drill-module.conf @@ -2,4 +2,4 @@ // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.logical.function.packages += "org.apache.hadoop.hive" +drill.classpath.scanning.packages += "org.apache.hadoop.hive" diff --git a/contrib/storage-hive/core/src/test/resources/logback.xml b/contrib/storage-hive/core/src/test/resources/logback.xml index a9e15865492..dbfca64569a 100644 --- a/contrib/storage-hive/core/src/test/resources/logback.xml +++ b/contrib/storage-hive/core/src/test/resources/logback.xml @@ -17,14 +17,13 @@ --> - - - true - 10000 - true - ${LILITH_HOSTNAME:-localhost} - + + + + + + + @@ -43,13 +42,13 @@ - + - - - - + + + + @@ -57,7 +56,7 @@ - + diff --git a/contrib/storage-hive/hive-exec-shade/src/main/resources/drill-module.conf b/contrib/storage-hive/hive-exec-shade/src/main/resources/drill-module.conf index 271395e5f27..7451eae4bef 100644 --- a/contrib/storage-hive/hive-exec-shade/src/main/resources/drill-module.conf +++ b/contrib/storage-hive/hive-exec-shade/src/main/resources/drill-module.conf @@ -2,4 +2,4 @@ // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.logical.function.packages += "org.apache.hadoop.hive" +drill.classpath.scanning.packages += "org.apache.hadoop.hive" diff --git a/contrib/storage-jdbc/src/main/resources/drill-module.conf b/contrib/storage-jdbc/src/main/resources/drill-module.conf index 721a599507b..0f98d051009 100755 --- a/contrib/storage-jdbc/src/main/resources/drill-module.conf +++ b/contrib/storage-jdbc/src/main/resources/drill-module.conf @@ -16,3 +16,6 @@ // This file tells Drill to consider this module when class path scanning. // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. +drill.classpath.scanning: { + packages += "org.apache.drill.exec.store.jdbc" +} diff --git a/contrib/storage-mongo/src/main/resources/drill-module.conf b/contrib/storage-mongo/src/main/resources/drill-module.conf index d44d3e4bb04..81490f0791d 100644 --- a/contrib/storage-mongo/src/main/resources/drill-module.conf +++ b/contrib/storage-mongo/src/main/resources/drill-module.conf @@ -16,7 +16,9 @@ // This file tells Drill to consider this module when class path scanning. // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. - +drill.classpath.scanning: { + packages += "org.apache.drill.exec.store.mongo" +} drill.exec: { sys.store.provider: { @@ -25,4 +27,4 @@ drill.exec: { } } -} \ No newline at end of file +} diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java index d050961587b..809fb755385 100644 --- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java +++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java @@ -20,6 +20,7 @@ public interface MongoTestConstants { public static final String LOCALHOST = "localhost"; + // TODO: DRILL-3934: add some randomization to this as it fails when running concurrent builds public static final int CONFIG_SERVER_PORT = 27019; public static final int MONGOD_1_PORT = 27020; public static final int MONGOD_2_PORT = 27021; diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index 0eaff60ea00..e3e00b91322 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -470,12 +470,10 @@ maven-resources-plugin - + copy-fmpp-resources initialize - - copy-resources - + copy-resources ${project.build.directory}/codegen @@ -486,36 +484,11 @@ - - copy-java-sources - process-sources - - copy-resources - - - ${basedir}/target/classes/ - - - src/main/java - true - - - src/test/java - true - - - target/generated-sources - - true - - - - + it under ${project.build.directory}/codegen/templates where all freemarker templates are. --> org.apache.maven.plugins maven-dependency-plugin 2.8 @@ -534,17 +507,17 @@ jar true ${project.build.directory}/ - **/Parser.jj + codegen/templates/Parser.jj - - com.googlecode.fmpp-maven-plugin - fmpp-maven-plugin - 1.0 + + org.apache.drill.tools + drill-fmpp-maven-plugin + ${project.version} org.freemarker @@ -554,49 +527,26 @@ - generate-fmpp-sources + generate-fmpp generate-sources - - generate - + generate - ${project.build.directory}/codegen/config.fmpp - target/generated-sources - ${project.build.directory}/codegen/templates + src/main/codegen/config.fmpp + ${project.build.directory}/generated-sources + ${project.build.directory}/codegen/templates - - org.codehaus.mojo - build-helper-maven-plugin - 1.5 - - - add-generated-sources - process-sources - - add-source - - - - ${project.build.directory}/generated-sources - - - - - - + org.codehaus.mojo javacc-maven-plugin 2.4 - generate-sources javacc - - javacc - + generate-sources + javacc ${project.build.directory}/generated-sources/ @@ -609,6 +559,70 @@ + + org.codehaus.mojo + build-helper-maven-plugin + 1.9.1 + + + add-sources-as-resources + process-sources + + add-resource + + + + + src/main/java + + + ${project.build.directory}/generated-sources + + + + + + add-test-sources-as-resources + process-test-sources + + add-test-resource + + + + + src/test/java + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + org.apache.drill + drill-common + ${project.version} + tests + + + + + process-classes + java + + + + org.apache.drill.common.scanner.BuildTimeScan + true + + ${project.build.outputDirectory} + + + org.apache.maven.plugins maven-surefire-plugin @@ -619,9 +633,7 @@ in a different profile --> default-test test - - test - + test **/TestLargeFileCompilation.java @@ -644,8 +656,8 @@ - com.googlecode.fmpp-maven-plugin - fmpp-maven-plugin + org.apache.drill.tools + drill-fmpp-maven-plugin [1.0,) generate diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index c9554af6a3f..db7f68dc1b8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -42,14 +42,12 @@ public interface ExecConstants { public static final String BIT_RETRY_TIMES = "drill.exec.rpc.bit.server.retry.count"; public static final String BIT_RETRY_DELAY = "drill.exec.rpc.bit.server.retry.delay"; public static final String BIT_TIMEOUT = "drill.exec.bit.timeout" ; - public static final String STORAGE_ENGINE_SCAN_PACKAGES = "drill.exec.storage.packages"; public static final String SERVICE_NAME = "drill.exec.cluster-id"; public static final String INITIAL_BIT_PORT = "drill.exec.rpc.bit.server.port"; public static final String BIT_RPC_TIMEOUT = "drill.exec.rpc.bit.timeout"; public static final String INITIAL_USER_PORT = "drill.exec.rpc.user.server.port"; public static final String USER_RPC_TIMEOUT = "drill.exec.rpc.user.timeout"; public static final String METRICS_CONTEXT_NAME = "drill.exec.metrics.context"; - public static final String FUNCTION_PACKAGES = "drill.exec.functions"; public static final String USE_IP_ADDRESS = "drill.exec.rpc.use.ip"; public static final String METRICS_JMX_OUTPUT_ENABLED = "drill.exec.metrics.jmx.enabled"; public static final String METRICS_LOG_OUTPUT_ENABLED = "drill.exec.metrics.log.enabled"; @@ -90,7 +88,6 @@ public interface ExecConstants { public static final String SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE = "drill.exec.sys.store.provider.local.write"; public static final String IMPERSONATION_ENABLED = "drill.exec.impersonation.enabled"; public static final String IMPERSONATION_MAX_CHAINED_USER_HOPS = "drill.exec.impersonation.max_chained_user_hops"; - public static final String USER_AUTHENTICATOR_IMPL_PACKAGES = "drill.exec.security.user.auth.packages"; public static final String USER_AUTHENTICATION_ENABLED = "drill.exec.security.user.auth.enabled"; public static final String USER_AUTHENTICATOR_IMPL = "drill.exec.security.user.auth.impl"; public static final String PAM_AUTHENTICATOR_PROFILES = "drill.exec.security.user.auth.pam_profiles"; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java index 61d9ed6bc4b..186f289a956 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java @@ -25,6 +25,7 @@ import org.apache.drill.common.DrillAutoCloseables; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.client.QuerySubmitter.Format; import org.apache.drill.exec.exception.SchemaChangeException; @@ -112,6 +113,7 @@ public void dataArrived(QueryDataBatch result, ConnectionThrottle throttle) { public int await() throws Exception { latch.await(); if (exception != null) { + exception.addSuppressed(new DrillRuntimeException("Exception in executor threadpool")); throw exception; } return count.get(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java index f0147ef5897..3f9da0721e6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java @@ -25,6 +25,7 @@ import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.ClassTransformationException; import org.apache.drill.exec.expr.CodeGenerator; +import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.SystemOptionManager; import org.apache.drill.exec.store.sys.local.LocalPStoreProvider; @@ -91,10 +92,6 @@ public GeneratedClassEntry(final Class clazz) { } } - public static CodeCompiler getTestCompiler(DrillConfig c) throws IOException{ - return new CodeCompiler(c, new SystemOptionManager(c, new LocalPStoreProvider(c)).init()); - } - /** * Flush the compiled classes from the cache. * diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java index efa8cc7dee1..1946ceeced5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/DotDrillFile.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.dotdrill; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.exec.store.dfs.DrillFileSystem; import org.apache.hadoop.fs.FileStatus; @@ -67,10 +68,10 @@ public String getBaseName() { return fileName.substring(0, fileName.lastIndexOf(type.getEnding())); } - public View getView(DrillConfig config) throws IOException { + public View getView(LogicalPlanPersistence lpPersistence) throws IOException { Preconditions.checkArgument(type == DotDrillType.VIEW); try(InputStream is = fs.open(status.getPath())){ - return config.getMapper().readValue(is, View.class); + return lpPersistence.getMapper().readValue(is, View.class); } } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java index 0dba6092317..96b0485ba96 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java @@ -60,7 +60,7 @@ public boolean isAggregating() { @Override public boolean isRandom() { - return holder.isRandom(); + return !holder.isDeterministic(); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java index 7cafab54ba8..be0fa1308e7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java @@ -57,7 +57,7 @@ NullHandling nulls() default NullHandling.INTERNAL; boolean isBinaryCommutative() default false; boolean isRandom() default false; - + String desc() default ""; FunctionCostCategory costCategory() default FunctionCostCategory.SIMPLE; public static enum NullHandling { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java index add37349d1c..9493b4dc4e7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java @@ -17,6 +17,9 @@ */ package org.apache.drill.exec.expr.fn; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + import java.util.List; import java.util.Map; @@ -44,35 +47,30 @@ import com.sun.codemodel.JType; import com.sun.codemodel.JVar; -class DrillAggFuncHolder extends DrillFuncHolder{ +class DrillAggFuncHolder extends DrillFuncHolder { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillAggFuncHolder.class); - private final String setup; - private final String reset; - private final String add; - private final String output; - private final String cleanup; - - public DrillAggFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports) { - this(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, FunctionCostCategory.getDefault()); + private String setup() { + return meth("setup"); + } + private String reset() { + return meth("reset", false); + } + private String add() { + return meth("add"); + } + private String output() { + return meth("output"); + } + private String cleanup() { + return meth("cleanup", false); } - public DrillAggFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionCostCategory costCategory) { - super(scope, nullHandling, isBinaryCommutative, isRandom, - registeredNames, parameters, returnValue, workspaceVars, methods, imports, costCategory); - Preconditions.checkArgument(nullHandling == NullHandling.INTERNAL, "An aggregation function is required to do its own null handling."); - setup = methods.get("setup"); - reset = methods.get("reset"); - add = methods.get("add"); - output = methods.get("output"); - cleanup = methods.get("cleanup"); - Preconditions.checkNotNull(add); - Preconditions.checkNotNull(output); - Preconditions.checkNotNull(reset); + public DrillAggFuncHolder( + FunctionAttributes attributes, + FunctionInitializer initializer) { + super(attributes, initializer); + checkArgument(attributes.getNullHandling() == NullHandling.INTERNAL, "An aggregation function is required to do its own null handling."); } @Override @@ -89,7 +87,7 @@ public boolean isAggregating() { public JVar[] renderStart(ClassGenerator g, HoldingContainer[] inputVariables) { if (!g.getMappingSet().isHashAggMapping()) { //Declare workspace vars for non-hash-aggregation. JVar[] workspaceJVars = declareWorkspaceVariables(g); - generateBody(g, BlockType.SETUP, setup, null, workspaceJVars, true); + generateBody(g, BlockType.SETUP, setup(), null, workspaceJVars, true); return workspaceJVars; } else { //Declare workspace vars and workspace vectors for hash aggregation. @@ -118,7 +116,7 @@ public JVar[] renderStart(ClassGenerator g, HoldingContainer[] inputVariables forLoop.test(ivar.lt(sizeVar)); forLoop.update(ivar.assignPlus(JExpr.lit(1))); - JBlock subBlock = generateInitWorkspaceBlockHA(g, BlockType.SETUP, setup, workspaceJVars, ivar); + JBlock subBlock = generateInitWorkspaceBlockHA(g, BlockType.SETUP, setup(), workspaceJVars, ivar); forLoop.body().add(subBlock); return workspaceJVars; } @@ -129,7 +127,7 @@ public JVar[] renderStart(ClassGenerator g, HoldingContainer[] inputVariables @Override public void renderMiddle(ClassGenerator g, HoldingContainer[] inputVariables, JVar[] workspaceJVars) { - addProtectedBlock(g, g.getBlock(BlockType.EVAL), add, inputVariables, workspaceJVars, false); + addProtectedBlock(g, g.getBlock(BlockType.EVAL), add(), inputVariables, workspaceJVars, false); } @@ -139,13 +137,13 @@ public HoldingContainer renderEnd(ClassGenerator g, HoldingContainer[] inputV JBlock sub = new JBlock(); g.getEvalBlock().add(sub); JVar internalOutput = sub.decl(JMod.FINAL, g.getHolderType(returnValue.type), returnValue.name, JExpr._new(g.getHolderType(returnValue.type))); - addProtectedBlock(g, sub, output, null, workspaceJVars, false); + addProtectedBlock(g, sub, output(), null, workspaceJVars, false); sub.assign(out.getHolder(), internalOutput); //hash aggregate uses workspace vectors. Initialization is done in "setup" and does not require "reset" block. if (!g.getMappingSet().isHashAggMapping()) { - generateBody(g, BlockType.RESET, reset, null, workspaceJVars, false); + generateBody(g, BlockType.RESET, reset(), null, workspaceJVars, false); } - generateBody(g, BlockType.CLEANUP, cleanup, null, workspaceJVars, false); + generateBody(g, BlockType.CLEANUP, cleanup(), null, workspaceJVars, false); return out; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillBooleanOPHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillBooleanOPHolder.java index d2708882b71..af1d84f30c6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillBooleanOPHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillBooleanOPHolder.java @@ -18,21 +18,10 @@ package org.apache.drill.exec.expr.fn; -import java.util.List; -import java.util.Map; - -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionCostCategory; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; - public class DrillBooleanOPHolder extends DrillSimpleFuncHolder{ - public DrillBooleanOPHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionCostCategory costCategory, Class drillFuncClass) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, - methods, imports, costCategory, drillFuncClass); + public DrillBooleanOPHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java index 9999c367b2a..f7ad1b3e144 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillComplexWriterFuncHolder.java @@ -18,21 +18,15 @@ package org.apache.drill.exec.expr.fn; -import java.util.List; -import java.util.Map; - import org.apache.drill.common.expression.FieldReference; import org.apache.drill.exec.expr.ClassGenerator; import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; import org.apache.drill.exec.physical.impl.project.ProjectRecordBatch; import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; import com.sun.codemodel.JBlock; import com.sun.codemodel.JClass; -import com.sun.codemodel.JConditional; import com.sun.codemodel.JExpr; import com.sun.codemodel.JExpression; import com.sun.codemodel.JInvocation; @@ -42,10 +36,8 @@ public class DrillComplexWriterFuncHolder extends DrillSimpleFuncHolder{ private FieldReference ref; - public DrillComplexWriterFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports); + public DrillComplexWriterFuncHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); } public void setReference(FieldReference ref) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAddFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAddFuncHolder.java index aae7da868cb..00ad1f5597d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAddFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAddFuncHolder.java @@ -32,11 +32,8 @@ public class DrillDecimalAddFuncHolder extends DrillSimpleFuncHolder{ - - public DrillDecimalAddFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionTemplate.FunctionCostCategory costCategory, Class drillFuncClass) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, costCategory, drillFuncClass); + public DrillDecimalAddFuncHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); } /* diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAggFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAggFuncHolder.java index 51d21aa0ddd..78d865c6bc1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAggFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalAggFuncHolder.java @@ -18,15 +18,14 @@ package org.apache.drill.exec.expr.fn; import java.util.List; -import java.util.Map; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; public class DrillDecimalAggFuncHolder extends DrillAggFuncHolder { - public DrillDecimalAggFuncHolder(FunctionTemplate.FunctionScope scope, FunctionTemplate.NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, Map methods, List imports) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports); + + public DrillDecimalAggFuncHolder(FunctionAttributes attributes, FunctionInitializer initializer) { + super(attributes, initializer); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalCastFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalCastFuncHolder.java index 73151c951eb..2fb9e288cad 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalCastFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalCastFuncHolder.java @@ -18,26 +18,19 @@ package org.apache.drill.exec.expr.fn; import java.util.List; -import java.util.Map; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.MajorType; -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; public class DrillDecimalCastFuncHolder extends DrillSimpleFuncHolder { - - public DrillDecimalCastFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionTemplate.FunctionCostCategory costCategory, Class drillFuncClass) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, costCategory, drillFuncClass); - } + public DrillDecimalCastFuncHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); + } @Override public MajorType getReturnType(List args) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java index cee7312c1b6..c867890f489 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalDivScaleFuncHolder.java @@ -18,25 +18,18 @@ package org.apache.drill.exec.expr.fn; import java.util.List; -import java.util.Map; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.util.DecimalScalePrecisionDivideFunction; -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; import org.apache.drill.exec.util.DecimalUtility; public class DrillDecimalDivScaleFuncHolder extends DrillSimpleFuncHolder{ - - public DrillDecimalDivScaleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionTemplate.FunctionCostCategory costCategory, Class drillFuncClass) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, costCategory, drillFuncClass); + public DrillDecimalDivScaleFuncHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); } /* diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalMaxScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalMaxScaleFuncHolder.java index 2aff09f5c0c..e0b8f9aefbb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalMaxScaleFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalMaxScaleFuncHolder.java @@ -28,15 +28,11 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; -public class DrillDecimalMaxScaleFuncHolder extends DrillSimpleFuncHolder{ +public class DrillDecimalMaxScaleFuncHolder extends DrillSimpleFuncHolder { - - public DrillDecimalMaxScaleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionTemplate.FunctionCostCategory costCategory, Class drillFuncClass) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, - methods, imports, costCategory, drillFuncClass); - } + public DrillDecimalMaxScaleFuncHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); + } @Override public MajorType getReturnType(List args) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalModScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalModScaleFuncHolder.java index 5faec84f577..b373fa9d2e5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalModScaleFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalModScaleFuncHolder.java @@ -32,11 +32,8 @@ public class DrillDecimalModScaleFuncHolder extends DrillSimpleFuncHolder{ - - public DrillDecimalModScaleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionTemplate.FunctionCostCategory costCategory, Class drillFuncClass) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, costCategory, drillFuncClass); + public DrillDecimalModScaleFuncHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); } /* diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSetScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSetScaleFuncHolder.java index 8e6bdb55383..a10d6eacc0b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSetScaleFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSetScaleFuncHolder.java @@ -18,24 +18,17 @@ package org.apache.drill.exec.expr.fn; import java.util.List; -import java.util.Map; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.MajorType; -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; public class DrillDecimalSetScaleFuncHolder extends DrillSimpleFuncHolder{ - - public DrillDecimalSetScaleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionTemplate.FunctionCostCategory costCategory, Class drillFuncClass) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, costCategory, drillFuncClass); + public DrillDecimalSetScaleFuncHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumAggFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumAggFuncHolder.java index 2988be557df..89ea7cc1b18 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumAggFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumAggFuncHolder.java @@ -18,15 +18,14 @@ package org.apache.drill.exec.expr.fn; import java.util.List; -import java.util.Map; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; public class DrillDecimalSumAggFuncHolder extends DrillAggFuncHolder { - public DrillDecimalSumAggFuncHolder(FunctionTemplate.FunctionScope scope, FunctionTemplate.NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, Map methods, List imports) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports); + + public DrillDecimalSumAggFuncHolder(FunctionAttributes attributes, FunctionInitializer initializer) { + super(attributes, initializer); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java index 24b1eb00063..d45e67490b3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalSumScaleFuncHolder.java @@ -18,26 +18,20 @@ package org.apache.drill.exec.expr.fn; import java.util.List; -import java.util.Map; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.util.DecimalScalePrecisionMulFunction; -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; import org.apache.drill.exec.util.DecimalUtility; public class DrillDecimalSumScaleFuncHolder extends DrillSimpleFuncHolder{ - public DrillDecimalSumScaleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionTemplate.FunctionCostCategory costCategory, Class drillFuncClass) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, costCategory, drillFuncClass); - } + public DrillDecimalSumScaleFuncHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); + } @Override public MajorType getReturnType(List args) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalZeroScaleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalZeroScaleFuncHolder.java index 6fb9ed67666..f941750b2bc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalZeroScaleFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillDecimalZeroScaleFuncHolder.java @@ -30,11 +30,8 @@ public class DrillDecimalZeroScaleFuncHolder extends DrillSimpleFuncHolder{ - - public DrillDecimalZeroScaleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionTemplate.FunctionCostCategory costCategory, Class drillFuncClass) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, costCategory, drillFuncClass); + public DrillDecimalZeroScaleFuncHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); } /* This function scope is used when we need to remove the scale part. diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java index 4e0f1671d90..fea3774126b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java @@ -19,9 +19,9 @@ import java.util.Arrays; import java.util.List; -import java.util.Map; import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.FunctionHolderExpression; import org.apache.drill.common.expression.LogicalExpression; @@ -37,57 +37,67 @@ import org.apache.drill.exec.expr.DrillFuncHolderExpr; import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionCostCategory; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; import org.apache.drill.exec.ops.UdfUtilities; import org.apache.drill.exec.vector.complex.reader.FieldReader; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.sun.codemodel.JBlock; import com.sun.codemodel.JExpr; import com.sun.codemodel.JType; import com.sun.codemodel.JVar; - public abstract class DrillFuncHolder extends AbstractFuncHolder { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class); + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFuncHolder.class); + private final FunctionAttributes attributes; protected final FunctionTemplate.FunctionScope scope; protected final FunctionTemplate.NullHandling nullHandling; protected final FunctionTemplate.FunctionCostCategory costCategory; protected final boolean isBinaryCommutative; protected final boolean isDeterministic; protected final String[] registeredNames; - protected final ImmutableList imports; protected final WorkspaceReference[] workspaceVars; protected final ValueReference[] parameters; protected final ValueReference returnValue; - protected final ImmutableMap methodMap; + private final FunctionInitializer initializer; - public DrillFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, - WorkspaceReference[] workspaceVars, Map methods, List imports, FunctionCostCategory costCategory) { + public DrillFuncHolder( + FunctionAttributes attributes, + FunctionInitializer initializer) { super(); - this.scope = scope; - this.nullHandling = nullHandling; - this.workspaceVars = workspaceVars; - this.isBinaryCommutative = isBinaryCommutative; - this.isDeterministic = ! isRandom; - this.registeredNames = registeredNames; - this.methodMap = ImmutableMap.copyOf(methods); - this.parameters = parameters; - this.returnValue = returnValue; - this.imports = ImmutableList.copyOf(imports); - this.costCategory = costCategory; + this.attributes = attributes; + this.scope = attributes.getScope(); + this.nullHandling = attributes.getNullHandling(); + this.costCategory = attributes.getCostCategory(); + this.isBinaryCommutative = attributes.isBinaryCommutative(); + this.isDeterministic = attributes.isDeterministic(); + this.registeredNames = attributes.getRegisteredNames(); + this.workspaceVars = attributes.getWorkspaceVars(); + this.parameters = attributes.getParameters(); + this.returnValue = attributes.getReturnValue(); + this.initializer = initializer; } - public List getImports() { - return imports; + protected String meth(String methodName) { + return meth(methodName, true); + } + + protected String meth(String methodName, boolean required) { + String method = initializer.getMethod(methodName); + if (method == null) { + if (!required) { + return ""; + } + throw UserException + .functionError() + .message("Failure while trying use function. No body found for required method %s.", methodName) + .addContext("FunctionClass", initializer.getClassName()) + .build(logger); + } + return method; } @Override @@ -115,16 +125,8 @@ public boolean isAggregating() { return false; } - /** - * @deprecated - use isDeterministic method instead - */ - @Deprecated - public boolean isRandom() { - return ! isDeterministic; - } - public boolean isDeterministic() { - return isDeterministic; + return attributes.isDeterministic(); } protected JVar[] declareWorkspaceVariables(ClassGenerator g) { @@ -271,19 +273,19 @@ public MajorType getReturnType(List args) { } public NullHandling getNullHandling() { - return this.nullHandling; + return attributes.getNullHandling(); } private boolean softCompare(MajorType a, MajorType b) { - return Types.softEquals(a, b, nullHandling == NullHandling.NULL_IF_NULL); + return Types.softEquals(a, b, getNullHandling() == NullHandling.NULL_IF_NULL); } public String[] getRegisteredNames() { - return registeredNames; + return attributes.getRegisteredNames(); } public int getCostCategory() { - return this.costCategory.getValue(); + return attributes.getCostCategory().getValue(); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionRegistry.java index 398295e4805..05439b36ec3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionRegistry.java @@ -17,22 +17,22 @@ */ package org.apache.drill.exec.expr.fn; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map.Entry; import java.util.Set; -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.common.util.PathScanner; -import org.apache.drill.exec.ExecConstants; +import org.apache.calcite.sql.SqlOperator; +import org.apache.drill.common.scanner.persistence.AnnotatedClassDescriptor; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.exec.expr.DrillFunc; import org.apache.drill.exec.planner.logical.DrillConstExecutor; import org.apache.drill.exec.planner.sql.DrillOperatorTable; import org.apache.drill.exec.planner.sql.DrillSqlAggOperator; import org.apache.drill.exec.planner.sql.DrillSqlOperator; -import org.apache.calcite.sql.SqlOperator; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Sets; @@ -48,11 +48,11 @@ public class DrillFunctionRegistry { */ private HashMap functionSignatureMap = new HashMap<>(); - public DrillFunctionRegistry(DrillConfig config) { + public DrillFunctionRegistry(ScanResult classpathScan) { FunctionConverter converter = new FunctionConverter(); - Set> providerClasses = PathScanner.scanForImplementations(DrillFunc.class, config.getStringList(ExecConstants.FUNCTION_PACKAGES)); - for (Class clazz : providerClasses) { - DrillFuncHolder holder = converter.getHolder(clazz); + List providerClasses = classpathScan.getAnnotatedClasses(); + for (AnnotatedClassDescriptor func : providerClasses) { + DrillFuncHolder holder = converter.getHolder(func); if (holder != null) { // register handle for each name the function can be referred to String[] names = holder.getRegisteredNames(); @@ -66,19 +66,20 @@ public DrillFunctionRegistry(DrillConfig config) { String functionName = name.toLowerCase(); methods.put(functionName, holder); String functionSignature = functionName + functionInput; - String existingImplementation; if ((existingImplementation = functionSignatureMap.get(functionSignature)) != null) { - throw new AssertionError(String.format("Conflicting functions with similar signature found. Func Name: %s, Class name: %s " + - " Class name: %s", functionName, clazz.getName(), existingImplementation)); - } else if (holder.isAggregating() && holder.isRandom() ) { - logger.warn("Aggregate functions cannot be random, did not register function {}", clazz.getName()); + throw new AssertionError( + String.format( + "Conflicting functions with similar signature found. Func Name: %s, Class name: %s " + + " Class name: %s", functionName, func.getClassName(), existingImplementation)); + } else if (holder.isAggregating() && !holder.isDeterministic() ) { + logger.warn("Aggregate functions must be deterministic, did not register function {}", func.getClassName()); } else { - functionSignatureMap.put(functionSignature, clazz.getName()); + functionSignatureMap.put(functionSignature, func.getClassName()); } } } else { - logger.warn("Unable to initialize function for class {}", clazz.getName()); + logger.warn("Unable to initialize function for class {}", func.getClassName()); } } if (logger.isTraceEnabled()) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java index 7e0f9c81d67..859f2704ba4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java @@ -17,8 +17,7 @@ */ package org.apache.drill.exec.expr.fn; -import java.util.List; -import java.util.Map; +import static com.google.common.base.Preconditions.checkNotNull; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.types.TypeProtos.DataMode; @@ -27,11 +26,8 @@ import org.apache.drill.exec.expr.ClassGenerator.BlockType; import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer; import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionCostCategory; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; -import com.google.common.base.Preconditions; import com.sun.codemodel.JBlock; import com.sun.codemodel.JConditional; import com.sun.codemodel.JExpr; @@ -39,32 +35,27 @@ import com.sun.codemodel.JMod; import com.sun.codemodel.JVar; -public class DrillSimpleFuncHolder extends DrillFuncHolder{ +public class DrillSimpleFuncHolder extends DrillFuncHolder { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSimpleFuncHolder.class); - private final String setupBody; - private final String evalBody; - private final String resetBody; - private final String cleanupBody; - private final Class drillFuncClass; + private final String drillFuncClass; - public DrillSimpleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports) { - this(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, FunctionCostCategory.getDefault(), null); + public DrillSimpleFuncHolder(FunctionAttributes functionAttributes, FunctionInitializer initializer) { + super(functionAttributes, initializer); + drillFuncClass = checkNotNull(initializer.getClassName()); } - public DrillSimpleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom, - String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, - Map methods, List imports, FunctionCostCategory costCategory, Class drillFuncClass) { - super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports, costCategory); - setupBody = methods.get("setup"); - evalBody = methods.get("eval"); - resetBody = methods.get("reset"); - cleanupBody = methods.get("cleanup"); - Preconditions.checkNotNull(evalBody); - - this.drillFuncClass = drillFuncClass; + private String setupBody() { + return meth("setup", false); + } + private String evalBody() { + return meth("eval"); + } + private String resetBody() { + return meth("reset", false); + } + private String cleanupBody() { + return meth("cleanup", false); } @Override @@ -73,10 +64,10 @@ public boolean isNested() { } public DrillSimpleFunc createInterpreter() throws Exception { - Preconditions.checkArgument(this.drillFuncClass != null, "drillFuncClass should not be null!"); - return drillFuncClass.newInstance(); + return (DrillSimpleFunc)Class.forName(drillFuncClass).newInstance(); } + @Override public HoldingContainer renderEnd(ClassGenerator g, HoldingContainer[] inputVariables, JVar[] workspaceJVars){ //If the function's annotation specifies a parameter has to be constant expression, but the HoldingContainer //for the argument is not, then raise exception. @@ -85,10 +76,10 @@ public HoldingContainer renderEnd(ClassGenerator g, HoldingContainer[] inputV throw new DrillRuntimeException(String.format("The argument '%s' of Function '%s' has to be constant!", parameters[i].name, this.getRegisteredNames()[0])); } } - generateBody(g, BlockType.SETUP, setupBody, inputVariables, workspaceJVars, true); - HoldingContainer c = generateEvalBody(g, inputVariables, evalBody, workspaceJVars); - generateBody(g, BlockType.RESET, resetBody, null, workspaceJVars, false); - generateBody(g, BlockType.CLEANUP, cleanupBody, null, workspaceJVars, false); + generateBody(g, BlockType.SETUP, setupBody(), inputVariables, workspaceJVars, true); + HoldingContainer c = generateEvalBody(g, inputVariables, evalBody(), workspaceJVars); + generateBody(g, BlockType.RESET, resetBody(), null, workspaceJVars, false); + generateBody(g, BlockType.CLEANUP, cleanupBody(), null, workspaceJVars, false); return c; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionAttributes.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionAttributes.java new file mode 100644 index 00000000000..4c58617d505 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionAttributes.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn; + +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionCostCategory; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; +import org.apache.drill.exec.expr.fn.DrillFuncHolder.ValueReference; +import org.apache.drill.exec.expr.fn.DrillFuncHolder.WorkspaceReference; + +/** + * Attributes of a function + * Those are used in code generation and optimization. + */ +public class FunctionAttributes { + private final FunctionScope scope; + private final NullHandling nullHandling; + private final boolean isBinaryCommutative; + private final boolean isDeterministic; + private final String[] registeredNames; + private final ValueReference[] parameters; + private final ValueReference returnValue; + private final WorkspaceReference[] workspaceVars; + private final FunctionCostCategory costCategory; + + public FunctionAttributes(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, + boolean isDeteministic, String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, + WorkspaceReference[] workspaceVars, FunctionCostCategory costCategory) { + super(); + this.scope = scope; + this.nullHandling = nullHandling; + this.isBinaryCommutative = isBinaryCommutative; + this.isDeterministic = isDeteministic; + this.registeredNames = registeredNames; + this.parameters = parameters; + this.returnValue = returnValue; + this.workspaceVars = workspaceVars; + this.costCategory = costCategory; + } + + public FunctionScope getScope() { + return scope; + } + + public NullHandling getNullHandling() { + return nullHandling; + } + + public boolean isBinaryCommutative() { + return isBinaryCommutative; + } + + @Deprecated + public boolean isRandom() { + return !isDeterministic; + } + + public boolean isDeterministic() { + return isDeterministic; + } + + public String[] getRegisteredNames() { + return registeredNames; + } + + public ValueReference[] getParameters() { + return parameters; + } + + public ValueReference getReturnValue() { + return returnValue; + } + + public WorkspaceReference[] getWorkspaceVars() { + return workspaceVars; + } + + public FunctionCostCategory getCostCategory() { + return costCategory; + } + + +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java index ef8a5b186c3..00be7aa0f0e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java @@ -17,22 +17,18 @@ */ package org.apache.drill.exec.expr.fn; -import com.google.common.base.Joiner; - -import java.io.IOException; -import java.io.InputStream; -import java.io.StringReader; +import java.lang.annotation.Annotation; import java.lang.reflect.Field; -import java.net.URL; +import java.util.Arrays; import java.util.List; -import java.util.Map; import javax.inject.Inject; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.scanner.persistence.FieldDescriptor; +import org.apache.drill.common.scanner.persistence.AnnotatedClassDescriptor; import org.apache.drill.common.types.TypeProtos.MajorType; -import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.expr.DrillFunc; -import org.apache.drill.exec.expr.DrillSimpleFunc; import org.apache.drill.exec.expr.annotations.FunctionTemplate; import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; import org.apache.drill.exec.expr.annotations.Output; @@ -44,16 +40,9 @@ import org.apache.drill.exec.ops.UdfUtilities; import org.apache.drill.exec.vector.complex.reader.FieldReader; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; -import org.codehaus.commons.compiler.CompileException; -import org.codehaus.janino.Java.CompilationUnit; -import org.codehaus.janino.Parser; -import org.codehaus.janino.Scanner; -import org.mortbay.util.IO; +import com.google.common.base.Joiner; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.io.InputSupplier; -import com.google.common.io.Resources; /** * Converts FunctionCalls to Java Expressions. @@ -61,59 +50,21 @@ public class FunctionConverter { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionConverter.class); - private Map functionUnits = Maps.newHashMap(); - - private CompilationUnit get(Class c) throws IOException{ - String path = c.getName(); - path = path.replaceFirst("\\$.*", ""); - path = path.replace(".", FileUtils.separator); - path = "/" + path + ".java"; - CompilationUnit cu = functionUnits.get(path); - if(cu != null) { - return cu; - } - - URL u = Resources.getResource(c, path); - InputSupplier supplier = Resources.newInputStreamSupplier(u); - try (InputStream is = supplier.getInput()) { - if (is == null) { - throw new IOException(String.format("Failure trying to located source code for Class %s, tried to read on classpath location %s", c.getName(), path)); - } - String body = IO.toString(is); - - //TODO: Hack to remove annotations so Janino doesn't choke. Need to reconsider this problem... - body = body.replaceAll("@\\w+(?:\\([^\\\\]*?\\))?", ""); - try{ - cu = new Parser(new Scanner(null, new StringReader(body))).parseCompilationUnit(); - functionUnits.put(path, cu); - return cu; - } catch (CompileException e) { - logger.warn("Failure while parsing function class:\n{}", body, e); - return null; - } - - } - - } - - /** - * Get the name of the class used for interpreted expression evaluation. - * - * @return - class name of interpreted evaluator - */ - private String getInterpreterClassName(Class clazz) { - return clazz.getName(); - } - - public DrillFuncHolder getHolder(Class clazz) { - FunctionTemplate template = clazz.getAnnotation(FunctionTemplate.class); + public DrillFuncHolder getHolder(AnnotatedClassDescriptor func) { + FunctionTemplate template = func.getAnnotationProxy(FunctionTemplate.class); if (template == null) { - return failure("Class does not declare FunctionTemplate annotation.", clazz); + return failure("Class does not declare FunctionTemplate annotation.", func); } - if ((template.name().isEmpty() && template.names().length == 0) || // none set - (!template.name().isEmpty() && template.names().length != 0)) { // both are set - return failure("Must use only one annotations 'name' or 'names', not both", clazz); + String name = template.name(); + List names = Arrays.asList(template.names()); + if (name.isEmpty() && names.isEmpty()) { + // none set + return failure("Must define 'name' or 'names'", func); + } + if (!name.isEmpty() && !names.isEmpty()) { + // both are set + return failure("Must use only one annotations 'name' or 'names', not both", func); } // start by getting field information. @@ -122,45 +73,39 @@ public DrillFuncHolder getHolder(Class clazz) { ValueReference outputField = null; - - for (Field field : clazz.getDeclaredFields()) { - - Param param = field.getAnnotation(Param.class); - Output output = field.getAnnotation(Output.class); - Workspace workspace = field.getAnnotation(Workspace.class); - Inject inject = field.getAnnotation(Inject.class); - - int i =0; - if (param != null) { - i++; - } - if (output != null) { - i++; - } - if (workspace != null) { - i++; - } - if (inject != null) { - i++; + for (FieldDescriptor field : func.getFields()) { + Param param = field.getAnnotationProxy(Param.class); + Output output = field.getAnnotationProxy(Output.class); + Workspace workspace = field.getAnnotationProxy(Workspace.class); + Inject inject = field.getAnnotationProxy(Inject.class); + + Annotation[] annotations = {param, output, workspace, inject}; + int annotationCount = 0; + for (Annotation annotationDescriptor : annotations) { + if (annotationDescriptor != null) { + annotationCount += 1; + } } - if (i == 0) { - return failure("The field must be either a @Param, @Output, @Inject or @Workspace field.", clazz, field); - } else if(i > 1) { - return failure("The field must be only one of @Param, @Output, @Inject or @Workspace. It currently has more than one of these annotations.", clazz, field); + if (annotationCount == 0) { + return failure("The field must be either a @Param, @Output, @Inject or @Workspace field.", func, field); + } else if(annotationCount > 1) { + return failure("The field must be only one of @Param, @Output, @Inject or @Workspace. It currently has more than one of these annotations.", func, field); } + // TODO(Julien): verify there are a few of those and we can load them + Class fieldClass = field.getFieldClass(); if (param != null || output != null) { // Special processing for @Param FieldReader - if (param != null && FieldReader.class.isAssignableFrom(field.getType())) { + if (param != null && FieldReader.class.isAssignableFrom(fieldClass)) { params.add(ValueReference.createFieldReaderRef(field.getName())); continue; } // Special processing for @Output ComplexWriter - if (output != null && ComplexWriter.class.isAssignableFrom(field.getType())) { + if (output != null && ComplexWriter.class.isAssignableFrom(fieldClass)) { if (outputField != null) { - return failure("You've declared more than one @Output field. You must declare one and only @Output field per Function class.", clazz, field); + return failure("You've declared more than one @Output field. You must declare one and only @Output field per Function class.", func, field); }else{ outputField = ValueReference.createComplexWriterRef(field.getName()); } @@ -168,28 +113,25 @@ public DrillFuncHolder getHolder(Class clazz) { } // check that param and output are value holders. - if (!ValueHolder.class.isAssignableFrom(field.getType())) { - return failure(String.format("The field doesn't holds value of type %s which does not implement the ValueHolder interface. All fields of type @Param or @Output must extend this interface..", field.getType()), clazz, field); + if (!ValueHolder.class.isAssignableFrom(fieldClass)) { + return failure(String.format("The field doesn't holds value of type %s which does not implement the ValueHolder interface. All fields of type @Param or @Output must extend this interface..", fieldClass), func, field); } // get the type field from the value holder. MajorType type = null; try { - type = getStaticFieldValue("TYPE", field.getType(), MajorType.class); + type = getStaticFieldValue("TYPE", fieldClass, MajorType.class); } catch (Exception e) { - return failure("Failure while trying to access the ValueHolder's TYPE static variable. All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, clazz, field.getName()); + return failure("Failure while trying to access the ValueHolder's TYPE static variable. All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, func, field); } - ValueReference p = new ValueReference(type, field.getName()); if (param != null) { - if (param.constant()) { - p.setConstant(true); - } + p.setConstant(param.constant()); params.add(p); } else { if (outputField != null) { - return failure("You've declared more than one @Output field. You must declare one and only @Output field per Function class.", clazz, field); + return failure("You've declared more than one @Output field. You must declare one and only @Output field per Function class.", func, field); } else { outputField = p; } @@ -198,23 +140,28 @@ public DrillFuncHolder getHolder(Class clazz) { } else { // workspace work. boolean isInject = inject != null; - if (isInject && UdfUtilities.INJECTABLE_GETTER_METHODS.get(field.getType()) == null) { - return failure(String.format("A %s cannot be injected into a %s, available injectable classes are: %s.", - field.getType(), DrillFunc.class.getSimpleName(), Joiner.on(",").join(UdfUtilities.INJECTABLE_GETTER_METHODS.keySet())), clazz, field); + if (isInject && UdfUtilities.INJECTABLE_GETTER_METHODS.get(fieldClass) == null) { + return failure( + String.format( + "A %s cannot be injected into a %s," + + " available injectable classes are: %s.", + fieldClass, DrillFunc.class.getSimpleName(), + Joiner.on(",").join(UdfUtilities.INJECTABLE_GETTER_METHODS.keySet()) + ), func, field); } - WorkspaceReference wsReference = new WorkspaceReference(field.getType(), field.getName(), isInject); + WorkspaceReference wsReference = new WorkspaceReference(fieldClass, field.getName(), isInject); - if (!isInject && template.scope() == FunctionScope.POINT_AGGREGATE && !ValueHolder.class.isAssignableFrom(field.getType()) ) { - return failure(String.format("Aggregate function '%s' workspace variable '%s' is of type '%s'. Please change it to Holder type.", template.name(), field.getName(), field.getType()), clazz, field); + if (!isInject && template.scope() == FunctionScope.POINT_AGGREGATE && !ValueHolder.class.isAssignableFrom(fieldClass) ) { + return failure(String.format("Aggregate function '%s' workspace variable '%s' is of type '%s'. Please change it to Holder type.", func.getClassName(), field.getName(), fieldClass), func, field); } //If the workspace var is of Holder type, get its MajorType and assign to WorkspaceReference. - if (ValueHolder.class.isAssignableFrom(field.getType())) { + if (ValueHolder.class.isAssignableFrom(fieldClass)) { MajorType majorType = null; try { - majorType = getStaticFieldValue("TYPE", field.getType(), MajorType.class); + majorType = getStaticFieldValue("TYPE", fieldClass, MajorType.class); } catch (Exception e) { - return failure("Failure while trying to access the ValueHolder's TYPE static variable. All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, clazz, field.getName()); + return failure("Failure while trying to access the ValueHolder's TYPE static variable. All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, func, field); } wsReference.setMajorType(majorType); } @@ -222,130 +169,65 @@ public DrillFuncHolder getHolder(Class clazz) { } } - // if (!workspaceFields.isEmpty()) return failure("This function declares one or more workspace fields. However, those have not yet been implemented.", clazz); if (outputField == null) { - return failure("This function declares zero output fields. A function must declare one output field.", clazz); - } - - // get function body. - - CompilationUnit cu; - try { - cu = get(clazz); - if (cu == null) { - return null; - } - } catch (IOException e) { - return failure("Failure while getting class body.", e, clazz); + return failure("This function declares zero output fields. A function must declare one output field.", func); } - + FunctionInitializer initializer = new FunctionInitializer(func.getClassName()); try{ - Map methods = MethodGrabbingVisitor.getMethods(cu, clazz); - List imports = ImportGrabber.getMethods(cu); // return holder ValueReference[] ps = params.toArray(new ValueReference[params.size()]); WorkspaceReference[] works = workspaceFields.toArray(new WorkspaceReference[workspaceFields.size()]); String[] registeredNames = ((template.name().isEmpty()) ? template.names() : new String[] {template.name()} ); + boolean isDeteministic = ! template.isRandom(); + FunctionAttributes functionAttributes = new FunctionAttributes( + template.scope(), + template.nulls(), + template.isBinaryCommutative(), + isDeteministic, registeredNames, ps, outputField, works, template.costCategory()); switch (template.scope()) { case POINT_AGGREGATE: - return new DrillAggFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports, template.costCategory()); + return new DrillAggFuncHolder(functionAttributes, initializer); case DECIMAL_AGGREGATE: - return new DrillDecimalAggFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports); + return new DrillDecimalAggFuncHolder(functionAttributes, initializer); case DECIMAL_SUM_AGGREGATE: - return new DrillDecimalSumAggFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports); + return new DrillDecimalSumAggFuncHolder(functionAttributes, initializer); case SIMPLE: if (outputField.isComplexWriter) { - return new DrillComplexWriterFuncHolder(template.scope(), template.nulls(), - template.isBinaryCommutative(), - template.isRandom(), registeredNames, - ps, outputField, works, methods, imports); + return new DrillComplexWriterFuncHolder(functionAttributes, initializer); } else { - return new DrillSimpleFuncHolder(template.scope(), template.nulls(), - template.isBinaryCommutative(), - template.isRandom(), registeredNames, - ps, outputField, works, methods, imports, template.costCategory(), - (Class)clazz - ); + return new DrillSimpleFuncHolder(functionAttributes, initializer); } case SC_BOOLEAN_OPERATOR: - return new DrillBooleanOPHolder(template.scope(), template.nulls(), - template.isBinaryCommutative(), - template.isRandom(), registeredNames, - ps, outputField, works, methods, imports, FunctionTemplate.FunctionCostCategory.getDefault(), (Class)clazz); - + return new DrillBooleanOPHolder(functionAttributes, initializer); case DECIMAL_MAX_SCALE: - return new DrillDecimalMaxScaleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports, - FunctionTemplate.FunctionCostCategory.getDefault(), (Class)clazz); + return new DrillDecimalMaxScaleFuncHolder(functionAttributes, initializer); case DECIMAL_MUL_SCALE: - return new DrillDecimalSumScaleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports, - FunctionTemplate.FunctionCostCategory.getDefault(), (Class)clazz); + return new DrillDecimalSumScaleFuncHolder(functionAttributes, initializer); case DECIMAL_ADD_SCALE: - return new DrillDecimalAddFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports, - FunctionTemplate.FunctionCostCategory.getDefault(), (Class)clazz); + return new DrillDecimalAddFuncHolder(functionAttributes, initializer); case DECIMAL_CAST: - return new DrillDecimalCastFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports, - FunctionTemplate.FunctionCostCategory.getDefault(), (Class)clazz); + return new DrillDecimalCastFuncHolder(functionAttributes, initializer); case DECIMAL_DIV_SCALE: - return new DrillDecimalDivScaleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports, - FunctionTemplate.FunctionCostCategory.getDefault(), (Class)clazz); + return new DrillDecimalDivScaleFuncHolder(functionAttributes, initializer); case DECIMAL_MOD_SCALE: - return new DrillDecimalModScaleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports, - FunctionTemplate.FunctionCostCategory.getDefault(), (Class)clazz); + return new DrillDecimalModScaleFuncHolder(functionAttributes, initializer); case DECIMAL_SET_SCALE: - return new DrillDecimalSetScaleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports, - FunctionTemplate.FunctionCostCategory.getDefault(), (Class)clazz); + return new DrillDecimalSetScaleFuncHolder(functionAttributes, initializer); case DECIMAL_ZERO_SCALE: - return new DrillDecimalZeroScaleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), - template.isRandom(), registeredNames, ps, outputField, works, methods, imports, - FunctionTemplate.FunctionCostCategory.getDefault(), (Class)clazz); + return new DrillDecimalZeroScaleFuncHolder(functionAttributes, initializer); case HOLISTIC_AGGREGATE: case RANGE_AGGREGATE: default: - return failure("Unsupported Function Type.", clazz); + return failure("Unsupported Function Type.", func); } } catch (Exception | NoSuchFieldError | AbstractMethodError ex) { - return failure("Failure while creating function holder.", ex, clazz); - } - - } - - - - private String getClassBody(Class c) throws CompileException, IOException{ - String path = c.getName(); - path = path.replaceFirst("\\$.*", ""); - path = path.replace(".", FileUtils.separator); - path = "/" + path + ".java"; - URL u = Resources.getResource(c, path); - InputSupplier supplier = Resources.newInputStreamSupplier(u); - try (InputStream is = supplier.getInput()) { - if (is == null) { - throw new IOException(String.format("Failure trying to located source code for Class %s, tried to read on classpath location %s", c.getName(), path)); - } - String body = IO.toString(is); - - //TODO: Hack to remove annotations so Janino doesn't choke. Need to reconsider this problem... - //return body.replaceAll("@(?:Output|Param|Workspace|Override|SuppressWarnings\\([^\\\\]*?\\)|FunctionTemplate\\([^\\\\]*?\\))", ""); - return body.replaceAll("@(?:\\([^\\\\]*?\\))?", ""); + return failure("Failure while creating function holder.", ex, func); } - } - - @SuppressWarnings("unchecked") private T getStaticFieldValue(String fieldName, Class valueType, Class c) throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException{ Field f = valueType.getDeclaredField(fieldName); @@ -353,28 +235,36 @@ private T getStaticFieldValue(String fieldName, Class valueType, Class return (T) val; } - private static DrillFuncHolder failure(String message, Throwable t, Class clazz, String fieldName) { - logger.warn("Failure loading function class {}, field {}. " + message, clazz.getName(), fieldName, t); - return null; + private static DrillFuncHolder failure(String message, Throwable t, AnnotatedClassDescriptor func, FieldDescriptor field) { + return fieldFailure(message, t, func.getClassName(), field.getName()); } - private DrillFuncHolder failure(String message, Class clazz, String fieldName) { - logger.warn("Failure loading function class {}, field {}. " + message, clazz.getName(), fieldName); - return null; + private static DrillFuncHolder failure(String message, AnnotatedClassDescriptor func, FieldDescriptor field) { + return fieldFailure(message, null, func.getClassName(), field.getName()); } - private DrillFuncHolder failure(String message, Class clazz) { - logger.warn("Failure loading function class [{}]. Message: {}", clazz.getName(), message); - return null; + private DrillFuncHolder failure(String message, AnnotatedClassDescriptor func) { + return classFailure(message, null, func.getClassName()); } - private DrillFuncHolder failure(String message, Throwable t, Class clazz) { - logger.warn("Failure loading function class [{}]. Message: {}", clazz.getName(), message, t); - return null; + private DrillFuncHolder failure(String message, Throwable t, AnnotatedClassDescriptor func) { + return classFailure(message, t, func.getClassName()); } - private DrillFuncHolder failure(String message, Class clazz, Field field) { - return failure(message, clazz, field.getName()); + private static DrillFuncHolder classFailure(String message, Throwable t, String funcClassName) { + return failure(String.format("Failure loading function class [%s]. Message: %s", funcClassName, message), t); + } + + private static DrillFuncHolder fieldFailure(String message, Throwable t, String funcClassName, String fieldName) { + return failure(String.format("Failure loading function class %s, field %s. Message: %s", funcClassName, fieldName, message), t); + } + + private static DrillFuncHolder failure(String message, Throwable t) { + if (t == null) { + t = new DrillRuntimeException(message); + } + logger.warn(message, t); + return null; } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java index 695cd461fd3..6be615d8e33 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java @@ -26,16 +26,17 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.fn.CastFunctions; -import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.common.types.TypeProtos.MajorType; -import org.apache.drill.common.util.PathScanner; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.planner.sql.DrillOperatorTable; import org.apache.drill.exec.resolver.FunctionResolver; +import org.apache.drill.exec.server.options.OptionManager; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; -import org.apache.drill.exec.server.options.OptionManager; public class FunctionImplementationRegistry implements FunctionLookupContext { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class); @@ -44,14 +45,19 @@ public class FunctionImplementationRegistry implements FunctionLookupContext { private List pluggableFuncRegistries = Lists.newArrayList(); private OptionManager optionManager = null; + @Deprecated @VisibleForTesting public FunctionImplementationRegistry(DrillConfig config){ + this(config, ClassPathScanner.fromPrescan(config)); + } + + public FunctionImplementationRegistry(DrillConfig config, ScanResult classpathScan){ Stopwatch w = new Stopwatch().start(); logger.debug("Generating function registry."); - drillFuncRegistry = new DrillFunctionRegistry(config); + drillFuncRegistry = new DrillFunctionRegistry(classpathScan); - Set> registryClasses = PathScanner.scanForImplementations( - PluggableFunctionRegistry.class, config.getStringList(ExecConstants.FUNCTION_PACKAGES)); + Set> registryClasses = + classpathScan.getImplementations(PluggableFunctionRegistry.class); for (Class clazz : registryClasses) { for (Constructor c : clazz.getConstructors()) { @@ -75,8 +81,8 @@ public FunctionImplementationRegistry(DrillConfig config){ logger.info("Function registry loaded. {} functions loaded in {} ms.", drillFuncRegistry.size(), w.elapsed(TimeUnit.MILLISECONDS)); } - public FunctionImplementationRegistry(DrillConfig config, OptionManager optionManager) { - this(config); + public FunctionImplementationRegistry(DrillConfig config, ScanResult classpathScan, OptionManager optionManager) { + this(config, classpathScan); this.optionManager = optionManager; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java new file mode 100644 index 00000000000..55d5d92823f --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionInitializer.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.expr.fn; + +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; +import java.net.URL; +import java.util.List; +import java.util.Map; + +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.common.util.FileUtils; +import org.codehaus.commons.compiler.CompileException; +import org.codehaus.janino.Java.CompilationUnit; +import org.codehaus.janino.Parser; +import org.codehaus.janino.Scanner; +import org.mortbay.util.IO; + +import com.google.common.collect.Maps; +import com.google.common.io.InputSupplier; +import com.google.common.io.Resources; + +/** + * To avoid the cost of initializing all functions up front, + * this class contains all informations required to initializing a function when it is used. + */ +public class FunctionInitializer { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionInitializer.class); + + private String className; + + private Map functionUnits = Maps.newHashMap(); + private Map methods; + private List imports; + private volatile boolean ready; + + /** + * @param className the fully qualified name of the class implementing the function + */ + public FunctionInitializer(String className) { + super(); + this.className = className; + } + + /** + * @return the fully qualified name of the class implementing the function + */ + public String getClassName() { + return className; + } + + /** + * @return the imports of this class (for java code gen) + */ + public List getImports() { + checkInit(); + return imports; + } + + /** + * @param methodName + * @return the content of the method (for java code gen inlining) + */ + public String getMethod(String methodName) { + checkInit(); + return methods.get(methodName); + } + + private void checkInit() { + if (ready) { + return; + } + + synchronized (this) { + if (ready) { + return; + } + + // get function body. + + try { + final Class clazz = Class.forName(className); + final CompilationUnit cu = get(clazz); + + if (cu == null) { + throw new IOException(String.format("Failure while loading class %s.", clazz.getName())); + } + + methods = MethodGrabbingVisitor.getMethods(cu, clazz); + imports = ImportGrabber.getMethods(cu); + + } catch (IOException | ClassNotFoundException e) { + throw UserException.functionError(e) + .message("Failure reading Function class.") + .addContext("Function Class", className) + .build(logger); + } + } + } + + private CompilationUnit get(Class c) throws IOException { + String path = c.getName(); + path = path.replaceFirst("\\$.*", ""); + path = path.replace(".", FileUtils.separator); + path = "/" + path + ".java"; + CompilationUnit cu = functionUnits.get(path); + if (cu != null) { + return cu; + } + + URL u = Resources.getResource(c, path); + InputSupplier supplier = Resources.newInputStreamSupplier(u); + try (InputStream is = supplier.getInput()) { + if (is == null) { + throw new IOException(String.format( + "Failure trying to located source code for Class %s, tried to read on classpath location %s", c.getName(), + path)); + } + String body = IO.toString(is); + + // TODO: Hack to remove annotations so Janino doesn't choke. Need to reconsider this problem... + body = body.replaceAll("@\\w+(?:\\([^\\\\]*?\\))?", ""); + try { + cu = new Parser(new Scanner(null, new StringReader(body))).parseCompilationUnit(); + functionUnits.put(path, cu); + return cu; + } catch (CompileException e) { + logger.warn("Failure while parsing function class:\n{}", body, e); + return null; + } + + } + + } + +} \ No newline at end of file diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java index d5e85eeff90..02580fcd60d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java @@ -145,7 +145,7 @@ public FragmentContext(final DrillbitContext dbContext, final PlanFragment fragm list = new OptionList(); } else { try { - list = dbContext.getConfig().getMapper().readValue(fragment.getOptionsJson(), OptionList.class); + list = dbContext.getLpPersistence().getMapper().readValue(fragment.getOptionsJson(), OptionList.class); } catch (final Exception e) { throw new ExecutionSetupException("Failure while reading plan options.", e); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java index 238907de9d4..b56c16c6a1a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java @@ -28,6 +28,7 @@ import org.apache.drill.common.AutoCloseables; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.common.exceptions.DrillRuntimeException; @@ -196,6 +197,10 @@ public StoragePluginRegistry getStorage() { return drillbitContext.getStorage(); } + public LogicalPlanPersistence getLpPersistence() { + return drillbitContext.getLpPersistence(); + } + public Collection getActiveEndpoints() { return drillbitContext.getBits(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperatorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperatorUtil.java index ba844293caa..0069ca06e12 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperatorUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperatorUtil.java @@ -17,15 +17,14 @@ */ package org.apache.drill.exec.physical.base; -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import org.apache.drill.common.config.CommonConstants; -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.util.PathScanner; +import java.util.List; +import java.util.Set; + +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.exec.physical.MinorFragmentEndpoint; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; -import java.util.List; +import com.google.common.collect.Lists; public class PhysicalOperatorUtil { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PhysicalOperatorUtil.class); @@ -33,14 +32,10 @@ public class PhysicalOperatorUtil { private PhysicalOperatorUtil() {} - public synchronized static Class[] getSubTypes(final DrillConfig config) { - final Class[] ops = - PathScanner.scanForImplementationsArr(PhysicalOperator.class, - config.getStringList(CommonConstants.PHYSICAL_OPERATOR_SCAN_PACKAGES)); - final String lineBrokenList = - ops.length == 0 ? "" : "\n\t- " + Joiner.on("\n\t- ").join(ops); - logger.debug("Found {} physical operator classes: {}.", ops.length, - lineBrokenList); + public static Set> getSubTypes(ScanResult classpathScan) { + final Set> ops = classpathScan.getImplementations(PhysicalOperator.class); + logger.debug("Found {} physical operator classes: {}.", ops.size(), + ops); return ops; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OperatorCreatorRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OperatorCreatorRegistry.java index 0be551b72e0..862aec7f4ef 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OperatorCreatorRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OperatorCreatorRegistry.java @@ -22,11 +22,10 @@ import java.lang.reflect.Type; import java.util.HashMap; import java.util.Map; +import java.util.Set; -import org.apache.drill.common.config.CommonConstants; -import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.util.PathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; public class OperatorCreatorRegistry { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OperatorCreatorRegistry.class); @@ -34,9 +33,9 @@ public class OperatorCreatorRegistry { private volatile Map, Constructor> constructorRegistry = new HashMap, Constructor>(); private volatile Map, Object> instanceRegistry = new HashMap, Object>(); - public OperatorCreatorRegistry(DrillConfig config) { - addImplementorsToMap(config, BatchCreator.class); - addImplementorsToMap(config, RootCreator.class); + public OperatorCreatorRegistry(ScanResult scanResult) { + addImplementorsToMap(scanResult, BatchCreator.class); + addImplementorsToMap(scanResult, RootCreator.class); logger.debug("Adding Operator Creator map: {}", constructorRegistry); } @@ -61,9 +60,8 @@ public synchronized Object getOperatorCreator(Class operator) throws Executio } } - private void addImplementorsToMap(DrillConfig config, Class baseInterface) { - Class[] providerClasses = PathScanner.scanForImplementationsArr(baseInterface, - config.getStringList(CommonConstants.PHYSICAL_OPERATOR_SCAN_PACKAGES)); + private void addImplementorsToMap(ScanResult scanResult, Class baseInterface) { + Set> providerClasses = scanResult.getImplementations(baseInterface); for (Class c : providerClasses) { Class operatorClass = c; boolean interfaceFound = false; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java index 8d77136e14d..f8284aab446 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java @@ -18,9 +18,12 @@ package org.apache.drill.exec.planner; import java.io.IOException; +import java.util.Set; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.logical.LogicalPlan; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.exec.coord.DrillbitEndpointSerDe; import org.apache.drill.exec.physical.PhysicalPlan; @@ -47,7 +50,7 @@ public class PhysicalPlanReader { private final ObjectReader operatorReader; private final ObjectReader logicalPlanReader; - public PhysicalPlanReader(DrillConfig config, ObjectMapper mapper, final DrillbitEndpoint endpoint, + public PhysicalPlanReader(DrillConfig config, ScanResult scanResult, LogicalPlanPersistence lpPersistance, final DrillbitEndpoint endpoint, final StoragePluginRegistry pluginRegistry) { // Endpoint serializer/deserializer. @@ -57,27 +60,22 @@ public PhysicalPlanReader(DrillConfig config, ObjectMapper mapper, final Drillbi .addSerializer(MajorType.class, new MajorTypeSerDe.Se()) .addDeserializer(MajorType.class, new MajorTypeSerDe.De()); - - mapper.registerModule(deserModule); - mapper.registerSubtypes(PhysicalOperatorUtil.getSubTypes(config)); + ObjectMapper lpMapper = lpPersistance.getMapper(); + lpMapper.registerModule(deserModule); + Set> subTypes = PhysicalOperatorUtil.getSubTypes(scanResult); + for (Class subType : subTypes) { + lpMapper.registerSubtypes(subType); + } InjectableValues injectables = new InjectableValues.Std() // .addValue(StoragePluginRegistry.class, pluginRegistry) // .addValue(DrillbitEndpoint.class, endpoint); // - this.mapper = mapper; + this.mapper = lpMapper; this.physicalPlanReader = mapper.reader(PhysicalPlan.class).with(injectables); this.operatorReader = mapper.reader(PhysicalOperator.class).with(injectables); this.logicalPlanReader = mapper.reader(LogicalPlan.class).with(injectables); } - // TODO - we do not want to storage engine registry generated here in production, this was created to keep old - // tests passing, this constructor should be removed and the tests should be updated to use the contstructor - // that takes a storage engine registry - @Deprecated - public PhysicalPlanReader(DrillConfig config, ObjectMapper mapper, final DrillbitEndpoint endpoint) { - this(config, mapper, endpoint, null); - } - public String writeJson(OptionList list) throws JsonProcessingException{ return mapper.writeValueAsString(list); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StoragePlugins.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StoragePlugins.java index 337bb0935f4..d8fd11cd9c1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StoragePlugins.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StoragePlugins.java @@ -25,7 +25,10 @@ import java.util.Map.Entry; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -43,12 +46,14 @@ public StoragePlugins(@JsonProperty("storage") Map public static void main(String[] args) throws Exception{ DrillConfig config = DrillConfig.create(); + ScanResult scanResult = ClassPathScanner.fromPrescan(config); + LogicalPlanPersistence lpp = new LogicalPlanPersistence(config, scanResult); String data = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8); - StoragePlugins se = config.getMapper().readValue(data, StoragePlugins.class); + StoragePlugins se = lpp.getMapper().readValue(data, StoragePlugins.class); ByteArrayOutputStream os = new ByteArrayOutputStream(); - config.getMapper().writeValue(System.out, se); - config.getMapper().writeValue(os, se); - se = config.getMapper().readValue(new ByteArrayInputStream(os.toByteArray()), StoragePlugins.class); + lpp.getMapper().writeValue(System.out, se); + lpp.getMapper().writeValue(os, se); + se = lpp.getMapper().readValue(new ByteArrayInputStream(os.toByteArray()), StoragePlugins.class); System.out.println(se); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java index 1eddcf3d483..9d11b960f46 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java @@ -148,7 +148,7 @@ protected void log(final String name, final Prel node, final Logger logger) { protected void log(final String name, final PhysicalPlan plan, final Logger logger) throws JsonProcessingException { if (logger.isDebugEnabled()) { - String planText = plan.unparse(context.getConfig().getMapper().writer()); + String planText = plan.unparse(context.getLpPersistence().getMapper().writer()); logger.debug(name + " : \n" + planText); } } @@ -248,7 +248,7 @@ protected DrillRel convertToDrel(RelNode relNode, RelDataType validatedRowType) final DrillRel convertedRelNode = convertToDrel(relNode); // Put a non-trivial topProject to ensure the final output field name is preserved, when necessary. - DrillRel topPreservedNameProj = addRenamedProject((DrillRel) convertedRelNode, validatedRowType); + DrillRel topPreservedNameProj = addRenamedProject(convertedRelNode, validatedRowType); return new DrillScreenRel(topPreservedNameProj.getCluster(), topPreservedNameProj.getTraitSet(), topPreservedNameProj); } @@ -339,12 +339,12 @@ protected Prel convertToPrel(RelNode drel) throws RelConversionException, SqlUns /* * 1.2) Break up all expressions with complex outputs into their own project operations */ - phyRelNode = ((Prel) phyRelNode).accept(new SplitUpComplexExpressions(planner.getTypeFactory(), context.getDrillOperatorTable(), context.getPlannerSettings().functionImplementationRegistry), null); + phyRelNode = phyRelNode.accept(new SplitUpComplexExpressions(planner.getTypeFactory(), context.getDrillOperatorTable(), context.getPlannerSettings().functionImplementationRegistry), null); /* * 1.3) Projections that contain reference to flatten are rewritten as Flatten operators followed by Project */ - phyRelNode = ((Prel) phyRelNode).accept(new RewriteProjectToFlatten(planner.getTypeFactory(), context.getDrillOperatorTable()), null); + phyRelNode = phyRelNode.accept(new RewriteProjectToFlatten(planner.getTypeFactory(), context.getDrillOperatorTable()), null); /* * 2.) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java index ac9af9d0fa3..502987cfbf1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java @@ -107,7 +107,7 @@ public LogicalExplain(RelNode node, SqlExplainLevel level, QueryContext context) DrillImplementor implementor = new DrillImplementor(new DrillParseContext(context.getPlannerSettings()), ResultMode.LOGICAL); implementor.go( (DrillRel) node); LogicalPlan plan = implementor.getPlan(); - this.json = plan.unparse(context.getConfig()); + this.json = plan.unparse(context.getLpPersistence()); } } @@ -117,7 +117,7 @@ public static class PhysicalExplain{ public PhysicalExplain(RelNode node, PhysicalPlan plan, SqlExplainLevel level, QueryContext context) { this.text = PrelSequencer.printWithIds((Prel) node, level); - this.json = plan.unparse(context.getConfig().getMapper().writer()); + this.json = plan.unparse(context.getLpPersistence().getMapper().writer()); } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java index 00bbcb4cb2e..df12fe77382 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/BasicServer.java @@ -27,9 +27,14 @@ import io.netty.channel.EventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.handler.timeout.ReadTimeoutHandler; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; import java.io.IOException; import java.net.BindException; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -39,6 +44,8 @@ import org.apache.drill.exec.proto.GeneralRPCProtos.RpcMode; import com.google.protobuf.Internal.EnumLite; +import com.google.common.base.Stopwatch; +import com.google.common.io.Closeables; import com.google.protobuf.MessageLite; import com.google.protobuf.Parser; @@ -216,7 +223,14 @@ public int bind(final int initialPort, boolean allowPortHunting) throws Drillbit @Override public void close() throws IOException { try { - eventLoopGroup.shutdownGracefully(0, 2, TimeUnit.SECONDS).get(); + Stopwatch watch = new Stopwatch().start(); + // this takes 1s to complete + // known issue: https://github.com/netty/netty/issues/2545 + eventLoopGroup.shutdownGracefully(0, 0, TimeUnit.SECONDS).get(); + long elapsed = watch.elapsed(MILLISECONDS); + if (elapsed > 500) { + logger.info("closed eventLoopGroup " + eventLoopGroup + " in " + elapsed + " ms"); + } } catch (final InterruptedException | ExecutionException e) { logger.warn("Failure while shutting down {}. ", this.getClass().getName(), e); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionCreator.java index a76d75391b4..ddb7c8e4a53 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/data/DataConnectionCreator.java @@ -66,6 +66,7 @@ public DataTunnel getTunnel(DrillbitEndpoint endpoint) { return new DataTunnel(newManager); } + @Override public void close() { Closeables.closeQuietly(server); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java index a1973562acd..beb34168fee 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserServer.java @@ -27,6 +27,7 @@ import java.util.UUID; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.DrillbitStartupException; import org.apache.drill.exec.memory.BufferAllocator; @@ -66,13 +67,14 @@ public class UserServer extends BasicServer> authImpls = - PathScanner.scanForImplementations(UserAuthenticator.class, - config.getStringList(ExecConstants.USER_AUTHENTICATOR_IMPL_PACKAGES)); + scan.getImplementations(UserAuthenticator.class); for(Class clazz : authImpls) { final UserAuthenticatorTemplate template = clazz.getAnnotation(UserAuthenticatorTemplate.class); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java index 903402ca7dc..ad757ef2101 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java @@ -21,6 +21,7 @@ import org.apache.drill.common.DrillAutoCloseables; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.RootAllocatorFactory; @@ -37,13 +38,15 @@ public class BootStrapContext implements AutoCloseable { private final EventLoopGroup loop2; private final MetricRegistry metrics; private final BufferAllocator allocator; + private final ScanResult classpathScan; - public BootStrapContext(DrillConfig config) { + public BootStrapContext(DrillConfig config, ScanResult classpathScan) { this.config = config; - loop = TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS), "BitServer-"); - loop2 = TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS), "BitClient-"); - metrics = DrillMetrics.getInstance(); - allocator = RootAllocatorFactory.newRoot(config); + this.classpathScan = classpathScan; + this.loop = TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS), "BitServer-"); + this.loop2 = TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS), "BitClient-"); + this.metrics = DrillMetrics.getInstance(); + this.allocator = RootAllocatorFactory.newRoot(config); } public DrillConfig getConfig() { @@ -66,6 +69,10 @@ public BufferAllocator getAllocator() { return allocator; } + public ScanResult getClasspathScan() { + return classpathScan; + } + @Override public void close() { try { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java index 892e9d9a83f..9267003474e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java @@ -23,6 +23,8 @@ import org.apache.drill.common.AutoCloseables; import org.apache.drill.common.StackTrace; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.coord.ClusterCoordinator; import org.apache.drill.exec.coord.ClusterCoordinator.RegistrationHandle; @@ -41,6 +43,7 @@ import org.apache.drill.exec.work.WorkManager; import org.apache.zookeeper.Environment; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; /** @@ -65,9 +68,11 @@ public static Drillbit start(final DrillConfig config) throws DrillbitStartupExc public static Drillbit start(final DrillConfig config, final RemoteServiceSet remoteServiceSet) throws DrillbitStartupException { logger.debug("Starting new Drillbit."); + // TODO: allow passing as a parameter + ScanResult classpathScan = ClassPathScanner.fromPrescan(config); Drillbit bit; try { - bit = new Drillbit(config, remoteServiceSet); + bit = new Drillbit(config, remoteServiceSet, classpathScan); } catch (final Exception ex) { throw new DrillbitStartupException("Failure while initializing values in Drillbit.", ex); } @@ -166,11 +171,21 @@ public static void main(final String[] cli) throws DrillbitStartupException { private final WebServer webServer; private RegistrationHandle registrationHandle; - public Drillbit(final DrillConfig config, final RemoteServiceSet serviceSet) throws Exception { + @VisibleForTesting + public Drillbit( + final DrillConfig config, + final RemoteServiceSet serviceSet) throws Exception { + this(config, serviceSet, ClassPathScanner.fromPrescan(config)); + } + + public Drillbit( + final DrillConfig config, + final RemoteServiceSet serviceSet, + final ScanResult classpathScan) throws Exception { final Stopwatch w = new Stopwatch().start(); logger.debug("Construction started."); final boolean allowPortHunting = serviceSet != null; - context = new BootStrapContext(config); + context = new BootStrapContext(config, classpathScan); manager = new WorkManager(context); engine = new ServiceEngine(manager.getControlMessageHandler(), manager.getUserWorker(), context, manager.getWorkBus(), manager.getDataHandler(), allowPortHunting); @@ -220,7 +235,6 @@ public synchronized void close() { if (coord != null && registrationHandle != null) { coord.unregister(registrationHandle); } - try { Thread.sleep(context.getConfig().getInt(ExecConstants.ZK_REFRESH) * 2); } catch (final InterruptedException e) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java index 6fdbfca1704..8a3a88806b2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java @@ -19,10 +19,14 @@ import io.netty.channel.EventLoopGroup; +import static com.google.common.base.Preconditions.checkNotNull; + import java.util.Collection; import java.util.concurrent.ExecutorService; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.exec.compile.CodeCompiler; import org.apache.drill.exec.coord.ClusterCoordinator; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; @@ -58,27 +62,33 @@ public class DrillbitContext { private final PStoreProvider provider; private final CodeCompiler compiler; private final ExecutorService executor; - - public DrillbitContext(DrillbitEndpoint endpoint, BootStrapContext context, ClusterCoordinator coord, - Controller controller, DataConnectionCreator connectionsPool, WorkEventBus workBus, PStoreProvider provider, + private final LogicalPlanPersistence lpPersistence; + private final ScanResult classpathScan; + + public DrillbitContext( + DrillbitEndpoint endpoint, + BootStrapContext context, + ClusterCoordinator coord, + Controller controller, + DataConnectionCreator connectionsPool, + WorkEventBus workBus, + PStoreProvider provider, ExecutorService executor) { - Preconditions.checkNotNull(endpoint); - Preconditions.checkNotNull(context); - Preconditions.checkNotNull(controller); - Preconditions.checkNotNull(connectionsPool); + this.classpathScan = context.getClasspathScan(); this.workBus = workBus; - this.controller = controller; - this.context = context; + this.controller = checkNotNull(controller); + this.context = checkNotNull(context); this.coord = coord; - this.connectionsPool = connectionsPool; - this.endpoint = endpoint; + this.connectionsPool = checkNotNull(connectionsPool); + this.endpoint = checkNotNull(endpoint); this.provider = provider; this.executor = executor; - this.storagePlugins = new StoragePluginRegistry(this); - this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(), endpoint, storagePlugins); - this.operatorCreatorRegistry = new OperatorCreatorRegistry(context.getConfig()); - this.systemOptions = new SystemOptionManager(context.getConfig(), provider); - this.functionRegistry = new FunctionImplementationRegistry(context.getConfig(), systemOptions); + this.lpPersistence = new LogicalPlanPersistence(context.getConfig(), classpathScan); + this.storagePlugins = new StoragePluginRegistry(this); // TODO change constructor + this.reader = new PhysicalPlanReader(context.getConfig(), classpathScan, lpPersistence, endpoint, storagePlugins); + this.operatorCreatorRegistry = new OperatorCreatorRegistry(classpathScan); + this.systemOptions = new SystemOptionManager(lpPersistence, provider); + this.functionRegistry = new FunctionImplementationRegistry(context.getConfig(), classpathScan, systemOptions); this.compiler = new CodeCompiler(context.getConfig(), systemOptions); } @@ -162,4 +172,12 @@ public ExecutorService getExecutor() { return executor; } + public LogicalPlanPersistence getLpPersistence() { + return lpPersistence; + } + + public ScanResult getClasspathScan() { + return classpathScan; + } + } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index 4cd61c2ad7d..b3bab2a672d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -26,7 +26,7 @@ import com.google.common.collect.Sets; import org.apache.commons.collections.IteratorUtils; -import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.map.CaseInsensitiveMap; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.ExecConstants; @@ -149,9 +149,9 @@ public class SystemOptionManager extends BaseOptionManager { */ private PStore options; - public SystemOptionManager(final DrillConfig config, final PStoreProvider provider) { + public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PStoreProvider provider) { this.provider = provider; - this.config = PStoreConfig.newJacksonBuilder(config.getMapper(), OptionValue.class) + this.config = PStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), OptionValue.class) .name("sys.options") .build(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java index 8c14587e8e5..7d2dfe846f2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java @@ -59,14 +59,14 @@ public DrillRestServer(final WorkManager workManager) { register(GenericExceptionMapper.class); JacksonJaxbJsonProvider provider = new JacksonJaxbJsonProvider(); - provider.setMapper(workManager.getContext().getConfig().getMapper()); + provider.setMapper(workManager.getContext().getLpPersistence().getMapper()); register(provider); register(new AbstractBinder() { @Override protected void configure() { bind(workManager).to(WorkManager.class); - bind(workManager.getContext().getConfig().getMapper()).to(ObjectMapper.class); + bind(workManager.getContext().getLpPersistence().getMapper()).to(ObjectMapper.class); bind(workManager.getContext().getPersistentStoreProvider()).to(PStoreProvider.class); bind(workManager.getContext().getStorage()).to(StoragePluginRegistry.class); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java index 25ea307cd1e..8c347e8a53f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/service/ServiceEngine.java @@ -17,12 +17,18 @@ */ package org.apache.drill.exec.service; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + import java.io.Closeable; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import io.netty.channel.EventLoopGroup; + import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.DrillbitStartupException; @@ -38,6 +44,7 @@ import org.apache.drill.exec.work.batch.ControlMessageHandler; import org.apache.drill.exec.work.user.UserWorker; +import com.google.common.base.Stopwatch; import com.google.common.io.Closeables; public class ServiceEngine implements Closeable{ @@ -54,7 +61,7 @@ public ServiceEngine(ControlMessageHandler controlMessageHandler, UserWorker use WorkEventBus workBus, DataResponseHandler dataHandler, boolean allowPortHunting) throws DrillbitStartupException { final EventLoopGroup eventLoopGroup = TransportCheck.createEventLoopGroup( context.getConfig().getInt(ExecConstants.USER_SERVER_RPC_THREADS), "UserServer-"); - this.userServer = new UserServer(context.getConfig(), context.getAllocator(), eventLoopGroup, userWorker); + this.userServer = new UserServer(context.getConfig(), context.getClasspathScan(), context.getAllocator(), eventLoopGroup, userWorker); this.controller = new ControllerImpl(context, controlMessageHandler, allowPortHunting); this.dataPool = new DataConnectionCreator(context, workBus, dataHandler, allowPortHunting); this.config = context.getConfig(); @@ -82,10 +89,33 @@ public Controller getController() { return controller; } + private void submit(ExecutorService p, final String name, final Closeable c) { + p.submit(new Runnable() { + @Override + public void run() { + Stopwatch watch = new Stopwatch().start(); + Closeables.closeQuietly(c); + long elapsed = watch.elapsed(MILLISECONDS); + if (elapsed > 500) { + logger.info("closed " + name + " in " + elapsed + " ms"); + } + } + }); + } + @Override public void close() throws IOException { - Closeables.closeQuietly(userServer); - Closeables.closeQuietly(dataPool); - Closeables.closeQuietly(controller); + // this takes time so close them in parallel + // Ideally though we fix this netty bug: https://github.com/netty/netty/issues/2545 + ExecutorService p = Executors.newFixedThreadPool(2); + submit(p, "userServer", userServer); + submit(p, "dataPool", dataPool); + submit(p, "controller", controller); + p.shutdown(); + try { + p.awaitTermination(3, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java index 6eb65e615fc..4673cb57d1f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java @@ -17,6 +17,8 @@ */ package org.apache.drill.exec.store; +import static com.google.common.base.Preconditions.checkNotNull; + import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -34,12 +36,14 @@ import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.tools.RuleSet; -import org.apache.drill.common.config.DrillConfig; + +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.logical.FormatPluginConfig; import org.apache.drill.common.logical.StoragePluginConfig; -import org.apache.drill.common.util.PathScanner; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.DrillbitStartupException; import org.apache.drill.exec.ops.OptimizerRulesContext; @@ -57,6 +61,7 @@ import com.google.common.base.Charsets; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet.Builder; @@ -78,22 +83,23 @@ public class StoragePluginRegistry implements Iterable pluginSystemTable; - private final Object updateLock = new Object(); - private volatile long lastUpdate = 0; - private static final long UPDATE_FREQUENCY = 2 * 60 * 1000; + private final LogicalPlanPersistence lpPersistence; + private final ScanResult classpathScan; public StoragePluginRegistry(DrillbitContext context) { + this.context = checkNotNull(context); + this.lpPersistence = checkNotNull(context.getLpPersistence()); + this.classpathScan = checkNotNull(context.getClasspathScan()); try { - this.context = context; this.pluginSystemTable = context // .getPersistentStoreProvider() // .getStore(PStoreConfig // - .newJacksonBuilder(context.getConfig().getMapper(), StoragePluginConfig.class) // + .newJacksonBuilder(lpPersistence.getMapper(), StoragePluginConfig.class) // .name("sys.storage_plugins") // .build()); } catch (IOException | RuntimeException e) { logger.error("Failure while loading storage plugin registry.", e); - throw new RuntimeException("Faiure while reading and loading storage plugin configuration.", e); + throw new RuntimeException("Failure while reading and loading storage plugin configuration.", e); } } @@ -103,11 +109,8 @@ public PStore getStore() { @SuppressWarnings("unchecked") public void init() throws DrillbitStartupException { - final DrillConfig config = context.getConfig(); final Collection> pluginClasses = - PathScanner.scanForImplementations( - StoragePlugin.class, - config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES)); + classpathScan.getImplementations(StoragePlugin.class); final String lineBrokenList = pluginClasses.size() == 0 ? "" : "\n\t- " + Joiner.on("\n\t- ").join(pluginClasses); @@ -146,13 +149,13 @@ private Map createPlugins() throws DrillbitStartupExcepti if (!pluginSystemTable.iterator().hasNext()) { // bootstrap load the config since no plugins are stored. logger.info("No storage plugin instances configured in persistent store, loading bootstrap configuration."); - Collection urls = PathScanner.forResource(ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE, false, Resources.class.getClassLoader()); + Collection urls = ClassPathScanner.forResource(ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE, false); if (urls != null && ! urls.isEmpty()) { logger.info("Loading the storage plugin configs from URLs {}.", urls); Map pluginURLMap = Maps.newHashMap(); for (URL url :urls) { String pluginsData = Resources.toString(url, Charsets.UTF_8); - StoragePlugins plugins = context.getConfig().getMapper().readValue(pluginsData, StoragePlugins.class); + StoragePlugins plugins = lpPersistence.getMapper().readValue(pluginsData, StoragePlugins.class); for (Map.Entry config : plugins) { if (!pluginSystemTable.putIfAbsent(config.getKey(), config.getValue())) { logger.warn("Duplicate plugin instance '{}' defined in [{}, {}], ignoring the later one.", diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java index 6f3ce27eff6..fb3e58aee2d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -17,20 +17,21 @@ */ package org.apache.drill.exec.store.dfs; +import static org.apache.drill.exec.store.dfs.FileSystemSchemaFactory.DEFAULT_WS_NAME; + import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.calcite.schema.SchemaPlus; - import org.apache.drill.common.JSONOptions; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.FormatPluginConfig; import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.ops.OptimizerRulesContext; -import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.AbstractStoragePlugin; @@ -46,8 +47,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import static org.apache.drill.exec.store.dfs.FileSystemSchemaFactory.DEFAULT_WS_NAME; - /** * A Storage engine associated with a Hadoop FileSystem Implementation. Examples include HDFS, MapRFS, QuantacastFileSystem, * LocalFileSystem, as well Apache Drill specific CachedFileSystem, ClassPathFileSystem and LocalSyncableFileSystem. @@ -61,20 +60,20 @@ public class FileSystemPlugin extends AbstractStoragePlugin{ private final Map formatPluginsByName; private final Map formatPluginsByConfig; private final FileSystemConfig config; - private final DrillbitContext context; private final Configuration fsConf; + private final LogicalPlanPersistence lpPersistance; public FileSystemPlugin(FileSystemConfig config, DrillbitContext context, String name) throws ExecutionSetupException{ + this.config = config; + this.lpPersistance = context.getLpPersistence(); try { - this.config = config; - this.context = context; fsConf = new Configuration(); fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.connection); fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName()); fsConf.set("fs.drill-local.impl", LocalSyncableFileSystem.class.getName()); - formatPluginsByName = FormatCreator.getFormatPlugins(context, fsConf, config); + formatPluginsByName = FormatCreator.getFormatPlugins(context, fsConf, config, context.getClasspathScan()); List matchers = Lists.newArrayList(); formatPluginsByConfig = Maps.newHashMap(); for (FormatPlugin p : formatPluginsByName.values()) { @@ -86,13 +85,13 @@ public FileSystemPlugin(FileSystemConfig config, DrillbitContext context, String List factories = Lists.newArrayList(); if (!noWorkspace) { for (Map.Entry space : config.workspaces.entrySet()) { - factories.add(new WorkspaceSchemaFactory(context.getConfig(), this, space.getKey(), name, space.getValue(), matchers)); + factories.add(new WorkspaceSchemaFactory(this, space.getKey(), name, space.getValue(), matchers, context.getLpPersistence())); } } // if the "default" workspace is not given add one. if (noWorkspace || !config.workspaces.containsKey(DEFAULT_WS_NAME)) { - factories.add(new WorkspaceSchemaFactory(context.getConfig(), this, DEFAULT_WS_NAME, name, WorkspaceConfig.DEFAULT, matchers)); + factories.add(new WorkspaceSchemaFactory(this, DEFAULT_WS_NAME, name, WorkspaceConfig.DEFAULT, matchers, context.getLpPersistence())); } this.schemaFactory = new FileSystemSchemaFactory(name, factories); @@ -114,7 +113,7 @@ public StoragePluginConfig getConfig() { @Override public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection, List columns) throws IOException { - FormatSelection formatSelection = selection.getWith(context.getConfig(), FormatSelection.class); + FormatSelection formatSelection = selection.getWith(lpPersistance, FormatSelection.class); FormatPlugin plugin; if (formatSelection.getFormat() instanceof NamedFormatPluginConfig) { plugin = formatPluginsByName.get( ((NamedFormatPluginConfig) formatSelection.getFormat()).name); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java index 729dc559425..3f932cd08a3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatCreator.java @@ -22,16 +22,14 @@ import java.util.Collection; import java.util.Map; -import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.logical.FormatPluginConfig; import org.apache.drill.common.logical.StoragePluginConfig; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.common.util.ConstructorChecker; -import org.apache.drill.common.util.PathScanner; -import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.server.DrillbitContext; +import org.apache.hadoop.conf.Configuration; import com.google.common.collect.Maps; -import org.apache.hadoop.conf.Configuration; public class FormatCreator { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatCreator.class); @@ -41,13 +39,14 @@ public class FormatCreator { private static final ConstructorChecker DEFAULT_BASED = new ConstructorChecker(String.class, DrillbitContext.class, Configuration.class, StoragePluginConfig.class); - static Map getFormatPlugins(DrillbitContext context, Configuration fsConf, - FileSystemConfig storageConfig) { - final DrillConfig config = context.getConfig(); + static Map getFormatPlugins( + DrillbitContext context, + Configuration fsConf, + FileSystemConfig storageConfig, + ScanResult classpathScan) { Map plugins = Maps.newHashMap(); - Collection> pluginClasses = - PathScanner.scanForImplementations(FormatPlugin.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES)); + Collection> pluginClasses = classpathScan.getImplementations(FormatPlugin.class); if (storageConfig.formats == null || storageConfig.formats.isEmpty()) { @@ -73,7 +72,7 @@ static Map getFormatPlugins(DrillbitContext context, Confi if (!FORMAT_BASED.check(c)) { continue; } - Class configClass = (Class) c.getParameterTypes()[4]; + Class configClass = c.getParameterTypes()[4]; constructors.put(configClass, c); } catch (Exception e) { logger.warn(String.format("Failure while trying instantiate FormatPlugin %s.", pluginClass.getName()), e); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java index fb48a80138a..bb42009370f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java @@ -28,14 +28,10 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.regex.Pattern; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; import org.apache.calcite.schema.Table; - -import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.exceptions.UserException; -import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.dotdrill.DotDrillFile; import org.apache.drill.exec.dotdrill.DotDrillType; @@ -55,13 +51,15 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.AccessControlException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.security.AccessControlException; public class WorkspaceSchemaFactory { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WorkspaceSchemaFactory.class); @@ -72,21 +70,26 @@ public class WorkspaceSchemaFactory { private final WorkspaceConfig config; private final Configuration fsConf; - private final DrillConfig drillConfig; private final String storageEngineName; private final String schemaName; private final FileSystemPlugin plugin; private final ObjectMapper mapper; + private final LogicalPlanPersistence logicalPlanPersistence; private final Path wsPath; - public WorkspaceSchemaFactory(DrillConfig drillConfig, FileSystemPlugin plugin, String schemaName, - String storageEngineName, WorkspaceConfig config, List formatMatchers) + public WorkspaceSchemaFactory( + FileSystemPlugin plugin, + String schemaName, + String storageEngineName, + WorkspaceConfig config, + List formatMatchers, + LogicalPlanPersistence logicalPlanPersistence) throws ExecutionSetupException, IOException { + this.logicalPlanPersistence = logicalPlanPersistence; this.fsConf = plugin.getFsConf(); this.plugin = plugin; - this.drillConfig = drillConfig; this.config = config; - this.mapper = drillConfig.getMapper(); + this.mapper = logicalPlanPersistence.getMapper(); this.fileMatchers = Lists.newArrayList(); this.dirMatchers = Lists.newArrayList(); this.storageEngineName = storageEngineName; @@ -226,7 +229,7 @@ public Set getTableNames() { private View getView(DotDrillFile f) throws IOException{ assert f.getType() == DotDrillType.VIEW; - return f.getView(drillConfig); + return f.getView(logicalPlanPersistence); } @Override @@ -237,7 +240,7 @@ public Table getTable(String name) { } // then look for files that start with this name and end in .drill. - List files = Collections.EMPTY_LIST; + List files = Collections.emptyList(); try { try { files = DotDrillUtil.getDotDrills(fs, new Path(config.getLocation()), name, DotDrillType.VIEW); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java index f939ba8265b..385c99fd0ca 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java @@ -62,7 +62,7 @@ public boolean supportsRead() { @Override public InfoSchemaGroupScan getPhysicalScan(String userName, JSONOptions selection, List columns) throws IOException { - SelectedTable table = selection.getWith(context.getConfig(), SelectedTable.class); + SelectedTable table = selection.getWith(context.getLpPersistence(), SelectedTable.class); return new InfoSchemaGroupScan(table); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java index bac02cada4a..33f030b6533 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java @@ -74,7 +74,7 @@ public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws @Override public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection, List columns) throws IOException { - SystemTable table = selection.getWith(context.getConfig(), SystemTable.class); + SystemTable table = selection.getWith(context.getLpPersistence(), SystemTable.class); return new SystemTableScan(table, this); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java index 8209277caa7..0bf16735a40 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java @@ -28,6 +28,7 @@ import org.apache.drill.common.SelfCleaningRunnable; import org.apache.drill.common.concurrent.ExtendedLatch; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.exec.coord.ClusterCoordinator; import org.apache.drill.exec.proto.BitControl.FragmentStatus; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; @@ -122,8 +123,12 @@ protected void afterExecute(final Runnable r, final Throwable t) { dataHandler = new DataResponseHandlerImpl(bee); // TODO only uses startFragmentPendingRemote() } - public void start(final DrillbitEndpoint endpoint, final Controller controller, - final DataConnectionCreator data, final ClusterCoordinator coord, final PStoreProvider provider) { + public void start( + final DrillbitEndpoint endpoint, + final Controller controller, + final DataConnectionCreator data, + final ClusterCoordinator coord, + final PStoreProvider provider) { dContext = new DrillbitContext(endpoint, bContext, coord, controller, data, workBus, provider, executor); statusThread.start(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index 2b17933e1e4..e17eba5a842 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -351,14 +351,14 @@ private void parseAndRunLogicalPlan(final String json) throws ExecutionSetupExce private void log(final LogicalPlan plan) { if (logger.isDebugEnabled()) { - logger.debug("Logical {}", plan.unparse(queryContext.getConfig())); + logger.debug("Logical {}", plan.unparse(queryContext.getLpPersistence())); } } private void log(final PhysicalPlan plan) { if (logger.isDebugEnabled()) { try { - final String planText = queryContext.getConfig().getMapper().writeValueAsString(plan); + final String planText = queryContext.getLpPersistence().getMapper().writeValueAsString(plan); logger.debug("Physical {}", planText); } catch (final IOException e) { logger.warn("Error while attempting to log physical plan.", e); @@ -367,7 +367,7 @@ private void log(final PhysicalPlan plan) { } private void returnPhysical(final PhysicalPlan plan) throws ExecutionSetupException { - final String jsonPlan = plan.unparse(queryContext.getConfig().getMapper().writer()); + final String jsonPlan = plan.unparse(queryContext.getLpPersistence().getMapper().writer()); runPhysicalPlan(DirectPlan.createDirectPlan(queryContext, new PhysicalFromLogicalExplain(jsonPlan))); } @@ -909,7 +909,7 @@ private void runSQL(final String sql) throws ExecutionSetupException { private PhysicalPlan convert(final LogicalPlan plan) throws OptimizerException { if (logger.isDebugEnabled()) { - logger.debug("Converting logical plan {}.", plan.toJsonStringSafe(queryContext.getConfig())); + logger.debug("Converting logical plan {}.", plan.toJsonStringSafe(queryContext.getLpPersistence())); } return new BasicOptimizer(queryContext, initiatingClient).optimize( new BasicOptimizer.BasicOptimizationContext(queryContext), plan); diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index cf0d86c3d2b..9bde6e0319b 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -17,7 +17,29 @@ // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl" +drill { + classpath.scanning { + base.classes : ${?drill.classpath.scanning.base.classes} [ + org.apache.drill.exec.expr.DrillFunc, + org.apache.drill.exec.expr.fn.PluggableFunctionRegistry, + org.apache.drill.exec.physical.base.PhysicalOperator, + org.apache.drill.exec.physical.impl.BatchCreator, + org.apache.drill.exec.physical.impl.RootCreator, + org.apache.drill.exec.rpc.user.security.UserAuthenticator, + org.apache.drill.exec.store.dfs.FormatPlugin, + org.apache.drill.exec.store.StoragePlugin + ], + + annotations += org.apache.drill.exec.expr.annotations.FunctionTemplate + + packages : ${?drill.classpath.scanning.packages} [ + org.apache.drill.exec.expr, + org.apache.drill.exec.physical, + org.apache.drill.exec.store, + org.apache.drill.exec.rpc.user.security + ] + } +} drill.client: { supports-complex-types: true @@ -49,15 +71,10 @@ drill.exec: { }, use.ip : false }, - operator: { - packages += "org.apache.drill.exec.physical.config" - }, optimizer: { implementation: "org.apache.drill.exec.opt.IdentityOptimizer" }, - functions: ["org.apache.drill.expr.fn.impl"], storage: { - packages += "org.apache.drill.exec.store", file: { text: { buffer.size: 262144, @@ -90,7 +107,6 @@ drill.exec: { ssl_enabled: false, port: 8047 }, - functions: ["org.apache.drill.expr.fn.impl"], network: { start: 35000 }, @@ -110,7 +126,6 @@ drill.exec: { }, security.user.auth { enabled: false, - packages += "org.apache.drill.exec.rpc.user.security", impl: "pam", pam_profiles: [ "sudo", "login" ] }, diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java index 9387662bf43..641357174dc 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java @@ -30,6 +30,8 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.client.DrillClient; @@ -113,9 +115,12 @@ protected void failed(Throwable e, Description description) { private int[] columnWidths = new int[] { 8 }; + private static ScanResult classpathScan; + @BeforeClass public static void setupDefaultTestCluster() throws Exception { config = DrillConfig.create(TEST_CONFIGURATIONS); + classpathScan = ClassPathScanner.fromPrescan(config); openClient(); } @@ -179,7 +184,7 @@ private static void openClient() throws Exception { bits = new Drillbit[drillbitCount]; for(int i = 0; i < drillbitCount; i++) { - bits[i] = new Drillbit(config, serviceSet); + bits[i] = new Drillbit(config, serviceSet, classpathScan); bits[i].run(); final StoragePluginRegistry pluginRegistry = bits[i].getContext().getStorage(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java index 05b72385a9e..db0a8110c9e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java @@ -26,6 +26,9 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.jdbc.SimpleCalciteSchema; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.common.util.TestTools; import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; @@ -33,6 +36,7 @@ import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.QueryContext; import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.planner.sql.DrillOperatorTable; import org.apache.drill.exec.planner.sql.DrillSqlWorker; @@ -74,8 +78,9 @@ protected void testSqlPlan(String sqlCommands) throws Exception { final String[] sqlStrings = sqlCommands.split(";"); final LocalPStoreProvider provider = new LocalPStoreProvider(config); provider.start(); - - final SystemOptionManager systemOptions = new SystemOptionManager(config, provider); + final ScanResult scanResult = ClassPathScanner.fromPrescan(config); + final LogicalPlanPersistence logicalPlanPersistence = new LogicalPlanPersistence(config, scanResult); + final SystemOptionManager systemOptions = new SystemOptionManager(logicalPlanPersistence , provider); systemOptions.init(); final UserSession userSession = UserSession.Builder.newBuilder().withOptionManager(systemOptions).build(); final SessionOptionManager sessionOptions = (SessionOptionManager) userSession.getOptions(); @@ -94,6 +99,10 @@ protected void testSqlPlan(String sqlCommands) throws Exception { result = systemOptions; dbContext.getPersistentStoreProvider(); result = provider; + dbContext.getClasspathScan(); + result = scanResult; + dbContext.getLpPersistence(); + result = logicalPlanPersistence; } }; @@ -130,6 +139,8 @@ protected void testSqlPlan(String sqlCommands) throws Exception { result = allocator; context.getExecutionControls(); result = executionControls; + dbContext.getLpPersistence(); + result = logicalPlanPersistence; } }; diff --git a/exec/java-exec/src/test/java/org/apache/drill/common/scanner/TestClassPathScanner.java b/exec/java-exec/src/test/java/org/apache/drill/common/scanner/TestClassPathScanner.java new file mode 100644 index 00000000000..824d6e8f72b --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/common/scanner/TestClassPathScanner.java @@ -0,0 +1,175 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.common.scanner; + +import static java.util.Arrays.asList; +import static java.util.Collections.sort; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.persistence.AnnotationDescriptor; +import org.apache.drill.common.scanner.persistence.FieldDescriptor; +import org.apache.drill.common.scanner.persistence.AnnotatedClassDescriptor; +import org.apache.drill.common.scanner.persistence.ScanResult; +import org.apache.drill.exec.expr.DrillFunc; +import org.apache.drill.exec.expr.annotations.FunctionTemplate; +import org.apache.drill.exec.fn.impl.testing.GeneratorFunctions.IncreasingBigInt; +import org.apache.drill.exec.fn.impl.testing.GeneratorFunctions.RandomBigIntGauss; +import org.apache.drill.exec.physical.base.PhysicalOperator; +import org.junit.Assert; +import org.junit.Test; + +public class TestClassPathScanner { + + @SafeVarargs + final private > void assertListEqualsUnordered(Collection list, T... expected) { + List expectedList = asList(expected); + sort(expectedList); + List gotList = new ArrayList<>(list); + sort(gotList); + assertEquals(expectedList.toString(), gotList.toString()); + } + + @Test + public void test() throws Exception { + ScanResult result = ClassPathScanner.fromPrescan(DrillConfig.create()); + // if the build has run properly. BuildTimeScan.REGISTRY_FILE was created with a prescan +// assertListEqualsUnordered(result.getPrescannedPackages(), +// "org.apache.drill.common.logical", +// "org.apache.drill.exec.expr", +// "org.apache.drill.exec.physical.base", +// "org.apache.drill.exec.expr.fn.impl", +// "org.apache.drill.exec.physical.impl", +// "org.apache.drill.exec.rpc.user.security", +// "org.apache.drill.exec.store", +// "org.apache.drill.exec.store.mock", +// "org.apache.drill.exec.physical.config", +// "org.apache.drill.storage" +// ); +// // this is added in the unit test folder that was not scanned so far +// assertListEqualsUnordered(result.getScannedPackages(), +// "org.apache.drill.exec.testing", +// "org.apache.drill.exec.fn.impl.testing", +// "org.apache.drill.exec.rpc.user.security.testing" +// ); + List functions = result.getAnnotatedClasses(); + Set scanned = new HashSet<>(); + AnnotatedClassDescriptor functionRandomBigIntGauss = null; + for (AnnotatedClassDescriptor function : functions) { + assertTrue(function.getClassName() + " scanned twice", scanned.add(function.getClassName())); + if (function.getClassName().equals(RandomBigIntGauss.class.getName())) { + functionRandomBigIntGauss = function; + } + } + if (functionRandomBigIntGauss == null) { + Assert.fail("functionRandomBigIntGauss not found"); + } + // TODO: use Andrew's randomized test framework to verify a subset of the functions + for (AnnotatedClassDescriptor function : functions) { + Class c = Class.forName(function.getClassName(), false, this.getClass().getClassLoader()); + + Field[] fields = c.getDeclaredFields(); + assertEquals("fields count for " + function, fields.length, function.getFields().size()); + for (int i = 0; i < fields.length; i++) { + FieldDescriptor fieldDescriptor = function.getFields().get(i); + Field field = fields[i]; + assertEquals( + "Class fields:\n" + Arrays.toString(fields) + "\n != \nDescriptor fields:\n" + function.getFields(), + field.getName(), fieldDescriptor.getName()); + verifyAnnotations(field.getDeclaredAnnotations(), fieldDescriptor.getAnnotations()); + assertEquals(field.getType(), fieldDescriptor.getFieldClass()); + } + + Annotation[] annotations = c.getDeclaredAnnotations(); + List scannedAnnotations = function.getAnnotations(); + verifyAnnotations(annotations, scannedAnnotations); + FunctionTemplate bytecodeAnnotation = function.getAnnotationProxy(FunctionTemplate.class); + FunctionTemplate reflectionAnnotation = c.getAnnotation(FunctionTemplate.class); + assertEquals(reflectionAnnotation.name(), bytecodeAnnotation.name()); + assertArrayEquals(reflectionAnnotation.names(), bytecodeAnnotation.names()); + assertEquals(reflectionAnnotation.scope(), bytecodeAnnotation.scope()); + assertEquals(reflectionAnnotation.nulls(), bytecodeAnnotation.nulls()); + assertEquals(reflectionAnnotation.isBinaryCommutative(), bytecodeAnnotation.isBinaryCommutative()); + assertEquals(reflectionAnnotation.desc(), bytecodeAnnotation.desc()); + assertEquals(reflectionAnnotation.costCategory(), bytecodeAnnotation.costCategory()); + } + for (String baseType : result.getScannedClasses()) { + validateType(result, baseType); + } + assertTrue(result.getImplementations(PhysicalOperator.class).size() > 0); + assertTrue(result.getImplementations(DrillFunc.class).size() > 0); + } + + private void validateType(ScanResult result, String baseType) throws ClassNotFoundException { + if (baseType.startsWith("org.apache.hadoop.hive")) { + return; + } + @SuppressWarnings("unchecked") + Class baseClass = (Class)Class.forName(baseType); + Set> impls = result.getImplementations(baseClass); + if (impls != null) { + for (Class impl : impls) { + assertTrue(impl + " extends " + baseType, baseClass.isAssignableFrom(impl)); + } + } + } + + private void verifyAnnotations(Annotation[] annotations, List scannedAnnotations) throws Exception { + assertEquals(Arrays.toString(annotations) + " expected but got " + scannedAnnotations, annotations.length, scannedAnnotations.size()); + for (int i = 0; i < annotations.length; i++) { + Annotation annotation = annotations[i]; + AnnotationDescriptor scannedAnnotation = scannedAnnotations.get(i); + Class annotationType = annotation.annotationType(); + assertEquals(annotationType.getName(), scannedAnnotation.getAnnotationType()); + if (annotation instanceof FunctionTemplate) { + FunctionTemplate ft = (FunctionTemplate)annotation; + if (ft.name() != null && !ft.name().equals("")) { + assertEquals(ft.name(), scannedAnnotation.getSingleValue("name")); + } + } + // generally verify all properties + Annotation proxy = scannedAnnotation.getProxy(annotationType); + Method[] declaredMethods = annotationType.getDeclaredMethods(); + for (Method method : declaredMethods) { + if (method.getParameterTypes().length == 0) { + Object reflectValue = method.invoke(annotation); + Object byteCodeValue = method.invoke(proxy); + String message = annotationType.getSimpleName() + "." + method.getName(); + if (method.getReturnType().isArray()) { + assertArrayEquals(message, (Object[])reflectValue, (Object[])byteCodeValue); + } else { + assertEquals(message, reflectValue, byteCodeValue); + } + } + } + } + } + +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java b/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java index 13f9563ade2..5928dce9640 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.PhysicalPlan; @@ -45,7 +46,7 @@ public class RunRootExec { public static void main(String args[]) throws Exception { String path = args[0]; int iterations = Integer.parseInt(args[1]); - Drillbit bit = new Drillbit(c, RemoteServiceSet.getLocalServiceSet()); + Drillbit bit = new Drillbit(c, RemoteServiceSet.getLocalServiceSet(), ClassPathScanner.fromPrescan(c)); bit.run(); DrillbitContext bitContext = bit.getContext(); PhysicalPlanReader reader = bitContext.getPlanReader(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestOpSerialization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestOpSerialization.java index 05105fc09c8..f4fe2da5ead 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestOpSerialization.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestOpSerialization.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.ValueExpressions; import org.apache.drill.common.logical.PlanProperties; @@ -32,6 +33,7 @@ import org.apache.drill.exec.physical.config.Screen; import org.apache.drill.exec.physical.config.UnionExchange; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.store.mock.MockSubScanPOP; import org.junit.Test; @@ -44,7 +46,7 @@ public class TestOpSerialization { @Test public void testSerializedDeserialize() throws Throwable { DrillConfig c = DrillConfig.create(); - PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); MockSubScanPOP s = new MockSubScanPOP("abc", null); s.setOperatorId(3); Filter f = new Filter(s, new ValueExpressions.BooleanExpression("true", ExpressionPosition.UNKNOWN), 0.1f); @@ -67,12 +69,10 @@ public void testSerializedDeserialize() throws Throwable { pops = Lists.reverse(pops); } PhysicalPlan plan1 = new PhysicalPlan(PlanProperties.builder().build(), pops); - String json = plan1.unparse(c.getMapper().writer()); - System.out.println(json); + LogicalPlanPersistence logicalPlanPersistence = PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c); + String json = plan1.unparse(logicalPlanPersistence.getMapper().writer()); PhysicalPlan plan2 = reader.readPhysicalPlan(json); - System.out.println("++++++++"); - System.out.println(plan2.unparse(c.getMapper().writer())); PhysicalOperator root = plan2.getSortedOperators(false).iterator().next(); assertEquals(0, root.getOperatorId()); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java index 0f437e909cc..0dd20b65b37 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java @@ -24,6 +24,7 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; import org.apache.drill.common.util.TestTools; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java index 6b6bd647e7f..d6c56883f30 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java @@ -17,18 +17,17 @@ */ package org.apache.drill.exec.client; +import static org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader; import static org.junit.Assert.assertTrue; import java.io.FileInputStream; -import mockit.Injectable; -import mockit.NonStrictExpectations; - import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -39,7 +38,6 @@ import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.proto.ExecProtos.FragmentHandle; import org.apache.drill.exec.proto.helper.QueryIdHelper; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; @@ -53,6 +51,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + /** * The unit test case will read a physical plan in json format. The physical plan contains a "trace" operator, * which will produce a dump file. The dump file will be input into DumpCat to test query mode and batch mode. @@ -69,12 +70,12 @@ public void testDumpCat(@Injectable final DrillbitContext bitContext, @Injectabl new NonStrictExpectations(){{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getConfig(); result = c; minTimes = 1; + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/trace/simple_trace.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java new file mode 100644 index 00000000000..9032946b9fa --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.compile; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.io.IOException; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.exec.server.options.SystemOptionManager; +import org.apache.drill.exec.store.sys.local.LocalPStoreProvider; + +public class CodeCompilerTestFactory { + public static CodeCompiler getTestCompiler(DrillConfig c) throws IOException { + DrillConfig config = checkNotNull(c); + LogicalPlanPersistence persistence = new LogicalPlanPersistence(config, ClassPathScanner.fromPrescan(config)); + LocalPStoreProvider provider = new LocalPStoreProvider(config); + SystemOptionManager systemOptionManager = new SystemOptionManager(persistence, provider); + return new CodeCompiler(config, systemOptionManager.init()); + } +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java index 345ac3ccc4a..84868018009 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java @@ -25,6 +25,8 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.compile.DrillCheckClassAdapter; import org.apache.drill.exec.compile.QueryClassLoader; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.server.options.SystemOptionManager; import org.apache.drill.exec.store.sys.local.LocalPStoreProvider; import org.objectweb.asm.ClassReader; @@ -55,7 +57,7 @@ public static void main(String[] args) throws Exception { check(output); final DrillConfig c = DrillConfig.forClient(); - final SystemOptionManager m = new SystemOptionManager(c, new LocalPStoreProvider(c)); + final SystemOptionManager m = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), new LocalPStoreProvider(c)); m.init(); try (QueryClassLoader ql = new QueryClassLoader(DrillConfig.create(), m)) { ql.injectByteCode("org.apache.drill.Pickle$OutgoingBatch", output); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java index 99ee904fee1..4865683f226 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java @@ -20,15 +20,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -38,8 +37,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.vector.Float8Vector; @@ -50,6 +49,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestMathFunctions extends ExecTest { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestMathFunctions.class); @@ -62,11 +64,11 @@ public void testBasicMathFunctions(@Injectable final DrillbitContext bitContext, bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/simple_math_functions.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java index 3447dae9f68..b8e7c3759ce 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java @@ -23,11 +23,9 @@ import java.math.BigDecimal; -import mockit.Injectable; -import mockit.NonStrictExpectations; - import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -37,8 +35,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.vector.ValueVector; @@ -49,6 +47,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestNewMathFunctions { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestNewMathFunctions.class); private final DrillConfig c = DrillConfig.create(); @@ -79,14 +80,14 @@ public void runTest(@Injectable final DrillbitContext bitContext, new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; final String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8); if (reader == null) { - reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); } if (registry == null) { registry = new FunctionImplementationRegistry(c); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java index 68c3b9c6728..81d1157558e 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java @@ -19,15 +19,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -37,8 +36,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.vector.BitVector; @@ -49,6 +48,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestRepeatedFunction extends ExecTest{ //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestRepeatedFunction.class); private final DrillConfig c = DrillConfig.create(); @@ -59,12 +61,12 @@ public void testRepeated(@Injectable final DrillbitContext bitContext, @Injectab new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/physical_repeated_1.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/GeneratorFunctions.java similarity index 85% rename from exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java rename to exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/GeneratorFunctions.java index 087595b34e3..93c67cc47fe 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/GeneratorFunctions.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.fn.impl; +package org.apache.drill.exec.fn.impl.testing; import java.util.Random; @@ -61,7 +61,7 @@ public void setup() { } public void eval() { - out.value = (long)(org.apache.drill.exec.fn.impl.GeneratorFunctions.random.nextGaussian() * range.value); + out.value = (long)(org.apache.drill.exec.fn.impl.testing.GeneratorFunctions.random.nextGaussian() * range.value); } } @@ -77,7 +77,7 @@ public void setup() { } public void eval() { - out.value = (long)(org.apache.drill.exec.fn.impl.GeneratorFunctions.random.nextFloat() * (max.value - min.value) + min.value); + out.value = (long)(org.apache.drill.exec.fn.impl.testing.GeneratorFunctions.random.nextFloat() * (max.value - min.value) + min.value); } } @@ -93,7 +93,7 @@ public void setup() { } public void eval() { - out.value = org.apache.drill.exec.fn.impl.GeneratorFunctions.random.nextGaussian() * range.value; + out.value = org.apache.drill.exec.fn.impl.testing.GeneratorFunctions.random.nextGaussian() * range.value; } } @@ -109,7 +109,7 @@ public void setup() { } public void eval() { - out.value = org.apache.drill.exec.fn.impl.GeneratorFunctions.random.nextFloat() * (max.value - min.value) + min.value; + out.value = org.apache.drill.exec.fn.impl.testing.GeneratorFunctions.random.nextFloat() * (max.value - min.value) + min.value; } } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java index f362257914f..03f701d25af 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestAllocators.java @@ -21,8 +21,6 @@ import static org.junit.Assert.fail; -import io.netty.buffer.DrillBuf; - import java.util.Iterator; import java.util.List; import java.util.Properties; @@ -38,8 +36,8 @@ import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.proto.UserBitShared; import org.apache.drill.exec.server.Drillbit; import org.apache.drill.exec.server.DrillbitContext; @@ -50,6 +48,8 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import io.netty.buffer.DrillBuf; + public class TestAllocators { private static final Properties TEST_CONFIGURATIONS = new Properties() { @@ -109,7 +109,7 @@ public void testAllocators() throws Exception { FragmentContext fragmentContext2 = new FragmentContext(bitContext, pf2, null, functionRegistry); // Get a few physical operators. Easiest way is to read a physical plan. - PhysicalPlanReader planReader = new PhysicalPlanReader(config, config.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), storageRegistry); + PhysicalPlanReader planReader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(bitContext, storageRegistry); PhysicalPlan plan = planReader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8)); List physicalOperators = plan.getSortedOperators(); Iterator physicalOperatorIterator = physicalOperators.iterator(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/opt/BasicOptimizerTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/opt/BasicOptimizerTest.java index 71e5d487642..a5f7bdc5b32 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/opt/BasicOptimizerTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/opt/BasicOptimizerTest.java @@ -18,9 +18,11 @@ package org.apache.drill.exec.opt; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.logical.LogicalPlan; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.junit.Test; public class BasicOptimizerTest extends ExecTest { @@ -28,8 +30,10 @@ public class BasicOptimizerTest extends ExecTest { @Test public void parseSimplePlan() throws Exception{ DrillConfig c = DrillConfig.create(); - LogicalPlan plan = LogicalPlan.parse(c, FileUtils.getResourceAsString("/scan_screen_logical.json")); - System.out.println(plan.unparse(c)); + LogicalPlanPersistence lpp = PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c); + LogicalPlan plan = LogicalPlan.parse(lpp, FileUtils.getResourceAsString("/scan_screen_logical.json")); + String unparse = plan.unparse(lpp); +// System.out.println(unparse); //System.out.println( new BasicOptimizer(DrillConfig.create()).convert(plan).unparse(c.getMapper().writer())); } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/config/TestParsePhysicalPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/config/TestParsePhysicalPlan.java index 4ad181f0111..3bfa347ae5a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/config/TestParsePhysicalPlan.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/config/TestParsePhysicalPlan.java @@ -18,10 +18,14 @@ package org.apache.drill.exec.physical.config; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.CoordinationProtos; import org.junit.Test; @@ -37,10 +41,14 @@ public class TestParsePhysicalPlan extends ExecTest { @Test public void parseSimplePlan() throws Exception{ DrillConfig c = DrillConfig.create(); - PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); - ObjectReader r = c.getMapper().reader(PhysicalPlan.class); - ObjectWriter writer = c.getMapper().writer(); + ScanResult scanResult = ClassPathScanner.fromPrescan(c); + LogicalPlanPersistence lpp = new LogicalPlanPersistence(c, scanResult); + + PhysicalPlanReader reader = new PhysicalPlanReader(c, scanResult, lpp, CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), null); + ObjectReader r = lpp.getMapper().reader(PhysicalPlan.class); + ObjectWriter writer = lpp.getMapper().writer(); PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/physical_test1.json"), Charsets.UTF_8)); - System.out.println(plan.unparse(writer)); + String unparse = plan.unparse(writer); +// System.out.println(unparse); } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java index 7e1da4481a4..a5bc6259ad1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java @@ -22,15 +22,13 @@ import java.util.List; -import mockit.Injectable; -import mockit.NonStrictExpectations; - import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.client.DrillClient; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers; import org.apache.drill.exec.expr.holders.BigIntHolder; @@ -45,9 +43,9 @@ import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.physical.base.FragmentRoot; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.record.VectorWrapper; @@ -68,6 +66,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestCastFunctions extends PopUnitTestBase{ //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class); private final DrillConfig c = DrillConfig.create(); @@ -82,11 +83,11 @@ public void testCastBigInt(@Injectable final DrillbitContext bitContext, bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = allocator; bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastBigInt.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -129,11 +130,11 @@ public void testCastInt(@Injectable final DrillbitContext bitContext, bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = allocator; bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastInt.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -173,11 +174,11 @@ public void testCastFloat4(@Injectable final DrillbitContext bitContext, bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = allocator; bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastFloat4.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -218,11 +219,11 @@ public void testCastFloat8(@Injectable final DrillbitContext bitContext, bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = allocator; bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastFloat8.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -264,11 +265,11 @@ public void testCastVarChar(@Injectable final DrillbitContext bitContext, bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = allocator; bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarChar.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -309,11 +310,11 @@ public void testCastVarBinary(@Injectable final DrillbitContext bitContext, bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = allocator; bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarBinary.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -354,11 +355,11 @@ public void testCastNested(@Injectable final DrillbitContext bitContext, bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = allocator; bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastNested.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -400,11 +401,11 @@ public void testCastNumException(@Injectable final DrillbitContext bitContext, bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = allocator; bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastNumException.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java index eb2e3c06a10..3664ef64fb1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctions.java @@ -19,20 +19,19 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.physical.base.FragmentRoot; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.server.DrillbitContext; import org.junit.Test; @@ -41,6 +40,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestComparisonFunctions extends ExecTest { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComparisonFunctions.class); @@ -54,14 +56,14 @@ public void runTest(@Injectable final DrillbitContext bitContext, new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; final String planString = Resources.toString(Resources.getResource(COMPARISON_TEST_PHYSICAL_PLAN), Charsets.UTF_8).replaceAll("EXPRESSION", expression); if (reader == null) { - reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); } if (registry == null) { registry = new FunctionImplementationRegistry(c); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java index 7f24c4c8698..e0595b901e1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestImplicitCastFunctions.java @@ -19,20 +19,19 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.physical.base.FragmentRoot; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.vector.ValueVector; @@ -42,6 +41,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestImplicitCastFunctions extends ExecTest { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestImplicitCastFunctions.class); @@ -70,14 +72,14 @@ public void runTest(@Injectable final DrillbitContext bitContext, new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; final String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8); if (reader == null) { - reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); } if (registry == null) { registry = new FunctionImplementationRegistry(c); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java index 80421677d11..f0ea8ccb0b6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java @@ -37,6 +37,7 @@ import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.physical.base.FragmentRoot; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; @@ -309,8 +310,7 @@ private SimpleRootExec doPhysicalTest(final DrillbitContext bitContext, UserClie final StoragePluginRegistry reg = new StoragePluginRegistry(bitContext); - final PhysicalPlanReader reader = new PhysicalPlanReader(config, config.getMapper(), - CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), reg); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(config, reg); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(config); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java index 4cbd4a9ecf3..bf51bb90afd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java @@ -24,9 +24,6 @@ import java.util.ArrayList; import java.util.List; -import mockit.Injectable; -import mockit.NonStrictExpectations; - import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.FunctionCall; @@ -34,11 +31,12 @@ import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.TypedNullConstant; import org.apache.drill.common.expression.ValueExpressions; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.DrillFuncHolder; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers; @@ -49,8 +47,8 @@ import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.physical.base.FragmentRoot; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.resolver.FunctionResolver; import org.apache.drill.exec.resolver.FunctionResolverFactory; import org.apache.drill.exec.rpc.user.UserServer; @@ -64,6 +62,9 @@ import com.google.common.io.Files; import com.sun.codemodel.JClassAlreadyExistsException; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestSimpleFunctions extends ExecTest { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class); private final DrillConfig c = DrillConfig.create(); @@ -154,12 +155,12 @@ public void testSubstring(@Injectable final DrillbitContext bitContext, new NonStrictExpectations(){{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testSubstring.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -193,12 +194,12 @@ public void testSubstringNegative(@Injectable final DrillbitContext bitContext, new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testSubstringNegative.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -233,12 +234,12 @@ public void testByteSubstring(@Injectable final DrillbitContext bitContext, new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testByteSubstring.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java index 32ed2205258..00d132e83a9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestStringFunctions.java @@ -19,20 +19,19 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.PhysicalPlan; import org.apache.drill.exec.physical.base.FragmentRoot; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.vector.ValueVector; @@ -43,6 +42,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestStringFunctions extends ExecTest { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestStringFunctions.class); @@ -74,14 +76,14 @@ public void runTest(@Injectable final DrillbitContext bitContext, new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; final String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8); if (reader == null) { - reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); } if (registry == null) { registry = new FunctionImplementationRegistry(c); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java index e00ff29a563..d97edd8fada 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestAgg.java @@ -19,14 +19,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -36,8 +35,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.vector.BigIntVector; @@ -49,6 +48,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestAgg extends ExecTest { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestAgg.class); private final DrillConfig c = DrillConfig.create(); @@ -57,12 +59,12 @@ private SimpleRootExec doTest(final DrillbitContext bitContext, UserClientConnec new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java index 6e3732f751a..1f7be1b4994 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java @@ -17,19 +17,12 @@ */ package org.apache.drill.exec.physical.impl.common; -import mockit.Injectable; -import mockit.NonStrictExpectations; - import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.expression.ExpressionPosition; -import org.apache.drill.common.expression.FieldReference; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.data.NamedExpression; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; -import org.apache.drill.exec.expr.holders.IntHolder; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.PhysicalPlan; @@ -38,17 +31,17 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; import org.apache.drill.exec.server.DrillbitContext; -import org.junit.Ignore; -import org.junit.Test; import com.codahale.metrics.MetricRegistry; import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.NonStrictExpectations; + public class TestHashTable extends ExecTest { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestHashTable.class); private final DrillConfig c = DrillConfig.create(); @@ -58,12 +51,12 @@ private SimpleRootExec doTest(final DrillbitContext bitContext, UserClientConnec new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(plan_path), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java index 07a665aae6d..de91b250eb2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestSimpleFilter.java @@ -19,13 +19,12 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -35,8 +34,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; import org.apache.drill.exec.server.DrillbitContext; import org.junit.Ignore; @@ -46,6 +45,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestSimpleFilter extends ExecTest { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFilter.class); private final DrillConfig c = DrillConfig.create(); @@ -56,12 +58,12 @@ public void testFilter(@Injectable final DrillbitContext bitContext, @Injectable new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/filter/test1.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -84,12 +86,12 @@ public void testSV4Filter(@Injectable final DrillbitContext bitContext, @Injecta new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/filter/test_sv4.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java index e7187d51391..bd3145f5c57 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java @@ -23,14 +23,14 @@ import java.util.Iterator; import java.util.List; -import mockit.Injectable; -import mockit.NonStrictExpectations; - import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; import org.apache.drill.common.util.FileUtils; import org.apache.drill.common.util.TestTools; import org.apache.drill.exec.client.DrillClient; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -40,9 +40,9 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.record.RecordBatchLoader; import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.rpc.user.QueryDataBatch; @@ -61,6 +61,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestHashJoin extends PopUnitTestBase { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestMergeJoin.class); @@ -72,18 +75,19 @@ public class TestHashJoin extends PopUnitTestBase { private void testHJMockScanCommon(final DrillbitContext bitContext, UserServer.UserClientConnection connection, String physicalPlan, int expectedRows) throws Throwable { final LocalPStoreProvider provider = new LocalPStoreProvider(c); provider.start(); - final SystemOptionManager opt = new SystemOptionManager(c, provider); + final SystemOptionManager opt = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), provider); opt.init(); new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; bitContext.getOptionManager(); result = opt; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); + bitContext.getLpPersistence(); result = new LogicalPlanPersistence(c, ClassPathScanner.fromPrescan(c)); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(physicalPlan), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java index 58c55b6f77f..3b363bd2c58 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java @@ -23,13 +23,11 @@ import java.util.ArrayList; import java.util.List; -import mockit.Injectable; -import mockit.NonStrictExpectations; - import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.client.DrillClient; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -39,9 +37,9 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.pop.PopUnitTestBase; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; @@ -58,6 +56,9 @@ import com.google.common.collect.Lists; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestMergeJoin extends PopUnitTestBase { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestMergeJoin.class); @@ -71,12 +72,12 @@ public void simpleEqualityJoin(@Injectable final DrillbitContext bitContext, new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/join/merge_join.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -125,11 +126,11 @@ public void orderedEqualityLeftJoin(@Injectable final DrillbitContext bitContext bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); bitContext.getConfig(); result = c; - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext)); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c, new StoragePluginRegistry(bitContext)); final PhysicalPlan plan = reader.readPhysicalPlan( Files.toString( FileUtils.getResourceAsFile("/join/merge_single_batch.json"), Charsets.UTF_8) @@ -181,11 +182,11 @@ public void orderedEqualityInnerJoin(@Injectable final DrillbitContext bitContex bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); bitContext.getConfig(); result = c; - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext)); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c, new StoragePluginRegistry(bitContext)); final PhysicalPlan plan = reader.readPhysicalPlan( Files.toString( FileUtils.getResourceAsFile("/join/merge_inner_single_batch.json"), Charsets.UTF_8) @@ -237,11 +238,11 @@ public void orderedEqualityMultiBatchJoin(@Injectable final DrillbitContext bitC bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); bitContext.getConfig(); result = c; - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StoragePluginRegistry(bitContext)); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c, new StoragePluginRegistry(bitContext)); final PhysicalPlan plan = reader.readPhysicalPlan( Files.toString( FileUtils.getResourceAsFile("/join/merge_multi_batch.json"), Charsets.UTF_8) @@ -290,11 +291,11 @@ public void testJoinBatchSize(@Injectable final DrillbitContext bitContext, @Inj bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); bitContext.getConfig(); result = c; - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/join/join_batchsize.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java index 97bbd2ca799..89e07ea783d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/limit/TestSimpleLimit.java @@ -19,13 +19,12 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -35,8 +34,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.vector.BigIntVector; @@ -47,6 +46,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestSimpleLimit extends ExecTest { private final DrillConfig c = DrillConfig.create(); @@ -55,9 +57,9 @@ public void testLimit(@Injectable final DrillbitContext bitContext, @Injectable new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; verifyLimitCount(bitContext, connection, "test1.json", 5); @@ -68,9 +70,9 @@ public void testLimitNoEnd(@Injectable final DrillbitContext bitContext, @Inject new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; verifyLimitCount(bitContext, connection, "test3.json", 95); @@ -86,9 +88,9 @@ public void testLimitAcrossBatches(@Injectable final DrillbitContext bitContext, new NonStrictExpectations(){{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; verifyLimitCount(bitContext, connection, "test2.json", 69999); @@ -102,7 +104,7 @@ public void testLimitAcrossBatches(@Injectable final DrillbitContext bitContext, } private void verifyLimitCount(DrillbitContext bitContext, UserServer.UserClientConnection connection, String testPlan, int expectedCount) throws Throwable { - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/limit/" + testPlan), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -122,7 +124,7 @@ private void verifyLimitCount(DrillbitContext bitContext, UserServer.UserClientC } private void verifySum(DrillbitContext bitContext, UserServer.UserClientConnection connection, String testPlan, int expectedCount, long expectedSum) throws Throwable { - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/limit/" + testPlan), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/orderedpartitioner/TestOrderedPartitionExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/orderedpartitioner/TestOrderedPartitionExchange.java index 7cd35be178b..8d690d3f058 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/orderedpartitioner/TestOrderedPartitionExchange.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/orderedpartitioner/TestOrderedPartitionExchange.java @@ -26,6 +26,7 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; @@ -80,7 +81,8 @@ public void twoBitTwoExchangeRun() throws Exception { if (b.getData() != null) { int rows = b.getHeader().getRowCount(); count += rows; - RecordBatchLoader loader = new RecordBatchLoader(new BootStrapContext(DrillConfig.create()).getAllocator()); + DrillConfig config = DrillConfig.create(); + RecordBatchLoader loader = new RecordBatchLoader(new BootStrapContext(config, ClassPathScanner.fromPrescan(config)).getAllocator()); loader.load(b.getHeader().getDef(), b.getData()); BigIntVector vv1 = (BigIntVector)loader.getValueAccessorById(BigIntVector.class, loader.getValueVectorId( new SchemaPath("col1", ExpressionPosition.UNKNOWN)).getFieldIds()).getValueVector(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java index 2ba72800bcc..3be6c606749 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/project/TestSimpleProjection.java @@ -19,15 +19,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -37,8 +36,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.util.VectorUtil; @@ -49,6 +48,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestSimpleProjection extends ExecTest { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleProjection.class); private final DrillConfig c = DrillConfig.create(); @@ -58,12 +60,12 @@ public void project(@Injectable final DrillbitContext bitContext, @Injectable Us new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/project/test1.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java index 4556eb7d3a9..f9f2a25b122 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/sort/TestSimpleSort.java @@ -19,15 +19,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -37,8 +36,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.vector.BigIntVector; @@ -50,6 +49,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + @Ignore public class TestSimpleSort extends ExecTest { // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleSort.class); @@ -60,12 +62,12 @@ public void sortOneKeyAscending(@Injectable final DrillbitContext bitContext, @I new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/sort/one_key_sort.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); @@ -104,12 +106,12 @@ public void sortTwoKeysOneAscendingOneDescending(@Injectable final DrillbitConte new NonStrictExpectations(){{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/sort/two_key_sort.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java index bd4c01f46bd..064ab7f68b4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceMultiRecordBatch.java @@ -18,13 +18,12 @@ package org.apache.drill.exec.physical.impl.trace; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -34,8 +33,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.vector.ValueVector; @@ -45,6 +44,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + /* * This test uses a physical plan with the mock scan that generates 100k records. * Here we inject the "trace" operator in two locations in the plan. @@ -63,11 +65,11 @@ public void testFilter(@Injectable final DrillbitContext bitContext, @Injectable bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); bitContext.getConfig(); result = c; - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/trace/multi_record_batch_trace.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java index 44f18af1d01..652c6936ff5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java @@ -18,15 +18,14 @@ package org.apache.drill.exec.physical.impl.trace; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.cache.VectorAccessibleSerializable; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -36,8 +35,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.proto.ExecProtos.FragmentHandle; import org.apache.drill.exec.proto.helper.QueryIdHelper; import org.apache.drill.exec.record.VectorAccessible; @@ -53,6 +52,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + /* * This test uses a simple physical plan with a mock-scan that * generates one row. The physical plan also consists of the @@ -77,11 +79,11 @@ public void testFilter(@Injectable final DrillbitContext bitContext, @Injectable bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); bitContext.getConfig(); result = c; - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/trace/simple_trace.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java index 7b0f1379c74..3d613ce6380 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/union/TestSimpleUnion.java @@ -19,13 +19,12 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import mockit.Injectable; -import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; -import org.apache.drill.exec.compile.CodeCompiler; +import org.apache.drill.exec.compile.CodeCompilerTestFactory; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.RootAllocatorFactory; import org.apache.drill.exec.ops.FragmentContext; @@ -35,8 +34,8 @@ import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry; import org.apache.drill.exec.physical.impl.SimpleRootExec; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.BitControl.PlanFragment; -import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.rpc.user.UserServer; import org.apache.drill.exec.server.DrillbitContext; import org.junit.Test; @@ -45,6 +44,9 @@ import com.google.common.base.Charsets; import com.google.common.io.Files; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestSimpleUnion extends ExecTest { //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleUnion.class); private final DrillConfig c = DrillConfig.create(); @@ -54,12 +56,12 @@ public void testUnion(@Injectable final DrillbitContext bitContext, @Injectable new NonStrictExpectations() {{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c); - bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); + bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(ClassPathScanner.fromPrescan(c)); bitContext.getConfig(); result = c; - bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); + bitContext.getCompiler(); result = CodeCompilerTestFactory.getTestCompiler(c); }}; - final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance()); + final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c); final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/union/test1.json"), Charsets.UTF_8)); final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c); final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/PhysicalPlanReaderTestFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/PhysicalPlanReaderTestFactory.java new file mode 100644 index 00000000000..81a05a62f65 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/PhysicalPlanReaderTestFactory.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.config.LogicalPlanPersistence; +import org.apache.drill.common.scanner.ClassPathScanner; +import org.apache.drill.common.scanner.persistence.ScanResult; +import org.apache.drill.exec.proto.CoordinationProtos; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.StoragePluginRegistry; + +public class PhysicalPlanReaderTestFactory { + + public static LogicalPlanPersistence defaultLogicalPlanPersistence(DrillConfig c) { + ScanResult scanResult = ClassPathScanner.fromPrescan(c); + return new LogicalPlanPersistence(c, scanResult); + } + + public static PhysicalPlanReader defaultPhysicalPlanReader( + DrillConfig c, + StoragePluginRegistry storageRegistry) { + ScanResult scanResult = ClassPathScanner.fromPrescan(c); + return new PhysicalPlanReader( + c, scanResult, new LogicalPlanPersistence(c, scanResult), + CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), + storageRegistry); + } + public static PhysicalPlanReader defaultPhysicalPlanReader(DrillConfig c) { + return defaultPhysicalPlanReader(c, null); + } + + public static PhysicalPlanReader defaultPhysicalPlanReader(DrillbitContext c) { + return defaultPhysicalPlanReader(c, null); + } + + public static PhysicalPlanReader defaultPhysicalPlanReader( + DrillbitContext c, + StoragePluginRegistry storageRegistry) { + return new PhysicalPlanReader( + c.getConfig(), + c.getClasspathScan(), + c.getLpPersistence(), + CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), + storageRegistry); + } + +} diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmentChecker.java b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmentChecker.java index 0fbf0bd259f..9e3bc4477fd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmentChecker.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmentChecker.java @@ -20,6 +20,7 @@ import java.util.List; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.planner.fragment.Fragment; import org.apache.drill.exec.planner.fragment.SimpleParallelizer; import org.apache.drill.exec.proto.BitControl.PlanFragment; @@ -48,7 +49,7 @@ public void checkSimpleExchangePlan() throws Exception{ private void print(String fragmentFile, int bitCount, int expectedFragmentCount) throws Exception{ System.out.println(String.format("=================Building plan fragments for [%s]. Allowing %d total Drillbits.==================", fragmentFile, bitCount)); - PhysicalPlanReader ppr = new PhysicalPlanReader(CONFIG, CONFIG.getMapper(), DrillbitEndpoint.getDefaultInstance()); + PhysicalPlanReader ppr = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); Fragment fragmentRoot = getRootFragment(ppr, fragmentFile); SimpleParallelizer par = new SimpleParallelizer(1000*1000, 5, 10, 1.2); List endpoints = Lists.newArrayList(); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmenter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmenter.java index 6491df6a1d6..6a49116ccab 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmenter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestFragmenter.java @@ -25,8 +25,8 @@ import org.apache.drill.exec.exception.FragmentSetupException; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.planner.fragment.Fragment; -import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.work.foreman.ForemanSetupException; import org.junit.Test; @@ -36,7 +36,7 @@ public class TestFragmenter extends PopUnitTestBase { @Test public void ensureOneFragment() throws FragmentSetupException, IOException, ForemanSetupException { - PhysicalPlanReader ppr = new PhysicalPlanReader(CONFIG, CONFIG.getMapper(), DrillbitEndpoint.getDefaultInstance()); + PhysicalPlanReader ppr = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); Fragment b = getRootFragment(ppr, "/physical_test1.json"); assertEquals(1, getFragmentCount(b)); assertEquals(0, b.getReceivingExchangePairs().size()); @@ -45,7 +45,7 @@ public void ensureOneFragment() throws FragmentSetupException, IOException, Fore @Test public void ensureThreeFragments() throws FragmentSetupException, IOException, ForemanSetupException { - PhysicalPlanReader ppr = new PhysicalPlanReader(CONFIG, CONFIG.getMapper(), DrillbitEndpoint.getDefaultInstance()); + PhysicalPlanReader ppr = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG); Fragment b = getRootFragment(ppr, "/physical_double_exchange.json"); logger.debug("Fragment Node {}", b); assertEquals(3, getFragmentCount(b)); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java index ac03f7aecaa..d184a124825 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/pop/TestInjectionValue.java @@ -28,6 +28,7 @@ import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.config.Screen; import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.junit.BeforeClass; import org.junit.Test; @@ -47,7 +48,7 @@ public static void setup(){ @Test public void testInjected() throws Exception{ - PhysicalPlanReader r = new PhysicalPlanReader(config, config.getMapper(), DrillbitEndpoint.getDefaultInstance()); + PhysicalPlanReader r = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(config); PhysicalPlan p = r.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/physical_screen.json"), Charsets.UTF_8)); List o = p.getSortedOperators(false); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java index d9b463446ee..c6e578f47e9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestCustomUserAuthenticator.java @@ -22,15 +22,16 @@ import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.rpc.RpcException; import org.apache.drill.exec.rpc.user.UserSession; +import org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl; import org.junit.BeforeClass; import org.junit.Test; import java.util.Properties; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_1; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_1_PASSWORD; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_2; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_2_PASSWORD; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_1; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_1_PASSWORD; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_2; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_2_PASSWORD; import static org.hamcrest.core.StringContains.containsString; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorTestImpl.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/testing/UserAuthenticatorTestImpl.java similarity index 91% rename from exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorTestImpl.java rename to exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/testing/UserAuthenticatorTestImpl.java index dc307970734..7cfab5ffd8c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/UserAuthenticatorTestImpl.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/testing/UserAuthenticatorTestImpl.java @@ -15,10 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.drill.exec.rpc.user.security; +package org.apache.drill.exec.rpc.user.security.testing; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.exception.DrillbitStartupException; +import org.apache.drill.exec.rpc.user.security.UserAuthenticationException; +import org.apache.drill.exec.rpc.user.security.UserAuthenticator; +import org.apache.drill.exec.rpc.user.security.UserAuthenticatorTemplate; import org.apache.drill.exec.util.ImpersonationUtil; import org.apache.hadoop.security.UserGroupInformation; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java index 71b0b1db2fc..107f9784ced 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestBitRpc.java @@ -18,20 +18,16 @@ package org.apache.drill.exec.server; import static org.junit.Assert.assertTrue; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.DrillBuf; import java.io.IOException; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import mockit.Injectable; -import mockit.NonStrictExpectations; - import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.scanner.ClassPathScanner; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; import org.apache.drill.exec.ExecTest; @@ -47,15 +43,12 @@ import org.apache.drill.exec.record.FragmentWritableBatch; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.WritableBatch; -import org.apache.drill.exec.rpc.RemoteConnection; -import org.apache.drill.exec.rpc.ResponseSender; import org.apache.drill.exec.rpc.RpcException; import org.apache.drill.exec.rpc.RpcOutcomeListener; import org.apache.drill.exec.rpc.control.WorkEventBus; import org.apache.drill.exec.rpc.data.AckSender; import org.apache.drill.exec.rpc.data.DataConnectionManager; import org.apache.drill.exec.rpc.data.DataResponseHandler; -import org.apache.drill.exec.rpc.data.DataRpcConfig; import org.apache.drill.exec.rpc.data.DataServer; import org.apache.drill.exec.rpc.data.DataTunnel; import org.apache.drill.exec.vector.Float8Vector; @@ -67,14 +60,21 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; +import mockit.Injectable; +import mockit.NonStrictExpectations; + public class TestBitRpc extends ExecTest { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestBitRpc.class); @Test public void testConnectionBackpressure(@Injectable WorkerBee bee, @Injectable final WorkEventBus workBus, @Injectable final FragmentManager fman, @Injectable final FragmentContext fcon) throws Exception { - final BootStrapContext c = new BootStrapContext(DrillConfig.create()); - BootStrapContext c2 = new BootStrapContext(DrillConfig.create()); + DrillConfig config1 = DrillConfig.create(); + final BootStrapContext c = new BootStrapContext(config1, ClassPathScanner.fromPrescan(config1)); + DrillConfig config2 = DrillConfig.create(); + BootStrapContext c2 = new BootStrapContext(config2, ClassPathScanner.fromPrescan(config2)); new NonStrictExpectations() {{ workBus.getFragmentManagerIfExists((FragmentHandle) any); result = fman; diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java index b029caa4ddb..8de77fbd6ed 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestOptionsAuthEnabled.java @@ -20,21 +20,21 @@ import org.apache.drill.BaseTestQuery; import org.apache.drill.common.config.DrillConfig; import org.apache.drill.exec.ExecConstants; -import org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl; +import org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl; import org.junit.BeforeClass; import org.junit.Test; -import java.util.Properties; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.ADMIN_GROUP; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.ADMIN_USER; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.ADMIN_USER_PASSWORD; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.PROCESS_USER; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.PROCESS_USER_PASSWORD; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_1; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_1_PASSWORD; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_2; +import static org.apache.drill.exec.rpc.user.security.testing.UserAuthenticatorTestImpl.TEST_USER_2_PASSWORD; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.ADMIN_GROUP; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.ADMIN_USER; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.ADMIN_USER_PASSWORD; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.PROCESS_USER; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.PROCESS_USER_PASSWORD; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_1; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_1_PASSWORD; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_2; -import static org.apache.drill.exec.rpc.user.security.UserAuthenticatorTestImpl.TEST_USER_2_PASSWORD; +import java.util.Properties; /** * Test setting system scoped options with user authentication enabled. (DRILL-3622) diff --git a/exec/java-exec/src/test/resources/drill-external-sort.conf b/exec/java-exec/src/test/resources/drill-external-sort.conf index 52e4303887a..aa3b43d07ec 100644 --- a/exec/java-exec/src/test/resources/drill-external-sort.conf +++ b/exec/java-exec/src/test/resources/drill-external-sort.conf @@ -2,9 +2,12 @@ // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl" +drill: { + classpath.scanning: { + packages += "org.apache.drill.exec.expr.fn.impl" + } -drill.exec: { + exec: { sort: { purge.threshold : 100, external: { @@ -18,4 +21,4 @@ sort: { } } } -} \ No newline at end of file +} diff --git a/exec/java-exec/src/test/resources/drill-module.conf b/exec/java-exec/src/test/resources/drill-module.conf index 76dff48c729..7925b5658b4 100644 --- a/exec/java-exec/src/test/resources/drill-module.conf +++ b/exec/java-exec/src/test/resources/drill-module.conf @@ -2,9 +2,15 @@ // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl" - -drill.exec: { +drill: { + classpath.scanning: { + packages += "org.apache.drill.exec.fn.impl.testing", + packages += "org.apache.drill.exec.physical.config", + packages += "org.apache.drill.exec.store", + packages += "org.apache.drill.exec.testing", + packages += "org.apache.drill.exec.rpc.user.security.testing" + } + exec: { cluster-id: "drillbits1" rpc: { user: { @@ -45,7 +51,7 @@ drill.exec: { zk: { connect: "localhost:2181", root: "drill/happy", - refresh: 500, + refresh: 5, timeout: 5000, retry: { count: 7200, @@ -61,4 +67,5 @@ drill.exec: { global.max.width: 100, executor.threads: 4 } + } } diff --git a/exec/java-exec/src/test/resources/drill-oom-xsort.conf b/exec/java-exec/src/test/resources/drill-oom-xsort.conf index c617a295bee..2b5ba8059ee 100644 --- a/exec/java-exec/src/test/resources/drill-oom-xsort.conf +++ b/exec/java-exec/src/test/resources/drill-oom-xsort.conf @@ -2,9 +2,11 @@ // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl" - -drill.exec: { +drill: { + classpath.scanning: { + packages += "org.apache.drill.exec.expr.fn.impl" + }, + exec: { memory: { fragment: { max: 50000000, @@ -15,4 +17,5 @@ drill.exec: { initial: 2000000 } } -} \ No newline at end of file + } +} diff --git a/exec/java-exec/src/test/resources/drill-spool-test-module.conf b/exec/java-exec/src/test/resources/drill-spool-test-module.conf index 89d248a9b95..b3058355323 100644 --- a/exec/java-exec/src/test/resources/drill-spool-test-module.conf +++ b/exec/java-exec/src/test/resources/drill-spool-test-module.conf @@ -2,9 +2,13 @@ // This file can also include any supplementary configuration information. // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. -drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl" - -drill.exec: { +drill: { + classpath.scanning: { + packages += "org.apache.drill.exec.expr.fn.impl" + packages += "org.apache.drill.exec.physical.config" + packages += "org.apache.drill.exec.store" + }, + exec: { cluster-id: "drillbits1" rpc: { user: { @@ -28,16 +32,10 @@ drill.exec: { }, use.ip : false }, - operator: { - packages += "org.apache.drill.exec.physical.config" - }, optimizer: { implementation: "org.apache.drill.exec.opt.IdentityOptimizer" }, functions: ["org.apache.drill.expr.fn.impl"], - storage: { - packages += "org.apache.drill.exec.store" - }, metrics : { context: "drillbit", jmx: { @@ -80,4 +78,5 @@ drill.exec: { delete: false, size: 0 } + } } diff --git a/exec/java-exec/src/test/resources/logback.xml b/exec/java-exec/src/test/resources/logback.xml index 54ccb427c29..bdc71da6286 100644 --- a/exec/java-exec/src/test/resources/logback.xml +++ b/exec/java-exec/src/test/resources/logback.xml @@ -16,13 +16,13 @@ limitations under the License. --> - - true - 10000 - true - ${LILITH_HOSTNAME:-localhost} - + + + + + + + @@ -31,15 +31,15 @@ - - - - + + + + - - - - + + + + @@ -47,7 +47,7 @@ - + diff --git a/exec/jdbc-all/example-conf/drill-module.conf b/exec/jdbc-all/example-conf/drill-module.conf index 082b8e60e94..6d8498c4c6c 100644 --- a/exec/jdbc-all/example-conf/drill-module.conf +++ b/exec/jdbc-all/example-conf/drill-module.conf @@ -1,13 +1,12 @@ drill: { - logical: { - operator.packages: ["org.apache.drill.common.logical.data"], - expression.packages: ["org.apache.drill.common.expression"], - function.packages: ["org.apache.drill.common.expression"], - storage.packages: [] - }, - physical: { - operator.packages: ["org.apache.drill.common.physical.pop"] + classpath.scanning: { + packages: [ + "org.apache.drill.common.logical.data", + "org.apache.drill.common.expression", + "org.apache.drill.common.expression", + "org.apache.drill.common.physical.pop", + "org.apache.drill.exec.store.mock" + ] } } -drill.exec.storage.packages += org.apache.drill.exec.store.mock diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml index f7b889c2ccf..1a66d2c1dfc 100644 --- a/exec/jdbc-all/pom.xml +++ b/exec/jdbc-all/pom.xml @@ -134,17 +134,6 @@ - - org.apache.drill - drill-common - ${project.version} - - - javassist - javassist - - - org.apache.drill.exec drill-jdbc diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java index 2792f913b48..b161e20c503 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java @@ -37,6 +37,7 @@ import org.apache.drill.common.logical.data.LogicalOperator; import org.apache.drill.common.util.Hook; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory; import org.apache.drill.jdbc.ConnectionFactory; import org.apache.drill.jdbc.ConnectionInfo; import org.junit.Assert; @@ -162,6 +163,7 @@ public static class ModelAndSchema { public ModelAndSchema(final Properties info, final ConnectionFactory factory) { this.info = info; this.adapter = new ConnectionFactoryAdapter() { + @Override public Connection createConnection() throws Exception { return factory.getConnection(new ConnectionInfo("jdbc:drill:zk=local", ModelAndSchema.this.info)); } @@ -307,6 +309,7 @@ public LogicalPlan logicalPlan() { Connection connection = null; Statement statement = null; final Hook.Closeable x = Hook.LOGICAL_PLAN.add(new Function() { + @Override public Void apply(String o) { plan0[0] = o; return null; @@ -317,7 +320,7 @@ public Void apply(String o) { statement = connection.prepareStatement(sql); statement.close(); final String plan = plan0[0].trim(); - return LogicalPlan.parse(DrillConfig.create(), plan); + return LogicalPlan.parse(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(DrillConfig.create()), plan); } catch (Exception e) { throw new RuntimeException(e); } finally { diff --git a/pom.xml b/pom.xml index f2242dd561e..80e72f0ec7d 100644 --- a/pom.xml +++ b/pom.xml @@ -43,6 +43,7 @@ --> 1.0.0 2.4.1 + 0.9.14 @@ -156,7 +157,6 @@ - false **/*.log @@ -202,7 +202,7 @@ checkstyle-validation - validate + verify **/*.java UTF-8 @@ -281,7 +281,7 @@ validate_java_and_maven_version - validate + verify enforce @@ -299,6 +299,7 @@ avoid_bad_dependencies + verify enforce @@ -366,7 +367,7 @@ org.apache.rat apache-rat-plugin - 0.10 + 0.11 org.apache.maven.plugins @@ -1885,6 +1886,7 @@ + tools/fmpp protocol common exec diff --git a/src/main/resources/checkstyle-config.xml b/src/main/resources/checkstyle-config.xml index 0758908c1ec..32bbbe85b18 100644 --- a/src/main/resources/checkstyle-config.xml +++ b/src/main/resources/checkstyle-config.xml @@ -36,6 +36,7 @@ + diff --git a/tools/fmpp/pom.xml b/tools/fmpp/pom.xml new file mode 100644 index 00000000000..77a00b7fcd5 --- /dev/null +++ b/tools/fmpp/pom.xml @@ -0,0 +1,82 @@ + + + + 4.0.0 + + + tools-parent + org.apache.drill.tools + 1.2.0-SNAPSHOT + + + drill-fmpp-maven-plugin + maven-plugin + tools/freemarker codegen tooling + + + + commons-io + commons-io + 2.4 + + + org.apache.maven + maven-core + 3.3.3 + + + commons-logging + commons-logging + + + + + org.apache.maven + maven-plugin-api + 3.3.3 + + + net.sourceforge.fmpp + fmpp + ${fmpp.version} + + + + + + org.apache.maven.plugins + maven-plugin-plugin + 3.4 + + drill-fmpp + + + + default-descriptor + + descriptor + + process-classes + + + help-descriptor + + helpmojo + + process-classes + + + + + + diff --git a/tools/fmpp/src/main/java/org/apache/drill/fmpp/mojo/FMPPMojo.java b/tools/fmpp/src/main/java/org/apache/drill/fmpp/mojo/FMPPMojo.java new file mode 100644 index 00000000000..6474bb7b6a5 --- /dev/null +++ b/tools/fmpp/src/main/java/org/apache/drill/fmpp/mojo/FMPPMojo.java @@ -0,0 +1,227 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.fmpp.mojo; + +import static java.lang.String.format; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.io.FileUtils; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.project.MavenProject; + +import com.google.common.base.Stopwatch; + +import fmpp.Engine; +import fmpp.ProgressListener; +import fmpp.progresslisteners.TerseConsoleProgressListener; +import fmpp.setting.Settings; +import fmpp.util.MiscUtil; + +/** + * a maven plugin to run the freemarker generation incrementally + * (if output has not changed, the files are not touched) + * @goal generate + * @phase generate-sources + */ +public class FMPPMojo extends AbstractMojo { + + /** + * Used to add new source directories to the build. + * @parameter default-value="${project}" + * @required + * @readonly + **/ + private MavenProject project; + + /** + * Where to find the FreeMarker template files. + * + * @parameter default-value="src/main/resources/fmpp/templates/" + * @required + */ + private File templates; + + + /** + * Where to write the generated files of the output files. + * + * @parameter default-value="${project.build.directory}/generated-sources/fmpp/" + * @required + */ + private File output; + + /** + * Location of the FreeMarker config file. + * + * @parameter default-value="src/main/resources/fmpp/config.fmpp" + * @required + */ + private File config; + + /** + * compilation scope to be added to ("compile" or "test") + * + * @parameter default-value="compile" + * @required + */ + private String scope; + + @Override + public void execute() throws MojoExecutionException, MojoFailureException { + if (project == null) { + throw new MojoExecutionException("This plugin can only be used inside a project."); + } + String outputPath = output.getAbsolutePath(); + if ((!output.exists() && !output.mkdirs()) || !output.isDirectory()) { + throw new MojoFailureException("can not write to output dir: " + outputPath); + } + String templatesPath = templates.getAbsolutePath(); + if (!templates.exists() || !templates.isDirectory()) { + throw new MojoFailureException("templates not found in dir: " + outputPath); + } + + // add the output directory path to the project source directories + switch (scope) { + case "compile": + project.addCompileSourceRoot(outputPath); + break; + case "test": + project.addTestCompileSourceRoot(outputPath); + break; + default: + throw new MojoFailureException("scope must be compile or test"); + } + + final Stopwatch sw = new Stopwatch().start(); + try { + getLog().info(format("Freemarker generation:\n scope: %s,\n config: %s,\n templates: %s", + scope, config.getAbsolutePath(), templatesPath)); + final File tmp = Files.createTempDirectory("freemarker-tmp").toFile(); + String tmpPath = tmp.getAbsolutePath(); + final String tmpPathNormalized = tmpPath.endsWith("/") ? tmpPath : tmpPath + "/"; + Settings settings = new Settings(new File(".")); + settings.set("sourceRoot", templatesPath); + settings.set("outputRoot", tmp.getAbsolutePath()); + settings.load(config); + settings.addProgressListener(new TerseConsoleProgressListener()); + settings.addProgressListener(new ProgressListener() { + @Override + public void notifyProgressEvent( + Engine engine, int event, + File src, int pMode, + Throwable error, Object param) + throws Exception { + if (event == EVENT_END_PROCESSING_SESSION) { + getLog().info(format("Freemarker generation took %dms", sw.elapsed(TimeUnit.MILLISECONDS))); + sw.reset(); + Report report = moveIfChanged(tmp, tmpPathNormalized); + getLog().info(format("Incremental output update took %dms", sw.elapsed(TimeUnit.MILLISECONDS))); + getLog().info(format("new: %d", report.newFiles)); + getLog().info(format("changed: %d", report.changedFiles)); + getLog().info(format("unchanged: %d", report.unchangedFiles)); + } + } + } ); + settings.execute(); + } catch (Exception e) { + throw new MojoFailureException(MiscUtil.causeMessages(e), e); + } + } + + private static final class Report { + int changedFiles; + int unchangedFiles; + int newFiles; + Report(int changedFiles, int unchangedFiles, int newFiles) { + super(); + this.changedFiles = changedFiles; + this.unchangedFiles = unchangedFiles; + this.newFiles = newFiles; + } + public Report() { + this(0, 0, 0); + } + void add(Report other) { + changedFiles += other.changedFiles; + unchangedFiles += other.unchangedFiles; + newFiles += other.newFiles; + } + public void addChanged() { + ++ changedFiles; + } + public void addNew() { + ++ newFiles; + } + public void addUnchanged() { + ++ unchangedFiles; + } + } + + private Report moveIfChanged(File root, String tmpPath) throws MojoFailureException, IOException { + Report report = new Report(); + for (File file : root.listFiles()) { + if (file.isDirectory()) { + report.add(moveIfChanged(file, tmpPath)); + if (!file.delete()) { + throw new MojoFailureException(format("can not delete %s", file)); + } + } else { + String absPath = file.getAbsolutePath(); + if (!absPath.startsWith(tmpPath)) { + throw new MojoFailureException(format("%s should start with %s", absPath, tmpPath)); + } + String relPath = absPath.substring(tmpPath.length()); + File outputFile = new File(output, relPath); + if (!outputFile.exists()) { + report.addNew(); + } else if (!FileUtils.contentEquals(file, outputFile)) { + getLog().info(format("%s has changed", relPath)); + if (!outputFile.delete()) { + throw new MojoFailureException(format("can not delete %s", outputFile)); + } + report.addChanged(); + } else { + report.addUnchanged(); + } + if (!outputFile.exists()) { + File parentDir = outputFile.getParentFile(); + if (parentDir.exists() && !parentDir.isDirectory()) { + throw new MojoFailureException(format("can not move %s to %s as %s is not a dir", file, outputFile, parentDir)); + } + if (!parentDir.exists() && !parentDir.mkdirs()) { + throw new MojoFailureException(format("can not move %s to %s as dir %s can not be created", file, outputFile, parentDir)); + } + FileUtils.moveFile(file, outputFile); + } else { + if (!file.delete()) { + throw new MojoFailureException(format("can not delete %s", file)); + } + } + } + } + return report; + } + +} diff --git a/tools/pom.xml b/tools/pom.xml new file mode 100644 index 00000000000..4e4a475456a --- /dev/null +++ b/tools/pom.xml @@ -0,0 +1,34 @@ + + + + 4.0.0 + + drill-root + org.apache.drill + 1.2.0-SNAPSHOT + + + org.apache.drill.tools + tools-parent + pom + tools/Parent Pom + + + fmpp + +