DRILL-4047: Support querying a table with options

- simplify DrillOperatorTable; separate TranslatableTable from DrillTable - fix table name in error message; improve tests debugability - FormatCreator refactor Update calcite fork version to r9 This closes #246
apache · Nov 25, 2015 · f7786cc · f7786cc
1 parent 367d74a
commit f7786cc
Show file tree

Hide file tree

Showing 21 changed files with 1,116 additions and 94 deletions.
diff --git a/...ib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HiveUDFOperator.java b/...ib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HiveUDFOperator.java
@@ -89,5 +89,10 @@ public SqlOperandCountRange getOperandCountRange() {
     public String getAllowedSignatures(SqlOperator op, String opName) {
       return opName + "(HiveUDF - Opaque)";
     }
+
+    @Override
+    public boolean isOptional(int arg) {
+      return false;
+    }
   }
 }
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
@@ -583,6 +583,11 @@
               <lookAhead>2</lookAhead>
               <isStatic>false</isStatic>
               <outputDirectory>${project.build.directory}/generated-sources/</outputDirectory>
+<!--
+              <debugParser>true</debugParser>
+              <debugLookAhead>true</debugLookAhead>
+              <debugTokenManager>true</debugTokenManager>
+ -->
             </configuration>
           </execution>
         </executions>

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java
@@ -18,6 +18,7 @@
 package org.apache.drill.exec.planner.common;
 
 import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.drill.exec.planner.logical.DrillTranslatableTable;
 import org.apache.calcite.rel.core.TableScan;
 import org.apache.calcite.plan.Convention;
 import org.apache.calcite.plan.RelOptCluster;
@@ -34,7 +35,11 @@ public abstract class DrillScanRelBase extends TableScan implements DrillRelNode
 
   public DrillScanRelBase(Convention convention, RelOptCluster cluster, RelTraitSet traits, RelOptTable table) {
     super(cluster, traits, table);
-    this.drillTable = table.unwrap(DrillTable.class);
+    DrillTable unwrap = table.unwrap(DrillTable.class);
+    if (unwrap == null) {
+      unwrap = table.unwrap(DrillTranslatableTable.class).getDrillTable();
+    }
+    this.drillTable = unwrap;
     assert drillTable != null;
   }
 

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
@@ -23,7 +23,6 @@
 import org.apache.calcite.schema.Statistic;
 import org.apache.calcite.schema.Statistics;
 import org.apache.calcite.schema.Table;
-
 import org.apache.drill.common.JSONOptions;
 import org.apache.drill.common.logical.StoragePluginConfig;
 import org.apache.drill.exec.physical.base.GroupScan;

diff --git a/...java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTranslatableTable.java b/...java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTranslatableTable.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelOptTable.ToRelContext;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.schema.Statistic;
+import org.apache.calcite.schema.TranslatableTable;
+import org.apache.calcite.schema.Schema.TableType;
+import org.apache.drill.exec.planner.logical.DrillTable;
+
+/**
+ * TableMacros must return a TranslatableTable
+ * This class adapts the existing DrillTable to a TranslatableTable
+ */
+public class DrillTranslatableTable implements TranslatableTable {
+
+  /** all calls will be delegated to this field */
+  private final DrillTable drillTable;
+
+  public DrillTranslatableTable(DrillTable drillTable) {
+    this.drillTable = drillTable;
+  }
+
+  public DrillTable getDrillTable() {
+    return drillTable;
+  }
+
+  @Override
+  public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+    return drillTable.getRowType(typeFactory);
+  }
+
+  @Override
+  public Statistic getStatistic() {
+    return drillTable.getStatistic();
+  }
+
+  @Override
+  public RelNode toRel(ToRelContext context, RelOptTable table) {
+    return drillTable.toRel(context, table);
+  }
+
+  @Override
+  public TableType getJdbcTableType() {
+    return drillTable.getJdbcTableType();
+  }
+
+  @Override
+  public int hashCode() {
+    return drillTable.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    return drillTable.equals(obj);
+  }
+
+  @Override
+  public String toString() {
+    return drillTable.toString();
+  }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/Checker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/Checker.java
@@ -49,4 +49,9 @@ public Consistency getConsistency() {
     return Consistency.NONE;
   }
 
+  @Override
+  public boolean isOptional(int i) {
+    return false;
+  }
+
 }
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillOperatorTable.java
@@ -50,22 +50,13 @@ public void add(String name, SqlOperator op) {
 
   @Override
   public void lookupOperatorOverloads(SqlIdentifier opName, SqlFunctionCategory category, SqlSyntax syntax, List<SqlOperator> operatorList) {
-    if (syntax == SqlSyntax.FUNCTION) {
-
-      // add optiq.
-      inner.lookupOperatorOverloads(opName, category, syntax, operatorList);
-
-      if(!operatorList.isEmpty()){
-        return;
-      }
+    inner.lookupOperatorOverloads(opName, category, syntax, operatorList);
 
+    if (operatorList.isEmpty() && syntax == SqlSyntax.FUNCTION && opName.isSimple()) {
       List<SqlOperator> drillOps = opMap.get(opName.getSimple().toLowerCase());
-      if(drillOps != null){
+      if (drillOps != null) {
         operatorList.addAll(drillOps);
       }
-
-    } else {
-      inner.lookupOperatorOverloads(opName, category, syntax, operatorList);
     }
   }
 

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -57,7 +57,7 @@ public class FileSystemPlugin extends AbstractStoragePlugin{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemPlugin.class);
 
   private final FileSystemSchemaFactory schemaFactory;
-  private final Map<String, FormatPlugin> formatPluginsByName;
+  private final FormatCreator formatCreator;
   private final Map<FormatPluginConfig, FormatPlugin> formatPluginsByConfig;
   private final FileSystemConfig config;
   private final Configuration fsConf;
@@ -73,10 +73,10 @@ public FileSystemPlugin(FileSystemConfig config, DrillbitContext context, String
       fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
       fsConf.set("fs.drill-local.impl", LocalSyncableFileSystem.class.getName());
 
-      formatPluginsByName = FormatCreator.getFormatPlugins(context, fsConf, config, context.getClasspathScan());
+      formatCreator = new FormatCreator(context, fsConf, config, context.getClasspathScan());
       List<FormatMatcher> matchers = Lists.newArrayList();
       formatPluginsByConfig = Maps.newHashMap();
-      for (FormatPlugin p : formatPluginsByName.values()) {
+      for (FormatPlugin p : formatCreator.getConfiguredFormatPlugins()) {
         matchers.add(p.getMatcher());
         formatPluginsByConfig.put(p.getConfig(), p);
       }
@@ -85,13 +85,13 @@ public FileSystemPlugin(FileSystemConfig config, DrillbitContext context, String
       List<WorkspaceSchemaFactory> factories = Lists.newArrayList();
       if (!noWorkspace) {
         for (Map.Entry<String, WorkspaceConfig> space : config.workspaces.entrySet()) {
-          factories.add(new WorkspaceSchemaFactory(this, space.getKey(), name, space.getValue(), matchers, context.getLpPersistence()));
+          factories.add(new WorkspaceSchemaFactory(this, space.getKey(), name, space.getValue(), matchers, context.getLpPersistence(), context.getClasspathScan()));
         }
       }
 
       // if the "default" workspace is not given add one.
       if (noWorkspace || !config.workspaces.containsKey(DEFAULT_WS_NAME)) {
-        factories.add(new WorkspaceSchemaFactory(this, DEFAULT_WS_NAME, name, WorkspaceConfig.DEFAULT, matchers, context.getLpPersistence()));
+        factories.add(new WorkspaceSchemaFactory(this, DEFAULT_WS_NAME, name, WorkspaceConfig.DEFAULT, matchers, context.getLpPersistence(), context.getClasspathScan()));
       }
 
       this.schemaFactory = new FileSystemSchemaFactory(name, factories);
@@ -116,12 +116,12 @@ public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection,
     FormatSelection formatSelection = selection.getWith(lpPersistance, FormatSelection.class);
     FormatPlugin plugin;
     if (formatSelection.getFormat() instanceof NamedFormatPluginConfig) {
-      plugin = formatPluginsByName.get( ((NamedFormatPluginConfig) formatSelection.getFormat()).name);
+      plugin = formatCreator.getFormatPluginByName( ((NamedFormatPluginConfig) formatSelection.getFormat()).name);
     } else {
       plugin = formatPluginsByConfig.get(formatSelection.getFormat());
     }
     if (plugin == null) {
-      throw new IOException(String.format("Failure getting requested format plugin named '%s'.  It was not one of the format plugins registered.", formatSelection.getFormat()));
+      plugin = formatCreator.newFormatPlugin(formatSelection.getFormat());
     }
     return plugin.getGroupScan(userName, formatSelection.getSelection(), columns);
   }
@@ -132,12 +132,12 @@ public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws
   }
 
   public FormatPlugin getFormatPlugin(String name) {
-    return formatPluginsByName.get(name);
+    return formatCreator.getFormatPluginByName(name);
   }
 
   public FormatPlugin getFormatPlugin(FormatPluginConfig config) {
     if (config instanceof NamedFormatPluginConfig) {
-      return formatPluginsByName.get(((NamedFormatPluginConfig) config).name);
+      return formatCreator.getFormatPluginByName(((NamedFormatPluginConfig) config).name);
     } else {
       return formatPluginsByConfig.get(config);
     }
@@ -146,7 +146,7 @@ public FormatPlugin getFormatPlugin(FormatPluginConfig config) {
   @Override
   public Set<StoragePluginOptimizerRule> getOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
     Builder<StoragePluginOptimizerRule> setBuilder = ImmutableSet.builder();
-    for(FormatPlugin plugin : this.formatPluginsByName.values()){
+    for(FormatPlugin plugin : formatCreator.getConfiguredFormatPlugins()){
       Set<StoragePluginOptimizerRule> rules = plugin.getOptimizerRules();
       if(rules != null && rules.size() > 0){
         setBuilder.addAll(rules);