DRILL-5152: Enhance the mock data source: better data, SQL access

exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information

exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java

@@ -67,7 +67,7 @@ protected final void setColumns(Collection<SchemaPath> projected) {
     Collection<SchemaPath> columnsToRead = projected;
 
     // If no column is required (SkipQuery), by default it will use DEFAULT_COLS_TO_READ .
-    // Handling SkipQuery is storage-plugin specif : JSON, text reader, parquet will override, in order to
+    // Handling SkipQuery is storage-plugin specific : JSON, text reader, parquet will override, in order to
     // improve query performance.
     if (projected.isEmpty()) {
       columnsToRead = getDefaultColumnsToRead();

exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -71,7 +71,7 @@ public interface StoragePlugin extends SchemaFactory, AutoCloseable {
   public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection, List<SchemaPath> columns)
       throws IOException;
 
-  /** Method returns a jackson serializable object that extends a StoragePluginConfig
+  /** Method returns a Jackson serializable object that extends a StoragePluginConfig
   * @return an extension of StoragePluginConfig
   */
   public StoragePluginConfig getConfig();
@@ -80,5 +80,4 @@ public AbstractGroupScan getPhysicalScan(String userName, JSONOptions selection,
    * Initialize the storage plugin. The storage plugin will not be used until this method is called.
    */
   public void start() throws IOException;
-
 }

exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -51,7 +51,7 @@ public interface StoragePluginRegistry extends Iterable<Map.Entry<String, Storag
    * @param name
    *          The name of the plugin
    * @param config
-   *          The plugin confgiruation
+   *          The plugin configuration
    * @param persist
    *          Whether to persist the plugin for later use or treat it as ephemeral.
    * @return The StoragePlugin instance.

exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -125,9 +125,10 @@ public void init() throws DrillbitStartupException {
     availablePlugins = findAvailablePlugins(classpathScan);
 
     // create registered plugins defined in "storage-plugins.json"
-    this.plugins.putAll(createPlugins());
+    plugins.putAll(createPlugins());
   }
 
+  @SuppressWarnings("resource")
   private Map<String, StoragePlugin> createPlugins() throws DrillbitStartupException {
     try {
       /*
@@ -145,7 +146,7 @@ private Map<String, StoragePlugin> createPlugins() throws DrillbitStartupExcepti
             String pluginsData = Resources.toString(url, Charsets.UTF_8);
             StoragePlugins plugins = lpPersistence.getMapper().readValue(pluginsData, StoragePlugins.class);
             for (Map.Entry<String, StoragePluginConfig> config : plugins) {
-              if (!pluginSystemTable.putIfAbsent(config.getKey(), config.getValue())) {
+              if (!definePluginConfig(config.getKey(), config.getValue())) {
                 logger.warn("Duplicate plugin instance '{}' defined in [{}, {}], ignoring the later one.",
                     config.getKey(), pluginURLMap.get(config.getKey()), url);
                 continue;
@@ -185,13 +186,32 @@ private Map<String, StoragePlugin> createPlugins() throws DrillbitStartupExcepti
     }
   }
 
+  /**
+   * Add a plugin and configuration. Assumes neither exists. Primarily
+   * for testing.
+   *
+   * @param name plugin name
+   * @param config plugin config
+   * @param plugin plugin implementation
+   */
+
+  public void definePlugin(String name, StoragePluginConfig config, StoragePlugin plugin) {
+    addPlugin(name, plugin);
+    definePluginConfig(name, config);
+  }
+
+  private boolean definePluginConfig(String name, StoragePluginConfig config) {
+    return pluginSystemTable.putIfAbsent(name, config);
+  }
+
   @Override
   public void addPlugin(String name, StoragePlugin plugin) {
     plugins.put(name, plugin);
   }
 
   @Override
   public void deletePlugin(String name) {
+    @SuppressWarnings("resource")
     StoragePlugin plugin = plugins.remove(name);
     closePlugin(plugin);
     pluginSystemTable.delete(name);
@@ -209,6 +229,7 @@ private void closePlugin(StoragePlugin plugin) {
     }
   }
 
+  @SuppressWarnings("resource")
   @Override
   public StoragePlugin createOrUpdate(String name, StoragePluginConfig config, boolean persist)
       throws ExecutionSetupException {
@@ -299,6 +320,7 @@ public StoragePlugin getPlugin(StoragePluginConfig config) throws ExecutionSetup
     }
   }
 
+  @SuppressWarnings("resource")
   @Override
   public FormatPlugin getFormatPlugin(StoragePluginConfig storageConfig, FormatPluginConfig formatConfig)
       throws ExecutionSetupException {
@@ -346,6 +368,7 @@ public SchemaFactory getSchemaFactory() {
 
   public class DrillSchemaFactory implements SchemaFactory {
 
+    @SuppressWarnings("resource")
     @Override
     public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException {
       Stopwatch watch = Stopwatch.createStarted();

exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/ColumnDef.java

@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.mock;
+
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.store.mock.MockGroupScanPOP.MockColumn;
+
+/**
+ * Defines a column for the "enhanced" version of the mock data
+ * source. This class is built from the column definitions in either
+ * the physical plan or an SQL statement (which gives rise to a
+ * physical plan.)
+ */
+
+public class ColumnDef {
+  public MockColumn mockCol;
+  public String name;
+  public int width;
+  public FieldGen generator;
+
+  public ColumnDef(MockColumn mockCol) {
+    this.mockCol = mockCol;
+    name = mockCol.getName();
+    width = TypeHelper.getSize(mockCol.getMajorType());
+    makeGenerator();
+  }
+
+  /**
+   * Create the data generator class for this column. The generator is
+   * created to match the data type by default. Or, the plan can
+   * specify a generator class (in which case the plan must ensure that
+   * the generator produces the correct value for the column data type.)
+   * The generator names a class: either a fully qualified name, or a
+   * class in this package.
+   */
+
+  private void makeGenerator() {
+    String genName = mockCol.getGenerator();
+    if (genName != null) {
+      if (! genName.contains(".")) {
+        genName = "org.apache.drill.exec.store.mock." + genName;
+      }
+      try {
+        ClassLoader cl = getClass().getClassLoader();
+        Class<?> genClass = cl.loadClass(genName);
+        generator = (FieldGen) genClass.newInstance();
+      } catch (ClassNotFoundException | InstantiationException
+          | IllegalAccessException | ClassCastException e) {
+        throw new IllegalArgumentException("Generator " + genName + " is undefined for mock field " + name);
+      }
+      generator.setup(this);
+      return;
+    }
+
+    makeDefaultGenerator();
+  }
+
+  private void makeDefaultGenerator() {
+
+    MinorType minorType = mockCol.getMinorType();
+    switch (minorType) {
+    case BIGINT:
+      break;
+    case BIT:
+      break;
+    case DATE:
+      break;
+    case DECIMAL18:
+      break;
+    case DECIMAL28DENSE:
+      break;
+    case DECIMAL28SPARSE:
+      break;
+    case DECIMAL38DENSE:
+      break;
+    case DECIMAL38SPARSE:
+      break;
+    case DECIMAL9:
+      break;
+    case FIXED16CHAR:
+      break;
+    case FIXEDBINARY:
+      break;
+    case FIXEDCHAR:
+      break;
+    case FLOAT4:
+      break;
+    case FLOAT8:
+      generator = new DoubleGen();
+      break;
+    case GENERIC_OBJECT:
+      break;
+    case INT:
+      generator = new IntGen();
+      break;
+    case INTERVAL:
+      break;
+    case INTERVALDAY:
+      break;
+    case INTERVALYEAR:
+      break;
+    case LATE:
+      break;
+    case LIST:
+      break;
+    case MAP:
+      break;
+    case MONEY:
+      break;
+    case NULL:
+      break;
+    case SMALLINT:
+      break;
+    case TIME:
+      break;
+    case TIMESTAMP:
+      break;
+    case TIMESTAMPTZ:
+      break;
+    case TIMETZ:
+      break;
+    case TINYINT:
+      break;
+    case UINT1:
+      break;
+    case UINT2:
+      break;
+    case UINT4:
+      break;
+    case UINT8:
+      break;
+    case UNION:
+      break;
+    case VAR16CHAR:
+      break;
+    case VARBINARY:
+      break;
+    case VARCHAR:
+      generator = new StringGen();
+      break;
+    default:
+      break;
+    }
+    if (generator == null) {
+      throw new IllegalArgumentException("No default column generator for column " + name + " of type " + minorType);
+    }
+    generator.setup(this);
+  }
+
+  public ColumnDef(MockColumn mockCol, int rep) {
+    this(mockCol);
+    name += Integer.toString(rep);
+  }
+
+  public MockColumn getConfig() {
+    return mockCol;
+  }
+
+  public String getName() {
+    return name;
+  }
+}

exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/DateGen.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.mock;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VarCharVector;
+
+/**
+ * Very simple date vaue generator that produces ISO dates
+ * uniformly distributed over the last year. ISO format
+ * is: 2016-12-07.
+ * <p>
+ * There are many possible date formats; this class does not
+ * attempt to generate all of them. Drill provides a date
+ * type, but we use a string format because example cases from
+ * people using the product often read text files. Finally, we
+ * (reluctantly) use the old-style date formats instead of the
+ * new Java 8 classes because Drill prefers to build with Java 7.
+ */
+
+public class DateGen implements FieldGen {
+
+  private final int ONE_DAY = 24 * 60 * 60 * 1000;
+  private final int ONE_YEAR = ONE_DAY * 365;
+
+  private final Random rand = new Random();
+  private long baseTime;
+  private SimpleDateFormat fmt;
+
+  public DateGen() {
+    // Start a year ago.
+    baseTime = System.currentTimeMillis() - ONE_YEAR;
+    fmt = new SimpleDateFormat("yyyy-mm-DD");
+  }
+
+  @Override
+  public void setup(ColumnDef colDef) { }
+
+  private long value() {
+    return baseTime + rand.nextInt(365) * ONE_DAY;
+  }
+
+  @Override
+  public void setValue(ValueVector v, int index) {
+    VarCharVector vector = (VarCharVector) v;
+    long randTime = baseTime + value();
+    String str = fmt.format(new Date(randTime));
+    vector.getMutator().setSafe(index, str.getBytes());
+  }
+}