DRILL-1545: Allow custom extensions for json files

Fix case where compressed files are skipped. Add jackson annotation to avoid serializing the default extensions list to enable forwards compatibility.
apache · May 7, 2015 · efb4e70 · efb4e70
1 parent b0ac3fb
commit efb4e70
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 14 deletions.
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/BasicFormatMatcher.java
@@ -89,17 +89,20 @@ protected final boolean isReadable(DrillFileSystem fs, FileStatus status) throws
     if (compressible) {
       codec = codecFactory.getCodec(status.getPath());
     }
-    String fileName;
+    String fileName = status.getPath().toString();
+    String fileNameHacked = null;
     if (codec != null) {
-      String path = status.getPath().toString();
+        fileNameHacked = fileName.substring(0, fileName.lastIndexOf('.'));
-      fileName = path.substring(0, path.lastIndexOf('.'));
-    } else {
-      fileName = status.getPath().toString();
     }
+
+    // Check for a matching pattern for compressed and uncompressed file name
     for (Pattern p : patterns) {
       if (p.matcher(fileName).matches()) {
         return true;
       }
+      if (fileNameHacked != null  &&  p.matcher(fileNameHacked).matches()) {
+        return true;
+      }
     }
 
     if (matcher.matches(fs, status)) {

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
@@ -21,7 +21,9 @@
 import java.util.List;
 import java.util.Map;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.logical.FormatPluginConfig;
@@ -45,20 +47,18 @@
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.Maps;
 
-public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> {
+public class
+        JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> {
 
   private static final boolean IS_COMPRESSIBLE = true;
   private static final String DEFAULT_NAME = "json";
-  private static final List<String> DEFAULT_EXTS = ImmutableList.of("json");
 
   public JSONFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig storageConfig) {
     this(name, context, fsConf, storageConfig, new JSONFormatConfig());
   }
 
-  public JSONFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig config,
+  public JSONFormatPlugin(String name, DrillbitContext context, Configuration fsConf, StoragePluginConfig config, JSONFormatConfig formatPluginConfig) {
-      JSONFormatConfig formatPluginConfig) {
+    super(name, context, fsConf, config, formatPluginConfig, true, false, false, IS_COMPRESSIBLE, formatPluginConfig.getExtensions(), DEFAULT_NAME);
-    super(name, context, fsConf, config, formatPluginConfig, true, false, false, IS_COMPRESSIBLE,
-        DEFAULT_EXTS, DEFAULT_NAME);
   }
 
   @Override
@@ -92,6 +92,18 @@ public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer)
   @JsonTypeName("json")
   public static class JSONFormatConfig implements FormatPluginConfig {
 
+    public List<String> extensions;
+    private static final List<String> DEFAULT_EXTS = ImmutableList.of("json");
+
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+    public List<String> getExtensions() {
+      if (extensions == null) {
+        // when loading an old JSONFormatConfig that doesn't contain an "extensions" attribute
+        return DEFAULT_EXTS;
+      }
+      return extensions;
+    }
+
     @Override
     public int hashCode() {
       return 31;
@@ -106,9 +118,9 @@ public boolean equals(Object obj) {
       } else if (getClass() == obj.getClass()) {
         return true;
       }
+
       return false;
     }
-
   }
 
   @Override

diff --git a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
@@ -33,7 +33,8 @@
           type: "parquet"
         },
         "json" : {
-          type: "json"
+          type: "json",
+          extensions: [ "json" ]
         },
         "avro" : {
           type: "avro"
@@ -56,7 +57,8 @@
                   delimiter: "\t"
         },
         "json" : {
-          type: "json"
+          type: "json",
+          extensions: [ "json" ]
         },
         "parquet" : {
           type: "parquet"