Support for MB tiles data provider, including tests

mrgeo-core/src/main/java/org/mrgeo/hdfs/utils/HadoopFileUtils.java

@@ -113,8 +113,8 @@ public static void copyFileToHdfs(String fromFile, String toFile,
 {
   Path toPath = new Path(toFile);
   Path fromPath = new Path(fromFile);
-  FileSystem srcFS = getFileSystem(toPath);
-  FileSystem dstFS = getFileSystem(fromPath);
+  FileSystem srcFS = getFileSystem(fromPath);
+  FileSystem dstFS = getFileSystem(toPath);
 
   Configuration conf = HadoopUtils.createConfiguration();
   InputStream in = null;
@@ -134,6 +134,30 @@ public static void copyFileToHdfs(String fromFile, String toFile,
   }
 }
 
+public static void copyFileFromHdfs(String fromFile, File toLocalFile) throws IOException
+{
+  Path fromPath = new Path(fromFile);
+  FileSystem srcFS = getFileSystem(fromPath);
+
+  Configuration conf = HadoopUtils.createConfiguration();
+  InputStream in = null;
+  OutputStream out = null;
+  try
+  {
+    in = srcFS.open(fromPath);
+    out = new FileOutputStream(toLocalFile);
+
+    IOUtils.copyBytes(in, out, conf, true);
+    toLocalFile.deleteOnExit();
+  }
+  catch (IOException e)
+  {
+    IOUtils.closeStream(out);
+    IOUtils.closeStream(in);
+    throw e;
+  }
+}
+
 public static void copyToHdfs(Path fromDir, Path toDir, String fileName)
     throws IOException
 {

mrgeo-core/src/main/java/org/mrgeo/hdfs/vector/HdfsVectorDataProvider.java

@@ -207,7 +207,7 @@ private static Path resolveName(Configuration conf, String input,
 
 @SuppressWarnings("squid:S1166") // Exception caught and handled
 @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "method only makes complete URI out of the name")
-private static Path resolveNameToPath(Configuration conf, String input,
+public static Path resolveNameToPath(Configuration conf, String input,
     ProviderProperties providerProperties, boolean mustExist) throws IOException
 {
   if (input.indexOf('/') >= 0)

mrgeo-core/src/main/java/org/mrgeo/hdfs/vector/HdfsVectorDataProviderFactory.java

@@ -127,7 +127,7 @@ public void delete(String name,
   }
 }
 
-private Path getBasePath(Configuration conf)
+public static Path getBasePath(Configuration conf)
 {
   return HdfsVectorDataProvider.getBasePath(conf);
 }

mrgeo-core/src/main/java/org/mrgeo/utils/FileUtils.java

@@ -32,12 +32,9 @@ public class FileUtils
 
 public static File createUniqueTmpDir() throws IOException
 {
-  File baseDir = new File(System.getProperty("java.io.tmpdir"));
-
-  String username = "mrgeo-" + System.getProperty("user.name");
-  String baseName = "-" + System.currentTimeMillis();
-
-  File tempDir = new File(baseDir, username + "/" + baseName);
+  File tmpUserDir = createTmpUserDir();
+  String baseName = "" + System.currentTimeMillis();
+  File tempDir = new File(tmpUserDir, baseName);
 
   return createDisposibleDirectory(tempDir);
 }

mrgeo-dataprovider/mrgeo-dataprovider-geowave/src/main/resources/META-INF/services/org.mrgeo.data.vector.VectorDataProviderFactory

@@ -13,35 +13,4 @@
 # See the License for the specific language governing permissions and limitations under the License.
 #
 
-#
-# Copyright 2009-2016 DigitalGlobe, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-#
-#
-
-#
-# Copyright 2009-2015 DigitalGlobe, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-#
-
 org.mrgeo.data.vector.geowave.GeoWaveVectorDataProviderFactory

mrgeo-dataprovider/mrgeo-dataprovider-mbvectortiles/src/main/java/org/mrgeo/data/vector/mbvectortiles/MbVectorTilesDataProvider.java

@@ -19,15 +19,22 @@
 import com.almworks.sqlite4java.SQLiteException;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.RecordWriter;
 import org.mrgeo.data.ProviderProperties;
 import org.mrgeo.data.vector.*;
 import org.mrgeo.geometry.Geometry;
+import org.mrgeo.hdfs.utils.HadoopFileUtils;
+import org.mrgeo.hdfs.vector.HdfsVectorDataProvider;
+import org.mrgeo.utils.FileUtils;
 import org.mrgeo.utils.tms.Bounds;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
@@ -36,14 +43,18 @@ public class MbVectorTilesDataProvider extends VectorDataProvider
 {
   static Logger log = LoggerFactory.getLogger(MbVectorTilesDataProvider.class);
 
-  protected static boolean canOpen(
+  private static Map<String, String> localCopies = new HashMap<String, String>();
+
+  private Configuration conf;
+
+  protected static boolean canOpen(Configuration conf,
           String input,
           ProviderProperties providerProperties) throws IOException
   {
-    MbVectorTilesSettings dbSettings = parseResourceName(input);
+    MbVectorTilesSettings dbSettings = parseResourceName(input, conf, providerProperties);
     SQLiteConnection conn = null;
     try {
-      conn = getDbConnection(dbSettings);
+      conn = getDbConnection(dbSettings, conf);
       return true;
     }
     catch(IOException e) {
@@ -58,23 +69,40 @@ protected static boolean canOpen(
   }
 
   @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "File must be specified by the user")
-  static SQLiteConnection getDbConnection(MbVectorTilesSettings dbSettings) throws IOException
+  static SQLiteConnection getDbConnection(MbVectorTilesSettings dbSettings,
+                                          Configuration conf) throws IOException
   {
-    // TODO: Download the file to a local directory if it is remote. See
-    // HadoopFileUtils for how we handle SequenceFile and MapFile. We should
-    // do something similar for these files. Keep in mind that we will also
-    // need to be able to copy files from HDFS in addition to S3 because the
-    // SQLite DB has to be on the file system.
     String filename = dbSettings.getFilename();
-    java.io.File dbFile = new java.io.File(filename);
-    if (!dbFile.exists()) {
-      throw new IOException("The MB tiles file must be in the file system: " + filename);
+    Path filepath = new Path(filename);
+    FileSystem fs = HadoopFileUtils.getFileSystem(conf, filepath);
+
+    File dbFile = null;
+    if (fs instanceof LocalFileSystem) {
+      dbFile = new File(filepath.toUri().getPath());
+    }
+    else {
+      String localName = localCopies.get(filename);
+      if (localName == null) {
+        dbFile = new File(FileUtils.createUniqueTmpDir(), new File(filename).getName());
+        Path localFilePath = new Path("file://" + dbFile.getAbsolutePath());
+        log.info("Attempting to copy MB tiles file " + filename +
+                " to the local machine at " + dbFile.getAbsolutePath());
+        fs.copyToLocalFile(false, filepath, localFilePath, true);
+        dbFile.deleteOnExit();
+        localCopies.put(filename, dbFile.getAbsolutePath());
+      }
+      else {
+        log.info("Using a copy of " + filename +
+                " already transferred to the local machine at " + localName);
+        dbFile = new File(localName);
+      }
     }
+
     try {
       return new SQLiteConnection(dbFile).open(false);
     }
     catch(SQLiteException e) {
-      throw new IOException("Unable to open MB tiles file: " + filename, e);
+      throw new IOException("Unable to open MB tiles file: " + dbFile.getAbsolutePath(), e);
     }
   }
 
@@ -83,30 +111,38 @@ public MbVectorTilesDataProvider(Configuration conf, String inputPrefix,
                                    ProviderProperties providerProperties)
   {
     super(inputPrefix, input, providerProperties);
+    this.conf = conf;
   }
 
   @Override
   public VectorMetadataReader getMetadataReader()
   {
-    return new MbVectorTilesMetadataReader(this);
+    // Not yet implemented. The metadata for mb tiles vector features
+    // is potentially different for every feature. So it doesn't make
+    // sense to provide metadata here.
+    return null;
   }
 
   @Override
   public VectorMetadataWriter getMetadataWriter()
   {
-    // Not yet implemented
+    // Not yet implemented. The metadata for mb tiles vector features
+    // is potentially different for every feature. So it doesn't make
+    // sense to write metadata here.
     return null;
   }
 
   @Override
   public VectorReader getVectorReader() throws IOException
   {
+    // Not yet implemented
     return null;
   }
 
   @Override
   public VectorReader getVectorReader(VectorReaderContext context) throws IOException
   {
+    // Not yet implemented
     return null;
   }
 
@@ -120,7 +156,7 @@ public VectorWriter getVectorWriter() throws IOException
   @Override
   public RecordReader<FeatureIdWritable, Geometry> getRecordReader() throws IOException
   {
-    MbVectorTilesSettings results = parseResourceName(getResourceName());
+    MbVectorTilesSettings results = parseResourceName(getResourceName(), conf, getProviderProperties());
     return new MbVectorTilesRecordReader(results);
   }
 
@@ -134,7 +170,7 @@ public RecordWriter<FeatureIdWritable, Geometry> getRecordWriter()
   @Override
   public VectorInputFormatProvider getVectorInputFormatProvider(VectorInputFormatContext context) throws IOException
   {
-    MbVectorTilesSettings results = parseResourceName(getResourceName());
+    MbVectorTilesSettings results = parseResourceName(getResourceName(), conf, getProviderProperties());
     return new MbVectorTilesInputFormatProvider(context, this, results);
   }
 
@@ -159,7 +195,7 @@ public void move(String toResource) throws IOException
 
   MbVectorTilesSettings parseResourceName() throws IOException
   {
-    return parseResourceName(getResourceName());
+    return parseResourceName(getResourceName(), conf, getProviderProperties());
   }
 
   /**
@@ -170,13 +206,17 @@ MbVectorTilesSettings parseResourceName() throws IOException
    *
    * @param input
    */
-  private static MbVectorTilesSettings parseResourceName(String input) throws IOException
+  private static MbVectorTilesSettings parseResourceName(String input,
+                                                         Configuration conf,
+                                                         ProviderProperties providerProperties) throws IOException
   {
     Map<String, String> settings = new HashMap<String, String>();
     parseDataSourceSettings(input, settings);
     String filename;
     if (settings.containsKey("filename")) {
-      filename = settings.get("filename");
+      filename = HdfsVectorDataProvider.resolveNameToPath(conf,
+              settings.get("filename"),
+              providerProperties, false).toString();
     }
     else {
       throw new IOException("Missing expected filename setting");