DRILL-5751: Fix unit tests to use local file system even if it is not…

… set by default

DRILL-5751: Changes after code review.

close #927

exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/CreateFunctionHandler.java

@@ -255,8 +255,8 @@ String getBinaryName() {
      * @throws IOException in case of binary or source absence or problems during copying jars
      */
     void initRemoteBackup() throws IOException {
-      fs.getFileStatus(stagingBinary);
-      fs.getFileStatus(stagingSource);
+      checkPathExistence(stagingBinary);
+      checkPathExistence(stagingSource);
       fs.mkdirs(remoteTmpDir);
       FileUtil.copy(fs, stagingBinary, fs, tmpRemoteBinary, false, true, fs.getConf());
       FileUtil.copy(fs, stagingSource, fs, tmpRemoteSource, false, true, fs.getConf());
@@ -315,6 +315,19 @@ void cleanUp() {
       deleteQuietly(remoteTmpDir, true);
     }
 
+    /**
+     * Checks if passed path exists on predefined file system.
+     *
+     * @param path path to be checked
+     * @throws IOException if path does not exist
+     */
+    private void checkPathExistence(Path path) throws IOException {
+      if (!fs.exists(path)) {
+        throw new IOException(String.format("File %s does not exist on file system %s",
+            path.toUri().getPath(), fs.getUri()));
+      }
+    }
+
     /**
      * Deletes quietly file or directory, in case of errors, logs warning and proceeds.
      *

exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java

@@ -59,7 +59,6 @@
 import org.apache.drill.exec.store.StoragePluginRegistry;
 import org.apache.drill.exec.util.TestUtilities;
 import org.apache.drill.exec.util.VectorUtil;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -83,7 +82,8 @@
 public class BaseTestQuery extends ExecTest {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class);
 
-  public static final String TEMP_SCHEMA = "dfs_test.tmp";
+  public static final String TEST_SCHEMA = "dfs_test";
+  public static final String TEMP_SCHEMA = TEST_SCHEMA + ".tmp";
 
   private static final int MAX_WIDTH_PER_NODE = 2;
 
@@ -138,9 +138,7 @@ public static void setupDefaultTestCluster() throws Exception {
     // turns on the verbose errors in tests
     // sever side stacktraces are added to the message before sending back to the client
     test("ALTER SESSION SET `exec.errors.verbose` = true");
-    Configuration conf = new Configuration();
-    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
-    fs = FileSystem.get(conf);
+    fs = getLocalFileSystem();
   }
 
   protected static void updateTestCluster(int newDrillbitCount, DrillConfig newConfig) {

exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java

@@ -23,6 +23,7 @@
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.UserRemoteException;
 import org.apache.drill.common.util.TestTools;
+import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.exception.VersionMismatchException;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
 import org.apache.drill.exec.expr.fn.registry.LocalFunctionRegistry;
@@ -35,6 +36,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -43,7 +45,6 @@
 import org.mockito.runners.MockitoJUnitRunner;
 import org.mockito.stubbing.Answer;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.List;
 import java.util.Properties;
@@ -67,18 +68,26 @@
 @RunWith(MockitoJUnitRunner.class)
 public class TestDynamicUDFSupport extends BaseTestQuery {
 
-  private static final File jars = new File(TestTools.getWorkingPath() + "/src/test/resources/jars");
+  private static final Path jars = new Path(TestTools.getWorkingPath(), "src/test/resources/jars");
   private static final String default_binary_name = "DrillUDF-1.0.jar";
   private static final String default_source_name = JarUtil.getSourceName(default_binary_name);
 
   @Rule
   public final TemporaryFolder base = new TemporaryFolder();
 
+  private static FileSystem localFileSystem;
+
+  @BeforeClass
+  public static void init() throws IOException {
+    localFileSystem = getLocalFileSystem();
+  }
+
   @Before
   public void setup() {
     Properties overrideProps = new Properties();
-    overrideProps.setProperty("drill.exec.udf.directory.root", base.getRoot().getPath());
-    overrideProps.setProperty("drill.tmp-dir", base.getRoot().getPath());
+    overrideProps.setProperty(ExecConstants.UDF_DIRECTORY_ROOT, base.getRoot().getPath());
+    overrideProps.setProperty(ExecConstants.DRILL_TMP_DIR, base.getRoot().getPath());
+    overrideProps.setProperty(ExecConstants.UDF_DIRECTORY_FS, FileSystem.DEFAULT_FS);
     updateTestCluster(1, DrillConfig.create(overrideProps));
   }
 
@@ -120,8 +129,10 @@ public void testDisableDynamicSupport() throws Exception {
   @Test
   public void testAbsentBinaryInStaging() throws Exception {
     Path staging = getDrillbitContext().getRemoteFunctionRegistry().getStagingArea();
+    FileSystem fs = getDrillbitContext().getRemoteFunctionRegistry().getFs();
 
-    String summary = String.format("File %s does not exist", new Path(staging, default_binary_name).toUri().getPath());
+    String summary = String.format("File %s does not exist on file system %s",
+        new Path(staging, default_binary_name).toUri().getPath(), fs.getUri());
 
     testBuilder()
         .sqlQuery("create function using jar '%s'", default_binary_name)
@@ -134,10 +145,11 @@ public void testAbsentBinaryInStaging() throws Exception {
   @Test
   public void testAbsentSourceInStaging() throws Exception {
     Path staging = getDrillbitContext().getRemoteFunctionRegistry().getStagingArea();
-    copyJar(getDrillbitContext().getRemoteFunctionRegistry().getFs(), new Path(jars.toURI()),
-        staging, default_binary_name);
+    FileSystem fs = getDrillbitContext().getRemoteFunctionRegistry().getFs();
+    copyJar(fs, jars, staging, default_binary_name);
 
-    String summary = String.format("File %s does not exist", new Path(staging, default_source_name).toUri().getPath());
+    String summary = String.format("File %s does not exist on file system %s",
+        new Path(staging, default_source_name).toUri().getPath(), fs.getUri());
 
     testBuilder()
         .sqlQuery("create function using jar '%s'", default_binary_name)
@@ -432,10 +444,11 @@ public void testLazyInit() throws Exception {
 
     Path localUdfDirPath = Deencapsulation.getField(
         getDrillbitContext().getFunctionImplementationRegistry(), "localUdfDir");
-    File localUdfDir = new File(localUdfDirPath.toUri().getPath());
 
-    assertTrue("Binary should exist in local udf directory", new File(localUdfDir, default_binary_name).exists());
-    assertTrue("Source should exist in local udf directory", new File(localUdfDir, default_source_name).exists());
+    assertTrue("Binary should exist in local udf directory",
+        localFileSystem.exists(new Path(localUdfDirPath, default_binary_name)));
+    assertTrue("Source should exist in local udf directory",
+        localFileSystem.exists(new Path(localUdfDirPath, default_source_name)));
   }
 
   @Test
@@ -498,10 +511,11 @@ public void testDropFunction() throws Exception {
 
     Path localUdfDirPath = Deencapsulation.getField(
         getDrillbitContext().getFunctionImplementationRegistry(), "localUdfDir");
-    File localUdfDir = new File(localUdfDirPath.toUri().getPath());
 
-    assertTrue("Binary should exist in local udf directory", new File(localUdfDir, default_binary_name).exists());
-    assertTrue("Source should exist in local udf directory", new File(localUdfDir, default_source_name).exists());
+    assertTrue("Binary should exist in local udf directory",
+        localFileSystem.exists(new Path(localUdfDirPath, default_binary_name)));
+    assertTrue("Source should exist in local udf directory",
+        localFileSystem.exists(new Path(localUdfDirPath, default_source_name)));
 
     String summary = "The following UDFs in jar %s have been unregistered:\n" +
         "[custom_lower(VARCHAR-REQUIRED)]";
@@ -530,9 +544,9 @@ public void testDropFunction() throws Exception {
         fs.exists(new Path(remoteFunctionRegistry.getRegistryArea(), default_source_name)));
 
     assertFalse("Binary should not be present in local udf directory",
-        new File(localUdfDir, default_binary_name).exists());
+        localFileSystem.exists(new Path(localUdfDirPath, default_binary_name)));
     assertFalse("Source should not be present in local udf directory",
-        new File(localUdfDir, default_source_name).exists());
+        localFileSystem.exists(new Path(localUdfDirPath, default_source_name)));
   }
 
   @Test
@@ -549,7 +563,7 @@ public void testReRegisterTheSameJarWithDifferentContent() throws Exception {
 
     Thread.sleep(1000);
 
-    Path src = new Path(jars.toURI().getPath(), "v2");
+    Path src = new Path(jars, "v2");
     copyJarsToStagingArea(src, default_binary_name, default_source_name);
     test("create function using jar '%s'", default_binary_name);
     testBuilder()
@@ -887,11 +901,11 @@ public Boolean answer(InvocationOnMock invocation) throws Throwable {
   }
 
   private void copyDefaultJarsToStagingArea() throws IOException {
-    copyJarsToStagingArea(new Path(jars.toURI()), default_binary_name, default_source_name);
+    copyJarsToStagingArea(jars, default_binary_name, default_source_name);
   }
 
   private void copyJarsToStagingArea(String binaryName, String sourceName) throws IOException  {
-    copyJarsToStagingArea(new Path(jars.toURI()), binaryName, sourceName);
+    copyJarsToStagingArea(jars, binaryName, sourceName);
   }
 
   private void copyJarsToStagingArea(Path src, String binaryName, String sourceName) throws IOException {

exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java

@@ -39,10 +39,13 @@
 import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
 import org.apache.drill.exec.util.GuavaPatcher;
 import org.apache.drill.test.DrillTest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.junit.After;
 import org.junit.BeforeClass;
 
 import java.io.File;
+import java.io.IOException;
 
 
 public class ExecTest extends DrillTest {
@@ -69,6 +72,17 @@ public static void setupOptionManager() throws Exception{
     optionManager.init();
   }
 
+  /**
+   * Creates instance of local file system.
+   *
+   * @return local file system
+   */
+  public static FileSystem getLocalFileSystem() throws IOException {
+    Configuration configuration = new Configuration();
+    configuration.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
+    return FileSystem.get(configuration);
+  }
+
   /**
    * Create a temp directory to store the given <i>dirName</i>.
    * Directory will be deleted on exit.

exec/java-exec/src/test/java/org/apache/drill/exec/TestRepeatedReaders.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -19,7 +19,6 @@
 package org.apache.drill.exec;
 
 import org.apache.drill.BaseTestQuery;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.junit.BeforeClass;
@@ -31,10 +30,7 @@ public class TestRepeatedReaders extends BaseTestQuery {
 
   @BeforeClass
   public static void initFs() throws Exception {
-    Configuration conf = new Configuration();
-    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
-
-    fs = FileSystem.get(conf);
+    fs = getLocalFileSystem();
   }
 
   private static void deleteTableIfExists(String tableName) {

exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -22,9 +22,6 @@
 
 import com.google.common.io.Files;
 import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.scanner.ClassPathScanner;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.common.util.TestTools;
@@ -42,7 +39,6 @@
 import org.apache.drill.exec.vector.IntVector;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.VarBinaryVector;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -97,12 +93,9 @@ public void test() throws Exception {
         VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(
             batch, context.getAllocator());
 
-        Configuration conf = new Configuration();
-        conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
-
         final VectorAccessibleSerializable newWrap = new VectorAccessibleSerializable(
             context.getAllocator());
-        try (final FileSystem fs = FileSystem.get(conf)) {
+        try (final FileSystem fs = getLocalFileSystem()) {
           final File tempDir = Files.createTempDir();
           tempDir.deleteOnExit();
           final Path path = new Path(tempDir.getAbsolutePath(), "drillSerializable");

exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java

@@ -119,10 +119,7 @@ public static Collection<Object[]> data() {
 
   @BeforeClass
   public static void initFs() throws Exception {
-    Configuration conf = new Configuration();
-    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
-
-    fs = FileSystem.get(conf);
+    fs = getLocalFileSystem();
     test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
   }
 
@@ -731,6 +728,7 @@ public void runTestAndValidate(String selection, String validationSelection, Str
           .go();
 
       Configuration hadoopConf = new Configuration();
+      hadoopConf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
       Path output = new Path(getDfsTestTmpSchemaLocation(), outputFile);
       FileSystem fs = output.getFileSystem(hadoopConf);
       for (FileStatus file : fs.listStatus(output)) {

exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriterEmptyFiles.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -19,7 +19,6 @@
 
 import org.apache.drill.BaseTestQuery;
 import org.apache.drill.exec.ExecConstants;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.junit.Assert;
@@ -32,11 +31,7 @@ public class TestParquetWriterEmptyFiles extends BaseTestQuery {
 
   @BeforeClass
   public static void initFs() throws Exception {
-    Configuration conf = new Configuration();
-    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
-
-    fs = FileSystem.get(conf);
-
+    fs = getLocalFileSystem();
     updateTestCluster(3, null);
   }
 

exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -30,7 +30,6 @@
 import org.apache.drill.exec.rpc.user.QueryDataBatch;
 import org.apache.drill.exec.vector.BigIntVector;
 import org.apache.drill.exec.vector.VarCharVector;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -48,10 +47,7 @@ public class TestWriter extends BaseTestQuery {
 
   @BeforeClass
   public static void initFs() throws Exception {
-    Configuration conf = new Configuration();
-    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
-
-    fs = FileSystem.get(conf);
+    fs = getLocalFileSystem();
   }
 
   @Test