SNOW-1369651: Do not fail file pattern expanding on file not found in…

… different pattern (#1811)

src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java

@@ -15,6 +15,7 @@
 import com.google.common.io.ByteStreams;
 import com.google.common.io.CountingOutputStream;
 import java.io.File;
+import java.io.FileFilter;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -74,6 +75,8 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.filefilter.WildcardFileFilter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Class for uploading/downloading files
@@ -97,6 +100,7 @@ public class SnowflakeFileTransferAgent extends SFBaseFileTransferAgent {
 
   private static final String localFSFileSep = systemGetProperty("file.separator");
   private static final int DEFAULT_PARALLEL = 10;
+  private static final Logger log = LoggerFactory.getLogger(SnowflakeFileTransferAgent.class);
 
   private final String command;
 
@@ -1938,7 +1942,7 @@ static Set<String> expandFileNames(String[] filePathList, String queryId)
     // For each location, list files and match against the patterns
     for (Map.Entry<String, List<String>> entry : locationToFilePatterns.entrySet()) {
       try {
-        java.io.File dir = new java.io.File(entry.getKey());
+        File dir = new File(entry.getKey());
 
         logger.debug(
             "Listing files under: {} with patterns: {}",
@@ -1950,11 +1954,15 @@ static Set<String> expandFileNames(String[] filePathList, String queryId)
             && injectedFileTransferException instanceof Exception) {
           throw (Exception) SnowflakeFileTransferAgent.injectedFileTransferException;
         }
-
         // The following currently ignore sub directories
-        for (Object file :
-            FileUtils.listFiles(dir, new WildcardFileFilter(entry.getValue()), null)) {
-          result.add(((java.io.File) file).getCanonicalPath());
+        File[] filesMatchingPattern =
+            dir.listFiles((FileFilter) new WildcardFileFilter(entry.getValue()));
+        if (filesMatchingPattern != null) {
+          for (File file : filesMatchingPattern) {
+            result.add(file.getCanonicalPath());
+          }
+        } else {
+          logger.debug("No files under {} matching pattern {}", entry.getKey(), entry.getValue());
         }
       } catch (Exception ex) {
         throw new SnowflakeSQLException(

src/test/java/net/snowflake/client/jdbc/FileUploaderExpandFileNamesTest.java

@@ -3,15 +3,25 @@
  */
 package net.snowflake.client.jdbc;
 
+import static net.snowflake.client.jdbc.SnowflakeUtil.systemGetProperty;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Properties;
 import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
 import net.snowflake.client.core.OCSPMode;
 import org.junit.Assert;
 import org.junit.Rule;
@@ -21,6 +31,8 @@
 /** Tests for SnowflakeFileTransferAgent.expandFileNames */
 public class FileUploaderExpandFileNamesTest {
   @Rule public TemporaryFolder folder = new TemporaryFolder();
+  @Rule public TemporaryFolder secondFolder = new TemporaryFolder();
+  private String localFSFileSep = systemGetProperty("file.separator");
 
   @Test
   public void testProcessFileNames() throws Exception {
@@ -126,4 +138,88 @@ public int read() throws IOException {
     assertEquals("dummy_dest_file_name", config.getDestFileName());
     assertEquals(expectedThrowCount, throwCount);
   }
+
+  /**
+   * We have N jobs expanding files with exclusive pattern, processing them and deleting. Expanding
+   * the list should not cause the error when file of another pattern is deleted which may happen
+   * when FileUtils.listFiles is used.
+   *
+   * <p>Fix available after version 3.16.1.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testFileListingDoesNotFailOnMissingFilesOfAnotherPattern() throws Exception {
+    folder.newFolder("TestFiles");
+    String folderName = folder.getRoot().getCanonicalPath();
+
+    int filePatterns = 10;
+    int filesPerPattern = 100;
+    IntStream.range(0, filesPerPattern * filePatterns)
+        .forEach(
+            id -> {
+              try {
+                File file =
+                    new File(
+                        folderName
+                            + localFSFileSep
+                            + "foo"
+                            + id % filePatterns
+                            + "-"
+                            + UUID.randomUUID());
+                assertTrue(file.createNewFile());
+              } catch (IOException e) {
+                throw new RuntimeException(e);
+              }
+            });
+
+    ExecutorService executorService = Executors.newFixedThreadPool(filePatterns / 3);
+    List<Future<Set<String>>> futures = new ArrayList<>();
+    for (int i = 0; i < filePatterns; ++i) {
+      String[] locations = {
+        folderName + localFSFileSep + "foo" + i + "*",
+      };
+      Future<Set<String>> future =
+          executorService.submit(
+              () -> {
+                try {
+                  Set<String> strings = SnowflakeFileTransferAgent.expandFileNames(locations, null);
+                  strings.forEach(
+                      fileName -> {
+                        try {
+                          File file = new File(fileName);
+                          Files.delete(file.toPath());
+                        } catch (IOException e) {
+                          throw new RuntimeException(e);
+                        }
+                      });
+                  return strings;
+                } catch (SnowflakeSQLException e) {
+                  throw new RuntimeException(e);
+                }
+              });
+      futures.add(future);
+    }
+    executorService.shutdown();
+    assertTrue(executorService.awaitTermination(60, TimeUnit.SECONDS));
+    assertEquals(filePatterns, futures.size());
+    for (Future<Set<String>> future : futures) {
+      assertTrue(future.isDone());
+      assertEquals(filesPerPattern, future.get().size());
+    }
+  }
+
+  @Test
+  public void testFileListingDoesNotFailOnNotExistingDirectory() throws Exception {
+    folder.newFolder("TestFiles");
+    String folderName = folder.getRoot().getCanonicalPath();
+    String[] locations = {
+      folderName + localFSFileSep + "foo*",
+    };
+    folder.delete();
+
+    Set<String> files = SnowflakeFileTransferAgent.expandFileNames(locations, null);
+
+    assertTrue(files.isEmpty());
+  }
 }