Skip to content

Commit

Permalink
DRILL-6858: Add functionality to list directories / files with except…
Browse files Browse the repository at this point in the history 
…ions suppression

1. Add listDirectoriesSafe, listFilesSafe, listAllSafe in FileSystemUtil and DrillFileSystemUtil classes.
2. Use FileSystemUtil.listAllSafe during listing files in show files command and information_schema.files table.

closes #1547
  • Loading branch information
@arina-ielchiieva @vvysotskyi
arina-ielchiieva authored and vvysotskyi committed Nov 26, 2018
1 parent 8bd8192 commit d0ba8ec
Show file tree
Hide file tree
Showing 8 changed files with 256 additions and 182 deletions.
5 changes: 2 additions & 3 deletions .../java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ShowFilesHandler.java
View file
Expand Up @@ -32,7 +32,6 @@
import org.apache.drill.exec.work.foreman.ForemanSetupException;
import org.apache.hadoop.fs.Path;

import java.io.IOException;
import java.sql.Timestamp;
import java.util.List;
import java.util.stream.Collectors;
Expand All @@ -46,7 +45,7 @@ public ShowFilesHandler(SqlHandlerConfig config) {
}

@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException, IOException {
public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException {
SchemaPlus defaultSchema = config.getConverter().getDefaultSchema();
SchemaPlus drillSchema = defaultSchema;
SqlShowFiles showFiles = unwrap(sqlNode, SqlShowFiles.class);
Expand Down Expand Up @@ -83,7 +82,7 @@ public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException, IOExc
}

Path path = new Path(wsSchema.getDefaultLocation(), fromDir);
List<ShowFilesCommandResult> records = FileSystemUtil.listAll(wsSchema.getFS(), path, false).stream()
List<ShowFilesCommandResult> records = FileSystemUtil.listAllSafe(wsSchema.getFS(), path, false).stream()
// use ShowFilesCommandResult for backward compatibility
.map(fileStatus -> new ShowFilesCommandResult(new Records.File(wsSchema.getFullSchemaName(), wsSchema, fileStatus)))
.collect(Collectors.toList());
Expand Down
5 changes: 1 addition & 4 deletions ...ava-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaRecordGenerator.java
View file
Expand Up @@ -31,7 +31,6 @@
import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.SHRD_COL_TABLE_SCHEMA;
import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.TBLS_COL_TABLE_TYPE;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -436,14 +435,12 @@ public void visitFiles(String schemaName, SchemaPlus schemaPlus) {
String defaultLocation = wsSchema.getDefaultLocation();
FileSystem fs = wsSchema.getFS();
boolean recursive = optionManager.getBoolean(ExecConstants.LIST_FILES_RECURSIVELY);
FileSystemUtil.listAll(fs, new Path(defaultLocation), recursive).forEach(
FileSystemUtil.listAllSafe(fs, new Path(defaultLocation), recursive).forEach(
fileStatus -> records.add(new Records.File(schemaName, wsSchema, fileStatus))
);
}
} catch (ClassCastException | UnsupportedOperationException e) {
// ignore the exception since either this is not a Drill schema or schema does not support files listing
} catch (IOException e) {
logger.warn("Failure while trying to list files", e);
}
}
}
Expand Down
57 changes: 51 additions & 6 deletions exec/java-exec/src/main/java/org/apache/drill/exec/util/DrillFileSystemUtil.java
View file
Expand Up @@ -36,12 +36,8 @@ public class DrillFileSystemUtil {
/**
* Path filter that skips all files and folders that start with dot or underscore.
*/
public static final PathFilter DRILL_SYSTEM_FILTER = new PathFilter() {
@Override
public boolean accept(Path path) {
return !path.getName().startsWith(DrillFileSystem.UNDERSCORE_PREFIX) && !path.getName().startsWith(DrillFileSystem.DOT_PREFIX);
}
};
public static final PathFilter DRILL_SYSTEM_FILTER = path ->
!path.getName().startsWith(DrillFileSystem.UNDERSCORE_PREFIX) && !path.getName().startsWith(DrillFileSystem.DOT_PREFIX);

/**
* Returns statuses of all directories present in given path applying custom filters if present.
Expand All @@ -58,6 +54,22 @@ public static List<FileStatus> listDirectories(final FileSystem fs, Path path, b
return FileSystemUtil.listDirectories(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
}

/**
* Returns statuses of all directories present in given path applying custom filters if present.
* Directories that start with dot or underscore are skipped.
* Will include nested directories if recursive flag is set to true.
* Will ignore all exceptions during listing if any.
*
* @param fs current file system
* @param path path to directory
* @param recursive true if nested directories should be included
* @param filters list of custom filters (optional)
* @return list of matching directory statuses
*/
public static List<FileStatus> listDirectoriesSafe(final FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
return FileSystemUtil.listDirectoriesSafe(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
}

/**
* Returns statuses of all files present in given path applying custom filters if present.
* Files and nested directories that start with dot or underscore are skipped.
Expand All @@ -73,6 +85,23 @@ public static List<FileStatus> listFiles(final FileSystem fs, Path path, boolean
return FileSystemUtil.listFiles(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
}

/**
* Returns statuses of all files present in given path applying custom filters if present.
* Files and nested directories that start with dot or underscore are skipped.
* Will include files from nested directories if recursive flag is set to true.
* Will ignore all exceptions during listing if any.
*
* @param fs current file system
* @param path path to file or directory
* @param recursive true if files in nested directories should be included
* @param filters list of custom filters (optional)
* @return list of matching file statuses
*/
public static List<FileStatus> listFilesSafe(final FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
return FileSystemUtil.listFilesSafe(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
}


/**
* Returns statuses of all directories and files present in given path applying custom filters if present.
* Directories and files that start with dot or underscore are skipped.
Expand All @@ -88,4 +117,20 @@ public static List<FileStatus> listAll(FileSystem fs, Path path, boolean recursi
return FileSystemUtil.listAll(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
}

/**
* Returns statuses of all directories and files present in given path applying custom filters if present.
* Directories and files that start with dot or underscore are skipped.
* Will include nested directories and their files if recursive flag is set to true.
* Will ignore all exceptions during listing if any.
*
* @param fs current file system
* @param path path to file or directory
* @param recursive true if nested directories and their files should be included
* @param filters list of custom filters (optional)
* @return list of matching directory and file statuses
*/
public static List<FileStatus> listAllSafe(FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
return FileSystemUtil.listAllSafe(fs, path, recursive, FileSystemUtil.mergeFilters(DRILL_SYSTEM_FILTER, filters));
}

}
159 changes: 120 additions & 39 deletions exec/java-exec/src/main/java/org/apache/drill/exec/util/FileSystemUtil.java
View file
Expand Up @@ -26,22 +26,20 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Stream;

/**
* Helper class that provides methods to list directories or file or both statuses.
* Can list statuses recursive and apply custom filters.
*/
public class FileSystemUtil {

private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemUtil.class);

/**
* Filter that will accept all files and directories.
*/
public static final PathFilter DUMMY_FILTER = new PathFilter() {
@Override
public boolean accept(Path path) {
return true;
}
};
public static final PathFilter DUMMY_FILTER = path -> true;

/**
* Returns statuses of all directories present in given path applying custom filters if present.
Expand All @@ -55,7 +53,28 @@ public boolean accept(Path path) {
*/
public static List<FileStatus> listDirectories(final FileSystem fs, Path path, boolean recursive, PathFilter... filters) throws IOException {
List<FileStatus> statuses = new ArrayList<>();
listDirectories(fs, path, recursive, statuses, mergeFilters(filters));
listDirectories(fs, path, recursive, false, statuses, mergeFilters(filters));
return statuses;
}

/**
* Returns statuses of all directories present in given path applying custom filters if present.
* Will also include nested directories if recursive flag is set to true.
* Will ignore all exceptions during listing if any.
*
* @param fs current file system
* @param path path to directory
* @param recursive true if nested directories should be included
* @param filters list of custom filters (optional)
* @return list of matching directory statuses
*/
public static List<FileStatus> listDirectoriesSafe(final FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
List<FileStatus> statuses = new ArrayList<>();
try {
listDirectories(fs, path, recursive, true, statuses, mergeFilters(filters));
} catch (Exception e) {
// all exceptions are ignored
}
return statuses;
}

Expand All @@ -71,7 +90,27 @@ public static List<FileStatus> listDirectories(final FileSystem fs, Path path, b
*/
public static List<FileStatus> listFiles(FileSystem fs, Path path, boolean recursive, PathFilter... filters) throws IOException {
List<FileStatus> statuses = new ArrayList<>();
listFiles(fs, path, recursive, statuses, mergeFilters(filters));
listFiles(fs, path, recursive, false, statuses, mergeFilters(filters));
return statuses;
}

/**
* Returns statuses of all files present in given path applying custom filters if present.
* Will also include files from nested directories if recursive flag is set to true.
*
* @param fs current file system
* @param path path to file or directory
* @param recursive true if files in nested directories should be included
* @param filters list of custom filters (optional)
* @return list of matching file statuses
*/
public static List<FileStatus> listFilesSafe(FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
List<FileStatus> statuses = new ArrayList<>();
try {
listFiles(fs, path, recursive, true, statuses, mergeFilters(filters));
} catch (Exception e) {
// all exceptions are ignored
}
return statuses;
}

Expand All @@ -87,7 +126,28 @@ public static List<FileStatus> listFiles(FileSystem fs, Path path, boolean recur
*/
public static List<FileStatus> listAll(FileSystem fs, Path path, boolean recursive, PathFilter... filters) throws IOException {
List<FileStatus> statuses = new ArrayList<>();
listAll(fs, path, recursive, statuses, mergeFilters(filters));
listAll(fs, path, recursive, false, statuses, mergeFilters(filters));
return statuses;
}

/**
* Returns statuses of all directories and files present in given path applying custom filters if present.
* Will also include nested directories and their files if recursive flag is set to true.
* Will ignore all exceptions during listing if any.
*
* @param fs current file system
* @param path path to file or directory
* @param recursive true if nested directories and their files should be included
* @param filters list of custom filters (optional)
* @return list of matching directory and file statuses
*/
public static List<FileStatus> listAllSafe(FileSystem fs, Path path, boolean recursive, PathFilter... filters) {
List<FileStatus> statuses = new ArrayList<>();
try {
listAll(fs, path, recursive, true, statuses, mergeFilters(filters));
} catch (Exception e) {
// all exceptions are ignored
}
return statuses;
}

Expand Down Expand Up @@ -122,63 +182,73 @@ public static PathFilter mergeFilters(final PathFilter... filters) {
return DUMMY_FILTER;
}

return new PathFilter() {
@Override
public boolean accept(Path path) {
for (PathFilter filter : filters) {
if (!filter.accept(path)) {
return false;
}
}
return true;
}
};
return path -> Stream.of(filters).allMatch(filter -> filter.accept(path));
}

/**
* Helper method that will store in given holder statuses of all directories present in given path applying custom filter.
* If recursive flag is set to true, will call itself recursively to add statuses of nested directories.
* If suppress exceptions flag is set to true, will ignore all exceptions during listing.
*
* @param fs current file system
* @param path path to directory
* @param recursive true if nested directories should be included
* @param suppressExceptions indicates if exceptions should be ignored during listing
* @param statuses holder for directory statuses
* @param filter custom filter
* @return holder with all matching directory statuses
*/
private static List<FileStatus> listDirectories(FileSystem fs, Path path, boolean recursive, List<FileStatus> statuses, PathFilter filter) throws IOException {
FileStatus[] fileStatuses = fs.listStatus(path, filter);
for (FileStatus status: fileStatuses) {
if (status.isDirectory()) {
statuses.add(status);
if (recursive) {
listDirectories(fs, status.getPath(), true, statuses, filter);
private static List<FileStatus> listDirectories(FileSystem fs, Path path, boolean recursive, boolean suppressExceptions,
List<FileStatus> statuses, PathFilter filter) throws IOException {
try {
for (FileStatus status : fs.listStatus(path, filter)) {
if (status.isDirectory()) {
statuses.add(status);
if (recursive) {
listDirectories(fs, status.getPath(), true, suppressExceptions, statuses, filter);
}
}
}
} catch (Exception e) {
if (suppressExceptions) {
logger.debug("Exception during listing file statuses", e);
} else {
throw e;
}
}
return statuses;
}

/**
* Helper method that will store in given holder statuses of all files present in given path applying custom filter.
* If recursive flag is set to true, will call itself recursively to add file statuses from nested directories.
* If suppress exceptions flag is set to true, will ignore all exceptions during listing.
*
* @param fs current file system
* @param path path to file or directory
* @param recursive true if files in nested directories should be included
* @param suppressExceptions indicates if exceptions should be ignored during listing
* @param statuses holder for file statuses
* @param filter custom filter
* @return holder with all matching file statuses
*/
private static List<FileStatus> listFiles(FileSystem fs, Path path, boolean recursive, List<FileStatus> statuses, PathFilter filter) throws IOException {
FileStatus[] fileStatuses = fs.listStatus(path, filter);
for (FileStatus status: fileStatuses) {
if (status.isDirectory()) {
if (recursive) {
listFiles(fs, status.getPath(), true, statuses, filter);
private static List<FileStatus> listFiles(FileSystem fs, Path path, boolean recursive, boolean suppressExceptions,
List<FileStatus> statuses, PathFilter filter) throws IOException {
try {
for (FileStatus status : fs.listStatus(path, filter)) {
if (status.isDirectory()) {
if (recursive) {
listFiles(fs, status.getPath(), true, suppressExceptions, statuses, filter);
}
} else {
statuses.add(status);
}
}
} catch (Exception e) {
if (suppressExceptions) {
logger.debug("Exception during listing file statuses", e);
} else {
statuses.add(status);
throw e;
}
}
return statuses;
Expand All @@ -187,19 +257,30 @@ private static List<FileStatus> listFiles(FileSystem fs, Path path, boolean recu
/**
* Helper method that will store in given holder statuses of all directories and files present in given path applying custom filter.
* If recursive flag is set to true, will call itself recursively to add nested directories and their file statuses.
* If suppress exceptions flag is set to true, will ignore all exceptions during listing.
*
* @param fs current file system
* @param path path to file or directory
* @param recursive true if nested directories and their files should be included
* @param suppressExceptions indicates if exceptions should be ignored during listing
* @param statuses holder for directory and file statuses
* @param filter custom filter
* @return holder with all matching directory and file statuses
*/
private static List<FileStatus> listAll(FileSystem fs, Path path, boolean recursive, List<FileStatus> statuses, PathFilter filter) throws IOException {
for (FileStatus status: fs.listStatus(path, filter)) {
statuses.add(status);
if (status.isDirectory() && recursive) {
listAll(fs, status.getPath(), true, statuses, filter);
private static List<FileStatus> listAll(FileSystem fs, Path path, boolean recursive, boolean suppressExceptions,
List<FileStatus> statuses, PathFilter filter) throws IOException {
try {
for (FileStatus status : fs.listStatus(path, filter)) {
statuses.add(status);
if (status.isDirectory() && recursive) {
listAll(fs, status.getPath(), true, suppressExceptions, statuses, filter);
}
}
} catch (Exception e) {
if (suppressExceptions) {
logger.debug("Exception during listing file statuses", e);
} else {
throw e;
}
}
return statuses;
Expand Down

0 comments on commit d0ba8ec

Please sign in to comment.