Skip to content

Commit

Permalink
Change return for list to UnderFileStatus
Browse files Browse the repository at this point in the history
  • Loading branch information
@madanadit
madanadit committed Nov 24, 2016
1 parent accebb6 commit c81d7b6
Show file tree
Hide file tree
Showing 11 changed files with 40 additions and 34 deletions.
10 changes: 5 additions & 5 deletions core/common/src/main/java/alluxio/underfs/BaseUnderFileSystem.java
View file
Expand Up @@ -85,12 +85,12 @@ public String[] listRecursive(String path) throws IOException {
List<String> returnPaths = new ArrayList<>();
Queue<String> pathsToProcess = new ArrayDeque<>();
// We call list initially, so we can return null if the path doesn't denote a directory
String[] subpaths = list(path);
UnderFileStatus[] subpaths = list(path);
if (subpaths == null) {
return null;
} else {
for (String subp : subpaths) {
pathsToProcess.add(PathUtils.concatPath(path, subp));
for (UnderFileStatus subp : subpaths) {
pathsToProcess.add(PathUtils.concatPath(path, subp.getName()));
}
}
while (!pathsToProcess.isEmpty()) {
Expand All @@ -99,8 +99,8 @@ public String[] listRecursive(String path) throws IOException {
// Add all of its subpaths
subpaths = list(p);
if (subpaths != null) {
for (String subp : subpaths) {
pathsToProcess.add(PathUtils.concatPath(p, subp));
for (UnderFileStatus subp : subpaths) {
pathsToProcess.add(PathUtils.concatPath(p, subp.getName()));
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions core/common/src/main/java/alluxio/underfs/ObjectUnderFileSystem.java
View file
Expand Up @@ -272,8 +272,8 @@ public String[] listRecursive(String path) throws IOException {
}

@Override
public String[] list(String path) throws IOException {
return UnderFileStatus.toListingResult(listInternal(path, false));
public UnderFileStatus[] list(String path) throws IOException {
return listInternal(path, false);
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions core/common/src/main/java/alluxio/underfs/UnderFileSystem.java
View file
Expand Up @@ -434,12 +434,12 @@ public int getValue() {
* order; they are not, in particular, guaranteed to appear in alphabetical order.
*
* @param path the abstract pathname to list
* @return An array of strings naming the files and directories in the directory denoted by this
* @return An array naming the files and directories in the directory denoted by this
* abstract pathname. The array will be empty if the directory is empty. Returns
* {@code null} if this abstract pathname does not denote a directory.
* @throws IOException if a non-Alluxio error occurs
*/
String[] list(String path) throws IOException;
UnderFileStatus[] list(String path) throws IOException;

/**
* Returns an array of strings naming the files and directories in the directory denoted by this
Expand Down
8 changes: 4 additions & 4 deletions core/server/src/main/java/alluxio/cli/Format.java
View file
Expand Up @@ -17,6 +17,7 @@
import alluxio.PropertyKeyFormat;
import alluxio.RuntimeConstants;
import alluxio.master.AlluxioMaster;
import alluxio.underfs.UnderFileStatus;
import alluxio.underfs.UnderFileSystem;
import alluxio.underfs.options.DeleteOptions;
import alluxio.util.UnderFileSystemUtils;
Expand All @@ -42,11 +43,10 @@ private static boolean formatFolder(String name, String folder) throws IOExcepti
UnderFileSystem ufs = UnderFileSystem.Factory.get(folder);
LOG.info("Formatting {}:{}", name, folder);
if (ufs.isDirectory(folder)) {
for (String p : ufs.list(folder)) {
String childPath = PathUtils.concatPath(folder, p);
for (UnderFileStatus p : ufs.list(folder)) {
String childPath = PathUtils.concatPath(folder, p.getName());
boolean failedToDelete;
// TODO(adit); eliminate this isDirectory call after list is updated to listStatus
if (ufs.isDirectory(childPath)) {
if (p.isDirectory()) {
failedToDelete = !ufs.deleteDirectory(childPath,
DeleteOptions.defaults().setRecursive(true));
} else {
Expand Down
7 changes: 4 additions & 3 deletions core/server/src/main/java/alluxio/master/AlluxioMaster.java
View file
Expand Up @@ -24,6 +24,7 @@
import alluxio.metrics.MetricsSystem;
import alluxio.metrics.sink.MetricsServlet;
import alluxio.security.authentication.TransportProvider;
import alluxio.underfs.UnderFileStatus;
import alluxio.underfs.UnderFileSystem;
import alluxio.util.CommonUtils;
import alluxio.util.ConfigurationUtils;
Expand Down Expand Up @@ -509,14 +510,14 @@ protected void stopServing() throws Exception {
*/
private boolean isJournalFormatted(String journalDirectory) throws IOException {
UnderFileSystem ufs = UnderFileSystem.Factory.get(journalDirectory);
String[] files = ufs.list(journalDirectory);
UnderFileStatus[] files = ufs.list(journalDirectory);
if (files == null) {
return false;
}
// Search for the format file.
String formatFilePrefix = Configuration.get(PropertyKey.MASTER_FORMAT_FILE_PREFIX);
for (String file : files) {
if (file.startsWith(formatFilePrefix)) {
for (UnderFileStatus file : files) {
if (file.getName().startsWith(formatFilePrefix)) {
return true;
}
}
Expand Down
11 changes: 7 additions & 4 deletions core/server/src/main/java/alluxio/master/file/FileSystemMaster.java
View file
Expand Up @@ -90,6 +90,7 @@
import alluxio.thrift.FileSystemMasterWorkerService;
import alluxio.thrift.PersistCommandOptions;
import alluxio.thrift.PersistFile;
import alluxio.underfs.UnderFileStatus;
import alluxio.underfs.UnderFileSystem;
import alluxio.underfs.options.DeleteOptions;
import alluxio.underfs.options.FileLocationOptions;
Expand Down Expand Up @@ -1945,15 +1946,17 @@ private long loadMetadataAndJournal(LockedInodePath inodePath, LoadMetadataOptio
InodeDirectory inode = (InodeDirectory) inodePath.getInode();

if (options.isLoadDirectChildren()) {
String[] files = ufs.list(ufsUri.getPath());
UnderFileStatus[] files = ufs.list(ufsUri.getPath());
LoadMetadataOptions loadMetadataOptions = LoadMetadataOptions.defaults();
loadMetadataOptions.setLoadDirectChildren(false).setCreateAncestors(false);

for (String file : files) {
if (PathUtils.isTemporaryFileName(file) || inode.getChild(file) != null) {
for (UnderFileStatus file : files) {
if (PathUtils.isTemporaryFileName(file.getName())
|| inode.getChild(file.getName()) != null) {
continue;
}
TempInodePathForChild tempInodePath = new TempInodePathForChild(inodePath, file);
TempInodePathForChild tempInodePath =
new TempInodePathForChild(inodePath, file.getName());
counter = loadMetadataAndJournal(tempInodePath, loadMetadataOptions);
}
inode.setDirectChildrenLoaded(true);
Expand Down
7 changes: 3 additions & 4 deletions minicluster/src/main/java/alluxio/underfs/UnderFileSystemCluster.java
View file
Expand Up @@ -138,10 +138,9 @@ public void cleanup() throws IOException {
if (isStarted()) {
String path = getUnderFilesystemAddress() + AlluxioURI.SEPARATOR;
UnderFileSystem ufs = UnderFileSystem.Factory.get(path);
for (String p : ufs.list(path)) {
String childPath = PathUtils.concatPath(path, p);
// TODO(adit); eliminate this isDirectory call after list is updated to listStatus
if (ufs.isDirectory(childPath)) {
for (UnderFileStatus p : ufs.list(path)) {
String childPath = PathUtils.concatPath(path, p.getName());
if (p.isDirectory()) {
ufs.deleteDirectory(childPath, DeleteOptions.defaults().setRecursive(true));
} else {
ufs.deleteFile(childPath);
Expand Down
3 changes: 2 additions & 1 deletion tests/src/test/java/alluxio/master/JournalIntegrationTest.java
View file
Expand Up @@ -38,6 +38,7 @@
import alluxio.master.journal.ReadWriteJournal;
import alluxio.security.authentication.AuthenticatedClientUser;
import alluxio.security.group.GroupMappingService;
import alluxio.underfs.UnderFileStatus;
import alluxio.underfs.UnderFileSystem;
import alluxio.util.CommonUtils;
import alluxio.util.IdUtils;
Expand Down Expand Up @@ -142,7 +143,7 @@ public void multipleFlush() throws Exception {
writer.getEntryOutputStream().flush();
writer.getEntryOutputStream().flush();
writer.getEntryOutputStream().flush();
String[] paths = UnderFileSystem.Factory.get(journalFolder)
UnderFileStatus[] paths = UnderFileSystem.Factory.get(journalFolder)
.list(journal.getCompletedDirectory());
// Make sure no new empty files were created.
Assert.assertTrue(paths == null || paths.length == 0);
Expand Down
6 changes: 3 additions & 3 deletions tests/src/test/java/alluxio/underfs/UnderStorageSystemInterfaceIntegrationTest.java
View file
Expand Up @@ -243,7 +243,7 @@ public void list() throws IOException {
String [] expectedResTopDir2 = new String[] {"/testDirNonEmpty2", "/testDirNonEmptyF"};
Arrays.sort(expectedResTopDir);
Arrays.sort(expectedResTopDir2);
String [] resTopDir = mUfs.list(testDirNonEmpty);
UnderFileStatus [] resTopDir = mUfs.list(testDirNonEmpty);
Arrays.sort(resTopDir);
Assert.assertTrue(Arrays.equals(expectedResTopDir, resTopDir)
|| Arrays.equals(expectedResTopDir2, resTopDir));
Expand All @@ -263,7 +263,7 @@ public void listLargeDirectory() throws IOException {
// See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
// https://cloud.google.com/storage/docs/consistency for more details.
// Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
String[] results = new String[] {};
UnderFileStatus[] results = new UnderFileStatus[] {};
for (int i = 0; i < 20; i++) {
results = mUfs.list(config.getTopLevelDirectory());
if (children.length == results.length) {
Expand All @@ -275,7 +275,7 @@ public void listLargeDirectory() throws IOException {

Arrays.sort(results);
for (int i = 0; i < children.length; ++i) {
Assert.assertTrue(results[i].equals(CommonUtils.stripPrefixIfPresent(children[i],
Assert.assertTrue(results[i].getName().equals(CommonUtils.stripPrefixIfPresent(children[i],
PathUtils.normalizePath(config.getTopLevelDirectory(), "/"))));
}
}
Expand Down
7 changes: 4 additions & 3 deletions underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java
View file
Expand Up @@ -19,6 +19,7 @@
import alluxio.retry.RetryPolicy;
import alluxio.security.authorization.Permission;
import alluxio.underfs.BaseUnderFileSystem;
import alluxio.underfs.UnderFileStatus;
import alluxio.underfs.UnderFileSystem;
import alluxio.underfs.options.CreateOptions;
import alluxio.underfs.options.DeleteOptions;
Expand Down Expand Up @@ -260,14 +261,14 @@ public boolean isFile(String path) throws IOException {
}

@Override
public String[] list(String path) throws IOException {
public UnderFileStatus[] list(String path) throws IOException {
FileStatus[] files = listStatus(path);
if (files != null && !isFile(path)) {
String[] rtn = new String[files.length];
UnderFileStatus[] rtn = new UnderFileStatus[files.length];
int i = 0;
for (FileStatus status : files) {
// only return the relative path, to keep consistent with java.io.File.list()
rtn[i++] = status.getPath().getName();
rtn[i++] = new UnderFileStatus(status.getPath().getName(), status.isDirectory());
}
return rtn;
} else {
Expand Down
7 changes: 4 additions & 3 deletions underfs/local/src/main/java/alluxio/underfs/local/LocalUnderFileSystem.java
View file
Expand Up @@ -18,6 +18,7 @@
import alluxio.security.authorization.Mode;
import alluxio.security.authorization.Permission;
import alluxio.underfs.BaseUnderFileSystem;
import alluxio.underfs.UnderFileStatus;
import alluxio.underfs.UnderFileSystem;
import alluxio.underfs.options.CreateOptions;
import alluxio.underfs.options.DeleteOptions;
Expand Down Expand Up @@ -195,15 +196,15 @@ public boolean isFile(String path) throws IOException {
}

@Override
public String[] list(String path) throws IOException {
public UnderFileStatus[] list(String path) throws IOException {
path = stripPath(path);
File file = new File(path);
File[] files = file.listFiles();
if (files != null) {
String[] rtn = new String[files.length];
UnderFileStatus[] rtn = new UnderFileStatus[files.length];
int i = 0;
for (File f : files) {
rtn[i++] = f.getName();
rtn[i++] = new UnderFileStatus(f.getName(), f.isDirectory());
}
return rtn;
} else {
Expand Down

0 comments on commit c81d7b6

Please sign in to comment.