From c81d7b6aa92467e694218d4c3903d2c50ec5c160 Mon Sep 17 00:00:00 2001 From: madanadit Date: Thu, 24 Nov 2016 06:26:41 -0500 Subject: [PATCH] Change return for list to UnderFileStatus --- .../java/alluxio/underfs/BaseUnderFileSystem.java | 10 +++++----- .../java/alluxio/underfs/ObjectUnderFileSystem.java | 4 ++-- .../main/java/alluxio/underfs/UnderFileSystem.java | 4 ++-- core/server/src/main/java/alluxio/cli/Format.java | 8 ++++---- .../src/main/java/alluxio/master/AlluxioMaster.java | 7 ++++--- .../java/alluxio/master/file/FileSystemMaster.java | 11 +++++++---- .../java/alluxio/underfs/UnderFileSystemCluster.java | 7 +++---- .../java/alluxio/master/JournalIntegrationTest.java | 3 ++- .../UnderStorageSystemInterfaceIntegrationTest.java | 6 +++--- .../alluxio/underfs/hdfs/HdfsUnderFileSystem.java | 7 ++++--- .../alluxio/underfs/local/LocalUnderFileSystem.java | 7 ++++--- 11 files changed, 40 insertions(+), 34 deletions(-) diff --git a/core/common/src/main/java/alluxio/underfs/BaseUnderFileSystem.java b/core/common/src/main/java/alluxio/underfs/BaseUnderFileSystem.java index 08dc168d6279..7b24e6edcd62 100644 --- a/core/common/src/main/java/alluxio/underfs/BaseUnderFileSystem.java +++ b/core/common/src/main/java/alluxio/underfs/BaseUnderFileSystem.java @@ -85,12 +85,12 @@ public String[] listRecursive(String path) throws IOException { List returnPaths = new ArrayList<>(); Queue pathsToProcess = new ArrayDeque<>(); // We call list initially, so we can return null if the path doesn't denote a directory - String[] subpaths = list(path); + UnderFileStatus[] subpaths = list(path); if (subpaths == null) { return null; } else { - for (String subp : subpaths) { - pathsToProcess.add(PathUtils.concatPath(path, subp)); + for (UnderFileStatus subp : subpaths) { + pathsToProcess.add(PathUtils.concatPath(path, subp.getName())); } } while (!pathsToProcess.isEmpty()) { @@ -99,8 +99,8 @@ public String[] listRecursive(String path) throws IOException { // Add all of its subpaths subpaths = list(p); if (subpaths != null) { - for (String subp : subpaths) { - pathsToProcess.add(PathUtils.concatPath(p, subp)); + for (UnderFileStatus subp : subpaths) { + pathsToProcess.add(PathUtils.concatPath(p, subp.getName())); } } } diff --git a/core/common/src/main/java/alluxio/underfs/ObjectUnderFileSystem.java b/core/common/src/main/java/alluxio/underfs/ObjectUnderFileSystem.java index 121580974da9..b0353e4a7c3d 100644 --- a/core/common/src/main/java/alluxio/underfs/ObjectUnderFileSystem.java +++ b/core/common/src/main/java/alluxio/underfs/ObjectUnderFileSystem.java @@ -272,8 +272,8 @@ public String[] listRecursive(String path) throws IOException { } @Override - public String[] list(String path) throws IOException { - return UnderFileStatus.toListingResult(listInternal(path, false)); + public UnderFileStatus[] list(String path) throws IOException { + return listInternal(path, false); } @Override diff --git a/core/common/src/main/java/alluxio/underfs/UnderFileSystem.java b/core/common/src/main/java/alluxio/underfs/UnderFileSystem.java index 761582d38f46..00e20cb15f63 100644 --- a/core/common/src/main/java/alluxio/underfs/UnderFileSystem.java +++ b/core/common/src/main/java/alluxio/underfs/UnderFileSystem.java @@ -434,12 +434,12 @@ public int getValue() { * order; they are not, in particular, guaranteed to appear in alphabetical order. * * @param path the abstract pathname to list - * @return An array of strings naming the files and directories in the directory denoted by this + * @return An array naming the files and directories in the directory denoted by this * abstract pathname. The array will be empty if the directory is empty. Returns * {@code null} if this abstract pathname does not denote a directory. * @throws IOException if a non-Alluxio error occurs */ - String[] list(String path) throws IOException; + UnderFileStatus[] list(String path) throws IOException; /** * Returns an array of strings naming the files and directories in the directory denoted by this diff --git a/core/server/src/main/java/alluxio/cli/Format.java b/core/server/src/main/java/alluxio/cli/Format.java index 1b37e98d39b8..7d8431f30fa9 100644 --- a/core/server/src/main/java/alluxio/cli/Format.java +++ b/core/server/src/main/java/alluxio/cli/Format.java @@ -17,6 +17,7 @@ import alluxio.PropertyKeyFormat; import alluxio.RuntimeConstants; import alluxio.master.AlluxioMaster; +import alluxio.underfs.UnderFileStatus; import alluxio.underfs.UnderFileSystem; import alluxio.underfs.options.DeleteOptions; import alluxio.util.UnderFileSystemUtils; @@ -42,11 +43,10 @@ private static boolean formatFolder(String name, String folder) throws IOExcepti UnderFileSystem ufs = UnderFileSystem.Factory.get(folder); LOG.info("Formatting {}:{}", name, folder); if (ufs.isDirectory(folder)) { - for (String p : ufs.list(folder)) { - String childPath = PathUtils.concatPath(folder, p); + for (UnderFileStatus p : ufs.list(folder)) { + String childPath = PathUtils.concatPath(folder, p.getName()); boolean failedToDelete; - // TODO(adit); eliminate this isDirectory call after list is updated to listStatus - if (ufs.isDirectory(childPath)) { + if (p.isDirectory()) { failedToDelete = !ufs.deleteDirectory(childPath, DeleteOptions.defaults().setRecursive(true)); } else { diff --git a/core/server/src/main/java/alluxio/master/AlluxioMaster.java b/core/server/src/main/java/alluxio/master/AlluxioMaster.java index cf46957cb98b..bec4409d7d5a 100644 --- a/core/server/src/main/java/alluxio/master/AlluxioMaster.java +++ b/core/server/src/main/java/alluxio/master/AlluxioMaster.java @@ -24,6 +24,7 @@ import alluxio.metrics.MetricsSystem; import alluxio.metrics.sink.MetricsServlet; import alluxio.security.authentication.TransportProvider; +import alluxio.underfs.UnderFileStatus; import alluxio.underfs.UnderFileSystem; import alluxio.util.CommonUtils; import alluxio.util.ConfigurationUtils; @@ -509,14 +510,14 @@ protected void stopServing() throws Exception { */ private boolean isJournalFormatted(String journalDirectory) throws IOException { UnderFileSystem ufs = UnderFileSystem.Factory.get(journalDirectory); - String[] files = ufs.list(journalDirectory); + UnderFileStatus[] files = ufs.list(journalDirectory); if (files == null) { return false; } // Search for the format file. String formatFilePrefix = Configuration.get(PropertyKey.MASTER_FORMAT_FILE_PREFIX); - for (String file : files) { - if (file.startsWith(formatFilePrefix)) { + for (UnderFileStatus file : files) { + if (file.getName().startsWith(formatFilePrefix)) { return true; } } diff --git a/core/server/src/main/java/alluxio/master/file/FileSystemMaster.java b/core/server/src/main/java/alluxio/master/file/FileSystemMaster.java index ae2d922c3f7f..f8f9f3054a01 100644 --- a/core/server/src/main/java/alluxio/master/file/FileSystemMaster.java +++ b/core/server/src/main/java/alluxio/master/file/FileSystemMaster.java @@ -90,6 +90,7 @@ import alluxio.thrift.FileSystemMasterWorkerService; import alluxio.thrift.PersistCommandOptions; import alluxio.thrift.PersistFile; +import alluxio.underfs.UnderFileStatus; import alluxio.underfs.UnderFileSystem; import alluxio.underfs.options.DeleteOptions; import alluxio.underfs.options.FileLocationOptions; @@ -1945,15 +1946,17 @@ private long loadMetadataAndJournal(LockedInodePath inodePath, LoadMetadataOptio InodeDirectory inode = (InodeDirectory) inodePath.getInode(); if (options.isLoadDirectChildren()) { - String[] files = ufs.list(ufsUri.getPath()); + UnderFileStatus[] files = ufs.list(ufsUri.getPath()); LoadMetadataOptions loadMetadataOptions = LoadMetadataOptions.defaults(); loadMetadataOptions.setLoadDirectChildren(false).setCreateAncestors(false); - for (String file : files) { - if (PathUtils.isTemporaryFileName(file) || inode.getChild(file) != null) { + for (UnderFileStatus file : files) { + if (PathUtils.isTemporaryFileName(file.getName()) + || inode.getChild(file.getName()) != null) { continue; } - TempInodePathForChild tempInodePath = new TempInodePathForChild(inodePath, file); + TempInodePathForChild tempInodePath = + new TempInodePathForChild(inodePath, file.getName()); counter = loadMetadataAndJournal(tempInodePath, loadMetadataOptions); } inode.setDirectChildrenLoaded(true); diff --git a/minicluster/src/main/java/alluxio/underfs/UnderFileSystemCluster.java b/minicluster/src/main/java/alluxio/underfs/UnderFileSystemCluster.java index a6667b99b971..29ad5e3667fb 100644 --- a/minicluster/src/main/java/alluxio/underfs/UnderFileSystemCluster.java +++ b/minicluster/src/main/java/alluxio/underfs/UnderFileSystemCluster.java @@ -138,10 +138,9 @@ public void cleanup() throws IOException { if (isStarted()) { String path = getUnderFilesystemAddress() + AlluxioURI.SEPARATOR; UnderFileSystem ufs = UnderFileSystem.Factory.get(path); - for (String p : ufs.list(path)) { - String childPath = PathUtils.concatPath(path, p); - // TODO(adit); eliminate this isDirectory call after list is updated to listStatus - if (ufs.isDirectory(childPath)) { + for (UnderFileStatus p : ufs.list(path)) { + String childPath = PathUtils.concatPath(path, p.getName()); + if (p.isDirectory()) { ufs.deleteDirectory(childPath, DeleteOptions.defaults().setRecursive(true)); } else { ufs.deleteFile(childPath); diff --git a/tests/src/test/java/alluxio/master/JournalIntegrationTest.java b/tests/src/test/java/alluxio/master/JournalIntegrationTest.java index b77d90cd5a89..f4fe85f5a68d 100644 --- a/tests/src/test/java/alluxio/master/JournalIntegrationTest.java +++ b/tests/src/test/java/alluxio/master/JournalIntegrationTest.java @@ -38,6 +38,7 @@ import alluxio.master.journal.ReadWriteJournal; import alluxio.security.authentication.AuthenticatedClientUser; import alluxio.security.group.GroupMappingService; +import alluxio.underfs.UnderFileStatus; import alluxio.underfs.UnderFileSystem; import alluxio.util.CommonUtils; import alluxio.util.IdUtils; @@ -142,7 +143,7 @@ public void multipleFlush() throws Exception { writer.getEntryOutputStream().flush(); writer.getEntryOutputStream().flush(); writer.getEntryOutputStream().flush(); - String[] paths = UnderFileSystem.Factory.get(journalFolder) + UnderFileStatus[] paths = UnderFileSystem.Factory.get(journalFolder) .list(journal.getCompletedDirectory()); // Make sure no new empty files were created. Assert.assertTrue(paths == null || paths.length == 0); diff --git a/tests/src/test/java/alluxio/underfs/UnderStorageSystemInterfaceIntegrationTest.java b/tests/src/test/java/alluxio/underfs/UnderStorageSystemInterfaceIntegrationTest.java index 09dd93e6cd2f..1e35cf4f3a15 100644 --- a/tests/src/test/java/alluxio/underfs/UnderStorageSystemInterfaceIntegrationTest.java +++ b/tests/src/test/java/alluxio/underfs/UnderStorageSystemInterfaceIntegrationTest.java @@ -243,7 +243,7 @@ public void list() throws IOException { String [] expectedResTopDir2 = new String[] {"/testDirNonEmpty2", "/testDirNonEmptyF"}; Arrays.sort(expectedResTopDir); Arrays.sort(expectedResTopDir2); - String [] resTopDir = mUfs.list(testDirNonEmpty); + UnderFileStatus [] resTopDir = mUfs.list(testDirNonEmpty); Arrays.sort(resTopDir); Assert.assertTrue(Arrays.equals(expectedResTopDir, resTopDir) || Arrays.equals(expectedResTopDir2, resTopDir)); @@ -263,7 +263,7 @@ public void listLargeDirectory() throws IOException { // See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and // https://cloud.google.com/storage/docs/consistency for more details. // Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval. - String[] results = new String[] {}; + UnderFileStatus[] results = new UnderFileStatus[] {}; for (int i = 0; i < 20; i++) { results = mUfs.list(config.getTopLevelDirectory()); if (children.length == results.length) { @@ -275,7 +275,7 @@ public void listLargeDirectory() throws IOException { Arrays.sort(results); for (int i = 0; i < children.length; ++i) { - Assert.assertTrue(results[i].equals(CommonUtils.stripPrefixIfPresent(children[i], + Assert.assertTrue(results[i].getName().equals(CommonUtils.stripPrefixIfPresent(children[i], PathUtils.normalizePath(config.getTopLevelDirectory(), "/")))); } } diff --git a/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java b/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java index 91bc10888944..47a041518d10 100644 --- a/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java +++ b/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java @@ -19,6 +19,7 @@ import alluxio.retry.RetryPolicy; import alluxio.security.authorization.Permission; import alluxio.underfs.BaseUnderFileSystem; +import alluxio.underfs.UnderFileStatus; import alluxio.underfs.UnderFileSystem; import alluxio.underfs.options.CreateOptions; import alluxio.underfs.options.DeleteOptions; @@ -260,14 +261,14 @@ public boolean isFile(String path) throws IOException { } @Override - public String[] list(String path) throws IOException { + public UnderFileStatus[] list(String path) throws IOException { FileStatus[] files = listStatus(path); if (files != null && !isFile(path)) { - String[] rtn = new String[files.length]; + UnderFileStatus[] rtn = new UnderFileStatus[files.length]; int i = 0; for (FileStatus status : files) { // only return the relative path, to keep consistent with java.io.File.list() - rtn[i++] = status.getPath().getName(); + rtn[i++] = new UnderFileStatus(status.getPath().getName(), status.isDirectory()); } return rtn; } else { diff --git a/underfs/local/src/main/java/alluxio/underfs/local/LocalUnderFileSystem.java b/underfs/local/src/main/java/alluxio/underfs/local/LocalUnderFileSystem.java index d9fdff287a2f..b77dfcc39872 100644 --- a/underfs/local/src/main/java/alluxio/underfs/local/LocalUnderFileSystem.java +++ b/underfs/local/src/main/java/alluxio/underfs/local/LocalUnderFileSystem.java @@ -18,6 +18,7 @@ import alluxio.security.authorization.Mode; import alluxio.security.authorization.Permission; import alluxio.underfs.BaseUnderFileSystem; +import alluxio.underfs.UnderFileStatus; import alluxio.underfs.UnderFileSystem; import alluxio.underfs.options.CreateOptions; import alluxio.underfs.options.DeleteOptions; @@ -195,15 +196,15 @@ public boolean isFile(String path) throws IOException { } @Override - public String[] list(String path) throws IOException { + public UnderFileStatus[] list(String path) throws IOException { path = stripPath(path); File file = new File(path); File[] files = file.listFiles(); if (files != null) { - String[] rtn = new String[files.length]; + UnderFileStatus[] rtn = new UnderFileStatus[files.length]; int i = 0; for (File f : files) { - rtn[i++] = f.getName(); + rtn[i++] = new UnderFileStatus(f.getName(), f.isDirectory()); } return rtn; } else {