From 97fb95521241676cf4208e06ac7e586cb0fa8657 Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Fri, 26 Jul 2024 12:00:36 +0530 Subject: [PATCH 1/8] HDDS-11238. Converge getBucket client-side calls for FileSystem client delete --- .../fs/ozone/BasicRootedOzoneFileSystem.java | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 23a921aaa10..d09a89219bb 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -704,6 +704,7 @@ private boolean deleteInSpan(Path f, boolean recursive) throws IOException { String key = pathToKey(f); OFSPath ofsPath = new OFSPath(key, ozoneConfiguration); + OzoneBucket bucket = adapterImpl.getBucket(ofsPath, false); // Handle rm root if (ofsPath.isRoot()) { // Intentionally drop support for rm root @@ -727,7 +728,7 @@ private boolean deleteInSpan(Path f, boolean recursive) throws IOException { // delete bucket if (ofsPath.isBucket()) { - return deleteBucket(f, recursive, ofsPath); + return deleteBucket(f, recursive, bucket); } // delete files and directory @@ -742,8 +743,6 @@ private boolean deleteInSpan(Path f, boolean recursive) throws IOException { boolean result; if (status.isDirectory()) { LOG.debug("delete: Path is a directory: {}", f); - - OzoneBucket bucket = adapterImpl.getBucket(ofsPath, false); if (bucket.getBucketLayout().isFileSystemOptimized()) { String ofsKeyPath = ofsPath.getNonKeyPathNoPrefixDelim() + OZONE_URI_DELIMITER + ofsPath.getKeyName(); @@ -766,15 +765,15 @@ private boolean deleteInSpan(Path f, boolean recursive) throws IOException { return result; } - private boolean deleteBucket(Path f, boolean recursive, OFSPath ofsPath) + private boolean deleteBucket(Path f, boolean recursive, OzoneBucket bucket) throws IOException { // check status of normal bucket try { getFileStatus(f); } catch (FileNotFoundException ex) { // remove orphan link bucket directly - if (isLinkBucket(f, ofsPath)) { - deleteBucketFromVolume(f, ofsPath); + if (isLinkBucket(f, bucket)) { + deleteBucketFromVolume(f, bucket); return true; } LOG.warn("delete: Path does not exist: {}", f); @@ -787,8 +786,8 @@ private boolean deleteBucket(Path f, boolean recursive, OFSPath ofsPath) boolean handleTrailingSlash = f.toString().endsWith(OZONE_URI_DELIMITER); // remove link bucket directly if link and // rm path does not have trailing slash - if (isLinkBucket(f, ofsPath) && !handleTrailingSlash) { - deleteBucketFromVolume(f, ofsPath); + if (isLinkBucket(f, bucket) && !handleTrailingSlash) { + deleteBucketFromVolume(f, bucket); return true; } @@ -799,14 +798,13 @@ private boolean deleteBucket(Path f, boolean recursive, OFSPath ofsPath) // if so, the contents of bucket were deleted and skip delete bucket // otherwise, Handle delete bucket if (!handleTrailingSlash) { - deleteBucketFromVolume(f, ofsPath); + deleteBucketFromVolume(f, bucket); } return result; } - private boolean isLinkBucket(Path f, OFSPath ofsPath) { + private boolean isLinkBucket(Path f, OzoneBucket bucket) { try { - OzoneBucket bucket = adapterImpl.getBucket(ofsPath, false); if (bucket.isLink()) { return true; } @@ -818,12 +816,12 @@ private boolean isLinkBucket(Path f, OFSPath ofsPath) { return false; } - private void deleteBucketFromVolume(Path f, OFSPath ofsPath) + private void deleteBucketFromVolume(Path f, OzoneBucket bucket) throws IOException { OzoneVolume volume = - adapterImpl.getObjectStore().getVolume(ofsPath.getVolumeName()); + adapterImpl.getObjectStore().getVolume(bucket.getVolumeName()); try { - volume.deleteBucket(ofsPath.getBucketName()); + volume.deleteBucket(bucket.getName()); } catch (OMException ex) { // bucket is not empty if (ex.getResult() == BUCKET_NOT_EMPTY) { From 84007afe8a3ce165d1bbec33cc953ada1f6330b3 Mon Sep 17 00:00:00 2001 From: tanvipenumudy <46785609+tanvipenumudy@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:14:29 +0530 Subject: [PATCH 2/8] Update hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index d09a89219bb..7490d3b9d23 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -704,7 +704,6 @@ private boolean deleteInSpan(Path f, boolean recursive) throws IOException { String key = pathToKey(f); OFSPath ofsPath = new OFSPath(key, ozoneConfiguration); - OzoneBucket bucket = adapterImpl.getBucket(ofsPath, false); // Handle rm root if (ofsPath.isRoot()) { // Intentionally drop support for rm root From 1d11d3eb415df23189700b6ac8a58d7cd6d64268 Mon Sep 17 00:00:00 2001 From: tanvipenumudy <46785609+tanvipenumudy@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:14:37 +0530 Subject: [PATCH 3/8] Update hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 7490d3b9d23..14e425502ba 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -727,7 +727,7 @@ private boolean deleteInSpan(Path f, boolean recursive) throws IOException { // delete bucket if (ofsPath.isBucket()) { - return deleteBucket(f, recursive, bucket); + return deleteBucket(f, recursive, ofsPath); } // delete files and directory From f1ab6e77beb79bf0f92522c07bef47c49f1474a4 Mon Sep 17 00:00:00 2001 From: tanvipenumudy <46785609+tanvipenumudy@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:17:43 +0530 Subject: [PATCH 4/8] Update hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../hadoop/fs/ozone/BasicRootedOzoneFileSystem.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 14e425502ba..6ea46a39014 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -764,8 +764,14 @@ private boolean deleteInSpan(Path f, boolean recursive) throws IOException { return result; } - private boolean deleteBucket(Path f, boolean recursive, OzoneBucket bucket) + private boolean deleteBucket(Path f, boolean recursive, OFSPath ofsPath) throws IOException { + OzoneBucket bucket; + try { + bucket = adapterImpl.getBucket(ofsPath, false); + } catch (Exception ex) { + return false; + } // check status of normal bucket try { getFileStatus(f); From 3f763a324e203d8c30835bce8d144a5954cd4bee Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Mon, 29 Jul 2024 12:27:10 +0530 Subject: [PATCH 5/8] Address review comments - make isLinkBucket inline --- .../fs/ozone/BasicRootedOzoneFileSystem.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 6ea46a39014..820c2f5a154 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -742,6 +742,8 @@ private boolean deleteInSpan(Path f, boolean recursive) throws IOException { boolean result; if (status.isDirectory()) { LOG.debug("delete: Path is a directory: {}", f); + + OzoneBucket bucket = adapterImpl.getBucket(ofsPath, false); if (bucket.getBucketLayout().isFileSystemOptimized()) { String ofsKeyPath = ofsPath.getNonKeyPathNoPrefixDelim() + OZONE_URI_DELIMITER + ofsPath.getKeyName(); @@ -777,7 +779,7 @@ private boolean deleteBucket(Path f, boolean recursive, OFSPath ofsPath) getFileStatus(f); } catch (FileNotFoundException ex) { // remove orphan link bucket directly - if (isLinkBucket(f, bucket)) { + if (bucket.isLink()) { deleteBucketFromVolume(f, bucket); return true; } @@ -791,7 +793,7 @@ private boolean deleteBucket(Path f, boolean recursive, OFSPath ofsPath) boolean handleTrailingSlash = f.toString().endsWith(OZONE_URI_DELIMITER); // remove link bucket directly if link and // rm path does not have trailing slash - if (isLinkBucket(f, bucket) && !handleTrailingSlash) { + if (bucket.isLink() && !handleTrailingSlash) { deleteBucketFromVolume(f, bucket); return true; } @@ -808,19 +810,6 @@ private boolean deleteBucket(Path f, boolean recursive, OFSPath ofsPath) return result; } - private boolean isLinkBucket(Path f, OzoneBucket bucket) { - try { - if (bucket.isLink()) { - return true; - } - } catch (Exception ex) { - LOG.error("Exception while getting bucket link information, " + - "considered it as false", ex); - return false; - } - return false; - } - private void deleteBucketFromVolume(Path f, OzoneBucket bucket) throws IOException { OzoneVolume volume = From e23a0300bc46cca681093b93aef286f4d8c03c6d Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Mon, 29 Jul 2024 12:42:10 +0530 Subject: [PATCH 6/8] Fix checkstyle --- .../hadoop/fs/ozone/BasicRootedOzoneFileSystem.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 820c2f5a154..7cb86ec29e0 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -768,12 +768,12 @@ private boolean deleteInSpan(Path f, boolean recursive) throws IOException { private boolean deleteBucket(Path f, boolean recursive, OFSPath ofsPath) throws IOException { - OzoneBucket bucket; - try { - bucket = adapterImpl.getBucket(ofsPath, false); - } catch (Exception ex) { - return false; - } + OzoneBucket bucket; + try { + bucket = adapterImpl.getBucket(ofsPath, false); + } catch (Exception ex) { + return false; + } // check status of normal bucket try { getFileStatus(f); From fe5c989fad0f540d136e06fc80567a049245874e Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Mon, 29 Jul 2024 13:18:40 +0530 Subject: [PATCH 7/8] Address review comments - add LOG.error --- .../org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 7cb86ec29e0..daf204f6790 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -772,6 +772,8 @@ private boolean deleteBucket(Path f, boolean recursive, OFSPath ofsPath) try { bucket = adapterImpl.getBucket(ofsPath, false); } catch (Exception ex) { + LOG.error("Exception while getting bucket link information, " + + "considered it as false", ex); return false; } // check status of normal bucket From d8e0a1e2d479da6896fbe744bb12df7da9763690 Mon Sep 17 00:00:00 2001 From: tanvipenumudy Date: Mon, 29 Jul 2024 13:47:51 +0530 Subject: [PATCH 8/8] Ignore OMException with BUCKET_NOT_FOUND or VOLUME_NOT_FOUND silently --- .../hadoop/fs/ozone/BasicRootedOzoneFileSystem.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index daf204f6790..224d94baa5d 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -91,6 +91,8 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_EMPTY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_EMPTY; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; /** * The minimal Rooted Ozone Filesystem implementation. @@ -771,9 +773,13 @@ private boolean deleteBucket(Path f, boolean recursive, OFSPath ofsPath) OzoneBucket bucket; try { bucket = adapterImpl.getBucket(ofsPath, false); + } catch (OMException ex) { + if (ex.getResult() != BUCKET_NOT_FOUND && ex.getResult() != VOLUME_NOT_FOUND) { + LOG.error("OMException while getting bucket information, considered it as false", ex); + } + return false; } catch (Exception ex) { - LOG.error("Exception while getting bucket link information, " + - "considered it as false", ex); + LOG.error("Exception while getting bucket information, considered it as false", ex); return false; } // check status of normal bucket