From 459e990086a3ad445942b4ad5a65ceb350eb4db6 Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Tue, 6 Apr 2021 15:36:18 +0530 Subject: [PATCH] HDDS-4495. [FSO]Delete : Implement async cleanup of garbage and orphan sub-dirs/files (#2093) --- .../src/main/resources/ozone-default.xml | 19 ++ .../java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../apache/hadoop/ozone/om/OMConfigKeys.java | 13 + ...DirectoryDeletingServiceWithFSOBucket.java | 318 ++++++++++++++++++ .../src/main/proto/OmClientProtocol.proto | 15 + .../hadoop/ozone/om/OMMetadataManager.java | 8 + .../ozone/om/DirectoryDeletingService.java | 286 ++++++++++++++++ .../apache/hadoop/ozone/om/KeyManager.java | 56 +++ .../hadoop/ozone/om/KeyManagerImpl.java | 112 ++++++ .../ozone/om/OmMetadataManagerImpl.java | 14 + .../hadoop/ozone/om/codec/OMDBDefinition.java | 9 +- .../ratis/utils/OzoneManagerRatisUtils.java | 3 + .../om/request/key/OMKeyDeleteRequestV1.java | 6 +- .../om/request/key/OMPathsPurgeRequestV1.java | 64 ++++ .../response/key/OMKeyDeleteResponseV1.java | 12 +- .../response/key/OMPathsPurgeResponseV1.java | 121 +++++++ .../key/TestOMKeyDeleteResponseV1.java | 4 +- 17 files changed, 1053 insertions(+), 8 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSOBucket.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DirectoryDeletingService.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMPathsPurgeRequestV1.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMPathsPurgeResponseV1.java diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 2b9da60a500..c3580a35907 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -2779,4 +2779,23 @@ existing unit test cases won't be affected. New OM version should be 'V1' + + ozone.directory.deleting.service.interval + 1m + OZONE, PERFORMANCE, OM + Time interval of the directory deleting service. It runs on OM + periodically and cleanup orphan directory and its sub-tree. For every + orphan directory it deletes the sub-path tree structure(dirs/files). It + sends sub-files to KeyDeletingService to deletes its blocks. Unit could + be defined with postfix (ns,ms,s,m,h,d) + + + + ozone.path.deleting.limit.per.task + 10000 + OZONE, PERFORMANCE, OM + A maximum number of paths(dirs/files) to be deleted by + directory deleting service per time interval. + + diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index c389becf6b8..5fedd835aa0 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -289,6 +289,7 @@ public static boolean isReadOnly( case PurgeKeys: case RecoverTrash: case DeleteOpenKeys: + case PurgePaths: return false; default: LOG.error("CmdType {} is not categorized as readOnly or not.", cmdType); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 71344f9126a..6a648183890 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -253,4 +253,17 @@ private OMConfigKeys() { public static final String OZONE_OM_LAYOUT_VERSION_DEFAULT = "V0"; public static final String OZONE_OM_LAYOUT_VERSION_V1 = "V1"; + + /** + * Configuration properties for Directory Deleting Service. + */ + public static final String OZONE_DIR_DELETING_SERVICE_INTERVAL = + "ozone.directory.deleting.service.interval"; + public static final String OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT + = "60s"; + + public static final String OZONE_PATH_DELETING_LIMIT_PER_TASK = + "ozone.path.deleting.limit.per.task"; + public static final int OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT = 10000; + } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSOBucket.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSOBucket.java new file mode 100644 index 00000000000..aa2d8e7b096 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSOBucket.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.ozone; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.om.DirectoryDeletingService; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; +import static org.junit.Assert.fail; + +/** + * Directory deletion service test cases. + */ +public class TestDirectoryDeletingServiceWithFSOBucket { + + private static final Logger LOG = + LoggerFactory.getLogger(TestDirectoryDeletingServiceWithFSOBucket.class); + + /** + * Set a timeout for each test. + */ + @Rule + public Timeout timeout = Timeout.seconds(300); + + private static boolean isBucketFSOptimized = true; + private static boolean enabledFileSystemPaths = true; + private static boolean omRatisEnabled = true; + + private static MiniOzoneCluster cluster; + private static FileSystem fs; + private static String volumeName; + private static String bucketName; + + @BeforeClass + public static void init() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 3); + conf.setInt(OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK, 5); + conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled); + conf.setBoolean(OZONE_ACL_ENABLED, true); + if (isBucketFSOptimized) { + TestOMRequestUtils.configureFSOptimizedPaths(conf, + enabledFileSystemPaths, OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1); + } else { + conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + enabledFileSystemPaths); + } + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(3) + .build(); + cluster.waitForClusterToBeReady(); + + // create a volume and a bucket to be used by OzoneFileSystem + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); + volumeName = bucket.getVolumeName(); + bucketName = bucket.getName(); + + String rootPath = String.format("%s://%s.%s/", + OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName); + + // Set the fs.defaultFS and start the filesystem + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); + // Set the number of keys to be processed during batch operate. + conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5); + + fs = FileSystem.get(conf); + } + + @AfterClass + public static void teardown() { + if (cluster != null) { + cluster.shutdown(); + } + IOUtils.closeQuietly(fs); + } + + @After + public void cleanup() { + try { + Path root = new Path("/"); + FileStatus[] fileStatuses = fs.listStatus(root); + for (FileStatus fileStatus : fileStatuses) { + fs.delete(fileStatus.getPath(), true); + } + } catch (IOException ex) { + fail("Failed to cleanup files."); + } + } + + @Test + public void testDeleteEmptyDirectory() throws Exception { + Path root = new Path("/rootDir"); + Path appRoot = new Path(root, "appRoot"); + fs.mkdirs(appRoot); + + Table deletedDirTable = + cluster.getOzoneManager().getMetadataManager().getDeletedDirTable(); + Table dirTable = + cluster.getOzoneManager().getMetadataManager().getDirectoryTable(); + + + DirectoryDeletingService dirDeletingService = + (DirectoryDeletingService) cluster.getOzoneManager().getKeyManager() + .getDirDeletingService(); + // Before delete + assertTableRowCount(deletedDirTable, 0); + assertTableRowCount(dirTable, 2); + + assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 0); + assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0); + + // Delete the appRoot, empty dir + fs.delete(appRoot, true); + + // After Delete + checkPath(appRoot); + + assertTableRowCount(deletedDirTable, 0); + assertTableRowCount(dirTable, 1); + + assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 1); + assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0); + + Assert.assertTrue(dirTable.iterator().hasNext()); + Assert.assertEquals(root.getName(), + dirTable.iterator().next().getValue().getName()); + + Assert.assertTrue(dirDeletingService.getRunCount() > 1); + } + + /** + * Tests verifies that directories and files are getting purged in multiple + * batches. + */ + @Test + public void testDeleteWithLargeSubPathsThanBatchSize() throws Exception { + Path root = new Path("/rootDir"); + Path appRoot = new Path(root, "appRoot"); + // Creates 2 parent dirs from root. + fs.mkdirs(appRoot); + + // create 2 more levels. In each level, creates 5 subdirs and 5 subfiles. + // This will create total of 3 parentDirs + (3 * 5) childDirs and + // Total of (3 * 5) childFiles + for (int i = 1; i <= 3; i++) { + Path childDir = new Path(appRoot, "parentDir" + i); + for (int j = 1; j <= 5; j++) { + // total 5 sub-dirs + 5 sub-files = 10 items in this level. + Path childSubDir = new Path(childDir, "childDir" + j); + Path childSubFile = new Path(childDir, "childFile" + j); + ContractTestUtils.touch(fs, childSubFile); // create sub file + fs.mkdirs(childSubDir); // create sub dir + } + } + + Table deletedDirTable = + cluster.getOzoneManager().getMetadataManager().getDeletedDirTable(); + Table keyTable = + cluster.getOzoneManager().getMetadataManager().getKeyTable(); + Table dirTable = + cluster.getOzoneManager().getMetadataManager().getDirectoryTable(); + + DirectoryDeletingService dirDeletingService = + (DirectoryDeletingService) cluster.getOzoneManager().getKeyManager() + .getDirDeletingService(); + + // Before delete + assertTableRowCount(deletedDirTable, 0); + assertTableRowCount(keyTable, 15); + assertTableRowCount(dirTable, 20); + + assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0); + assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 0); + + // Delete the appRoot + fs.delete(appRoot, true); + + // After Delete + checkPath(appRoot); + + assertTableRowCount(deletedDirTable, 0); + assertTableRowCount(keyTable, 0); + assertTableRowCount(dirTable, 1); + + assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 15); + assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 19); + + Assert.assertTrue(dirDeletingService.getRunCount() > 1); + } + + @Test + public void testDeleteWithMultiLevels() throws Exception { + Path root = new Path("/rootDir"); + Path appRoot = new Path(root, "appRoot"); + + for (int i = 1; i <= 3; i++) { + Path parent = new Path(appRoot, "parentDir" + i); + Path child = new Path(parent, "childFile"); + ContractTestUtils.touch(fs, child); + } + + Table deletedDirTable = + cluster.getOzoneManager().getMetadataManager().getDeletedDirTable(); + Table keyTable = + cluster.getOzoneManager().getMetadataManager().getKeyTable(); + Table dirTable = + cluster.getOzoneManager().getMetadataManager().getDirectoryTable(); + + DirectoryDeletingService dirDeletingService = + (DirectoryDeletingService) cluster.getOzoneManager().getKeyManager() + .getDirDeletingService(); + + // Before delete + assertTableRowCount(deletedDirTable, 0); + assertTableRowCount(dirTable, 5); + assertTableRowCount(keyTable, 3); + + assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0); + assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 0); + + // Delete the rootDir, which should delete all keys. + fs.delete(root, true); + + // After Delete + checkPath(root); + + assertTableRowCount(deletedDirTable, 0); + assertTableRowCount(keyTable, 0); + assertTableRowCount(dirTable, 0); + + assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 3); + assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 5); + + Assert.assertTrue(dirDeletingService.getRunCount() > 1); + } + + private void assertSubPathsCount(long pathCount, long expectedCount) + throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor(() -> pathCount >= expectedCount, 1000, 120000); + } + + private void assertTableRowCount(Table table, int count) + throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor(() -> assertTableRowCount(count, table), 1000, + 120000); // 2 minutes + } + + private boolean assertTableRowCount(int expectedCount, + Table table) { + long count = 0L; + try { + count = cluster.getOzoneManager().getMetadataManager() + .countRowsInTable(table); + LOG.info("{} actual row count={}, expectedCount={}", table.getName(), + count, expectedCount); + } catch (IOException ex) { + fail("testDoubleBuffer failed with: " + ex); + } + return count == expectedCount; + } + + private void checkPath(Path path) { + try { + fs.getFileStatus(path); + fail("testRecursiveDelete failed"); + } catch (IOException ex) { + Assert.assertTrue(ex instanceof FileNotFoundException); + Assert.assertTrue(ex.getMessage().contains("No such file or directory")); + } + } + +} diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 2bae4e5c7ca..3bacb0e1b1b 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -93,6 +93,8 @@ enum Type { ListTrash = 91; RecoverTrash = 92; + + PurgePaths = 93; } message OMRequest { @@ -165,6 +167,8 @@ message OMRequest { optional ListTrashRequest listTrashRequest = 91; optional RecoverTrashRequest RecoverTrashRequest = 92; + + optional PurgePathsRequest purgePathsRequest = 93; } message OMResponse { @@ -235,6 +239,7 @@ message OMResponse { optional ListTrashResponse listTrashResponse = 91; optional RecoverTrashResponse RecoverTrashResponse = 92; + optional PurgePathsResponse purgePathsResponse = 93; } enum Status { @@ -956,6 +961,16 @@ message PurgeKeysResponse { } +message PurgePathsRequest { + repeated string deletedDirs = 1; + repeated KeyInfo deletedSubFiles = 2; + repeated KeyInfo markDeletedSubDirs = 3; +} + +message PurgePathsResponse { + +} + message DeleteOpenKeysRequest { repeated OpenKeyBucket openKeysPerBucket = 1; } diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 4c66040b53f..bcbef0ca60b 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -436,4 +436,12 @@ Set getMultipartUploadKeys(String volumeName, * @return bytes of DB key. */ String getMultipartKey(long parentObjectId, String fileName, String uploadId); + + /** + * Get Deleted Directory Table. + * + * @return Deleted Directory Table. + */ + Table getDeletedDirTable(); + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DirectoryDeletingService.java new file mode 100644 index 00000000000..ec5c3a91171 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DirectoryDeletingService.java @@ -0,0 +1,286 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om; + +import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ServiceException; +import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.hdds.utils.BackgroundTask; +import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; +import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.util.Time; +import org.apache.ratis.protocol.ClientId; +import org.apache.ratis.protocol.Message; +import org.apache.ratis.protocol.RaftClientRequest; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.hadoop.ozone.ClientVersions.CURRENT_VERSION; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT; + +/** + * This is a background service to delete orphan directories and its + * sub paths(sub-dirs and sub-files). + * + *

+ * This will scan the metadata of om periodically to get the orphan dirs from + * DeletedDirectoryTable and find its sub paths. It will fetch all sub-files + * from KeyTable and move those to DeletedTable so that OM's + * KeyDeletingService will cleanup those files later. It will fetch all + * sub-directories from the DirectoryTable and move those to + * DeletedDirectoryTable so that these will be visited in next iterations. + * + *

+ * After moving all sub-files and sub-dirs the parent orphan directory will be + * deleted by this service. It will continue traversing until all the leaf path + * components of an orphan directory is visited. + */ +public class DirectoryDeletingService extends BackgroundService { + + private final KeyManager keyManager; + private final OzoneManager ozoneManager; + private AtomicLong deletedDirsCount; + private AtomicLong deletedFilesCount; + private final AtomicLong runCount; + + private static ClientId clientId = ClientId.randomId(); + + // Use only a single thread for DirDeletion. Multiple threads would read + // or write to same tables and can send deletion requests for same key + // multiple times. + private static final int DIR_DELETING_CORE_POOL_SIZE = 1; + + // Number of items(dirs/files) to be batched in an iteration. + private final long pathLimitPerTask; + + public DirectoryDeletingService(long interval, TimeUnit unit, + long serviceTimeout, OzoneManager ozoneManager) { + super("DirectoryDeletingService", interval, unit, + DIR_DELETING_CORE_POOL_SIZE, serviceTimeout); + this.keyManager = ozoneManager.getKeyManager(); + this.ozoneManager = ozoneManager; + this.deletedDirsCount = new AtomicLong(0); + this.deletedFilesCount = new AtomicLong(0); + this.runCount = new AtomicLong(0); + this.pathLimitPerTask = ozoneManager.getConfiguration() + .getInt(OZONE_PATH_DELETING_LIMIT_PER_TASK, + OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT); + } + + private boolean shouldRun() { + if (ozoneManager == null) { + // OzoneManager can be null for testing + return true; + } + return ozoneManager.isLeaderReady(); + } + + private boolean isRatisEnabled() { + if (ozoneManager == null) { + return false; + } + return ozoneManager.isRatisEnabled(); + } + + @Override + public BackgroundTaskQueue getTasks() { + BackgroundTaskQueue queue = new BackgroundTaskQueue(); + queue.add(new DirectoryDeletingService.DirDeletingTask()); + return queue; + } + + private class DirDeletingTask implements BackgroundTask { + + @Override + public int getPriority() { + return 0; + } + + @Override + public BackgroundTaskResult call() throws Exception { + if (shouldRun()) { + runCount.incrementAndGet(); + long count = pathLimitPerTask; + try { + long startTime = Time.monotonicNow(); + // step-1) Get one pending deleted directory + OmKeyInfo pendingDeletedDirInfo = keyManager.getPendingDeletionDir(); + if (pendingDeletedDirInfo != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Pending deleted dir name: {}", + pendingDeletedDirInfo.getKeyName()); + } + // step-1: get all sub directories under the deletedDir + List dirs = + keyManager.getPendingDeletionSubDirs(pendingDeletedDirInfo, + count); + count = count - dirs.size(); + List deletedSubDirList = new ArrayList<>(); + for (OmKeyInfo dirInfo : dirs) { + deletedSubDirList.add(dirInfo); + if (LOG.isDebugEnabled()) { + LOG.debug("deleted sub dir name: {}", + dirInfo.getKeyName()); + } + } + + // step-2: get all sub files under the deletedDir + List purgeDeletedFiles = + keyManager.getPendingDeletionSubFiles(pendingDeletedDirInfo, + count); + count = count - purgeDeletedFiles.size(); + + if (LOG.isDebugEnabled()) { + for (OmKeyInfo fileInfo : purgeDeletedFiles) { + LOG.debug("deleted sub file name: {}", fileInfo.getKeyName()); + } + } + + // step-3: Since there is a boundary condition of 'numEntries' in + // each batch, check whether the sub paths count reached batch size + // limit. If count reached limit then there can be some more child + // paths to be visited and will keep the parent deleted directory + // for one more pass. + List purgeDeletedDirs = new ArrayList<>(); + if (count > 0) { + // TODO: Now, there is only one entry in this list. Maintained + // list data structure becuase this can be extended to add + // more directories within the batchSize limit. + purgeDeletedDirs.add(pendingDeletedDirInfo.getPath()); + } + + if (isRatisEnabled()) { + submitPurgePaths(purgeDeletedDirs, purgeDeletedFiles, + deletedSubDirList); + } + // TODO: need to handle delete with non-ratis + + deletedDirsCount.addAndGet(purgeDeletedDirs.size()); + deletedFilesCount.addAndGet(purgeDeletedFiles.size()); + if (LOG.isDebugEnabled()) { + LOG.debug("Number of dirs deleted: {}, Number of files moved:" + + " {} to DeletedTable, elapsed time: {}ms", + deletedDirsCount, deletedFilesCount, + Time.monotonicNow() - startTime); + } + } + } catch (IOException e) { + LOG.error("Error while running delete directories and files " + + "background task. Will retry at next run.", e); + } + } + + // place holder by returning empty results of this call back. + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + } + + /** + * Returns the number of dirs deleted by the background service. + * + * @return Long count. + */ + @VisibleForTesting + public long getDeletedDirsCount() { + return deletedDirsCount.get(); + } + + /** + * Returns the number of files moved to DeletedTable by the background + * service. + * + * @return Long count. + */ + @VisibleForTesting + public long getMovedFilesCount() { + return deletedFilesCount.get(); + } + + /** + * Returns the number of times this Background service has run. + * + * @return Long, run count. + */ + @VisibleForTesting + public long getRunCount() { + return runCount.get(); + } + + private int submitPurgePaths(List purgeDeletedDirs, + List purgeDeletedFiles, List markDirsAsDeleted) { + // Put all keys to be purged in a list + int deletedCount = 0; + OzoneManagerProtocolProtos.PurgePathsRequest.Builder purgePathsRequest = + OzoneManagerProtocolProtos.PurgePathsRequest.newBuilder(); + for (String purgeDir : purgeDeletedDirs) { + purgePathsRequest.addDeletedDirs(purgeDir); + } + for (OmKeyInfo purgeFile : purgeDeletedFiles) { + purgePathsRequest.addDeletedSubFiles( + purgeFile.getProtobuf(CURRENT_VERSION)); + } + + // Add these directories to deletedDirTable, so that its sub-paths will be + // traversed in next iteration to ensure cleanup all sub-children. + for (OmKeyInfo dir : markDirsAsDeleted) { + purgePathsRequest.addMarkDeletedSubDirs(dir.getProtobuf(CURRENT_VERSION)); + } + + OzoneManagerProtocolProtos.OMRequest omRequest = + OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.PurgePaths) + .setPurgePathsRequest(purgePathsRequest) + .setClientId(clientId.toString()) + .build(); + + // Submit Purge paths request to OM + try { + RaftClientRequest raftClientRequest = + createRaftClientRequestForDelete(omRequest); + ozoneManager.getOmRatisServer().submitRequest(omRequest, + raftClientRequest); + } catch (ServiceException e) { + LOG.error("PurgePaths request failed. Will retry at next run."); + return 0; + } + return deletedCount; + } + + + private RaftClientRequest createRaftClientRequestForDelete( + OzoneManagerProtocolProtos.OMRequest omRequest) { + return RaftClientRequest.newBuilder() + .setClientId(clientId) + .setServerId(ozoneManager.getOmRatisServer().getRaftPeerId()) + .setGroupId(ozoneManager.getOmRatisServer().getRaftGroupId()) + .setCallId(runCount.get()) + .setMessage( + Message.valueOf( + OMRatisHelper.convertRequestToByteString(omRequest))) + .setType(RaftClientRequest.writeRequestType()) + .build(); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 658f503a1a7..b569b5dec60 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -273,4 +273,60 @@ OmMultipartUploadListParts listParts(String volumeName, String bucketName, * @param key */ void refresh(OmKeyInfo key) throws IOException; + + /** + * Assume OM has FS namespace like below, deleteDirTable stores absolute + * path name as existing KeyDeletionService expects full key name. + * For example, if user deletes directory 'd1' then the entry in OM DB looks + * like, DBKey = 1030/d3 and DBValue = KeyInfo with keyName "a/b2/d3" + * + * vol1 + * | + * buck-1 + * | + * a + * | + * ----------------------------------- + * | | | + * b1 b2 b3 + * ----- --------- ---------- + * | | | | | | | | + * c1 c2 d1 d2 d3 e1 e2 e3 + * | | + * -------- | + * | | | + * d21.txt d22.txt e11.txt + * + * @return OmKeyInfo + * @throws IOException + */ + OmKeyInfo getPendingDeletionDir() throws IOException; + + /** + * Returns all sub directories under the given parent directory. + * + * @param parentInfo + * @param numEntries + * @return list of dirs + * @throws IOException + */ + List getPendingDeletionSubDirs(OmKeyInfo parentInfo, + long numEntries) throws IOException; + + /** + * Returns all sub files under the given parent directory. + * + * @param parentInfo + * @param numEntries + * @return list of files + * @throws IOException + */ + List getPendingDeletionSubFiles(OmKeyInfo parentInfo, + long numEntries) throws IOException; + + /** + * Returns the instance of Directory Deleting Service. + * @return Background service. + */ + BackgroundService getDirDeletingService(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 422a915131a..38a43070b88 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -134,6 +134,8 @@ import static org.apache.hadoop.ozone.ClientVersions.CURRENT_VERSION; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DIRECTORY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; @@ -176,6 +178,7 @@ public class KeyManagerImpl implements KeyManager { private final PrefixManager prefixManager; private final boolean enableFileSystemPaths; + private BackgroundService dirDeletingService; @VisibleForTesting @@ -250,6 +253,22 @@ public void start(OzoneConfiguration configuration) { serviceTimeout, configuration); keyDeletingService.start(); } + + // Start directory deletion service for FSO buckets. + if (OzoneManagerRatisUtils.isBucketFSOptimized() + && dirDeletingService == null) { + long dirDeleteInterval = configuration.getTimeDuration( + OZONE_DIR_DELETING_SERVICE_INTERVAL, + OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); + long serviceTimeout = configuration.getTimeDuration( + OZONE_BLOCK_DELETING_SERVICE_TIMEOUT, + OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT, + TimeUnit.MILLISECONDS); + dirDeletingService = new DirectoryDeletingService(dirDeleteInterval, + TimeUnit.SECONDS, serviceTimeout, ozoneManager); + dirDeletingService.start(); + } } KeyProviderCryptoExtension getKMSProvider() { @@ -262,6 +281,10 @@ public void stop() throws IOException { keyDeletingService.shutdown(); keyDeletingService = null; } + if (dirDeletingService != null) { + dirDeletingService.shutdown(); + dirDeletingService = null; + } } private OmBucketInfo getBucketInfo(String volumeName, String bucketName) @@ -979,6 +1002,11 @@ public BackgroundService getDeletingService() { return keyDeletingService; } + @Override + public BackgroundService getDirDeletingService() { + return dirDeletingService; + } + @Override public OmMultipartInfo initiateMultipartUpload(OmKeyArgs omKeyArgs) throws IOException { @@ -2894,4 +2922,88 @@ private static List toNodeUuid(Collection nodes) { } return nodeSet; } + + @Override + public OmKeyInfo getPendingDeletionDir() throws IOException { + OmKeyInfo omKeyInfo = null; + try (TableIterator> + deletedDirItr = metadataManager.getDeletedDirTable().iterator()) { + if (deletedDirItr.hasNext()) { + Table.KeyValue keyValue = deletedDirItr.next(); + if (keyValue != null) { + omKeyInfo = keyValue.getValue(); + } + } + } + return omKeyInfo; + } + + @Override + public List getPendingDeletionSubDirs(OmKeyInfo parentInfo, + long numEntries) throws IOException { + List directories = new ArrayList<>(); + String seekDirInDB = metadataManager.getOzonePathKey( + parentInfo.getObjectID(), ""); + long countEntries = 0; + + Table dirTable = metadataManager.getDirectoryTable(); + TableIterator> + iterator = dirTable.iterator(); + + iterator.seek(seekDirInDB); + + while (iterator.hasNext() && numEntries - countEntries > 0) { + OmDirectoryInfo dirInfo = iterator.value().getValue(); + if (!OMFileRequest.isImmediateChild(dirInfo.getParentObjectID(), + parentInfo.getObjectID())) { + break; + } + String dirName = OMFileRequest.getAbsolutePath(parentInfo.getKeyName(), + dirInfo.getName()); + OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo( + parentInfo.getVolumeName(), parentInfo.getBucketName(), dirInfo, + dirName); + directories.add(omKeyInfo); + countEntries++; + + // move to next entry in the DirTable + iterator.next(); + } + + return directories; + } + + @Override + public List getPendingDeletionSubFiles(OmKeyInfo parentInfo, + long numEntries) throws IOException { + List files = new ArrayList<>(); + String seekFileInDB = metadataManager.getOzonePathKey( + parentInfo.getObjectID(), ""); + long countEntries = 0; + + Table fileTable = metadataManager.getKeyTable(); + TableIterator> + iterator = fileTable.iterator(); + + iterator.seek(seekFileInDB); + + while (iterator.hasNext() && numEntries - countEntries > 0) { + OmKeyInfo fileInfo = iterator.value().getValue(); + if (!OMFileRequest.isImmediateChild(fileInfo.getParentObjectID(), + parentInfo.getObjectID())) { + break; + } + fileInfo.setFileName(fileInfo.getKeyName()); + String fullKeyPath = OMFileRequest.getAbsolutePath( + parentInfo.getKeyName(), fileInfo.getKeyName()); + fileInfo.setKeyName(fullKeyPath); + + files.add(fileInfo); + countEntries++; + // move to next entry in the KeyTable + iterator.next(); + } + + return files; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 4e4f91bdd0e..b67346ebbee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -137,6 +137,8 @@ public class OmMetadataManagerImpl implements OMMetadataManager { * |----------------------------------------------------------------------| * | multipartFileInfoTable | parentId/fileName/uploadId ->... | * |----------------------------------------------------------------------| + * | deletedDirTable | parentId/directoryName -> KeyInfo | + * |----------------------------------------------------------------------| * | transactionInfoTable | #TRANSACTIONINFO -> OMTransactionInfo | * |----------------------------------------------------------------------| */ @@ -155,6 +157,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager { public static final String FILE_TABLE = "fileTable"; public static final String OPEN_FILE_TABLE = "openFileTable"; public static final String MULTIPARTFILEINFO_TABLE = "multipartFileInfoTable"; + public static final String DELETED_DIR_TABLE = "deletedDirectoryTable"; public static final String TRANSACTION_INFO_TABLE = "transactionInfoTable"; @@ -180,6 +183,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager { private boolean isRatisEnabled; private boolean ignorePipelineinKey; private Table multipartFileInfoTable; + private Table deletedDirTable; // Epoch is used to generate the objectIDs. The most significant 2 bits of // objectIDs is set to this epoch. For clusters before HDDS-4315 there is @@ -255,6 +259,11 @@ public Table getDeletedTable() { return deletedTable; } + @Override + public Table getDeletedDirTable() { + return deletedDirTable; + } + @Override public Table getOpenKeyTable() { if (OzoneManagerRatisUtils.isBucketFSOptimized()) { @@ -372,6 +381,7 @@ public static DBStoreBuilder addOMTablesAndCodecs(DBStoreBuilder builder) { .addTable(FILE_TABLE) .addTable(OPEN_FILE_TABLE) .addTable(MULTIPARTFILEINFO_TABLE) + .addTable(DELETED_DIR_TABLE) .addTable(TRANSACTION_INFO_TABLE) .addCodec(OzoneTokenIdentifier.class, new TokenIdentifierCodec()) .addCodec(OmKeyInfo.class, new OmKeyInfoCodec(true)) @@ -454,6 +464,10 @@ protected void initializeOmTables() throws IOException { String.class, OmMultipartKeyInfo.class); checkTableStatus(multipartFileInfoTable, MULTIPARTFILEINFO_TABLE); + deletedDirTable = this.store.getTable(DELETED_DIR_TABLE, String.class, + OmKeyInfo.class); + checkTableStatus(deletedDirTable, DELETED_DIR_TABLE); + transactionInfoTable = this.store.getTable(TRANSACTION_INFO_TABLE, String.class, TransactionInfo.class); checkTableStatus(transactionInfoTable, TRANSACTION_INFO_TABLE); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index 77b9e044340..f3716dbb92d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -180,6 +180,12 @@ public class OMDBDefinition implements DBDefinition { OmMultipartKeyInfo.class, new OmMultipartKeyInfoCodec()); + public static final DBColumnFamilyDefinition + DELETED_DIR_TABLE = + new DBColumnFamilyDefinition<>(OmMetadataManagerImpl.DELETED_DIR_TABLE, + String.class, new StringCodec(), OmKeyInfo.class, + new OmKeyInfoCodec(true)); + @Override public String getName() { return OzoneConsts.OM_DB_NAME; @@ -196,7 +202,8 @@ public DBColumnFamilyDefinition[] getColumnFamilies() { VOLUME_TABLE, OPEN_KEY_TABLE, KEY_TABLE, BUCKET_TABLE, MULTIPART_INFO_TABLE, PREFIX_TABLE, DTOKEN_TABLE, S3_SECRET_TABLE, TRANSACTION_INFO_TABLE, DIRECTORY_TABLE, - FILE_TABLE, OPEN_FILE_TABLE, MULTIPART_FILEINFO_TABLE}; + FILE_TABLE, OPEN_FILE_TABLE, MULTIPART_FILEINFO_TABLE, + DELETED_DIR_TABLE}; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index b3e6c4fd5ee..f4628a93071 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -48,6 +48,7 @@ import org.apache.hadoop.ozone.om.request.key.OMKeyRenameRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyRenameRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeysRenameRequest; +import org.apache.hadoop.ozone.om.request.key.OMPathsPurgeRequestV1; import org.apache.hadoop.ozone.om.request.key.OMTrashRecoverRequest; import org.apache.hadoop.ozone.om.request.key.acl.OMKeyAddAclRequest; import org.apache.hadoop.ozone.om.request.key.acl.OMKeyAddAclRequestV1; @@ -184,6 +185,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest) { return new OMFileCreateRequest(omRequest); case PurgeKeys: return new OMKeyPurgeRequest(omRequest); + case PurgePaths: + return new OMPathsPurgeRequestV1(omRequest); case InitiateMultiPartUpload: if (isBucketFSOptimized()) { return new S3InitiateMultipartUploadRequestV1(omRequest); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestV1.java index dbf564513ed..87427f897e8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequestV1.java @@ -160,9 +160,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, // TODO: Revisit if we need it later. omClientResponse = new OMKeyDeleteResponseV1(omResponse - .setDeleteKeyResponse(DeleteKeyResponse.newBuilder()).build(), - omKeyInfo, ozoneManager.isRatisEnabled(), - omBucketInfo.copyObject(), keyStatus.isDirectory()); + .setDeleteKeyResponse(DeleteKeyResponse.newBuilder()).build(), + keyName, omKeyInfo, ozoneManager.isRatisEnabled(), + omBucketInfo.copyObject(), keyStatus.isDirectory()); result = Result.SUCCESS; } catch (IOException ex) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMPathsPurgeRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMPathsPurgeRequestV1.java new file mode 100644 index 00000000000..b9d6066143e --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMPathsPurgeRequestV1.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.key; + +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.key.OMPathsPurgeResponseV1; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + +import java.util.List; + +/** + * Handles purging of keys from OM DB. + */ +public class OMPathsPurgeRequestV1 extends OMKeyRequest { + + public OMPathsPurgeRequestV1(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { + OzoneManagerProtocolProtos.PurgePathsRequest purgePathsRequest = + getOmRequest().getPurgePathsRequest(); + + List deletedDirsList = purgePathsRequest.getDeletedDirsList(); + List deletedSubFilesList = + purgePathsRequest.getDeletedSubFilesList(); + List markDeletedSubDirsList = + purgePathsRequest.getMarkDeletedSubDirsList(); + + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( + getOmRequest()); + + OMClientResponse omClientResponse = new OMPathsPurgeResponseV1( + omResponse.build(), markDeletedSubDirsList, deletedSubFilesList, + deletedDirsList, ozoneManager.isRatisEnabled()); + addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, + omDoubleBufferHelper); + + return omClientResponse; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseV1.java index 15c1ba64249..69e87df3985 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyDeleteResponseV1.java @@ -40,12 +40,14 @@ public class OMKeyDeleteResponseV1 extends OMKeyDeleteResponse { private boolean isDeleteDirectory; + private String keyName; public OMKeyDeleteResponseV1(@Nonnull OMResponse omResponse, - @Nonnull OmKeyInfo omKeyInfo, boolean isRatisEnabled, - @Nonnull OmBucketInfo omBucketInfo, + @Nonnull String keyName, @Nonnull OmKeyInfo omKeyInfo, + boolean isRatisEnabled, @Nonnull OmBucketInfo omBucketInfo, @Nonnull boolean isDeleteDirectory) { super(omResponse, omKeyInfo, isRatisEnabled, omBucketInfo); + this.keyName = keyName; this.isDeleteDirectory = isDeleteDirectory; } @@ -69,6 +71,12 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, if (isDeleteDirectory) { omMetadataManager.getDirectoryTable().deleteWithBatch(batchOperation, ozoneDbKey); + OmKeyInfo omKeyInfo = getOmKeyInfo(); + // Sets full absolute key name to OmKeyInfo, which is + // required for moving the sub-files to KeyDeletionService. + omKeyInfo.setKeyName(keyName); + omMetadataManager.getDeletedDirTable().putWithBatch( + batchOperation, ozoneDbKey, omKeyInfo); } else { Table keyTable = omMetadataManager.getKeyTable(); addDeletionToBatch(omMetadataManager, batchOperation, keyTable, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMPathsPurgeResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMPathsPurgeResponseV1.java new file mode 100644 index 00000000000..b6f5299dcb2 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMPathsPurgeResponseV1.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.key; + +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.request.key.OMPathsPurgeRequestV1; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_DIR_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; + +/** + * Response for {@link OMPathsPurgeRequestV1} request. + */ +@CleanupTableInfo(cleanupTables = {DELETED_TABLE, DELETED_DIR_TABLE, + DIRECTORY_TABLE, FILE_TABLE}) +public class OMPathsPurgeResponseV1 extends OMClientResponse { + private static final Logger LOG = + LoggerFactory.getLogger(OMPathsPurgeResponseV1.class); + + private List markDeletedDirList; + private List dirList; + private List fileList; + private boolean isRatisEnabled; + + + public OMPathsPurgeResponseV1(@Nonnull OMResponse omResponse, + @Nonnull List markDeletedDirs, + @Nonnull List files, + @Nonnull List dirs, boolean isRatisEnabled) { + super(omResponse); + this.markDeletedDirList = markDeletedDirs; + this.dirList = dirs; + this.fileList = files; + this.isRatisEnabled = isRatisEnabled; + } + + @Override + public void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { + + // Add all sub-directories to deleted directory table. + for (OzoneManagerProtocolProtos.KeyInfo key : markDeletedDirList) { + OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key); + String ozoneDbKey = omMetadataManager.getOzonePathKey( + keyInfo.getParentObjectID(), keyInfo.getFileName()); + omMetadataManager.getDeletedDirTable().putWithBatch(batchOperation, + ozoneDbKey, keyInfo); + + omMetadataManager.getDirectoryTable().deleteWithBatch(batchOperation, + ozoneDbKey); + + if (LOG.isDebugEnabled()) { + LOG.debug("markDeletedDirList KeyName: {}, DBKey: {}", + keyInfo.getKeyName(), ozoneDbKey); + } + } + + // Delete all the visited directories from deleted directory table + for (String key : dirList) { + omMetadataManager.getDeletedDirTable().deleteWithBatch(batchOperation, + key); + + if (LOG.isDebugEnabled()) { + LOG.info("Purge Deleted Directory DBKey: {}", key); + } + } + for (OzoneManagerProtocolProtos.KeyInfo key : fileList) { + OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key); + String ozoneDbKey = omMetadataManager.getOzonePathKey( + keyInfo.getParentObjectID(), keyInfo.getFileName()); + omMetadataManager.getKeyTable().deleteWithBatch(batchOperation, + ozoneDbKey); + + if (LOG.isDebugEnabled()) { + LOG.info("Move keyName:{} to DeletedTable DBKey: {}", + keyInfo.getKeyName(), ozoneDbKey); + } + + RepeatedOmKeyInfo repeatedOmKeyInfo = null; + repeatedOmKeyInfo = OmUtils.prepareKeyForDelete(keyInfo, + repeatedOmKeyInfo, keyInfo.getUpdateID(), isRatisEnabled); + + omMetadataManager.getDeletedTable().putWithBatch(batchOperation, + keyInfo.getPath(), repeatedOmKeyInfo); + + } + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java index d46fe729698..4422527922d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java @@ -36,8 +36,8 @@ public class TestOMKeyDeleteResponseV1 extends TestOMKeyDeleteResponse { @Override protected OMKeyDeleteResponse getOmKeyDeleteResponse(OmKeyInfo omKeyInfo, OzoneManagerProtocolProtos.OMResponse omResponse) { - return new OMKeyDeleteResponseV1(omResponse, omKeyInfo, - true, getOmBucketInfo(), false); + return new OMKeyDeleteResponseV1(omResponse, omKeyInfo.getKeyName(), + omKeyInfo, true, getOmBucketInfo(), false); } @Override