From a7fea98d0b343ea54b86a21fcc2c308fb231ae27 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sun, 30 Nov 2025 02:13:04 -0500 Subject: [PATCH 1/2] HDDS-14037. Create DBDefinition and corresponding MetadataManager for SnapshotDiff DB Change-Id: I30eb4fe01cd73fcf8113b2896e6eaff4b77eeb26 --- .../src/main/proto/OmServerProtocol.proto | 37 +++++ .../hadoop/ozone/om/OmSnapshotManager.java | 47 +----- .../om/snapshot/SnapshotDiffManager.java | 13 +- .../snapshot/db/SnapshotDiffDBDefinition.java | 145 +++++++++++++++++ .../db/SnapshotDiffMetadataManager.java | 51 ++++++ .../db/SnapshotDiffMetadataManagerImpl.java | 153 ++++++++++++++++++ .../ozone/om/snapshot/db/package-info.java | 21 +++ .../diff/helper/SnapshotDiffObjectInfo.java | 61 +++++++ .../om/snapshot/diff/helper/package-info.java | 21 +++ .../om/snapshot/TestSnapshotDiffManager.java | 4 +- 10 files changed, 502 insertions(+), 51 deletions(-) create mode 100644 hadoop-hdds/interface-server/src/main/proto/OmServerProtocol.proto create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffDBDefinition.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManager.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManagerImpl.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/package-info.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/package-info.java diff --git a/hadoop-hdds/interface-server/src/main/proto/OmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/OmServerProtocol.proto new file mode 100644 index 000000000000..54a3dc665bbe --- /dev/null +++ b/hadoop-hdds/interface-server/src/main/proto/OmServerProtocol.proto @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * These .proto interfaces are private and unstable. + * Please see http://wiki.apache.org/hadoop/Compatibility + * for what changes are allowed for a *unstable* .proto interface. + */ +syntax = "proto2"; +option java_package = "org.apache.hadoop.hdds.protocol.proto"; +option java_outer_classname = "OmServerProtocolProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.hdds.block; + +import "hdds.proto"; + +// OM Server protocol +message SnapDiffObjectInfo { + optional uint64 objectID = 1; + optional string keyName = 2; +} \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index a1c7aa918fa0..fa7d137df945 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -53,6 +53,7 @@ import static org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager.getSnapshotRootPath; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotActive; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.dropColumnFamilyHandle; +import static org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_PURGED_JOB_TABLE_NAME; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE; import com.google.common.annotations.VisibleForTesting; @@ -104,6 +105,7 @@ import org.apache.hadoop.ozone.om.snapshot.SnapshotCache; import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; @@ -133,43 +135,6 @@ public final class OmSnapshotManager implements AutoCloseable { public static final String DELIMITER = "-"; - /** - * Contains all the snap diff job which are either queued, in_progress or - * done. This table is used to make sure that there is only single job for - * requests with the same snapshot pair at any point of time. - * |------------------------------------------------| - * | KEY | VALUE | - * |------------------------------------------------| - * | fromSnapshotId-toSnapshotId | SnapshotDiffJob | - * |------------------------------------------------| - */ - public static final String SNAP_DIFF_JOB_TABLE_NAME = - "snap-diff-job-table"; - - /** - * Global table to keep the diff report. Each key is prefixed by the jobId - * to improve look up and clean up. JobId comes from snap-diff-job-table. - * |--------------------------------| - * | KEY | VALUE | - * |--------------------------------| - * | jobId-index | DiffReportEntry | - * |--------------------------------| - */ - public static final String SNAP_DIFF_REPORT_TABLE_NAME = - "snap-diff-report-table"; - - /** - * Contains all the snap diff job which can be purged either due to max - * allowed time is over, FAILED or REJECTED. - * |-------------------------------------------| - * | KEY | VALUE | - * |-------------------------------------------| - * | jobId | numOfTotalEntriesInReportTable | - * |-------------------------------------------| - */ - private static final String SNAP_DIFF_PURGED_JOB_TABLE_NAME = - "snap-diff-purged-job-table"; - /** * For snapshot compaction we need to capture SST files following column * families before compaction. @@ -249,9 +214,9 @@ public OmSnapshotManager(OzoneManager ozoneManager) throws IOException { this.snapshotDiffDb = createRocksDbForSnapshotDiff(options, dbPath, columnFamilyDescriptors, columnFamilyHandles); - snapDiffJobCf = getOrCreateColumnFamily(SNAP_DIFF_JOB_TABLE_NAME, + snapDiffJobCf = getOrCreateColumnFamily(SnapshotDiffDBDefinition.SNAP_DIFF_JOB_TABLE_NAME, columnFamilyDescriptors, columnFamilyHandles); - snapDiffReportCf = getOrCreateColumnFamily(SNAP_DIFF_REPORT_TABLE_NAME, + snapDiffReportCf = getOrCreateColumnFamily(SnapshotDiffDBDefinition.SNAP_DIFF_REPORT_TABLE_NAME, columnFamilyDescriptors, columnFamilyHandles); snapDiffPurgedJobCf = getOrCreateColumnFamily( SNAP_DIFF_PURGED_JOB_TABLE_NAME, columnFamilyDescriptors, @@ -1113,8 +1078,8 @@ private void dropUnknownColumnFamilies( List familyHandles ) { Set allowedColumnFamilyOnStartUp = new HashSet<>( - Arrays.asList(DEFAULT_COLUMN_FAMILY_NAME, SNAP_DIFF_JOB_TABLE_NAME, - SNAP_DIFF_REPORT_TABLE_NAME, SNAP_DIFF_PURGED_JOB_TABLE_NAME)); + Arrays.asList(DEFAULT_COLUMN_FAMILY_NAME, SnapshotDiffDBDefinition.SNAP_DIFF_JOB_TABLE_NAME, + SnapshotDiffDBDefinition.SNAP_DIFF_REPORT_TABLE_NAME, SNAP_DIFF_PURGED_JOB_TABLE_NAME)); try { for (ColumnFamilyHandle columnFamilyHandle : familyHandles) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index ccb9e8c792d5..4eeec3dc4c62 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -116,6 +116,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithObjectID; import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; +import org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition; import org.apache.hadoop.ozone.om.snapshot.diff.delta.CompositeDeltaDiffComputer; import org.apache.hadoop.ozone.om.snapshot.diff.delta.DeltaFileComputer; import org.apache.hadoop.ozone.om.snapshot.util.TableMergeIterator; @@ -141,11 +142,7 @@ * Class to generate snapshot diff. */ public class SnapshotDiffManager implements AutoCloseable { - private static final Logger LOG = - LoggerFactory.getLogger(SnapshotDiffManager.class); - private static final String FROM_SNAP_TABLE_SUFFIX = "-from-snap"; - private static final String TO_SNAP_TABLE_SUFFIX = "-to-snap"; - private static final String UNIQUE_IDS_TABLE_SUFFIX = "-unique-ids"; + private static final Logger LOG = LoggerFactory.getLogger(SnapshotDiffManager.class); private static final String DELETE_DIFF_TABLE_SUFFIX = "-delete-diff"; private static final String RENAME_DIFF_TABLE_SUFFIX = "-rename-diff"; private static final String CREATE_DIFF_TABLE_SUFFIX = "-create-diff"; @@ -816,11 +813,11 @@ void generateSnapshotDiffReport(final String jobKey, // JobId is prepended to column families name to make them unique // for request. fromSnapshotColumnFamily = - createColumnFamily(jobId + FROM_SNAP_TABLE_SUFFIX); + createColumnFamily(jobId + SnapshotDiffDBDefinition.SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_NAME); toSnapshotColumnFamily = - createColumnFamily(jobId + TO_SNAP_TABLE_SUFFIX); + createColumnFamily(jobId + SnapshotDiffDBDefinition.SNAP_DIFF_TO_SNAP_OBJECT_TABLE_NAME); objectIDsColumnFamily = - createColumnFamily(jobId + UNIQUE_IDS_TABLE_SUFFIX); + createColumnFamily(jobId + SnapshotDiffDBDefinition.SNAP_DIFF_UNIQUE_IDS_TABLE_NAME); // ObjectId to keyName map to keep key info for fromSnapshot. // objectIdToKeyNameMap is used to identify what keys were touched diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffDBDefinition.java new file mode 100644 index 000000000000..1add663c7f1c --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffDBDefinition.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.db; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DB_NAME; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DB_DIR; +import static org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone.getDiffReportEntryCodec; + +import java.util.Map; +import org.apache.hadoop.hdds.utils.db.BooleanCodec; +import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; +import org.apache.hadoop.hdds.utils.db.DBDefinition; +import org.apache.hadoop.hdds.utils.db.LongCodec; +import org.apache.hadoop.hdds.utils.db.StringCodec; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; +import org.apache.hadoop.ozone.om.snapshot.diff.helper.SnapshotDiffObjectInfo; + +/** + * The SnapshotDiffDBDefinition class defines the schema for the snapshot + * difference database tables used in snapshot diff operations. Each table + * corresponds to a specific aspect of snapshot diff job processing and data + * storage, ensuring proper organization and efficient access. + */ +public final class SnapshotDiffDBDefinition extends DBDefinition.WithMap { + /** + * Contains all the snap diff job which are either queued, in_progress or + * done. This table is used to make sure that there is only single job for + * requests with the same snapshot pair at any point of time. + * |------------------------------------------------| + * | KEY | VALUE | + * |------------------------------------------------| + * | fromSnapshotId-toSnapshotId | SnapshotDiffJob | + * |------------------------------------------------| + */ + public static final String SNAP_DIFF_JOB_TABLE_NAME = "snap-diff-job-table"; + + public static final DBColumnFamilyDefinition SNAP_DIFF_JOB_TABLE_DEF + = new DBColumnFamilyDefinition<>(SNAP_DIFF_JOB_TABLE_NAME, StringCodec.get(), SnapshotDiffJob.getCodec()); + /** + * Global table to keep the diff report. Each key is prefixed by the jobId + * to improve look up and clean up. JobId comes from snap-diff-job-table. + * |--------------------------------| + * | KEY | VALUE | + * |--------------------------------| + * | jobId-index | DiffReportEntry | + * |--------------------------------| + */ + public static final String SNAP_DIFF_REPORT_TABLE_NAME = "snap-diff-report-table"; + public static final DBColumnFamilyDefinition SNAP_DIFF_REPORT_TABLE_NAME_DEF + = new DBColumnFamilyDefinition<>(SNAP_DIFF_REPORT_TABLE_NAME, StringCodec.get(), getDiffReportEntryCodec()); + /** + * Contains all the snap diff job which can be purged either due to max + * allowed time is over, FAILED or REJECTED. + * |-------------------------------------------| + * | KEY | VALUE | + * |-------------------------------------------| + * | jobId | numOfTotalEntriesInReportTable | + * |-------------------------------------------| + */ + public static final String SNAP_DIFF_PURGED_JOB_TABLE_NAME = "snap-diff-purged-job-table"; + public static final DBColumnFamilyDefinition SNAP_DIFF_PURGED_JOB_TABLE_DEF + = new DBColumnFamilyDefinition<>(SNAP_DIFF_PURGED_JOB_TABLE_NAME, StringCodec.get(), LongCodec.get()); + /** + * Contains all the snap diff job intermediate object output for from snapshot. + * |----------------------------------------------------| + * | KEY | VALUE | + * |----------------------------------------------------| + * | jobId-objectId | SnapshotDiffObjectInfo | + * |----------------------------------------------------| + */ + public static final String SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_NAME = "-from-snap"; + public static final DBColumnFamilyDefinition SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_DEF + = new DBColumnFamilyDefinition<>(SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_NAME, StringCodec.get(), + SnapshotDiffObjectInfo.getCodec()); + + /** + * Contains all the snap diff job intermediate object output for to snapshot. + * |----------------------------------------------------| + * | KEY | VALUE | + * |----------------------------------------------------| + * | jobId-objectId | SnapshotDiffObjectInfo | + * |----------------------------------------------------| + */ + public static final String SNAP_DIFF_TO_SNAP_OBJECT_TABLE_NAME = "-to-snap"; + public static final DBColumnFamilyDefinition SNAP_DIFF_TO_SNAP_OBJECT_TABLE_DEF + = new DBColumnFamilyDefinition<>(SNAP_DIFF_TO_SNAP_OBJECT_TABLE_NAME, StringCodec.get(), + SnapshotDiffObjectInfo.getCodec()); + + /** + * Contains all the snap diff job intermediate object output for to snapshot. + * |----------------------------------------------------| + * | KEY | VALUE | + * |----------------------------------------------------| + * | jobId-objectId | boolean | + * |----------------------------------------------------| + */ + public static final String SNAP_DIFF_UNIQUE_IDS_TABLE_NAME = "-unique-ids"; + public static final DBColumnFamilyDefinition SNAP_DIFF_UNIQUE_IDS_TABLE_DEF + = new DBColumnFamilyDefinition<>(SNAP_DIFF_UNIQUE_IDS_TABLE_NAME, StringCodec.get(), BooleanCodec.get()); + + //---------------------------------------------------------------------------// + private static final Map> COLUMN_FAMILIES = + DBColumnFamilyDefinition.newUnmodifiableMap(SNAP_DIFF_JOB_TABLE_DEF, + SNAP_DIFF_REPORT_TABLE_NAME_DEF, + SNAP_DIFF_PURGED_JOB_TABLE_DEF, + SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_DEF, + SNAP_DIFF_TO_SNAP_OBJECT_TABLE_DEF, + SNAP_DIFF_UNIQUE_IDS_TABLE_DEF); + + private static final SnapshotDiffDBDefinition INSTANCE = new SnapshotDiffDBDefinition(); + + public static SnapshotDiffDBDefinition get() { + return INSTANCE; + } + + private SnapshotDiffDBDefinition() { + super(COLUMN_FAMILIES); + } + + @Override + public String getName() { + return OM_SNAPSHOT_DIFF_DB_NAME; + } + + @Override + public String getLocationConfigKey() { + return OZONE_OM_SNAPSHOT_DIFF_DB_DIR; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManager.java new file mode 100644 index 000000000000..667a49380c56 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManager.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.db; + +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; +import org.apache.hadoop.ozone.om.snapshot.diff.helper.SnapshotDiffObjectInfo; + +/** + * Interface representing the metadata manager for snapshot difference operations. + * This interface provides methods to access various tables storing metadata + * related to snapshot diff jobs, diff reports, purged jobs, and object information + * for snapshots. + * + * Implementations of this interface should handle the setup, maintenance, and + * querying of the relevant metadata tables. + * + * The interface extends {@link AutoCloseable}, requiring implementations to + * handle proper resource cleanup. + */ +public interface SnapshotDiffMetadataManager extends AutoCloseable { + + Table getSnapshotDiffJobTable(); + + Table getSnapshotDiffReportTable(); + + Table getSnapshotDiffPurgedJobTable(); + + Table getSnapshotDiffFromSnapshotObjectInfoTable(); + + Table getSnapshotDiffToSnapshotObjectInfoTable(); + + Table getSnapshotDiffUniqueObjectIdsTable(); + +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManagerImpl.java new file mode 100644 index 000000000000..686981606238 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManagerImpl.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.db; + +import static org.apache.commons.io.FileUtils.deleteDirectory; +import static org.apache.hadoop.hdds.utils.db.DBStoreBuilder.getDBDirPath; +import static org.apache.hadoop.hdds.utils.db.cache.TableCache.CacheType.PARTIAL_CACHE; +import static org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_DEF; +import static org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_JOB_TABLE_DEF; +import static org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_PURGED_JOB_TABLE_DEF; +import static org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_REPORT_TABLE_NAME_DEF; +import static org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_TO_SNAP_OBJECT_TABLE_DEF; +import static org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_UNIQUE_IDS_TABLE_DEF; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; +import org.apache.hadoop.hdds.utils.db.StringCodec; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; +import org.apache.hadoop.ozone.om.snapshot.diff.helper.SnapshotDiffObjectInfo; + +/** + * Implementation of the {@link SnapshotDiffMetadataManager} interface. + * Provides functionality for managing metadata related to snapshot difference + * operations, including managing tables for snapshot diff jobs, reports, purged jobs, + * and snapshot object information. + * + * This class handles the persistent storage of snapshot diff metadata using a database. + * It ensures proper initialization and versioning of the metadata schema. + */ +public class SnapshotDiffMetadataManagerImpl implements SnapshotDiffMetadataManager { + + private static final String VERSION_FILE = "VERSION"; + private static final String VERSION_NUMBER = "1"; + + private final DBStore dbStore; + + private final Table snapshotDiffJobTable; + private final Table snapshotDiffReportTable; + private final Table snapshotDiffPurgedJobTable; + private final Table snapshotDiffFromSnapshotObjectInfoTable; + private final Table snapshotDiffToSnapshotObjectInfoTable; + private final Table snapshotDiffUniqueObjectIdsTable; + + public SnapshotDiffMetadataManagerImpl(ConfigurationSource conf) throws IOException { + File dbPath = getDBDirPath(SnapshotDiffDBDefinition.get(), conf).toPath() + .resolve(SnapshotDiffDBDefinition.get().getName()).toFile(); + Path versionFilePath = dbPath.toPath().resolve(VERSION_FILE); + boolean versionContentMatches = checkAndDeleteIfVersionMismatches(dbPath, versionFilePath); + + this.dbStore = DBStoreBuilder.newBuilder(conf, SnapshotDiffDBDefinition.get(), dbPath).build(); + + this.snapshotDiffJobTable = SNAP_DIFF_JOB_TABLE_DEF.getTable(dbStore, PARTIAL_CACHE); + this.snapshotDiffReportTable = SNAP_DIFF_REPORT_TABLE_NAME_DEF.getTable(dbStore, PARTIAL_CACHE); + this.snapshotDiffPurgedJobTable = SNAP_DIFF_PURGED_JOB_TABLE_DEF.getTable(dbStore, PARTIAL_CACHE); + this.snapshotDiffFromSnapshotObjectInfoTable = + SNAP_DIFF_FROM_SNAP_OBJECT_TABLE_DEF.getTable(dbStore, PARTIAL_CACHE); + this.snapshotDiffToSnapshotObjectInfoTable = SNAP_DIFF_TO_SNAP_OBJECT_TABLE_DEF.getTable(dbStore, PARTIAL_CACHE); + this.snapshotDiffUniqueObjectIdsTable = SNAP_DIFF_UNIQUE_IDS_TABLE_DEF.getTable(dbStore, PARTIAL_CACHE); + if (!versionContentMatches) { + writeVersionFileContent(versionFilePath); + } + } + + /** + * Checks if the version specified in the version file matches the expected version. + * If the versions do not match and the database path exists, the database directory + * is deleted. + * + * @param dbPath the path to the database directory that may be deleted in case of + * a version mismatch + * @param versionFilePath the path to the file containing the version information + * @return {@code true} if the version matches, {@code false} otherwise + * @throws IOException if an I/O error occurs while reading the version file or deleting + * the database directory + */ + private boolean checkAndDeleteIfVersionMismatches(File dbPath, Path versionFilePath) throws IOException { + boolean versionContentMatches = VERSION_NUMBER.equals(getVersionFileContent(versionFilePath)); + if (!versionContentMatches) { + if (dbPath.exists()) { + deleteDirectory(dbPath); + } + } + return versionContentMatches; + } + + private static String getVersionFileContent(Path versionFilePath) throws IOException { + if (versionFilePath.toFile().exists()) { + return StringCodec.get().fromPersistedFormat(Files.readAllBytes(versionFilePath)); + } + return null; + } + + private static void writeVersionFileContent(Path versionFilePath) throws IOException { + Files.write(versionFilePath, StringCodec.get().toPersistedFormat(VERSION_NUMBER)); + } + + @Override + public Table getSnapshotDiffJobTable() { + return snapshotDiffJobTable; + } + + @Override + public Table getSnapshotDiffReportTable() { + return snapshotDiffReportTable; + } + + @Override + public Table getSnapshotDiffPurgedJobTable() { + return snapshotDiffPurgedJobTable; + } + + @Override + public Table getSnapshotDiffFromSnapshotObjectInfoTable() { + return snapshotDiffFromSnapshotObjectInfoTable; + } + + @Override + public Table getSnapshotDiffToSnapshotObjectInfoTable() { + return snapshotDiffToSnapshotObjectInfoTable; + } + + @Override + public Table getSnapshotDiffUniqueObjectIdsTable() { + return snapshotDiffUniqueObjectIdsTable; + } + + @Override + public void close() throws Exception { + dbStore.close(); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/package-info.java new file mode 100644 index 000000000000..3dc2ab8285f5 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains classes for managing snapshot diff db. + */ +package org.apache.hadoop.ozone.om.snapshot.db; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java new file mode 100644 index 000000000000..7d2e3e64a8b5 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.diff.helper; + +import org.apache.hadoop.hdds.protocol.proto.OmServerProtocolProtos.SnapDiffObjectInfo; +import org.apache.hadoop.hdds.utils.db.Codec; +import org.apache.hadoop.hdds.utils.db.DelegatedCodec; +import org.apache.hadoop.hdds.utils.db.Proto2Codec; + +/** + * Represents information about an object in a snapshot difference. + * This class provides methods for converting between its internal + * representation and a Protobuf-based representation using a custom + * serialization codec. + */ +public class SnapshotDiffObjectInfo { + + private static final Codec CODEC = + new DelegatedCodec<>(Proto2Codec.get(SnapDiffObjectInfo.getDefaultInstance()), + SnapshotDiffObjectInfo::getFromProtobuf, + SnapshotDiffObjectInfo::toProtobuf, + SnapshotDiffObjectInfo.class); + + private long objectId; + private String key; + + public SnapshotDiffObjectInfo(long objectId, String key) { + this.objectId = objectId; + this.key = key; + } + + public static Codec getCodec() { + return CODEC; + } + + private static SnapshotDiffObjectInfo getFromProtobuf(SnapDiffObjectInfo objectInfo) { + return new SnapshotDiffObjectInfo(objectInfo.getObjectID(), objectInfo.getKeyName()); + } + + private SnapDiffObjectInfo toProtobuf() { + return SnapDiffObjectInfo.newBuilder() + .setObjectID(objectId) + .setKeyName(key) + .build(); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/package-info.java new file mode 100644 index 000000000000..7a96cecf8147 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Helper classes for computing the delta between two snapshots. + */ +package org.apache.hadoop.ozone.om.snapshot.diff.helper; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 6ff6f235a63a..5d681640249b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -30,13 +30,13 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT; import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER; -import static org.apache.hadoop.ozone.om.OmSnapshotManager.SNAP_DIFF_JOB_TABLE_NAME; -import static org.apache.hadoop.ozone.om.OmSnapshotManager.SNAP_DIFF_REPORT_TABLE_NAME; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.LEGACY; import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.getTableKey; +import static org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_JOB_TABLE_NAME; +import static org.apache.hadoop.ozone.om.snapshot.db.SnapshotDiffDBDefinition.SNAP_DIFF_REPORT_TABLE_NAME; import static org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_ALREADY_CANCELLED_JOB; import static org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_ALREADY_DONE_JOB; import static org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_ALREADY_FAILED_JOB; From 5e3b2b6c5bb64b2809f5dba5bd9d8425a8ae2ffa Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 12 Dec 2025 02:53:05 -0500 Subject: [PATCH 2/2] HDDS-14037. Move SnapshotDiffObjectInfo proto to interface storage Change-Id: I385d44a6390218b8ff67b9cbdc5908222118ad78 --- .../src/main/proto/OmServerProtocol.proto | 37 ------------------- .../src/main/proto/OmStorageProtocol.proto | 5 +++ .../diff/helper/SnapshotDiffObjectInfo.java | 2 +- 3 files changed, 6 insertions(+), 38 deletions(-) delete mode 100644 hadoop-hdds/interface-server/src/main/proto/OmServerProtocol.proto diff --git a/hadoop-hdds/interface-server/src/main/proto/OmServerProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/OmServerProtocol.proto deleted file mode 100644 index 54a3dc665bbe..000000000000 --- a/hadoop-hdds/interface-server/src/main/proto/OmServerProtocol.proto +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * These .proto interfaces are private and unstable. - * Please see http://wiki.apache.org/hadoop/Compatibility - * for what changes are allowed for a *unstable* .proto interface. - */ -syntax = "proto2"; -option java_package = "org.apache.hadoop.hdds.protocol.proto"; -option java_outer_classname = "OmServerProtocolProtos"; -option java_generic_services = true; -option java_generate_equals_and_hash = true; -package hadoop.hdds.block; - -import "hdds.proto"; - -// OM Server protocol -message SnapDiffObjectInfo { - optional uint64 objectID = 1; - optional string keyName = 2; -} \ No newline at end of file diff --git a/hadoop-ozone/interface-storage/src/main/proto/OmStorageProtocol.proto b/hadoop-ozone/interface-storage/src/main/proto/OmStorageProtocol.proto index ee57233f6de1..2381e18d5390 100644 --- a/hadoop-ozone/interface-storage/src/main/proto/OmStorageProtocol.proto +++ b/hadoop-ozone/interface-storage/src/main/proto/OmStorageProtocol.proto @@ -73,4 +73,9 @@ message FileSizeCountKeyProto { required string volume = 1; required string bucket = 2; required int64 fileSizeUpperBound = 3; +} + +message SnapDiffObjectInfo { + optional uint64 objectID = 1; + optional string keyName = 2; } \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java index 7d2e3e64a8b5..25c0f6d15268 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java @@ -17,10 +17,10 @@ package org.apache.hadoop.ozone.om.snapshot.diff.helper; -import org.apache.hadoop.hdds.protocol.proto.OmServerProtocolProtos.SnapDiffObjectInfo; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.DelegatedCodec; import org.apache.hadoop.hdds.utils.db.Proto2Codec; +import org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.SnapDiffObjectInfo; /** * Represents information about an object in a snapshot difference.