From 6e241f65e1d1d2469f5e46b2cde7ecce44a6e0c0 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 8 Oct 2025 09:39:47 -0400 Subject: [PATCH] HDDS-13765. SnapshotLocalData yaml should also track snapshotId Change-Id: Iba47aeb21663dfa407ab71339cef02c0d74b49f2 --- .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../hadoop/ozone/om/OmSnapshotLocalData.java | 11 +++++++++- .../ozone/om/OmSnapshotLocalDataYaml.java | 8 +++++--- .../hadoop/ozone/om/OmSnapshotManager.java | 4 ++-- .../ozone/om/TestOmSnapshotLocalDataYaml.java | 20 +++++++++++++------ 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index bb6eef205e44..c9064da1781c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -213,6 +213,7 @@ public final class OzoneConsts { public static final String OM_SLD_LAST_DEFRAG_TIME = "lastDefragTime"; public static final String OM_SLD_NEEDS_DEFRAG = "needsDefrag"; public static final String OM_SLD_VERSION_SST_FILE_INFO = "versionSstFileInfos"; + public static final String OM_SLD_SNAP_ID = "snapshotId"; public static final String OM_SLD_PREV_SNAP_ID = "previousSnapshotId"; public static final String OM_SLD_VERSION_META_SST_FILES = "sstFiles"; public static final String OM_SLD_VERSION_META_PREV_SNAP_VERSION = "previousSnapshotVersion"; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java index 7a351ba5c337..5f65fd4c0d08 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java @@ -41,6 +41,9 @@ */ public abstract class OmSnapshotLocalData { + // Unique identifier for the snapshot. This is used to identify the snapshot. + private UUID snapshotId; + // Version of the snapshot local data. 0 indicates not defragged snapshot. // defragged snapshots will have version > 0. private int version; @@ -70,7 +73,8 @@ public abstract class OmSnapshotLocalData { /** * Creates a OmSnapshotLocalData object with default values. */ - public OmSnapshotLocalData(List notDefraggedSSTFileList, UUID previousSnapshotId) { + public OmSnapshotLocalData(UUID snapshotId, List notDefraggedSSTFileList, UUID previousSnapshotId) { + this.snapshotId = snapshotId; this.isSSTFiltered = false; this.lastDefragTime = 0L; this.needsDefrag = false; @@ -93,6 +97,7 @@ public OmSnapshotLocalData(OmSnapshotLocalData source) { this.needsDefrag = source.needsDefrag; this.checksum = source.checksum; this.version = source.version; + this.snapshotId = source.snapshotId; this.previousSnapshotId = source.previousSnapshotId; this.versionSstFileInfos = new LinkedHashMap<>(); setVersionSstFileInfos(source.versionSstFileInfos); @@ -167,6 +172,10 @@ public UUID getPreviousSnapshotId() { return previousSnapshotId; } + public UUID getSnapshotId() { + return snapshotId; + } + public void setPreviousSnapshotId(UUID previousSnapshotId) { this.previousSnapshotId = previousSnapshotId; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java index 3a80915e6eac..1d4fedfacaaf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java @@ -70,8 +70,8 @@ public final class OmSnapshotLocalDataYaml extends OmSnapshotLocalData { /** * Creates a new OmSnapshotLocalDataYaml with default values. */ - public OmSnapshotLocalDataYaml(List liveFileMetaDatas, UUID previousSnapshotId) { - super(liveFileMetaDatas, previousSnapshotId); + public OmSnapshotLocalDataYaml(UUID snapshotId, List liveFileMetaDatas, UUID previousSnapshotId) { + super(snapshotId, liveFileMetaDatas, previousSnapshotId); } /** @@ -227,8 +227,10 @@ private final class ConstructSnapshotLocalData extends AbstractConstruct { public Object construct(Node node) { MappingNode mnode = (MappingNode) node; Map nodes = constructMapping(mnode); + UUID snapId = UUID.fromString((String) nodes.get(OzoneConsts.OM_SLD_SNAP_ID)); UUID prevSnapId = UUID.fromString((String) nodes.get(OzoneConsts.OM_SLD_PREV_SNAP_ID)); - OmSnapshotLocalDataYaml snapshotLocalData = new OmSnapshotLocalDataYaml(Collections.emptyList(), prevSnapId); + OmSnapshotLocalDataYaml snapshotLocalData = new OmSnapshotLocalDataYaml(snapId, Collections.emptyList(), + prevSnapId); // Set version from YAML Integer version = (Integer) nodes.get(OzoneConsts.OM_SLD_VERSION); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index d86b1ce6473f..d531f95c46b6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -645,8 +645,8 @@ public static void createNewOmSnapshotLocalDataFile(OmSnapshotManager snapshotMa SnapshotInfo snapshotInfo) throws IOException { Path snapshotLocalDataPath = Paths.get(getSnapshotLocalPropertyYamlPath(snapshotStore.getDbLocation().toPath())); Files.deleteIfExists(snapshotLocalDataPath); - OmSnapshotLocalDataYaml snapshotLocalDataYaml = new OmSnapshotLocalDataYaml(getSnapshotSSTFileList(snapshotStore), - snapshotInfo.getPathPreviousSnapshotId()); + OmSnapshotLocalDataYaml snapshotLocalDataYaml = new OmSnapshotLocalDataYaml(snapshotInfo.getSnapshotId(), + getSnapshotSSTFileList(snapshotStore), snapshotInfo.getPathPreviousSnapshotId()); snapshotLocalDataYaml.writeToYaml(snapshotManager, snapshotLocalDataPath.toFile()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java index eda95dc7b31d..8b41e5072185 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java @@ -104,7 +104,7 @@ private LiveFileMetaData createLiveFileMetaData(String fileName, String table, S /** * Creates a snapshot local data YAML file. */ - private Pair writeToYaml(String snapshotName) throws IOException { + private Pair writeToYaml(UUID snapshotId, String snapshotName) throws IOException { String yamlFilePath = snapshotName + ".yaml"; UUID previousSnapshotId = UUID.randomUUID(); // Create snapshot data with not defragged SST files @@ -112,7 +112,8 @@ private Pair writeToYaml(String snapshotName) throws IOException { createLiveFileMetaData("sst1", "table1", "k1", "k2"), createLiveFileMetaData("sst2", "table1", "k3", "k4"), createLiveFileMetaData("sst3", "table2", "k4", "k5")); - OmSnapshotLocalDataYaml dataYaml = new OmSnapshotLocalDataYaml(notDefraggedSSTFileList, previousSnapshotId); + OmSnapshotLocalDataYaml dataYaml = new OmSnapshotLocalDataYaml(snapshotId, notDefraggedSSTFileList, + previousSnapshotId); // Set version dataYaml.setVersion(42); @@ -146,7 +147,8 @@ private Pair writeToYaml(String snapshotName) throws IOException { @Test public void testWriteToYaml() throws IOException { - Pair yamlFilePrevIdPair = writeToYaml("snapshot1"); + UUID snapshotId = UUID.randomUUID(); + Pair yamlFilePrevIdPair = writeToYaml(snapshotId, "snapshot1"); File yamlFile = yamlFilePrevIdPair.getLeft(); UUID prevSnapId = yamlFilePrevIdPair.getRight(); @@ -172,6 +174,7 @@ public void testWriteToYaml() throws IOException { assertEquals(2, defraggedSSTFiles.get(43).getSstFiles().size()); assertEquals(1, defraggedSSTFiles.get(44).getSstFiles().size()); assertEquals(prevSnapId, snapshotData.getPreviousSnapshotId()); + assertEquals(snapshotId, snapshotData.getSnapshotId()); assertEquals(ImmutableMap.of( 0, new VersionMeta(0, ImmutableList.of(new SstFileInfo("sst1", "k1", "k2", "table1"), @@ -186,7 +189,8 @@ public void testWriteToYaml() throws IOException { @Test public void testUpdateSnapshotDataFile() throws IOException { - Pair yamlFilePrevIdPair = writeToYaml("snapshot2"); + UUID snapshotId = UUID.randomUUID(); + Pair yamlFilePrevIdPair = writeToYaml(snapshotId, "snapshot2"); File yamlFile = yamlFilePrevIdPair.getLeft(); // Read from YAML file OmSnapshotLocalDataYaml dataYaml = @@ -228,7 +232,8 @@ public void testEmptyFile() throws IOException { @Test public void testChecksum() throws IOException { - Pair yamlFilePrevIdPair = writeToYaml("snapshot3"); + UUID snapshotId = UUID.randomUUID(); + Pair yamlFilePrevIdPair = writeToYaml(snapshotId, "snapshot3"); File yamlFile = yamlFilePrevIdPair.getLeft(); // Read from YAML file OmSnapshotLocalDataYaml snapshotData = OmSnapshotLocalDataYaml.getFromYamlFile(omSnapshotManager, yamlFile); @@ -244,7 +249,8 @@ public void testChecksum() throws IOException { @Test public void testYamlContainsAllFields() throws IOException { - Pair yamlFilePrevIdPair = writeToYaml("snapshot4"); + UUID snapshotId = UUID.randomUUID(); + Pair yamlFilePrevIdPair = writeToYaml(snapshotId, "snapshot4"); File yamlFile = yamlFilePrevIdPair.getLeft(); String content = FileUtils.readFileToString(yamlFile, Charset.defaultCharset()); @@ -255,5 +261,7 @@ public void testYamlContainsAllFields() throws IOException { assertThat(content).contains(OzoneConsts.OM_SLD_LAST_DEFRAG_TIME); assertThat(content).contains(OzoneConsts.OM_SLD_NEEDS_DEFRAG); assertThat(content).contains(OzoneConsts.OM_SLD_VERSION_SST_FILE_INFO); + assertThat(content).contains(OzoneConsts.OM_SLD_SNAP_ID); + assertThat(content).contains(OzoneConsts.OM_SLD_PREV_SNAP_ID); } }