Skip to content

HDDS-13765. SnapshotLocalData yaml should also track snapshotId #9123

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
swamirishi merged 1 commit into from
Oct 8, 2025
Merged

HDDS-13765. SnapshotLocalData yaml should also track snapshotId #9123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ public final class OzoneConsts {
public static final String OM_SLD_LAST_DEFRAG_TIME = "lastDefragTime";
public static final String OM_SLD_NEEDS_DEFRAG = "needsDefrag";
public static final String OM_SLD_VERSION_SST_FILE_INFO = "versionSstFileInfos";
public static final String OM_SLD_SNAP_ID = "snapshotId";
public static final String OM_SLD_PREV_SNAP_ID = "previousSnapshotId";
public static final String OM_SLD_VERSION_META_SST_FILES = "sstFiles";
public static final String OM_SLD_VERSION_META_PREV_SNAP_VERSION = "previousSnapshotVersion";
Expand Down
11 changes: 10 additions & 1 deletion hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@
*/
public abstract class OmSnapshotLocalData {

// Unique identifier for the snapshot. This is used to identify the snapshot.
private UUID snapshotId;

// Version of the snapshot local data. 0 indicates not defragged snapshot.
// defragged snapshots will have version > 0.
private int version;
Expand Down Expand Up @@ -70,7 +73,8 @@ public abstract class OmSnapshotLocalData {
/**
* Creates a OmSnapshotLocalData object with default values.
*/
public OmSnapshotLocalData(List<LiveFileMetaData> notDefraggedSSTFileList, UUID previousSnapshotId) {
public OmSnapshotLocalData(UUID snapshotId, List<LiveFileMetaData> notDefraggedSSTFileList, UUID previousSnapshotId) {
this.snapshotId = snapshotId;
this.isSSTFiltered = false;
this.lastDefragTime = 0L;
this.needsDefrag = false;
Expand All @@ -93,6 +97,7 @@ public OmSnapshotLocalData(OmSnapshotLocalData source) {
this.needsDefrag = source.needsDefrag;
this.checksum = source.checksum;
this.version = source.version;
this.snapshotId = source.snapshotId;
this.previousSnapshotId = source.previousSnapshotId;
this.versionSstFileInfos = new LinkedHashMap<>();
setVersionSstFileInfos(source.versionSstFileInfos);
Expand Down Expand Up @@ -167,6 +172,10 @@ public UUID getPreviousSnapshotId() {
return previousSnapshotId;
}

public UUID getSnapshotId() {
return snapshotId;
}

public void setPreviousSnapshotId(UUID previousSnapshotId) {
this.previousSnapshotId = previousSnapshotId;
}
Expand Down
8 changes: 5 additions & 3 deletions ...ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ public final class OmSnapshotLocalDataYaml extends OmSnapshotLocalData {
/**
* Creates a new OmSnapshotLocalDataYaml with default values.
*/
public OmSnapshotLocalDataYaml(List<LiveFileMetaData> liveFileMetaDatas, UUID previousSnapshotId) {
super(liveFileMetaDatas, previousSnapshotId);
public OmSnapshotLocalDataYaml(UUID snapshotId, List<LiveFileMetaData> liveFileMetaDatas, UUID previousSnapshotId) {
super(snapshotId, liveFileMetaDatas, previousSnapshotId);
}

/**
Expand Down Expand Up @@ -227,8 +227,10 @@ private final class ConstructSnapshotLocalData extends AbstractConstruct {
public Object construct(Node node) {
MappingNode mnode = (MappingNode) node;
Map<Object, Object> nodes = constructMapping(mnode);
UUID snapId = UUID.fromString((String) nodes.get(OzoneConsts.OM_SLD_SNAP_ID));
UUID prevSnapId = UUID.fromString((String) nodes.get(OzoneConsts.OM_SLD_PREV_SNAP_ID));
OmSnapshotLocalDataYaml snapshotLocalData = new OmSnapshotLocalDataYaml(Collections.emptyList(), prevSnapId);
OmSnapshotLocalDataYaml snapshotLocalData = new OmSnapshotLocalDataYaml(snapId, Collections.emptyList(),
prevSnapId);

// Set version from YAML
Integer version = (Integer) nodes.get(OzoneConsts.OM_SLD_VERSION);
Expand Down
4 changes: 2 additions & 2 deletions hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -645,8 +645,8 @@ public static void createNewOmSnapshotLocalDataFile(OmSnapshotManager snapshotMa
SnapshotInfo snapshotInfo) throws IOException {
Path snapshotLocalDataPath = Paths.get(getSnapshotLocalPropertyYamlPath(snapshotStore.getDbLocation().toPath()));
Files.deleteIfExists(snapshotLocalDataPath);
OmSnapshotLocalDataYaml snapshotLocalDataYaml = new OmSnapshotLocalDataYaml(getSnapshotSSTFileList(snapshotStore),
snapshotInfo.getPathPreviousSnapshotId());
OmSnapshotLocalDataYaml snapshotLocalDataYaml = new OmSnapshotLocalDataYaml(snapshotInfo.getSnapshotId(),
getSnapshotSSTFileList(snapshotStore), snapshotInfo.getPathPreviousSnapshotId());
snapshotLocalDataYaml.writeToYaml(snapshotManager, snapshotLocalDataPath.toFile());
}

Expand Down
20 changes: 14 additions & 6 deletions ...e/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,15 +104,16 @@ private LiveFileMetaData createLiveFileMetaData(String fileName, String table, S
/**
* Creates a snapshot local data YAML file.
*/
private Pair<File, UUID> writeToYaml(String snapshotName) throws IOException {
private Pair<File, UUID> writeToYaml(UUID snapshotId, String snapshotName) throws IOException {
String yamlFilePath = snapshotName + ".yaml";
UUID previousSnapshotId = UUID.randomUUID();
// Create snapshot data with not defragged SST files
List<LiveFileMetaData> notDefraggedSSTFileList = asList(
createLiveFileMetaData("sst1", "table1", "k1", "k2"),
createLiveFileMetaData("sst2", "table1", "k3", "k4"),
createLiveFileMetaData("sst3", "table2", "k4", "k5"));
OmSnapshotLocalDataYaml dataYaml = new OmSnapshotLocalDataYaml(notDefraggedSSTFileList, previousSnapshotId);
OmSnapshotLocalDataYaml dataYaml = new OmSnapshotLocalDataYaml(snapshotId, notDefraggedSSTFileList,
previousSnapshotId);

// Set version
dataYaml.setVersion(42);
Expand Down Expand Up @@ -146,7 +147,8 @@ private Pair<File, UUID> writeToYaml(String snapshotName) throws IOException {

@Test
public void testWriteToYaml() throws IOException {
Pair<File, UUID> yamlFilePrevIdPair = writeToYaml("snapshot1");
UUID snapshotId = UUID.randomUUID();
Pair<File, UUID> yamlFilePrevIdPair = writeToYaml(snapshotId, "snapshot1");
File yamlFile = yamlFilePrevIdPair.getLeft();
UUID prevSnapId = yamlFilePrevIdPair.getRight();

Expand All @@ -172,6 +174,7 @@ public void testWriteToYaml() throws IOException {
assertEquals(2, defraggedSSTFiles.get(43).getSstFiles().size());
assertEquals(1, defraggedSSTFiles.get(44).getSstFiles().size());
assertEquals(prevSnapId, snapshotData.getPreviousSnapshotId());
assertEquals(snapshotId, snapshotData.getSnapshotId());
assertEquals(ImmutableMap.of(
0, new VersionMeta(0,
ImmutableList.of(new SstFileInfo("sst1", "k1", "k2", "table1"),
Expand All @@ -186,7 +189,8 @@ public void testWriteToYaml() throws IOException {

@Test
public void testUpdateSnapshotDataFile() throws IOException {
Pair<File, UUID> yamlFilePrevIdPair = writeToYaml("snapshot2");
UUID snapshotId = UUID.randomUUID();
Pair<File, UUID> yamlFilePrevIdPair = writeToYaml(snapshotId, "snapshot2");
File yamlFile = yamlFilePrevIdPair.getLeft();
// Read from YAML file
OmSnapshotLocalDataYaml dataYaml =
Expand Down Expand Up @@ -228,7 +232,8 @@ public void testEmptyFile() throws IOException {

@Test
public void testChecksum() throws IOException {
Pair<File, UUID> yamlFilePrevIdPair = writeToYaml("snapshot3");
UUID snapshotId = UUID.randomUUID();
Pair<File, UUID> yamlFilePrevIdPair = writeToYaml(snapshotId, "snapshot3");
File yamlFile = yamlFilePrevIdPair.getLeft();
// Read from YAML file
OmSnapshotLocalDataYaml snapshotData = OmSnapshotLocalDataYaml.getFromYamlFile(omSnapshotManager, yamlFile);
Expand All @@ -244,7 +249,8 @@ public void testChecksum() throws IOException {

@Test
public void testYamlContainsAllFields() throws IOException {
Pair<File, UUID> yamlFilePrevIdPair = writeToYaml("snapshot4");
UUID snapshotId = UUID.randomUUID();
Pair<File, UUID> yamlFilePrevIdPair = writeToYaml(snapshotId, "snapshot4");
File yamlFile = yamlFilePrevIdPair.getLeft();
String content = FileUtils.readFileToString(yamlFile, Charset.defaultCharset());

Expand All @@ -255,5 +261,7 @@ public void testYamlContainsAllFields() throws IOException {
assertThat(content).contains(OzoneConsts.OM_SLD_LAST_DEFRAG_TIME);
assertThat(content).contains(OzoneConsts.OM_SLD_NEEDS_DEFRAG);
assertThat(content).contains(OzoneConsts.OM_SLD_VERSION_SST_FILE_INFO);
assertThat(content).contains(OzoneConsts.OM_SLD_SNAP_ID);
assertThat(content).contains(OzoneConsts.OM_SLD_PREV_SNAP_ID);
}
}