Skip to content

HDDS-13861. Intermittent failure in testOverWriteKeyWithAndWithOutVersioning [2/2] #9261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
peterxcli merged 6 commits into from
Nov 7, 2025
Merged

HDDS-13861. Intermittent failure in testOverWriteKeyWithAndWithOutVersioning [2/2] #9261

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b77698e
Revert "HDDS-13861. Intermittent failure in testOverWriteKeyWithAndWi…
peterxcli Nov 7, 2025
5558385
Suspend KeyDeletingService during versioning test to prevent prematur…
peterxcli Nov 7, 2025
0c5fe24
Ensure double buffer flush for deleted table after suspending KeyDele…
peterxcli Nov 7, 2025
8d941f7
code cleanup
peterxcli Nov 7, 2025
995adbd
address comments
peterxcli Nov 7, 2025
fa27790
fix pmd
peterxcli Nov 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 16 additions & 35 deletions ...ntegration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,8 @@ abstract class OzoneRpcClientTests extends OzoneTestBase {
private static MessageDigest eTagProvider;
private static Set<OzoneClient> ozoneClients = new HashSet<>();
private static GenericTestUtils.PrintStreamCapturer output;
private static final BucketLayout VERSIONING_TEST_BUCKET_LAYOUT =
BucketLayout.OBJECT_STORE;

@BeforeAll
public static void initialize() throws NoSuchAlgorithmException, UnsupportedEncodingException {
Expand Down Expand Up @@ -4452,12 +4454,12 @@ public void testKeyReadWriteForGDPR() throws Exception {
}
//Step 4
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(getBucketLayout())
OmKeyInfo omKeyInfo = omMetadataManager.getKeyTable(BucketLayout.OBJECT_STORE)
.get(omMetadataManager.getOzoneKey(volumeName, bucketName, keyName));

omKeyInfo.getMetadata().remove(OzoneConsts.GDPR_FLAG);

omMetadataManager.getKeyTable(getBucketLayout())
omMetadataManager.getKeyTable(BucketLayout.OBJECT_STORE)
.put(omMetadataManager.getOzoneKey(volumeName, bucketName, keyName),
omKeyInfo);

Expand Down Expand Up @@ -4605,10 +4607,6 @@ public void testHeadObject() throws IOException {

}

private BucketLayout getBucketLayout() {
return BucketLayout.DEFAULT;
}

private void createRequiredForVersioningTest(String volumeName,
String bucketName, String keyName, boolean versioning) throws Exception {

Expand All @@ -4624,7 +4622,7 @@ private void createRequiredForVersioningTest(String volumeName,
// information. This is easier to do with object store keys.
volume.createBucket(bucketName, BucketArgs.newBuilder()
.setVersioning(versioning)
.setBucketLayout(BucketLayout.OBJECT_STORE).build());
.setBucketLayout(VERSIONING_TEST_BUCKET_LAYOUT).build());
OzoneBucket bucket = volume.getBucket(bucketName);

TestDataUtil.createKey(bucket, keyName,
Expand All @@ -4637,52 +4635,35 @@ private void createRequiredForVersioningTest(String volumeName,

private void checkExceptedResultForVersioningTest(String volumeName,
String bucketName, String keyName, int expectedCount) throws Exception {
// Get actual bucket layout from the bucket itself to avoid mismatch
String bucketKey = cluster.getOzoneManager().getMetadataManager()
.getBucketKey(volumeName, bucketName);
BucketLayout actualLayout = cluster.getOzoneManager().getMetadataManager()
.getBucketTable().get(bucketKey).getBucketLayout();

OmKeyInfo omKeyInfo = cluster.getOzoneManager().getMetadataManager()
.getKeyTable(actualLayout).get(
cluster.getOzoneManager().getMetadataManager()
.getOzoneKey(volumeName, bucketName, keyName));
OMMetadataManager metadataManager = cluster.getOzoneManager().getMetadataManager();
String ozoneKey = metadataManager.getOzoneKey(volumeName, bucketName, keyName);

OmKeyInfo omKeyInfo = metadataManager.getKeyTable(VERSIONING_TEST_BUCKET_LAYOUT).get(ozoneKey);

assertNotNull(omKeyInfo);
assertEquals(expectedCount,
omKeyInfo.getKeyLocationVersions().size());
assertEquals(expectedCount, omKeyInfo.getKeyLocationVersions().size());

// Suspend KeyDeletingService to prevent it from purging entries from deleted table
cluster.getOzoneManager().getKeyManager().getDeletingService().suspend();
// ensure flush double buffer for deleted Table
cluster.getOzoneManager().awaitDoubleBufferFlush();

if (expectedCount == 1) {
// Wait for deleted key to become visible in deletedTable
String ozoneKey = cluster.getOzoneManager().getMetadataManager()
.getOzoneKey(volumeName, bucketName, keyName);
GenericTestUtils.waitFor((CheckedSupplier<Boolean, IOException>) () -> {
List<? extends Table.KeyValue<String, RepeatedOmKeyInfo>> rangeKVs
= cluster.getOzoneManager().getMetadataManager().getDeletedTable()
.getRangeKVs(null, 100, ozoneKey);
return !rangeKVs.isEmpty();
}, 100, 60000);

List<? extends Table.KeyValue<String, RepeatedOmKeyInfo>> rangeKVs
= cluster.getOzoneManager().getMetadataManager().getDeletedTable()
.getRangeKVs(null, 100, ozoneKey);
= metadataManager.getDeletedTable().getRangeKVs(null, 100, ozoneKey);

assertThat(rangeKVs).isNotEmpty();
assertEquals(expectedCount,
rangeKVs.get(0).getValue().getOmKeyInfoList().size());
} else {
// If expectedCount is greater than 1 means versioning enabled,
// so delete table should be empty.
RepeatedOmKeyInfo repeatedOmKeyInfo = cluster
.getOzoneManager().getMetadataManager()
.getDeletedTable().get(cluster.getOzoneManager().getMetadataManager()
.getOzoneKey(volumeName, bucketName, keyName));
RepeatedOmKeyInfo repeatedOmKeyInfo =
metadataManager.getDeletedTable().get(ozoneKey);

assertNull(repeatedOmKeyInfo);
}
cluster.getOzoneManager().getKeyManager().getDeletingService().resume();
}

@Test
Expand Down