From b1bf0aa5787741975df161ad4d4b357a085ce9f6 Mon Sep 17 00:00:00 2001 From: Jyotirmoy Sinha Date: Wed, 13 Sep 2023 10:31:48 +0530 Subject: [PATCH 1/4] HDDS-9269. [snapshot] Add unit-testcase for Snapshot Quota handling --- .../hadoop/ozone/om/TestOmSnapshot.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java index dea28e03d818..5a826826614e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.client.OzoneQuota; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; @@ -1786,6 +1787,39 @@ public void testDeleteSnapshotMissingMandatoryParams() throws Exception { () -> store.deleteSnapshot(nullstr, bucket, snap1)); } + @Test + public void testSnapshotQuotaHandling() throws Exception { + String volume = "vol-" + counter.incrementAndGet(); + String bucket = "buck-" + counter.incrementAndGet(); + store.createVolume(volume); + OzoneVolume volume1 = store.getVolume(volume); + volume1.createBucket(bucket); + OzoneBucket bucket1 = volume1.getBucket(bucket); + bucket1.setQuota(OzoneQuota.parseQuota("102400000", "500")); + volume1.setQuota(OzoneQuota.parseQuota("204800000", "1000")); + // Create Key1 and take snapshot + String key1 = "key-1-"; + createFileKeyWithPrefix(bucket1, key1); + + long volNameQuotaBefore = volume1.getQuotaInNamespace(); + long volSpaceQuotaBefore = volume1.getQuotaInBytes(); + long buckNameQuotaBefore = bucket1.getQuotaInNamespace(); + long buckSpaceQuotaBefore = bucket1.getQuotaInBytes(); + + String snap1 = "snap" + counter.incrementAndGet(); + createSnapshot(volume, bucket, snap1); + + long volNameQuotaAfter = volume1.getQuotaInNamespace(); + long volSpaceQuotaAfter = volume1.getQuotaInBytes(); + long buckNameQuotaAfter = bucket1.getQuotaInNamespace(); + long buckSpaceQuotaAfter = bucket1.getQuotaInBytes(); + + assertEquals(volNameQuotaBefore, volNameQuotaAfter); + assertEquals(volSpaceQuotaBefore, volSpaceQuotaAfter); + assertEquals(buckNameQuotaBefore, buckNameQuotaAfter); + assertEquals(buckSpaceQuotaBefore, buckSpaceQuotaAfter); + } + @NotNull private static List getKeyTableSstFiles() throws IOException { From c573540ce026f2c29aa421cd60caea2015f66533 Mon Sep 17 00:00:00 2001 From: Jyotirmoy Sinha Date: Wed, 13 Sep 2023 12:00:18 +0530 Subject: [PATCH 2/4] Add delete snapshot scenario --- .../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java index 5a826826614e..4a2042e6498c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java @@ -1818,6 +1818,18 @@ public void testSnapshotQuotaHandling() throws Exception { assertEquals(volSpaceQuotaBefore, volSpaceQuotaAfter); assertEquals(buckNameQuotaBefore, buckNameQuotaAfter); assertEquals(buckSpaceQuotaBefore, buckSpaceQuotaAfter); + + store.deleteSnapshot(volume, bucket, snap1); + + long volNameQuotaFinal = volume1.getQuotaInNamespace(); + long volSpaceQuotaFinal = volume1.getQuotaInBytes(); + long buckNameQuotaFinal = bucket1.getQuotaInNamespace(); + long buckSpaceQuotaFinal = bucket1.getQuotaInBytes(); + + assertEquals(volNameQuotaBefore, volNameQuotaFinal); + assertEquals(volSpaceQuotaBefore, volSpaceQuotaFinal); + assertEquals(buckNameQuotaBefore, buckNameQuotaFinal); + assertEquals(buckSpaceQuotaBefore, buckSpaceQuotaFinal); } @NotNull From b3888d8ada8a3adb9b941967e4979404570e9f25 Mon Sep 17 00:00:00 2001 From: Jyotirmoy Sinha Date: Wed, 13 Sep 2023 12:30:03 +0530 Subject: [PATCH 3/4] Added checks for used namespaces and quota for volume/bucket --- .../hadoop/ozone/om/TestOmSnapshot.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java index 4a2042e6498c..7a7f0351d41c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java @@ -1801,33 +1801,28 @@ public void testSnapshotQuotaHandling() throws Exception { String key1 = "key-1-"; createFileKeyWithPrefix(bucket1, key1); - long volNameQuotaBefore = volume1.getQuotaInNamespace(); - long volSpaceQuotaBefore = volume1.getQuotaInBytes(); - long buckNameQuotaBefore = bucket1.getQuotaInNamespace(); - long buckSpaceQuotaBefore = bucket1.getQuotaInBytes(); + long volNameQuotaBefore = volume1.getUsedNamespace(); + long buckNameQuotaBefore = bucket1.getUsedNamespace(); + long buckSpaceQuotaBefore = bucket1.getUsedBytes(); String snap1 = "snap" + counter.incrementAndGet(); createSnapshot(volume, bucket, snap1); - long volNameQuotaAfter = volume1.getQuotaInNamespace(); - long volSpaceQuotaAfter = volume1.getQuotaInBytes(); - long buckNameQuotaAfter = bucket1.getQuotaInNamespace(); - long buckSpaceQuotaAfter = bucket1.getQuotaInBytes(); + long volNameQuotaAfter = volume1.getUsedNamespace(); + long buckNameQuotaAfter = bucket1.getUsedNamespace(); + long buckSpaceQuotaAfter = bucket1.getUsedBytes(); assertEquals(volNameQuotaBefore, volNameQuotaAfter); - assertEquals(volSpaceQuotaBefore, volSpaceQuotaAfter); assertEquals(buckNameQuotaBefore, buckNameQuotaAfter); assertEquals(buckSpaceQuotaBefore, buckSpaceQuotaAfter); store.deleteSnapshot(volume, bucket, snap1); - long volNameQuotaFinal = volume1.getQuotaInNamespace(); - long volSpaceQuotaFinal = volume1.getQuotaInBytes(); - long buckNameQuotaFinal = bucket1.getQuotaInNamespace(); - long buckSpaceQuotaFinal = bucket1.getQuotaInBytes(); + long volNameQuotaFinal = volume1.getUsedNamespace(); + long buckNameQuotaFinal = bucket1.getUsedNamespace(); + long buckSpaceQuotaFinal = bucket1.getUsedBytes(); assertEquals(volNameQuotaBefore, volNameQuotaFinal); - assertEquals(volSpaceQuotaBefore, volSpaceQuotaFinal); assertEquals(buckNameQuotaBefore, buckNameQuotaFinal); assertEquals(buckSpaceQuotaBefore, buckSpaceQuotaFinal); } From 8d1ee8fe1d7e450418289907a5cbfec78b316f96 Mon Sep 17 00:00:00 2001 From: Jyotirmoy Sinha Date: Wed, 13 Sep 2023 15:06:24 +0530 Subject: [PATCH 4/4] Change variable names and add delete scenario --- .../hadoop/ozone/om/TestOmSnapshot.java | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java index 7a7f0351d41c..d61789495cf7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java @@ -1797,34 +1797,44 @@ public void testSnapshotQuotaHandling() throws Exception { OzoneBucket bucket1 = volume1.getBucket(bucket); bucket1.setQuota(OzoneQuota.parseQuota("102400000", "500")); volume1.setQuota(OzoneQuota.parseQuota("204800000", "1000")); - // Create Key1 and take snapshot + + long volUsedNamespaceInitial = volume1.getUsedNamespace(); + long buckUsedNamspaceInitial = bucket1.getUsedNamespace(); + long buckUsedBytesIntial = bucket1.getUsedBytes(); + String key1 = "key-1-"; - createFileKeyWithPrefix(bucket1, key1); + key1 = createFileKeyWithPrefix(bucket1, key1); - long volNameQuotaBefore = volume1.getUsedNamespace(); - long buckNameQuotaBefore = bucket1.getUsedNamespace(); - long buckSpaceQuotaBefore = bucket1.getUsedBytes(); + long volUsedNamespaceBefore = volume1.getUsedNamespace(); + long buckUsedNamspaceBefore = bucket1.getUsedNamespace(); + long buckUsedBytesBefore = bucket1.getUsedBytes(); String snap1 = "snap" + counter.incrementAndGet(); createSnapshot(volume, bucket, snap1); - long volNameQuotaAfter = volume1.getUsedNamespace(); - long buckNameQuotaAfter = bucket1.getUsedNamespace(); - long buckSpaceQuotaAfter = bucket1.getUsedBytes(); + long volUsedNamespaceAfter = volume1.getUsedNamespace(); + long buckUsedNamespaceAfter = bucket1.getUsedNamespace(); + long buckUsedBytesAfter = bucket1.getUsedBytes(); - assertEquals(volNameQuotaBefore, volNameQuotaAfter); - assertEquals(buckNameQuotaBefore, buckNameQuotaAfter); - assertEquals(buckSpaceQuotaBefore, buckSpaceQuotaAfter); + assertEquals(volUsedNamespaceBefore, volUsedNamespaceAfter); + assertEquals(buckUsedNamspaceBefore, buckUsedNamespaceAfter); + assertEquals(buckUsedBytesBefore, buckUsedBytesAfter); store.deleteSnapshot(volume, bucket, snap1); - long volNameQuotaFinal = volume1.getUsedNamespace(); - long buckNameQuotaFinal = bucket1.getUsedNamespace(); - long buckSpaceQuotaFinal = bucket1.getUsedBytes(); + long volUsedNamespaceFinal = volume1.getUsedNamespace(); + long buckUsedNamespaceFinal = bucket1.getUsedNamespace(); + long buckUsedBytesFinal = bucket1.getUsedBytes(); + + assertEquals(volUsedNamespaceBefore, volUsedNamespaceFinal); + assertEquals(buckUsedNamspaceBefore, buckUsedNamespaceFinal); + assertEquals(buckUsedBytesBefore, buckUsedBytesFinal); + + bucket1.deleteKey(key1); - assertEquals(volNameQuotaBefore, volNameQuotaFinal); - assertEquals(buckNameQuotaBefore, buckNameQuotaFinal); - assertEquals(buckSpaceQuotaBefore, buckSpaceQuotaFinal); + assertEquals(volUsedNamespaceInitial, volume1.getUsedNamespace()); + assertEquals(buckUsedNamspaceInitial, bucket1.getUsedNamespace()); + assertEquals(buckUsedBytesIntial, bucket1.getUsedBytes()); } @NotNull