From 0a901ba95b6b4f19ac32b8a3d2d4076a74d1fd2f Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 1 Apr 2020 16:16:40 +0530 Subject: [PATCH] =?UTF-8?q?HBASE-23678=20:=20Builder=20API=20for=20version?= =?UTF-8?q?=20management=20-=20setVersionsWithTim=E2=80=A6=20(#1381)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Xu Cang --- .../client/ColumnFamilyDescriptorBuilder.java | 23 +++ .../hbase/regionserver/TestMinVersions.java | 191 ++++++++++++------ 2 files changed, 152 insertions(+), 62 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java index 0b78c9d4d5f6..8e9ae47edd98 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ColumnFamilyDescriptorBuilder.java @@ -572,6 +572,12 @@ public ColumnFamilyDescriptorBuilder setValue(final String key, final String val return this; } + public ColumnFamilyDescriptorBuilder setVersionsWithTimeToLive(final int retentionInterval, + final int versionAfterInterval) { + desc.setVersionsWithTimeToLive(retentionInterval, versionAfterInterval); + return this; + } + /** * An ModifyableFamilyDescriptor contains information about a column family such as the * number of versions, compression settings, etc. @@ -942,6 +948,23 @@ public ModifyableColumnFamilyDescriptor setMinVersions(int minVersions) { return setValue(MIN_VERSIONS_BYTES, Integer.toString(minVersions)); } + /** + * Retain all versions for a given TTL(retentionInterval), and then only a specific number + * of versions(versionAfterInterval) after that interval elapses. + * + * @param retentionInterval Retain all versions for this interval + * @param versionAfterInterval Retain no of versions to retain after retentionInterval + * @return this (for chained invocation) + */ + public ModifyableColumnFamilyDescriptor setVersionsWithTimeToLive( + final int retentionInterval, final int versionAfterInterval) { + ModifyableColumnFamilyDescriptor modifyableColumnFamilyDescriptor = + setVersions(versionAfterInterval, Integer.MAX_VALUE); + modifyableColumnFamilyDescriptor.setTimeToLive(retentionInterval); + modifyableColumnFamilyDescriptor.setKeepDeletedCells(KeepDeletedCells.TTL); + return modifyableColumnFamilyDescriptor; + } + @Override public boolean isBlockCacheEnabled() { return getStringOrDefault(BLOCKCACHE_BYTES, Boolean::valueOf, DEFAULT_BLOCKCACHE); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java index 5482678f2e1e..a31a6be72b19 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java @@ -20,6 +20,8 @@ import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; + +import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hbase.Cell; @@ -528,80 +530,145 @@ public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception { int ttl = 4; ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(c0) - .setMinVersions(2).setMaxVersions(Integer.MAX_VALUE).setTimeToLive(ttl). - setKeepDeletedCells(KeepDeletedCells.TTL).build(); + .setVersionsWithTimeToLive(ttl, 2).build(); + verifyVersionedCellKeyValues(ttl, cfd); + + cfd = ColumnFamilyDescriptorBuilder.newBuilder(c0) + .setMinVersions(2) + .setMaxVersions(Integer.MAX_VALUE) + .setTimeToLive(ttl) + .setKeepDeletedCells(KeepDeletedCells.TTL) + .build(); + verifyVersionedCellKeyValues(ttl, cfd); + } + private void verifyVersionedCellKeyValues(int ttl, ColumnFamilyDescriptor cfd) + throws IOException { TableDescriptor htd = TableDescriptorBuilder. newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build(); HRegion region = hbu.createLocalHRegion(htd, null, null); - long startTS = EnvironmentEdgeManager.currentTime(); - ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge(); - injectEdge.setValue(startTS); - EnvironmentEdgeManager.injectEdge(injectEdge); + try { + long startTS = EnvironmentEdgeManager.currentTime(); + ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge(); + injectEdge.setValue(startTS); + EnvironmentEdgeManager.injectEdge(injectEdge); + + long ts = startTS - 2000; + putFourVersions(region, ts); + + Get get; + Result result; + + //check we can still see all versions before compaction + get = new Get(T1); + get.readAllVersions(); + get.setTimeRange(0, ts); + result = region.get(get); + checkResult(result, c0, T4, T3, T2, T1); + + region.flush(true); + region.compact(true); + Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime()); + long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 4; + Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000)); + //check that nothing was purged yet + verifyBeforeCompaction(region, ts); + + injectEdge.incValue(ttl * 1000); + + region.flush(true); + region.compact(true); + verifyAfterTtl(region, ts); + } finally { + HBaseTestingUtility.closeRegionAndWAL(region); + } + } + + private void verifyAfterTtl(HRegion region, long ts) throws IOException { + Get get; + Result result; + //check that after compaction (which is after TTL) that only T1 && T2 were purged + get = new Get(T1); + get.readAllVersions(); + get.setTimeRange(0, ts); + result = region.get(get); + checkResult(result, c0, T4, T3); + + get = new Get(T1); + get.readAllVersions(); + get.setTimeRange(0, ts - 1); + result = region.get(get); + checkResult(result, c0, T3); + + get = new Get(T1); + get.readAllVersions(); + get.setTimestamp(ts - 2); + result = region.get(get); + checkResult(result, c0, T3); + + get = new Get(T1); + get.readAllVersions(); + get.setTimestamp(ts - 3); + result = region.get(get); + Assert.assertEquals(result.getColumnCells(c0, c0).size(), 0); + + get = new Get(T1); + get.readAllVersions(); + get.setTimeRange(0, ts - 2); + result = region.get(get); + Assert.assertEquals(result.getColumnCells(c0, c0).size(), 0); + } + + private void verifyBeforeCompaction(HRegion region, long ts) throws IOException { + Get get; + Result result; + get = new Get(T1); + get.readAllVersions(); + get.setTimeRange(0, ts); + result = region.get(get); + checkResult(result, c0, T4, T3, T2, T1); + + get = new Get(T1); + get.readAllVersions(); + get.setTimeRange(0, ts - 1); + result = region.get(get); + checkResult(result, c0, T3, T2, T1); + + get = new Get(T1); + get.readAllVersions(); + get.setTimeRange(0, ts - 2); + result = region.get(get); + checkResult(result, c0, T2, T1); + + get = new Get(T1); + get.readAllVersions(); + get.setTimeRange(0, ts - 3); + result = region.get(get); + checkResult(result, c0, T1); + } - long ts = startTS - 2000; + private void putFourVersions(HRegion region, long ts) throws IOException { // 1st version - Put p = new Put(T1, ts-3); - p.addColumn(c0, c0, T1); - region.put(p); + Put put = new Put(T1, ts - 4); + put.addColumn(c0, c0, T1); + region.put(put); // 2nd version - p = new Put(T1, ts-2); - p.addColumn(c0, c0, T2); - region.put(p); + put = new Put(T1, ts - 3); + put.addColumn(c0, c0, T2); + region.put(put); // 3rd version - p = new Put(T1, ts-1); - p.addColumn(c0, c0, T3); - region.put(p); - - Get g; - Result r; - - //check we can still see all versions before compaction - g = new Get(T1); - g.readAllVersions(); - g.setTimeRange(0, ts); - r = region.get(g); - checkResult(r, c0, T3, T2, T1); - - region.flush(true); - region.compact(true); - Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime()); - long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 3; - Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000)); - //check that nothing was purged yet - g = new Get(T1); - g.readAllVersions(); - g.setTimeRange(0, ts); - r = region.get(g); - checkResult(r, c0, T3, T2, T1); - - g = new Get(T1); - g.readAllVersions(); - g.setTimeRange(0, ts -1); - r = region.get(g); - checkResult(r, c0, T2, T1); - - injectEdge.incValue(ttl * 1000); - - region.flush(true); - region.compact(true); - - //check that after compaction (which is after TTL) that only T1 was purged - g = new Get(T1); - g.readAllVersions(); - g.setTimeRange(0, ts); - r = region.get(g); - checkResult(r, c0, T3, T2); - - g = new Get(T1); - g.readAllVersions(); - g.setTimestamp(ts -2); - r = region.get(g); - checkResult(r, c0, T2); + put = new Put(T1, ts - 2); + put.addColumn(c0, c0, T3); + region.put(put); + + // 4th version + put = new Put(T1, ts - 1); + put.addColumn(c0, c0, T4); + region.put(put); } private void checkResult(Result r, byte[] col, byte[] ... vals) {