diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java index 2a5e2a5aa39d..6638fd2049c6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -47,6 +47,9 @@ public class DataTieringManager { "hbase.regionserver.datatiering.enable"; public static final boolean DEFAULT_GLOBAL_DATA_TIERING_ENABLED = false; // disabled by default public static final String DATATIERING_KEY = "hbase.hstore.datatiering.type"; + public static final String HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY = + "hbase.hstore.datatiering.grace.period.millis"; + public static final long DEFAULT_DATATIERING_GRACE_PERIOD = 0; public static final String DATATIERING_HOT_DATA_AGE_KEY = "hbase.hstore.datatiering.hot.age.millis"; public static final DataTieringType DEFAULT_DATATIERING = DataTieringType.NONE; @@ -139,6 +142,9 @@ public boolean isHotData(BlockCacheKey key) throws DataTieringException { * @return {@code true} if the data is hot, {@code false} otherwise */ public boolean isHotData(long maxTimestamp, Configuration conf) { + if (isWithinGracePeriod(maxTimestamp, conf)) { + return true; + } DataTieringType dataTieringType = getDataTieringType(conf); if ( @@ -170,8 +176,11 @@ public boolean isHotData(Path hFilePath) throws DataTieringException { throw new DataTieringException( "Store file corresponding to " + hFilePath + " doesn't exist"); } - return hotDataValidator(dataTieringType.getInstance().getTimestamp(getHStoreFile(hFilePath)), - getDataTieringHotDataAge(configuration)); + long maxTimestamp = dataTieringType.getInstance().getTimestamp(hStoreFile); + if (isWithinGracePeriod(maxTimestamp, configuration)) { + return true; + } + return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration)); } // DataTieringType.NONE or other types are considered hot by default return true; @@ -189,13 +198,21 @@ public boolean isHotData(Path hFilePath) throws DataTieringException { public boolean isHotData(HFileInfo hFileInfo, Configuration configuration) { DataTieringType dataTieringType = getDataTieringType(configuration); if (hFileInfo != null && !dataTieringType.equals(DataTieringType.NONE)) { - return hotDataValidator(dataTieringType.getInstance().getTimestamp(hFileInfo), - getDataTieringHotDataAge(configuration)); + long maxTimestamp = dataTieringType.getInstance().getTimestamp(hFileInfo); + if (isWithinGracePeriod(maxTimestamp, configuration)) { + return true; + } + return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration)); } // DataTieringType.NONE or other types are considered hot by default return true; } + private boolean isWithinGracePeriod(long maxTimestamp, Configuration conf) { + long gracePeriod = getDataTieringGracePeriod(conf); + return gracePeriod > 0 && (getCurrentTimestamp() - maxTimestamp) < gracePeriod; + } + private boolean hotDataValidator(long maxTimestamp, long hotDataAge) { long currentTimestamp = getCurrentTimestamp(); long diff = currentTimestamp - maxTimestamp; @@ -275,6 +292,11 @@ private long getDataTieringHotDataAge(Configuration conf) { conf.get(DATATIERING_HOT_DATA_AGE_KEY, String.valueOf(DEFAULT_DATATIERING_HOT_DATA_AGE))); } + private long getDataTieringGracePeriod(Configuration conf) { + return Long.parseLong(conf.get(HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY, + String.valueOf(DEFAULT_DATATIERING_GRACE_PERIOD))); + } + /* * This API traverses through the list of online regions and returns a subset of these files-names * that are cold. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index 507f14a86946..21e2315ae881 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -231,6 +231,57 @@ public void testHotDataWithPath() throws IOException { new DataTieringException("Store file corresponding to " + hFilePath + " doesn't exist")); } + @Test + public void testGracePeriodMakesColdFileHot() throws IOException, DataTieringException { + initializeTestEnvironment(); + + long hotAge = 1 * DAY; + long gracePeriod = 3 * DAY; + + long currentTime = System.currentTimeMillis(); + long fileTimestamp = currentTime - (2 * DAY); + + Configuration conf = getConfWithGracePeriod(hotAge, gracePeriod); + HRegion region = createHRegion("tableGracePeriod", conf); + HStore hStore = createHStore(region, "cf1", conf); + + HStoreFile file = createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(), + hStore.getReadOnlyConfiguration(), fileTimestamp, region.getRegionFileSystem()); + file.initReader(); + + hStore.refreshStoreFiles(); + region.stores.put(Bytes.toBytes("cf1"), hStore); + testOnlineRegions.put(region.getRegionInfo().getEncodedName(), region); + Path hFilePath = file.getPath(); + assertTrue("File should be hot due to grace period", dataTieringManager.isHotData(hFilePath)); + } + + @Test + public void testFileIsColdWithoutGracePeriod() throws IOException, DataTieringException { + initializeTestEnvironment(); + + long hotAge = 1 * DAY; + long gracePeriod = 0; + long currentTime = System.currentTimeMillis(); + long fileTimestamp = currentTime - (2 * DAY); + + Configuration conf = getConfWithGracePeriod(hotAge, gracePeriod); + HRegion region = createHRegion("tableNoGracePeriod", conf); + HStore hStore = createHStore(region, "cf1", conf); + + HStoreFile file = createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(), + hStore.getReadOnlyConfiguration(), fileTimestamp, region.getRegionFileSystem()); + file.initReader(); + + hStore.refreshStoreFiles(); + region.stores.put(Bytes.toBytes("cf1"), hStore); + testOnlineRegions.put(region.getRegionInfo().getEncodedName(), region); + + Path hFilePath = file.getPath(); + assertFalse("File should be cold without grace period", + dataTieringManager.isHotData(hFilePath)); + } + @Test public void testPrefetchWhenDataTieringEnabled() throws IOException { setPrefetchBlocksOnOpen(); @@ -770,6 +821,8 @@ private static HRegion createHRegion(String table, Configuration conf) throws IO .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY)) .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY)) + .setValue(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY, + conf.get(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY)) .build(); RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build(); @@ -797,6 +850,8 @@ private static HStore createHStore(HRegion region, String columnFamily, Configur .setValue(DataTieringManager.DATATIERING_KEY, conf.get(DataTieringManager.DATATIERING_KEY)) .setValue(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY, conf.get(DataTieringManager.DATATIERING_HOT_DATA_AGE_KEY)) + .setValue(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY, + conf.get(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY)) .build(); return new HStore(region, columnFamilyDescriptor, conf, false); @@ -809,6 +864,13 @@ private static Configuration getConfWithTimeRangeDataTieringEnabled(long hotData return conf; } + private static Configuration getConfWithGracePeriod(long hotDataAge, long gracePeriod) { + Configuration conf = getConfWithTimeRangeDataTieringEnabled(hotDataAge); + conf.set(DataTieringManager.HSTORE_DATATIERING_GRACE_PERIOD_MILLIS_KEY, + String.valueOf(gracePeriod)); + return conf; + } + static HStoreFile createHStoreFile(Path storeDir, Configuration conf, long timestamp, HRegionFileSystem regionFs) throws IOException { String columnFamily = storeDir.getName();