Skip to content
Permalink
Browse files
Revert "HBASE-26274 Create an option to reintroduce BlockCache to map…
…reduce job (#3684)" (#3695)

This reverts commit 6556a5e.
  • Loading branch information
taklwu committed Sep 22, 2021
1 parent 6556a5e commit ecd35f77ae7af74eb72ce68875ac9b3044492a1b
Show file tree
Hide file tree
Showing 9 changed files with 8 additions and 238 deletions.
@@ -1049,17 +1049,6 @@ public enum OperationStatusCode {

public static final float HFILE_BLOCK_CACHE_SIZE_DEFAULT = 0.4f;

/**
* Configuration key for setting the fix size of the block size, default do nothing and it should
* be explicitly set by user or only used within ClientSideRegionScanner. if it's set less than
* current max on heap size, it overrides the max size of block cache
*/
public static final String HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY =
"hfile.onheap.block.cache.fixed.size";
public static final long HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_DEFAULT = 0L;
public static final long HBASE_CLIENT_SCANNER_ONHEAP_BLOCK_CACHE_FIXED_SIZE_DEFAULT =
32 * 1024 * 1024L;

/*
* Minimum percentage of free heap necessary for a successful cluster startup.
*/
@@ -25,10 +25,8 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory;
import org.apache.hadoop.hbase.mob.MobFileCache;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
@@ -62,15 +60,6 @@ public ClientSideRegionScanner(Configuration conf, FileSystem fs,
region = HRegion.newHRegion(CommonFSUtils.getTableDir(rootDir, htd.getTableName()), null, fs,
conf, hri, htd, null);
region.setRestoredRegion(true);
// non RS process does not have a block cache, and this a client side scanner,
// create one for MapReduce jobs to cache the INDEX block by setting to use
// IndexOnlyLruBlockCache and set a value to HBASE_CLIENT_SCANNER_BLOCK_CACHE_SIZE_KEY
conf.set(BlockCacheFactory.BLOCKCACHE_POLICY_KEY, "IndexOnlyLRU");
conf.setIfUnset(HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY,
String.valueOf(HConstants.HBASE_CLIENT_SCANNER_ONHEAP_BLOCK_CACHE_FIXED_SIZE_DEFAULT));
// don't allow L2 bucket cache for non RS process to avoid unexpected disk usage.
conf.unset(HConstants.BUCKET_CACHE_IOENGINE_KEY);
region.setBlockCache(BlockCacheFactory.createBlockCache(conf));
// we won't initialize the MobFileCache when not running in RS process. so provided an
// initialized cache. Consider the case: an CF was set from an mob to non-mob. if we only
// initialize cache for MOB region, NPE from HMobStore will still happen. So Initialize the
@@ -133,10 +122,6 @@ public void close() {
}
}

HRegion getRegion() {
return region;
}

@Override
public boolean renewLease() {
throw new UnsupportedOperationException();
@@ -147,13 +147,4 @@ default Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repe
* @return The list of sub blockcaches that make up this one; returns null if no sub caches.
*/
BlockCache [] getBlockCaches();

/**
* Check if block type is meta or index block
* @param blockType block type of a given HFile block
* @return true if block type is non-data block
*/
default boolean isMetaBlock(BlockType blockType) {
return blockType != null && blockType.getCategory() != BlockType.BlockCategory.DATA;
}
}
@@ -43,7 +43,7 @@ public final class BlockCacheFactory {
*/

/**
* Configuration key to cache block policy (Lru, TinyLfu, AdaptiveLRU, IndexOnlyLRU).
* Configuration key to cache block policy (Lru, TinyLfu).
*/
public static final String BLOCKCACHE_POLICY_KEY = "hfile.block.cache.policy";
public static final String BLOCKCACHE_POLICY_DEFAULT = "LRU";
@@ -129,8 +129,6 @@ private static FirstLevelBlockCache createFirstLevelCache(final Configuration c)
StringUtils.byteDesc(cacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
if (policy.equalsIgnoreCase("LRU")) {
return new LruBlockCache(cacheSize, blockSize, true, c);
} else if (policy.equalsIgnoreCase("IndexOnlyLRU")) {
return new IndexOnlyLruBlockCache(cacheSize, blockSize, true, c);
} else if (policy.equalsIgnoreCase("TinyLFU")) {
return new TinyLfuBlockCache(cacheSize, blockSize, ForkJoinPool.commonPool(), c);
} else if (policy.equalsIgnoreCase("AdaptiveLRU")) {
@@ -22,6 +22,7 @@

import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;

/**
@@ -70,6 +71,10 @@ public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
cacheBlock(cacheKey, buf, false);
}

private boolean isMetaBlock(BlockType blockType) {
return blockType.getCategory() != BlockCategory.DATA;
}

@Override
public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching,
boolean repeat, boolean updateCacheMetrics) {

This file was deleted.

@@ -228,13 +228,9 @@ public static long getOnHeapCacheSize(final Configuration conf) {
if (usage != null) {
max = usage.getMax();
}
float onHeapCacheFixedSize = (float) conf
.getLong(HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY,
HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_DEFAULT) / max;

// Calculate the amount of heap to give the heap.
return (onHeapCacheFixedSize > 0 && onHeapCacheFixedSize < cachePercentage) ?
(long) (max * onHeapCacheFixedSize) :
(long) (max * cachePercentage);
return (long) (max * cachePercentage);
}

/**

This file was deleted.

@@ -374,34 +374,4 @@ public void testL2CacheWithInvalidBucketSize() {
} catch (IllegalArgumentException e) {
}
}

@Test
public void testIndexOnlyLruBlockCache() {
CacheConfig cc = new CacheConfig(this.conf);
conf.set(BlockCacheFactory.BLOCKCACHE_POLICY_KEY, "IndexOnlyLRU");
BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf);
assertTrue(blockCache instanceof IndexOnlyLruBlockCache);
// reject data block
long initialBlockCount = blockCache.getBlockCount();
BlockCacheKey bck = new BlockCacheKey("bck", 0);
Cacheable c = new DataCacheEntry();
blockCache.cacheBlock(bck, c, true);
// accept index block
Cacheable indexCacheEntry = new IndexCacheEntry();
blockCache.cacheBlock(bck, indexCacheEntry, true);
assertEquals(initialBlockCount + 1, blockCache.getBlockCount());
}

@Test
public void testGetOnHeapCacheSize() {
Configuration copyConf = new Configuration(conf);
long fixedSize = 1024 * 1024L;
long onHeapCacheSize = MemorySizeUtil.getOnHeapCacheSize(copyConf);
assertEquals(null, copyConf.get(HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY));
assertTrue(onHeapCacheSize > 0 && onHeapCacheSize != fixedSize);
// when HBASE_BLOCK_CACHE_FIXED_SIZE_KEY is set, it will be a fixed size
copyConf.setLong(HConstants.HFILE_ONHEAP_BLOCK_CACHE_FIXED_SIZE_KEY, fixedSize);
onHeapCacheSize = MemorySizeUtil.getOnHeapCacheSize(copyConf);
assertEquals(fixedSize, onHeapCacheSize);
}
}

0 comments on commit ecd35f7

Please sign in to comment.