KYLIN-4896 Optimize the process of writing big resource files into HDFS.

(cherry picked from commit 985ff83)
zhangayqian · Jun 11, 2021 · bfdbaba · bfdbaba
1 parent 3855c13
commit bfdbaba
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -87,8 +87,10 @@ public HBaseResourceStore(KylinConfig kylinConfig) throws IOException {
         tableName = metadataUrl.getIdentifier();
         createHTableIfNeeded(tableName);
 
-        kvSizeLimit = Integer
+        int kvSizeLimitActual = Integer
                 .parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
+        kvSizeLimit = kvSizeLimitActual > 10485760 ? kvSizeLimitActual : 10485760;
+        logger.debug("hbase.client.keyvalue.maxsize is {}, kvSizeLimit is set to {}", kvSizeLimitActual, kvSizeLimit);
     }
 
     Connection getConnection() throws IOException {
@@ -323,6 +325,7 @@ protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldT
             byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
 
             if (content.length > kvSizeLimit) {
+                logger.info("Length of content exceeds the limit of {} bytes, push down {} to HDFS", kvSizeLimit, resPath);
                 pushdown = writePushdown(resPath, ContentWriter.create(content));
                 content = BytesUtil.EMPTY_BYTE_ARRAY;
             }