improve spillableMapBasePath disk directory is full

apache · Oct 9, 2022 · 0882f24 · 0882f24
1 parent 29d5956
commit 0882f24
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 1 deletion.
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -40,6 +40,7 @@
 import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
 import org.apache.hudi.common.model.WriteConcurrencyMode;
 import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode;
 import org.apache.hudi.common.table.log.block.HoodieLogBlock;
 import org.apache.hudi.common.table.marker.MarkerType;
@@ -1915,7 +1916,7 @@ public int getMaxDFSStreamBufferSize() {
   }
 
   public String getSpillableMapBasePath() {
-    return getString(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH);
+    return getBasePath() + HoodieTableMetaClient.SPILLABLE_MAP_RELATIVE_PATH;
   }
 
   public double getWriteStatusFailureFraction() {

diff --git a/...client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java b/...client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
@@ -28,6 +28,7 @@
 import org.apache.hudi.common.model.HoodieTableType;
 import org.apache.hudi.common.model.WriteConcurrencyMode;
 import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.marker.MarkerType;
 import org.apache.hudi.config.HoodieWriteConfig.Builder;
 import org.apache.hudi.index.HoodieIndex;
@@ -415,6 +416,16 @@ public void testSimpleBucketIndexPartitionerConfig() {
     assertEquals("org.apache.hudi.table.action.commit.UpsertPartitioner", overwritePartitioner.getString(HoodieLayoutConfig.LAYOUT_PARTITIONER_CLASS_NAME));
   }
 
+  @Test
+  public void testSpillableMapBasePath() {
+    HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath("tmp").build();
+    assertEquals(writeConfig.getSpillableMapBasePath(), HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.defaultValue());
+
+    String tableBasePath = "base_path";
+    writeConfig = HoodieWriteConfig.newBuilder().withPath(tableBasePath).withMemoryConfig(HoodieMemoryConfig.newBuilder().build()).build();
+    assertEquals(writeConfig.getSpillableMapBasePath(), tableBasePath + HoodieTableMetaClient.SPILLABLE_MAP_RELATIVE_PATH);
+  }
+
   private HoodieWriteConfig createWriteConfig(Map<String, String> configs) {
     final Properties properties = new Properties();
     configs.forEach(properties::setProperty);

diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -93,6 +93,8 @@ public class HoodieTableMetaClient implements Serializable {
   public static final String BOOTSTRAP_INDEX_BY_FILE_ID_FOLDER_PATH = BOOTSTRAP_INDEX_ROOT_FOLDER_PATH + Path.SEPARATOR
       + ".fileids";
 
+  public static final String SPILLABLE_MAP_RELATIVE_PATH = METAFOLDER_NAME + Path.SEPARATOR + ".spillable_map" + Path.SEPARATOR+ ".temp";
+
   public static final String SCHEMA_FOLDER_NAME = ".schema";
 
   public static final String MARKER_EXTN = ".marker";