Skip to content

Commit

Permalink
[HUDI-2151] Part1 Setting default parallelism to 200 for some of writ…
Browse files Browse the repository at this point in the history
…e configs (#3948)
  • Loading branch information
nsivabalan committed Nov 12, 2021
1 parent bc511ed commit 4f217fe
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -159,12 +159,12 @@ public class HoodieWriteConfig extends HoodieConfig {

public static final ConfigProperty<String> INSERT_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.insert.shuffle.parallelism")
.defaultValue("1500")
.defaultValue("200")
.withDocumentation("Parallelism for inserting records into the table. Inserts can shuffle data before writing to tune file sizes and optimize the storage layout.");

public static final ConfigProperty<String> BULKINSERT_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.bulkinsert.shuffle.parallelism")
.defaultValue("1500")
.defaultValue("200")
.withDocumentation("For large initial imports using bulk_insert operation, controls the parallelism to use for sort modes or custom partitioning done"
+ "before writing records to the table.");

Expand All @@ -183,13 +183,13 @@ public class HoodieWriteConfig extends HoodieConfig {

public static final ConfigProperty<String> UPSERT_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.upsert.shuffle.parallelism")
.defaultValue("1500")
.defaultValue("200")
.withDocumentation("Parallelism to use for upsert operation on the table. Upserts can shuffle data to perform index lookups, file sizing, bin packing records optimally"
+ "into file groups.");

public static final ConfigProperty<String> DELETE_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.delete.shuffle.parallelism")
.defaultValue("1500")
.defaultValue("200")
.withDocumentation("Parallelism used for “delete” operation. Delete operations also performs shuffles, similar to upsert operation.");

public static final ConfigProperty<String> ROLLBACK_PARALLELISM_VALUE = ConfigProperty
Expand Down Expand Up @@ -241,7 +241,7 @@ public class HoodieWriteConfig extends HoodieConfig {

public static final ConfigProperty<String> FINALIZE_WRITE_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.finalize.write.parallelism")
.defaultValue("1500")
.defaultValue("200")
.withDocumentation("Parallelism for the write finalization internal operation, which involves removing any partially written "
+ "files from lake storage, before committing the write. Reduce this value, if the high number of tasks incur delays for smaller tables "
+ "or low latency writes.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public final class HoodieMetadataConfig extends HoodieConfig {

public static final ConfigProperty<Integer> FILE_LISTING_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.file.listing.parallelism")
.defaultValue(1500)
.defaultValue(200)
.sinceVersion("0.7.0")
.withDocumentation("Parallelism to use, when listing the table on lake storage.");

Expand Down

0 comments on commit 4f217fe

Please sign in to comment.