chore: increase parallel loads (#3073)

rudderlabs · Mar 7, 2023 · 7dcc756 · 7dcc756
1 parent 3ec2433
commit 7dcc756
Showing 1 changed file with 10 additions and 7 deletions.
diff --git a/warehouse/upload.go b/warehouse/upload.go
@@ -147,13 +147,16 @@ func init() {
 
 func setMaxParallelLoads() {
 	maxParallelLoads = map[string]int{
-		warehouseutils.BQ:         config.GetInt("Warehouse.bigquery.maxParallelLoads", 20),
-		warehouseutils.RS:         config.GetInt("Warehouse.redshift.maxParallelLoads", 3),
-		warehouseutils.POSTGRES:   config.GetInt("Warehouse.postgres.maxParallelLoads", 3),
-		warehouseutils.MSSQL:      config.GetInt("Warehouse.mssql.maxParallelLoads", 3),
-		warehouseutils.SNOWFLAKE:  config.GetInt("Warehouse.snowflake.maxParallelLoads", 3),
-		warehouseutils.CLICKHOUSE: config.GetInt("Warehouse.clickhouse.maxParallelLoads", 3),
-		warehouseutils.DELTALAKE:  config.GetInt("Warehouse.deltalake.maxParallelLoads", 3),
+		warehouseutils.BQ:             config.GetInt("Warehouse.bigquery.maxParallelLoads", 20),
+		warehouseutils.RS:             config.GetInt("Warehouse.redshift.maxParallelLoads", 8),
+		warehouseutils.POSTGRES:       config.GetInt("Warehouse.postgres.maxParallelLoads", 8),
+		warehouseutils.MSSQL:          config.GetInt("Warehouse.mssql.maxParallelLoads", 8),
+		warehouseutils.SNOWFLAKE:      config.GetInt("Warehouse.snowflake.maxParallelLoads", 8),
+		warehouseutils.CLICKHOUSE:     config.GetInt("Warehouse.clickhouse.maxParallelLoads", 8),
+		warehouseutils.DELTALAKE:      config.GetInt("Warehouse.deltalake.maxParallelLoads", 8),
+		warehouseutils.S3_DATALAKE:    config.GetInt("Warehouse.s3_datalake.maxParallelLoads", 8),
+		warehouseutils.GCS_DATALAKE:   config.GetInt("Warehouse.gcs_datalake.maxParallelLoads", 8),
+		warehouseutils.AZURE_DATALAKE: config.GetInt("Warehouse.azure_datalake.maxParallelLoads", 8),
 	}
 	columnCountLimitMap = map[string]int{
 		warehouseutils.AZURE_SYNAPSE: config.GetInt("Warehouse.azure_synapse.columnCountLimit", 1024),