HIVE-27060: Exception in add partitions with SQL Server when number o…

…f parameters exceed 2100 (Venu Reddy, reviewed by Zhihua Deng) Closes apache#4044
yeahyung · Jul 20, 2023 · e83ae7c · e83ae7c
1 parent 9fa5e8f
commit e83ae7c
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 7 deletions.
diff --git a/.../metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java b/.../metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.hive.metastore;
 
 import static org.apache.commons.lang3.StringUtils.repeat;
-import static org.apache.hadoop.hive.metastore.Batchable.NO_BATCHING;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -83,7 +82,14 @@ private void insertInBatch(String tableName, String columns, int columnCount, in
     if (rowCount == 0 || columnCount == 0) {
       return;
     }
-    int maxRowsInBatch = (batchSize == NO_BATCHING) ? rowCount : batchSize;
+    int maxRowsInBatch = batchSize > 0 ? batchSize : rowCount;
+    if (dbType.isSQLSERVER()) {
+      // SQL Server supports a maximum of 2100 parameters in a request. Adjust the maxRowsInBatch accordingly
+      int maxAllowedRows = (2100 - columnCount) / columnCount;
+      if (maxRowsInBatch > maxAllowedRows) {
+        maxRowsInBatch = maxAllowedRows;
+      }
+    }
     int maxBatches = rowCount / maxRowsInBatch;
     int last = rowCount % maxRowsInBatch;
     String rowFormat = "(" + repeat(",?", columnCount).substring(1) + ")";

diff --git a/...e/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/...e/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -195,16 +195,12 @@ public MetaStoreDirectSql(PersistenceManager pm, Configuration conf, String sche
 
     this.dbType = dbType;
     int batchSize = MetastoreConf.getIntVar(conf, ConfVars.DIRECT_SQL_PARTITION_BATCH_SIZE);
+    this.directSqlInsertPart = new DirectSqlInsertPart(pm, dbType, batchSize);
     if (batchSize == DETECT_BATCHING) {
       batchSize = dbType.needsInBatching() ? 1000 : NO_BATCHING;
     }
     this.batchSize = batchSize;
     this.updateStat = new DirectSqlUpdateStat(pm, conf, dbType, batchSize);
-
-    // TODO: Oracle supports to insert more than 1000 rows with a single insert query. Can use NO_BATCHING for oracle db
-    //  too during batch detection(DETECT_BATCHING) for insert queries as future improvement. Currently, used the same
-    //  limit as IN clause/operator limit(i.e., 1000) during batch detection.
-    this.directSqlInsertPart = new DirectSqlInsertPart(pm, dbType, batchSize);
     ImmutableMap.Builder<String, String> fieldNameToTableNameBuilder =
         new ImmutableMap.Builder<>();