diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java index 182768402f64..be17470edd66 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.metastore; import static org.apache.commons.lang3.StringUtils.repeat; -import static org.apache.hadoop.hive.metastore.Batchable.NO_BATCHING; import java.util.ArrayList; import java.util.Collections; @@ -83,7 +82,14 @@ private void insertInBatch(String tableName, String columns, int columnCount, in if (rowCount == 0 || columnCount == 0) { return; } - int maxRowsInBatch = (batchSize == NO_BATCHING) ? rowCount : batchSize; + int maxRowsInBatch = batchSize > 0 ? batchSize : rowCount; + if (dbType.isSQLSERVER()) { + // SQL Server supports a maximum of 2100 parameters in a request. Adjust the maxRowsInBatch accordingly + int maxAllowedRows = (2100 - columnCount) / columnCount; + if (maxRowsInBatch > maxAllowedRows) { + maxRowsInBatch = maxAllowedRows; + } + } int maxBatches = rowCount / maxRowsInBatch; int last = rowCount % maxRowsInBatch; String rowFormat = "(" + repeat(",?", columnCount).substring(1) + ")"; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index a39da29cbb86..5789b535debd 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -195,16 +195,12 @@ public MetaStoreDirectSql(PersistenceManager pm, Configuration conf, String sche this.dbType = dbType; int batchSize = MetastoreConf.getIntVar(conf, ConfVars.DIRECT_SQL_PARTITION_BATCH_SIZE); + this.directSqlInsertPart = new DirectSqlInsertPart(pm, dbType, batchSize); if (batchSize == DETECT_BATCHING) { batchSize = dbType.needsInBatching() ? 1000 : NO_BATCHING; } this.batchSize = batchSize; this.updateStat = new DirectSqlUpdateStat(pm, conf, dbType, batchSize); - - // TODO: Oracle supports to insert more than 1000 rows with a single insert query. Can use NO_BATCHING for oracle db - // too during batch detection(DETECT_BATCHING) for insert queries as future improvement. Currently, used the same - // limit as IN clause/operator limit(i.e., 1000) during batch detection. - this.directSqlInsertPart = new DirectSqlInsertPart(pm, dbType, batchSize); ImmutableMap.Builder fieldNameToTableNameBuilder = new ImmutableMap.Builder<>();