Skip to content
Permalink
Browse files

fix: improve insert values(...) batch rewrite

Now it supports VALUES (?,1), and splits very long batch into series of smaller ones
to keep overall number of bind variables <32768 (that is PG's limit in v3 protocol)

Trigger property: reWriteBatchedInserts=true. Default value is still "false".

closes #580
closes #584
  • Loading branch information
Chrriis authored and vlsi committed Jun 22, 2016
1 parent 1eb4085 commit 510e6e0fec5e3a44ffda854f5cf808ce9f72ee2e
Showing with 1,458 additions and 1,384 deletions.
  1. +23 −10 pgjdbc/src/main/java/org/postgresql/core/DMLCommand.java
  2. +31 −3 pgjdbc/src/main/java/org/postgresql/core/NativeQuery.java
  3. +88 −86 pgjdbc/src/main/java/org/postgresql/core/Parser.java
  4. +0 −13 pgjdbc/src/main/java/org/postgresql/core/Query.java
  5. +1 −17 pgjdbc/src/main/java/org/postgresql/core/v2/V2Query.java
  6. +152 −0 pgjdbc/src/main/java/org/postgresql/core/v3/BatchedQuery.java
  7. +0 −311 pgjdbc/src/main/java/org/postgresql/core/v3/BatchedQueryDecorator.java
  8. +0 −12 pgjdbc/src/main/java/org/postgresql/core/v3/CompositeQuery.java
  9. +9 −6 pgjdbc/src/main/java/org/postgresql/core/v3/QueryExecutorImpl.java
  10. +9 −7 pgjdbc/src/main/java/org/postgresql/core/v3/SimpleParameterList.java
  11. +14 −26 pgjdbc/src/main/java/org/postgresql/core/v3/SimpleQuery.java
  12. +51 −5 pgjdbc/src/main/java/org/postgresql/jdbc/BatchResultHandler.java
  13. +2 −3 pgjdbc/src/main/java/org/postgresql/jdbc/CallableBatchResultHandler.java
  14. +2 −2 pgjdbc/src/main/java/org/postgresql/jdbc/PgCallableStatement.java
  15. +47 −44 pgjdbc/src/main/java/org/postgresql/jdbc/PgPreparedStatement.java
  16. +25 −35 pgjdbc/src/main/java/org/postgresql/jdbc/PgStatement.java
  17. +137 −268 pgjdbc/src/test/java/org/postgresql/jdbc/DeepBatchedInsertStatementTest.java
  18. +370 −0 pgjdbc/src/test/java/org/postgresql/jdbc/InsertRewriteWithAlternatingTypesIssue584.java
  19. +45 −0 pgjdbc/src/test/java/org/postgresql/test/core/NativeQueryBindLengthTest.java
  20. +0 −14 pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchExecuteBinaryTest.java
  21. +178 −70 pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchExecuteTest.java
  22. +12 −4 pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchFailureTest.java
  23. +0 −88 pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchedInsertDoubleRowInSingleBatch.java
  24. +248 −129 pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchedInsertReWriteEnabledTest.java
  25. +0 −226 pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchedInsertStatementPreparingTest.java
  26. +4 −5 pgjdbc/src/test/java/org/postgresql/test/jdbc2/Jdbc2TestSuite.java
  27. +10 −0 ubenchmark/src/main/java/org/postgresql/benchmark/statement/InsertBatch.java
@@ -14,12 +14,21 @@
* Data Modification Language inspection support.
*
* @author Jeremy Whiting jwhiting@redhat.com
* @author Christopher Deckers (chrriis@gmail.com)
*
*/
public class DMLCommand {

public boolean isBatchedReWriteCompatible() {
return batchedReWriteCompatible;
return valuesBraceOpenPosition >= 0;
}

public int getBatchRewriteValuesBraceOpenPosition() {
return valuesBraceOpenPosition;
}

public int getBatchRewriteValuesBraceClosePosition() {
return valuesBraceClosePosition;
}

public DMLCommandType getType() {
@@ -32,33 +41,37 @@ public boolean isReturningKeywordPresent() {

public static DMLCommand createStatementTypeInfo(DMLCommandType type,
boolean isBatchedReWritePropertyConfigured,
boolean isBatchedReWriteCompatible, boolean isRETURNINGkeywordPresent,
boolean autocommit, int priorQueryCount) {
int valuesBraceOpenPosition, int valuesBraceClosePosition, boolean isRETURNINGkeywordPresent,
int priorQueryCount) {
return new DMLCommand(type, isBatchedReWritePropertyConfigured,
isBatchedReWriteCompatible, isRETURNINGkeywordPresent, autocommit,
valuesBraceOpenPosition, valuesBraceClosePosition, isRETURNINGkeywordPresent,
priorQueryCount);
}

public static DMLCommand createStatementTypeInfo(DMLCommandType type) {
return new DMLCommand(type, false, false, false, false,0);
return new DMLCommand(type, false, -1, -1, false, 0);
}

public static DMLCommand createStatementTypeInfo(DMLCommandType type,
boolean isRETURNINGkeywordPresent) {
return new DMLCommand(type, false, false, isRETURNINGkeywordPresent, false,0);
return new DMLCommand(type, false, -1, -1, isRETURNINGkeywordPresent, 0);
}

private DMLCommand(DMLCommandType type, boolean isBatchedReWriteConfigured,
boolean isCompatible, boolean isPresent, boolean isautocommitConfigured,
int valuesBraceOpenPosition, int valuesBraceClosePosition, boolean isPresent,
int priorQueryCount) {
commandType = type;
parsedSQLhasRETURNINGKeyword = isPresent;
batchedReWriteCompatible = (type == INSERT) && isBatchedReWriteConfigured
&& isCompatible && !isautocommitConfigured
boolean batchedReWriteCompatible = (type == INSERT) && isBatchedReWriteConfigured
&& valuesBraceOpenPosition >= 0 && valuesBraceClosePosition > valuesBraceOpenPosition
&& !isPresent && priorQueryCount == 0;
this.valuesBraceOpenPosition = batchedReWriteCompatible ? valuesBraceOpenPosition : -1;
this.valuesBraceClosePosition = batchedReWriteCompatible ? valuesBraceClosePosition : -1;
}

private final DMLCommandType commandType;
private final boolean parsedSQLhasRETURNINGKeyword;
private final boolean batchedReWriteCompatible;
private final int valuesBraceOpenPosition;
private final int valuesBraceClosePosition;

}
@@ -14,7 +14,7 @@
* replaced with $1, $2, etc.
*/
public class NativeQuery {
private final static String[] BIND_NAMES = new String[128];
private final static String[] BIND_NAMES = new String[128 * 10];
private final static int[] NO_BINDS = new int[0];

public final String nativeSql;
@@ -79,8 +79,36 @@ public static String bindName(int index) {
return index < BIND_NAMES.length ? BIND_NAMES[index] : "$" + index;
}

public static int bindCount() {
return BIND_NAMES.length;
public static StringBuilder appendBindName(StringBuilder sb, int index) {
if (index < BIND_NAMES.length) {
return sb.append(bindName(index));
}
sb.append('$');
sb.append(index);
return sb;
}

/**
* Calculate the text length required for the given number of bind variables
* including dollars.
* Do this to avoid repeated calls to
* AbstractStringBuilder.expandCapacity(...) and Arrays.copyOf
*
* @param bindCount total number of parameters in a query
* @return int total character length for $xyz kind of binds
*/
public static int calculateBindLength(int bindCount) {
int res = 0;
int bindLen = 2; // $1
int maxBindsOfLen = 9; // $0 .. $9
while (bindCount > 0) {
int numBinds = Math.min(maxBindsOfLen, bindCount);
bindCount -= numBinds;
res += bindLen * numBinds;
bindLen++;
maxBindsOfLen *= 10; // $0..$9 (9 items) -> $10..$99 (90 items)
}
return res;
}

public DMLCommand getCommand() {

0 comments on commit 510e6e0

Please sign in to comment.
You can’t perform that action at this time.