Skip to content
Permalink
Browse files

perf: Add optimization to re-write batched insert stataments.

  • Loading branch information
whitingjr authored and davecramer committed Apr 15, 2016
1 parent 1e1f3c4 commit ac8abf3f2acbef3f52eb7874b6815d1adc974299
Showing with 2,248 additions and 17 deletions.
  1. +7 −1 pgjdbc/src/main/java/org/postgresql/PGProperty.java
  2. +6 −0 pgjdbc/src/main/java/org/postgresql/core/BaseConnection.java
  3. +8 −2 pgjdbc/src/main/java/org/postgresql/core/NativeQuery.java
  4. +19 −0 pgjdbc/src/main/java/org/postgresql/core/ParameterList.java
  5. +99 −2 pgjdbc/src/main/java/org/postgresql/core/Parser.java
  6. +17 −0 pgjdbc/src/main/java/org/postgresql/core/Query.java
  7. +33 −0 pgjdbc/src/main/java/org/postgresql/core/v2/FastpathParameterList.java
  8. +38 −0 pgjdbc/src/main/java/org/postgresql/core/v2/SimpleParameterList.java
  9. +23 −0 pgjdbc/src/main/java/org/postgresql/core/v2/V2Query.java
  10. +242 −0 pgjdbc/src/main/java/org/postgresql/core/v3/BatchedQueryDecorator.java
  11. +29 −0 pgjdbc/src/main/java/org/postgresql/core/v3/CompositeParameterList.java
  12. +19 −0 pgjdbc/src/main/java/org/postgresql/core/v3/CompositeQuery.java
  13. +14 −3 pgjdbc/src/main/java/org/postgresql/core/v3/QueryExecutorImpl.java
  14. +47 −0 pgjdbc/src/main/java/org/postgresql/core/v3/SimpleParameterList.java
  15. +38 −4 pgjdbc/src/main/java/org/postgresql/core/v3/SimpleQuery.java
  16. +16 −0 pgjdbc/src/main/java/org/postgresql/core/v3/V3ParameterList.java
  17. +14 −0 pgjdbc/src/main/java/org/postgresql/ds/common/BaseDataSource.java
  18. +13 −1 pgjdbc/src/main/java/org/postgresql/jdbc/PgConnection.java
  19. +49 −4 pgjdbc/src/main/java/org/postgresql/jdbc/PgPreparedStatement.java
  20. +28 −0 pgjdbc/src/main/java/org/postgresql/jdbc/PgStatement.java
  21. +94 −0 pgjdbc/src/test/java/org/postgresql/test/core/v2/V2ParameterListTests.java
  22. +227 −0 pgjdbc/src/test/java/org/postgresql/test/core/v3/V3ParameterListTests.java
  23. +64 −0 pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchExecuteTest.java
  24. +101 −0 pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchedInsertDoubleRowInSingleBatch.java
  25. +283 −0 pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchedInsertReWriteEnabledTest.java
  26. +260 −0 pgjdbc/src/test/java/org/postgresql/test/jdbc2/BatchedInsertStatementPreparingTest.java
  27. +451 −0 pgjdbc/src/test/java/org/postgresql/test/jdbc2/DeepBatchedInsertStatementTest.java
  28. +9 −0 pgjdbc/src/test/java/org/postgresql/test/jdbc2/Jdbc2TestSuite.java
@@ -334,7 +334,13 @@
"If disabled hosts are connected in the given order. If enabled hosts are chosen randomly from the set of suitable candidates"),

HOST_RECHECK_SECONDS("hostRecheckSeconds", "10",
"Specifies period (seconds) after host statuses are checked again in case they have changed");
"Specifies period (seconds) after host statuses are checked again in case they have changed"),

/**
* Configure optimization to enable batch insert re-writing.
*/
REWRITE_BATCHED_INSERTS ("reWriteBatchedInserts", "false",
"Enable optimization to rewrite and collapse compatible INSERT statements that are batched.");

private String _name;
private String _defaultValue;
@@ -231,4 +231,10 @@ public ResultSet execSQLQuery(String s, int resultSetType, int resultSetConcurre
* Invoke purge() on the underlying shared Timer so that internal resources will be released.
*/
public void purgeTimerTasks();

/**
* To be used for checking if the batched insert re-write optimization is enabled.
* @return true if re-write feature is enabled
*/
public boolean isReWriteBatchedInsertsEnabled();
}
@@ -19,6 +19,7 @@

public final String nativeSql;
public final int[] bindPositions;
public final boolean isBatchedReWriteCompatible;

static {
for (int i = 1; i < BIND_NAMES.length; i++) {
@@ -27,13 +28,14 @@
}

public NativeQuery(String nativeSql) {
this(nativeSql, NO_BINDS);
this(nativeSql, NO_BINDS, false);
}

public NativeQuery(String nativeSql, int[] bindPositions) {
public NativeQuery(String nativeSql, int[] bindPositions, boolean compatible) {
this.nativeSql = nativeSql;
this.bindPositions =
bindPositions == null || bindPositions.length == 0 ? NO_BINDS : bindPositions;
this.isBatchedReWriteCompatible = compatible;
}

/**
@@ -76,4 +78,8 @@ public String toString(ParameterList parameters) {
public static String bindName(int index) {
return index < BIND_NAMES.length ? BIND_NAMES[index] : "$" + index;
}

public static int bindsCount() {
return BIND_NAMES.length;
}
}
@@ -170,4 +170,23 @@
* @return a string representation of the parameter.
*/
String toString(int index);

/**
* Overwrite current parameters with parameters in the provided list.
* Current parameters are cleared.
* @param list of parameters to overwrite with.
*/
void addAll(ParameterList list);

/**
* Use this operation to append more parameters to the current list.
* @param list of parameters to append with.
*/
void appendAll(ParameterList list);

/**
* Returns the bound parameter values.
* @return Object array containing the parameter values.
*/
Object[] getValues();
}
@@ -50,6 +50,8 @@
StringBuilder nativeSql = new StringBuilder(query.length() + 10);
List<Integer> bindPositions = null; // initialized on demand
List<NativeQuery> nativeQueries = null;
boolean isCurrentReWriteCompatible = false;
int afterValuesParens = 0;

boolean whitespaceOnly = true;
for (int i = 0; i < aChars.length; ++i) {
@@ -79,6 +81,10 @@

case '(':
inParen++;
afterValuesParens++;
if (afterValuesParens > 1) {
isCurrentReWriteCompatible = false;
}
break;

case ')':
@@ -117,13 +123,44 @@
nativeQueries = new ArrayList<NativeQuery>();
}

nativeQueries.add(new NativeQuery(nativeSql.toString(), toIntArray(bindPositions)));
nativeQueries.add(new NativeQuery(nativeSql.toString(), toIntArray(bindPositions),isCurrentReWriteCompatible));
}
// Prepare for next query
if (bindPositions != null) {
bindPositions.clear();
}
nativeSql.setLength(0);
isCurrentReWriteCompatible = false;
}
break;

case 'i':
isCurrentReWriteCompatible = isCurrentReWriteCompatible || (Parser.parseInsertKeyword(aChars, i, false));
break;

case 'I':
isCurrentReWriteCompatible = isCurrentReWriteCompatible || (Parser.parseInsertKeyword(aChars, i, true));
break;

case 'r':
// exclude insert statements with returning keyword
isCurrentReWriteCompatible = isCurrentReWriteCompatible && (Parser.parseReturningKeyword(aChars, i, false) == false);
break;

case 'R':
// exclude insert statements with RETURNING keyword
isCurrentReWriteCompatible = isCurrentReWriteCompatible && (Parser.parseReturningKeyword(aChars, i, true) == false);
break;

case 'v':
if (Parser.parseValuesKeyword(aChars, i, false)) {
afterValuesParens = 0 ;
}
break;

case 'V':
if (Parser.parseValuesKeyword(aChars, i, true)) {
afterValuesParens = 0 ;
}
break;

@@ -140,7 +177,7 @@
return nativeQueries != null ? nativeQueries : Collections.<NativeQuery>emptyList();
}

NativeQuery lastQuery = new NativeQuery(nativeSql.toString(), toIntArray(bindPositions));
NativeQuery lastQuery = new NativeQuery(nativeSql.toString(), toIntArray(bindPositions), isCurrentReWriteCompatible);

if (nativeQueries == null) {
return Collections.singletonList(lastQuery);
@@ -338,6 +375,66 @@ public static int parseBlockComment(final char[] query, int offset) {
return offset;
}

/**
* Parse string to check presence of INSERT keyword.
* @param query char[] of the query statement
* @param offset position of query to start checking
* @param isUpper is the text expected to be upper/lower case
* @return boolean indicates presence of word
*/
public static boolean parseInsertKeyword(final char[] query, int offset, boolean isUpper) {
if (query.length < (offset + 7)) {
return false;
}

if (isUpper && query[offset] == 'I' && query[offset + 1] == 'N' && query[offset + 2] == 'S' && query[offset + 3] == 'E' && query[offset + 4] == 'R' && query[offset + 5] == 'T' ) {
return true;
} else if ( !isUpper && query[offset] == 'i' && query[offset + 1] == 'n' && query[offset + 2] == 's' && query[offset + 3] == 'e' && query[offset + 4] == 'r' && query[offset + 5] == 't') {
return true;
}
return false;
}

/**
* Parse string to check presence of RETURNING keyword
* @param query char[] of the query statement
* @param offset position of query to start checking
* @param isUpper is the text expected to be upper/lower case
* @return boolean indicates presence of word
*/
public static boolean parseReturningKeyword(final char[] query, int offset, boolean isUpper) {
if (query.length < (offset + 9)) {
return false;
}

if ( isUpper && query[offset] == 'R' && query[offset + 1] == 'E' && query[offset + 2] == 'T' && query[offset + 3] == 'U' && query[offset + 4] == 'R' && query[offset + 5] == 'N' && query[offset + 6] == 'I' && query[offset + 7] == 'N' && query[offset + 8] == 'G' ) {
return true;
} else if ( !isUpper && query[offset] == 'r' && query[offset + 1] == 'e' && query[offset + 2] == 't' && query[offset + 3] == 'u' && query[offset + 4] == 'r' && query[offset + 5] == 'n' && query[offset + 6] == 'i' && query[offset + 7] == 'n' && query[offset + 8] == 'g' ) {
return true;
}
return false;
}

/**
* Parse string to check presence of VALUES keyword
* @param query char[] of the query statement
* @param offset position of query to start checking
* @param isUpper is the text expected to be upper/lower case
* @return boolean indicates presence of word
*/
public static boolean parseValuesKeyword(final char[] query, int offset, boolean isUpper) {
if (query.length < (offset + 6)) {
return false;
}

if ( isUpper && query[offset] == 'V' && query[offset + 1] == 'A' && query[offset + 2] == 'L' && query[offset + 3] == 'U' && query[offset + 4] == 'E' && query[offset + 5] == 'S' ) {
return true;
} else if ( !isUpper && query[offset] == 'v' && query[offset + 1] == 'a' && query[offset + 2] == 'l' && query[offset + 3] == 'u' && query[offset + 4] == 'e' && query[offset + 5] == 's' ) {
return true;
}
return false;
}

/**
* @param c character
* @return true if the character is a whitespace character as defined in the backend's parser
@@ -51,4 +51,21 @@
boolean isStatementDescribed();

boolean isEmpty();

/**
* Convenience to check if the Query has an insert statement
* that can be re-written.
*/
boolean isStatementReWritableInsert();

/**
* Increment the number of batched rows for this Query.
*/
void incrementBatchSize();

/**
* Get the number of times this Query has been batched.
* @return number of times <code>addBatch()</code> has been called.
*/
int getBatchSize();
}
@@ -182,5 +182,38 @@ public void setBinaryParameter(int index, byte[] value, int oid) {
}

private final Object[] paramValues;

@Override
public Object[] getValues() {
return this.paramValues;
}

@Override
/**
* Replace all parameters with new values in provided list.
*/
public void addAll(ParameterList list) {
if (list instanceof SimpleParameterList ) {
// only v2.SimpleParameterList is compatible with this type
SimpleParameterList spl = (SimpleParameterList) list;
Arrays.fill(paramValues, null);
System.arraycopy(spl.getValues(), 0, paramValues, 0,
spl.getInParameterCount());
}
}

@Override
/**
* Append parameters to the list.
*/
public void appendAll(ParameterList list) {
if (list instanceof SimpleParameterList ) {
// only v2.SimpleParameterList is compatible with this type
SimpleParameterList spl = (SimpleParameterList) list;
int count = spl.getInParameterCount();
System.arraycopy(spl.getValues(), 0, paramValues,
getInParameterCount() - count, count);
}
}
}

@@ -208,6 +208,44 @@ public void setBinaryParameter(int index, byte[] value, int oid) {
throw new UnsupportedOperationException();
}

@Override
public Object[] getValues() {
return paramValues;
}

@Override
/**
* Replace all parameters with new values in provided list.
*/
public void addAll(ParameterList list) {
if (list instanceof SimpleParameterList ) {
/* only v2.SimpleParameterList is compatible with this type. The list
of parameters is expected to already be cloned from the values
passed by application. */
SimpleParameterList spl = (SimpleParameterList) list;
Arrays.fill(paramValues, null);
System.arraycopy(spl.getValues(), 0, paramValues, 0,
spl.getInParameterCount());
}
}

@Override
/**
* Append parameters to the list.
*/
public void appendAll(ParameterList list) {
if (list instanceof SimpleParameterList ) {
/* only v2.SimpleParameterList is compatible with this type. The list
of parameters is expected to already be cloned from the values
passed by application. */
SimpleParameterList spl = (SimpleParameterList) list;
int count = spl.getInParameterCount();
System.arraycopy(spl.getValues(), 0, paramValues,
getInParameterCount() - count,
count);
}
}

private final Object[] paramValues;

private final boolean useEStringSyntax;
@@ -60,10 +60,33 @@ public boolean isEmpty() {
return nativeQuery.nativeSql.isEmpty();
}

@Override
public boolean isStatementReWritableInsert() {
return statementReWritableInsert;
}

public void setStatementReWritableInsert(boolean canReWrite) {
statementReWritableInsert = canReWrite;
}

@Override
public void incrementBatchSize() {
batchSize += 1;
}

@Override
public int getBatchSize() {
return batchSize;
}

private static final ParameterList NO_PARAMETERS = new SimpleParameterList(0, false);

private final NativeQuery nativeQuery;

private final boolean useEStringSyntax; // whether escaped string syntax should be used

private boolean statementReWritableInsert;

private int batchSize = 0;
}

0 comments on commit ac8abf3

Please sign in to comment.
You can’t perform that action at this time.