Skip to content
Permalink
Browse files
feat: Add maxResultBuffer property (#1657)
* feat: Add maxResultBuffer property

Implementation of new property - maxResultBuffer. Enables max bytes read during reading result set.
Enable declare in two styles:
- as size of bytes (i.e. 100, 150M, 300K, 400G);
- as percent of max heap memory (i.e. 10p, 15pct, 20percent).
Also validates if defined size isn't bigger than enabled. Max possible size is 90% of max heap memory.
If maxResultBuffer property isn't declared, work of driver is not changed.

Commit made for Heimdalldata's request to solve memory problem during reading result set.

* feat: Add maxResultBuffer property

Add PGPropertyMaxResultBufferParser test cases.

* feat: Add maxResultBuffer property

Update of docs for maxResultBuffer property, and change in comments for javadoc to describe new exceptions.
  • Loading branch information
adrklos authored and davecramer committed Dec 30, 2019
1 parent 6331680 commit 557e2de462b0f52ddc0b151971a9aa6e2d553622
@@ -152,6 +152,7 @@ In addition to the standard connection parameters the driver supports a number o
| preferQueryMode | String | extended | Specifies which mode is used to execute queries to database, possible values: extended, extendedForPrepared, extendedCacheEverything, simple |
| reWriteBatchedInserts | Boolean | false | Enable optimization to rewrite and collapse compatible INSERT statements that are batched. |
| escapeSyntaxCallMode | String | select | Specifies how JDBC escape call syntax is transformed into underlying SQL (CALL/SELECT), for invoking procedures or functions (requires server version >= 11), possible values: select, callIfNoReturn, call |
| maxResultBuffer | String | null | Specifies size of result buffer in bytes, which can't be exceeded during reading result set. Can be specified as particular size (i.e. "100", "200M" "2G") or as percent of max heap memory (i.e. "10p", "20pct", "50percent") |

## Contributing
For information on how to contribute to the project see the [Contributing Guidelines](CONTRIBUTING.md)
@@ -482,7 +482,17 @@ Connection conn = DriverManager.getConnection(url);

The default is `select`

* **maxResultBuffer** = String

Specifies size of result buffer in bytes, which can't be exceeded during reading result set.
Property can be specified in two styles:
- as size of bytes (i.e. 100, 150M, 300K, 400G, 1T);
- as percent of max heap memory (i.e. 10p, 15pct, 20percent);

A limit during setting of property is 90% of max heap memory. All given values, which gonna be higher than limit, gonna lowered to the limit.

By default, maxResultBuffer is not set (is null), what means that reading of results gonna be performed without limits.

<a name="unix sockets"></a>
## Unix sockets

@@ -476,7 +476,14 @@ public enum PGProperty {
+ "When 'transaction' setting readOnly to 'true' will cause transactions to BEGIN READ ONLY if autocommit is 'false'. "
+ "When 'always' setting readOnly to 'true' will set the session to READ ONLY if autoCommit is 'true' "
+ "and the transaction to BEGIN READ ONLY if autocommit is 'false'.",
false, "ignore", "transaction", "always");
false, "ignore", "transaction", "always"),

/**
* Specifies size of buffer during fetching result set. Can be specified as specified size or
* percent of heap memory.
*/
MAX_RESULT_BUFFER("maxResultBuffer", null,
"Specifies size of buffer during fetching result set. Can be specified as specified size or percent of heap memory.");

private final String name;
private final String defaultValue;
@@ -8,6 +8,7 @@
import org.postgresql.util.ByteStreamWriter;
import org.postgresql.util.GT;
import org.postgresql.util.HostSpec;
import org.postgresql.util.PGPropertyMaxResultBufferParser;
import org.postgresql.util.PSQLException;
import org.postgresql.util.PSQLState;

@@ -54,6 +55,9 @@ public class PGStream implements Closeable, Flushable {
private Encoding encoding;
private Writer encodingWriter;

private long maxResultBuffer = -1;
private long resultBufferByteCount = 0;

/**
* Constructor: Connect to the PostgreSQL back end and return a stream connection.
*
@@ -463,12 +467,16 @@ public String receiveString() throws IOException {
*
* @return tuple from the back end
* @throws IOException if a data I/O error occurs
* @throws SQLException if read more bytes than set maxResultBuffer
*/
public byte[][] receiveTupleV3() throws IOException, OutOfMemoryError {
receiveInteger4(); // MESSAGE SIZE
public byte[][] receiveTupleV3() throws IOException, OutOfMemoryError, SQLException {
int messageSize = receiveInteger4(); // MESSAGE SIZE
int nf = receiveInteger2();
//size = messageSize - 4 bytes of message size - 2 bytes of field count - 4 bytes for each column length
int dataToReadSize = messageSize - 4 - 2 - 4 * nf;
byte[][] answer = new byte[nf][];

increaseByteCounter(dataToReadSize);
OutOfMemoryError oom = null;
for (int i = 0; i < nf; ++i) {
int size = receiveInteger4();
@@ -619,4 +627,41 @@ public void setNetworkTimeout(int milliseconds) throws IOException {
public int getNetworkTimeout() throws IOException {
return connection.getSoTimeout();
}

/**
* Method to set MaxResultBuffer inside PGStream.
*
* @param value value of new max result buffer as string (cause we can expect % or chars to use
* multiplier)
* @throws PSQLException exception returned when occurred parsing problem.
*/
public void setMaxResultBuffer(String value) throws PSQLException {
maxResultBuffer = PGPropertyMaxResultBufferParser.parseProperty(value);
}

/**
* Method to clear count of byte buffer.
*/
public void clearResultBufferCount() {
resultBufferByteCount = 0;
}

/**
* Method to increase actual count of buffer. If buffer count is bigger than max result buffer
* limit, then gonna return an exception.
*
* @param value size of bytes to add to byte buffer.
* @throws SQLException exception returned when result buffer count is bigger than max result
* buffer.
*/
private void increaseByteCounter(long value) throws SQLException {
if (maxResultBuffer != -1) {
resultBufferByteCount += value;
if (resultBufferByteCount > maxResultBuffer) {
throw new PSQLException(GT.tr(
"Result set exceeded maxResultBuffer limit. Received: {0}; Current limit: {1}",
String.valueOf(resultBufferByteCount), String.valueOf(maxResultBuffer)),PSQLState.COMMUNICATION_ERROR);
}
}
}
}
@@ -97,6 +97,9 @@ private PGStream tryConnect(String user, String database,
newStream.getSocket().setSoTimeout(socketTimeout * 1000);
}

String maxResultBuffer = PGProperty.MAX_RESULT_BUFFER.get(info);
newStream.setMaxResultBuffer(maxResultBuffer);

// Enable TCP keep-alive probe if required.
boolean requireTCPKeepAlive = PGProperty.TCP_KEEP_ALIVE.getBoolean(info);
newStream.getSocket().setKeepAlive(requireTCPKeepAlive);
@@ -2204,8 +2204,9 @@ protected void processResults(ResultHandler handler, int flags) throws IOExcepti
new PSQLException(GT.tr("Ran out of memory retrieving query results."),
PSQLState.OUT_OF_MEMORY, oome));
}
} catch (SQLException e) {
handler.handleError(e);
}

if (!noResults) {
if (tuples == null) {
tuples = new ArrayList<byte[][]>();
@@ -2300,6 +2301,7 @@ protected void processResults(ResultHandler handler, int flags) throws IOExcepti
receiveRFQ();
if (!pendingExecuteQueue.isEmpty() && pendingExecuteQueue.peekFirst().asSimple) {
tuples = null;
pgStream.clearResultBufferCount();

ExecuteRequest executeRequest = pendingExecuteQueue.removeFirst();
// Simple queries might return several resultsets, thus we clear
@@ -1511,6 +1511,14 @@ public void setHideUnprivilegedObjects(boolean hideUnprivileged) {
PGProperty.HIDE_UNPRIVILEGED_OBJECTS.set(properties, hideUnprivileged);
}

public String getMaxResultBuffer() {
return PGProperty.MAX_RESULT_BUFFER.get(properties);
}

public void setMaxResultBuffer(String maxResultBuffer) {
PGProperty.MAX_RESULT_BUFFER.set(properties, maxResultBuffer);
}

//#if mvn.project.property.postgresql.jdbc.spec >= "JDBC4.1"
public java.util.logging.Logger getParentLogger() {
return Logger.getLogger("org.postgresql");
@@ -0,0 +1,225 @@
/*
* Copyright (c) 2019, PostgreSQL Global Development Group
* See the LICENSE file in the project root for more information.
*/

package org.postgresql.util;

import java.lang.management.ManagementFactory;
import java.util.logging.Level;
import java.util.logging.Logger;

public class PGPropertyMaxResultBufferParser {

private static final Logger LOGGER = Logger.getLogger(PGPropertyMaxResultBufferParser.class.getName());

private static final String[] PERCENT_PHRASES = new String[]{
"p",
"pct",
"percent"
};

/**
* Method to parse value of max result buffer size.
*
* @param value string containing size of bytes with optional multiplier (T, G, M or K) or percent
* value to declare max percent of heap memory to use.
* @return value of max result buffer size.
* @throws PSQLException Exception when given value can't be parsed.
*/
public static long parseProperty(String value) throws PSQLException {
long result = -1;
if (checkIfValueContainsPercent(value)) {
result = parseBytePercentValue(value);
} else if (checkIfValueExistsToBeParsed(value)) {
result = parseByteValue(value);
}
result = adjustResultSize(result);
return result;
}

/**
* Method to check if given value can contain percent declaration of size of max result buffer.
*
* @param value Value to check.
* @return Result if value contains percent.
*/
private static boolean checkIfValueContainsPercent(String value) {
return (value != null) && (getPercentPhraseLengthIfContains(value) != -1);
}

/**
* Method to get percent value of max result buffer size dependable on actual free memory. This
* method doesn't check other possibilities of value declaration.
*
* @param value string containing percent used to define max result buffer.
* @return percent value of max result buffer size.
* @throws PSQLException Exception when given value can't be parsed.
*/
private static long parseBytePercentValue(String value) throws PSQLException {
long result = -1;
int length;

if (checkIfValueExistsToBeParsed(value)) {
length = getPercentPhraseLengthIfContains(value);

if (length == -1) {
throwExceptionAboutParsingError(
"Received MaxResultBuffer parameter can't be parsed. Value received to parse: {0}",
value);
}

result = calculatePercentOfMemory(value, length);
}
return result;
}

/**
* Method to get length of percent phrase existing in given string, only if one of phrases exist
* on the length of string.
*
* @param valueToCheck String which is gonna be checked if contains percent phrase.
* @return Length of phrase inside string, returns -1 when no phrase found.
*/
private static int getPercentPhraseLengthIfContains(String valueToCheck) {
int result = -1;
for (String phrase : PERCENT_PHRASES) {
int indx = getPhraseLengthIfContains(valueToCheck, phrase);
if (indx != -1) {
result = indx;
}
}
return result;
}

/**
* Method to get length of given phrase in given string to check, method checks if phrase exist on
* the end of given string.
*
* @param valueToCheck String which gonna be checked if contains phrase.
* @param phrase Phrase to be looked for on the end of given string.
* @return Length of phrase inside string, returns -1 when phrase wasn't found.
*/
private static int getPhraseLengthIfContains(String valueToCheck, String phrase) {
int searchValueLength = phrase.length();

if (valueToCheck.length() > searchValueLength) {
String subValue = valueToCheck.substring(valueToCheck.length() - searchValueLength);
if (subValue.equals(phrase)) {
return searchValueLength;
}
}
return -1;
}

/**
* Method to calculate percent of given max heap memory.
*
* @param value String which contains percent + percent phrase which gonna be used
* during calculations.
* @param percentPhraseLength Length of percent phrase inside given value.
* @return Size of byte buffer based on percent of max heap memory.
*/
private static long calculatePercentOfMemory(String value, int percentPhraseLength) {
String realValue = value.substring(0, value.length() - percentPhraseLength);
double percent = Double.parseDouble(realValue) / 100;
long result = (long) (percent * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax());
return result;
}

/**
* Method to check if given value has any chars to be parsed.
*
* @param value Value to be checked.
* @return Result if value can be parsed.
*/
private static boolean checkIfValueExistsToBeParsed(String value) {
return value != null && value.length() != 0;
}

/**
* Method to get size based on given string value. String can contains just a number or number +
* multiplier sign (like T, G, M or K).
*
* @param value Given string to be parsed.
* @return Size based on given string.
* @throws PSQLException Exception when given value can't be parsed.
*/
private static long parseByteValue(String value) throws PSQLException {
long result = -1;
long multiplier = 1;
long mul = 1000;
String realValue;
char sign = value.charAt(value.length() - 1);

switch (sign) {

case 'T':
case 't':
multiplier *= mul;

case 'G':
case 'g':
multiplier *= mul;

case 'M':
case 'm':
multiplier *= mul;

case 'K':
case 'k':
multiplier *= mul;
realValue = value.substring(0, value.length() - 1);
result = Integer.parseInt(realValue) * multiplier;
break;

case '%':
return result;

default:
if (sign >= '0' && sign <= '9') {
result = Long.parseLong(value);
} else {
throwExceptionAboutParsingError(
"Received MaxResultBuffer parameter can't be parsed. Value received to parse: {0}",
value);
}
break;
}
return result;
}

/**
* Method to adjust result memory limit size. If given memory is larger than 90% of max heap
* memory then it gonna be reduced to 90% of max heap memory.
*
* @param value Size to be adjusted.
* @return Adjusted size (original size or 90% of max heap memory)
*/
private static long adjustResultSize(long value) {
if (value > 0.9 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) {
long newResult = (long) (0.9 * ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax());

LOGGER.log(Level.WARNING, GT.tr(
"WARNING! Required to allocate {0} bytes, which exceeded possible heap memory size. Assigned {1} bytes as limit.",
String.valueOf(value), String.valueOf(newResult)));

value = newResult;
}
return value;
}

/**
* Method to throw message for parsing MaxResultBuffer.
*
* @param message Message to be added to exception.
* @param values Values to be put inside exception message.
* @throws PSQLException Exception when given value can't be parsed.
*/
private static void throwExceptionAboutParsingError(String message, Object... values) throws PSQLException {
throw new PSQLException(GT.tr(
message,
values),
PSQLState.SYNTAX_ERROR);
}
}

0 comments on commit 557e2de

Please sign in to comment.