Skip to content
Permalink
Browse files

[WFS provider] Fix excessive memory consumption on big layers (refs #…

…1870)

There was a pseudo memory leak in qgsgml.cpp
And the WFS downloader could also have to process big replies, causing a
lot of features to be instanciated at once.

Was seen on the 'portugal_addresses' layer of http://www.naturalgis.pt/cgi-bin/opendata/mapserv?
  • Loading branch information
rouault committed May 21, 2018
1 parent 19b3325 commit 0addae5b783e541953522eabe178426e820bf852
Showing with 17 additions and 3 deletions.
  1. +1 −1 src/core/qgsgml.cpp
  2. +11 −2 src/providers/wfs/qgswfsfeatureiterator.cpp
  3. +5 −0 src/providers/wfs/qgswfsrequest.cpp
@@ -844,7 +844,7 @@ void QgsGmlStreamingParser::endElement( const XML_Char *el )
const int localNameLen = ( pszSep ) ? ( int )( elLen - nsLen ) - 1 : elLen;
ParseMode parseMode( mParseModeStack.isEmpty() ? None : mParseModeStack.top() );

mDimension = mDimensionStack.isEmpty() ? 0 : mDimensionStack.top();
mDimension = mDimensionStack.isEmpty() ? 0 : mDimensionStack.pop();

const bool isGMLNS = ( nsLen == mGMLNameSpaceURI.size() && mGMLNameSpaceURIPtr && memcmp( el, mGMLNameSpaceURIPtr, nsLen ) == 0 );

@@ -456,6 +456,7 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
int pagingIter = 1;
QString gmlIdFirstFeatureFirstIter;
bool disablePaging = false;
// Top level loop to do feature paging in WFS 2
while ( true )
{
success = true;
@@ -476,9 +477,14 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
false /* cache */ );

int featureCountForThisResponse = 0;
bool bytesStillAvailableInReply = false;
// Loop until there is no data coming from the current requst
while ( true )
{
loop.exec( QEventLoop::ExcludeUserInputEvents );
if ( !bytesStillAvailableInReply )
{
loop.exec( QEventLoop::ExcludeUserInputEvents );
}
if ( mStop )
{
interrupted = true;
@@ -495,7 +501,10 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
bool finished = false;
if ( mReply )
{
data = mReply->readAll();
// Limit the number of bytes to process at once, to avoid the GML parser to
// create too many objects.
data = mReply->read( 10 * 1024 * 1024 );
bytesStillAvailableInReply = mReply->bytesAvailable() > 0;
}
else
{
@@ -25,6 +25,8 @@
#include <QNetworkCacheMetaData>
#include <QCryptographicHash> // just for testin file:// fake_qgis_http_endpoint hack

const qint64 READ_BUFFER_SIZE_HINT = 1024 * 1024;

QgsWfsRequest::QgsWfsRequest( const QgsWFSDataSourceURI &uri )
: mUri( uri )
, mErrorCode( QgsWfsRequest::NoError )
@@ -125,6 +127,7 @@ bool QgsWfsRequest::sendGET( const QUrl &url, bool synchronous, bool forceRefres
}

mReply = QgsNetworkAccessManager::instance()->get( request );
mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
if ( !mUri.auth().setAuthorizationReply( mReply ) )
{
mErrorCode = QgsWfsRequest::NetworkError;
@@ -176,6 +179,7 @@ bool QgsWfsRequest::sendPOST( const QUrl &url, const QString &contentTypeHeader,
request.setHeader( QNetworkRequest::ContentTypeHeader, contentTypeHeader );

mReply = QgsNetworkAccessManager::instance()->post( request, data );
mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
if ( !mUri.auth().setAuthorizationReply( mReply ) )
{
mErrorCode = QgsWfsRequest::NetworkError;
@@ -266,6 +270,7 @@ void QgsWfsRequest::replyFinished()

QgsDebugMsg( QString( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ) );
mReply = QgsNetworkAccessManager::instance()->get( request );
mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
if ( !mUri.auth().setAuthorizationReply( mReply ) )
{
mResponse.clear();

0 comments on commit 0addae5

Please sign in to comment.
You can’t perform that action at this time.