Skip to content
Permalink
Browse files

[WFS provider] Fix excessive memory consumption on big layers (refs #…

…18740)

There was a pseudo memory leak in qgsgml.cpp
And the WFS downloader could also have to process big replies, causing a
lot of features to be instanciated at once.

Was seen on the 'portugal_addresses' layer of http://www.naturalgis.pt/cgi-bin/opendata/mapserv?

Backport of
0addae5
c3c5c97
6cf1c50
  • Loading branch information
rouault committed May 21, 2018
1 parent e8e15d5 commit 1121565d05c27f5a8c32bb1f19ce1412b3607ad3
Showing with 17 additions and 3 deletions.
  1. +1 −1 src/core/qgsgml.cpp
  2. +11 −2 src/providers/wfs/qgswfsfeatureiterator.cpp
  3. +5 −0 src/providers/wfs/qgswfsrequest.cpp
@@ -856,7 +856,7 @@ void QgsGmlStreamingParser::endElement( const XML_Char* el )
const int localNameLen = ( pszSep ) ? ( int )( elLen - nsLen ) - 1 : elLen;
ParseMode theParseMode( mParseModeStack.isEmpty() ? none : mParseModeStack.top() );

mDimension = mDimensionStack.isEmpty() ? 0 : mDimensionStack.top() ;
mDimension = mDimensionStack.isEmpty() ? 0 : mDimensionStack.pop() ;

const bool isGMLNS = ( nsLen == mGMLNameSpaceURI.size() && mGMLNameSpaceURIPtr && memcmp( el, mGMLNameSpaceURIPtr, nsLen ) == 0 );

@@ -436,6 +436,7 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
int pagingIter = 1;
QString gmlIdFirstFeatureFirstIter;
bool disablePaging = false;
// Top level loop to do feature paging in WFS 2
while ( true )
{
success = true;
@@ -456,9 +457,14 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
false /* cache */ );

int featureCountForThisResponse = 0;
bool bytesStillAvailableInReply = false;
// Loop until there is no data coming from the current request
while ( true )
{
loop.exec( QEventLoop::ExcludeUserInputEvents );
if ( !bytesStillAvailableInReply )
{
loop.exec( QEventLoop::ExcludeUserInputEvents );
}
if ( mStop )
{
interrupted = true;
@@ -475,7 +481,10 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
bool finished = false;
if ( mReply )
{
data = mReply->readAll();
// Limit the number of bytes to process at once, to avoid the GML parser to
// create too many objects.
data = mReply->read( 10 * 1024 * 1024 );
bytesStillAvailableInReply = mReply->bytesAvailable() > 0;
}
else
{
@@ -24,6 +24,8 @@
#include <QNetworkCacheMetaData>
#include <QCryptographicHash> // just for testin file:// fake_qgis_http_endpoint hack

const qint64 READ_BUFFER_SIZE_HINT = 1024 * 1024;

QgsWFSRequest::QgsWFSRequest( const QString& theUri )
: mUri( theUri )
, mReply( nullptr )
@@ -116,6 +118,7 @@ bool QgsWFSRequest::sendGET( const QUrl& url, bool synchronous, bool forceRefres
}

mReply = QgsNetworkAccessManager::instance()->get( request );
mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
if ( !mUri.auth().setAuthorizationReply( mReply ) )
{
mErrorCode = QgsWFSRequest::NetworkError;
@@ -167,6 +170,7 @@ bool QgsWFSRequest::sendPOST( const QUrl& url, const QString& contentTypeHeader,
request.setHeader( QNetworkRequest::ContentTypeHeader, contentTypeHeader );

mReply = QgsNetworkAccessManager::instance()->post( request, data );
mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
if ( !mUri.auth().setAuthorizationReply( mReply ) )
{
mErrorCode = QgsWFSRequest::NetworkError;
@@ -257,6 +261,7 @@ void QgsWFSRequest::replyFinished()

QgsDebugMsg( QString( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ) );
mReply = QgsNetworkAccessManager::instance()->get( request );
mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
if ( !mUri.auth().setAuthorizationReply( mReply ) )
{
mResponse.clear();

0 comments on commit 1121565

Please sign in to comment.
You can’t perform that action at this time.