Skip to content

Commit

Permalink
Dynamically adjust postgres feature queue size
Browse files Browse the repository at this point in the history
Lower the default queue size, but automatically adjust it
based on how long each fetch takes. This change keeps fetching
responsive even for slow connections or databases. The current
approach with a fixed queue size can result in very slow feature
fetching, which prevents UI updates for multiple seconds.

(cherry-picked from fbe4be8)
  • Loading branch information
nyalldawson committed Mar 5, 2017
1 parent d79bb3a commit 37a7a22
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 7 deletions.
19 changes: 14 additions & 5 deletions src/providers/postgres/qgspostgresfeatureiterator.cpp
Expand Up @@ -22,16 +22,13 @@
#include "qgslogger.h"
#include "qgsmessagelog.h"

#include <QElapsedTimer>
#include <QObject>
#include <QSettings>


const int QgsPostgresFeatureIterator::sFeatureQueueSize = 2000;


QgsPostgresFeatureIterator::QgsPostgresFeatureIterator( QgsPostgresFeatureSource* source, bool ownSource, const QgsFeatureRequest& request )
: QgsAbstractFeatureIteratorFromSource<QgsPostgresFeatureSource>( source, ownSource, request )
, mFeatureQueueSize( sFeatureQueueSize )
, mFeatureQueueSize( 1 )
, mFetched( 0 )
, mFetchGeometry( false )
, mExpressionCompiled( false )
Expand Down Expand Up @@ -227,6 +224,9 @@ bool QgsPostgresFeatureIterator::fetchFeature( QgsFeature& feature )

if ( mFeatureQueue.empty() && !mLastFetch )
{
QElapsedTimer timer;
timer.start();

QString fetch = QString( "FETCH FORWARD %1 FROM %2" ).arg( mFeatureQueueSize ).arg( mCursorName );
QgsDebugMsgLevel( QString( "fetching %1 features." ).arg( mFeatureQueueSize ), 4 );

Expand Down Expand Up @@ -262,6 +262,15 @@ bool QgsPostgresFeatureIterator::fetchFeature( QgsFeature& feature )
} // for each row in queue
}
unlock();

if ( timer.elapsed() > 500 && mFeatureQueueSize > 1 )
{
mFeatureQueueSize /= 2;
}
else if ( timer.elapsed() < 50 && mFeatureQueueSize < 10000 )
{
mFeatureQueueSize *= 2;
}
}

if ( mFeatureQueue.empty() )
Expand Down
2 changes: 0 additions & 2 deletions src/providers/postgres/qgspostgresfeatureiterator.h
Expand Up @@ -118,8 +118,6 @@ class QgsPostgresFeatureIterator : public QgsAbstractFeatureIteratorFromSource<Q

bool mIsTransactionConnection;

static const int sFeatureQueueSize;

private:
//! returns whether the iterator supports simplify geometries on provider side
virtual bool providerCanSimplify( QgsSimplifyMethod::MethodType methodType ) const override;
Expand Down

0 comments on commit 37a7a22

Please sign in to comment.