Skip to content
Permalink
Browse files

When we are compiling expressions to handoff to backend providers,

check if a node has already been determined to evaluate to a static,
precalculated value, and if so, use this value for the node instead
of attempting to compile the actual contents of the node itself

If we are certain that a node is static and will never change,
the this potentially allows us to short-cut a large part of the
filter expressions content. We already use this short-cut when
evaluating expressions on the QGIS side since years, and its
proven to be stable and reliable. By respecting this during
expression compilation we can offer a huge speed up to certain
filter expressions, especially those which utilise QGIS variables
which are known to be static (such as atlas variables, map scales,
etc). Previously ANY use of a qgis variable would always cause
expression compilation to fail and require a full set of feature
fetching from the provider.

(Resulted in orders of magnitude faster atlas export for a complex
atlas.)
  • Loading branch information
nyalldawson committed Feb 11, 2021
1 parent 0ddff71 commit 8fa29ac6d2166a2623f0c2019203a55f526c5fcf
Showing with 154 additions and 47 deletions.
  1. +18 −0 python/core/auto_generated/expression/qgsexpressionnode.sip.in
  2. +2 −1 python/core/auto_generated/qgsfeaturerequest.sip.in
  3. +16 −0 src/core/expression/qgsexpressionnode.h
  4. +6 −2 src/core/providers/ogr/qgsogrexpressioncompiler.cpp
  5. +1 −1 src/core/providers/ogr/qgsogrexpressioncompiler.h
  6. +2 −2 src/core/providers/ogr/qgsogrfeatureiterator.cpp
  7. +2 −1 src/core/qgsfeaturerequest.h
  8. +30 −1 src/core/qgssqlexpressioncompiler.cpp
  9. +13 −1 src/core/qgssqlexpressioncompiler.h
  10. +6 −2 src/core/qgssqliteexpressioncompiler.cpp
  11. +4 −1 src/core/qgssqliteexpressioncompiler.h
  12. +6 −3 src/providers/db2/qgsdb2expressioncompiler.cpp
  13. +1 −1 src/providers/db2/qgsdb2expressioncompiler.h
  14. +2 −2 src/providers/db2/qgsdb2featureiterator.cpp
  15. +6 −2 src/providers/hana/qgshanaexpressioncompiler.cpp
  16. +1 −1 src/providers/hana/qgshanaexpressioncompiler.h
  17. +1 −1 src/providers/hana/qgshanafeatureiterator.cpp
  18. +6 −2 src/providers/mssql/qgsmssqlexpressioncompiler.cpp
  19. +1 −1 src/providers/mssql/qgsmssqlexpressioncompiler.h
  20. +2 −2 src/providers/mssql/qgsmssqlfeatureiterator.cpp
  21. +6 −2 src/providers/oracle/qgsoracleexpressioncompiler.cpp
  22. +1 −1 src/providers/oracle/qgsoracleexpressioncompiler.h
  23. +1 −1 src/providers/oracle/qgsoraclefeatureiterator.cpp
  24. +6 −2 src/providers/postgres/qgspostgresexpressioncompiler.cpp
  25. +1 −1 src/providers/postgres/qgspostgresexpressioncompiler.h
  26. +1 −1 src/providers/postgres/qgspostgresfeatureiterator.cpp
  27. +2 −2 src/providers/spatialite/qgsspatialitefeatureiterator.cpp
  28. +1 −1 tests/src/core/testqgssqliteexpressioncompiler.cpp
  29. +5 −5 tests/src/python/featuresourcetestbase.py
  30. +4 −4 tests/src/python/providertestbase.py
@@ -242,6 +242,24 @@ work like for example resolving a column name to an attribute index.

int parserLastColumn;

bool hasCachedStaticValue() const;
%Docstring
Returns ``True`` if the node can be replaced by a static cached value.

.. seealso:: :py:func:`cachedStaticValue`

.. versionadded:: 3.18
%End

QVariant cachedStaticValue() const;
%Docstring
Returns the node's static cached value. Only valid if :py:func:`~QgsExpressionNode.hasCachedStaticValue` is ``True``.

.. seealso:: :py:func:`hasCachedStaticValue`

.. versionadded:: 3.18
%End

protected:


@@ -63,7 +63,8 @@ Examples:
NoFlags,
NoGeometry,
SubsetOfAttributes,
ExactIntersect
ExactIntersect,
IgnoreStaticNodesDuringExpressionCompilation,
};
typedef QFlags<QgsFeatureRequest::Flag> Flags;

@@ -299,6 +299,22 @@ class CORE_EXPORT QgsExpressionNode SIP_ABSTRACT
*/
int parserLastColumn = 0;

/**
* Returns TRUE if the node can be replaced by a static cached value.
*
* \see cachedStaticValue()
* \since QGIS 3.18
*/
bool hasCachedStaticValue() const { return mHasCachedValue; }

/**
* Returns the node's static cached value. Only valid if hasCachedStaticValue() is TRUE.
*
* \see hasCachedStaticValue()
* \since QGIS 3.18
*/
QVariant cachedStaticValue() const { return mCachedStaticValue; }

protected:

/**
@@ -19,9 +19,9 @@
#include "qgsexpressionnodeimpl.h"
#include "qgsogrprovider.h"

QgsOgrExpressionCompiler::QgsOgrExpressionCompiler( QgsOgrFeatureSource *source )
QgsOgrExpressionCompiler::QgsOgrExpressionCompiler( QgsOgrFeatureSource *source, bool ignoreStaticNodes )
: QgsSqlExpressionCompiler( source->mFields, QgsSqlExpressionCompiler::CaseInsensitiveStringMatch | QgsSqlExpressionCompiler::NoNullInBooleanLogic
| QgsSqlExpressionCompiler::NoUnaryMinus | QgsSqlExpressionCompiler::IntegerDivisionResultsInInteger )
| QgsSqlExpressionCompiler::NoUnaryMinus | QgsSqlExpressionCompiler::IntegerDivisionResultsInInteger, ignoreStaticNodes )
, mSource( source )
{
}
@@ -50,6 +50,10 @@ QgsSqlExpressionCompiler::Result QgsOgrExpressionCompiler::compile( const QgsExp

QgsSqlExpressionCompiler::Result QgsOgrExpressionCompiler::compileNode( const QgsExpressionNode *node, QString &result )
{
QgsSqlExpressionCompiler::Result staticRes = replaceNodeByStaticCachedValueIfPossible( node, result );
if ( staticRes != Fail )
return staticRes;

switch ( node->nodeType() )
{
case QgsExpressionNode::ntBinaryOperator:
@@ -30,7 +30,7 @@ class QgsOgrExpressionCompiler : public QgsSqlExpressionCompiler
{
public:

explicit QgsOgrExpressionCompiler( QgsOgrFeatureSource *source );
explicit QgsOgrExpressionCompiler( QgsOgrFeatureSource *source, bool ignoreStaticNodes = false );

Result compile( const QgsExpression *exp ) override;

@@ -201,11 +201,11 @@ QgsOgrFeatureIterator::QgsOgrFeatureIterator( QgsOgrFeatureSource *source, bool
QgsSqlExpressionCompiler *compiler = nullptr;
if ( source->mDriverName == QLatin1String( "SQLite" ) || source->mDriverName == QLatin1String( "GPKG" ) )
{
compiler = new QgsSQLiteExpressionCompiler( source->mFields );
compiler = new QgsSQLiteExpressionCompiler( source->mFields, request.flags() & QgsFeatureRequest::IgnoreStaticNodesDuringExpressionCompilation );
}
else
{
compiler = new QgsOgrExpressionCompiler( source );
compiler = new QgsOgrExpressionCompiler( source, request.flags() & QgsFeatureRequest::IgnoreStaticNodesDuringExpressionCompilation );
}

QgsSqlExpressionCompiler::Result result = compiler->compile( request.filterExpression() );
@@ -80,7 +80,8 @@ class CORE_EXPORT QgsFeatureRequest
NoFlags = 0,
NoGeometry = 1, //!< Geometry is not required. It may still be returned if e.g. required for a filter condition.
SubsetOfAttributes = 2, //!< Fetch only a subset of attributes (setSubsetOfAttributes sets this flag)
ExactIntersect = 4 //!< Use exact geometry intersection (slower) instead of bounding boxes
ExactIntersect = 4, //!< Use exact geometry intersection (slower) instead of bounding boxes
IgnoreStaticNodesDuringExpressionCompilation = 8, //!< If a feature request uses a filter expression which can be partially precalculated due to static nodes in the expression, setting this flag will prevent these precalculated values from being utilized during compilation of the filter for the backend provider. This flag significantly slows down feature requests and should be used for debugging purposes only. (Since QGIS 3.18)
};
Q_DECLARE_FLAGS( Flags, Flag )

@@ -18,10 +18,11 @@
#include "qgsexpressionfunction.h"
#include "qgsexpression.h"

QgsSqlExpressionCompiler::QgsSqlExpressionCompiler( const QgsFields &fields, Flags flags )
QgsSqlExpressionCompiler::QgsSqlExpressionCompiler( const QgsFields &fields, Flags flags, bool ignoreStaticNodes )
: mResult( None )
, mFields( fields )
, mFlags( flags )
, mIgnoreStaticNodes( ignoreStaticNodes )
{
}

@@ -88,6 +89,10 @@ QString QgsSqlExpressionCompiler::quotedValue( const QVariant &value, bool &ok )

QgsSqlExpressionCompiler::Result QgsSqlExpressionCompiler::compileNode( const QgsExpressionNode *node, QString &result )
{
QgsSqlExpressionCompiler::Result staticRes = replaceNodeByStaticCachedValueIfPossible( node, result );
if ( staticRes != Fail )
return staticRes;

switch ( node->nodeType() )
{
case QgsExpressionNode::ntUnaryOperator:
@@ -440,6 +445,30 @@ QString QgsSqlExpressionCompiler::castToInt( const QString &value ) const
return QString();
}

QgsSqlExpressionCompiler::Result QgsSqlExpressionCompiler::replaceNodeByStaticCachedValueIfPossible( const QgsExpressionNode *node, QString &result )
{
if ( mIgnoreStaticNodes )
return Fail;

if ( node->hasCachedStaticValue() )
{
bool ok = false;
if ( mFlags.testFlag( CaseInsensitiveStringMatch ) && node->cachedStaticValue().type() == QVariant::String )
{
// provider uses case insensitive matching, so if literal was a string then we only have a Partial compilation and need to
// double check results using QGIS' expression engine
result = quotedValue( node->cachedStaticValue(), ok );
return ok ? Partial : Fail;
}
else
{
result = quotedValue( node->cachedStaticValue(), ok );
return ok ? Complete : Fail;
}
}
return Fail;
}

bool QgsSqlExpressionCompiler::nodeIsNullLiteral( const QgsExpressionNode *node ) const
{
if ( node->nodeType() != QgsExpressionNode::ntLiteral )
@@ -67,8 +67,11 @@ class CORE_EXPORT QgsSqlExpressionCompiler
* Constructor for expression compiler.
* \param fields fields from provider
* \param flags flags which control how expression is compiled
* \param ignoreStaticNodes If an expression has been partially precalculated due to static nodes in the expression, setting this argument to FALSE
* will prevent these precalculated values from being utilized during compilation of the expression. This flag significantly limits the effectiveness
* of compilation and should be used for debugging purposes only. (Since QGIS 3.18)
*/
explicit QgsSqlExpressionCompiler( const QgsFields &fields, QgsSqlExpressionCompiler::Flags flags = Flags() );
explicit QgsSqlExpressionCompiler( const QgsFields &fields, QgsSqlExpressionCompiler::Flags flags = Flags(), bool ignoreStaticNodes = false );
virtual ~QgsSqlExpressionCompiler() = default;

/**
@@ -173,13 +176,22 @@ class CORE_EXPORT QgsSqlExpressionCompiler
*/
virtual QString castToInt( const QString &value ) const;

/**
* Tries to replace a node by its static cached value where possible.
*
* \since QGIS 3.18
*/
virtual Result replaceNodeByStaticCachedValueIfPossible( const QgsExpressionNode *node, QString &str );

QString mResult;
QgsFields mFields;

private:

Flags mFlags;

bool mIgnoreStaticNodes = false;

bool nodeIsNullLiteral( const QgsExpressionNode *node ) const;

};
@@ -22,13 +22,17 @@
#include "qgsexpression.h"
#include "qgssqliteutils.h"

QgsSQLiteExpressionCompiler::QgsSQLiteExpressionCompiler( const QgsFields &fields )
: QgsSqlExpressionCompiler( fields, QgsSqlExpressionCompiler::LikeIsCaseInsensitive | QgsSqlExpressionCompiler::IntegerDivisionResultsInInteger )
QgsSQLiteExpressionCompiler::QgsSQLiteExpressionCompiler( const QgsFields &fields, bool ignoreStaticNodes )
: QgsSqlExpressionCompiler( fields, QgsSqlExpressionCompiler::LikeIsCaseInsensitive | QgsSqlExpressionCompiler::IntegerDivisionResultsInInteger, ignoreStaticNodes )
{
}

QgsSqlExpressionCompiler::Result QgsSQLiteExpressionCompiler::compileNode( const QgsExpressionNode *node, QString &result )
{
QgsSqlExpressionCompiler::Result staticRes = replaceNodeByStaticCachedValueIfPossible( node, result );
if ( staticRes != Fail )
return staticRes;

switch ( node->nodeType() )
{
case QgsExpressionNode::ntBinaryOperator:
@@ -41,8 +41,11 @@ class CORE_EXPORT QgsSQLiteExpressionCompiler : public QgsSqlExpressionCompiler
/**
* Constructor for expression compiler.
* \param fields fields from provider
* \param ignoreStaticNodes If an expression has been partially precalculated due to static nodes in the expression, setting this argument to FALSE
* will prevent these precalculated values from being utilized during compilation of the expression. This flag significantly limits the effectiveness
* of compilation and should be used for debugging purposes only. (Since QGIS 3.18)
*/
explicit QgsSQLiteExpressionCompiler( const QgsFields &fields );
explicit QgsSQLiteExpressionCompiler( const QgsFields &fields, bool ignoreStaticNodes = false );

protected:

@@ -20,9 +20,8 @@
#include "qgsexpressionnodeimpl.h"
#include "qgslogger.h"

QgsDb2ExpressionCompiler::QgsDb2ExpressionCompiler( QgsDb2FeatureSource *source )
: QgsSqlExpressionCompiler( source->mFields
)
QgsDb2ExpressionCompiler::QgsDb2ExpressionCompiler( QgsDb2FeatureSource *source, bool ignoreStaticNodes )
: QgsSqlExpressionCompiler( source->mFields, Flags(), ignoreStaticNodes )
{

}
@@ -54,6 +53,10 @@ QString resultType( QgsSqlExpressionCompiler::Result result )

QgsSqlExpressionCompiler::Result QgsDb2ExpressionCompiler::compileNode( const QgsExpressionNode *node, QString &result )
{
QgsSqlExpressionCompiler::Result staticRes = replaceNodeByStaticCachedValueIfPossible( node, result );
if ( staticRes != Fail )
return staticRes;

QgsDebugMsg( QStringLiteral( "nodeType: %1" ).arg( nodeType( node ) ) );
if ( node->nodeType() == QgsExpressionNode::ntColumnRef )
{
@@ -26,7 +26,7 @@ class QgsDb2ExpressionCompiler : public QgsSqlExpressionCompiler
{
public:

explicit QgsDb2ExpressionCompiler( QgsDb2FeatureSource *source );
explicit QgsDb2ExpressionCompiler( QgsDb2FeatureSource *source, bool ignoreStaticNodes = false );

protected:
Result compileNode( const QgsExpressionNode *node, QString &result ) override;
@@ -191,7 +191,7 @@ void QgsDb2FeatureIterator::BuildStatement( const QgsFeatureRequest &request )
mCompileStatus = NoCompilation;
if ( request.filterType() == QgsFeatureRequest::FilterExpression )
{
QgsDb2ExpressionCompiler compiler = QgsDb2ExpressionCompiler( mSource );
QgsDb2ExpressionCompiler compiler = QgsDb2ExpressionCompiler( mSource, request.flags() & QgsFeatureRequest::IgnoreStaticNodesDuringExpressionCompilation );
QgsDebugMsg( "expression dump: " + request.filterExpression()->dump() );
QgsDebugMsg( "expression expression: " + request.filterExpression()->expression() );
QgsSqlExpressionCompiler::Result result = compiler.compile( request.filterExpression() );
@@ -232,7 +232,7 @@ void QgsDb2FeatureIterator::BuildStatement( const QgsFeatureRequest &request )
break;
}

QgsDb2ExpressionCompiler compiler = QgsDb2ExpressionCompiler( mSource );
QgsDb2ExpressionCompiler compiler = QgsDb2ExpressionCompiler( mSource, request.flags() & QgsFeatureRequest::IgnoreStaticNodesDuringExpressionCompilation );
QgsExpression expression = clause.expression();
QgsDebugMsg( "expression: " + expression.dump() );
if ( compiler.compile( &expression ) == QgsSqlExpressionCompiler::Complete )
@@ -19,9 +19,9 @@
#include "qgshanautils.h"
#include "qgssqlexpressioncompiler.h"

QgsHanaExpressionCompiler::QgsHanaExpressionCompiler( QgsHanaFeatureSource *source )
QgsHanaExpressionCompiler::QgsHanaExpressionCompiler( QgsHanaFeatureSource *source, bool ignoreStaticNodes )
: QgsSqlExpressionCompiler( source->mFields, QgsSqlExpressionCompiler::IntegerDivisionResultsInInteger |
QgsSqlExpressionCompiler::NoNullInBooleanLogic )
QgsSqlExpressionCompiler::NoNullInBooleanLogic, ignoreStaticNodes )
, mGeometryColumn( source->mGeometryColumn )
{
}
@@ -90,6 +90,10 @@ QString QgsHanaExpressionCompiler::castToText( const QString &value ) const
QgsSqlExpressionCompiler::Result QgsHanaExpressionCompiler::compileNode(
const QgsExpressionNode *node, QString &result )
{
QgsSqlExpressionCompiler::Result staticRes = replaceNodeByStaticCachedValueIfPossible( node, result );
if ( staticRes != Fail )
return staticRes;

switch ( node->nodeType() )
{
case QgsExpressionNode::ntFunction:
@@ -24,7 +24,7 @@
class QgsHanaExpressionCompiler : public QgsSqlExpressionCompiler
{
public:
explicit QgsHanaExpressionCompiler( QgsHanaFeatureSource *source );
explicit QgsHanaExpressionCompiler( QgsHanaFeatureSource *source, bool ignoreStaticNodes = false );

protected:
QString quotedIdentifier( const QString &identifier ) override;
@@ -398,7 +398,7 @@ QString QgsHanaFeatureIterator::buildSqlQuery( const QgsFeatureRequest &request
{
if ( QgsSettings().value( QStringLiteral( "qgis/compileExpressions" ), true ).toBool() )
{
QgsHanaExpressionCompiler compiler = QgsHanaExpressionCompiler( mSource );
QgsHanaExpressionCompiler compiler = QgsHanaExpressionCompiler( mSource, request.flags() & QgsFeatureRequest::IgnoreStaticNodesDuringExpressionCompilation );
QgsSqlExpressionCompiler::Result result = compiler.compile( request.filterExpression() );
switch ( result )
{
@@ -16,16 +16,20 @@
#include "qgsmssqlexpressioncompiler.h"
#include "qgsexpressionnodeimpl.h"

QgsMssqlExpressionCompiler::QgsMssqlExpressionCompiler( QgsMssqlFeatureSource *source )
QgsMssqlExpressionCompiler::QgsMssqlExpressionCompiler( QgsMssqlFeatureSource *source, bool ignoreStaticNodes )
: QgsSqlExpressionCompiler( source->mFields,
QgsSqlExpressionCompiler::LikeIsCaseInsensitive |
QgsSqlExpressionCompiler::CaseInsensitiveStringMatch |
QgsSqlExpressionCompiler::IntegerDivisionResultsInInteger )
QgsSqlExpressionCompiler::IntegerDivisionResultsInInteger, ignoreStaticNodes )
{
}

QgsSqlExpressionCompiler::Result QgsMssqlExpressionCompiler::compileNode( const QgsExpressionNode *node, QString &result )
{
QgsSqlExpressionCompiler::Result staticRes = replaceNodeByStaticCachedValueIfPossible( node, result );
if ( staticRes != Fail )
return staticRes;

switch ( node->nodeType() )
{
case QgsExpressionNode::ntBinaryOperator:
@@ -24,7 +24,7 @@ class QgsMssqlExpressionCompiler : public QgsSqlExpressionCompiler
{
public:

explicit QgsMssqlExpressionCompiler( QgsMssqlFeatureSource *source );
explicit QgsMssqlExpressionCompiler( QgsMssqlFeatureSource *source, bool ignoreStaticNodes = false );

protected:
Result compileNode( const QgsExpressionNode *node, QString &result ) override;
@@ -330,7 +330,7 @@ void QgsMssqlFeatureIterator::BuildStatement( const QgsFeatureRequest &request )
mCompileStatus = NoCompilation;
if ( request.filterType() == QgsFeatureRequest::FilterExpression )
{
QgsMssqlExpressionCompiler compiler = QgsMssqlExpressionCompiler( mSource );
QgsMssqlExpressionCompiler compiler = QgsMssqlExpressionCompiler( mSource, request.flags() & QgsFeatureRequest::IgnoreStaticNodesDuringExpressionCompilation );
QgsSqlExpressionCompiler::Result result = compiler.compile( request.filterExpression() );
if ( result == QgsSqlExpressionCompiler::Complete || result == QgsSqlExpressionCompiler::Partial )
{
@@ -364,7 +364,7 @@ void QgsMssqlFeatureIterator::BuildStatement( const QgsFeatureRequest &request )
break;
}

QgsMssqlExpressionCompiler compiler = QgsMssqlExpressionCompiler( mSource );
QgsMssqlExpressionCompiler compiler = QgsMssqlExpressionCompiler( mSource, request.flags() & QgsFeatureRequest::IgnoreStaticNodesDuringExpressionCompilation );
QgsExpression expression = clause.expression();
if ( compiler.compile( &expression ) == QgsSqlExpressionCompiler::Complete )
{
@@ -17,13 +17,17 @@
#include "qgssqlexpressioncompiler.h"
#include "qgsexpressionnodeimpl.h"

QgsOracleExpressionCompiler::QgsOracleExpressionCompiler( QgsOracleFeatureSource *source )
: QgsSqlExpressionCompiler( source->mFields )
QgsOracleExpressionCompiler::QgsOracleExpressionCompiler( QgsOracleFeatureSource *source, bool ignoreStaticNodes )
: QgsSqlExpressionCompiler( source->mFields, Flags(), ignoreStaticNodes )
{
}

QgsSqlExpressionCompiler::Result QgsOracleExpressionCompiler::compileNode( const QgsExpressionNode *node, QString &result )
{
QgsSqlExpressionCompiler::Result staticRes = replaceNodeByStaticCachedValueIfPossible( node, result );
if ( staticRes != Fail )
return staticRes;

switch ( node->nodeType() )
{
case QgsExpressionNode::ntBinaryOperator:

0 comments on commit 8fa29ac

Please sign in to comment.