[FEATURE] Expression compiler for mssql provider

Also make mssql provider handle requests which cannot be executed
gracefully (was crashing before)

Author: nyalldawson

src/core/qgssqlexpressioncompiler.cpp

@@ -43,8 +43,10 @@ QString QgsSqlExpressionCompiler::quotedIdentifier( const QString& identifier )
   return quoted;
 }
 
-QString QgsSqlExpressionCompiler::quotedValue( const QVariant& value )
+QString QgsSqlExpressionCompiler::quotedValue( const QVariant& value, bool& ok )
 {
+  ok = true;
+
   if ( value.isNull() )
     return "NULL";
 
@@ -231,17 +233,18 @@ QgsSqlExpressionCompiler::Result QgsSqlExpressionCompiler::compileNode( const Qg
     case QgsExpression::ntLiteral:
     {
       const QgsExpression::NodeLiteral* n = static_cast<const QgsExpression::NodeLiteral*>( node );
+      bool ok = false;
       if ( mFlags.testFlag( CaseInsensitiveStringMatch ) && n->value().type() == QVariant::String )
       {
         // provider uses case insensitive matching, so if literal was a string then we only have a Partial compilation and need to
         // double check results using QGIS' expression engine
-        result = quotedValue( n->value() );
-        return Partial;
+        result = quotedValue( n->value(), ok );
+        return ok ? Partial : Fail;
       }
       else
       {
-        result = quotedValue( n->value() );
-        return Complete;
+        result = quotedValue( n->value(), ok );
+        return ok ? Complete : Fail;
       }
     }
 

src/core/qgssqlexpressioncompiler.h

@@ -78,9 +78,11 @@ class CORE_EXPORT QgsSqlExpressionCompiler
 
     /** Returns a quoted attribute value, in the format expected by the provider.
      * Derived classes should override this if special handling of attribute values is required.
+     * @param value value to quote
+     * @param ok wil be set to true if value can be compiled
      * @see quotedIdentifier()
      */
-    virtual QString quotedValue( const QVariant& value );
+    virtual QString quotedValue( const QVariant& value, bool &ok );
 
     /** Compiles an expression node and returns the result of the compilation.
      * @param node expression node to compile

src/providers/mssql/CMakeLists.txt

@@ -1,5 +1,25 @@
-SET (MSSQL_SRCS qgsmssqlprovider.cpp qgsmssqlgeometryparser.cpp qgsmssqlsourceselect.cpp qgsmssqltablemodel.cpp qgsmssqlnewconnection.cpp qgsmssqldataitems.cpp qgsmssqlfeatureiterator.cpp)
-SET (MSSQL_MOC_HDRS qgsmssqlprovider.h qgsmssqlsourceselect.h qgsmssqltablemodel.h qgsmssqlnewconnection.h qgsmssqldataitems.h)
+SET(MSSQL_SRCS
+  qgsmssqlprovider.cpp
+  qgsmssqlgeometryparser.cpp
+  qgsmssqlsourceselect.cpp
+  qgsmssqltablemodel.cpp
+  qgsmssqlnewconnection.cpp
+  qgsmssqldataitems.cpp
+  qgsmssqlexpressioncompiler.cpp
+  qgsmssqlfeatureiterator.cpp
+)
+
+SET(MSSQL_MOC_HDRS
+  qgsmssqlprovider.h
+  qgsmssqlsourceselect.h
+  qgsmssqltablemodel.h
+  qgsmssqlnewconnection.h
+  qgsmssqldataitems.h
+)
+
+SET(MSSQL_HDRS
+  qgsmssqlexpressioncompiler.h
+)
 
 ########################################################
 # Build
@@ -23,7 +43,7 @@ INCLUDE_DIRECTORIES(
 )
 
 
-ADD_LIBRARY(mssqlprovider MODULE ${MSSQL_SRCS} ${MSSQL_MOC_SRCS})
+ADD_LIBRARY(mssqlprovider MODULE ${MSSQL_SRCS} ${MSSQL_MOC_SRCS} ${MSSQL_HDRS})
 
 TARGET_LINK_LIBRARIES(mssqlprovider
   qgis_core

src/providers/mssql/qgsmssqlexpressioncompiler.cpp

@@ -0,0 +1,79 @@
+/***************************************************************************
+ qgsmssqlexpressioncompiler.cpp
+ ------------------------------
+ begin                : 9.12.2015
+ copyright            : (C) 2015 by Nyall Dawson
+ email                : nyall dot dawson at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsmssqlexpressioncompiler.h"
+
+QgsMssqlExpressionCompiler::QgsMssqlExpressionCompiler( QgsMssqlFeatureSource* source )
+    : QgsSqlExpressionCompiler( source->mFields,
+                                QgsSqlExpressionCompiler::LikeIsCaseInsensitive | QgsSqlExpressionCompiler::CaseInsensitiveStringMatch )
+{
+
+}
+
+QgsSqlExpressionCompiler::Result QgsMssqlExpressionCompiler::compileNode( const QgsExpression::Node* node, QString& result )
+{
+  if ( node->nodeType() == QgsExpression::ntBinaryOperator )
+  {
+    const QgsExpression::NodeBinaryOperator *bin( static_cast<const QgsExpression::NodeBinaryOperator*>( node ) );
+    QString op1, op2;
+
+    Result result1 = compileNode( bin->opLeft(), op1 );
+    Result result2 = compileNode( bin->opRight(), op2 );
+    if ( result1 == Fail || result2 == Fail )
+      return Fail;
+
+    switch ( bin->op() )
+    {
+      case QgsExpression::boPow:
+        result = QString( "power(%1,%2)" ).arg( op1, op2 );
+        return result1 == Partial || result2 == Partial ? Partial : Complete;
+
+      case QgsExpression::boRegexp:
+        return Fail; //not supported, regexp syntax is too different to Qt
+
+      case QgsExpression::boConcat:
+        result = QString( "%1 + %2" ).arg( op1, op2 );
+        return result1 == Partial || result2 == Partial ? Partial : Complete;
+
+      default:
+        break;
+    }
+
+  }
+
+  //fallback to default handling
+  return QgsSqlExpressionCompiler::compileNode( node, result );
+}
+
+QString QgsMssqlExpressionCompiler::quotedValue( const QVariant& value, bool& ok )
+{
+  ok = true;
+  if ( value.isNull() )
+  {
+    //no NULL literal support
+    ok = false;
+    return QString();
+  }
+
+  switch ( value.type() )
+  {
+    case QVariant::Bool:
+      //no boolean literal support in mssql, so fake it
+      return value.toBool() ? "(1=1)" : "(1=0)";
+
+    default:
+      return QgsSqlExpressionCompiler::quotedValue( value, ok );
+  }
+}

src/providers/mssql/qgsmssqlexpressioncompiler.h

@@ -0,0 +1,35 @@
+/***************************************************************************
+ qgsmssqlexpressioncompiler.h
+ ----------------------------
+ begin                : 9.12.2015
+ copyright            : (C) 2015 by Nyall Dawson
+ email                : nyall dot dawson at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSMSSQLEXPRESSIONCOMPILER_H
+#define QGSMSSQLEXPRESSIONCOMPILER_H
+
+#include "qgssqlexpressioncompiler.h"
+#include "qgsexpression.h"
+#include "qgsmssqlfeatureiterator.h"
+
+class QgsMssqlExpressionCompiler : public QgsSqlExpressionCompiler
+{
+  public:
+
+    explicit QgsMssqlExpressionCompiler( QgsMssqlFeatureSource* source );
+
+  protected:
+    virtual Result compileNode( const QgsExpression::Node* node, QString& result ) override;
+    virtual QString quotedValue( const QVariant& value, bool& ok ) override;
+
+};
+
+#endif // QGSMSSQLEXPRESSIONCOMPILER_H

src/providers/mssql/qgsmssqlfeatureiterator.cpp

@@ -16,16 +16,19 @@
  ***************************************************************************/
 
 #include "qgsmssqlfeatureiterator.h"
+#include "qgsmssqlexpressioncompiler.h"
 #include "qgsmssqlprovider.h"
 #include "qgslogger.h"
 
 #include <QObject>
 #include <QTextStream>
 #include <QSqlRecord>
+#include <QSettings>
 
 
 QgsMssqlFeatureIterator::QgsMssqlFeatureIterator( QgsMssqlFeatureSource* source, bool ownSource, const QgsFeatureRequest& request )
     : QgsAbstractFeatureIteratorFromSource<QgsMssqlFeatureSource>( source, ownSource, request )
+    , mExpressionCompiled( false )
 {
   mClosed = false;
   mQuery = NULL;
@@ -60,12 +63,14 @@ QgsMssqlFeatureIterator::~QgsMssqlFeatureIterator()
 
 void QgsMssqlFeatureIterator::BuildStatement( const QgsFeatureRequest& request )
 {
-  // build sql statement
-  mStatement = QString( "SELECT " );
+  mFallbackStatement.clear();
+  mStatement.clear();
+
+  bool limitAtProvider = ( mRequest.limit() >= 0 );
 
-  if ( request.limit() >= 0 && request.filterType() != QgsFeatureRequest::FilterExpression )
-    mStatement += QString( "TOP %1 " ).arg( mRequest.limit() );
+  // build sql statement
 
+  // note: 'SELECT ' is added later, to account for 'SELECT TOP...' type queries
   mStatement += QString( "[%1]" ).arg( mSource->mFidColName );
   mFidCol = mSource->mFields.indexFromName( mSource->mFidColName );
   mAttributesToFetch.append( mFidCol );
@@ -131,6 +136,52 @@ void QgsMssqlFeatureIterator::BuildStatement( const QgsFeatureRequest& request )
       mStatement += " WHERE (" + mSource->mSqlWhereClause + ')';
     else
       mStatement += " AND (" + mSource->mSqlWhereClause + ')';
+    filterAdded = true;
+  }
+
+  //NOTE - must be last added!
+  mExpressionCompiled = false;
+  if ( request.filterType() == QgsFeatureRequest::FilterExpression )
+  {
+    if ( QSettings().value( "/qgis/compileExpressions", true ).toBool() )
+    {
+      QgsMssqlExpressionCompiler compiler = QgsMssqlExpressionCompiler( mSource );
+      QgsSqlExpressionCompiler::Result result = compiler.compile( request.filterExpression() );
+      if ( result == QgsSqlExpressionCompiler::Complete || result == QgsSqlExpressionCompiler::Partial )
+      {
+        mFallbackStatement = mStatement;
+        if ( !filterAdded )
+          mStatement += " WHERE (" + compiler.result() + ')';
+        else
+          mStatement += " AND (" + compiler.result() + ')';
+        filterAdded = true;
+
+        //if only partial success when compiling expression, we need to double-check results using QGIS' expressions
+        mExpressionCompiled = ( result == QgsSqlExpressionCompiler::Complete );
+        limitAtProvider = mExpressionCompiled;
+      }
+      else
+      {
+        limitAtProvider = false;
+      }
+    }
+    else
+    {
+      limitAtProvider = false;
+    }
+  }
+
+  if ( request.limit() >= 0 && limitAtProvider )
+  {
+    mStatement.prepend( QString( "SELECT TOP %1 " ).arg( mRequest.limit() ) );
+    if ( !mFallbackStatement.isEmpty() )
+      mFallbackStatement.prepend( QString( "SELECT TOP %1 " ).arg( mRequest.limit() ) );
+  }
+  else
+  {
+    mStatement.prepend( "SELECT " );
+    if ( !mFallbackStatement.isEmpty() )
+      mFallbackStatement.prepend( "SELECT " );
   }
 
   QgsDebugMsg( mStatement );
@@ -188,6 +239,13 @@ bool QgsMssqlFeatureIterator::fetchFeature( QgsFeature& feature )
   return false;
 }
 
+bool QgsMssqlFeatureIterator::nextFeatureFilterExpression( QgsFeature& f )
+{
+  if ( !mExpressionCompiled )
+    return QgsAbstractFeatureIterator::nextFeatureFilterExpression( f );
+  else
+    return fetchFeature( f );
+}
 
 bool QgsMssqlFeatureIterator::rewind()
 {
@@ -205,10 +263,28 @@ bool QgsMssqlFeatureIterator::rewind()
 
   mQuery->clear();
   mQuery->setForwardOnly( true );
-  if ( !mQuery->exec( mStatement ) )
+
+  bool result = mQuery->exec( mStatement );
+  if ( !result && !mFallbackStatement.isEmpty() )
+  {
+    //try with fallback statement
+    result = mQuery->exec( mFallbackStatement );
+    mExpressionCompiled = false;
+  }
+
+  if ( !result )
   {
     QString msg = mQuery->lastError().text();
     QgsDebugMsg( msg );
+    delete mQuery;
+    mQuery = 0;
+    if ( mDatabase.isOpen() )
+      mDatabase.close();
+
+    iteratorClosed();
+
+    mClosed = true;
+    return false;
   }
 
   return true;
@@ -231,7 +307,10 @@ bool QgsMssqlFeatureIterator::close()
   }
 
   if ( mQuery )
+  {
     delete mQuery;
+    mQuery = 0;
+  }
 
   if ( mDatabase.isOpen() )
     mDatabase.close();

src/providers/mssql/qgsmssqlfeatureiterator.h

@@ -65,6 +65,7 @@ class QgsMssqlFeatureSource : public QgsAbstractFeatureSource
     bool isSpatial() { return !mGeometryColName.isEmpty() || !mGeometryColType.isEmpty(); }
 
     friend class QgsMssqlFeatureIterator;
+    friend class QgsMssqlExpressionCompiler;
 };
 
 class QgsMssqlFeatureIterator : public QgsAbstractFeatureIteratorFromSource<QgsMssqlFeatureSource>
@@ -87,6 +88,9 @@ class QgsMssqlFeatureIterator : public QgsAbstractFeatureIteratorFromSource<QgsM
     //! fetch next feature, return true on success
     virtual bool fetchFeature( QgsFeature& feature ) override;
 
+    //! fetch next feature filter expression
+    bool nextFeatureFilterExpression( QgsFeature& f ) override;
+
     // The current database
     QSqlDatabase mDatabase;
 
@@ -96,6 +100,8 @@ class QgsMssqlFeatureIterator : public QgsAbstractFeatureIteratorFromSource<QgsM
     // The current sql statement
     QString mStatement;
 
+    QString mFallbackStatement;
+
     // Field index of FID column
     long mFidCol;
 
@@ -104,6 +110,8 @@ class QgsMssqlFeatureIterator : public QgsAbstractFeatureIteratorFromSource<QgsM
 
     // for parsing sql geometries
     QgsMssqlGeometryParser mParser;
+
+    bool mExpressionCompiled;
 };
 
 #endif // QGSMSSQLFEATUREITERATOR_H

src/providers/ogr/qgsogrexpressioncompiler.cpp

@@ -89,7 +89,7 @@ QString QgsOgrExpressionCompiler::quotedIdentifier( const QString& identifier )
   return mSource->mProvider->quotedIdentifier( identifier.toUtf8() );
 }
 
-QString QgsOgrExpressionCompiler::quotedValue( const QVariant& value )
+QString QgsOgrExpressionCompiler::quotedValue( const QVariant& value, bool& )
 {
   return QgsOgrUtils::quotedValue( value );
 }