[FEATURE] [WFS provider] Add support for WFS 2.0 joins and other impr…

…ovements

Second part of qgis/QGIS-Enhancement-Proposals#53
(QEP 35: WFS provider enhancements)

- URI parameter with sql with SELECT / FROM / JOIN / WHERE / ORDER BY clauses
- handle WFS 2.0 joins
- handle DateTime fields
- enable "Only request features overlapping the view extent" by default (and memorize the settings)
- rework DescribeFeatureType parsing to handle responses with several documents, and some support for attribute types being complexType
- rework feature transfer between downloader and iterator so as to avoid uncontrolled RAM usage when the iterator cannot keep up with the downloader
- turn on WAL journaling for better reader / writer concurrency
- add retry logic based on the 'Max retry in case of tile request errors' setting (renamed 'Max retry in case of tile or feature request errors')
- error to MessageBar in case of failed download
- in progress dialog, add a "Hide" button to mask the dialog
- improve automated testing
- add testing of the GUI of QgsWFSSourceSelect

src/core/qgsvectorlayer.h

@@ -205,6 +205,7 @@ struct CORE_EXPORT QgsVectorJoinInfo
  * - password=string
  * - authcfg=string
  * - version=auto/1.0.0/1.1.0/2.0.0
+ *  -sql=string: full SELECT SQL statement with optional WHERE, ORDER BY and possibly with JOIN if supported on server
  * - filter=string: QGIS expression or OGC/FES filter
  * - retrictToRequestBBOX=1: to download only features in the view extent (or more generally
  *   in the bounding box of the feature iterator)

src/providers/wfs/qgswfscapabilities.cpp

@@ -16,6 +16,7 @@
  ***************************************************************************/
 #include "qgswfscapabilities.h"
 #include "qgswfsconstants.h"
+#include "qgswfsutils.h"
 #include "qgslogger.h"
 #include "qgsmessagelog.h"
 #include "qgsogcutils.h"
@@ -63,8 +64,23 @@ void QgsWFSCapabilities::Capabilities::clear()
   maxFeatures = 0;
   supportsHits = false;
   supportsPaging = false;
+  supportsJoins = false;
   version = "";
   featureTypes.clear();
+  spatialPredicatesList.clear();
+  functionList.clear();
+  setAllTypenames.clear();
+  mapUnprefixedTypenameToPrefixedTypename.clear();
+  setAmbiguousUnprefixedTypename.clear();
+}
+
+QString QgsWFSCapabilities::Capabilities::addPrefixIfNeeded( const QString& name ) const
+{
+  if ( name.contains( ':' ) )
+    return name;
+  if ( setAmbiguousUnprefixedTypename.contains( name ) )
+    return "";
+  return mapUnprefixedTypenameToPrefixedTypename[name];
 }
 
 void QgsWFSCapabilities::capabilitiesReplyFinished()
@@ -158,6 +174,16 @@ void QgsWFSCapabilities::capabilitiesReplyFinished()
           QgsDebugMsg( "Supports paging" );
         }
       }
+      else if ( contraint.attribute( "name" ) == "ImplementsStandardJoins" ||
+                contraint.attribute( "name" ) == "ImplementsSpatialJoins" /* WFS 2.0 */ )
+      {
+        QDomElement value = contraint.firstChildElement( "DefaultValue" );
+        if ( !value.isNull() && value.text() == "TRUE" )
+        {
+          mCaps.supportsJoins = true;
+          QgsDebugMsg( "Supports joins" );
+        }
+      }
     }
 
     // In WFS 2.0, max features can also be set in Operation.GetFeature (e.g. GeoServer)
@@ -323,6 +349,50 @@ void QgsWFSCapabilities::capabilitiesReplyFinished()
     mCaps.featureTypes.push_back( featureType );
   }
 
+  Q_FOREACH ( const FeatureType& f, mCaps.featureTypes )
+  {
+    mCaps.setAllTypenames.insert( f.name );
+    QString unprefixed( QgsWFSUtils::removeNamespacePrefix( f.name ) );
+    if ( !mCaps.setAmbiguousUnprefixedTypename.contains( unprefixed ) )
+    {
+      if ( mCaps.mapUnprefixedTypenameToPrefixedTypename.contains( unprefixed ) )
+      {
+        mCaps.setAmbiguousUnprefixedTypename.insert( unprefixed );
+        mCaps.mapUnprefixedTypenameToPrefixedTypename.remove( unprefixed );
+      }
+      else
+      {
+        mCaps.mapUnprefixedTypenameToPrefixedTypename[unprefixed] = f.name;
+      }
+    }
+  }
+
+  //go to <Filter_Capabilities>
+  QDomElement filterCapabilitiesElem = doc.firstChildElement( "Filter_Capabilities" );
+  if ( !filterCapabilitiesElem.isNull() )
+    parseFilterCapabilities( filterCapabilitiesElem );
+
+  // Hard-coded functions
+  Function f_ST_GeometryFromText( "ST_GeometryFromText", 1, 2 );
+  f_ST_GeometryFromText.returnType = "gml:AbstractGeometryType";
+  f_ST_GeometryFromText.argumentList << Argument( "wkt", "xs:string" );
+  f_ST_GeometryFromText.argumentList << Argument( "srsname", "xs:string" );
+  mCaps.functionList << f_ST_GeometryFromText;
+
+  Function f_ST_GeomFromGML( "ST_GeomFromGML", 1 );
+  f_ST_GeomFromGML.returnType = "gml:AbstractGeometryType";
+  f_ST_GeomFromGML.argumentList << Argument( "gml", "xs:string" );
+  mCaps.functionList << f_ST_GeomFromGML;
+
+  Function f_ST_MakeEnvelope( "ST_MakeEnvelope", 4, 5 );
+  f_ST_MakeEnvelope.returnType = "gml:AbstractGeometryType";
+  f_ST_MakeEnvelope.argumentList << Argument( "minx", "xs:double" );
+  f_ST_MakeEnvelope.argumentList << Argument( "miny", "xs:double" );
+  f_ST_MakeEnvelope.argumentList << Argument( "maxx", "xs:double" );
+  f_ST_MakeEnvelope.argumentList << Argument( "maxy", "xs:double" );
+  f_ST_MakeEnvelope.argumentList << Argument( "srsname", "xs:string" );
+  mCaps.functionList << f_ST_MakeEnvelope;
+
   emit gotCapabilities();
 }
 
@@ -404,6 +474,146 @@ void QgsWFSCapabilities::parseSupportedOperations( const QDomElement& operations
   }
 }
 
+static QgsWFSCapabilities::Function getSpatialPredicate( const QString& name )
+{
+  QgsWFSCapabilities::Function f;
+  // WFS 1.0 advertize Intersect, but for conveniency we internally convert it to Intersects
+  if ( name == "Intersect" )
+    f.name = "ST_Intersects";
+  else
+    f.name = ( name == "BBOX" ) ? "BBOX" : "ST_" + name;
+  f.returnType = "xs:boolean";
+  if ( name == "DWithin" || name == "Beyond" )
+  {
+    f.minArgs = 3;
+    f.maxArgs = 3;
+    f.argumentList << QgsWFSCapabilities::Argument( "geometry", "gml:AbstractGeometryType" );
+    f.argumentList << QgsWFSCapabilities::Argument( "geometry", "gml:AbstractGeometryType" );
+    f.argumentList << QgsWFSCapabilities::Argument( "distance" );
+  }
+  else
+  {
+    f.minArgs = 2;
+    f.maxArgs = 2;
+    f.argumentList << QgsWFSCapabilities::Argument( "geometry", "gml:AbstractGeometryType" );
+    f.argumentList << QgsWFSCapabilities::Argument( "geometry", "gml:AbstractGeometryType" );
+  }
+  return f;
+}
+
+void QgsWFSCapabilities::parseFilterCapabilities( const QDomElement& filterCapabilitiesElem )
+{
+  // WFS 1.0
+  QDomElement spatial_Operators = filterCapabilitiesElem.firstChildElement( "Spatial_Capabilities" ).firstChildElement( "Spatial_Operators" );
+  QDomElement spatial_Operator = spatial_Operators.firstChildElement();
+  while ( !spatial_Operator.isNull() )
+  {
+    QString name = spatial_Operator.tagName();
+    if ( !name.isEmpty() )
+    {
+      mCaps.spatialPredicatesList << getSpatialPredicate( name );
+    }
+    spatial_Operator = spatial_Operator.nextSiblingElement();
+  }
+
+  // WFS 1.1 and 2.0
+  QDomElement spatialOperators = filterCapabilitiesElem.firstChildElement( "Spatial_Capabilities" ).firstChildElement( "SpatialOperators" );
+  QDomElement spatialOperator = spatialOperators.firstChildElement( "SpatialOperator" );
+  while ( !spatialOperator.isNull() )
+  {
+    QString name = spatialOperator.attribute( "name" );
+    if ( !name.isEmpty() )
+    {
+      mCaps.spatialPredicatesList << getSpatialPredicate( name );
+    }
+    spatialOperator = spatialOperator.nextSiblingElement( "SpatialOperator" );
+  }
+
+  // WFS 1.0
+  QDomElement function_Names = filterCapabilitiesElem.firstChildElement( "Scalar_Capabilities" )
+                               .firstChildElement( "Arithmetic_Operators" )
+                               .firstChildElement( "Functions" )
+                               .firstChildElement( "Function_Names" );
+  QDomElement function_NameElem = function_Names.firstChildElement( "Function_Name" );
+  while ( !function_NameElem.isNull() )
+  {
+    Function f;
+    f.name = function_NameElem.text();
+    bool ok;
+    int nArgs = function_NameElem.attribute( "nArgs" ).toInt( &ok );
+    if ( ok )
+    {
+      if ( nArgs >= 0 )
+      {
+        f.minArgs = nArgs;
+        f.maxArgs = nArgs;
+      }
+      else
+      {
+        f.minArgs = -nArgs;
+      }
+    }
+    mCaps.functionList << f;
+    function_NameElem = function_NameElem.nextSiblingElement( "Function_Name" );
+  }
+
+  // WFS 1.1
+  QDomElement functionNames = filterCapabilitiesElem.firstChildElement( "Scalar_Capabilities" )
+                              .firstChildElement( "ArithmeticOperators" )
+                              .firstChildElement( "Functions" )
+                              .firstChildElement( "FunctionNames" );
+  QDomElement functionNameElem = functionNames.firstChildElement( "FunctionName" );
+  while ( !functionNameElem.isNull() )
+  {
+    Function f;
+    f.name = functionNameElem.text();
+    bool ok;
+    int nArgs = functionNameElem.attribute( "nArgs" ).toInt( &ok );
+    if ( ok )
+    {
+      if ( nArgs >= 0 )
+      {
+        f.minArgs = nArgs;
+        f.maxArgs = nArgs;
+      }
+      else
+      {
+        f.minArgs = -nArgs;
+      }
+    }
+    mCaps.functionList << f;
+    functionNameElem = functionNameElem.nextSiblingElement( "FunctionName" );
+  }
+
+  QDomElement functions = filterCapabilitiesElem.firstChildElement( "Functions" );
+  QDomElement functionElem = functions.firstChildElement( "Function" );
+  while ( !functionElem.isNull() )
+  {
+    QString name = functionElem.attribute( "name" );
+    if ( !name.isEmpty() )
+    {
+      Function f;
+      f.name = name;
+      QDomElement returnsElem = functionElem.firstChildElement( "Returns" );
+      f.returnType = returnsElem.text();
+      QDomElement argumentsElem = functionElem.firstChildElement( "Arguments" );
+      QDomElement argumentElem = argumentsElem.firstChildElement( "Argument" );
+      while ( !argumentElem.isNull() )
+      {
+        Argument arg;
+        arg.name = argumentElem.attribute( "name" );
+        arg.type = argumentElem.firstChildElement( "Type" ).text();
+        f.argumentList << arg;
+        argumentElem = argumentElem.nextSiblingElement( "Argument" );
+      }
+      f.minArgs = f.argumentList.count();
+      f.maxArgs = f.argumentList.count();
+      mCaps.functionList << f;
+    }
+    functionElem = functionElem.nextSiblingElement( "Function" );
+  }
+}
+
 QString QgsWFSCapabilities::errorMessageWithReason( const QString& reason )
 {
   return tr( "Download of capabilities failed: %1" ).arg( reason );

src/providers/wfs/qgswfscapabilities.h

@@ -45,17 +45,60 @@ class QgsWFSCapabilities : public QgsWFSRequest
       bool deleteCap;
     };
 
+    //! argument of a function
+    struct Argument
+    {
+      //! name
+      QString name;
+      //! type, or empty if unknown
+      QString type;
+
+      //! constructor
+      Argument( const QString& nameIn = QString(), const QString& typeIn = QString() ) : name( nameIn ), type( typeIn ) {}
+    };
+
+    //! description of server functions
+    struct Function
+    {
+      //! name
+      QString name;
+      //! return type, or empty if unknown
+      QString returnType;
+      //! minimum number of argument (or -1 if unknown)
+      int minArgs;
+      //! maximum number of argument (or -1 if unknown)
+      int maxArgs;
+      //! list of arguments. May be empty despite minArgs > 0
+      QList<Argument> argumentList;
+
+      //! constructor with name and fixed number of arguments
+      Function( const QString& nameIn, int args ) : name( nameIn ), minArgs( args ), maxArgs( args ) {}
+      //! constructor with name and min,max number of arguments
+      Function( const QString& nameIn, int minArgs, int maxArgsIn ) : name( nameIn ), minArgs( minArgs ), maxArgs( maxArgsIn ) {}
+      //! default constructor
+      Function() : minArgs( -1 ), maxArgs( -1 ) {}
+    };
+
     //! parsed get capabilities document
     struct Capabilities
     {
       Capabilities();
-      void clear();
 
       QString version;
       bool supportsHits;
       bool supportsPaging;
+      bool supportsJoins;
       int maxFeatures;
       QList<FeatureType> featureTypes;
+      QList<Function> spatialPredicatesList;
+      QList<Function> functionList;
+
+      QSet< QString > setAllTypenames;
+      QMap< QString, QString> mapUnprefixedTypenameToPrefixedTypename;
+      QSet< QString > setAmbiguousUnprefixedTypename;
+
+      void clear();
+      QString addPrefixIfNeeded( const QString& name ) const;
     };
 
     //! return parsed capabilities - requestCapabilities() must be called before
@@ -81,6 +124,8 @@ class QgsWFSCapabilities : public QgsWFSRequest
                                    bool& updateCap,
                                    bool& deleteCap );
 
+    void parseFilterCapabilities( const QDomElement& filterCapabilitiesElem );
+
     static QString NormalizeSRSName( QString crsName );
 };
 

src/providers/wfs/qgswfsconstants.cpp

@@ -33,6 +33,8 @@ const QString QgsWFSConstants::URI_PARAM_RESTRICT_TO_REQUEST_BBOX( "retrictToReq
 const QString QgsWFSConstants::URI_PARAM_MAXNUMFEATURES( "maxNumFeatures" );
 const QString QgsWFSConstants::URI_PARAM_IGNOREAXISORIENTATION( "IgnoreAxisOrientation" );
 const QString QgsWFSConstants::URI_PARAM_INVERTAXISORIENTATION( "InvertAxisOrientation" );
+const QString QgsWFSConstants::URI_PARAM_VALIDATESQLFUNCTIONS( "validateSQLFunctions" );
+const QString QgsWFSConstants::URI_PARAM_HIDEDOWNLOADPROGRESSDIALOG( "hideDownloadProgressDialog" );
 
 const QString QgsWFSConstants::VERSION_AUTO( "auto" );
 
@@ -42,4 +44,5 @@ const QString QgsWFSConstants::SETTINGS_MAXNUMFEATURES( "maxnumfeatures" );
 
 const QString QgsWFSConstants::FIELD_GEN_COUNTER( "__qgis_gen_counter" );
 const QString QgsWFSConstants::FIELD_GMLID( "__qgis_gmlid" );
-const QString QgsWFSConstants::FIELD_HEXWKB_GEOM( "__qgis_hexwkb_geom" );
+const QString QgsWFSConstants::FIELD_HEXWKB_GEOM( "__qgis_hexwkb_geom" );
+const QString QgsWFSConstants::FIELD_MD5( "__qgis_md5" );

src/providers/wfs/qgswfsconstants.h

@@ -39,6 +39,8 @@ struct QgsWFSConstants
   static const QString URI_PARAM_MAXNUMFEATURES;
   static const QString URI_PARAM_IGNOREAXISORIENTATION;
   static const QString URI_PARAM_INVERTAXISORIENTATION;
+  static const QString URI_PARAM_VALIDATESQLFUNCTIONS;
+  static const QString URI_PARAM_HIDEDOWNLOADPROGRESSDIALOG;
 
   //
   static const QString VERSION_AUTO;
@@ -52,6 +54,7 @@ struct QgsWFSConstants
   static const QString FIELD_GEN_COUNTER;
   static const QString FIELD_GMLID;
   static const QString FIELD_HEXWKB_GEOM;
+  static const QString FIELD_MD5;
 };
 
 #endif // QGSWFSCONSTANTS_H

src/providers/wfs/qgswfsdataitems.cpp

@@ -30,7 +30,9 @@
 QgsWFSLayerItem::QgsWFSLayerItem( QgsDataItem* parent, QString name, const QgsDataSourceURI& uri, QString featureType, QString title, QString crsString )
     : QgsLayerItem( parent, title, parent->path() + '/' + name, QString(), QgsLayerItem::Vector, "WFS" )
 {
-  mUri = QgsWFSDataSourceURI::build( uri.uri(), featureType, crsString );
+  QSettings settings;
+  bool useCurrentViewExtent = settings.value( "/Windows/WFSSourceSelect/FeatureCurrentViewExtent", true ).toBool();
+  mUri = QgsWFSDataSourceURI::build( uri.uri(), featureType, crsString, QString(), useCurrentViewExtent );
   setState( Populated );
   mIconName = "mIconConnect.png";
 }
@@ -208,7 +210,7 @@ QGISEXTERN QgsDataItem * dataItem( QString thePath, QgsDataItem* parentItem )
     if ( QgsWFSConnection::connectionList().contains( connectionName ) )
     {
       QgsWFSConnection connection( connectionName );
-      return new QgsWFSConnectionItem( parentItem, "WMS", thePath, connection.uri().uri() );
+      return new QgsWFSConnectionItem( parentItem, "WFS", thePath, connection.uri().uri() );
     }
   }