From 9206eaa221b8f16ad7e41d0e0eff60e6884ad72e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 15 Jan 2024 16:06:40 +0100 Subject: [PATCH 01/21] QgsCPLHTTPFetchOverrider: export it, and better take into account mFeedback to cancel document downloading before starting --- src/core/qgscplhttpfetchoverrider.cpp | 7 +++++++ src/core/qgscplhttpfetchoverrider.h | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/qgscplhttpfetchoverrider.cpp b/src/core/qgscplhttpfetchoverrider.cpp index c69f74a9d7ff..b981771058bd 100644 --- a/src/core/qgscplhttpfetchoverrider.cpp +++ b/src/core/qgscplhttpfetchoverrider.cpp @@ -63,6 +63,13 @@ CPLHTTPResult *QgsCPLHTTPFetchOverrider::callback( const char *pszURL, } } + if ( pThis->mFeedback && pThis->mFeedback->isCanceled() ) + { + psResult->nStatus = 1; + psResult->pszErrBuf = CPLStrdup( "download interrupted by user" ); + return psResult; + } + QgsBlockingNetworkRequest blockingRequest; blockingRequest.setAuthCfg( pThis->mAuthCfg ); diff --git a/src/core/qgscplhttpfetchoverrider.h b/src/core/qgscplhttpfetchoverrider.h index f9f8550cf376..bcf4654a7202 100644 --- a/src/core/qgscplhttpfetchoverrider.h +++ b/src/core/qgscplhttpfetchoverrider.h @@ -42,7 +42,7 @@ class QgsFeedback; * \note not available in Python bindings * \since QGIS 3.18 */ -class QgsCPLHTTPFetchOverrider +class CORE_EXPORT QgsCPLHTTPFetchOverrider { public: //! Installs the redirection for the current thread From 0108743033c327c79cb4f8d54c7d5bb09e565649 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 15 Jan 2024 16:07:36 +0100 Subject: [PATCH 02/21] QgsGmlStreamingParser: take into account complexContent and JSON'ify it --- .../PyQt6/core/auto_generated/qgsgml.sip.in | 1 + python/core/auto_generated/qgsgml.sip.in | 1 + src/core/qgsgml.cpp | 259 +++++++++++++++++- src/core/qgsgml.h | 25 +- tests/src/core/testqgsgml.cpp | 84 ++++++ 5 files changed, 363 insertions(+), 7 deletions(-) diff --git a/python/PyQt6/core/auto_generated/qgsgml.sip.in b/python/PyQt6/core/auto_generated/qgsgml.sip.in index beefc32e4857..befa06cb44ff 100644 --- a/python/PyQt6/core/auto_generated/qgsgml.sip.in +++ b/python/PyQt6/core/auto_generated/qgsgml.sip.in @@ -11,6 +11,7 @@ + class QgsGml : QObject { %Docstring(signature="appended") diff --git a/python/core/auto_generated/qgsgml.sip.in b/python/core/auto_generated/qgsgml.sip.in index beefc32e4857..befa06cb44ff 100644 --- a/python/core/auto_generated/qgsgml.sip.in +++ b/python/core/auto_generated/qgsgml.sip.in @@ -11,6 +11,7 @@ + class QgsGml : QObject { %Docstring(signature="appended") diff --git a/src/core/qgsgml.cpp b/src/core/qgsgml.cpp index fd7f1cc451f8..2d64e139d9b7 100644 --- a/src/core/qgsgml.cpp +++ b/src/core/qgsgml.cpp @@ -39,6 +39,8 @@ #include +using namespace nlohmann; + static const char NS_SEPARATOR = '?'; static const char *GML_NAMESPACE = "http://www.opengis.net/gml"; static const char *GML32_NAMESPACE = "http://www.opengis.net/gml/3.2"; @@ -339,7 +341,7 @@ static QString stripNS( const QString &string ) QgsGmlStreamingParser::QgsGmlStreamingParser( const QList &layerProperties, const QgsFields &fields, - const QMap< QString, QPair > &mapFieldNameToSrcLayerNameFieldName, + const QMap< QString, QPair > &fieldNameToSrcLayerNameFieldNameMap, AxisOrientationLogic axisOrientationLogic, bool invertAxisOrientation ) : mLayerProperties( layerProperties ) @@ -367,8 +369,8 @@ QgsGmlStreamingParser::QgsGmlStreamingParser( const QList &laye mThematicAttributes.clear(); for ( int i = 0; i < fields.size(); i++ ) { - const QMap< QString, QPair >::const_iterator att_it = mapFieldNameToSrcLayerNameFieldName.constFind( fields.at( i ).name() ); - if ( att_it != mapFieldNameToSrcLayerNameFieldName.constEnd() ) + const QMap< QString, QPair >::const_iterator att_it = fieldNameToSrcLayerNameFieldNameMap.constFind( fields.at( i ).name() ); + if ( att_it != fieldNameToSrcLayerNameFieldNameMap.constEnd() ) { if ( mLayerProperties.size() == 1 ) mThematicAttributes.insert( att_it.value().second, qMakePair( i, fields.at( i ) ) ); @@ -416,6 +418,19 @@ QgsGmlStreamingParser::QgsGmlStreamingParser( const QList &laye } +void QgsGmlStreamingParser::setFieldsXPath( + const QMap> &fieldNameToXPathMapAndIsNestedContent, + const QMap &mapNamespacePrefixToURI ) +{ + for ( auto iter = fieldNameToXPathMapAndIsNestedContent.constBegin(); iter != fieldNameToXPathMapAndIsNestedContent.constEnd(); ++iter ) + { + mMapXPathToFieldNameAndIsNestedContent[iter.value().first] = QPair( iter.key(), iter.value().second ); + } + for ( auto iter = mapNamespacePrefixToURI.constBegin(); iter != mapNamespacePrefixToURI.constEnd(); ++iter ) + mMapNamespaceURIToNamespacePrefix[iter.value()] = iter.key(); +} + + QgsGmlStreamingParser::~QgsGmlStreamingParser() { XML_ParserFree( mParser ); @@ -493,6 +508,37 @@ QVector QgsGmlStreamingParser: return ret; } +/** + * Returns a json string or number from the provided string. When a string + * looks like a number, a json number is returned. + */ +static json jsonFromString( const QString &s ) +{ + bool conversionOk; + + // Does it look like a floating-point value ? + if ( s.indexOf( '.' ) >= 0 || s.indexOf( 'e' ) >= 0 ) + { + const auto doubleVal = s.toDouble( &conversionOk ); + if ( conversionOk ) + { + return json( doubleVal ); + } + } + // Does it look like an integer? (but don't recognize strings starting with + // 0) + else if ( !s.isEmpty() && s[0] != '0' ) + { + const auto longlongVal = s.toLongLong( &conversionOk ); + if ( conversionOk ) + { + return json( longlongVal ); + } + } + + return json( s.toStdString() ); +} + #define LOCALNAME_EQUALS(string_constant) \ ( localNameLen == static_cast(strlen( string_constant )) && memcmp(pszLocalName, string_constant, localNameLen) == 0 ) @@ -698,6 +744,7 @@ void QgsGmlStreamingParser::startElement( const XML_Char *el, const XML_Char **a const QgsAttributes attributes( mThematicAttributes.size() ); //add empty attributes mCurrentFeature->setAttributes( attributes ); mParseModeStack.push( QgsGmlStreamingParser::Feature ); + mCurrentXPathWithinFeature.clear(); mCurrentFeatureId = readAttribute( QStringLiteral( "fid" ), attr ); if ( mCurrentFeatureId.isEmpty() ) { @@ -777,9 +824,39 @@ void QgsGmlStreamingParser::startElement( const XML_Char *el, const XML_Char **a else if ( parseMode == Feature ) { const QString localName( QString::fromUtf8( pszLocalName, localNameLen ) ); - if ( mThematicAttributes.contains( localName ) ) + if ( !mMapXPathToFieldNameAndIsNestedContent.isEmpty() ) + { + const QString nsURI( nsLen ? QString::fromUtf8( el, nsLen ) : QString() ); + const auto nsIter = mMapNamespaceURIToNamespacePrefix.constFind( nsURI ); + if ( !mCurrentXPathWithinFeature.isEmpty() ) + mCurrentXPathWithinFeature.append( '/' ); + if ( nsIter != mMapNamespaceURIToNamespacePrefix.constEnd() ) + { + mCurrentXPathWithinFeature.append( *nsIter ); + mCurrentXPathWithinFeature.append( ':' ); + } + mCurrentXPathWithinFeature.append( localName ); + const auto xpathIter = mMapXPathToFieldNameAndIsNestedContent.constFind( mCurrentXPathWithinFeature ); + mAttributeValIsNested = false; + if ( xpathIter != mMapXPathToFieldNameAndIsNestedContent.end() ) + { + mParseModeStack.push( QgsGmlStreamingParser::Attribute ); + mAttributeDepth = mParseDepth; + mAttributeName = xpathIter->first; + mAttributeValIsNested = xpathIter->second; + if ( mAttributeValIsNested ) + { + mAttributeJson = json::object(); + mAttributeJsonCurrentStack.clear(); + mAttributeJsonCurrentStack.push( &mAttributeJson ); + } + mStringCash.clear(); + } + } + else if ( mThematicAttributes.contains( localName ) ) { mParseModeStack.push( QgsGmlStreamingParser::Attribute ); + mAttributeDepth = mParseDepth; mAttributeName = localName; mStringCash.clear(); } @@ -798,6 +875,39 @@ void QgsGmlStreamingParser::startElement( const XML_Char *el, const XML_Char **a } } } + else if ( parseMode == Attribute && mAttributeValIsNested ) + { + const std::string localName( pszLocalName, localNameLen ); + const QString nsURI( nsLen ? QString::fromUtf8( el, nsLen ) : QString() ); + const auto nsIter = mMapNamespaceURIToNamespacePrefix.constFind( nsURI ); + const std::string nodeName = nsIter != mMapNamespaceURIToNamespacePrefix.constEnd() ? ( *nsIter ).toStdString() + ':' + localName : localName; + + addStringContentToJson(); + + auto &jsonParent = *( mAttributeJsonCurrentStack.top() ); + auto iter = jsonParent.find( nodeName ); + if ( iter != jsonParent.end() ) + { + if ( iter->type() != json::value_t::array ) + { + auto array = json::array(); + array.emplace_back( std::move( *iter ) ); + *iter = array; + } + iter->push_back( json::object() ); + mAttributeJsonCurrentStack.push( &( iter->back() ) ); + } + else + { + auto res = jsonParent.emplace( nodeName, json::object() ); + // res.first is a json::iterator + // Dereferencing it leads to a json reference + // And taking a reference on it gets a pointer + nlohmann::json *ptr = &( *( res.first ) ); + // cppcheck-suppress danglingLifetime + mAttributeJsonCurrentStack.push( ptr ); + } + } else if ( mParseDepth == 0 && LOCALNAME_EQUALS( "FeatureCollection" ) ) { QString numberReturned = readAttribute( QStringLiteral( "numberReturned" ), attr ); // WFS 2.0 @@ -847,6 +957,55 @@ void QgsGmlStreamingParser::startElement( const XML_Char *el, const XML_Char **a mFoundUnhandledGeometryElement = true; } + // Handle XML attributes in XPath mode + if ( !mParseModeStack.isEmpty() && + ( mParseModeStack.back() == Feature || + mParseModeStack.back() == Attribute ) && + !mMapXPathToFieldNameAndIsNestedContent.isEmpty() ) + { + for ( const XML_Char **attrIter = attr; attrIter && *attrIter; attrIter += 2 ) + { + const char *questionMark = strchr( attrIter[0], '?' ); + QString key( '@' ); + if ( questionMark ) + { + const QString nsURI( QString::fromUtf8( attrIter[0], static_cast( questionMark - attrIter[0] ) ) ); + const QString localName( QString::fromUtf8( questionMark + 1 ) ); + const auto nsIter = mMapNamespaceURIToNamespacePrefix.constFind( nsURI ); + if ( nsIter != mMapNamespaceURIToNamespacePrefix.constEnd() ) + { + key.append( *nsIter ); + key.append( ':' ); + } + key.append( localName ); + } + else + { + const QString localName( QString::fromUtf8( attrIter[0] ) ); + key.append( localName ); + } + + if ( mAttributeValIsNested && mParseModeStack.back() == Attribute ) + { + mAttributeJsonCurrentStack.top()->emplace( + key.toStdString(), + jsonFromString( QString::fromUtf8( attrIter[1] ) ) ); + } + else + { + QString xpath( mCurrentXPathWithinFeature ); + if ( !xpath.isEmpty() ) + xpath.append( '/' ); + xpath.append( key ); + const auto xpathIter = mMapXPathToFieldNameAndIsNestedContent.constFind( xpath ); + if ( xpathIter != mMapXPathToFieldNameAndIsNestedContent.end() ) + { + setAttribute( xpathIter->first, QString::fromUtf8( attrIter[1] ) ); + } + } + } + } + if ( !mGeometryString.empty() ) isGeom = true; @@ -902,6 +1061,37 @@ void QgsGmlStreamingParser::endElement( const XML_Char *el ) const bool isGMLNS = ( nsLen == mGMLNameSpaceURI.size() && mGMLNameSpaceURIPtr && memcmp( el, mGMLNameSpaceURIPtr, nsLen ) == 0 ); + if ( parseMode == Feature || ( parseMode == Attribute && mAttributeDepth == mParseDepth ) ) + { + if ( !mMapXPathToFieldNameAndIsNestedContent.isEmpty() ) + { + const auto nPos = mCurrentXPathWithinFeature.lastIndexOf( '/' ); + if ( nPos < 0 ) + mCurrentXPathWithinFeature.clear(); + else + mCurrentXPathWithinFeature.resize( nPos ); + } + } + + if ( parseMode == Attribute && mAttributeValIsNested ) + { + if ( !mStringCash.isEmpty() ) + { + auto &jsonParent = *( mAttributeJsonCurrentStack.top() ); + if ( jsonParent.type() == json::value_t::object && jsonParent.empty() ) + { + jsonParent = jsonFromString( mStringCash ); + } + else if ( jsonParent.type() == json::value_t::object ) + { + addStringContentToJson(); + } + mStringCash.clear(); + } + + mAttributeJsonCurrentStack.pop(); + } + if ( parseMode == Coordinate && isGMLNS && LOCALNAME_EQUALS( "coordinates" ) ) { mParseModeStack.pop(); @@ -919,11 +1109,43 @@ void QgsGmlStreamingParser::endElement( const XML_Char *el ) setAttribute( mAttributeName, mStringCash ); } - else if ( parseMode == Attribute && QString::fromUtf8( pszLocalName, localNameLen ) == mAttributeName ) //add a thematic attribute to the feature + else if ( parseMode == Attribute && mAttributeDepth == mParseDepth ) //add a thematic attribute to the feature { mParseModeStack.pop(); + mParseDepth = -1; - setAttribute( mAttributeName, mStringCash ); + if ( mAttributeValIsNested ) + { + //find index with attribute name + const QMap >::const_iterator att_it = mThematicAttributes.constFind( mAttributeName ); + Q_ASSERT( mCurrentFeature ); + const int attrIndex = att_it.value().first; + auto attrVal = mCurrentFeature->attribute( attrIndex ); + if ( attrVal.isNull() ) + { + mCurrentFeature->setAttribute( attrIndex, QString::fromStdString( mAttributeJson.dump() ) ); + } + else + { + QString str = attrVal.toString(); + if ( str[0] == '[' && str.back() == ']' ) + { + str.back() = ','; + } + else + { + str.insert( 0, '[' ); + str.append( ',' ); + } + str.append( QString::fromStdString( mAttributeJson.dump() ) ); + str.append( ']' ); + mCurrentFeature->setAttribute( attrIndex, str ); + } + } + else + { + setAttribute( mAttributeName, mStringCash ); + } } else if ( parseMode == Geometry && localNameLen == static_cast( mGeometryAttributeUTF8Len ) && @@ -1218,6 +1440,31 @@ void QgsGmlStreamingParser::characters( const XML_Char *chars, int len ) } } +void QgsGmlStreamingParser::addStringContentToJson() +{ + const QString s( mStringCash.trimmed() ); + if ( !s.isEmpty() ) + { + auto &jsonParent = *( mAttributeJsonCurrentStack.top() ); + auto textIter = jsonParent.find( "_text" ); + if ( textIter != jsonParent.end() ) + { + if ( textIter->type() != json::value_t::array ) + { + auto array = json::array(); + array.emplace_back( std::move( *textIter ) ); + *textIter = array; + } + textIter->emplace_back( jsonFromString( s ) ); + } + else + { + jsonParent.emplace( "_text", jsonFromString( s ) ); + } + } + mStringCash.clear(); +} + void QgsGmlStreamingParser::setAttribute( const QString &name, const QString &value ) { //find index with attribute name diff --git a/src/core/qgsgml.h b/src/core/qgsgml.h index 0f1635de1f69..ca24317e8cba 100644 --- a/src/core/qgsgml.h +++ b/src/core/qgsgml.h @@ -32,6 +32,10 @@ #include +#ifndef SIP_RUN +#include "json.hpp" +#endif + class QgsCoordinateReferenceSystem; class QTextCodec; @@ -90,7 +94,7 @@ class CORE_EXPORT QgsGmlStreamingParser //! Constructor for a join layer, or dealing with renamed fields QgsGmlStreamingParser( const QList &layerProperties, const QgsFields &fields, - const QMap< QString, QPair > &mapFieldNameToSrcLayerNameFieldName, + const QMap< QString, QPair > &fieldNameToSrcLayerNameFieldNameMap, AxisOrientationLogic axisOrientationLogic = Honour_EPSG_if_urn, bool invertAxisOrientation = false ); ~QgsGmlStreamingParser(); @@ -100,6 +104,15 @@ class CORE_EXPORT QgsGmlStreamingParser //! QgsGmlStreamingParser cannot be copied. QgsGmlStreamingParser &operator=( const QgsGmlStreamingParser &other ) = delete; + /** + * Define the XPath of the attributes and whether they are made of nested + * content. Also provides a map from namespace prefix to namespace URI, + * to help decoding the XPath. + */ + void setFieldsXPath( + const QMap> &fieldNameToSrcLayerNameFieldNameMap, + const QMap &namespacePrefixToURIMap ); + /** * Process a new chunk of data. atEnd must be set to TRUE when this is * the last chunk of data. @@ -189,6 +202,9 @@ class CORE_EXPORT QgsGmlStreamingParser static_cast( data )->characters( chars, len ); } + // Add mStringCash to the current json object + void addStringContentToJson(); + // Set current feature attribute void setAttribute( const QString &name, const QString &value ); @@ -284,6 +300,8 @@ class CORE_EXPORT QgsGmlStreamingParser QgsFields mFields; QMap > mThematicAttributes; + QMap> mMapXPathToFieldNameAndIsNestedContent; + QMap mMapNamespaceURIToNamespacePrefix; bool mIsException; QString mExceptionText; @@ -294,6 +312,7 @@ class CORE_EXPORT QgsGmlStreamingParser QString mCurrentTypename; //!< Used to track the current (unprefixed) typename for wfs:Member in join layer //! Keep track about the most important nested elements QStack mParseModeStack; + QString mCurrentXPathWithinFeature; //! This contains the character data if an important element has been encountered QString mStringCash; QgsFeature *mCurrentFeature = nullptr; @@ -313,6 +332,10 @@ class CORE_EXPORT QgsGmlStreamingParser */ QList< QList > mCurrentWKBFragments; QString mAttributeName; + int mAttributeDepth = -1; + bool mAttributeValIsNested = false; + nlohmann::json mAttributeJson; + QStack mAttributeJsonCurrentStack; char mEndian; //! Coordinate separator for coordinate strings. Usually "," QString mCoordinateSeparator; diff --git a/tests/src/core/testqgsgml.cpp b/tests/src/core/testqgsgml.cpp index 4b15c859f101..9b95a3beda9c 100644 --- a/tests/src/core/testqgsgml.cpp +++ b/tests/src/core/testqgsgml.cpp @@ -87,6 +87,7 @@ class TestQgsGML : public QObject void testUnknownEncoding_data(); void testUnknownEncoding(); void testUnhandledEncoding(); + void testXPath(); }; const QString data1( "" + "" + "unknown" + "" + "" + "foo" + "" + "bar" + "baz" + "" + "" + "" + "x" + "" + "y" + "123456789012312011.2512345678901234567890123456789" + "ab" + "ab" + "abc" + "x" + "" + "" + "" + "foo" + "bar" + "" + "" + "10,20" + "" + "" + "" + "" + "" ); + + QgsFields fields; + fields.append( QgsField( QStringLiteral( "fid" ), QVariant::String, QStringLiteral( "fid" ) ) ); + fields.append( QgsField( QStringLiteral( "my_attr" ), QVariant::String, QStringLiteral( "string" ) ) ); + fields.append( QgsField( QStringLiteral( "strfield" ), QVariant::String, QStringLiteral( "string" ) ) ); + fields.append( QgsField( QStringLiteral( "nested_strfield2" ), QVariant::String, QStringLiteral( "string" ) ) ); + fields.append( QgsField( QStringLiteral( "nested_strfield2_attr" ), QVariant::String, QStringLiteral( "string" ) ) ); + fields.append( QgsField( QStringLiteral( "nested_strfield3" ), QVariant::String, QStringLiteral( "string" ) ) ); + fields.append( QgsField( QStringLiteral( "complex" ), QVariant::String, QStringLiteral( "string" ) ) ); + fields.append( QgsField( QStringLiteral( "complex2" ), QVariant::String, QStringLiteral( "string" ) ) ); + fields.append( QgsField( QStringLiteral( "complex_repeated" ), QVariant::String, QStringLiteral( "string" ) ) ); + + QgsGmlStreamingParser gmlParser( QStringLiteral( "mytypename" ), QStringLiteral( "mygeom" ), fields ); + + QMap> mapFieldNameToXPathAndIsNestedContent; + QMap mapNamespacePrefixToURI; + mapFieldNameToXPathAndIsNestedContent[QStringLiteral( "fid" )] = QPair( QStringLiteral( "@fid" ), false ); + mapFieldNameToXPathAndIsNestedContent[QStringLiteral( "my_attr" )] = QPair( QStringLiteral( "@myns:my_attr" ), false ); + mapFieldNameToXPathAndIsNestedContent[QStringLiteral( "strfield" )] = QPair( QStringLiteral( "myns:strfield" ), false ); + mapFieldNameToXPathAndIsNestedContent[QStringLiteral( "nested_strfield2" )] = QPair( QStringLiteral( "myns:nested/myns:strfield2" ), false ); + mapFieldNameToXPathAndIsNestedContent[QStringLiteral( "nested_strfield2_attr" )] = QPair( QStringLiteral( "myns:nested/myns:strfield2/@attr" ), false ); + mapFieldNameToXPathAndIsNestedContent[QStringLiteral( "nested_strfield3" )] = QPair( QStringLiteral( "myns:nested/myns:strfield3" ), false ); + mapFieldNameToXPathAndIsNestedContent[QStringLiteral( "complex" )] = QPair( QStringLiteral( "myns:complex" ), true ); + mapFieldNameToXPathAndIsNestedContent[QStringLiteral( "complex2" )] = QPair( QStringLiteral( "myns:complex2" ), true ); + mapFieldNameToXPathAndIsNestedContent[QStringLiteral( "complex_repeated" )] = QPair( QStringLiteral( "myns:complex_repeated" ), true ); + mapNamespacePrefixToURI[QStringLiteral( "myns" )] = QStringLiteral( "http://myns" ); + gmlParser.setFieldsXPath( mapFieldNameToXPathAndIsNestedContent, mapNamespacePrefixToURI ); + + QCOMPARE( gmlParser.processData( data.toUtf8(), true ), true ); + QVector features = gmlParser.getAndStealReadyFeatures(); + QCOMPARE( features.size(), 1 ); + auto &f = *( features[ 0 ].first ); + QCOMPARE( f.attributes().size(), 9 ); + QCOMPARE( f.attribute( QStringLiteral( "fid" ) ).toString(), QStringLiteral( "mytypename.1" ) ); + QCOMPARE( f.attribute( QStringLiteral( "my_attr" ) ).toString(), QStringLiteral( "my_value" ) ); + QCOMPARE( f.attribute( QStringLiteral( "strfield" ) ).toString(), QStringLiteral( "foo" ) ); + QCOMPARE( f.attribute( QStringLiteral( "nested_strfield2" ) ).toString(), QStringLiteral( "bar" ) ); + QCOMPARE( f.attribute( QStringLiteral( "nested_strfield2_attr" ) ).toString(), QStringLiteral( "attr_val" ) ); + QCOMPARE( f.attribute( QStringLiteral( "nested_strfield3" ) ).toString(), QStringLiteral( "baz" ) ); + QCOMPARE( f.attribute( QStringLiteral( "complex" ) ).toString(), QStringLiteral( "{\"@foo\":\"bar\",\"myns:l1\":{\"c\":{\"k1\":1234567890123,\"k2\":12,\"k3\":\"01\",\"k4\":1.25,\"k5\":\"12345678901234567890123456789\"},\"myns:a\":\"x\",\"myns:b\":{\"@k\":\"v\",\"_text\":\"y\"},\"myns:d\":[{\"_text\":\"a\",\"i\":\"b\"},{\"_text\":\"b\",\"i\":\"a\"},{\"_text\":[\"a\",\"c\"],\"i\":\"b\"},\"x\"]}}" ) ); + QCOMPARE( f.attribute( QStringLiteral( "complex2" ) ).toString(), QStringLiteral( "{}" ) ); + QCOMPARE( f.attribute( QStringLiteral( "complex_repeated" ) ).toString(), QStringLiteral( "[\"foo\",\"bar\"]" ) ); +} + QGSTEST_MAIN( TestQgsGML ) #include "testqgsgml.moc" From be57a36ffd868a6950d4ad02d177bb59a3efcfbe Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 15 Jan 2024 16:08:17 +0100 Subject: [PATCH 03/21] QgsOgcUtils: take into account properties with a complex XPath --- .../core/auto_generated/qgsogcutils.sip.in | 4 +- python/core/auto_generated/qgsogcutils.sip.in | 4 +- src/core/qgsogcutils.cpp | 96 +++++++++++++++++-- src/core/qgsogcutils.h | 25 ++++- tests/src/core/testqgsogcutils.cpp | 80 ++++++++++++++++ 5 files changed, 194 insertions(+), 15 deletions(-) diff --git a/python/PyQt6/core/auto_generated/qgsogcutils.sip.in b/python/PyQt6/core/auto_generated/qgsogcutils.sip.in index 6e8a54d5cf0c..c9b82863c138 100644 --- a/python/PyQt6/core/auto_generated/qgsogcutils.sip.in +++ b/python/PyQt6/core/auto_generated/qgsogcutils.sip.in @@ -197,7 +197,9 @@ Creates an ElseFilter from ``doc`` bool honourAxisOrientation, bool invertAxisOrientation, QString *errorMessage = 0, - bool requiresFilterElement = false ); + bool requiresFilterElement = false, + const QMap &fieldNameToXPathMap = QMap(), + const QMap &namespacePrefixToUriMap = QMap() ); %Docstring Creates an OGC expression XML element from the ``exp`` expression. diff --git a/python/core/auto_generated/qgsogcutils.sip.in b/python/core/auto_generated/qgsogcutils.sip.in index 5ad1e9f7ccba..8755e727d658 100644 --- a/python/core/auto_generated/qgsogcutils.sip.in +++ b/python/core/auto_generated/qgsogcutils.sip.in @@ -197,7 +197,9 @@ Creates an ElseFilter from ``doc`` bool honourAxisOrientation, bool invertAxisOrientation, QString *errorMessage = 0, - bool requiresFilterElement = false ); + bool requiresFilterElement = false, + const QMap &fieldNameToXPathMap = QMap(), + const QMap &namespacePrefixToUriMap = QMap() ); %Docstring Creates an OGC expression XML element from the ``exp`` expression. diff --git a/src/core/qgsogcutils.cpp b/src/core/qgsogcutils.cpp index ce1eb9bcbbab..3658e3211e36 100644 --- a/src/core/qgsogcutils.cpp +++ b/src/core/qgsogcutils.cpp @@ -54,7 +54,9 @@ QgsOgcUtilsExprToFilter::QgsOgcUtilsExprToFilter( QDomDocument &doc, const QString &geometryName, const QString &srsName, bool honourAxisOrientation, - bool invertAxisOrientation ) + bool invertAxisOrientation, + const QMap &fieldNameToXPathMap, + const QMap &namespacePrefixToUriMap ) : mDoc( doc ) , mGMLUsed( false ) , mGMLVersion( gmlVersion ) @@ -64,6 +66,8 @@ QgsOgcUtilsExprToFilter::QgsOgcUtilsExprToFilter( QDomDocument &doc, , mGeometryName( geometryName ) , mSrsName( srsName ) , mInvertAxisOrientation( invertAxisOrientation ) + , mFieldNameToXPathMap( fieldNameToXPathMap ) + , mNamespacePrefixToUriMap( namespacePrefixToUriMap ) , mFilterPrefix( ( filterVersion == QgsOgcUtils::FILTER_FES_2_0 ) ? "fes" : "ogc" ) , mPropertyName( ( filterVersion == QgsOgcUtils::FILTER_FES_2_0 ) ? "ValueReference" : "PropertyName" ) , mGeomId( 1 ) @@ -1893,7 +1897,9 @@ QDomElement QgsOgcUtils::expressionToOgcFilter( const QgsExpression &expression, const QString &srsName, bool honourAxisOrientation, bool invertAxisOrientation, - QString *errorMessage ) + QString *errorMessage, + const QMap &fieldNameToXPathMap, + const QMap &namespacePrefixToUriMap ) { if ( !expression.rootNode() ) return QDomElement(); @@ -1902,7 +1908,7 @@ QDomElement QgsOgcUtils::expressionToOgcFilter( const QgsExpression &expression, QgsExpressionContext context; context << QgsExpressionContextUtils::globalScope(); - QgsOgcUtilsExprToFilter utils( doc, gmlVersion, filterVersion, namespacePrefix, namespaceURI, geometryName, srsName, honourAxisOrientation, invertAxisOrientation ); + QgsOgcUtilsExprToFilter utils( doc, gmlVersion, filterVersion, namespacePrefix, namespaceURI, geometryName, srsName, honourAxisOrientation, invertAxisOrientation, fieldNameToXPathMap, namespacePrefixToUriMap ); const QDomElement exprRootElem = utils.expressionNodeToOgcFilter( exp.rootNode(), &exp, &context ); if ( errorMessage ) *errorMessage = utils.errorMessage(); @@ -1931,7 +1937,9 @@ QDomElement QgsOgcUtils::expressionToOgcExpression( const QgsExpression &express bool honourAxisOrientation, bool invertAxisOrientation, QString *errorMessage, - bool requiresFilterElement ) + bool requiresFilterElement, + const QMap &fieldNameToXPathMap, + const QMap &namespacePrefixToUriMap ) { QgsExpressionContext context; context << QgsExpressionContextUtils::globalScope(); @@ -1948,7 +1956,7 @@ QDomElement QgsOgcUtils::expressionToOgcExpression( const QgsExpression &express case QgsExpressionNode::ntLiteral: case QgsExpressionNode::ntColumnRef: { - QgsOgcUtilsExprToFilter utils( doc, gmlVersion, filterVersion, QString(), QString(), geometryName, srsName, honourAxisOrientation, invertAxisOrientation ); + QgsOgcUtilsExprToFilter utils( doc, gmlVersion, filterVersion, QString(), QString(), geometryName, srsName, honourAxisOrientation, invertAxisOrientation, fieldNameToXPathMap, namespacePrefixToUriMap ); const QDomElement exprRootElem = utils.expressionNodeToOgcFilter( node, &exp, &context ); if ( errorMessage ) @@ -1985,14 +1993,16 @@ QDomElement QgsOgcUtils::SQLStatementToOgcFilter( const QgsSQLStatement &stateme bool honourAxisOrientation, bool invertAxisOrientation, const QMap< QString, QString> &mapUnprefixedTypenameToPrefixedTypename, - QString *errorMessage ) + QString *errorMessage, + const QMap &fieldNameToXPathMap, + const QMap &namespacePrefixToUriMap ) { if ( !statement.rootNode() ) return QDomElement(); QgsOgcUtilsSQLStatementToFilter utils( doc, gmlVersion, filterVersion, layerProperties, honourAxisOrientation, invertAxisOrientation, - mapUnprefixedTypenameToPrefixedTypename ); + mapUnprefixedTypenameToPrefixedTypename, fieldNameToXPathMap, namespacePrefixToUriMap ); const QDomElement exprRootElem = utils.toOgcFilter( statement.rootNode() ); if ( errorMessage ) *errorMessage = utils.errorMessage(); @@ -2192,6 +2202,39 @@ QDomElement QgsOgcUtilsExprToFilter::expressionColumnRefToOgcFilter( const QgsEx Q_UNUSED( expression ) Q_UNUSED( context ) QDomElement propElem = mDoc.createElement( mFilterPrefix + ":" + mPropertyName ); + if ( !mFieldNameToXPathMap.isEmpty() ) + { + const auto iterFieldName = mFieldNameToXPathMap.constFind( node->name() ); + if ( iterFieldName != mFieldNameToXPathMap.constEnd() ) + { + const QString xpath( *iterFieldName ); + + if ( !mNamespacePrefixToUriMap.isEmpty() ) + { + const QStringList parts = xpath.split( '/' ); + QSet setNamespacePrefix; + for ( const QString &part : std::as_const( parts ) ) + { + const QStringList subparts = part.split( ':' ); + if ( subparts.size() == 2 && !setNamespacePrefix.contains( subparts[0] ) ) + { + const auto iterNamespacePrefix = mNamespacePrefixToUriMap.constFind( subparts[0] ); + if ( iterNamespacePrefix != mNamespacePrefixToUriMap.constEnd() ) + { + setNamespacePrefix.insert( subparts[0] ); + QDomAttr attr = mDoc.createAttribute( QStringLiteral( "xmlns:" ) + subparts[0] ); + attr.setValue( *iterNamespacePrefix ); + propElem.setAttributeNode( attr ); + } + } + } + } + + propElem.appendChild( mDoc.createTextNode( xpath ) ); + + return propElem; + } + } QString columnRef( node->name() ); if ( !mNamespacePrefix.isEmpty() && !mNamespaceURI.isEmpty() ) columnRef = mNamespacePrefix + QStringLiteral( ":" ) + columnRef; @@ -2477,7 +2520,9 @@ QgsOgcUtilsSQLStatementToFilter::QgsOgcUtilsSQLStatementToFilter( QDomDocument & const QList &layerProperties, bool honourAxisOrientation, bool invertAxisOrientation, - const QMap< QString, QString> &mapUnprefixedTypenameToPrefixedTypename ) + const QMap< QString, QString> &mapUnprefixedTypenameToPrefixedTypename, + const QMap &fieldNameToXPathMap, + const QMap &namespacePrefixToUriMap ) : mDoc( doc ) , mGMLUsed( false ) , mGMLVersion( gmlVersion ) @@ -2489,6 +2534,8 @@ QgsOgcUtilsSQLStatementToFilter::QgsOgcUtilsSQLStatementToFilter( QDomDocument & , mPropertyName( ( filterVersion == QgsOgcUtils::FILTER_FES_2_0 ) ? "ValueReference" : "PropertyName" ) , mGeomId( 1 ) , mMapUnprefixedTypenameToPrefixedTypename( mapUnprefixedTypenameToPrefixedTypename ) + , mFieldNameToXPathMap( fieldNameToXPathMap ) + , mNamespacePrefixToUriMap( namespacePrefixToUriMap ) { } @@ -2685,6 +2732,39 @@ QDomElement QgsOgcUtilsSQLStatementToFilter::toOgcFilter( const QgsSQLStatement: QDomElement propElem = mDoc.createElement( mFilterPrefix + ":" + mPropertyName ); if ( node->tableName().isEmpty() || mLayerProperties.size() == 1 ) { + if ( !mFieldNameToXPathMap.isEmpty() ) + { + const auto iterFieldName = mFieldNameToXPathMap.constFind( node->name() ); + if ( iterFieldName != mFieldNameToXPathMap.constEnd() ) + { + const QString xpath( *iterFieldName ); + + if ( !mNamespacePrefixToUriMap.isEmpty() ) + { + const QStringList parts = xpath.split( '/' ); + QSet setNamespacePrefix; + for ( const QString &part : std::as_const( parts ) ) + { + const QStringList subparts = part.split( ':' ); + if ( subparts.size() == 2 && !setNamespacePrefix.contains( subparts[0] ) ) + { + const auto iterNamespacePrefix = mNamespacePrefixToUriMap.constFind( subparts[0] ); + if ( iterNamespacePrefix != mNamespacePrefixToUriMap.constEnd() ) + { + setNamespacePrefix.insert( subparts[0] ); + QDomAttr attr = mDoc.createAttribute( QStringLiteral( "xmlns:" ) + subparts[0] ); + attr.setValue( *iterNamespacePrefix ); + propElem.setAttributeNode( attr ); + } + } + } + } + + propElem.appendChild( mDoc.createTextNode( xpath ) ); + + return propElem; + } + } if ( mLayerProperties.size() == 1 && !mLayerProperties[0].mNamespacePrefix.isEmpty() && !mLayerProperties[0].mNamespaceURI.isEmpty() ) propElem.appendChild( mDoc.createTextNode( mLayerProperties[0].mNamespacePrefix + QStringLiteral( ":" ) + node->name() ) ); diff --git a/src/core/qgsogcutils.h b/src/core/qgsogcutils.h index 06a988e10853..4c5860fcc2d2 100644 --- a/src/core/qgsogcutils.h +++ b/src/core/qgsogcutils.h @@ -207,7 +207,9 @@ class CORE_EXPORT QgsOgcUtils const QString &srsName, bool honourAxisOrientation, bool invertAxisOrientation, - QString *errorMessage = nullptr ) SIP_SKIP; + QString *errorMessage = nullptr, + const QMap &fieldNameToXPathMap = QMap(), + const QMap &namespacePrefixToUriMap = QMap() ) SIP_SKIP; /** * Creates an OGC expression XML element from the \a exp expression @@ -239,7 +241,9 @@ class CORE_EXPORT QgsOgcUtils bool honourAxisOrientation, bool invertAxisOrientation, QString *errorMessage = nullptr, - bool requiresFilterElement = false ); + bool requiresFilterElement = false, + const QMap &fieldNameToXPathMap = QMap(), + const QMap &namespacePrefixToUriMap = QMap() ); #ifndef SIP_RUN @@ -292,7 +296,9 @@ class CORE_EXPORT QgsOgcUtils bool honourAxisOrientation, bool invertAxisOrientation, const QMap< QString, QString> &mapUnprefixedTypenameToPrefixedTypename, - QString *errorMessage = nullptr ) SIP_SKIP; + QString *errorMessage = nullptr, + const QMap &fieldNameToXPathMap = QMap(), + const QMap &namespacePrefixToUriMap = QMap() ) SIP_SKIP; private: @@ -393,7 +399,10 @@ class QgsOgcUtilsExprToFilter const QString &geometryName, const QString &srsName, bool honourAxisOrientation, - bool invertAxisOrientation ); + bool invertAxisOrientation, + const QMap &fieldNameToXPathMap, + const QMap &namespacePrefixToUriMap + ); //! Convert an expression to a OGC filter QDomElement expressionNodeToOgcFilter( const QgsExpressionNode *node, QgsExpression *expression, const QgsExpressionContext *context ); @@ -414,6 +423,8 @@ class QgsOgcUtilsExprToFilter const QString &mGeometryName; const QString &mSrsName; bool mInvertAxisOrientation; + const QMap &mFieldNameToXPathMap; + const QMap &mNamespacePrefixToUriMap; QString mErrorMessage; QString mFilterPrefix; QString mPropertyName; @@ -529,7 +540,9 @@ class QgsOgcUtilsSQLStatementToFilter const QList &layerProperties, bool honourAxisOrientation, bool invertAxisOrientation, - const QMap< QString, QString> &mapUnprefixedTypenameToPrefixedTypename ); + const QMap< QString, QString> &mapUnprefixedTypenameToPrefixedTypename, + const QMap &fieldNameToXPathMap, + const QMap &namespacePrefixToUriMap ); //! Convert a SQL statement to a OGC filter QDomElement toOgcFilter( const QgsSQLStatement::Node *node ); @@ -555,6 +568,8 @@ class QgsOgcUtilsSQLStatementToFilter QString mCurrentSRSName; QMap mMapTableAliasToNames; const QMap< QString, QString> &mMapUnprefixedTypenameToPrefixedTypename; + const QMap &mFieldNameToXPathMap; + const QMap &mNamespacePrefixToUriMap; QDomElement toOgcFilter( const QgsSQLStatement::NodeUnaryOperator *node ); QDomElement toOgcFilter( const QgsSQLStatement::NodeBinaryOperator *node ); diff --git a/tests/src/core/testqgsogcutils.cpp b/tests/src/core/testqgsogcutils.cpp index ae87f97923f1..2bfcf26b8caa 100644 --- a/tests/src/core/testqgsogcutils.cpp +++ b/tests/src/core/testqgsogcutils.cpp @@ -73,6 +73,10 @@ class TestQgsOgcUtils : public QObject void testExpressionToOgcFilterWFS20(); void testExpressionToOgcFilterWFS20_data(); + void testExpressionToOgcFilterWithXPath(); + + void testSQLStatementToOgcFilterWithXPath(); + void testSQLStatementToOgcFilter(); void testSQLStatementToOgcFilter_data(); @@ -1288,6 +1292,82 @@ void TestQgsOgcUtils::testSQLStatementToOgcFilter_data() "" ); } +void TestQgsOgcUtils::testExpressionToOgcFilterWithXPath() +{ + const QgsExpression exp( "a = 1" ); + QString errorMsg; + + QMap mapFieldNameToXPath; + mapFieldNameToXPath["a"] = "myns:foo/myns:bar/otherns:a"; + + QMap mapNamespacePrefixToUri; + mapNamespacePrefixToUri["myns"] = "https://myns"; + mapNamespacePrefixToUri["otherns"] = "https://otherns"; + + QDomDocument doc; + const QDomElement filterElem = QgsOgcUtils::expressionToOgcFilter( exp, doc, + QgsOgcUtils::GML_3_2_1, QgsOgcUtils::FILTER_FES_2_0, + QString(), QString(), + QStringLiteral( "my_geometry_name" ), QString(), true, false, &errorMsg, mapFieldNameToXPath, mapNamespacePrefixToUri ); + + if ( !errorMsg.isEmpty() ) + qDebug( "ERROR: %s", errorMsg.toLatin1().data() ); + + QDomElement xmlElem = comparableElement( QStringLiteral( "myns:foo/myns:bar/otherns:a1" ) ); + doc.appendChild( filterElem ); + qDebug( "OGC : %s", doc.toString( -1 ).toLatin1().data() ); + + QDomElement ogcElem = comparableElement( doc.toString( -1 ) ); + QVERIFY( compareElements( xmlElem, ogcElem ) ); +} + +void TestQgsOgcUtils::testSQLStatementToOgcFilterWithXPath() +{ + + const QgsSQLStatement statement( "SELECT * FROM t WHERE a = 1" ); + if ( !statement.hasParserError() ) + { + qDebug( "%s", statement.parserErrorString().toLatin1().data() ); + QVERIFY( !statement.hasParserError() ); + } + + QMap mapFieldNameToXPath; + mapFieldNameToXPath["a"] = "myns:foo/myns:bar/otherns:a"; + + QMap mapNamespacePrefixToUri; + mapNamespacePrefixToUri["myns"] = "https://myns"; + mapNamespacePrefixToUri["otherns"] = "https://otherns"; + + QString errorMsg; + QDomDocument doc; + const bool honourAxisOrientation = true; + const bool invertAxisOrientation = false; + QList layerProperties; + QgsOgcUtils::LayerProperties prop; + prop.mSRSName = QStringLiteral( "urn:ogc:def:crs:EPSG::4326" ); + prop.mGeometryAttribute = QStringLiteral( "geom" ); + layerProperties.append( prop ); + const QDomElement filterElem = QgsOgcUtils::SQLStatementToOgcFilter( statement, + doc, + QgsOgcUtils::GML_3_2_1, + QgsOgcUtils::FILTER_FES_2_0, + layerProperties, + honourAxisOrientation, + invertAxisOrientation, + QMap(), + &errorMsg, mapFieldNameToXPath, mapNamespacePrefixToUri ); + + if ( !errorMsg.isEmpty() ) + qDebug( "ERROR: %s", errorMsg.toLatin1().data() ); + + QDomElement xmlElem = comparableElement( QStringLiteral( "myns:foo/myns:bar/otherns:a1" ) ); + doc.appendChild( filterElem ); + qDebug( "OGC : %s", doc.toString( -1 ).toLatin1().data() ); + + QDomElement ogcElem = comparableElement( doc.toString( -1 ) ); + QVERIFY( compareElements( xmlElem, ogcElem ) ); +} + void TestQgsOgcUtils::testParseCrsName() { From 83fe2fea87bc9ff3c45a41c5983c5746ae8e7898 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 15 Jan 2024 16:09:09 +0100 Subject: [PATCH 04/21] [WFS provider] Handle documents with Complex Feature schemas (using OGR GMLAS driver), and JSON'ify content of complex properties Funded by QGIS user group Germany (QGIS Anwendergruppe Deutschland e.V.) --- src/providers/wfs/CMakeLists.txt | 2 + src/providers/wfs/qgswfsfeatureiterator.cpp | 14 +- src/providers/wfs/qgswfsprovider.cpp | 517 +++++++++++++- src/providers/wfs/qgswfsprovider.h | 20 +- src/providers/wfs/qgswfsprovidermetadata.cpp | 2 + src/providers/wfs/qgswfsshareddata.cpp | 84 ++- src/providers/wfs/qgswfsshareddata.h | 6 + tests/src/python/test_provider_wfs.py | 54 ++ .../describefeaturetype.xml | 6 + .../getcapabilities.xml | 652 ++++++++++++++++++ .../inspire_complexfeatures/getfeature.xml | 59 ++ .../getfeature_hits.xml | 3 + 12 files changed, 1383 insertions(+), 36 deletions(-) create mode 100644 tests/testdata/provider/wfs/inspire_complexfeatures/describefeaturetype.xml create mode 100644 tests/testdata/provider/wfs/inspire_complexfeatures/getcapabilities.xml create mode 100644 tests/testdata/provider/wfs/inspire_complexfeatures/getfeature.xml create mode 100644 tests/testdata/provider/wfs/inspire_complexfeatures/getfeature_hits.xml diff --git a/src/providers/wfs/CMakeLists.txt b/src/providers/wfs/CMakeLists.txt index 045f8757be8f..c553d933653e 100644 --- a/src/providers/wfs/CMakeLists.txt +++ b/src/providers/wfs/CMakeLists.txt @@ -63,6 +63,7 @@ target_include_directories(provider_wfs_a PUBLIC target_link_libraries(provider_wfs_a qgis_core + GDAL::GDAL ) # require c++17 @@ -120,6 +121,7 @@ else() target_link_libraries (provider_wfs qgis_core + GDAL::GDAL ) if (WITH_GUI) diff --git a/src/providers/wfs/qgswfsfeatureiterator.cpp b/src/providers/wfs/qgswfsfeatureiterator.cpp index a88e1d7101b7..a10192bb8558 100644 --- a/src/providers/wfs/qgswfsfeatureiterator.cpp +++ b/src/providers/wfs/qgswfsfeatureiterator.cpp @@ -213,12 +213,24 @@ QUrl QgsWFSFeatureDownloaderImpl::buildURL( qint64 startIndex, long long maxFeat arg( minx ).arg( miny ).arg( maxx ).arg( maxy ) ); QgsExpression bboxExp( filterBbox ); QDomDocument bboxDoc; + + QMap fieldNameToXPathMap; + if ( !mShared->mFieldNameToXPathAndIsNestedContentMap.isEmpty() ) + { + for ( auto iterFieldName = mShared->mFieldNameToXPathAndIsNestedContentMap.constBegin(); iterFieldName != mShared->mFieldNameToXPathAndIsNestedContentMap.constEnd(); ++iterFieldName ) + { + const QString &fieldName = iterFieldName.key(); + const auto &value = iterFieldName.value(); + fieldNameToXPathMap[fieldName] = value.first; + } + } + QDomElement bboxElem = QgsOgcUtils::expressionToOgcFilter( bboxExp, bboxDoc, gmlVersion, filterVersion, mShared->mLayerPropertiesList.size() == 1 ? mShared->mLayerPropertiesList[0].mNamespacePrefix : QString(), mShared->mLayerPropertiesList.size() == 1 ? mShared->mLayerPropertiesList[0].mNamespaceURI : QString(), geometryAttribute, mShared->srsName(), - honourAxisOrientation, mShared->mURI.invertAxisOrientation() ); + honourAxisOrientation, mShared->mURI.invertAxisOrientation(), nullptr, fieldNameToXPathMap, mShared->mNamespacePrefixToURIMap ); bboxDoc.appendChild( bboxElem ); filters.push_back( bboxDoc.toString() ); diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index 2405a9988e50..e50d58cfe37e 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -16,13 +16,17 @@ ***************************************************************************/ #include "qgis.h" +#include "qgscplhttpfetchoverrider.h" #include "qgsfeature.h" +#include "qgsfeedback.h" #include "qgsfields.h" #include "qgsgeometry.h" #include "qgscoordinatereferencesystem.h" #include "qgslogger.h" #include "qgsmessagelog.h" #include "qgsogcutils.h" +#include "qgsogrutils.h" +#include "qgssqliteutils.h" #include "qgswfsconstants.h" #include "qgswfsfeatureiterator.h" #include "qgswfsprovider.h" @@ -33,6 +37,10 @@ #include "qgswfsutils.h" #include "qgssettings.h" +#include "cpl_string.h" +#include "gdal.h" + +#include #include #include #include @@ -605,6 +613,8 @@ bool QgsWFSProvider::processSQL( const QString &sqlString, QString &errorMsg, QS QgsFields fields; Qgis::WkbType geomType; if ( !readAttributesFromSchema( describeFeatureDocument, + response, + /* singleLayerContext = */ typenameList.size() == 1, typeName, geometryAttribute, fields, geomType, errorMsg ) ) { @@ -624,7 +634,7 @@ bool QgsWFSProvider::processSQL( const QString &sqlString, QString &errorMsg, QS } } - setLayerPropertiesListFromDescribeFeature( describeFeatureDocument, typenameList, errorMsg ); + setLayerPropertiesListFromDescribeFeature( describeFeatureDocument, response, typenameList, errorMsg ); const QString &defaultTypeName = mShared->mURI.typeName(); QgsWFSProviderSQLColumnRefValidator oColumnValidator( @@ -789,7 +799,7 @@ bool QgsWFSProvider::processSQL( const QString &sqlString, QString &errorMsg, QS return true; } -bool QgsWFSProvider::setLayerPropertiesListFromDescribeFeature( QDomDocument &describeFeatureDocument, const QStringList &typenameList, QString &errorMsg ) +bool QgsWFSProvider::setLayerPropertiesListFromDescribeFeature( QDomDocument &describeFeatureDocument, const QByteArray &response, const QStringList &typenameList, QString &errorMsg ) { mShared->mLayerPropertiesList.clear(); for ( const QString &typeName : typenameList ) @@ -798,6 +808,8 @@ bool QgsWFSProvider::setLayerPropertiesListFromDescribeFeature( QDomDocument &de QgsFields fields; Qgis::WkbType geomType; if ( !readAttributesFromSchema( describeFeatureDocument, + response, + /* singleLayerContext = */ typenameList.size() == 1, typeName, geometryAttribute, fields, geomType, errorMsg ) ) { @@ -1513,6 +1525,8 @@ bool QgsWFSProvider::describeFeatureType( QString &geometryAttribute, QgsFields } if ( !readAttributesFromSchema( describeFeatureDocument, + response, + /* singleLayerContext = */ true, mShared->mURI.typeName(), geometryAttribute, fields, geomType, errorMsg ) ) { @@ -1522,18 +1536,478 @@ bool QgsWFSProvider::describeFeatureType( QString &geometryAttribute, QgsFields return false; } - setLayerPropertiesListFromDescribeFeature( describeFeatureDocument, {mShared->mURI.typeName()}, errorMsg ); + setLayerPropertiesListFromDescribeFeature( describeFeatureDocument, response, {mShared->mURI.typeName()}, errorMsg ); return true; } + bool QgsWFSProvider::readAttributesFromSchema( QDomDocument &schemaDoc, + const QByteArray &response, + bool singleLayerContext, const QString &prefixedTypename, QString &geometryAttribute, QgsFields &fields, Qgis::WkbType &geomType, QString &errorMsg ) { + bool mayTryWithGMLAS = false; + bool ret = readAttributesFromSchemaWithoutGMLAS( schemaDoc, prefixedTypename, geometryAttribute, fields, geomType, errorMsg, mayTryWithGMLAS ); + if ( singleLayerContext && + mayTryWithGMLAS && + GDALGetDriverByName( "GMLAS" ) ) + { + QgsFields fieldsGMLAS; + Qgis::WkbType geomTypeGMLAS; + QString errorMsgGMLAS; + if ( readAttributesFromSchemaWithGMLAS( response, prefixedTypename, geometryAttribute, fieldsGMLAS, geomTypeGMLAS, errorMsgGMLAS ) ) + { + fields = fieldsGMLAS; + geomType = geomTypeGMLAS; + ret = true; + } + else if ( !ret ) + { + errorMsg = errorMsgGMLAS; + } + } + return ret; +} + +static QVariant::Type getVariantTypeFromXML( const QString &xmlType ) +{ + QVariant::Type attributeType = QVariant::Invalid; + + const QString type = QString( xmlType ) + .replace( QLatin1String( "xs:" ), QString() ) + .replace( QLatin1String( "xsd:" ), QString() ); + + if ( type.compare( QLatin1String( "string" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "token" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "NMTOKEN" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "NCName" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "QName" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "ID" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "IDREF" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "anyURI" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "anySimpleType" ), Qt::CaseInsensitive ) == 0 ) + { + attributeType = QVariant::String; + } + else if ( type.compare( QLatin1String( "boolean" ), Qt::CaseInsensitive ) == 0 ) + { + attributeType = QVariant::Bool; + } + else if ( type.compare( QLatin1String( "double" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "float" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "decimal" ), Qt::CaseInsensitive ) == 0 ) + { + attributeType = QVariant::Double; + } + else if ( type.compare( QLatin1String( "byte" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "unsignedByte" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "int" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "short" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "unsignedShort" ), Qt::CaseInsensitive ) == 0 ) + { + attributeType = QVariant::Int; + } + else if ( type.compare( QLatin1String( "long" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "unsignedLong" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "integer" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "negativeInteger" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "nonNegativeInteger" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "positiveInteger" ), Qt::CaseInsensitive ) == 0 ) + { + attributeType = QVariant::LongLong; + } + else if ( type.compare( QLatin1String( "date" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "gYear" ), Qt::CaseInsensitive ) == 0 || + type.compare( QLatin1String( "gYearMonth" ), Qt::CaseInsensitive ) == 0 ) + { + attributeType = QVariant::Date; + } + else if ( type.compare( QLatin1String( "time" ), Qt::CaseInsensitive ) == 0 ) + { + attributeType = QVariant::Time; + } + else if ( type.compare( QLatin1String( "dateTime" ), Qt::CaseInsensitive ) == 0 ) + { + attributeType = QVariant::DateTime; + } + return attributeType; +} + +bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &response, + const QString &prefixedTypename, + QString &geometryAttribute, + QgsFields &fields, + Qgis::WkbType &geomType, + QString &errorMsg ) +{ + QUrl url( mShared->mURI.requestUrl( QStringLiteral( "DescribeFeatureType" ) ) ); + QUrlQuery query( url ); + query.addQueryItem( QStringLiteral( "TYPENAME" ), prefixedTypename ); + url.setQuery( query ); + + // If a previous attempt with the same URL failed because of cancellation + // in the past second, do not retry. + // The main use case for that is when QgsWfsProviderMetadata::querySublayers() + // is called when adding a layer, and several QgsWFSProvider instances are + // quickly created. + static QMutex mutex; + static QUrl lastCanceledURL; + static QDateTime lastCanceledDateTime; + { + QMutexLocker lock( &mutex ); + if ( lastCanceledURL == url && lastCanceledDateTime + 1 > QDateTime::currentDateTime() ) + { + mMetadataRetrievalCanceled = true; + return false; + } + } + + // Create a unique /vsimem/ filename + constexpr int TEMP_FILENAME_SIZE = 128; + void *p = malloc( TEMP_FILENAME_SIZE ); + char *pszSchemaTempFilename = static_cast( p ); + snprintf( pszSchemaTempFilename, TEMP_FILENAME_SIZE, "/vsimem/schema_%p.xsd", p ); + + // Serialize the main schema into a temporary /vsimem/ filename + char *pszSchema = VSIStrdup( response.constData() ); + VSILFILE *fp = VSIFileFromMemBuffer( pszSchemaTempFilename, + reinterpret_cast( pszSchema ), strlen( pszSchema ), /* bTakeOwnership=*/ true ); + if ( fp ) + VSIFCloseL( fp ); + + QgsFeedback feedback; + GDALDatasetH hDS = nullptr; + + // Analyze the DescribeFeatureType response schema with the OGR GMLAS driver + // in a thread, so it can get interrupted (with GDAL 3.9: https://github.com/OSGeo/gdal/pull/9019) + const auto downloaderLambda = [pszSchemaTempFilename, &feedback, &hDS]() + { + QgsCPLHTTPFetchOverrider cplHTTPFetchOverrider( QString(), &feedback ); + QgsSetCPLHTTPFetchOverriderInitiatorClass( cplHTTPFetchOverrider, QStringLiteral( "WFSProviderDownloadSchema" ) ) + + char **papszOpenOptions = nullptr; + papszOpenOptions = CSLSetNameValue( papszOpenOptions, "XSD", pszSchemaTempFilename ); + hDS = GDALOpenEx( "GMLAS:", GDAL_OF_VECTOR, nullptr, papszOpenOptions, nullptr ); + CSLDestroy( papszOpenOptions ); + }; + + std::unique_ptr<_DownloaderThread> downloaderThread = + std::make_unique<_DownloaderThread>( downloaderLambda ); + downloaderThread->start(); + + QTimer timerForHits; + + QMessageBox *box = nullptr; + QWidget *parentWidget = nullptr; + if ( qApp->thread() == QThread::currentThread() ) + { + parentWidget = QApplication::activeWindow(); + if ( !parentWidget ) + { + const QWidgetList widgets = QgsApplication::topLevelWidgets(); + for ( QWidget *widget : widgets ) + { + if ( widget->objectName() == QLatin1String( "QgisApp" ) ) + { + parentWidget = widget; + break; + } + } + } + } + if ( parentWidget ) + { + // Display an information box if within 2 seconds, the schema has not + // been analyzed. + box = new QMessageBox( + QMessageBox::Information, tr( "Information" ), tr( "Download of schemas in progress..." ), + QMessageBox::Cancel, + parentWidget ); +#if GDAL_VERSION_NUM >= GDAL_COMPUTE_VERSION(3,9,0) + connect( box, &QDialog::rejected, &feedback, &QgsFeedback::cancel ); +#else + box->button( QMessageBox::Cancel )->setEnabled( false ); +#endif + + QgsSettings s; + const double settingDefaultValue = 2.0; + const QString settingName = QStringLiteral( "qgis/wfsDownloadSchemasPopupTimeout" ); + if ( !s.contains( settingName ) ) + { + s.setValue( settingName, settingDefaultValue ); + } + const double timeout = s.value( settingName, settingDefaultValue ).toDouble(); + if ( timeout > 0 ) + { + timerForHits.setInterval( static_cast( 1000 * timeout ) ); + timerForHits.setSingleShot( true ); + timerForHits.start(); + connect( &timerForHits, &QTimer::timeout, box, &QDialog::exec ); + } + + // Close dialog when download theread finishes. + // Will actually trigger the QDialog::rejected signal... + connect( downloaderThread.get(), &QThread::finished, box, &QDialog::accept ); + } + + // Run an event loop until download thread finishes + QEventLoop loop; + connect( downloaderThread.get(), &QThread::finished, &loop, &QEventLoop::quit ); + loop.exec( QEventLoop::ExcludeUserInputEvents ); + downloaderThread->wait(); + + VSIUnlink( pszSchemaTempFilename ); + VSIFree( pszSchemaTempFilename ); + + bool ret = hDS != nullptr; + if ( feedback.isCanceled() && !ret ) + { + QMutexLocker lock( &mutex ); + mMetadataRetrievalCanceled = true; + lastCanceledURL = url; + lastCanceledDateTime = QDateTime::currentDateTime(); + errorMsg = tr( "Schema analysis interrupted by user." ); + return false; + } + if ( !ret ) + { + errorMsg = tr( "Cannot analyze schema indicated in DescribeFeatureType response." ); + return false; + } + + gdal::dataset_unique_ptr oDSCloser( hDS ); + + // Retrieve namespace prefix and URIs + OGRLayerH hOtherMetadataLayer = GDALDatasetGetLayerByName( hDS, "_ogr_other_metadata" ); + if ( !hOtherMetadataLayer ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find _ogr_other_metadata layer" ), 4 ); + return false; + } + + auto hOtherMetadataLayerDefn = OGR_L_GetLayerDefn( hOtherMetadataLayer ); + + const int keyIdx = OGR_FD_GetFieldIndex( hOtherMetadataLayerDefn, "key" ); + if ( keyIdx < 0 ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find key field in _ogr_other_metadata" ), 4 ); + return false; + } + + const int valueIdx = OGR_FD_GetFieldIndex( hOtherMetadataLayerDefn, "value" ); + if ( valueIdx < 0 ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find value field in _ogr_other_metadata" ), 4 ); + return false; + } + + std::map> mapPrefixIdxToPrefixAndUri; + while ( true ) + { + gdal::ogr_feature_unique_ptr hFeatureOtherMD( + OGR_L_GetNextFeature( hOtherMetadataLayer ) ); + if ( !hFeatureOtherMD ) + break; + + const QString key = QString::fromUtf8( + OGR_F_GetFieldAsString( hFeatureOtherMD.get(), keyIdx ) ); + const QString value = QString::fromUtf8( + OGR_F_GetFieldAsString( hFeatureOtherMD.get(), valueIdx ) ); + + if ( key.startsWith( QLatin1String( "namespace_prefix_" ) ) ) + { + mapPrefixIdxToPrefixAndUri[key.mid( int( strlen( "namespace_prefix_" ) ) ).toInt()].first = value; + } + else if ( key.startsWith( QLatin1String( "namespace_uri_" ) ) ) + { + mapPrefixIdxToPrefixAndUri[key.mid( int( strlen( "namespace_uri_" ) ) ).toInt()].second = value; + } + } + for ( const auto &kv : mapPrefixIdxToPrefixAndUri ) + { + if ( !kv.second.first.isEmpty() && !kv.second.second.isEmpty() ) + { + mShared->mNamespacePrefixToURIMap[kv.second.first] = kv.second.second; + QgsDebugMsgLevel( QStringLiteral( "%1 -> %2" ).arg( kv.second.first ).arg( kv.second.second ), 4 ); + } + } + + // Find the layer of interest + OGRLayerH hLayersMetadata = GDALDatasetGetLayerByName( hDS, "_ogr_layers_metadata" ); + if ( !hLayersMetadata ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find _ogr_layers_metadata layer" ), 4 ); + return false; + } + OGR_L_SetAttributeFilter( hLayersMetadata, + ( "layer_xpath = " + QgsSqliteUtils::quotedString( prefixedTypename ).toStdString() ).c_str() ); + gdal::ogr_feature_unique_ptr hFeatureLayersMD( OGR_L_GetNextFeature( hLayersMetadata ) ); + if ( !hFeatureLayersMD ) + { + QgsDebugMsgLevel( + QStringLiteral( "Cannot find feature with layer_xpath = %1 in _ogr_layers_metadata" ).arg( prefixedTypename ), 4 ); + return false; + } + const int fldIdx = OGR_F_GetFieldIndex( hFeatureLayersMD.get(), "layer_name" ); + if ( fldIdx < 0 ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find layer_name field in _ogr_layers_metadata" ), 4 ); + return false; + } + const QString layerName = QString::fromUtf8( + OGR_F_GetFieldAsString( hFeatureLayersMD.get(), fldIdx ) ); + + OGRLayerH hLayer = GDALDatasetGetLayerByName( + hDS, layerName.toStdString().c_str() ); + if ( !hLayer ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find %& layer" ).arg( layerName ), 4 ); + return false; + } + + // Get field information + OGRLayerH hFieldsMetadata = GDALDatasetGetLayerByName( hDS, "_ogr_fields_metadata" ); + if ( !hFieldsMetadata ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find _ogr_fields_metadata layer" ), 4 ); + return false; + } + OGR_L_SetAttributeFilter( hFieldsMetadata, + ( "layer_name = " + QgsSqliteUtils::quotedString( layerName ).toStdString() ).c_str() ); + + auto hFieldsMetadataDefn = OGR_L_GetLayerDefn( hFieldsMetadata ); + + const int fieldNameIdx = OGR_FD_GetFieldIndex( hFieldsMetadataDefn, "field_name" ); + if ( fieldNameIdx < 0 ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find field_name field in _ogr_fields_metadata" ), 4 ); + return false; + } + + const int fieldXPathIdx = OGR_FD_GetFieldIndex( hFieldsMetadataDefn, "field_xpath" ); + if ( fieldXPathIdx < 0 ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find field_xpath field in _ogr_fields_metadata" ), 4 ); + return false; + } + + const int fieldIsListIdx = OGR_FD_GetFieldIndex( hFieldsMetadataDefn, "field_is_list" ); + if ( fieldIsListIdx < 0 ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find field_is_list field in _ogr_fields_metadata" ), 4 ); + return false; + } + + const int fieldTypeIdx = OGR_FD_GetFieldIndex( hFieldsMetadataDefn, "field_type" ); + if ( fieldTypeIdx < 0 ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find field_type field in _ogr_fields_metadata" ), 4 ); + return false; + } + + const int fieldCategoryIdx = OGR_FD_GetFieldIndex( hFieldsMetadataDefn, "field_category" ); + if ( fieldCategoryIdx < 0 ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find field_category field in _ogr_fields_metadata" ), 4 ); + return false; + } + + mShared->mFieldNameToXPathAndIsNestedContentMap.clear(); + while ( true ) + { + gdal::ogr_feature_unique_ptr hFeatureFieldsMD( OGR_L_GetNextFeature( hFieldsMetadata ) ); + if ( !hFeatureFieldsMD ) + break; + + QString fieldName = QString::fromUtf8( OGR_F_GetFieldAsString( hFeatureFieldsMD.get(), fieldNameIdx ) ); + const char *fieldXPath = OGR_F_GetFieldAsString( hFeatureFieldsMD.get(), fieldXPathIdx ); + // The xpath includes the one of the feature itself. We can strip it off + const char *slash = strchr( fieldXPath, '/' ); + if ( slash ) + fieldXPath = slash + 1; + const bool fieldIsList = OGR_F_GetFieldAsInteger( hFeatureFieldsMD.get(), fieldIsListIdx ) == 1; + const char *fieldType = OGR_F_GetFieldAsString( hFeatureFieldsMD.get(), fieldTypeIdx ); + const char *fieldCategory = OGR_F_GetFieldAsString( hFeatureFieldsMD.get(), fieldCategoryIdx ); + + // For fields that should be linked to other tables and that we will + // get as JSON, remove the "_pkid" suffix from the name created by GMLAS. + if ( EQUAL( fieldCategory, "PATH_TO_CHILD_ELEMENT_WITH_LINK" ) && + fieldName.endsWith( QLatin1String( "_pkid" ) ) ) + { + fieldName.resize( fieldName.size() - int( strlen( "_pkid" ) ) ); + } + + QgsDebugMsgLevel( + QStringLiteral( "field %1: xpath=%2 is_list=%3 type=%4 category=%5" ). + arg( fieldName ).arg( fieldXPath ).arg( fieldIsList ).arg( fieldType ).arg( fieldCategory ), 5 ); + if ( EQUAL( fieldCategory, "REGULAR" ) && EQUAL( fieldType, "geometry" ) ) + { + if ( geometryAttribute.isEmpty() ) + { + mShared->mFieldNameToXPathAndIsNestedContentMap[fieldName] = + QPair( fieldXPath, false ); + geometryAttribute = fieldName; + geomType = QgsOgrUtils::ogrGeometryTypeToQgsWkbType( + OGR_L_GetGeomType( hLayer ) ); + } + } + else if ( EQUAL( fieldCategory, "REGULAR" ) && !fieldIsList ) + { + QVariant::Type type = getVariantTypeFromXML( QString::fromUtf8( fieldType ) ); + if ( type != QVariant::Invalid ) + { + fields.append( QgsField( fieldName, type, fieldType ) ); + } + else + { + // unhandled:duration, base64Binary, hexBinary, anyType + QgsDebugMsgLevel( + QStringLiteral( "unhandled type for field %1: xpath=%2 is_list=%3 type=%4 category=%5" ). + arg( fieldName ).arg( fieldXPath ).arg( fieldIsList ).arg( fieldType ).arg( fieldCategory ), 3 ); + fields.append( QgsField( fieldName, QVariant::String, fieldType ) ); + } + mShared->mFieldNameToXPathAndIsNestedContentMap[fieldName] = + QPair( fieldXPath, false ); + } + else + { + QgsField field( fieldName, QVariant::String ); + field.setEditorWidgetSetup( QgsEditorWidgetSetup( QStringLiteral( "JsonEdit" ), QVariantMap() ) ); + fields.append( field ); + mShared->mFieldNameToXPathAndIsNestedContentMap[fieldName] = + QPair( fieldXPath, true ); + } + } + + return true; +} + +bool QgsWFSProvider::readAttributesFromSchemaWithoutGMLAS( QDomDocument &schemaDoc, + const QString &prefixedTypename, + QString &geometryAttribute, + QgsFields &fields, + Qgis::WkbType &geomType, + QString &errorMsg, bool &mayTryWithGMLAS ) +{ + mayTryWithGMLAS = false; + //get the root element QDomNodeList schemaNodeList = schemaDoc.elementsByTagNameNS( QgsWFSConstants::XMLSCHEMA_NAMESPACE, QStringLiteral( "schema" ) ); if ( schemaNodeList.length() < 1 ) @@ -1615,6 +2089,7 @@ bool QgsWFSProvider::readAttributesFromSchema( QDomDocument &schemaDoc, if ( foundImport && onlyIncludeOrImport ) { errorMsg = tr( "It is probably a schema for Complex Features." ); + mayTryWithGMLAS = true; } // e.g http://services.cuzk.cz/wfs/inspire-CP-wfs.asp?SERVICE=WFS&VERSION=2.0.0&REQUEST=DescribeFeatureType // which has a single @@ -1643,17 +2118,18 @@ bool QgsWFSProvider::readAttributesFromSchema( QDomDocument &schemaDoc, arg( schemaLocation, errorMsg ); } - return readAttributesFromSchema( describeFeatureDocument, - prefixedTypename, - geometryAttribute, - fields, - geomType, - errorMsg ); + return readAttributesFromSchemaWithoutGMLAS( describeFeatureDocument, + prefixedTypename, + geometryAttribute, + fields, + geomType, + errorMsg, mayTryWithGMLAS ); } else { errorMsg = tr( "Cannot find element '%1'" ).arg( unprefixedTypename ); + mayTryWithGMLAS = true; } return false; } @@ -1772,27 +2248,18 @@ bool QgsWFSProvider::readAttributesFromSchema( QDomDocument &schemaDoc, propertyType = propertyType.at( 0 ).toUpper() + propertyType.mid( 1 ); geomType = geomTypeFromPropertyType( geometryAttribute, propertyType ); } - else if ( !name.isEmpty() ) //todo: distinguish between numerical and non-numerical types + else if ( !name.isEmpty() ) { - QVariant::Type attributeType = QVariant::String; //string is default type - if ( type.contains( QLatin1String( "double" ), Qt::CaseInsensitive ) || type.contains( QLatin1String( "float" ), Qt::CaseInsensitive ) || type.contains( QLatin1String( "decimal" ), Qt::CaseInsensitive ) ) + const QVariant::Type attributeType = getVariantTypeFromXML( type ); + if ( attributeType != QVariant::Invalid ) { - attributeType = QVariant::Double; + fields.append( QgsField( name, attributeType, type ) ); } - else if ( type.contains( QLatin1String( "int" ), Qt::CaseInsensitive ) || - type.contains( QLatin1String( "short" ), Qt::CaseInsensitive ) ) - { - attributeType = QVariant::Int; - } - else if ( type.contains( QLatin1String( "long" ), Qt::CaseInsensitive ) ) - { - attributeType = QVariant::LongLong; - } - else if ( type.contains( QLatin1String( "dateTime" ), Qt::CaseInsensitive ) ) + else { - attributeType = QVariant::DateTime; + mayTryWithGMLAS = true; + fields.append( QgsField( name, QVariant::String, type ) ); } - fields.append( QgsField( name, attributeType, type ) ); } } if ( !foundGeometryAttribute ) diff --git a/src/providers/wfs/qgswfsprovider.h b/src/providers/wfs/qgswfsprovider.h index acc22442f24c..eb698527ca41 100644 --- a/src/providers/wfs/qgswfsprovider.h +++ b/src/providers/wfs/qgswfsprovider.h @@ -138,6 +138,9 @@ class QgsWFSProvider final: public QgsVectorDataProvider //! Perform an initial GetFeature request with a 1-feature limit. void issueInitialGetFeature(); + //! Return whether metadata retrieval has been canceled (typically download of the schema) + bool metadataRetrievalCanceled() const { return mMetadataRetrievalCanceled; } + private slots: void featureReceivedAnalyzeOneFeature( QVector ); @@ -171,11 +174,24 @@ class QgsWFSProvider final: public QgsVectorDataProvider QDomElement geometryElement( const QgsGeometry &geometry, QDomDocument &transactionDoc ); //! Set mShared->mLayerPropertiesList from describeFeatureDocument - bool setLayerPropertiesListFromDescribeFeature( QDomDocument &describeFeatureDocument, const QStringList &typenameList, QString &errorMsg ); + bool setLayerPropertiesListFromDescribeFeature( QDomDocument &describeFeatureDocument, const QByteArray &response, const QStringList &typenameList, QString &errorMsg ); //! backup of mShared->mLayerPropertiesList on the feature type when there is no sql request QList< QgsOgcUtils::LayerProperties > mLayerPropertiesListWhenNoSqlRequest; + //! Set if metadata retrieval has been canceled (typically download of the schema) + bool mMetadataRetrievalCanceled = false; + + bool readAttributesFromSchemaWithoutGMLAS( QDomDocument &schemaDoc, + const QString &prefixedTypename, + QString &geometryAttribute, + QgsFields &fields, Qgis::WkbType &geomType, QString &errorMsg, bool &mayTryWithGMLAS ); + + bool readAttributesFromSchemaWithGMLAS( const QByteArray &response, + const QString &prefixedTypename, + QString &geometryAttribute, + QgsFields &fields, Qgis::WkbType &geomType, QString &errorMsg ); + protected: //! String used to define a subset of the layer @@ -206,6 +222,8 @@ class QgsWFSProvider final: public QgsVectorDataProvider * thematic attributes and their types from a dom document. Returns true in case of success. */ bool readAttributesFromSchema( QDomDocument &schemaDoc, + const QByteArray &response, + bool singleLayerContext, const QString &prefixedTypename, QString &geometryAttribute, QgsFields &fields, Qgis::WkbType &geomType, QString &errorMsg ); diff --git a/src/providers/wfs/qgswfsprovidermetadata.cpp b/src/providers/wfs/qgswfsprovidermetadata.cpp index 8edef0c520f8..a4f79bc03cc4 100644 --- a/src/providers/wfs/qgswfsprovidermetadata.cpp +++ b/src/providers/wfs/qgswfsprovidermetadata.cpp @@ -236,6 +236,8 @@ QList QgsWfsProviderMetadata::querySublayers( const QgsWFSProvider provider( uri + " " + QgsWFSConstants::URI_PARAM_SKIP_INITIAL_GET_FEATURE + "='true'", QgsDataProvider::ProviderOptions(), caps ); + if ( provider.metadataRetrievalCanceled() ) + return res; QgsProviderSublayerDetails details; details.setType( Qgis::LayerType::Vector ); details.setProviderKey( QgsWFSProvider::WFS_PROVIDER_KEY ); diff --git a/src/providers/wfs/qgswfsshareddata.cpp b/src/providers/wfs/qgswfsshareddata.cpp index 0cce64716cd5..2c332b056426 100644 --- a/src/providers/wfs/qgswfsshareddata.cpp +++ b/src/providers/wfs/qgswfsshareddata.cpp @@ -54,6 +54,8 @@ QgsWFSSharedData *QgsWFSSharedData::clone() const copy->mGeometryAttribute = mGeometryAttribute; copy->mLayerPropertiesList = mLayerPropertiesList; copy->mMapFieldNameToSrcLayerNameFieldName = mMapFieldNameToSrcLayerNameFieldName; + copy->mFieldNameToXPathAndIsNestedContentMap = mFieldNameToXPathAndIsNestedContentMap; + copy->mNamespacePrefixToURIMap = mNamespacePrefixToURIMap; copy->mPageSize = mPageSize; copy->mCaps = mCaps; copy->mHasWarnedAboutMissingFeatureId = mHasWarnedAboutMissingFeatureId; @@ -105,8 +107,23 @@ QString QgsWFSSharedData::computedExpression( const QgsExpression &expression ) bool honourAxisOrientation = false; getVersionValues( gmlVersion, filterVersion, honourAxisOrientation ); + QMap fieldNameToXPathMap; + if ( !mFieldNameToXPathAndIsNestedContentMap.isEmpty() ) + { + for ( auto iterFieldName = mFieldNameToXPathAndIsNestedContentMap.constBegin(); iterFieldName != mFieldNameToXPathAndIsNestedContentMap.constEnd(); ++iterFieldName ) + { + const QString &fieldName = iterFieldName.key(); + const auto &value = iterFieldName.value(); + fieldNameToXPathMap[fieldName] = value.first; + } + } + QDomDocument expressionDoc; - QDomElement expressionElem = QgsOgcUtils::expressionToOgcExpression( expression, expressionDoc, gmlVersion, filterVersion, mGeometryAttribute, srsName(), honourAxisOrientation, mURI.invertAxisOrientation(), nullptr, true ); + QDomElement expressionElem = QgsOgcUtils::expressionToOgcExpression( + expression, expressionDoc, gmlVersion, filterVersion, mGeometryAttribute, + srsName(), honourAxisOrientation, mURI.invertAxisOrientation(), nullptr, + true, + fieldNameToXPathMap, mNamespacePrefixToURIMap ); if ( !expressionElem.isNull() ) { @@ -155,12 +172,23 @@ bool QgsWFSSharedData::computeFilter( QString &errorMsg ) } } + QMap fieldNameToXPathMap; + if ( !mFieldNameToXPathAndIsNestedContentMap.isEmpty() ) + { + for ( auto iterFieldName = mFieldNameToXPathAndIsNestedContentMap.constBegin(); iterFieldName != mFieldNameToXPathAndIsNestedContentMap.constEnd(); ++iterFieldName ) + { + const QString &fieldName = iterFieldName.key(); + const auto &value = iterFieldName.value(); + fieldNameToXPathMap[fieldName] = value.first; + } + } + QDomDocument filterDoc; const QDomElement filterElem = QgsOgcUtils::SQLStatementToOgcFilter( sql, filterDoc, gmlVersion, filterVersion, mLayerPropertiesList, honourAxisOrientation, mURI.invertAxisOrientation(), mCaps.mapUnprefixedTypenameToPrefixedTypename, - &errorMsg ); + &errorMsg, fieldNameToXPathMap, mNamespacePrefixToURIMap ); if ( !errorMsg.isEmpty() ) { errorMsg = tr( "SQL statement to OGC Filter error: " ) + errorMsg; @@ -188,13 +216,24 @@ bool QgsWFSSharedData::computeFilter( QString &errorMsg ) //if not, if must be a QGIS expression const QgsExpression filterExpression( filter ); + QMap fieldNameToXPathMap; + if ( !mFieldNameToXPathAndIsNestedContentMap.isEmpty() ) + { + for ( auto iterFieldName = mFieldNameToXPathAndIsNestedContentMap.constBegin(); iterFieldName != mFieldNameToXPathAndIsNestedContentMap.constEnd(); ++iterFieldName ) + { + const QString &fieldName = iterFieldName.key(); + const auto &value = iterFieldName.value(); + fieldNameToXPathMap[fieldName] = value.first; + } + } + const QDomElement filterElem = QgsOgcUtils::expressionToOgcFilter( filterExpression, filterDoc, gmlVersion, filterVersion, mLayerPropertiesList.size() == 1 ? mLayerPropertiesList[0].mNamespacePrefix : QString(), mLayerPropertiesList.size() == 1 ? mLayerPropertiesList[0].mNamespaceURI : QString(), mGeometryAttribute, srsName(), honourAxisOrientation, mURI.invertAxisOrientation(), - &errorMsg ); + &errorMsg, fieldNameToXPathMap, mNamespacePrefixToURIMap ); if ( !errorMsg.isEmpty() ) { @@ -261,11 +300,16 @@ QgsGmlStreamingParser *QgsWFSSharedData::createParser() const } else { - return new QgsGmlStreamingParser( mURI.typeName(), - mGeometryAttribute, - mFields, - axisOrientationLogic, - mURI.invertAxisOrientation() ); + auto parser = new QgsGmlStreamingParser( mURI.typeName(), + mGeometryAttribute, + mFields, + axisOrientationLogic, + mURI.invertAxisOrientation() ); + if ( !mFieldNameToXPathAndIsNestedContentMap.isEmpty() ) + { + parser->setFieldsXPath( mFieldNameToXPathAndIsNestedContentMap, mNamespacePrefixToURIMap ); + } + return parser; } } @@ -329,6 +373,29 @@ QString QgsWFSSharedData::combineWFSFilters( const std::vector &filters } envelopeFilterDoc.firstChildElement().appendChild( andElem ); + QSet setNamespaceURI; + for ( auto iterFieldName = mFieldNameToXPathAndIsNestedContentMap.constBegin(); iterFieldName != mFieldNameToXPathAndIsNestedContentMap.constEnd(); ++iterFieldName ) + { + const auto &value = iterFieldName.value(); + const QStringList parts = value.first.split( '/' ); + for ( const QString &part : parts ) + { + const QStringList subparts = part.split( ':' ); + if ( subparts.size() == 2 && subparts[0] != QLatin1String( "gml" ) ) + { + const auto iter = mNamespacePrefixToURIMap.constFind( subparts[0] ); + if ( iter != mNamespacePrefixToURIMap.constEnd() && + !setNamespaceURI.contains( *iter ) ) + { + setNamespaceURI.insert( *iter ); + QDomAttr attr = envelopeFilterDoc.createAttribute( QStringLiteral( "xmlns:" ) + subparts[0] ); + attr.setValue( *iter ); + envelopeFilterDoc.firstChildElement().setAttributeNode( attr ); + } + } + } + } + if ( mLayerPropertiesList.size() == 1 && envelopeFilterDoc.firstChildElement().hasAttribute( QStringLiteral( "xmlns:" ) + mLayerPropertiesList[0].mNamespacePrefix ) ) { @@ -337,7 +404,6 @@ QString QgsWFSSharedData::combineWFSFilters( const std::vector &filters else { // add xmls:PREFIX=URI attributes to top element - QSet setNamespaceURI; for ( const QgsOgcUtils::LayerProperties &props : std::as_const( mLayerPropertiesList ) ) { if ( !props.mNamespacePrefix.isEmpty() && !props.mNamespaceURI.isEmpty() && diff --git a/src/providers/wfs/qgswfsshareddata.h b/src/providers/wfs/qgswfsshareddata.h index df5082ac665e..654e20684c52 100644 --- a/src/providers/wfs/qgswfsshareddata.h +++ b/src/providers/wfs/qgswfsshareddata.h @@ -97,6 +97,12 @@ class QgsWFSSharedData : public QObject, public QgsBackgroundCachedSharedData //! Map a field name to the pair (typename, fieldname) that describes its source field QMap< QString, QPair > mMapFieldNameToSrcLayerNameFieldName; + //! Map a field name to the pair (xpath, isNestedContent) + QMap > mFieldNameToXPathAndIsNestedContentMap; + + //! Map a namespace prefix to its URI + QMap mNamespacePrefixToURIMap; + //! Page size for WFS 2.0. 0 = disabled long long mPageSize = 0; diff --git a/tests/src/python/test_provider_wfs.py b/tests/src/python/test_provider_wfs.py index bd72c415010a..4113f51ddf8a 100644 --- a/tests/src/python/test_provider_wfs.py +++ b/tests/src/python/test_provider_wfs.py @@ -51,11 +51,18 @@ QObject, Qt, QTime, + QVariant, ) + import unittest from qgis.testing import start_app, QgisTestCase from utilities import compareWkt, unitTestDataPath +from osgeo import gdal + +# Default value is 2 second, which is too short when run under Valgrind +gdal.SetConfigOption('OGR_GMLAS_XERCES_MAX_TIME', '20') + TEST_DATA_DIR = unitTestDataPath() @@ -6427,6 +6434,53 @@ def testDeegreeServerWithUnknownGeometryTypeErrorSituation(self): self.assertEqual(sublayers[3].featureCount(), -1) self.assertEqual(sublayers[4].featureCount(), -1) + @unittest.skipIf(gdal.GetDriverByName("GMLAS") is None, "OGR GMLAS driver required") + def testWFSComplexFeatures(self): + """Test reading complex features""" + + endpoint = self.__class__.basetestpath + '/fake_qgis_http_endpoint_WFS_complex_features' + + shutil.copy(os.path.join(TEST_DATA_DIR, 'provider', 'wfs', 'inspire_complexfeatures', 'getcapabilities.xml'), sanitize(endpoint, '?SERVICE=WFS?REQUEST=GetCapabilities&VERSION=2.0.0')) + shutil.copy(os.path.join(TEST_DATA_DIR, 'provider', 'wfs', 'inspire_complexfeatures', 'describefeaturetype.xml'), sanitize(endpoint, '?SERVICE=WFS&REQUEST=DescribeFeatureType&VERSION=2.0.0&TYPENAMES=ps:ProtectedSite&TYPENAME=ps:ProtectedSite')) + shutil.copy(os.path.join(TEST_DATA_DIR, 'provider', 'wfs', 'inspire_complexfeatures', 'getfeature_hits.xml'), sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=ps:ProtectedSite&RESULTTYPE=hits')) + shutil.copy(os.path.join(TEST_DATA_DIR, 'provider', 'wfs', 'inspire_complexfeatures', 'getfeature.xml'), sanitize(endpoint, '?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=ps:ProtectedSite&SRSNAME=urn:ogc:def:crs:EPSG::25833')) + + vl = QgsVectorLayer("url='http://" + endpoint + "' typename='ps:ProtectedSite' version='2.0.0' skipInitialGetFeature='true'", 'test', 'WFS') + self.assertTrue(vl.isValid()) + self.assertEqual(vl.featureCount(), 1228) + self.assertEqual(len(vl.fields()), 26) + self.assertEqual([field.name() for field in vl.fields()], ['id', 'metadataproperty', 'description_href', 'description_title', 'description_nilreason', 'description', 'descriptionreference_href', 'descriptionreference_title', 'descriptionreference_nilreason', 'identifier_codespace', 'identifier', 'name', 'location_location', 'inspireid_identifier_localid', 'inspireid_identifier_namespace', 'inspireid_identifier_versionid_nilreason', 'inspireid_identifier_versionid_nil', 'inspireid_identifier_versionid', 'legalfoundationdate_nilreason', 'legalfoundationdate', 'legalfoundationdocument_nilreason', 'legalfoundationdocument_owns', 'legalfoundationdocument_ci_citation', 'sitedesignation', 'sitename', 'siteprotectionclassification']) + self.assertEqual(vl.fields()["sitedesignation"].type(), QVariant.String) + + got_f = [f for f in vl.getFeatures()] + self.assertEqual(len(got_f), 1) + geom = got_f[0].geometry() + self.assertFalse(geom.isNull()) + self.assertEqual(got_f[0]["id"], 'ProtectedSite_FFH_553_DE4546-303') + self.assertEqual(got_f[0]["sitedesignation"], '{"ps:DesignationType":{"ps:designation":{"@xlink:href":"http://inspire.ec.europa.eu/codelist/Natura2000DesignationValue/specialAreaOfConservation"},"ps:designationScheme":{"@xlink:href":"http://inspire.ec.europa.eu/codelist/DesignationSchemeValue/natura2000"}}}') + + vl = QgsVectorLayer("url='http://" + endpoint + "' typename='ps:ProtectedSite' version='2.0.0' skipInitialGetFeature='true'", 'test', 'WFS') + vl.setSubsetString("inspireid_identifier_localid = 'ProtectedSite_FFH_553_DE4546'") + + if int(QT_VERSION_STR.split('.')[0]) >= 6: + attrs = 'xmlns:base="http://inspire.ec.europa.eu/schemas/base/3.3" xmlns:ps="http://inspire.ec.europa.eu/schemas/ps/4.0"' + else: + attrs = 'xmlns:ps="http://inspire.ec.europa.eu/schemas/ps/4.0" xmlns:base="http://inspire.ec.europa.eu/schemas/base/3.3"' + with open(sanitize(endpoint, + f"""?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=ps:ProtectedSite&SRSNAME=urn:ogc:def:crs:EPSG::25833&FILTER= + + ps:inspireID/base:Identifier/base:localId + ProtectedSite_FFH_553_DE4546 + + +"""), + 'wb') as f: + with open(os.path.join(TEST_DATA_DIR, 'provider', 'wfs', 'inspire_complexfeatures', 'getfeature.xml'), "rb") as f_source: + f.write(f_source.read()) + + got_f = [f for f in vl.getFeatures()] + self.assertEqual(len(got_f), 1) + if __name__ == '__main__': unittest.main() diff --git a/tests/testdata/provider/wfs/inspire_complexfeatures/describefeaturetype.xml b/tests/testdata/provider/wfs/inspire_complexfeatures/describefeaturetype.xml new file mode 100644 index 000000000000..65b9794b8d55 --- /dev/null +++ b/tests/testdata/provider/wfs/inspire_complexfeatures/describefeaturetype.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/tests/testdata/provider/wfs/inspire_complexfeatures/getcapabilities.xml b/tests/testdata/provider/wfs/inspire_complexfeatures/getcapabilities.xml new file mode 100644 index 000000000000..56635efeb0af --- /dev/null +++ b/tests/testdata/provider/wfs/inspire_complexfeatures/getcapabilities.xml @@ -0,0 +1,652 @@ + + + + INSPIRE Download Service: Protected Sites / Schutzgebiete des Landes Brandenburg (WFS-PS-SCHUTZG) + Der interoperable INSPIRE-Downloaddienst (WFS) Protected Sites (Schutzgebiete des Landes Brandenburg) beinhaltet Informationen zu den Schutzgebieten nach Naturschutzrecht des Landes Brandenburg und Europäische Schutzgebiete. Zu den Schutzgebieten nach Naturschutzrecht des Landes Brandenburg zählen Naturschutzgebiete, Landschaftsschutzgebiete, Biosphärenreservate, Naturparks und Nationalparke. Zum europäischen Schutzgebietssystem Natura 2000 zählen Vogelschutzgebiete (Special Protection Area (SPA)) und Fauna-Flora-Habitat-Gebiete (FFH-Gebiete). Gemäß der INSPIRE-Datenspezifikation Protected Sites (D2.8.I.9_v3.2) liegen die Inhalte der Schutzgebiete INSPIRE-konform vor. Der WFS beinhaltet den FeatureType ProtectedSite. --- The compliant INSPIRE download service (WFS) Protected Sites (Schutzgebiete des Landes Brandenburg) provides information on protected sites in the state of Brandenburg that are legally protected by state environmental law or according to the European Natura 2000 protected areas network. Areas legally protected by state environmental law include nature reserves, protected landscapes, biosphere reserves, nature parks and national parks. Areas according to the European Natura 2000 network include Special Protection Areas (SPA) for birds and Special Areas of Conservation (SAC) defined by the European Union's Habitats Directive (92/43/EEC). The content of the protected areas is compliant to the INSPIRE data specification for the annex theme Protected Sites (D2.8.I.9_v3.2). The WFS consists of the FeatureType ProtectedSite. + + Schutzgebiet + Naturschutz + Natur + Umwelt + Biosphärenreservat + Landschaft + Landschaftsschutzgebiet + Nationalpark + Naturpark + Naturschutzgebiet + Naturschutzrecht + Vogelschutzgebiet + Schutzgebiete + Großschutzgebiet + ProtectedSite + Brandenburg + WFS + interoperabel + interoperability + inspireidentifiziert + + WFS + 2.0.0 + Nutzung erfolgt derzeit kostenfrei unter Beachtung des Urheberrechts. + Es gelten die Bedingungen der Datenlizenz Deutschland – Namensnennung – Version 2.0: https://www.govdata.de/dl-de/by-2-0. Als Bezeichnung des Bereitstellers ist „© Landesamt für Umwelt Brandenburg“ anzugeben. + + + Landesvermessung und Geobasisinformation Brandenburg (LGB) + + + INSPIRE-Zentrale im Land Brandenburg + Kundenservice + + + +49-331-8844-123 + +49-331-8844-16123 + + + Heinrich-Mann-Allee 104 B + Potsdam + Brandenburg + 14473 + Deutschland + kundenservice@inspire.brandenburg.de + + + 24/7 + + + ServiceCenter + + + + + + ps:ProtectedSite + ProtectedSite + Naturschutzgebiete (NSG), Landschaftsschutzgebiete (LSG), Nationalparke (NatP), Naturparke (NP), Biospärenreservate (BR), Fauna-Flora-Habitat-Gebiete (FFH) und Vogelschutzgebiete (SPA, Special Protection Area) des Landes Brandenburg + http://www.opengis.net/def/crs/EPSG/0/25833 + http://www.opengis.net/def/crs/EPSG/0/25832 + http://www.opengis.net/def/crs/EPSG/0/4326 + http://www.opengis.net/def/crs/EPSG/0/4258 + http://www.opengis.net/def/crs/EPSG/0/3034 + http://www.opengis.net/def/crs/EPSG/0/3035 + http://www.opengis.net/def/crs/EPSG/0/3044 + http://www.opengis.net/def/crs/EPSG/0/3045 + http://www.opengis.net/def/crs/EPSG/0/3857 + http://www.opengis.net/def/crs/EPSG/0/4839 + + application/gml+xml; version=3.2 + text/xml; subtype=gml/3.2.1 + + + 11.071522 51.292131 + 14.777398 53.617170 + + + + + + + + + TRUE + + + + TRUE + + + + TRUE + + + + TRUE + + + + TRUE + + + + TRUE + + + + TRUE + + + + TRUE + + + + TRUE + + + + TRUE + + + + FALSE + + + + TRUE + + + + FALSE + + + + TRUE + + + + FALSE + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + xsd:double + + + gml:_Geometry + + + + + xsd:double + + + gml32:AbstractGeometry + + + + + gml:Point + + + gml:_Geometry + + + + + gml32:Point + + + gml32:AbstractGeometry + + + + + xsd:anyType + + + xsd:string + + + xsd:anyType + + + + + xsd:anyType + + + xsd:string + + + + + gml:_Geometry + + + xsd:string + + + xsd:string + + + + + gml32:AbstractGeometry + + + xsd:string + + + xsd:string + + + + + xsd:double + + + xsd:double + + + xsd:double + + + xsd:double + + + + + xsd:integer + + + xsd:integer + + + xsd:integer + + + + + xsd:integer + + + xsd:integer + + + xsd:integer + + + + + gml:Point + + + gml:_Geometry + + + + + gml32:Point + + + gml32:AbstractGeometry + + + + + xsd:boolean + + + gml:_Geometry + + + + + xsd:boolean + + + gml32:AbstractGeometry + + + + + xsd:boolean + + + gml:_Geometry + + + + + xsd:boolean + + + gml32:AbstractGeometry + + + + + xsd:boolean + + + gml:_Geometry + + + + + xsd:boolean + + + gml32:AbstractGeometry + + + + + xsd:double + + + gml:_Geometry + + + + + xsd:double + + + gml32:AbstractGeometry + + + + + xsd:integer + + + xsd:double + + + + + xsd:double + + + gml:_Geometry + + + + + xsd:double + + + gml32:AbstractGeometry + + + + + xsd:integer + + + xsd:double + + + + + + diff --git a/tests/testdata/provider/wfs/inspire_complexfeatures/getfeature.xml b/tests/testdata/provider/wfs/inspire_complexfeatures/getfeature.xml new file mode 100644 index 000000000000..63c0894220e6 --- /dev/null +++ b/tests/testdata/provider/wfs/inspire_complexfeatures/getfeature.xml @@ -0,0 +1,59 @@ + + + + + + https://registry.gdi-de.org/id/de.bb.inspire.ps.schutzg/ProtectedSite_FFH_553_DE4546-303 + + + + + + 393417.723 5703577.814 393373.225 5703627.160 393355.567 5703647.893 393059.411 5703439.935 392935.194 5703238.164 392910.426 5703197.931 392748.503 5703042.996 392607.511 5702922.719 392568.349 5702894.811 392510.694 5702830.645 392494.856 5702773.768 392479.853 5702712.855 392464.829 5702651.443 392419.821 5702468.704 392405.314 5702444.287 392374.601 5702415.032 392317.941 5702387.341 392275.050 5702366.089 392244.501 5702340.829 392230.178 5702320.907 392212.278 5702299.130 392194.242 5702261.852 392177.115 5702173.514 392168.678 5702126.338 392170.581 5702111.753 392173.880 5702094.610 392168.231 5702078.834 392154.072 5702062.907 392138.272 5702043.546 392120.281 5702031.777 392096.319 5702020.754 392051.341 5702009.592 391982.372 5701998.913 391908.470 5701989.937 391815.141 5701970.752 391763.622 5701970.863 391732.955 5701967.118 391702.766 5701962.853 391675.512 5701956.967 391636.232 5701950.573 391594.038 5701946.300 391567.222 5701938.895 391522.887 5701931.208 391461.032 5701923.239 391411.142 5701914.279 391377.895 5701908.639 391340.695 5701904.161 391303.514 5701900.182 391272.601 5701890.444 391251.434 5701898.816 391230.366 5701885.172 391188.154 5701843.883 391165.473 5701815.299 391150.049 5701792.921 391126.053 5701768.893 391086.318 5701727.001 391041.242 5701676.825 390984.667 5701614.614 390944.993 5701574.221 390904.301 5701533.369 390872.966 5701501.138 390840.837 5701473.942 390827.136 5701456.995 390787.463 5701416.602 390740.978 5701356.478 390722.189 5701325.234 390712.013 5701296.637 390691.381 5700793.249 390735.499 5700661.380 390804.628 5700407.431 390815.234 5700385.486 390821.415 5700353.218 390821.570 5700308.191 390823.253 5700251.595 390826.024 5700184.951 390750.464 5700196.552 390739.130 5700127.484 390737.877 5700072.510 390741.111 5700029.357 390736.727 5699959.004 390727.022 5699880.865 390720.018 5699868.647 390718.180 5699848.213 390731.148 5699810.664 390754.653 5699761.678 390768.267 5699752.116 390819.267 5699690.499 390838.363 5699668.206 390845.788 5699641.890 390849.329 5699606.228 390852.408 5699547.075 390853.845 5699484.487 390856.887 5699387.818 390863.167 5699211.479 390869.103 5699112.190 390875.285 5698957.865 390881.127 5698905.102 390882.965 5698803.479 390882.585 5698720.956 390887.277 5698676.744 390876.847 5698654.161 390861.219 5698626.788 390849.209 5698602.269 390826.233 5698578.699 390800.515 5698549.238 390780.678 5698529.041 390747.887 5698497.870 390692.925 5698450.599 390652.351 5698449.260 390627.075 5698442.792 390576.722 5698410.339 390548.720 5698373.969 390517.660 5698336.224 390486.051 5698309.506 390436.492 5698272.018 390388.019 5698260.998 390366.156 5698252.389 390357.233 5698217.739 390361.363 5698172.048 390359.759 5698145.102 390348.556 5698128.052 390317.794 5698109.804 390304.262 5698060.335 390292.449 5697991.787 390292.708 5697949.256 390280.915 5697881.207 390369.881 5697891.569 390416.611 5697896.658 390448.339 5697901.861 390454.806 5697937.613 390471.224 5697996.468 390486.491 5698063.874 390502.971 5698124.227 390514.718 5698166.765 390527.903 5698207.745 390553.805 5698241.700 390606.736 5698276.048 390674.093 5698332.816 390816.316 5698446.544 390900.389 5698520.635 390921.707 5698564.782 390956.902 5698581.348 391089.748 5698685.956 391280.702 5698829.201 391600.508 5699077.217 391631.830 5699096.943 391654.703 5699118.016 391718.339 5699169.433 391862.834 5699277.564 391895.920 5699303.721 392014.517 5699402.408 391999.969 5699315.965 391990.207 5699199.811 391960.425 5698997.939 391907.753 5698860.033 391946.996 5698889.940 391967.442 5698900.607 391990.230 5698895.171 391997.615 5698867.856 391998.331 5698824.307 392000.996 5698791.683 392028.283 5698749.546 392069.495 5698729.849 392090.404 5698702.981 392100.584 5698695.060 392142.807 5698748.855 392140.645 5698805.970 392140.198 5698880.522 392145.511 5698961.341 392167.475 5699094.503 392185.223 5699185.817 392182.493 5699253.460 392172.030 5699278.900 392171.268 5699296.940 392167.768 5699333.600 392165.509 5699351.701 392160.261 5699418.946 392149.172 5699477.927 392135.786 5699517.494 392126.293 5699554.399 392128.553 5699597.326 392126.559 5699621.919 392130.071 5699658.792 392134.680 5699673.610 392140.124 5699684.392 392146.313 5699701.146 392150.149 5699721.498 392158.213 5699735.174 392168.868 5699763.251 392175.924 5699788.973 392180.137 5699806.309 392182.638 5699830.717 392190.685 5699868.405 392150.042 5699877.574 392161.360 5699909.625 392162.932 5699923.567 392162.936 5699948.078 392161.777 5699968.635 392162.752 5700004.611 392165.355 5700031.517 392178.323 5700054.997 392177.704 5700076.532 392182.531 5700108.849 392187.242 5700126.164 392188.601 5700147.118 392187.381 5700166.176 392181.716 5700174.412 392180.271 5700187.977 392184.446 5700228.825 392194.008 5700242.440 392205.781 5700248.960 392218.695 5700258.936 392231.041 5700279.439 392238.617 5700305.641 392233.149 5700330.876 392221.982 5700351.343 392228.437 5700496.645 392273.882 5700494.783 392325.159 5700513.191 392313.092 5700572.712 392334.152 5700598.361 392345.671 5700610.895 392350.821 5700626.691 392373.461 5700654.276 392393.502 5700679.466 392412.422 5700701.702 392422.913 5700725.783 392433.863 5700748.845 392441.931 5700787.032 392453.269 5700819.582 392475.472 5700848.685 392492.946 5700884.486 392526.127 5700925.145 392563.557 5700959.627 392596.807 5700989.779 392630.475 5701017.912 392667.659 5701046.402 392714.274 5701097.515 392728.159 5701118.956 392742.093 5701129.390 392759.330 5701147.192 392767.762 5701169.857 392776.002 5701200.033 392785.188 5701216.664 392789.216 5701229.505 392783.645 5701252.243 392785.536 5701286.181 392785.729 5701339.698 392788.226 5701400.623 392786.679 5701472.719 392788.947 5701503.640 392792.222 5701522.515 392792.054 5701555.036 392823.458 5701576.760 392866.308 5701597.013 392812.399 5701624.234 392787.024 5701639.781 392775.632 5701654.754 392764.138 5701667.231 392752.595 5701690.715 392747.880 5701709.917 392748.281 5701731.910 392738.351 5701770.334 392724.954 5701797.396 392725.629 5701813.875 392643.125 5701973.328 392657.698 5702023.754 392683.166 5702022.710 392653.572 5702093.955 392601.432 5702152.118 392579.627 5702181.525 392486.501 5702386.934 392483.366 5702408.072 392487.652 5702439.411 392530.904 5702603.713 392566.395 5702737.320 392575.572 5702765.957 392596.195 5702793.124 392634.501 5702824.569 392690.482 5702872.296 392750.471 5702932.365 392786.304 5702927.895 392883.874 5703013.936 392978.010 5703101.618 393027.168 5703178.138 393128.389 5703359.122 393129.729 5703361.518 393350.448 5703523.046 393417.723 5703577.814 + + + + + 392388.973 5701374.469 392385.166 5701403.638 392375.162 5701428.059 392318.559 5701438.383 392280.880 5701434.425 392191.051 5701439.607 392134.195 5701455.944 392090.916 5701486.231 392075.042 5701513.894 392058.464 5701536.584 392033.417 5701560.121 392016.503 5701586.826 392008.386 5701620.674 392003.245 5701653.900 391992.852 5701668.832 391971.898 5701670.192 391951.211 5701678.043 391933.683 5701689.766 391931.006 5701709.885 391922.091 5701724.257 391912.218 5701739.669 391903.241 5701752.542 391891.931 5701769.513 391888.694 5701788.155 391885.227 5701837.819 391883.217 5701886.424 391886.463 5701916.805 391991.716 5701934.000 392072.855 5701948.683 392106.498 5701951.806 392144.497 5701975.760 392165.545 5701988.903 392213.188 5702028.469 392222.374 5702045.101 392232.583 5702086.701 392238.762 5702139.523 392241.945 5702167.448 392245.449 5702200.200 392248.220 5702216.688 392252.308 5702236.463 392264.151 5702265.280 392276.620 5702284.172 392287.511 5702299.272 392301.460 5702314.836 392313.047 5702325.095 392332.774 5702335.490 392372.413 5702351.193 392400.320 5702364.522 392426.175 5702376.610 392440.002 5702383.276 392451.108 5702358.705 392469.537 5702311.906 392488.528 5702268.914 392507.985 5702222.863 392525.997 5702182.682 392543.296 5702146.070 392555.948 5702123.003 392563.452 5702095.433 392570.086 5702041.684 392576.321 5701995.824 392563.979 5701873.025 392556.840 5701784.278 392486.463 5701763.652 392493.417 5701725.850 392506.155 5701658.297 392511.152 5701560.548 392512.470 5701458.447 392521.127 5701425.578 392514.144 5701413.858 392443.247 5701392.753 392388.973 5701374.469 + + + + + 390948.449 5698619.212 390958.133 5698696.851 390950.855 5698751.174 390945.709 5698820.918 390944.312 5698884.504 390943.718 5698931.050 390943.706 5698979.573 390940.472 5699022.725 390937.794 5699103.872 390931.829 5699214.668 390929.692 5699296.794 390925.463 5699364.498 390924.579 5699440.569 390929.724 5699492.883 390931.635 5699527.320 390915.577 5699611.517 390912.187 5699638.669 390947.034 5699646.745 390941.942 5699668.964 390945.094 5699684.842 390959.776 5699725.760 390967.366 5699764.467 390956.985 5699791.905 390929.849 5699825.532 390885.236 5699859.875 390822.684 5699895.954 390807.633 5699907.076 390821.927 5699938.505 390854.023 5700013.725 390864.240 5700043.320 390878.338 5700179.806 390869.717 5700433.276 390851.235 5700531.579 390840.113 5700577.556 390820.494 5700635.886 390792.753 5700764.583 390780.215 5700812.618 390785.156 5700859.938 390786.376 5700901.908 390784.415 5700939.505 390788.812 5700961.335 390788.063 5700991.880 390786.233 5701020.468 390789.655 5701067.350 390791.915 5701110.277 390794.543 5701162.194 390794.257 5701216.230 390797.941 5701245.093 390799.402 5701268.544 390804.134 5701286.358 390819.976 5701306.719 390855.093 5701345.798 390876.387 5701364.935 390888.884 5701376.928 390905.602 5701394.251 390935.623 5701431.038 390956.917 5701450.175 390977.691 5701468.832 391166.102 5701684.213 391220.262 5701748.523 391251.221 5701783.771 391279.608 5701805.118 391300.382 5701823.775 391326.781 5701833.198 391796.794 5701901.472 391797.420 5701867.931 391833.216 5701825.945 391835.332 5701804.348 391835.205 5701776.841 391832.074 5701761.462 391839.724 5701740.639 391855.815 5701730.475 391876.593 5701712.616 391888.067 5701699.640 391900.539 5701686.623 391902.860 5701670.020 391926.450 5701647.543 391942.144 5701639.897 391953.658 5701627.920 391967.772 5701618.337 391976.810 5701606.961 391983.789 5701594.170 391988.709 5701579.962 391993.588 5701564.755 392000.842 5701546.449 391995.459 5701537.165 391987.743 5701531.979 392000.379 5701522.957 392026.691 5701505.872 392049.782 5701483.415 392105.713 5701420.095 392123.842 5701410.848 392142.144 5701393.590 392213.130 5701343.659 392256.250 5701333.888 392297.802 5701334.686 392326.623 5701330.003 392350.348 5701323.028 392363.062 5701291.492 392368.653 5701269.253 392353.168 5701245.377 392328.579 5701206.867 392318.800 5701175.753 392330.691 5701160.759 392347.932 5701142.044 392340.048 5701108.352 392308.779 5700980.074 392295.520 5700925.092 392269.167 5700880.152 392231.680 5700807.654 392198.994 5700681.436 392180.128 5700611.677 392146.345 5700568.541 392137.393 5700545.397 392129.783 5700506.191 392197.974 5700497.894 392190.144 5700477.705 392184.610 5700415.904 392173.271 5700322.326 392165.064 5700244.126 392158.789 5700200.863 392154.470 5700156.520 392148.171 5700088.247 392124.655 5699941.643 392136.914 5699935.638 392137.471 5699912.605 392132.445 5699899.805 392123.660 5699905.167 392118.613 5699891.868 392096.313 5699762.723 392079.538 5699707.384 392070.054 5699671.256 392041.434 5699534.366 392019.465 5699437.722 391677.237 5699179.621 391642.566 5699188.045 391638.874 5699171.188 391650.348 5699158.213 391632.837 5699145.924 390948.449 5698619.212 + + + + + + + ProtectedSite_FFH_553_DE4546-303 + https://registry.gdi-de.org/id/de.bb.inspire.ps.schutzg + + + + + + + + + + + + + deu + + + + + + + Große Röder + latn + + + + + natureConservation + + + \ No newline at end of file diff --git a/tests/testdata/provider/wfs/inspire_complexfeatures/getfeature_hits.xml b/tests/testdata/provider/wfs/inspire_complexfeatures/getfeature_hits.xml new file mode 100644 index 000000000000..8e345f70c4d4 --- /dev/null +++ b/tests/testdata/provider/wfs/inspire_complexfeatures/getfeature_hits.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file From b46a4520a847f2d73102850445a13040c5ef480a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 16 Jan 2024 19:30:58 +0100 Subject: [PATCH 05/21] [WFS provider] Better take into account case of GML geometry element (readAttributesFromSchemaWithGMLAS) --- src/providers/wfs/qgswfsprovider.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index e50d58cfe37e..ca1d43839d16 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -1963,7 +1963,17 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon { mShared->mFieldNameToXPathAndIsNestedContentMap[fieldName] = QPair( fieldXPath, false ); - geometryAttribute = fieldName; + geometryAttribute = fieldXPath; + { + const auto parts = geometryAttribute.split( '/' ); + if ( parts.size() > 1 ) + geometryAttribute = parts[0]; + } + { + const auto parts = geometryAttribute.split( ':' ); + if ( parts.size() == 2 ) + geometryAttribute = parts[1]; + } geomType = QgsOgrUtils::ogrGeometryTypeToQgsWkbType( OGR_L_GetGeomType( hLayer ) ); } From c67d4e0f731d968d181caaeac06b83bb6d3fd0b8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 17 Jan 2024 13:32:09 +0100 Subject: [PATCH 06/21] [WFS provider] Better take into account case where geometry element is optional (minOccurs=0) --- src/providers/wfs/qgswfsprovider.cpp | 42 +++++++++++--- src/providers/wfs/qgswfsprovider.h | 22 +++++-- src/providers/wfs/qgswfsprovidermetadata.cpp | 61 ++++++++++++++++---- 3 files changed, 101 insertions(+), 24 deletions(-) diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index ca1d43839d16..267e0cda7768 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -133,7 +133,7 @@ QgsWFSProvider::QgsWFSProvider( const QString &uri, const ProviderOptions &optio //fetch attributes of layer and type of its geometry attribute //WBC 111221: extracting geometry type here instead of getFeature allows successful //layer creation even when no features are retrieved (due to, e.g., BBOX or FILTER) - if ( !describeFeatureType( mShared->mGeometryAttribute, mShared->mFields, mShared->mWKBType ) ) + if ( !describeFeatureType( mShared->mGeometryAttribute, mShared->mFields, mShared->mWKBType, mGeometryMaybeMissing ) ) { mValid = false; return; @@ -612,11 +612,16 @@ bool QgsWFSProvider::processSQL( const QString &sqlString, QString &errorMsg, QS QString geometryAttribute; QgsFields fields; Qgis::WkbType geomType; + bool geometryMaybeMissing; if ( !readAttributesFromSchema( describeFeatureDocument, response, /* singleLayerContext = */ typenameList.size() == 1, typeName, - geometryAttribute, fields, geomType, errorMsg ) ) + geometryAttribute, + fields, + geomType, + geometryMaybeMissing, + errorMsg ) ) { errorMsg = tr( "Analysis of DescribeFeatureType response failed for url %1, typeName %2: %3" ). arg( dataSourceUri(), typeName, errorMsg ); @@ -630,6 +635,7 @@ bool QgsWFSProvider::processSQL( const QString &sqlString, QString &errorMsg, QS { mShared->mGeometryAttribute = geometryAttribute; mShared->mWKBType = geomType; + mGeometryMaybeMissing = geometryMaybeMissing; mThisTypenameFields = fields; } } @@ -807,11 +813,12 @@ bool QgsWFSProvider::setLayerPropertiesListFromDescribeFeature( QDomDocument &de QString geometryAttribute; QgsFields fields; Qgis::WkbType geomType; + bool geometryMaybeMissing; if ( !readAttributesFromSchema( describeFeatureDocument, response, /* singleLayerContext = */ typenameList.size() == 1, typeName, - geometryAttribute, fields, geomType, errorMsg ) ) + geometryAttribute, fields, geomType, geometryMaybeMissing, errorMsg ) ) { errorMsg = tr( "Analysis of DescribeFeatureType response failed for url %1, typeName %2: %3" ). arg( dataSourceUri(), typeName, errorMsg ); @@ -1496,7 +1503,7 @@ void QgsWFSProvider::handlePostCloneOperations( QgsVectorDataProvider *source ) mShared = qobject_cast( source )->mShared; }; -bool QgsWFSProvider::describeFeatureType( QString &geometryAttribute, QgsFields &fields, Qgis::WkbType &geomType ) +bool QgsWFSProvider::describeFeatureType( QString &geometryAttribute, QgsFields &fields, Qgis::WkbType &geomType, bool &geometryMaybeMissing ) { fields.clear(); @@ -1528,7 +1535,7 @@ bool QgsWFSProvider::describeFeatureType( QString &geometryAttribute, QgsFields response, /* singleLayerContext = */ true, mShared->mURI.typeName(), - geometryAttribute, fields, geomType, errorMsg ) ) + geometryAttribute, fields, geomType, geometryMaybeMissing, errorMsg ) ) { QgsDebugMsgLevel( response, 4 ); QgsMessageLog::logMessage( tr( "Analysis of DescribeFeatureType response failed for url %1: %2" ). @@ -1549,8 +1556,10 @@ bool QgsWFSProvider::readAttributesFromSchema( QDomDocument &schemaDoc, QString &geometryAttribute, QgsFields &fields, Qgis::WkbType &geomType, + bool &geometryMaybeMissing, QString &errorMsg ) { + geometryMaybeMissing = false; bool mayTryWithGMLAS = false; bool ret = readAttributesFromSchemaWithoutGMLAS( schemaDoc, prefixedTypename, geometryAttribute, fields, geomType, errorMsg, mayTryWithGMLAS ); if ( singleLayerContext && @@ -1560,7 +1569,7 @@ bool QgsWFSProvider::readAttributesFromSchema( QDomDocument &schemaDoc, QgsFields fieldsGMLAS; Qgis::WkbType geomTypeGMLAS; QString errorMsgGMLAS; - if ( readAttributesFromSchemaWithGMLAS( response, prefixedTypename, geometryAttribute, fieldsGMLAS, geomTypeGMLAS, errorMsgGMLAS ) ) + if ( readAttributesFromSchemaWithGMLAS( response, prefixedTypename, geometryAttribute, fieldsGMLAS, geomTypeGMLAS, geometryMaybeMissing, errorMsgGMLAS ) ) { fields = fieldsGMLAS; geomType = geomTypeGMLAS; @@ -1643,8 +1652,11 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon QString &geometryAttribute, QgsFields &fields, Qgis::WkbType &geomType, + bool &geometryMaybeMissing, QString &errorMsg ) { + geometryMaybeMissing = false; + QUrl url( mShared->mURI.requestUrl( QStringLiteral( "DescribeFeatureType" ) ) ); QUrlQuery query( url ); query.addQueryItem( QStringLiteral( "TYPENAME" ), prefixedTypename ); @@ -1913,6 +1925,14 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon return false; } + const int fieldMinOccursIdx = OGR_FD_GetFieldIndex( hFieldsMetadataDefn, "field_min_occurs" ); + if ( fieldMinOccursIdx < 0 ) + { + // should not happen + QgsDebugMsgLevel( QStringLiteral( "Cannot find field_min_occurs field in _ogr_fields_metadata" ), 4 ); + return false; + } + const int fieldTypeIdx = OGR_FD_GetFieldIndex( hFieldsMetadataDefn, "field_type" ); if ( fieldTypeIdx < 0 ) { @@ -1974,8 +1994,14 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon if ( parts.size() == 2 ) geometryAttribute = parts[1]; } - geomType = QgsOgrUtils::ogrGeometryTypeToQgsWkbType( - OGR_L_GetGeomType( hLayer ) ); + geomType = QgsWkbTypes::multiType( QgsOgrUtils::ogrGeometryTypeToQgsWkbType( + OGR_L_GetGeomType( hLayer ) ) ); + if ( geomType == Qgis::WkbType::MultiPolygon ) + geomType = Qgis::WkbType::MultiSurface; + else if ( geomType == Qgis::WkbType::MultiLineString ) + geomType = Qgis::WkbType::MultiCurve; + + geometryMaybeMissing = OGR_F_GetFieldAsInteger( hFeatureFieldsMD.get(), fieldMinOccursIdx ) == 0; } } else if ( EQUAL( fieldCategory, "REGULAR" ) && !fieldIsList ) diff --git a/src/providers/wfs/qgswfsprovider.h b/src/providers/wfs/qgswfsprovider.h index eb698527ca41..02e1afc2190d 100644 --- a/src/providers/wfs/qgswfsprovider.h +++ b/src/providers/wfs/qgswfsprovider.h @@ -141,6 +141,9 @@ class QgsWFSProvider final: public QgsVectorDataProvider //! Return whether metadata retrieval has been canceled (typically download of the schema) bool metadataRetrievalCanceled() const { return mMetadataRetrievalCanceled; } + //! Return whether the geometry may be missing + bool geometryMaybeMissing() const { return mGeometryMaybeMissing; } + private slots: void featureReceivedAnalyzeOneFeature( QVector ); @@ -161,6 +164,9 @@ class QgsWFSProvider final: public QgsVectorDataProvider //! Field set by featureReceivedAnalyzeOneFeature() if a "name" field is set in the sample feature bool mSampleFeatureHasName = false; + //! Whether the geometry may be missing + bool mGeometryMaybeMissing = false; + /** * Invalidates cache of shared object */ @@ -185,12 +191,16 @@ class QgsWFSProvider final: public QgsVectorDataProvider bool readAttributesFromSchemaWithoutGMLAS( QDomDocument &schemaDoc, const QString &prefixedTypename, QString &geometryAttribute, - QgsFields &fields, Qgis::WkbType &geomType, QString &errorMsg, bool &mayTryWithGMLAS ); + QgsFields &fields, Qgis::WkbType &geomType, + QString &errorMsg, bool &mayTryWithGMLAS ); bool readAttributesFromSchemaWithGMLAS( const QByteArray &response, const QString &prefixedTypename, QString &geometryAttribute, - QgsFields &fields, Qgis::WkbType &geomType, QString &errorMsg ); + QgsFields &fields, + Qgis::WkbType &geomType, + bool &geometryMaybeMissing, + QString &errorMsg ); protected: @@ -215,7 +225,8 @@ class QgsWFSProvider final: public QgsVectorDataProvider * the geometry attribute and the thematic attributes with their types. */ bool describeFeatureType( QString &geometryAttribute, - QgsFields &fields, Qgis::WkbType &geomType ); + QgsFields &fields, Qgis::WkbType &geomType, + bool &geometryMaybeMissing ); /** * For a given typename, reads the name of the geometry attribute, the @@ -226,7 +237,10 @@ class QgsWFSProvider final: public QgsVectorDataProvider bool singleLayerContext, const QString &prefixedTypename, QString &geometryAttribute, - QgsFields &fields, Qgis::WkbType &geomType, QString &errorMsg ); + QgsFields &fields, + Qgis::WkbType &geomType, + bool &geometryMaybeMissing, + QString &errorMsg ); //helper methods for WFS-T diff --git a/src/providers/wfs/qgswfsprovidermetadata.cpp b/src/providers/wfs/qgswfsprovidermetadata.cpp index a4f79bc03cc4..63ff9ebbbafd 100644 --- a/src/providers/wfs/qgswfsprovidermetadata.cpp +++ b/src/providers/wfs/qgswfsprovidermetadata.cpp @@ -255,10 +255,12 @@ QList QgsWfsProviderMetadata::querySublayers( const return res; } - if ( provider.wkbType() == Qgis::WkbType::Unknown && + if ( ( provider.wkbType() == Qgis::WkbType::Unknown || + ( provider.wkbType() != Qgis::WkbType::NoGeometry && + provider.geometryMaybeMissing() ) ) && provider.sharedData()->layerProperties().size() == 1 ) { - std::vector> requests; + std::map> requests; std::set finishedRequests; constexpr int INDEX_ALL = 0; constexpr int INDEX_NULL = 1; @@ -273,13 +275,42 @@ QList QgsWfsProviderMetadata::querySublayers( const QStringLiteral( "IsSurface" ) }; + constexpr int INDEX_GEOMETRYCOLLECTION = 5; + std::vector featureCounts( INDEX_GEOMETRYCOLLECTION + 1, -1 ); + const auto downloaderLambda = [ &, feedback]() { QEventLoop loop; QTimer timerForHits; - for ( QString function : filterNames ) + for ( int i = 0; i <= INDEX_SURFACE; ++ i ) { + if ( provider.wkbType() == Qgis::WkbType::MultiPoint ) + { + if ( i != INDEX_ALL && i != INDEX_NULL && i != INDEX_POINT ) + { + featureCounts[i] = 0; + continue; + } + } + else if ( provider.wkbType() == Qgis::WkbType::MultiCurve ) + { + if ( i != INDEX_ALL && i != INDEX_NULL && i != INDEX_CURVE ) + { + featureCounts[i] = 0; + continue; + } + } + else if ( provider.wkbType() == Qgis::WkbType::MultiSurface ) + { + if ( i != INDEX_ALL && i != INDEX_NULL && i != INDEX_SURFACE ) + { + featureCounts[i] = 0; + continue; + } + } + QString filter; + const QString &function = filterNames[i]; if ( function == QLatin1String( "IsNull" ) ) { filter = QgsWFSProvider::buildIsNullGeometryFilter( caps, provider.geometryAttribute() ); @@ -296,16 +327,16 @@ QList QgsWfsProviderMetadata::querySublayers( const filter = provider.sharedData()->combineWFSFilters( {filter, provider.sharedData()->WFSFilter()} ); } - requests.emplace_back( std::make_unique( wfsUri ) ); + requests[i] = std::make_unique( wfsUri ); + QgsWFSGetFeature *thisRequest = requests[i].get(); - requests.back()->request( /* synchronous = */ false, + thisRequest->request( /* synchronous = */ false, caps.version, wfsUri.typeName(), filter, /* hitsOnly = */ true, caps ); - QgsWFSGetFeature *thisRequest = requests.back().get(); const auto downloadFinishedLambda = [ &, thisRequest]() { finishedRequests.insert( thisRequest ); @@ -331,9 +362,9 @@ QList QgsWfsProviderMetadata::querySublayers( const loop.exec( QEventLoop::ExcludeUserInputEvents ); // Make sure to terminate requests in this thread, to avoid potential // crash in main thread when "requests" goes out of scope. - for ( auto &request : requests ) + for ( auto &pair : requests ) { - request->abort(); + pair.second->abort(); } }; @@ -342,15 +373,14 @@ QList QgsWfsProviderMetadata::querySublayers( const downloaderThread->start(); downloaderThread->wait(); - constexpr int INDEX_GEOMETRYCOLLECTION = 5; - std::vector featureCounts( INDEX_GEOMETRYCOLLECTION + 1, -1 ); bool countsAllValid = false; if ( finishedRequests.size() == requests.size() ) { countsAllValid = true; - for ( size_t i = 0; i < requests.size(); ++i ) + for ( const auto &pair : requests ) { - QByteArray data = requests[i]->response(); + const int i = pair.first; + QByteArray data = pair.second->response(); QgsGmlStreamingParser gmlParser( ( QString() ), ( QString() ), QgsFields() ); QString errorMsg; if ( gmlParser.processData( data, true, errorMsg ) ) @@ -386,6 +416,13 @@ QList QgsWfsProviderMetadata::querySublayers( const res.clear(); for ( const auto &tuple : types ) { + if ( provider.wkbType() == Qgis::WkbType::MultiPoint && tuple.index != INDEX_NULL && tuple.index != INDEX_POINT ) + continue; + if ( provider.wkbType() == Qgis::WkbType::MultiCurve && tuple.index != INDEX_NULL && tuple.index != INDEX_CURVE ) + continue; + if ( provider.wkbType() == Qgis::WkbType::MultiSurface && tuple.index != INDEX_NULL && tuple.index != INDEX_SURFACE ) + continue; + if ( !countsAllValid || featureCounts[tuple.index] > 0 || ( tuple.wkbType == Qgis::WkbType::NoGeometry && featureCounts[INDEX_ALL] == 0 ) ) { From e3d24f81c4c08d894d30f30d4e35355eb8f1777a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 17 Jan 2024 14:44:16 +0100 Subject: [PATCH 07/21] [WFS provider] Set OGR GMLAS XSD cache in a subdirectory of 'cache/directory' setting, and make it clearable through Settings -> Options-> Network -> Cache Settings --- src/app/options/qgsoptions.cpp | 14 ++++++++ src/providers/wfs/qgswfsprovider.cpp | 52 ++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/src/app/options/qgsoptions.cpp b/src/app/options/qgsoptions.cpp index 36ae57767b76..393684fa9198 100644 --- a/src/app/options/qgsoptions.cpp +++ b/src/app/options/qgsoptions.cpp @@ -2263,6 +2263,20 @@ void QgsOptions::browseCacheDirectory() void QgsOptions::clearCache() { QgsNetworkAccessManager::instance()->cache()->clear(); + + // Clear WFS XSD cache used by OGR GMLAS driver + QString cacheDirectory = mSettings->value( QStringLiteral( "cache/directory" ) ).toString(); + if ( cacheDirectory.isEmpty() ) + cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation ); + if ( !cacheDirectory.endsWith( QDir::separator() ) ) + { + cacheDirectory.push_back( QDir::separator() ); + } + // Must be kept in sync with QgsWFSProvider::readAttributesFromSchemaWithGMLAS() + cacheDirectory += QLatin1String( "gmlas_xsd_cache" ); + QDir dir( cacheDirectory ); + dir.removeRecursively(); + QMessageBox::information( this, tr( "Clear Cache" ), tr( "Content cache has been cleared." ) ); } diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index 267e0cda7768..09359ab39043 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +56,7 @@ #include #include #include +#include #include @@ -1704,6 +1706,56 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon char **papszOpenOptions = nullptr; papszOpenOptions = CSLSetNameValue( papszOpenOptions, "XSD", pszSchemaTempFilename ); + + QgsSettings settings; + QString cacheDirectory = settings.value( QStringLiteral( "cache/directory" ) ).toString(); + if ( cacheDirectory.isEmpty() ) + cacheDirectory = QStandardPaths::writableLocation( QStandardPaths::CacheLocation ); + if ( !cacheDirectory.endsWith( QDir::separator() ) ) + { + cacheDirectory.push_back( QDir::separator() ); + } + // Must be kept in sync with QgsOptions::clearCache() + cacheDirectory += QLatin1String( "gmlas_xsd_cache" ); + QgsDebugMsgLevel( QStringLiteral( "cacheDirectory = %1" ).arg( cacheDirectory ), 4 ); + char *pszEscaped = CPLEscapeString( cacheDirectory.toStdString().c_str(), -1, CPLES_XML ); + QString config = QStringLiteral( "%1" + "" + " true" + " " + " " + " " + " " + " " + " gml:boundedBy" + " gml32:boundedBy" + " gml:priorityLocation" + " gml32:priorityLocation" + " gml32:descriptionReference/@owns" + " @xlink:show" + " @xlink:type" + " @xlink:role" + " @xlink:arcrole" + " @xlink:actuate" + " @gml:remoteSchema" + " @gml32:remoteSchema" + " swe:Quantity/swe:extension" + " swe:Quantity/@referenceFrame" + " swe:Quantity/@axisID" + " swe:Quantity/@updatable" + " swe:Quantity/@optional" + " swe:Quantity/@id" + " swe:Quantity/swe:identifier" + " " + " swe:Quantity/swe:label" + " swe:Quantity/swe:nilValues" + " swe:Quantity/swe:constraint" + " swe:Quantity/swe:quality" + "" + "" ).arg( pszEscaped ); + CPLFree( pszEscaped ); + papszOpenOptions = CSLSetNameValue( papszOpenOptions, "CONFIG_FILE", config.toStdString().c_str() ); + hDS = GDALOpenEx( "GMLAS:", GDAL_OF_VECTOR, nullptr, papszOpenOptions, nullptr ); CSLDestroy( papszOpenOptions ); }; From 1b436dfe2b1b3e59d2ade62c5a90b21c79a1ae6d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 20 Jan 2024 12:32:20 +0100 Subject: [PATCH 08/21] [WFS provider] Silence harmless warnings coming from GMLAS driver --- src/providers/wfs/qgswfsprovider.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index 09359ab39043..885bbf089428 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -1649,6 +1649,18 @@ static QVariant::Type getVariantTypeFromXML( const QString &xmlType ) return attributeType; } +static void CPL_STDCALL QgsWFSProviderGMLASErrorHandler( CPLErr eErr, CPLErrorNum /*eErrorNum*/, const char *pszErrorMsg ) +{ + // Silence harmless warnings like "GeographicalName_pronunciation_PronunciationOfName_pronunciationSoundLink_nilReason identifier truncated to geographicalname_pronunciation_pronunciationofname_pronunciatio" + if ( !( eErr == CE_Warning && strstr( pszErrorMsg, " truncated to " ) ) ) + { + if ( eErr == CE_Failure ) + QgsMessageLog::logMessage( QObject::tr( "GMLAS error: %1" ).arg( pszErrorMsg ), QObject::tr( "WFS" ) ); + else + QgsDebugMsgLevel( QStringLiteral( "GMLAS eErr=%1, msg=%2" ).arg( eErr ).arg( pszErrorMsg ), 4 ); + } +} + bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &response, const QString &prefixedTypename, QString &geometryAttribute, @@ -1756,7 +1768,9 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon CPLFree( pszEscaped ); papszOpenOptions = CSLSetNameValue( papszOpenOptions, "CONFIG_FILE", config.toStdString().c_str() ); + CPLPushErrorHandler( QgsWFSProviderGMLASErrorHandler ); hDS = GDALOpenEx( "GMLAS:", GDAL_OF_VECTOR, nullptr, papszOpenOptions, nullptr ); + CPLPopErrorHandler(); CSLDestroy( papszOpenOptions ); }; From 95e3e372e3caf77c7fa59a8bbbbc98237968fecf Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 20 Jan 2024 12:59:36 +0100 Subject: [PATCH 09/21] QgsGmlStreamingParser: fix quadratic performance pattern when parsing huge JSON content Addresses issue with https://github.com/qgis/QGIS/pull/55847#issuecomment-1902043499 and parsing of "https://www.wfs.nrw.de/geobasis/wfs_nw_inspire-gewaesser-netzwerk_atkis-basis-dlm?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=net:Network&COUNT=1&SRSNAME=urn:ogc:def:crs:EPSG::25832&NAMESPACES=xmlns(net,http://inspire.ec.europa.eu/schemas/net/4.0)&NAMESPACE=xmlns(net,http://inspire.ec.europa.eu/schemas/net/4.0" which is a 44 MB single WFS feature... --- src/core/qgsgml.cpp | 21 ++++++++++++--------- src/core/qgsgml.h | 2 ++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/core/qgsgml.cpp b/src/core/qgsgml.cpp index 2d64e139d9b7..7d3c7962b1dc 100644 --- a/src/core/qgsgml.cpp +++ b/src/core/qgsgml.cpp @@ -1116,18 +1116,14 @@ void QgsGmlStreamingParser::endElement( const XML_Char *el ) if ( mAttributeValIsNested ) { - //find index with attribute name - const QMap >::const_iterator att_it = mThematicAttributes.constFind( mAttributeName ); - Q_ASSERT( mCurrentFeature ); - const int attrIndex = att_it.value().first; - auto attrVal = mCurrentFeature->attribute( attrIndex ); - if ( attrVal.isNull() ) + auto iter = mMapFieldNameToJSONContent.find( mAttributeName ); + if ( iter == mMapFieldNameToJSONContent.end() ) { - mCurrentFeature->setAttribute( attrIndex, QString::fromStdString( mAttributeJson.dump() ) ); + mMapFieldNameToJSONContent[mAttributeName] = QString::fromStdString( mAttributeJson.dump() ); } else { - QString str = attrVal.toString(); + QString &str = iter.value(); if ( str[0] == '[' && str.back() == ']' ) { str.back() = ','; @@ -1139,7 +1135,6 @@ void QgsGmlStreamingParser::endElement( const XML_Char *el ) } str.append( QString::fromStdString( mAttributeJson.dump() ) ); str.append( ']' ); - mCurrentFeature->setAttribute( attrIndex, str ); } } else @@ -1249,6 +1244,14 @@ void QgsGmlStreamingParser::endElement( const XML_Char *el ) } mCurrentFeature->setValid( true ); + for ( auto iter = mMapFieldNameToJSONContent.constBegin(); iter != mMapFieldNameToJSONContent.constEnd(); ++iter ) + { + const QMap >::const_iterator att_it = mThematicAttributes.constFind( iter.key() ); + const int attrIndex = att_it.value().first; + mCurrentFeature->setAttribute( attrIndex, iter.value() ); + } + mMapFieldNameToJSONContent.clear(); + mFeatureList.push_back( QgsGmlFeaturePtrGmlIdPair( mCurrentFeature, mCurrentFeatureId ) ); mCurrentFeature = nullptr; diff --git a/src/core/qgsgml.h b/src/core/qgsgml.h index ca24317e8cba..9d1ed3458d8f 100644 --- a/src/core/qgsgml.h +++ b/src/core/qgsgml.h @@ -334,6 +334,8 @@ class CORE_EXPORT QgsGmlStreamingParser QString mAttributeName; int mAttributeDepth = -1; bool mAttributeValIsNested = false; + //! Map from field name to JSON content. + QMap< QString, QString > mMapFieldNameToJSONContent; nlohmann::json mAttributeJson; QStack mAttributeJsonCurrentStack; char mEndian; From 3c846a4ffabfd70d84fed5b89bfc2694d3a8e0da Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 20 Jan 2024 20:50:57 +0100 Subject: [PATCH 10/21] [WFS provider] readAttributesFromSchemaWithGMLAS(): make sure geomType is properly initialized to NoGeometry when there's no geometry field --- src/providers/wfs/qgswfsprovider.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index 885bbf089428..fbe499ee9792 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -1669,6 +1669,7 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon bool &geometryMaybeMissing, QString &errorMsg ) { + geomType = Qgis::WkbType::NoGeometry; geometryMaybeMissing = false; QUrl url( mShared->mURI.requestUrl( QStringLiteral( "DescribeFeatureType" ) ) ); From 4a12c842e9741e3cee7e324312f4fe0e51b496b7 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 21 Jan 2024 00:05:40 +0100 Subject: [PATCH 11/21] JSONEditWidget: fix performance issues with too large arrays in tree view mode --- src/gui/editorwidgets/qgsjsoneditwidget.cpp | 57 +++++++++++++++++---- src/gui/editorwidgets/qgsjsoneditwidget.h | 2 + 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/gui/editorwidgets/qgsjsoneditwidget.cpp b/src/gui/editorwidgets/qgsjsoneditwidget.cpp index 15976e955238..5981300aa944 100644 --- a/src/gui/editorwidgets/qgsjsoneditwidget.cpp +++ b/src/gui/editorwidgets/qgsjsoneditwidget.cpp @@ -268,12 +268,32 @@ void QgsJsonEditWidget::refreshTreeView( const QJsonDocument &jsonDocument ) } else if ( jsonDocument.isArray() ) { - for ( int index = 0; index < jsonDocument.array().size(); index++ ) + const QJsonArray array = jsonDocument.array(); + const auto arraySize = array.size(); + // Limit the number of rows we display, otherwise for pathological cases + // like https://github.com/qgis/QGIS/pull/55847#issuecomment-1902077683 + // a unbounded number of elements will just stall the GUI forever. + constexpr decltype( arraySize ) MAX_ELTS = 200; + // If there are too many elements, disable URL highighting as it + // performs very poorly. + if ( arraySize > MAX_ELTS ) + mEnableUrlHighlighting = false; + for ( auto index = decltype( arraySize ) {0}; index < arraySize; index++ ) { QTreeWidgetItem *treeWidgetItem = new QTreeWidgetItem( mTreeWidget, QStringList() << QString::number( index ) ); - refreshTreeViewItem( treeWidgetItem, jsonDocument.array().at( index ) ); - mTreeWidget->addTopLevelItem( treeWidgetItem ); - mTreeWidget->expandItem( treeWidgetItem ); + if ( arraySize <= MAX_ELTS || ( index < MAX_ELTS / 2 || index + MAX_ELTS / 2 > arraySize ) ) + { + refreshTreeViewItem( treeWidgetItem, array.at( index ) ); + mTreeWidget->addTopLevelItem( treeWidgetItem ); + mTreeWidget->expandItem( treeWidgetItem ); + } + else if ( index == MAX_ELTS / 2 ) + { + index = arraySize - MAX_ELTS / 2; + refreshTreeViewItem( treeWidgetItem, tr( "... truncated ..." ) ); + mTreeWidget->addTopLevelItem( treeWidgetItem ); + mTreeWidget->expandItem( treeWidgetItem ); + } } } @@ -304,7 +324,7 @@ void QgsJsonEditWidget::refreshTreeViewItem( QTreeWidgetItem *treeWidgetItem, co case QJsonValue::String: { const QString jsonValueString = jsonValue.toString(); - if ( QUrl( jsonValueString ).scheme().isEmpty() ) + if ( !mEnableUrlHighlighting || QUrl( jsonValueString ).scheme().isEmpty() ) { refreshTreeViewItemValue( treeWidgetItem, jsonValueString, @@ -327,12 +347,31 @@ void QgsJsonEditWidget::refreshTreeViewItem( QTreeWidgetItem *treeWidgetItem, co case QJsonValue::Array: { const QJsonArray jsonArray = jsonValue.toArray(); - for ( int index = 0; index < jsonArray.size(); index++ ) + const auto arraySize = jsonArray.size(); + // Limit the number of rows we display, otherwise for pathological cases + // like https://github.com/qgis/QGIS/pull/55847#issuecomment-1902077683 + // a unbounded number of elements will just stall the GUI forever. + constexpr decltype( arraySize ) MAX_ELTS = 200; + // If there are too many elements, disable URL highighting as it + // performs very poorly. + if ( arraySize > MAX_ELTS ) + mEnableUrlHighlighting = false; + for ( auto index = decltype( arraySize ) {0}; index < arraySize; index++ ) { QTreeWidgetItem *treeWidgetItemChild = new QTreeWidgetItem( treeWidgetItem, QStringList() << QString::number( index ) ); - refreshTreeViewItem( treeWidgetItemChild, jsonArray.at( index ) ); - treeWidgetItem->addChild( treeWidgetItemChild ); - treeWidgetItem->setExpanded( true ); + if ( arraySize <= MAX_ELTS || ( index < MAX_ELTS / 2 || index + MAX_ELTS / 2 > arraySize ) ) + { + refreshTreeViewItem( treeWidgetItemChild, jsonArray.at( index ) ); + treeWidgetItem->addChild( treeWidgetItemChild ); + treeWidgetItem->setExpanded( true ); + } + else if ( index == MAX_ELTS / 2 ) + { + index = arraySize - MAX_ELTS / 2; + refreshTreeViewItem( treeWidgetItemChild, tr( "... truncated ..." ) ); + treeWidgetItem->addChild( treeWidgetItemChild ); + treeWidgetItem->setExpanded( true ); + } } } break; diff --git a/src/gui/editorwidgets/qgsjsoneditwidget.h b/src/gui/editorwidgets/qgsjsoneditwidget.h index 488efa9fd9fb..89bf1cb421a9 100644 --- a/src/gui/editorwidgets/qgsjsoneditwidget.h +++ b/src/gui/editorwidgets/qgsjsoneditwidget.h @@ -124,6 +124,8 @@ class GUI_EXPORT QgsJsonEditWidget : public QWidget, private Ui::QgsJsonEditWidg QAction *mCopyValueAction; QAction *mCopyKeyAction; + + bool mEnableUrlHighlighting = true; }; #endif // QGSJSONEDITWIDGET_H From c87df1fa6109ef8df0ddfbf6e1f8d4925da45d82 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 21 Jan 2024 14:09:22 +0100 Subject: [PATCH 12/21] [GUI] Attribute table: truncate too large strings to avoid performance issues --- src/gui/attributetable/qgsattributetablemodel.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/gui/attributetable/qgsattributetablemodel.cpp b/src/gui/attributetable/qgsattributetablemodel.cpp index d099d8af0ed8..f3a453035e4f 100644 --- a/src/gui/attributetable/qgsattributetablemodel.cpp +++ b/src/gui/attributetable/qgsattributetablemodel.cpp @@ -734,7 +734,15 @@ QVariant QgsAttributeTableModel::data( const QModelIndex &index, int role ) cons case Qt::DisplayRole: { const WidgetData &widgetData = getWidgetData( index.column() ); - return widgetData.fieldFormatter->representValue( mLayer, fieldId, widgetData.config, widgetData.cache, val ); + QString s = widgetData.fieldFormatter->representValue( mLayer, fieldId, widgetData.config, widgetData.cache, val ); + // In table view, too long strings kill performance. Just truncate them + constexpr int MAX_STRING_LENGTH = 10 * 1000; + if ( static_cast( s.size() ) > static_cast( MAX_STRING_LENGTH ) ) + { + s.resize( MAX_STRING_LENGTH ); + s.append( tr( "... truncated ..." ) ); + } + return s; } case Qt::ToolTipRole: { From c61231f67fb8c4840082f9b8c1ba9f75c8a6ab35 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 21 Jan 2024 14:09:31 +0100 Subject: [PATCH 13/21] [WFS provider] Change feature paging enabled option to be tri-state (default, enabled, disabled), to allow a user to effectively override server capabilities --- .../qgsnewhttpconnection.sip.in | 1 + .../qgsnewhttpconnection.sip.in | 1 + src/core/qgsowsconnection.cpp | 2 +- src/core/qgsowsconnection.h | 2 +- src/gui/qgsmanageconnectionsdialog.cpp | 2 +- src/gui/qgsnewhttpconnection.cpp | 53 ++++++++++++++----- src/gui/qgsnewhttpconnection.h | 18 +++++-- src/providers/wfs/oapif/qgsoapifprovider.cpp | 9 ++-- src/providers/wfs/qgswfsconnection.cpp | 2 +- src/providers/wfs/qgswfsdatasourceuri.cpp | 12 +++-- src/providers/wfs/qgswfsdatasourceuri.h | 10 +++- src/providers/wfs/qgswfsnewconnection.cpp | 8 ++- src/providers/wfs/qgswfsprovider.cpp | 18 +++++-- src/ui/qgsnewhttpconnectionbase.ui | 12 ++--- tests/src/python/test_provider_wfs_gui.py | 4 +- 15 files changed, 112 insertions(+), 42 deletions(-) diff --git a/python/PyQt6/gui/auto_generated/qgsnewhttpconnection.sip.in b/python/PyQt6/gui/auto_generated/qgsnewhttpconnection.sip.in index de35f2dfd0d9..55187f5d5ce5 100644 --- a/python/PyQt6/gui/auto_generated/qgsnewhttpconnection.sip.in +++ b/python/PyQt6/gui/auto_generated/qgsnewhttpconnection.sip.in @@ -83,6 +83,7 @@ Returns the current connection url. WFS_VERSION_API_FEATURES_1_0, }; + virtual bool validate(); %Docstring Returns ``True`` if dialog settings are valid, or ``False`` if current diff --git a/python/gui/auto_generated/qgsnewhttpconnection.sip.in b/python/gui/auto_generated/qgsnewhttpconnection.sip.in index 98e2e2aa29e5..6802b819feec 100644 --- a/python/gui/auto_generated/qgsnewhttpconnection.sip.in +++ b/python/gui/auto_generated/qgsnewhttpconnection.sip.in @@ -83,6 +83,7 @@ Returns the current connection url. WFS_VERSION_API_FEATURES_1_0, }; + virtual bool validate(); %Docstring Returns ``True`` if dialog settings are valid, or ``False`` if current diff --git a/src/core/qgsowsconnection.cpp b/src/core/qgsowsconnection.cpp index 818706565b32..2a145710aecd 100644 --- a/src/core/qgsowsconnection.cpp +++ b/src/core/qgsowsconnection.cpp @@ -67,7 +67,7 @@ const QgsSettingsEntryEnumFlag *QgsOwsConnection::settingsDpiMode const QgsSettingsEntryEnumFlag *QgsOwsConnection::settingsTilePixelRatio = new QgsSettingsEntryEnumFlag( QStringLiteral( "tile-pixel-ratio" ), sTreeOwsConnections, Qgis::TilePixelRatio::Undefined, QString(), Qgis::SettingsOption::SaveEnumFlagAsInt ) ; const QgsSettingsEntryString *QgsOwsConnection::settingsMaxNumFeatures = new QgsSettingsEntryString( QStringLiteral( "max-num-features" ), sTreeOwsConnections ) ; const QgsSettingsEntryString *QgsOwsConnection::settingsPagesize = new QgsSettingsEntryString( QStringLiteral( "page-size" ), sTreeOwsConnections ) ; -const QgsSettingsEntryBool *QgsOwsConnection::settingsPagingEnabled = new QgsSettingsEntryBool( QStringLiteral( "paging-enabled" ), sTreeOwsConnections, true ) ; +const QgsSettingsEntryString *QgsOwsConnection::settingsPagingEnabled = new QgsSettingsEntryString( QStringLiteral( "paging-enabled" ), sTreeOwsConnections, QString( "default" ) ) ; const QgsSettingsEntryBool *QgsOwsConnection::settingsPreferCoordinatesForWfsT11 = new QgsSettingsEntryBool( QStringLiteral( "prefer-coordinates-for-wfs-T11" ), sTreeOwsConnections, false ) ; const QgsSettingsEntryBool *QgsOwsConnection::settingsIgnoreAxisOrientation = new QgsSettingsEntryBool( QStringLiteral( "ignore-axis-orientation" ), sTreeOwsConnections, false ) ; const QgsSettingsEntryBool *QgsOwsConnection::settingsInvertAxisOrientation = new QgsSettingsEntryBool( QStringLiteral( "invert-axis-orientation" ), sTreeOwsConnections, false ) ; diff --git a/src/core/qgsowsconnection.h b/src/core/qgsowsconnection.h index 555e417b60ca..f438b4bf71df 100644 --- a/src/core/qgsowsconnection.h +++ b/src/core/qgsowsconnection.h @@ -108,7 +108,7 @@ class CORE_EXPORT QgsOwsConnection : public QObject static const QgsSettingsEntryEnumFlag *settingsTilePixelRatio; static const QgsSettingsEntryString *settingsMaxNumFeatures; static const QgsSettingsEntryString *settingsPagesize; - static const QgsSettingsEntryBool *settingsPagingEnabled; + static const QgsSettingsEntryString *settingsPagingEnabled; static const QgsSettingsEntryBool *settingsPreferCoordinatesForWfsT11; static const QgsSettingsEntryBool *settingsIgnoreAxisOrientation; static const QgsSettingsEntryBool *settingsInvertAxisOrientation; diff --git a/src/gui/qgsmanageconnectionsdialog.cpp b/src/gui/qgsmanageconnectionsdialog.cpp index 16244414444c..90ba1c6a6081 100644 --- a/src/gui/qgsmanageconnectionsdialog.cpp +++ b/src/gui/qgsmanageconnectionsdialog.cpp @@ -996,7 +996,7 @@ void QgsManageConnectionsDialog::loadWfsConnections( const QDomDocument &doc, co QgsOwsConnection::settingsVersion->setValue( child.attribute( QStringLiteral( "version" ) ), {QStringLiteral( "wfs" ), connectionName} ); QgsOwsConnection::settingsMaxNumFeatures->setValue( child.attribute( QStringLiteral( "maxnumfeatures" ) ), {QStringLiteral( "wfs" ), connectionName} ); QgsOwsConnection::settingsPagesize->setValue( child.attribute( QStringLiteral( "pagesize" ) ), {QStringLiteral( "wfs" ), connectionName} ); - QgsOwsConnection::settingsPagingEnabled->setValue( child.attribute( QStringLiteral( "pagingenabled" ) ).toInt(), {QStringLiteral( "wfs" ), connectionName} ); + QgsOwsConnection::settingsPagingEnabled->setValue( child.attribute( QStringLiteral( "pagingenabled" ) ), {QStringLiteral( "wfs" ), connectionName} ); QgsOwsConnection::settingsIgnoreAxisOrientation->setValue( child.attribute( QStringLiteral( "ignoreAxisOrientation" ) ).toInt(), {QStringLiteral( "wfs" ), connectionName} ); QgsOwsConnection::settingsInvertAxisOrientation->setValue( child.attribute( QStringLiteral( "invertAxisOrientation" ) ).toInt(), {QStringLiteral( "wfs" ), connectionName} ); diff --git a/src/gui/qgsnewhttpconnection.cpp b/src/gui/qgsnewhttpconnection.cpp index 257200b92648..9f735bb2c3e1 100644 --- a/src/gui/qgsnewhttpconnection.cpp +++ b/src/gui/qgsnewhttpconnection.cpp @@ -88,8 +88,13 @@ QgsNewHttpConnection::QgsNewHttpConnection( QWidget *parent, ConnectionTypes typ static_cast( &QComboBox::currentIndexChanged ), this, &QgsNewHttpConnection::wfsVersionCurrentIndexChanged ); - connect( cbxWfsFeaturePaging, &QCheckBox::stateChanged, - this, &QgsNewHttpConnection::wfsFeaturePagingStateChanged ); + cmbFeaturePaging->clear(); + cmbFeaturePaging->addItem( tr( "Default (trust server capabilities)" ) ); + cmbFeaturePaging->addItem( tr( "Enabled" ) ); + cmbFeaturePaging->addItem( tr( "Disabled" ) ); + connect( cmbFeaturePaging, + static_cast( &QComboBox::currentIndexChanged ), + this, &QgsNewHttpConnection::wfsFeaturePagingCurrentIndexChanged ); if ( !connectionName.isEmpty() ) { @@ -176,18 +181,20 @@ QgsNewHttpConnection::QgsNewHttpConnection( QWidget *parent, ConnectionTypes typ void QgsNewHttpConnection::wfsVersionCurrentIndexChanged( int index ) { // For now 2019-06-06, leave paging checkable for some WFS version 1.1 servers with support - cbxWfsFeaturePaging->setEnabled( index == WFS_VERSION_MAX || index >= WFS_VERSION_2_0 ); - lblPageSize->setEnabled( cbxWfsFeaturePaging->isChecked() && ( index == WFS_VERSION_MAX || index >= WFS_VERSION_1_1 ) ); - txtPageSize->setEnabled( cbxWfsFeaturePaging->isChecked() && ( index == WFS_VERSION_MAX || index >= WFS_VERSION_1_1 ) ); + cmbFeaturePaging->setEnabled( index == WFS_VERSION_MAX || index >= WFS_VERSION_2_0 ); + const bool pagingNotDisabled = cmbFeaturePaging->currentIndex() != static_cast( QgsNewHttpConnection::WfsFeaturePagingIndex::DISABLED ); + lblPageSize->setEnabled( pagingNotDisabled && ( index == WFS_VERSION_MAX || index >= WFS_VERSION_1_1 ) ); + txtPageSize->setEnabled( pagingNotDisabled && ( index == WFS_VERSION_MAX || index >= WFS_VERSION_1_1 ) ); cbxWfsIgnoreAxisOrientation->setEnabled( index != WFS_VERSION_1_0 && index != WFS_VERSION_API_FEATURES_1_0 ); cbxWfsInvertAxisOrientation->setEnabled( index != WFS_VERSION_API_FEATURES_1_0 ); wfsUseGml2EncodingForTransactions()->setEnabled( index == WFS_VERSION_1_1 ); } -void QgsNewHttpConnection::wfsFeaturePagingStateChanged( int state ) +void QgsNewHttpConnection::wfsFeaturePagingCurrentIndexChanged( int index ) { - lblPageSize->setEnabled( state == Qt::Checked ); - txtPageSize->setEnabled( state == Qt::Checked ); + const bool pagingNotDisabled = index != static_cast( QgsNewHttpConnection::WfsFeaturePagingIndex::DISABLED ); + lblPageSize->setEnabled( pagingNotDisabled ); + txtPageSize->setEnabled( pagingNotDisabled ); } QString QgsNewHttpConnection::name() const @@ -268,9 +275,9 @@ QComboBox *QgsNewHttpConnection::wfsVersionComboBox() return cmbVersion; } -QCheckBox *QgsNewHttpConnection::wfsPagingEnabledCheckBox() +QComboBox *QgsNewHttpConnection::wfsPagingComboBox() { - return cbxWfsFeaturePaging; + return cmbFeaturePaging; } QCheckBox *QgsNewHttpConnection::wfsUseGml2EncodingForTransactions() @@ -333,9 +340,15 @@ void QgsNewHttpConnection::updateServiceSpecificSettings() txtMaxNumFeatures->setText( QgsOwsConnection::settingsMaxNumFeatures->value( detailsParameters ) ); // Only default to paging enabled if WFS 2.0.0 or higher - const bool pagingEnabled = QgsOwsConnection::settingsPagingEnabled->valueWithDefaultOverride( versionIdx == WFS_VERSION_MAX || versionIdx >= WFS_VERSION_2_0, detailsParameters ); + const QString pagingEnabled = QgsOwsConnection::settingsPagingEnabled->value( detailsParameters ); + if ( pagingEnabled == QLatin1String( "enabled" ) ) + cmbFeaturePaging->setCurrentIndex( static_cast( QgsNewHttpConnection::WfsFeaturePagingIndex::ENABLED ) ); + else if ( pagingEnabled == QLatin1String( "disabled" ) ) + cmbFeaturePaging->setCurrentIndex( static_cast( QgsNewHttpConnection::WfsFeaturePagingIndex::DISABLED ) ); + else + cmbFeaturePaging->setCurrentIndex( static_cast( QgsNewHttpConnection::WfsFeaturePagingIndex::DEFAULT ) ); + txtPageSize->setText( QgsOwsConnection::settingsPagesize->value( detailsParameters ) ); - cbxWfsFeaturePaging->setChecked( pagingEnabled ); } QUrl QgsNewHttpConnection::urlTrimmed() const @@ -438,7 +451,21 @@ void QgsNewHttpConnection::accept() QgsOwsConnection::settingsVersion->setValue( version, detailsParameters ); QgsOwsConnection::settingsMaxNumFeatures->setValue( txtMaxNumFeatures->text(), detailsParameters ); QgsOwsConnection::settingsPagesize->setValue( txtPageSize->text(), detailsParameters ); - QgsOwsConnection::settingsPagingEnabled->setValue( cbxWfsFeaturePaging->isChecked(), detailsParameters ); + + QString pagingEnabled = QStringLiteral( "default" ); + switch ( cmbFeaturePaging->currentIndex() ) + { + case static_cast( QgsNewHttpConnection::WfsFeaturePagingIndex::DEFAULT ): + pagingEnabled = QStringLiteral( "default" ); + break; + case static_cast( QgsNewHttpConnection::WfsFeaturePagingIndex::ENABLED ): + pagingEnabled = QStringLiteral( "enabled" ); + break; + case static_cast( QgsNewHttpConnection::WfsFeaturePagingIndex::DISABLED ): + pagingEnabled = QStringLiteral( "disabled" ); + break; + } + QgsOwsConnection::settingsPagingEnabled->setValue( pagingEnabled, detailsParameters ); } QStringList credentialsParameters = {mServiceName.toLower(), newConnectionName}; diff --git a/src/gui/qgsnewhttpconnection.h b/src/gui/qgsnewhttpconnection.h index c7df8b6f0694..d80a857bc117 100644 --- a/src/gui/qgsnewhttpconnection.h +++ b/src/gui/qgsnewhttpconnection.h @@ -95,7 +95,7 @@ class GUI_EXPORT QgsNewHttpConnection : public QDialog, private Ui::QgsNewHttpCo void urlChanged( const QString & ); void updateOkButtonState(); void wfsVersionCurrentIndexChanged( int index ); - void wfsFeaturePagingStateChanged( int state ); + void wfsFeaturePagingCurrentIndexChanged( int index ); protected: @@ -109,6 +109,16 @@ class GUI_EXPORT QgsNewHttpConnection : public QDialog, private Ui::QgsNewHttpCo WFS_VERSION_API_FEATURES_1_0 = 4, }; +#ifndef SIP_RUN + //! Index of wfsFeaturePaging + enum class WfsFeaturePagingIndex + { + DEFAULT = 0, + ENABLED = 1, + DISABLED = 2, + }; +#endif + /** * Returns TRUE if dialog settings are valid, or FALSE if current * settings are not valid and the dialog should not be acceptable. @@ -139,10 +149,10 @@ class GUI_EXPORT QgsNewHttpConnection : public QDialog, private Ui::QgsNewHttpCo QComboBox *wfsVersionComboBox() SIP_SKIP; /** - * Returns the "WFS paging enabled" checkbox - * \since QGIS 3.2 + * Returns the "WFS paging" combobox + * \since QGIS 3.36 */ - QCheckBox *wfsPagingEnabledCheckBox() SIP_SKIP; + QComboBox *wfsPagingComboBox() SIP_SKIP; /** * Returns the "Use GML2 encoding for transactions" checkbox diff --git a/src/providers/wfs/oapif/qgsoapifprovider.cpp b/src/providers/wfs/oapif/qgsoapifprovider.cpp index 080eb106c83f..9c282ae18560 100644 --- a/src/providers/wfs/oapif/qgsoapifprovider.cpp +++ b/src/providers/wfs/oapif/qgsoapifprovider.cpp @@ -109,7 +109,8 @@ bool QgsOapifProvider::init() mShared->mServerMaxFeatures = apiRequest.maxLimit(); - if ( mShared->mURI.maxNumFeatures() > 0 && mShared->mServerMaxFeatures > 0 && !mShared->mURI.pagingEnabled() ) + const bool pagingEnabled = mShared->mURI.pagingStatus() != QgsWFSDataSourceURI::PagingStatus::DISABLED; + if ( mShared->mURI.maxNumFeatures() > 0 && mShared->mServerMaxFeatures > 0 && !pagingEnabled ) { mShared->mMaxFeatures = std::min( mShared->mURI.maxNumFeatures(), mShared->mServerMaxFeatures ); } @@ -117,12 +118,12 @@ bool QgsOapifProvider::init() { mShared->mMaxFeatures = mShared->mURI.maxNumFeatures(); } - else if ( mShared->mServerMaxFeatures > 0 && !mShared->mURI.pagingEnabled() ) + else if ( mShared->mServerMaxFeatures > 0 && !pagingEnabled ) { mShared->mMaxFeatures = mShared->mServerMaxFeatures; } - if ( mShared->mURI.pagingEnabled() && mShared->mURI.pageSize() > 0 ) + if ( pagingEnabled && mShared->mURI.pageSize() > 0 ) { if ( mShared->mServerMaxFeatures > 0 ) { @@ -133,7 +134,7 @@ bool QgsOapifProvider::init() mShared->mPageSize = mShared->mURI.pageSize(); } } - else if ( mShared->mURI.pagingEnabled() ) + else if ( pagingEnabled ) { if ( apiRequest.defaultLimit() > 0 && apiRequest.maxLimit() > 0 ) { diff --git a/src/providers/wfs/qgswfsconnection.cpp b/src/providers/wfs/qgswfsconnection.cpp index cb6ed2f788ac..155008ed537e 100644 --- a/src/providers/wfs/qgswfsconnection.cpp +++ b/src/providers/wfs/qgswfsconnection.cpp @@ -50,7 +50,7 @@ QgsWfsConnection::QgsWfsConnection( const QString &connName ) if ( settingsPagingEnabled->exists( detailsParameters ) ) { mUri.removeParam( QgsWFSConstants::URI_PARAM_PAGING_ENABLED ); // setParam allow for duplicates! - mUri.setParam( QgsWFSConstants::URI_PARAM_PAGING_ENABLED, settingsPagingEnabled->value( detailsParameters ) ? QStringLiteral( "true" ) : QStringLiteral( "false" ) ); + mUri.setParam( QgsWFSConstants::URI_PARAM_PAGING_ENABLED, settingsPagingEnabled->value( detailsParameters ) ); } if ( settingsPreferCoordinatesForWfsT11->exists( detailsParameters ) ) diff --git a/src/providers/wfs/qgswfsdatasourceuri.cpp b/src/providers/wfs/qgswfsdatasourceuri.cpp index db27f4ee8c66..0300352748d4 100644 --- a/src/providers/wfs/qgswfsdatasourceuri.cpp +++ b/src/providers/wfs/qgswfsdatasourceuri.cpp @@ -334,11 +334,17 @@ long long QgsWFSDataSourceURI::pageSize() const return mURI.param( QgsWFSConstants::URI_PARAM_PAGE_SIZE ).toLongLong(); } -bool QgsWFSDataSourceURI::pagingEnabled() const +QgsWFSDataSourceURI::PagingStatus QgsWFSDataSourceURI::pagingStatus() const { if ( !mURI.hasParam( QgsWFSConstants::URI_PARAM_PAGING_ENABLED ) ) - return true; - return mURI.param( QgsWFSConstants::URI_PARAM_PAGING_ENABLED ) == QLatin1String( "true" ); + return PagingStatus::DEFAULT; + const QString val = mURI.param( QgsWFSConstants::URI_PARAM_PAGING_ENABLED ); + if ( val == QLatin1String( "true" ) || val == QLatin1String( "enabled" ) ) + return PagingStatus::ENABLED; + else if ( val == QLatin1String( "false" ) || val == QLatin1String( "disabled" ) ) + return PagingStatus::DISABLED; + else + return PagingStatus::DEFAULT; } void QgsWFSDataSourceURI::setTypeName( const QString &typeName ) diff --git a/src/providers/wfs/qgswfsdatasourceuri.h b/src/providers/wfs/qgswfsdatasourceuri.h index 193b129bccc7..9bc074b4c1ff 100644 --- a/src/providers/wfs/qgswfsdatasourceuri.h +++ b/src/providers/wfs/qgswfsdatasourceuri.h @@ -69,8 +69,16 @@ class QgsWFSDataSourceURI //! Returns user defined limit page size. 0=server udefault long long pageSize() const; + //! Whether paging is enabled + enum class PagingStatus + { + DEFAULT, //! For WFS <= 1.1, no paging. For WFS 2.0, trust GetCapabilities "ImplementsResultPaging" + ENABLED, //! Enabled + DISABLED // Disabled + }; + //! Returns whether paging is enabled. - bool pagingEnabled() const; + PagingStatus pagingStatus() const; //! Gets typename (with prefix) QString typeName() const; diff --git a/src/providers/wfs/qgswfsnewconnection.cpp b/src/providers/wfs/qgswfsnewconnection.cpp index 2e3a0bc4b7d1..25ac88499811 100644 --- a/src/providers/wfs/qgswfsnewconnection.cpp +++ b/src/providers/wfs/qgswfsnewconnection.cpp @@ -106,7 +106,11 @@ void QgsWFSNewConnection::capabilitiesReplyFinished() wfsPageSizeLineEdit()->setText( QString::number( caps.maxFeatures ) ); } wfsVersionComboBox()->setCurrentIndex( versionIdx ); - wfsPagingEnabledCheckBox()->setChecked( caps.supportsPaging ); + + wfsPagingComboBox()->setCurrentIndex( + static_cast( caps.supportsPaging ? + QgsNewHttpConnection::WfsFeaturePagingIndex::ENABLED : + QgsNewHttpConnection::WfsFeaturePagingIndex::DISABLED ) ); mCapabilities.reset(); } @@ -159,7 +163,7 @@ void QgsWFSNewConnection::oapifLandingPageReplyFinished() } wfsVersionComboBox()->setCurrentIndex( WFS_VERSION_API_FEATURES_1_0 ); - wfsPagingEnabledCheckBox()->setChecked( true ); + wfsPagingComboBox()->setCurrentIndex( static_cast( QgsNewHttpConnection::WfsFeaturePagingIndex::ENABLED ) ); mCapabilities.reset(); diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index fbe499ee9792..65359a8c24e1 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -2582,7 +2582,19 @@ bool QgsWFSProvider::getCapabilities() mShared->mURI.setPostEndpoints( mShared->mCaps.operationPostEndpoints ); mShared->mWFSVersion = mShared->mCaps.version; - if ( mShared->mURI.maxNumFeatures() > 0 && mShared->mCaps.maxFeatures > 0 && !( mShared->mCaps.supportsPaging && mShared->mURI.pagingEnabled() ) ) + + bool pagingEnabled = false; + if ( mShared->mURI.pagingStatus() == QgsWFSDataSourceURI::PagingStatus::ENABLED ) + { + pagingEnabled = true; + } + else if ( mShared->mWFSVersion.startsWith( QLatin1String( "2.0" ) ) ) + { + if ( mShared->mCaps.supportsPaging && mShared->mURI.pagingStatus() == QgsWFSDataSourceURI::PagingStatus::DEFAULT ) + pagingEnabled = true; + } + + if ( mShared->mURI.maxNumFeatures() > 0 && mShared->mCaps.maxFeatures > 0 && !pagingEnabled ) { mShared->mMaxFeatures = std::min( mShared->mURI.maxNumFeatures(), mShared->mCaps.maxFeatures ); } @@ -2590,7 +2602,7 @@ bool QgsWFSProvider::getCapabilities() { mShared->mMaxFeatures = mShared->mURI.maxNumFeatures(); } - else if ( mShared->mCaps.maxFeatures > 0 && !( mShared->mCaps.supportsPaging && mShared->mURI.pagingEnabled() ) ) + else if ( mShared->mCaps.maxFeatures > 0 && !pagingEnabled ) { mShared->mMaxFeatures = mShared->mCaps.maxFeatures; } @@ -2599,7 +2611,7 @@ bool QgsWFSProvider::getCapabilities() mShared->mMaxFeatures = 0; } - if ( mShared->mCaps.supportsPaging && mShared->mURI.pagingEnabled() ) + if ( pagingEnabled ) { if ( mShared->mURI.pageSize() > 0 ) { diff --git a/src/ui/qgsnewhttpconnectionbase.ui b/src/ui/qgsnewhttpconnectionbase.ui index d4215e5a9859..0e8c049d271f 100644 --- a/src/ui/qgsnewhttpconnectionbase.ui +++ b/src/ui/qgsnewhttpconnectionbase.ui @@ -84,16 +84,16 @@ - - + + - Enable feature paging - - - true + Feature paging + + + diff --git a/tests/src/python/test_provider_wfs_gui.py b/tests/src/python/test_provider_wfs_gui.py index 261c337a4c0b..4eff718f703b 100644 --- a/tests/src/python/test_provider_wfs_gui.py +++ b/tests/src/python/test_provider_wfs_gui.py @@ -228,7 +228,7 @@ def test(self): self.addWfsLayer_layer_name = None main_dialog.addVectorLayer.connect(self.slotAddWfsLayer) QTest.mouseClick(buttonAdd, Qt.MouseButton.LeftButton) - self.assertEqual(self.addWfsLayer_uri, ' pagingEnabled=\'true\' preferCoordinatesForWfsT11=\'false\' restrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\'') + self.assertEqual(self.addWfsLayer_uri, ' pagingEnabled=\'default\' preferCoordinatesForWfsT11=\'false\' restrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\'') self.assertEqual(self.addWfsLayer_layer_name, 'my:typename') # Click on Build Query @@ -301,7 +301,7 @@ def test(self): self.addWfsLayer_layer_name = None main_dialog.addVectorLayer.connect(self.slotAddWfsLayer) QTest.mouseClick(buttonAdd, Qt.MouseButton.LeftButton) - self.assertEqual(self.addWfsLayer_uri, ' pagingEnabled=\'true\' preferCoordinatesForWfsT11=\'false\' restrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\' sql=SELECT * FROM typename WHERE 1 = 1') + self.assertEqual(self.addWfsLayer_uri, ' pagingEnabled=\'default\' preferCoordinatesForWfsT11=\'false\' restrictToRequestBBOX=\'1\' srsname=\'EPSG:4326\' typename=\'my:typename\' url=\'' + "http://" + expected_endpoint + '\' version=\'auto\' sql=SELECT * FROM typename WHERE 1 = 1') self.assertEqual(self.addWfsLayer_layer_name, 'my:typename') # main_dialog.setProperty("hideDialogs", None) From 34a5f1ec0e639b49d0968257efa4fabdf3a2c777 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 21 Jan 2024 20:22:54 +0100 Subject: [PATCH 14/21] [WFS provider] Fix discovery of geometry attribute when partially recognized by regular analysis of DescribeFeatureType and then handled by GMLAS based analysis --- src/providers/wfs/qgswfsprovider.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index 65359a8c24e1..a1cf297249c0 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -1568,11 +1568,13 @@ bool QgsWFSProvider::readAttributesFromSchema( QDomDocument &schemaDoc, mayTryWithGMLAS && GDALGetDriverByName( "GMLAS" ) ) { + QString geometryAttributeGMLAS; QgsFields fieldsGMLAS; Qgis::WkbType geomTypeGMLAS; QString errorMsgGMLAS; - if ( readAttributesFromSchemaWithGMLAS( response, prefixedTypename, geometryAttribute, fieldsGMLAS, geomTypeGMLAS, geometryMaybeMissing, errorMsgGMLAS ) ) + if ( readAttributesFromSchemaWithGMLAS( response, prefixedTypename, geometryAttributeGMLAS, fieldsGMLAS, geomTypeGMLAS, geometryMaybeMissing, errorMsgGMLAS ) ) { + geometryAttribute = geometryAttributeGMLAS; fields = fieldsGMLAS; geomType = geomTypeGMLAS; ret = true; From 5d472d2495669c516794bed7935aa124cc2a420d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 21 Jan 2024 21:35:01 +0100 Subject: [PATCH 15/21] QgsGmlStreamingParser: in JSON attribute mode, robustify against geometry elements in unexpected places --- src/core/qgsgml.cpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/core/qgsgml.cpp b/src/core/qgsgml.cpp index 7d3c7962b1dc..44704c4e37bd 100644 --- a/src/core/qgsgml.cpp +++ b/src/core/qgsgml.cpp @@ -607,7 +607,7 @@ void QgsGmlStreamingParser::startElement( const XML_Char *el, const XML_Char **a mGeometryString.append( ">", 1 ); } - if ( isGMLNS && LOCALNAME_EQUALS( "coordinates" ) ) + if ( !mAttributeValIsNested && isGMLNS && LOCALNAME_EQUALS( "coordinates" ) ) { mParseModeStack.push( Coordinate ); mCoorMode = QgsGmlStreamingParser::Coordinate; @@ -623,7 +623,7 @@ void QgsGmlStreamingParser::startElement( const XML_Char *el, const XML_Char **a mTupleSeparator = ' '; } } - else if ( isGMLNS && + else if ( !mAttributeValIsNested && isGMLNS && ( LOCALNAME_EQUALS( "pos" ) || LOCALNAME_EQUALS( "posList" ) ) ) { mParseModeStack.push( QgsGmlStreamingParser::PosList ); @@ -650,7 +650,7 @@ void QgsGmlStreamingParser::startElement( const XML_Char *el, const XML_Char **a mGeometryString.clear(); } //else if ( mParseModeStack.size() == 0 && elementName == mGMLNameSpaceURI + NS_SEPARATOR + "boundedBy" ) - else if ( isGMLNS && LOCALNAME_EQUALS( "boundedBy" ) ) + else if ( !mAttributeValIsNested && isGMLNS && LOCALNAME_EQUALS( "boundedBy" ) ) { mParseModeStack.push( QgsGmlStreamingParser::BoundingBox ); mCurrentExtent = QgsRectangle(); @@ -778,35 +778,35 @@ void QgsGmlStreamingParser::startElement( const XML_Char *el, const XML_Char **a { isGeom = true; } - else if ( isGMLNS && LOCALNAME_EQUALS( "Point" ) ) + else if ( !mAttributeValIsNested && isGMLNS && LOCALNAME_EQUALS( "Point" ) ) { isGeom = true; } - else if ( isGMLNS && LOCALNAME_EQUALS( "LineString" ) ) + else if ( !mAttributeValIsNested && isGMLNS && LOCALNAME_EQUALS( "LineString" ) ) { isGeom = true; } - else if ( isGMLNS && + else if ( !mAttributeValIsNested && isGMLNS && localNameLen == static_cast( strlen( "Polygon" ) ) && memcmp( pszLocalName, "Polygon", localNameLen ) == 0 ) { isGeom = true; mCurrentWKBFragments.push_back( QList() ); } - else if ( isGMLNS && LOCALNAME_EQUALS( "MultiPoint" ) ) + else if ( !mAttributeValIsNested && isGMLNS && LOCALNAME_EQUALS( "MultiPoint" ) ) { isGeom = true; mParseModeStack.push( QgsGmlStreamingParser::MultiPoint ); //we need one nested list for intermediate WKB mCurrentWKBFragments.push_back( QList() ); } - else if ( isGMLNS && ( LOCALNAME_EQUALS( "MultiLineString" ) || LOCALNAME_EQUALS( "MultiCurve" ) ) ) + else if ( !mAttributeValIsNested && isGMLNS && ( LOCALNAME_EQUALS( "MultiLineString" ) || LOCALNAME_EQUALS( "MultiCurve" ) ) ) { isGeom = true; mParseModeStack.push( QgsGmlStreamingParser::MultiLine ); //we need one nested list for intermediate WKB mCurrentWKBFragments.push_back( QList() ); } - else if ( isGMLNS && ( LOCALNAME_EQUALS( "MultiPolygon" ) || LOCALNAME_EQUALS( "MultiSurface" ) ) ) + else if ( !mAttributeValIsNested && isGMLNS && ( LOCALNAME_EQUALS( "MultiPolygon" ) || LOCALNAME_EQUALS( "MultiSurface" ) ) ) { isGeom = true; mParseModeStack.push( QgsGmlStreamingParser::MultiPolygon ); @@ -1116,6 +1116,7 @@ void QgsGmlStreamingParser::endElement( const XML_Char *el ) if ( mAttributeValIsNested ) { + mAttributeValIsNested = false; auto iter = mMapFieldNameToJSONContent.find( mAttributeName ); if ( iter == mMapFieldNameToJSONContent.end() ) { @@ -1258,7 +1259,7 @@ void QgsGmlStreamingParser::endElement( const XML_Char *el ) ++mFeatureCount; mParseModeStack.pop(); } - else if ( isGMLNS && LOCALNAME_EQUALS( "Point" ) ) + else if ( !mAttributeValIsNested && isGMLNS && LOCALNAME_EQUALS( "Point" ) ) { QList pointList; if ( pointsFromString( pointList, mStringCash ) != 0 ) @@ -1300,7 +1301,8 @@ void QgsGmlStreamingParser::endElement( const XML_Char *el ) } } } - else if ( isGMLNS && ( LOCALNAME_EQUALS( "LineString" ) || LOCALNAME_EQUALS( "LineStringSegment" ) ) ) + else if ( !mAttributeValIsNested && + isGMLNS && ( LOCALNAME_EQUALS( "LineString" ) || LOCALNAME_EQUALS( "LineStringSegment" ) ) ) { //add WKB point to the feature From 2dae2d7270a63b6dc13eefc30e9b4ca1c346cce8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 21 Jan 2024 22:38:24 +0100 Subject: [PATCH 16/21] [WFS provider] readAttributesFromSchemaWithoutGMLAS(): also retry with GMLAS driver when no attribute has been found Fixes case of https://github.com/qgis/QGIS/pull/55847/commits/8eb189b383a6fe8928ccee94c823c29fa181b17f --- src/providers/wfs/qgswfsprovider.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index a1cf297249c0..3b660959cb75 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -2270,6 +2270,7 @@ bool QgsWFSProvider::readAttributesFromSchemaWithoutGMLAS( QDomDocument &schemaD if ( attributeNodeList.size() < 1 ) { errorMsg = tr( "Cannot find attribute elements" ); + mayTryWithGMLAS = true; return false; } From e198ffd5e9d2a93c646dae1f01cda20daaaa4a77 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 21 Jan 2024 22:38:01 +0100 Subject: [PATCH 17/21] [WFS provider] Add support for geometry element using gml:GeometricPrimitivePropertyType --- src/providers/wfs/qgswfsprovider.cpp | 35 ++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index 3b660959cb75..72054708835c 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -2046,13 +2046,39 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon QgsDebugMsgLevel( QStringLiteral( "field %1: xpath=%2 is_list=%3 type=%4 category=%5" ). arg( fieldName ).arg( fieldXPath ).arg( fieldIsList ).arg( fieldType ).arg( fieldCategory ), 5 ); - if ( EQUAL( fieldCategory, "REGULAR" ) && EQUAL( fieldType, "geometry" ) ) + if ( EQUAL( fieldCategory, "REGULAR" ) && ( EQUAL( fieldType, "geometry" ) || fieldName.endsWith( QLatin1String( "_abstractgeometricprimitive" ) ) ) ) { if ( geometryAttribute.isEmpty() ) { + geomType = QgsWkbTypes::multiType( QgsOgrUtils::ogrGeometryTypeToQgsWkbType( + OGR_L_GetGeomType( hLayer ) ) ); + + QString qFieldXPath = QString::fromUtf8( fieldXPath ); + if ( fieldName.endsWith( QLatin1String( "_abstractgeometricprimitive" ) ) && strstr( fieldXPath, "/gml:Point" ) ) + { + // Note: this particular case will not be needed in GDAL >= 3.8.4 + // The _abstractgeometricprimitive case is for a layer like + // "https://www.wfs.nrw.de/geobasis/wfs_nw_inspire-gewaesser-physisch_atkis-basis-dlm?SERVICE=WFS&REQUEST=DescribeFeatureType&VERSION=2.0.0&TYPENAMES=hy-p:Embankment&NAMESPACES=xmlns(hy-p,http://inspire.ec.europa.eu/schemas/hy-p/4.0)&TYPENAME=hy-p:Embankment&NAMESPACE=xmlns(hy-p,http://inspire.ec.europa.eu/schemas/hy-p/4.0)" + // which has a geometry element as: + // layer_name (String) = embankment + // field_index (Integer) = 30 + // field_name (String) = geometry_abstractgeometricprimitive + // field_xpath (String) = hy-p:Embankment/hy-p:geometry/gml:Point,hy-p:Embankment/hy-p:geometry/gml:LineString,hy-p:Embankment/hy-p:geometry/gml:LinearRing,hy-p:Embankment/hy-p:geometry/gml:Ring,hy-p:Embankment/hy-p:geometry/gml:Curve,hy-p:Embankment/hy-p:geometry/gml:OrientableCurve,hy-p:Embankment/hy-p:geometry/gml:CompositeCurve,hy-p:Embankment/hy-p:geometry/gml:Polygon,hy-p:Embankment/hy-p:geometry/gml:Surface,hy-p:Embankment/hy-p:geometry/gml:PolyhedralSurface,hy-p:Embankment/hy-p:geometry/gml:TriangulatedSurface,hy-p:Embankment/hy-p:geometry/gml:Tin,hy-p:Embankment/hy-p:geometry/gml:OrientableSurface,hy-p:Embankment/hy-p:geometry/gml:Shell,hy-p:Embankment/hy-p:geometry/gml:CompositeSurface,hy-p:Embankment/hy-p:geometry/gml:Solid,hy-p:Embankment/hy-p:geometry/gml:CompositeSolid + // field_type (String) = anyType + // field_is_list (Integer(Boolean)) = 0 + // field_min_occurs (Integer) = 0 + // field_max_occurs (Integer) = 1 + // field_category (String) = REGULAR + + const auto pos_gmlPoint = qFieldXPath.indexOf( QStringLiteral( "/gml:Point," ) ); + qFieldXPath.resize( pos_gmlPoint ); + geomType = Qgis::WkbType::Unknown; + } + mShared->mFieldNameToXPathAndIsNestedContentMap[fieldName] = - QPair( fieldXPath, false ); - geometryAttribute = fieldXPath; + QPair( qFieldXPath, false ); + geometryAttribute = qFieldXPath; + { const auto parts = geometryAttribute.split( '/' ); if ( parts.size() > 1 ) @@ -2063,13 +2089,12 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon if ( parts.size() == 2 ) geometryAttribute = parts[1]; } - geomType = QgsWkbTypes::multiType( QgsOgrUtils::ogrGeometryTypeToQgsWkbType( - OGR_L_GetGeomType( hLayer ) ) ); if ( geomType == Qgis::WkbType::MultiPolygon ) geomType = Qgis::WkbType::MultiSurface; else if ( geomType == Qgis::WkbType::MultiLineString ) geomType = Qgis::WkbType::MultiCurve; + QgsDebugMsgLevel( QStringLiteral( "geometry field: %1, xpath: %2" ).arg( geometryAttribute ).arg( qFieldXPath ), 4 ); geometryMaybeMissing = OGR_F_GetFieldAsInteger( hFeatureFieldsMD.get(), fieldMinOccursIdx ) == 0; } } From dee8b4841385e11bda268c297388e60701eef273 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 22 Jan 2024 01:09:57 +0100 Subject: [PATCH 18/21] QgsGmlStreamingParser::startElement(): make clang-tidy happy --- src/core/qgsgml.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/qgsgml.cpp b/src/core/qgsgml.cpp index 44704c4e37bd..f22c99c397b1 100644 --- a/src/core/qgsgml.cpp +++ b/src/core/qgsgml.cpp @@ -736,6 +736,7 @@ void QgsGmlStreamingParser::startElement( const XML_Char *el, const XML_Char **a } else if ( parseMode == None && localNameLen == static_cast( mTypeNameUTF8Len ) && + mTypeNamePtr && memcmp( pszLocalName, mTypeNamePtr, mTypeNameUTF8Len ) == 0 ) { Q_ASSERT( !mCurrentFeature ); From 3966603683b2d0948174712c7efb0baa0b65290f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 22 Jan 2024 14:27:03 +0100 Subject: [PATCH 19/21] [WFS provider] querySublayers(): workaround servers that declare geometry type filters, but which are not working --- src/providers/wfs/qgswfsprovidermetadata.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/providers/wfs/qgswfsprovidermetadata.cpp b/src/providers/wfs/qgswfsprovidermetadata.cpp index 63ff9ebbbafd..146fcf94c40d 100644 --- a/src/providers/wfs/qgswfsprovidermetadata.cpp +++ b/src/providers/wfs/qgswfsprovidermetadata.cpp @@ -19,6 +19,7 @@ #include "qgis.h" #include "qgsfeedback.h" +#include "qgslogger.h" #include "qgsogcutils.h" #include "qgsoapifprovider.h" #include "qgswfsdataitems.h" @@ -394,6 +395,23 @@ QList QgsWfsProviderMetadata::querySublayers( const if ( countsAllValid ) { + // Some servers are buggy and actually ignore the geometry type filter + // So if the Point, Curve and Surface filters return the same number of + // features than the No filter request, consider that the geometry type + // is unknown and try sampling one feature to guess the type + // e.g with https://geodienste.komm.one/ows/services/org.273.561ba9e8-9b66-45a2-98db-17920e10c53d_wfs?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=xplan:BP_Plan&FILTER=%3Cfes:Filter%20xmlns:fes%3D%22http://www.opengis.net/fes/2.0%22%3E%0A%20%3Cfes:PropertyIsEqualTo%3E%0A%20%20%3Cfes:Function%20name%3D%22IsCurve%22%3E%0A%20%20%20%3Cfes:ValueReference%3EraeumlicherGeltungsbereich%3C/fes:ValueReference%3E%0A%20%20%3C/fes:Function%3E%0A%20%20%3Cfes:Literal%3Etrue%3C/fes:Literal%3E%0A%20%3C/fes:PropertyIsEqualTo%3E%0A%3C/fes:Filter%3E%0A&RESULTTYPE=hits + if ( featureCounts[INDEX_ALL] > 0 && + featureCounts[INDEX_POINT] == featureCounts[INDEX_ALL] && + featureCounts[INDEX_CURVE] == featureCounts[INDEX_ALL] && + featureCounts[INDEX_SURFACE] == featureCounts[INDEX_ALL] ) + { + QgsDebugMsgLevel( QString( "%1 declares geometry filters, but they are not working. Guessing the geometry type from one sample" ).arg( uri ), 2 ); + provider.issueInitialGetFeature(); + details.setWkbType( provider.wkbType() ); + details.setFeatureCount( featureCounts[INDEX_ALL] ); + return res; + } + // Deduce numbers of geometry collections from other types featureCounts[INDEX_GEOMETRYCOLLECTION] = featureCounts[INDEX_ALL] - ( featureCounts[INDEX_NULL] + featureCounts[INDEX_POINT] + featureCounts[INDEX_CURVE] + featureCounts[INDEX_SURFACE] ); From 927d7abad8a5370f3cbef1ac15fcb6b2b37b784d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 22 Jan 2024 16:48:27 +0100 Subject: [PATCH 20/21] [WFS provider] If GMLAS analysis failed, write error to message log --- src/providers/wfs/qgswfsprovider.cpp | 30 ++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/providers/wfs/qgswfsprovider.cpp b/src/providers/wfs/qgswfsprovider.cpp index 72054708835c..2fae077f2c8f 100644 --- a/src/providers/wfs/qgswfsprovider.cpp +++ b/src/providers/wfs/qgswfsprovider.cpp @@ -1542,6 +1542,7 @@ bool QgsWFSProvider::describeFeatureType( QString &geometryAttribute, QgsFields QgsDebugMsgLevel( response, 4 ); QgsMessageLog::logMessage( tr( "Analysis of DescribeFeatureType response failed for url %1: %2" ). arg( dataSourceUri(), errorMsg ), tr( "WFS" ) ); + pushError( errorMsg ); return false; } @@ -1583,6 +1584,10 @@ bool QgsWFSProvider::readAttributesFromSchema( QDomDocument &schemaDoc, { errorMsg = errorMsgGMLAS; } + else + { + pushError( errorMsgGMLAS ); + } } return ret; } @@ -1657,9 +1662,21 @@ static void CPL_STDCALL QgsWFSProviderGMLASErrorHandler( CPLErr eErr, CPLErrorNu if ( !( eErr == CE_Warning && strstr( pszErrorMsg, " truncated to " ) ) ) { if ( eErr == CE_Failure ) + { + void *pUserData = CPLGetErrorHandlerUserData(); + QString *pString = static_cast( pUserData ); + if ( pString->isEmpty() ) + *pString = QObject::tr( "Error while analyzing schema: %1" ).arg( pszErrorMsg ); QgsMessageLog::logMessage( QObject::tr( "GMLAS error: %1" ).arg( pszErrorMsg ), QObject::tr( "WFS" ) ); + } + else if ( eErr == CE_Debug ) + { + QgsDebugMsgLevel( QStringLiteral( "GMLAS debug msg: %1" ).arg( pszErrorMsg ), 5 ); + } else - QgsDebugMsgLevel( QStringLiteral( "GMLAS eErr=%1, msg=%2" ).arg( eErr ).arg( pszErrorMsg ), 4 ); + { + QgsDebugMsgLevel( QStringLiteral( "GMLAS eErr=%1, msg=%2" ).arg( eErr ).arg( pszErrorMsg ), 2 ); + } } } @@ -1714,7 +1731,8 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon // Analyze the DescribeFeatureType response schema with the OGR GMLAS driver // in a thread, so it can get interrupted (with GDAL 3.9: https://github.com/OSGeo/gdal/pull/9019) - const auto downloaderLambda = [pszSchemaTempFilename, &feedback, &hDS]() + + const auto downloaderLambda = [pszSchemaTempFilename, &feedback, &hDS, &errorMsg]() { QgsCPLHTTPFetchOverrider cplHTTPFetchOverrider( QString(), &feedback ); QgsSetCPLHTTPFetchOverriderInitiatorClass( cplHTTPFetchOverrider, QStringLiteral( "WFSProviderDownloadSchema" ) ) @@ -1771,7 +1789,7 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon CPLFree( pszEscaped ); papszOpenOptions = CSLSetNameValue( papszOpenOptions, "CONFIG_FILE", config.toStdString().c_str() ); - CPLPushErrorHandler( QgsWFSProviderGMLASErrorHandler ); + CPLPushErrorHandlerEx( QgsWFSProviderGMLASErrorHandler, &errorMsg ); hDS = GDALOpenEx( "GMLAS:", GDAL_OF_VECTOR, nullptr, papszOpenOptions, nullptr ); CPLPopErrorHandler(); CSLDestroy( papszOpenOptions ); @@ -1845,6 +1863,9 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon VSIUnlink( pszSchemaTempFilename ); VSIFree( pszSchemaTempFilename ); + if ( !errorMsg.isEmpty() ) + return false; + bool ret = hDS != nullptr; if ( feedback.isCanceled() && !ret ) { @@ -1857,7 +1878,8 @@ bool QgsWFSProvider::readAttributesFromSchemaWithGMLAS( const QByteArray &respon } if ( !ret ) { - errorMsg = tr( "Cannot analyze schema indicated in DescribeFeatureType response." ); + if ( errorMsg.isEmpty() ) + errorMsg = tr( "Cannot analyze schema indicated in DescribeFeatureType response." ); return false; } From f017314e97e2a91adab39e236e6be21c4513feb3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 25 Jan 2024 17:29:44 +0100 Subject: [PATCH 21/21] QgsVectorLayerUtils::guessFriendlyIdentifierField(): improve heuristics to work better with WFS layers analyzed with the GMLAS driver --- src/core/vector/qgsvectorlayerutils.cpp | 30 +++++++++++++++++++- tests/src/python/test_qgsvectorlayerutils.py | 13 +++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/core/vector/qgsvectorlayerutils.cpp b/src/core/vector/qgsvectorlayerutils.cpp index fc5468155a64..600563841bbf 100644 --- a/src/core/vector/qgsvectorlayerutils.cpp +++ b/src/core/vector/qgsvectorlayerutils.cpp @@ -1239,9 +1239,37 @@ QString QgsVectorLayerUtils::guessFriendlyIdentifierField( const QgsFields &fiel break; } - const QString candidateName = bestCandidateName.isEmpty() ? bestCandidateNameWithAntiCandidate : bestCandidateName; + QString candidateName = bestCandidateName.isEmpty() ? bestCandidateNameWithAntiCandidate : bestCandidateName; if ( !candidateName.isEmpty() ) { + // Special case for layers got from WFS using the OGR GMLAS field parsing logic. + // Such layers contain a "id" field (the gml:id attribute of the object), + // as well as a gml_name (a ) element. However this gml:name is often + // absent, partly because it is a property of the base class in GML schemas, and + // that a lot of readers are not able to deduce its potential presence. + // So try to look at another field whose name would end with _name + // And fallback to using the "id" field that should always be filled. + if ( candidateName == QLatin1String( "gml_name" ) && + fields.indexOf( QStringLiteral( "id" ) ) >= 0 ) + { + candidateName.clear(); + // Try to find a field ending with "_name", which is not "gml_name" + for ( const QgsField &field : std::as_const( fields ) ) + { + const QString fldName = field.name(); + if ( fldName != QLatin1String( "gml_name" ) && fldName.endsWith( QLatin1String( "_name" ) ) ) + { + candidateName = fldName; + break; + } + } + if ( candidateName.isEmpty() ) + { + // Fallback to "id" + candidateName = QStringLiteral( "id" ); + } + } + if ( foundFriendly ) *foundFriendly = true; return candidateName; diff --git a/tests/src/python/test_qgsvectorlayerutils.py b/tests/src/python/test_qgsvectorlayerutils.py index c09f0b4bf586..5e5e9bece97a 100644 --- a/tests/src/python/test_qgsvectorlayerutils.py +++ b/tests/src/python/test_qgsvectorlayerutils.py @@ -880,6 +880,19 @@ def testGuessFriendlyIdentifierField(self): fields.append(QgsField('org', QVariant.String)) self.assertEqual(QgsVectorLayerUtils.guessFriendlyIdentifierField(fields), 'station') + # Particular case for WFS layers analyzed with the GMLAS driver. + # We prioritize a field ending with _name, but which is not gml_name + fields = QgsFields() + fields.append(QgsField('id', QVariant.String)) + fields.append(QgsField('gml_name', QVariant.String)) + fields.append(QgsField('other_name', QVariant.String)) + self.assertEqual(QgsVectorLayerUtils.guessFriendlyIdentifierField(fields), 'other_name') + + fields = QgsFields() + fields.append(QgsField('id', QVariant.String)) + fields.append(QgsField('gml_name', QVariant.String)) + self.assertEqual(QgsVectorLayerUtils.guessFriendlyIdentifierField(fields), 'id') + if __name__ == '__main__': unittest.main()