Read ESRI tilemaps in vtpk and arcgis tile services

This allows us to correctly handle "indexed" vector tile sets,
where higher zoom level tiles may be missing where a lower
zoom level tile is deemed to have sufficient detail for higher
zoom levels.

python/core/auto_additions/qgis.py

@@ -3613,3 +3613,11 @@
 Qgis.FeatureSymbologyExport.__doc__ = 'Options for exporting features considering their symbology.\n\n.. note::\n\n   Prior to QGIS 3.32 this was available as :py:class:`QgsVectorFileWriter`.SymbologyExport.\n\n.. versionadded:: 3.32\n\n' + '* ``NoSymbology``: ' + Qgis.FeatureSymbologyExport.NoSymbology.__doc__ + '\n' + '* ``FeatureSymbology``: ' + Qgis.FeatureSymbologyExport.PerFeature.__doc__ + '\n' + '* ``SymbolLayerSymbology``: ' + Qgis.FeatureSymbologyExport.PerSymbolLayer.__doc__
 # --
 Qgis.FeatureSymbologyExport.baseClass = Qgis
+# monkey patching scoped based enum
+Qgis.TileAvailability.Available.__doc__ = "Tile is available within the matrix"
+Qgis.TileAvailability.NotAvailable.__doc__ = "Tile is not available within the matrix, e.g. there is no content for the tile"
+Qgis.TileAvailability.AvailableNoChildren.__doc__ = "Tile is available within the matrix, and is known to have no children (ie no higher zoom level tiles exist covering this tile's region)"
+Qgis.TileAvailability.UseLowerZoomLevelTile.__doc__ = "Tile is not available at the requested zoom level, it should be replaced by a tile from a lower zoom level instead182"
+Qgis.TileAvailability.__doc__ = 'Possible availability states for a tile within a tile matrix.\n\n.. versionadded:: 3.32\n\n' + '* ``Available``: ' + Qgis.TileAvailability.Available.__doc__ + '\n' + '* ``NotAvailable``: ' + Qgis.TileAvailability.NotAvailable.__doc__ + '\n' + '* ``AvailableNoChildren``: ' + Qgis.TileAvailability.AvailableNoChildren.__doc__ + '\n' + '* ``UseLowerZoomLevelTile``: ' + Qgis.TileAvailability.UseLowerZoomLevelTile.__doc__
+# --
+Qgis.TileAvailability.baseClass = Qgis

python/core/auto_generated/qgis.sip.in

@@ -2047,6 +2047,14 @@ The development version
       PerSymbolLayer
     };
 
+    enum class TileAvailability
+    {
+      Available,
+      NotAvailable,
+      AvailableNoChildren,
+      UseLowerZoomLevelTile,
+    };
+
     static const double DEFAULT_SEARCH_RADIUS_MM;
 
     static const float DEFAULT_MAPTOPIXEL_THRESHOLD;

python/core/auto_generated/qgstiles.sip.in

@@ -242,6 +242,8 @@ Defines a set of tile matrices for multiple zoom levels.
 %End
   public:
 
+    QgsTileMatrixSet();
+
     virtual ~QgsTileMatrixSet();
 
     bool isEmpty() const;
@@ -298,6 +300,21 @@ Returns the maximum zoom level for tiles present in the set.
 %Docstring
 Deletes any existing matrices which fall outside the zoom range specified
 by ``minimumZoom`` to ``maximumZoom``, inclusive.
+%End
+
+    Qgis::TileAvailability tileAvailability( QgsTileXYZ id ) const;
+%Docstring
+Returns the availability of the given tile in this matrix.
+
+This method can be used to determine whether a particular tile actually
+exists within the matrix, or is not available (e.g. due to holes within the matrix).
+
+This method returns :py:class:`Qgis`.TileAvailability.Available by default, unless specific
+tile availability is known for the given ``id``.
+
+.. seealso:: :py:func:`defaultAvailability`
+
+.. versionadded:: 3.32
 %End
 
     QgsCoordinateReferenceSystem crs() const;
@@ -374,6 +391,8 @@ Returns a list of tiles in the given tile range.
 .. versionadded:: 3.32
 %End
 
+  protected:
+
 };
 
 /************************************************************************

python/core/auto_generated/vectortile/qgsvectortilematrixset.sip.in

@@ -27,13 +27,15 @@ Encapsulates properties of a vector tile matrix set, including tile origins and
 Returns a vector tile structure corresponding to the standard web mercator/GoogleCRS84Quad setup.
 %End
 
-    bool fromEsriJson( const QVariantMap &json );
+    bool fromEsriJson( const QVariantMap &json, const QVariantMap &rootTileMap = QVariantMap() );
 %Docstring
 Initializes the tile structure settings from an ESRI REST VectorTileService ``json`` map.
 
 .. note::
 
    This same structure is utilized in ESRI vtpk archives in the root.json file.
+
+Optionally, a ``rootTileMap`` can be specified for indexed vector tile datasets (since QGIS 3.32)
 %End
 
 };

python/core/auto_generated/vectortile/qgsvtpktiles.sip.in

@@ -65,6 +65,15 @@ Returns the VTPK sprite image, if it exists.
     QgsLayerMetadata layerMetadata() const;
 %Docstring
 Reads layer metadata from the VTPK file.
+%End
+
+    QVariantMap rootTileMap() const;
+%Docstring
+Returns the root tilemap content, if it exists.
+
+This method returns the contents of the "tilemap/root.json" file.
+
+.. versionadded:: 3.32
 %End
 
     QgsVectorTileMatrixSet matrixSet() const;

src/core/qgis.h

@@ -3553,6 +3553,20 @@ class CORE_EXPORT Qgis
     };
     Q_ENUM( FeatureSymbologyExport )
 
+    /**
+     * Possible availability states for a tile within a tile matrix.
+     *
+     * \since QGIS 3.32
+     */
+    enum class TileAvailability
+    {
+      Available, //!< Tile is available within the matrix
+      NotAvailable, //!< Tile is not available within the matrix, e.g. there is no content for the tile
+      AvailableNoChildren, //!< Tile is available within the matrix, and is known to have no children (ie no higher zoom level tiles exist covering this tile's region)
+      UseLowerZoomLevelTile, //!< Tile is not available at the requested zoom level, it should be replaced by a tile from a lower zoom level instead182
+    };
+    Q_ENUM( TileAvailability )
+
     /**
      * Identify search radius in mm
      * \since QGIS 2.3

src/core/qgstiles.cpp

@@ -129,6 +129,12 @@ QPointF QgsTileMatrix::mapToTileCoordinates( const QgsPointXY &mapPoint ) const
 // QgsTileMatrixSet
 //
 
+QgsTileMatrixSet::QgsTileMatrixSet()
+{
+  mTileAvailabilityFunction = []( QgsTileXYZ ) { return Qgis::TileAvailability::Available; };
+  mTileReplacementFunction = []( QgsTileXYZ id, QgsTileXYZ & replacement ) { replacement = id; return Qgis::TileAvailability::Available; };
+}
+
 bool QgsTileMatrixSet::isEmpty() const
 {
   return mTileMatrices.isEmpty();
@@ -204,6 +210,11 @@ void QgsTileMatrixSet::dropMatricesOutsideZoomRange( int minimumZoom, int maximu
   }
 }
 
+Qgis::TileAvailability QgsTileMatrixSet::tileAvailability( QgsTileXYZ id ) const
+{
+  return mTileAvailabilityFunction( id );
+}
+
 QgsCoordinateReferenceSystem QgsTileMatrixSet::crs() const
 {
   if ( mTileMatrices.empty() )
@@ -408,7 +419,20 @@ QVector<QgsTileXYZ> QgsTileMatrixSet::tilesInRange( QgsTileRange range, int zoom
   {
     for ( int tileColumn = range.startColumn(); tileColumn <= range.endColumn(); ++tileColumn )
     {
-      tiles.append( QgsTileXYZ( tileColumn, tileRow, zoomLevel ) );
+      QgsTileXYZ tile( tileColumn, tileRow, zoomLevel );
+      QgsTileXYZ replacement;
+      switch ( mTileReplacementFunction( tile, replacement ) )
+      {
+        case Qgis::TileAvailability::NotAvailable:
+          break;
+
+        case Qgis::TileAvailability::Available:
+        case Qgis::TileAvailability::AvailableNoChildren:
+        case Qgis::TileAvailability::UseLowerZoomLevelTile:
+          if ( !tiles.contains( replacement ) )
+            tiles.append( replacement );
+          break;
+      }
     }
   }
   return tiles;

src/core/qgstiles.h

@@ -251,6 +251,8 @@ class CORE_EXPORT QgsTileMatrixSet
 
   public:
 
+    QgsTileMatrixSet();
+
     virtual ~QgsTileMatrixSet() = default;
 
     /**
@@ -309,6 +311,20 @@ class CORE_EXPORT QgsTileMatrixSet
      */
     void dropMatricesOutsideZoomRange( int minimumZoom, int maximumZoom );
 
+    /**
+     * Returns the availability of the given tile in this matrix.
+     *
+     * This method can be used to determine whether a particular tile actually
+     * exists within the matrix, or is not available (e.g. due to holes within the matrix).
+     *
+     * This method returns Qgis::TileAvailability::Available by default, unless specific
+     * tile availability is known for the given \a id.
+     *
+     * \see defaultAvailability()
+     * \since QGIS 3.32
+     */
+    Qgis::TileAvailability tileAvailability( QgsTileXYZ id ) const;
+
     /**
      * Returns the coordinate reference system associated with the tiles.
      *
@@ -383,7 +399,9 @@ class CORE_EXPORT QgsTileMatrixSet
      */
     QVector<QgsTileXYZ> tilesInRange( QgsTileRange range, int zoomLevel ) const;
 
-  private:
+  protected:
+    std::function< Qgis::TileAvailability( QgsTileXYZ id ) > mTileAvailabilityFunction;
+    std::function< Qgis::TileAvailability( QgsTileXYZ id, QgsTileXYZ &replacement ) > mTileReplacementFunction;
 
     // Usually corresponds to zoom level 0, even if that zoom level is NOT present in the actual tile matrices for this set
     QgsTileMatrix mRootMatrix;

src/core/vectortile/qgsarcgisvectortileservicedataprovider.cpp

@@ -256,6 +256,39 @@ bool QgsArcGisVectorTileServiceDataProvider::setupArcgisVectorTileServiceConnect
     }
   }
 
+  // read tileMap if available
+  QVariantMap tileMap;
+  const QString tileMapEndpoint = mArcgisLayerConfiguration.value( QStringLiteral( "tileMap" ) ).toString();
+  if ( !tileMapEndpoint.isEmpty() )
+  {
+    QUrl tilemapUrl( tileServiceUri + '/' + tileMapEndpoint );
+    tilemapUrl.setQuery( query );
+
+    QNetworkRequest tileMapRequest = QNetworkRequest( tilemapUrl );
+    QgsSetRequestInitiatorClass( tileMapRequest, QStringLiteral( "QgsVectorTileLayer" ) )
+
+    QgsBlockingNetworkRequest tileMapNetworkRequest;
+    switch ( tileMapNetworkRequest.get( tileMapRequest ) )
+    {
+      case QgsBlockingNetworkRequest::NoError:
+        break;
+
+      case QgsBlockingNetworkRequest::NetworkError:
+      case QgsBlockingNetworkRequest::TimeoutError:
+      case QgsBlockingNetworkRequest::ServerExceptionError:
+        return false;
+    }
+
+    const QgsNetworkReplyContent tileMapContent = tileMapNetworkRequest.reply();
+    const QByteArray tileMapRaw = tileMapContent.content();
+
+    const QJsonDocument tileMapDoc = QJsonDocument::fromJson( tileMapRaw, &err );
+    if ( !tileMapDoc.isNull() )
+    {
+      tileMap = tileMapDoc.object().toVariantMap();
+    }
+  }
+
   mSourcePath = tileServiceUri + '/' + mArcgisLayerConfiguration.value( QStringLiteral( "tiles" ) ).toList().value( 0 ).toString();
   if ( !QgsVectorTileUtils::checkXYZUrlTemplate( mSourcePath ) )
   {
@@ -265,7 +298,7 @@ bool QgsArcGisVectorTileServiceDataProvider::setupArcgisVectorTileServiceConnect
 
   mArcgisLayerConfiguration.insert( QStringLiteral( "serviceUri" ), tileServiceUri );
 
-  mMatrixSet.fromEsriJson( mArcgisLayerConfiguration );
+  mMatrixSet.fromEsriJson( mArcgisLayerConfiguration, tileMap );
   mCrs = mMatrixSet.crs();
 
   // if hardcoded zoom limits aren't specified, take them from the server