feat(pmtiles): Experimental PMTilesLoader (#3023)

modules/mvt/src/lib/get-schemas-from-tilejson.ts

@@ -0,0 +1,64 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Schema, Field, DataType, SchemaMetadata, FieldMetadata} from '@loaders.gl/schema';
+import type {TileJSONLayer, TileJSONField} from './parse-tilejson';
+
+// LAYERS
+
+export function getSchemaFromTileJSONLayer(layer: TileJSONLayer): Schema {
+  const fields: Field[] = [];
+  if (layer.fields) {
+    for (const field of layer.fields) {
+      fields.push({
+        name: field.name,
+        type: getDataTypeFromTileJSONField(field),
+        metadata: getMetadataFromTileJSONField(field)
+      });
+    }
+  }
+  return {
+    metadata: getMetadataFromTileJSONLayer(layer),
+    fields
+  };
+}
+
+function getMetadataFromTileJSONLayer(layer: TileJSONLayer): SchemaMetadata {
+  const metadata: Record<string, string> = {};
+  for (const [key, value] of Object.entries(layer)) {
+    if (key !== 'fields' && value) {
+      metadata[key] = JSON.stringify(value);
+    }
+  }
+  return metadata;
+}
+
+// FIELDS
+
+function getDataTypeFromTileJSONField(field: TileJSONField): DataType {
+  switch (field.type.toLowerCase()) {
+    case 'float32':
+      return 'float32';
+    case 'number':
+    case 'float64':
+      return 'float64';
+    case 'string':
+    case 'utf8':
+      return 'utf8';
+    case 'boolean':
+      return 'bool';
+    default:
+      return 'null';
+  }
+}
+
+function getMetadataFromTileJSONField(field: TileJSONField): FieldMetadata {
+  const metadata: Record<string, string> = {};
+  for (const [key, value] of Object.entries(field)) {
+    if (key !== 'name' && value) {
+      metadata[key] = JSON.stringify(value);
+    }
+  }
+  return metadata;
+}

modules/mvt/src/lib/parse-tilejson.ts

@@ -2,20 +2,31 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import {Schema} from '@loaders.gl/schema';
+import {getSchemaFromTileJSONLayer} from './get-schemas-from-tilejson';
+
 export type TileJSONOptions = {
   /** max number of values. If not provided, include all values in the source tilestats */
   maxValues?: number;
 };
 
 /** Parsed and typed TileJSON, merges Tilestats information if present */
 export type TileJSON = {
+  /** Name of the tileset (for presentation in UI) */
   name?: string;
+  /** A description of the contents or purpose of the tileset */
   description?: string;
+  /** The version of the tileset */
   version?: string;
 
   tileFormat?: string;
   tilesetType?: string;
 
+  /** Generating application. Tippecanoe adds this. */
+  generator?: string;
+  /** Generating application options. Tippecanoe adds this. */
+  generatorOptions?: string;
+
   /** Tile indexing scheme */
   scheme?: 'xyz' | 'tms';
   /** Sharded URLs */
@@ -33,11 +44,6 @@ export type TileJSON = {
   // Combination of tilestats (if present) and tilejson layer information
   layers?: TileJSONLayer[];
 
-  /** Generating application. Tippecanoe adds this. */
-  generator?: string;
-  /** Generating application options. Tippecanoe adds this. */
-  generatorOptions?: string;
-
   /** Any nested JSON metadata */
   metaJson?: any | null;
 };
@@ -61,6 +67,8 @@ export type TileJSONLayer = {
   minZoom?: number;
   maxZoom?: number;
   fields: TileJSONField[];
+
+  schema?: Schema;
 };
 
 export type TileJSONField = {
@@ -262,17 +270,16 @@ function parseTilestatsForLayer(layer: TilestatsLayer, options: TileJSONOptions)
 }
 
 function mergeLayers(layers: TileJSONLayer[], tilestatsLayers: TileJSONLayer[]): TileJSONLayer[] {
-  return layers.map((layer) => {
+  return layers.map((layer: TileJSONLayer): TileJSONLayer => {
     const tilestatsLayer = tilestatsLayers.find((tsLayer) => tsLayer.name === layer.name);
-    // For aesthetics in JSON dumps, we preserve field order (make sure layers is last)
-    const fields = tilestatsLayer?.fields || [];
-    const layer2: Partial<TileJSONLayer> = {...layer};
-    delete layer2.fields;
-    return {
-      ...layer2,
+    const fields = tilestatsLayer?.fields || layer.fields || [];
+    const mergedLayer = {
+      ...layer,
       ...tilestatsLayer,
       fields
     } as TileJSONLayer;
+    mergedLayer.schema = getSchemaFromTileJSONLayer(mergedLayer);
+    return mergedLayer;
   });
 }
 
@@ -419,7 +426,7 @@ function attributeTypeToFieldType(aType: string): {type: string} {
   const type = aType.toLowerCase();
   if (!type || !attrTypeMap[type]) {
     // console.warn(
-    //   `cannot convert feature type ${type} to loaders.gl data type, use string by default`
+    //   `cannot convert attribute type ${type} to loaders.gl data type, use string by default`
     // );
   }
   return attrTypeMap[type] || {type: 'string'};

modules/mvt/src/lib/types.ts

@@ -1,8 +1,46 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
-// Copyright vis.gl contributors
+// Copyright (c) vis.gl contributors
+
+/** For local coordinates, the tileIndex is not required */
+type MVTLocalCoordinatesOptions = {
+  /**
+   * When set to `local`, the parser will return a flat array of GeoJSON objects with local coordinates decoded from tile origin.
+   */
+  coordinates: 'local';
+  tileIndex: null;
+};
+
+/** In WGS84 coordinates, the tileIndex is required */
+type MVTWgs84CoordinatesOptions = {
+  /**
+   * When set to `wgs84`, the parser will return a flat array of GeoJSON objects with coordinates in longitude, latitude decoded from the provided tile index.
+   */
+  coordinates?: 'wgs84';
+
+  /**
+   * Mandatory with `wgs84` coordinates option. An object containing tile index values (`x`, `y`,
+   * `z`) to reproject features' coordinates into WGS84.
+   */
+  tileIndex?: {x: number; y: number; z: number};
+};
+
+export type MVTOptions = (MVTLocalCoordinatesOptions | MVTWgs84CoordinatesOptions) & {
+  shape?: 'geojson-table' | 'columnar-table' | 'geojson' | 'binary' | 'binary-geometry';
+  /**
+   * When non-`null`, the layer name of each feature is added to
+   * `feature.properties[layerProperty]`. (A `feature.properties` object is created if the feature
+   * has no existing properties). If set to `null`, a layer name property will not be added.
+   */
+  layerProperty?: string | number;
+
+  /**
+   * Optional list of layer names. If not `null`, only features belonging to the named layers will
+   * be included in the output. If `null`, features from all layers are returned.
+   */
+  layers?: string[];
+};
 
-/** TODO where is this used? */
 export type MVTMapboxGeometry = {
   type?: string;
   id?: number;

modules/mvt/src/mvt-loader.ts

@@ -26,6 +26,8 @@ export type MVTLoaderOptions = LoaderOptions & {
     workerUrl?: string;
   };
   gis?: {
+    /** @deprecated Use options.mvt.shape === 'binary-geometry' */
+    binary?: boolean;
     /** @deprecated. Use options.mvt.shape */
     format?: 'geojson-table' | 'columnar-table' | 'geojson' | 'binary' | 'binary-geometry';
   };

modules/mvt/test/data/tilejson/tilejson.ts

@@ -9,3 +9,7 @@ export const TILEJSONS = [
   {url: '@loaders.gl/mvt/test/data/tilejson/world-bright-ssl.tilejson', bad: false},
   {url: '@loaders.gl/mvt/test/data/tilejson/world-bright.tilejson', bad: false}
 ];
+
+export const TILEJSONS_WITH_TILESTATS = [
+  {url: '@loaders.gl/mvt/test/data/tilejson/tippecanoe.tilejson', bad: false}
+];

modules/pmtiles/src/index.ts

@@ -7,3 +7,6 @@ export {PMTilesSource} from './pmtiles-source';
 export type {PMTilesMetadata} from './lib/parse-pmtiles';
 export type {PMTilesTileSourceProps} from './pmtiles-source';
 export {PMTilesTileSource} from './pmtiles-source';
+
+export {PMTilesLoader as _PMTilesLoader} from './pmtiles-loader';
+export type {PMTilesLoaderOptions} from './pmtiles-loader';

modules/pmtiles/src/lib/parse-pmtiles.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {LoaderOptions} from '@loaders.gl/loader-utils';
+import {LoaderOptions} from '@loaders.gl/loader-utils';
 import type {TileJSON} from '@loaders.gl/mvt';
 import {TileJSONLoader} from '@loaders.gl/mvt';
 // import {Source, PMTiles, Header, TileType} from 'pmtiles';
@@ -61,12 +61,10 @@ export type PMTilesMetadata = {
  */
 export function parsePMTilesHeader(
   header: pmtiles.Header,
-  pmmetadata: unknown,
+  pmtilesMetadata: Record<string, unknown> | null,
   options?: {includeFormatHeader?: boolean},
   loadOptions?: LoaderOptions
 ): PMTilesMetadata {
-  const pmtilesMetadata = pmmetadata as Record<string, unknown> | null;
-
   // Ironically, to use the TileJSON loader we need to stringify the metadata again.
   // This is the price of integrating with the existing pmtiles library.
   // TODO - provide a non-standard TileJSONLoader parsers that accepts a JSON object?

modules/pmtiles/src/lib/version.ts

@@ -0,0 +1,8 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// Version constant cannot be imported, it needs to correspond to the build version of **this** module.
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+export const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';

modules/pmtiles/src/pmtiles-loader.ts

@@ -0,0 +1,65 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {LoaderWithParser, LoaderOptions, ReadableFile} from '@loaders.gl/loader-utils';
+import {BlobFile} from '@loaders.gl/loader-utils';
+import {VERSION} from './lib/version';
+
+import {VectorSourceInfo, ImageSourceInfo} from './source-info';
+import {PMTilesTileSource, PMTilesTileSourceProps} from './pmtiles-source';
+
+export type PMTilesLoaderOptions = LoaderOptions & {
+  pmtiles?: PMTilesTileSourceProps['pmtiles'];
+};
+
+/**
+ * Loader for PMTiles metadata
+ * @note This loader is intended to allow PMTiles to be treated like other file types in top-level loading logic.
+ * @note For actual access to the tile data, use the PMTilesSource class.
+ */
+export const PMTilesLoader = {
+  name: 'PMTiles',
+  id: 'pmtiles',
+  module: 'pmtiles',
+  version: VERSION,
+  extensions: ['pmtiles'],
+  mimeTypes: ['application/octet-stream'],
+  tests: ['PMTiles'],
+  options: {
+    pmtiles: {}
+  },
+  parse: async (arrayBuffer: ArrayBuffer, options?: PMTilesLoaderOptions) =>
+    parseFileAsPMTiles(new BlobFile(new Blob([arrayBuffer])), options),
+  parseFile: parseFileAsPMTiles
+} as const satisfies LoaderWithParser<
+  VectorSourceInfo | ImageSourceInfo,
+  never,
+  PMTilesLoaderOptions
+>;
+
+async function parseFileAsPMTiles(
+  file: ReadableFile,
+  options?: PMTilesLoaderOptions
+): Promise<VectorSourceInfo | ImageSourceInfo> {
+  const source = new PMTilesTileSource(file.handle as string | Blob, {
+    pmtiles: options?.pmtiles || {}
+  });
+  const formatSpecificMetadata = await source.getMetadata();
+  const {tileMIMEType, tilejson = {}} = formatSpecificMetadata;
+  const {layers = []} = tilejson;
+  switch (tileMIMEType) {
+    case 'application/vnd.mapbox-vector-tile':
+      return {
+        shape: 'vector-source',
+        layers: layers.map((layer) => ({name: layer.name, schema: layer.schema})),
+        tables: [],
+        formatSpecificMetadata
+      } as VectorSourceInfo;
+    case 'image/png':
+    case 'image/jpeg':
+      return {shape: 'image-source', formatSpecificMetadata} as ImageSourceInfo;
+    default:
+      throw new Error(`PMTilesLoader: Unsupported tile MIME type ${tileMIMEType}`);
+  }
+}

modules/pmtiles/src/pmtiles-source.ts

@@ -44,7 +44,6 @@ export const PMTilesSource = {
 } as const satisfies Source<PMTilesTileSource, PMTilesTileSourceProps>;
 
 export type PMTilesTileSourceProps = DataSourceProps & {
-  url: string | Blob;
   attributions?: string[];
   pmtiles?: {
     loadOptions?: TileJSONLoaderOptions & MVTLoaderOptions & ImageLoaderOptions;
@@ -67,7 +66,7 @@ export class PMTilesTileSource extends DataSource implements ImageTileSource, Ve
     super(props);
     this.props = props;
     const url = typeof data === 'string' ? resolvePath(data) : new BlobSource(data, 'pmtiles');
-    this.data = props.url;
+    this.data = data;
     this.pmtiles = new PMTiles(url);
     this.getTileData = this.getTileData.bind(this);
     this.metadata = this.getMetadata();
@@ -79,7 +78,7 @@ export class PMTilesTileSource extends DataSource implements ImageTileSource, Ve
 
   async getMetadata(): Promise<PMTilesMetadata> {
     const pmtilesHeader = await this.pmtiles.getHeader();
-    const pmtilesMetadata = await this.pmtiles.getMetadata();
+    const pmtilesMetadata = ((await this.pmtiles.getMetadata()) as Record<string, unknown>) || {};
     const metadata: PMTilesMetadata = parsePMTilesHeader(
       pmtilesHeader,
       pmtilesMetadata,

modules/pmtiles/src/source-info.ts

@@ -0,0 +1,25 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Schema} from '@loaders.gl/schema';
+
+export type SourceInfo = VectorSourceInfo | ImageSourceInfo;
+
+/** Information about a vector source */
+export type VectorSourceInfo = {
+  shape: 'vector-source';
+  /** List of geospatial tables */
+  layers: {name: string; schema: Schema}[];
+  /** List of nongeospatial tables */
+  tables: {name: string; schema: Schema}[];
+  /** Format specific metadata */
+  formatSpecificMetadata?: Record<string, any>;
+};
+
+/** Information about an image source */
+export type ImageSourceInfo = {
+  shape: 'image-source';
+  /** Format specific metadata */
+  formatSpecificMetadata?: Record<string, any>;
+};

modules/pmtiles/test/data/tilesets.ts

@@ -11,3 +11,13 @@ export const PMTILESETS = [
 export const PMTILESETS_INVALID = ['empty.pmtiles', 'invalid.pmtiles', 'invalid_v4.pmtiles'].map(
   (tileset) => `@loaders.gl/pmtiles/test/data/${tileset}`
 );
+
+export const PMTILESETS_V3 = [
+  'protomaps(vector)ODbL_firenze.pmtiles',
+  'stamen_toner(raster)CC-BY+ODbL_z3.pmtiles',
+  'usgs-mt-whitney-8-15-webp-512.pmtiles'
+].map((tileset) => `@loaders.gl/pmtiles/test/data/pmtiles-v3/${tileset}`);
+
+export const PMTILESETS_VECTOR = ['protomaps(vector)ODbL_firenze.pmtiles'].map(
+  (tileset) => `@loaders.gl/pmtiles/test/data/pmtiles-v3/${tileset}`
+);

modules/pmtiles/test/index.ts

@@ -3,3 +3,5 @@
 // Copyright (c) vis.gl contributors
 
 import './pmtiles-source.spec';
+
+import './pmtiles-loader.spec';