CARTO: Polygon triangulation in CartoVectorTileLoader

modules/carto/src/api/maps-v3-client.ts

@@ -32,7 +32,7 @@ import {assert} from '../utils';
 
 const MAX_GET_LENGTH = 8192;
 const DEFAULT_CLIENT = 'deck-gl-carto';
-const V3_MINOR_VERSION = '3.1';
+const V3_MINOR_VERSION = '3.2';
 
 export type Headers = Record<string, string>;
 interface RequestParams {

modules/carto/src/layers/schema/carto-raster-tile-loader.ts

@@ -31,7 +31,7 @@ const CartoRasterTileLoader: LoaderWithParser = {
   parse: async (arrayBuffer, options?: CartoRasterTileLoaderOptions) =>
     parseCartoRasterTile(arrayBuffer, options),
   parseSync: parseCartoRasterTile,
-  worker: false, // TODO set to true once workers deployed to unpkg
+  worker: true,
   options: DEFAULT_OPTIONS
 };
 

modules/carto/src/layers/schema/carto-spatial-tile-loader.ts

@@ -33,7 +33,7 @@ const CartoSpatialTileLoader: LoaderWithParser = {
   parse: async (arrayBuffer, options?: CartoSpatialTileLoaderOptions) =>
     parseCartoSpatialTile(arrayBuffer, options),
   parseSync: parseCartoSpatialTile,
-  worker: false, // TODO set to true once workers deployed to unpkg
+  worker: true,
   options: DEFAULT_OPTIONS
 };
 

modules/carto/src/layers/schema/carto-vector-tile-loader.ts

@@ -1,5 +1,6 @@
+import earcut from 'earcut';
 import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
-import type {BinaryFeatures} from '@loaders.gl/schema';
+import type {BinaryFeatures, BinaryPolygonFeatures, TypedArray} from '@loaders.gl/schema';
 
 import {TileReader} from './carto-tile';
 import {parsePbf} from './tile-loader-utils';
@@ -31,19 +32,71 @@
   parse: async (arrayBuffer, options?: CartoVectorTileLoaderOptions) =>
     parseCartoVectorTile(arrayBuffer, options),
   parseSync: parseCartoVectorTile,
-  worker: false, // TODO set to true once workers deployed to unpkg
+  worker: true,
   options: DEFAULT_OPTIONS
 };
 
+function triangulatePolygon(
+  polygons: BinaryPolygonFeatures,
+  target: number[],
+  {
+    startPosition,
+    endPosition,
+    indices
+  }: {startPosition: number; endPosition: number; indices: TypedArray}
+): void {
+  const coordLength = polygons.positions.size;
+  const start = startPosition * coordLength;
+  const end = endPosition * coordLength;
+
+  // Extract positions and holes for just this polygon
+  const polygonPositions = polygons.positions.value.subarray(start, end);
+
+  // Holes are referenced relative to outer polygon
+  const holes = indices.slice(1).map((n: number) => n - startPosition);
+
+  // Compute triangulation
+  const triangles = earcut(polygonPositions, holes, coordLength);
+
+  // Indices returned by triangulation are relative to start
+  // of polygon, so we need to offset
+  for (let t = 0, tl = triangles.length; t < tl; ++t) {
+    target.push(startPosition + triangles[t]);
+  }
+}
+
+function triangulate(polygons: BinaryPolygonFeatures) {
+  const {polygonIndices, positions, primitivePolygonIndices} = polygons;
+  const triangles = [];
+
+  let rangeStart = 0;
+  for (let i = 0; i < polygonIndices.value.length - 1; i++) {
+    const startPosition = polygonIndices.value[i];
+    const endPosition = polygonIndices.value[i + 1];
+
+    // Extract hole indices between start & end position
+    const rangeEnd = primitivePolygonIndices.value.indexOf(endPosition);
+    const indices = primitivePolygonIndices.value.subarray(rangeStart, rangeEnd);
+    rangeStart = rangeEnd;
+
+    triangulatePolygon(polygons, triangles, {startPosition, endPosition, indices});
+  }
+
+  polygons.triangles = {value: new Uint32Array(triangles), size: 1};
+}
+
 function parseCartoVectorTile(
   arrayBuffer: ArrayBuffer,
   options?: CartoVectorTileLoaderOptions
 ): BinaryFeatures | null {
   if (!arrayBuffer) return null;
   const tile = parsePbf(arrayBuffer, TileReader);
 
-  // Note: there is slight, difference in `numericProps` type, however geojson/mvtlayer can cope with this
-  return tile as unknown as BinaryFeatures;
+  if (tile.polygons && !tile.polygons.triangles) {
+    triangulate(tile.polygons);
+  }
+
+  return tile;
 }
 
 export default CartoVectorTileLoader;

test/modules/carto/api/maps-api-client.spec.ts

@@ -368,21 +368,21 @@ test(`getDataV2#versionError`, async t => {
   {
     props: {},
     mapInstantiationUrl:
-      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.1&name=table'
+      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.2&name=table'
   },
   {
     props: {
       format: FORMATS.GEOJSON
     },
     mapInstantiationUrl:
-      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.1&name=table'
+      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.2&name=table'
   },
   {
     props: {
       format: FORMATS.NDJSON
     },
     mapInstantiationUrl:
-      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.1&name=table'
+      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.2&name=table'
   },
   {
     props: {
@@ -393,37 +393,37 @@ test(`getDataV2#versionError`, async t => {
       }
     },
     mapInstantiationUrl:
-      'http://carto-api-with-slash/v3/maps/connection_name/table?client=deck-gl-carto&v=3.1&name=table'
+      'http://carto-api-with-slash/v3/maps/connection_name/table?client=deck-gl-carto&v=3.2&name=table'
   },
   {
     props: {geoColumn: 'geog'},
     mapInstantiationUrl:
-      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.1&name=table&geo_column=geog'
+      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.2&name=table&geo_column=geog'
   },
   {
     props: {columns: ['a', 'b', 'c']},
     mapInstantiationUrl:
-      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.1&name=table&columns=a%2Cb%2Cc'
+      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.2&name=table&columns=a%2Cb%2Cc'
   },
   {
     props: {columns: ['a', 'b', 'c'], geoColumn: 'geog'},
     mapInstantiationUrl:
-      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.1&name=table&geo_column=geog&columns=a%2Cb%2Cc'
+      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.2&name=table&geo_column=geog&columns=a%2Cb%2Cc'
   },
   {
     props: {geoColumn: 'geog', aggregationExp: 'sum(col) as value'},
     mapInstantiationUrl:
-      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.1&name=table&geo_column=geog&aggregationExp=sum(col)%20as%20value'
+      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.2&name=table&geo_column=geog&aggregationExp=sum(col)%20as%20value'
   },
   {
     props: {geoColumn: 'quadbin:quadbin', aggregationExp: 'sum(col) as v', aggregationResLevel: 7},
     mapInstantiationUrl:
-      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.1&name=table&geo_column=quadbin%3Aquadbin&aggregationExp=sum(col)%20as%20v&aggregationResLevel=7'
+      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.2&name=table&geo_column=quadbin%3Aquadbin&aggregationExp=sum(col)%20as%20v&aggregationResLevel=7'
   },
   {
     props: {geoColumn: 'h3:h3', aggregationResLevel: 7},
     mapInstantiationUrl:
-      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.1&name=table&geo_column=h3%3Ah3&aggregationExp=1%20AS%20value&aggregationResLevel=7'
+      'http://carto-api/v3/maps/connection_name/table?client=deck-gl-carto&v=3.2&name=table&geo_column=h3%3Ah3&aggregationExp=1%20AS%20value&aggregationResLevel=7'
   },
   {
     props: {
@@ -432,7 +432,7 @@ test(`getDataV2#versionError`, async t => {
       queryParameters: {end: '2021-09-17', start: '2021-09-15'}
     },
     mapInstantiationUrl:
-      'http://carto-api/v3/maps/connection_name/query?client=deck-gl-carto&v=3.1&q=select%20*%20from%20table&queryParameters=%7B%22end%22%3A%222021-09-17%22%2C%22start%22%3A%222021-09-15%22%7D'
+      'http://carto-api/v3/maps/connection_name/query?client=deck-gl-carto&v=3.2&q=select%20*%20from%20table&queryParameters=%7B%22end%22%3A%222021-09-17%22%2C%22start%22%3A%222021-09-15%22%7D'
   }
 ].forEach(({props, mapInstantiationUrl}) => {
   for (const useSetDefaultCredentials of [true, false]) {
@@ -557,7 +557,7 @@ test('fetchLayerData#post', async t => {
       const body = JSON.parse(options.body);
       t.is(body.q, source, 'should have query in body');
       t.is(body.client, 'deck-gl-carto', 'should have client in body');
-      t.is(body.v, '3.1', 'should have v=3.1 in body');
+      t.is(body.v, '3.2', 'should have v=3.2 in body');
       for (const p in props) {
         // Special case for geoColumn
         const prop = p === 'geoColumn' ? 'geo_column' : p;
@@ -825,7 +825,7 @@ test('fetchMap#geoColumn', async t => {
   const geoColumn = 'geo_column';
   const columns = ['a', 'b'];
 
-  const mapInstantiationUrl = `https://gcp-us-east1.api.carto.com/v3/maps/${connectionName}/table?client=deck-gl-carto&v=3.1&name=${source}&geo_column=${geoColumn}&columns=a%2Cb`;
+  const mapInstantiationUrl = `https://gcp-us-east1.api.carto.com/v3/maps/${connectionName}/table?client=deck-gl-carto&v=3.2&name=${source}&geo_column=${geoColumn}&columns=a%2Cb`;
   const table = {type: MAP_TYPES.TABLE, columns, geoColumn, connectionName, source};
 
   const mapResponse = {

test/modules/carto/carto-layer.spec.ts

@@ -88,43 +88,45 @@ mockedV3Test('CartoLayer#v3', async t => {
     }
   };
 
+  const props = {
+    connection: 'conn_name',
+    credentials: CREDENTIALS_V3,
+    loadOptions: {worker: false}
+  };
+
   await testLayerAsync({
     Layer: CartoLayer,
     testCases: [
       {
         props: {
+          ...props,
           data: 'select * from table',
-          type: MAP_TYPES.QUERY,
-          connection: 'conn_name',
-          credentials: CREDENTIALS_V3
+          type: MAP_TYPES.QUERY
         },
         onAfterUpdate
       },
       {
         props: {
+          ...props,
           data: 'tileset',
-          connection: 'conn_name',
-          type: MAP_TYPES.TILESET,
-          credentials: CREDENTIALS_V3
+          type: MAP_TYPES.TILESET
         },
         onAfterUpdate
       },
       {
         props: {
+          ...props,
           data: 'table',
-          connection: 'conn_name',
-          type: MAP_TYPES.TABLE,
-          credentials: CREDENTIALS_V3
+          type: MAP_TYPES.TABLE
         },
         onAfterUpdate
       },
       {
         props: {
+          ...props,
           data: 'dynamic_tileset',
-          connection: 'conn_name',
           type: MAP_TYPES.TABLE,
-          formatTiles: TILE_FORMATS.BINARY,
-          credentials: CREDENTIALS_V3
+          formatTiles: TILE_FORMATS.BINARY
         },
         onAfterUpdate
       }
@@ -164,7 +166,8 @@ mockedV3Test('CartoLayer#loadOptions', async t => {
           credentials: CREDENTIALS_V3,
           loadOptions: {
             custom: 'value',
-            fetch: {headers: {'Custom-Header': 'Header-Value'}}
+            fetch: {headers: {'Custom-Header': 'Header-Value'}},
+            worker: false
           }
         },
         onAfterUpdate
@@ -326,7 +329,8 @@ mockedV3Test('CartoLayer#_updateData executed when props changes', async t => {
         type: MAP_TYPES.TABLE,
         data: 'table',
         connection: 'connection_name',
-        credentials: CREDENTIALS_V3
+        credentials: CREDENTIALS_V3,
+        loadOptions: {worker: false}
       },
       onAfterUpdate({layer, spies}) {
         if (layer.isLoaded) {
@@ -428,7 +432,8 @@ mockedV3Test('CartoLayer#_updateData invalid apiVersion', async t => {
         type: MAP_TYPES.TABLE,
         data: 'table',
         connection: 'connection_name',
-        credentials: CREDENTIALS_V3
+        credentials: CREDENTIALS_V3,
+        loadOptions: {worker: false}
       }
     },
     {
@@ -465,6 +470,7 @@ mockedV3Test('CartoLayer#onDataLoad', async t => {
         type: MAP_TYPES.TILESET,
         connection: 'connection_name',
         credentials: CREDENTIALS_V3,
+        loadOptions: {worker: false},
         onDataLoad
       },
       onAfterUpdate: ({layer}) => {
@@ -509,6 +515,7 @@ mockedV3Test('CartoLayer#onDataError', async t => {
         type: MAP_TYPES.TILESET,
         connection: 'connection_name',
         credentials: CREDENTIALS_V3,
+        loadOptions: {worker: false},
         onDataError
       },
       onAfterUpdate: ({layer}) => {
@@ -550,7 +557,7 @@ mockedV3Test('CartoLayer#dynamic', async t => {
         formatTiles: TILE_FORMATS.BINARY,
         connection: 'connection_name',
         credentials: CREDENTIALS_V3,
-
+        loadOptions: {worker: false},
         onDataLoad
       },
       onAfterUpdate: ({layer, subLayers}) => {

test/modules/carto/data/binaryTilePolygonNoTri.json

@@ -0,0 +1 @@
+[10,34,10,2,16,2,18,2,16,1,26,2,16,1,42,20,10,16,103,114,111,115,115,70,108,111,111,114,65,114,101,97,77,50,18,0,18,41,10,2,16,2,18,5,10,1,0,16,1,26,2,16,1,34,2,16,1,50,20,10,16,103,114,111,115,115,70,108,111,111,114,65,114,101,97,77,50,18,0,26,149,11,10,197,6,10,192,6,0,0,0,32,137,189,13,192,0,0,0,224,10,54,68,64,0,0,0,224,1,190,13,192,0,0,0,224,5,54,68,64,0,0,0,64,83,189,13,192,0,0,0,128,1,54,68,64,0,0,0,96,34,189,13,192,0,0,0,32,5,54,68,64,0,0,0,128,38,189,13,192,0,0,0,96,6,54,68,64,0,0,0,32,137,189,13,192,0,0,0,224,10,54,68,64,0,0,0,128,149,189,13,192,0,0,0,192,7,54,68,64,0,0,0,64,169,189,13,192,0,0,0,128,6,54,68,64,0,0,0,96,186,189,13,192,0,0,0,96,7,54,68,64,0,0,0,32,162,189,13,192,0,0,0,96,8,54,68,64,0,0,0,128,149,189,13,192,0,0,0,192,7,54,68,64,0,0,0,192,141,189,13,192,0,0,0,128,3,54,68,64,0,0,0,192,167,189,13,192,0,0,0,64,4,54,68,64,0,0,0,128,159,189,13,192,0,0,0,224,4,54,68,64,0,0,0,160,133,189,13,192,0,0,0,64,4,54,68,64,0,0,0,192,141,189,13,192,0,0,0,128,3,54,68,64,0,0,0,160,125,189,13,192,0,0,0,160,7,54,68,64,0,0,0,96,133,189,13,192,0,0,0,32,7,54,68,64,0,0,0,32,150,189,13,192,0,0,0,128,7,54,68,64,0,0,0,160,141,189,13,192,0,0,0,32,8,54,68,64,0,0,0,160,125,189,13,192,0,0,0,160,7,54,68,64,0,0,0,128,112,189,13,192,0,0,0,160,5,54,68,64,0,0,0,96,103,189,13,192,0,0,0,96,6,54,68,64,0,0,0,32,83,189,13,192,0,0,0,224,5,54,68,64,0,0,0,224,92,189,13,192,0,0,0,32,5,54,68,64,0,0,0,128,112,189,13,192,0,0,0,160,5,54,68,64,0,0,0,128,202,186,13,192,0,0,0,0,4,54,68,64,0,0,0,160,111,186,13,192,0,0,0,32,255,53,68,64,0,0,0,192,93,186,13,192,0,0,0,128,255,53,68,64,0,0,0,128,102,186,13,192,0,0,0,0,0,54,68,64,0,0,0,96,52,186,13,192,0,0,0,32,2,54,68,64,0,0,0,96,148,186,13,192,0,0,0,32,7,54,68,64,0,0,0,32,193,186,13,192,0,0,0,224,4,54,68,64,0,0,0,224,188,186,13,192,0,0,0,160,4,54,68,64,0,0,0,128,202,186,13,192,0,0,0,0,4,54,68,64,0,0,0,64,6,187,13,192,0,0,0,0,1,54,68,64,0,0,0,224,222,186,13,192,0,0,0,224,252,53,68,64,0,0,0,96,171,186,13,192,0,0,0,224,253,53,68,64,0,0,0,64,179,186,13,192,0,0,0,192,254,53,68,64,0,0,0,224,162,186,13,192,0,0,0,32,255,53,68,64,0,0,0,32,155,186,13,192,0,0,0,64,254,53,68,64,0,0,0,160,111,186,13,192,0,0,0,32,255,53,68,64,0,0,0,32,127,186,13,192,0,0,0,0,0,54,68,64,0,0,0,128,148,186,13,192,0,0,0,128,255,53,68,64,0,0,0,0,157,186,13,192,0,0,0,0,0,54,68,64,0,0,0,192,137,186,13,192,0,0,0,128,0,54,68,64,0,0,0,160,158,186,13,192,0,0,0,160,1,54,68,64,0,0,0,224,175,186,13,192,0,0,0,0,1,54,68,64,0,0,0,192,188,186,13,192,0,0,0,160,1,54,68,64,0,0,0,64,172,186,13,192,0,0,0,96,2,54,68,64,0,0,0,128,202,186,13,192,0,0,0,0,4,54,68,64,0,0,0,64,6,187,13,192,0,0,0,0,1,54,68,64,16,2,18,8,10,4,0,26,35,52,16,1,26,56,10,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,16,1,34,56,10,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,16,1,42,12,10,8,0,6,11,16,21,26,35,52,16,1,58,0,58,0,58,0,66,184,3,10,16,103,114,111,115,115,70,108,111,111,114,65,114,101,97,77,50,18,163,3,10,160,3,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,160,169,64,0,0,0,0,0,254,165,64,0,0,0,0,0,254,165,64,0,0,0,0,0,254,165,64,0,0,0,0,0,254,165,64,0,0,0,0,0,254,165,64,0,0,0,0,0,254,165,64,0,0,0,0,0,254,165,64,0,0,0,0,0,254,165,64,0,0,0,0,0,254,165,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64,0,0,0,0,0,52,157,64]

test/modules/carto/layers/carto-vector-tile.spec.ts

@@ -4,13 +4,29 @@ import CartoVectoTileLoader from '@deck.gl/carto/layers/schema/carto-vector-tile
 
 // See test/modules/carto/responseToJson for details for creating test data
 import binaryVectorTileData from '../data/binaryTilePolygon.json';
+import binaryNoTrianglesTileData from '../data/binaryTilePolygonNoTri.json';
 const BINARY_VECTOR_TILE = new Uint8Array(binaryVectorTileData).buffer;
+const BINARY_VECTOR_TILE_NOTRI = new Uint8Array(binaryNoTrianglesTileData).buffer;
 
 test('Parse Carto Vector Tile', async t => {
   const {polygons} = CartoVectoTileLoader.parseSync(BINARY_VECTOR_TILE);
-  t.deepEqual(polygons.positions.value.length, 2 * 151, 'Positions correctly decoded');
-  t.deepEqual(polygons.globalFeatureIds.value.length, 151, 'globalFeatureIds correctly decoded');
-  t.deepEqual(polygons.properties, [{DO_LABEL: 'Puerto Rico'}], 'Properites correctly decoded');
+  t.equal(polygons.positions.value.length, 2 * 151, 'Positions correctly decoded');
+  t.equal(polygons.globalFeatureIds.value.length, 151, 'globalFeatureIds correctly decoded');
+  t.deepEqual(polygons.properties, [{DO_LABEL: 'Puerto Rico'}], 'Properties correctly decoded');
   t.deepEqual(polygons.fields, [{id: 31}], 'Fields correctly decoded');
   t.end();
 });
+
+test('Carto Vector Tile triangulation', async t => {
+  const {polygons} = CartoVectoTileLoader.parseSync(BINARY_VECTOR_TILE_NOTRI);
+  t.equal(polygons.positions.value.length, 2 * 52, 'Positions correctly decoded');
+  t.equal(polygons.globalFeatureIds.value.length, 52, 'globalFeatureIds correctly decoded');
+  t.equal(
+    polygons.numericProps.grossFloorAreaM2.value.length,
+    52,
+    'Numeric Properties correctly decoded'
+  );
+  t.ok(polygons.triangles, 'triangles array added');
+  t.equal(polygons.triangles.value.length, 141, 'Polygons triangulated correctly');
+  t.end();
+});