Merge 9c4785d into c29c929

docs/specifications/loader-object-format.md

@@ -39,3 +39,14 @@ Each (non-worker) loader should define a `parse` function. Additional parsing fu
 Synchronous parsers are more flexible as they can support synchronous parsing which can simplify application logic and debugging, and iterator-based parsers are more flexible as they can support batched loading of large data sets in addition to atomic loading.
 
 You are encouraged to provide the most capable parser function you can (e.g. `parseSync` or `parseToIterator` if possible). Unless you are writing a completely new loader from scratch, the appropriate choice often depends on the capabilities of an existing external "loader" that you are working with.
+
+### Parser Function Signatures
+
+- `async parse(data : ArrayBuffer, options : Object, context : Object) : Object`
+- `parseSync(data : ArrayBuffer, options : Object, context : Object) : Object`
+- `parseInBatches(data : AsyncIterator, options : Object, context : Object) : AsyncIterator`
+
+The `context` parameter will contain the foolowing fields
+
+- `parse` or `parseSync`
+- `url` if available

examples/3d-tiles/webpack.config.js

@@ -26,7 +26,7 @@ const CONFIG = {
             [
               '@babel/preset-env',
               {
-                exclude: ['transform-regenerator']
+                exclude: ['@babel/transform-regenerator']
               }
             ],
             '@babel/preset-react'

modules/3d-tiles/src/encoders/helpers/encode-3d-tile-header.js

@@ -1,5 +1,5 @@
 // HELPER ENCODERS
-import {assert} from '@loaders.gl/core';
+import {assert} from '@loaders.gl/loader-utils';
 
 export function encode3DTileHeader(tile, dataView, byteOffset) {
   const HEADER_SIZE = 12;

modules/3d-tiles/src/parsers/helpers/parse-utils.js

@@ -2,7 +2,7 @@
 // See LICENSE.md and https://github.com/AnalyticalGraphicsInc/cesium/blob/master/LICENSE.md
 
 /* global TextDecoder */
-import {assert} from '@loaders.gl/core';
+import {assert} from '@loaders.gl/loader-utils';
 
 // Decode the JSON binary array into clear text
 export function getStringFromArrayBuffer(arrayBuffer, byteOffset, byteLength) {

modules/3d-tiles/src/parsers/parse-3d-tile-batched-model.js

@@ -9,11 +9,19 @@ import {parse3DTileHeaderSync} from './helpers/parse-3d-tile-header';
 import {parse3DTileTablesHeaderSync, parse3DTileTablesSync} from './helpers/parse-3d-tile-tables';
 import {parse3DTileGLTFViewSync, extractGLTF, GLTF_FORMAT} from './helpers/parse-3d-tile-gltf-view';
 
-export async function parseBatchedModel3DTile(tile, arrayBuffer, byteOffset, options) {
-  return parseBatchedModel3DTileSync(tile, arrayBuffer, byteOffset, options);
+export async function parseBatchedModel3DTile(tile, arrayBuffer, byteOffset, options, context) {
+  byteOffset = parseBatchedModel(tile, arrayBuffer, byteOffset, options, context);
+  await extractGLTF(tile, GLTF_FORMAT.EMBEDDED, options, context);
+  return byteOffset;
+}
+
+export function parseBatchedModel3DTileSync(tile, arrayBuffer, byteOffset, options, context) {
+  byteOffset = parseBatchedModel(tile, arrayBuffer, byteOffset, options, context);
+  extractGLTF(tile, GLTF_FORMAT.EMBEDDED, options, context);
+  return byteOffset;
 }
 
-export function parseBatchedModel3DTileSync(tile, arrayBuffer, byteOffset, options) {
+export function parseBatchedModel(tile, arrayBuffer, byteOffset, options, context) {
   byteOffset = parse3DTileHeaderSync(tile, arrayBuffer, byteOffset, options);
 
   byteOffset = parse3DTileTablesHeaderSync(tile, arrayBuffer, byteOffset, options);
@@ -24,17 +32,7 @@ export function parseBatchedModel3DTileSync(tile, arrayBuffer, byteOffset, optio
   const featureTable = new Tile3DFeatureTable(tile.featureTableJson, tile.featureTableBinary);
   tile.rtcCenter = featureTable.getGlobalProperty('RTC_CENTER', GL.FLOAT, 3);
 
-  extractGLTF(tile, GLTF_FORMAT.EMBEDDED, options);
-
-  /* TODO - Remove. This was a shot in the dark, didn't work...
-  if (tile.rtcCenter) {
-    const instanceTransform = new Matrix4();
-    Ellipsoid.WGS84.eastNorthUpToFixedFrame(tile.rtcCenter, instanceTransform);
-    const modelMatrix = new Matrix4();
-    instanceTransform.getRotationMatrix3(modelMatrix);
-    tile.instances = [{modelMatrix: modelMatrix.invert()}];
-  }
-  */
+  extractGLTF(tile, GLTF_FORMAT.EMBEDDED, options, context);
 
   return byteOffset;
 }

modules/3d-tiles/src/parsers/parse-3d-tile-composite.js

@@ -6,7 +6,15 @@
 
 import {parse3DTileHeaderSync} from './helpers/parse-3d-tile-header';
 
-export async function parseComposite3DTile(tile, arrayBuffer, byteOffset, options, parse3DTile) {
+// eslint-disable-next-line max-params
+export async function parseComposite3DTile(
+  tile,
+  arrayBuffer,
+  byteOffset,
+  options,
+  context,
+  parse3DTile
+) {
   byteOffset = parse3DTileHeaderSync(tile, arrayBuffer, byteOffset, options);
 
   const view = new DataView(arrayBuffer);
@@ -20,14 +28,22 @@ export async function parseComposite3DTile(tile, arrayBuffer, byteOffset, option
   while (tile.tiles.length < tile.tilesLength && tile.byteLength - byteOffset > 12) {
     const subtile = {};
     tile.tiles.push(subtile);
-    byteOffset = await parse3DTile(arrayBuffer, byteOffset, options, subtile);
+    byteOffset = await parse3DTile(arrayBuffer, byteOffset, options, context, subtile);
     // TODO - do we need to add any padding in between tiles?
   }
 
   return byteOffset;
 }
 
-export function parseComposite3DTileSync(tile, arrayBuffer, byteOffset, options, parse3DTileSync) {
+// eslint-disable-next-line max-params
+export function parseComposite3DTileSync(
+  tile,
+  arrayBuffer,
+  byteOffset,
+  options,
+  context,
+  parse3DTileSync
+) {
   byteOffset = parse3DTileHeaderSync(tile, arrayBuffer, byteOffset, options);
 
   const view = new DataView(arrayBuffer);
@@ -41,7 +57,7 @@ export function parseComposite3DTileSync(tile, arrayBuffer, byteOffset, options,
   while (tile.tiles.length < tile.tilesLength && tile.byteLength - byteOffset > 12) {
     const subtile = {};
     tile.tiles.push(subtile);
-    byteOffset = parse3DTileSync(arrayBuffer, byteOffset, options, subtile);
+    byteOffset = parse3DTileSync(arrayBuffer, byteOffset, options, context, subtile);
     // TODO - do we need to add any padding in between tiles?
   }
 

modules/3d-tiles/src/parsers/parse-3d-tile-instanced-model.js

@@ -11,13 +11,21 @@ import {parse3DTileHeaderSync} from './helpers/parse-3d-tile-header';
 import {parse3DTileTablesHeaderSync, parse3DTileTablesSync} from './helpers/parse-3d-tile-tables';
 import {parse3DTileGLTFViewSync, extractGLTF} from './helpers/parse-3d-tile-gltf-view';
 
-export async function parseInstancedModel3DTile(tile, arrayBuffer, byteOffset, options) {
-  return parseInstancedModel3DTileSync(tile, arrayBuffer, byteOffset, options);
+export async function parseInstancedModel3DTile(tile, arrayBuffer, byteOffset, options, context) {
+  byteOffset = parseInstancedModel(tile, arrayBuffer, byteOffset, options, context);
+  await extractGLTF(tile, tile.gltfFormat, options, context);
+  return byteOffset;
 }
 
 // Reference code:
 // https://github.com/AnalyticalGraphicsInc/cesium/blob/master/Source/Scene/Instanced3DModel3DTileContent.js#L190
-export function parseInstancedModel3DTileSync(tile, arrayBuffer, byteOffset, options) {
+export function parseInstancedModel3DTileSync(tile, arrayBuffer, byteOffset, options, context) {
+  byteOffset = parseInstancedModel(tile, arrayBuffer, byteOffset, options, context);
+  // extractGLTFSync(tile, tile.gltfFormat, options, context);
+  return byteOffset;
+}
+
+function parseInstancedModel(tile, arrayBuffer, byteOffset, options, context) {
   byteOffset = parse3DTileHeaderSync(tile, arrayBuffer, byteOffset, options);
   if (tile.version !== 1) {
     throw new Error(`Instanced 3D Model version ${tile.version} is not supported`);
@@ -58,8 +66,6 @@ export function parseInstancedModel3DTileSync(tile, arrayBuffer, byteOffset, opt
     instancesLength
   );
 
-  extractGLTF(tile, tile.gltfFormat, options);
-
   extractInstancedAttributes(tile, featureTable, batchTable, instancesLength);
 
   return byteOffset;
@@ -80,15 +86,6 @@ function extractInstancedAttributes(tile, featureTable, batchTable, instancesLen
     // TODO - tileset is not available at this stage, tile is parsed independently
     // upAxis: (tileset && tileset._gltfUpAxis) || [0, 1, 0],
     forwardAxis: [1, 0, 0]
-
-    // Cesium internals
-    // opaquePass: Pass.CESIUM_3D_TILE, // Draw opaque portions during the 3D Tiles pass
-    // pickIdLoaded: getPickIdCallback(tile),
-    // imageBasedLightingFactor: tileset.imageBasedLightingFactor,
-    // lightColor: tileset.lightColor,
-    // luminanceAtZenith: tileset.luminanceAtZenith,
-    // sphericalHarmonicCoefficients: tileset.sphericalHarmonicCoefficients,
-    // specularEnvironmentMaps: tileset.specularEnvironmentMaps
   };
 
   const instances = collectionOptions.instances;

modules/3d-tiles/src/parsers/parse-3d-tile-point-cloud.js

@@ -4,7 +4,6 @@
 import {Vector3} from 'math.gl';
 import {GL} from '@loaders.gl/math'; // 'math.gl/geometry';
 
-import {parse} from '@loaders.gl/core';
 import Tile3DFeatureTable from '../classes/tile-3d-feature-table';
 import Tile3DBatchTable from '../classes/tile-3d-batch-table';
 import {parse3DTileHeaderSync} from './helpers/parse-3d-tile-header';
@@ -13,47 +12,40 @@ import {normalize3DTileColorAttribute} from './helpers/normalize-3d-tile-colors'
 import {normalize3DTileNormalAttribute} from './helpers/normalize-3d-tile-normals';
 import {normalize3DTilePositionAttribute} from './helpers/normalize-3d-tile-positions';
 
-export async function parsePointCloud3DTile(tile, arrayBuffer, byteOffset, options) {
+export async function parsePointCloud3DTile(tile, arrayBuffer, byteOffset, options, context) {
   byteOffset = parse3DTileHeaderSync(tile, arrayBuffer, byteOffset, options);
   byteOffset = parse3DTileTablesHeaderSync(tile, arrayBuffer, byteOffset, options);
   byteOffset = parse3DTileTablesSync(tile, arrayBuffer, byteOffset, options);
-
-  await extractPointCloud(tile, options);
-
-  return byteOffset;
-}
-
-export function parsePointCloud3DTileSync(tile, arrayBuffer, byteOffset, options) {
-  byteOffset = parse3DTileHeaderSync(tile, arrayBuffer, byteOffset, options);
-  byteOffset = parse3DTileTablesHeaderSync(tile, arrayBuffer, byteOffset, options);
-  byteOffset = parse3DTileTablesSync(tile, arrayBuffer, byteOffset, options);
-
-  extractPointCloudSync(tile, options);
-
-  return byteOffset;
-}
-
-// eslint-disable-next-line max-statements, complexity
-async function extractPointCloud(tile, options) {
   initializeTile(tile);
 
   const {featureTable, batchTable} = parsePointCloudTables(tile);
 
-  await parseDraco(tile, featureTable, batchTable, options);
+  await parseDraco(tile, featureTable, batchTable, options, context);
 
   parsePositions(tile, featureTable, options);
   parseColors(tile, featureTable, batchTable, options);
   parseNormals(tile, featureTable, options);
+
+  return byteOffset;
 }
 
-function extractPointCloudSync(tile, options) {
+// TODO - is there really a need for sync tile parsing?
+export function parsePointCloud3DTileSync(tile, arrayBuffer, byteOffset, options, context) {
+  byteOffset = parse3DTileHeaderSync(tile, arrayBuffer, byteOffset, options);
+  byteOffset = parse3DTileTablesHeaderSync(tile, arrayBuffer, byteOffset, options);
+  byteOffset = parse3DTileTablesSync(tile, arrayBuffer, byteOffset, options);
+
   initializeTile(tile);
 
   const {featureTable} = parsePointCloudTables(tile);
 
+  // parseDracoSync(tile, featureTable, batchTable, options);
+
   parsePositions(tile, featureTable, options);
   parseColors(tile, featureTable, options);
   parseNormals(tile, featureTable, options);
+
+  return byteOffset;
 }
 
 function initializeTile(tile) {
@@ -180,7 +172,7 @@ function parseBatchIds(tile, featureTable) {
 }
 
 // eslint-disable-next-line complexity
-async function parseDraco(tile, featureTable, batchTable, options) {
+async function parseDraco(tile, featureTable, batchTable, options, context) {
   let dracoBuffer;
   let dracoFeatureTableProperties;
   let dracoBatchTableProperties;
@@ -217,13 +209,15 @@ async function parseDraco(tile, featureTable, batchTable, options) {
       batchTableProperties: dracoBatchTableProperties,
       dequantizeInShader: false
     };
-    await loadDraco(tile, dracoData, options);
+
+    await loadDraco(tile, dracoData, options, context);
   }
 }
 
-/* eslint-disable complexity, max-statements */
-export async function loadDraco(tile, dracoData, options) {
-  const data = await parse(dracoData.buffer);
+// eslint-disable-next-line complexity, max-statements
+export async function loadDraco(tile, dracoData, options, context) {
+  const {parse} = context;
+  const data = await parse(dracoData.buffer, options.DracoLoader);
   const decodedPositions = data.attributes.POSITION && data.attributes.POSITION.value;
   const decodedColors = data.attributes.COLOR_0 && data.attributes.COLOR_0.value;
   const decodedNormals = data.attributes.NORMAL && data.attributes.NORMAL.value;

modules/3d-tiles/src/parsers/parse-3d-tile.js

@@ -13,46 +13,60 @@ import {
 import {parseComposite3DTile, parseComposite3DTileSync} from './parse-3d-tile-composite';
 
 // Extracts
-export async function parse3DTile(arrayBuffer, byteOffset = 0, options = {}, tile = {}) {
+export async function parse3DTile(arrayBuffer, byteOffset = 0, options, context, tile = {}) {
   tile.byteOffset = byteOffset;
   tile.type = getMagicString(arrayBuffer, byteOffset);
 
   switch (tile.type) {
     case TILE3D_TYPE.COMPOSITE:
       // Note: We pass this function as argument so that embedded tiles can be parsed recursively
-      return await parseComposite3DTile(tile, arrayBuffer, byteOffset, options, parse3DTile);
+      return await parseComposite3DTile(
+        tile,
+        arrayBuffer,
+        byteOffset,
+        options,
+        context,
+        parse3DTile
+      );
 
     case TILE3D_TYPE.BATCHED_3D_MODEL:
-      return await parseBatchedModel3DTile(tile, arrayBuffer, byteOffset, options);
+      return await parseBatchedModel3DTile(tile, arrayBuffer, byteOffset, options, context);
 
     case TILE3D_TYPE.INSTANCED_3D_MODEL:
-      return await parseInstancedModel3DTile(tile, arrayBuffer, byteOffset, options);
+      return await parseInstancedModel3DTile(tile, arrayBuffer, byteOffset, options, context);
 
     case TILE3D_TYPE.POINT_CLOUD:
-      return await parsePointCloud3DTile(tile, arrayBuffer, byteOffset, options);
+      return await parsePointCloud3DTile(tile, arrayBuffer, byteOffset, options, context);
 
     default:
       throw new Error(`3DTileLoader: unknown type ${tile.type}`); // eslint-disable-line
   }
 }
 
-export function parse3DTileSync(arrayBuffer, byteOffset = 0, options = {}, tile = {}) {
+export function parse3DTileSync(arrayBuffer, byteOffset = 0, options, context, tile = {}) {
   tile.byteOffset = byteOffset;
   tile.type = getMagicString(arrayBuffer, byteOffset);
 
   switch (tile.type) {
     case TILE3D_TYPE.COMPOSITE:
       // Note: We pass this function as argument so that embedded tiles can be parsed recursively
-      return parseComposite3DTileSync(tile, arrayBuffer, byteOffset, options, parse3DTileSync);
+      return parseComposite3DTileSync(
+        tile,
+        arrayBuffer,
+        byteOffset,
+        options,
+        context,
+        parse3DTileSync
+      );
 
     case TILE3D_TYPE.BATCHED_3D_MODEL:
-      return parseBatchedModel3DTileSync(tile, arrayBuffer, byteOffset, options);
+      return parseBatchedModel3DTileSync(tile, arrayBuffer, byteOffset, options, context);
 
     case TILE3D_TYPE.INSTANCED_3D_MODEL:
-      return parseInstancedModel3DTileSync(tile, arrayBuffer, byteOffset, options);
+      return parseInstancedModel3DTileSync(tile, arrayBuffer, byteOffset, options, context);
 
     case TILE3D_TYPE.POINT_CLOUD:
-      return parsePointCloud3DTileSync(tile, arrayBuffer, byteOffset, options);
+      return parsePointCloud3DTileSync(tile, arrayBuffer, byteOffset, options, context);
 
     default:
       throw new Error(`3DTileLoader: unknown type ${tile.type}`); // eslint-disable-line

modules/3d-tiles/src/tile-3d-loader.js

@@ -1,16 +1,16 @@
 import {parse3DTile, parse3DTileSync} from './parsers/parse-3d-tile';
 
-async function parse(arrayBuffer, options, url, loader) {
+async function parse(arrayBuffer, options, context, loader) {
   const tile = {};
   const byteOffset = 0;
-  await parse3DTile(arrayBuffer, byteOffset, options, tile);
+  await parse3DTile(arrayBuffer, byteOffset, options, context, tile);
   return tile;
 }
 
-function parseSync(arrayBuffer, options, url, loader) {
+function parseSync(arrayBuffer, options, context, loader) {
   const tile = {};
   const byteOffset = 0;
-  parse3DTileSync(arrayBuffer, byteOffset, options, tile);
+  parse3DTileSync(arrayBuffer, byteOffset, options, context, tile);
   return tile;
 }
 

modules/3d-tiles/src/tileset/helpers/transform-utils.js

@@ -1,4 +1,4 @@
-import {assert} from '@loaders.gl/core';
+import {assert} from '@loaders.gl/loader-utils';
 import {Ellipsoid} from '@math.gl/geospatial';
 import {Matrix4, Vector3} from 'math.gl';
 

modules/core/src/index.js

@@ -1,4 +1,5 @@
-import {parse, parseSync} from './lib/parse';
+import {parse} from './lib/parse';
+import {parseSync} from './lib/parse-sync';
 import {fetchFile} from './lib/fetch/fetch-file';
 import {load} from './lib/load';
 import {resolvePath} from './lib/fetch/file-aliases';
@@ -23,7 +24,10 @@ export {
 export {registerLoaders} from './lib/register-loaders';
 
 // LOADING (READING + PARSING)
-export {parse, parseSync, parseInBatches, parseInBatchesSync} from './lib/parse';
+export {parse} from './lib/parse';
+export {parseSync} from './lib/parse-sync';
+export {parseInBatches} from './lib/parse-in-batches';
+export {parseInBatchesSync} from './lib/parse-in-batches-sync';
 export {load, loadInBatches} from './lib/load';
 
 // ENCODING AND SAVING