Merge 44defce into a4ab6a8

modules/core/src/lib/encode.js

@@ -1,9 +1,9 @@
-export function encode(data, writer, options, url) {
+export async function encode(data, writer, options, url) {
   if (writer.encode) {
-    return writer.encode(data, options);
+    return await writer.encode(data, options);
   }
   if (writer.encodeSync) {
-    return Promise.resolve(writer.encodeSync(data, options));
+    return writer.encodeSync(data, options);
   }
   // TODO - Use encodeToBatches?
   throw new Error('Writer could not encode data');

modules/core/src/lib/loader-utils/get-data.js

@@ -59,8 +59,9 @@ export async function getArrayBufferOrStringFromData(data, loader) {
   }
 
   if (isFetchResponse(data)) {
-    await checkFetchResponseStatus(data);
-    return loader.binary ? await data.arrayBuffer() : await data.text();
+    const response = data;
+    await checkFetchResponseStatus(response);
+    return loader.binary ? await response.arrayBuffer() : await response.text();
   }
 
   // if (isIterable(data) || isAsyncIterable(data)) {

modules/core/src/lib/loader-utils/parse-with-worker.js

@@ -4,6 +4,8 @@ import WorkerFarm from '../../worker-utils/worker-farm';
 import {getTransferList} from '../../worker-utils/get-transfer-list';
 import {parse} from '../parse';
 
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
 export function canParseWithWorker(loader, data, options, context) {
   if (!WorkerFarm.isSupported()) {
     return false;
@@ -37,10 +39,12 @@ export default function parseWithWorker(loader, data, options, context) {
   // TODO - decide how to handle logging on workers
   options = JSON.parse(JSON.stringify(options));
 
-  return workerFarm.process(workerSource, `loaders.gl-${workerName}`, {
+  const warning = loader.version !== VERSION ? `(core version ${VERSION})` : '';
+
+  return workerFarm.process(workerSource, `loaders.gl@${loader.version}${warning}:${workerName}`, {
     arraybuffer: toArrayBuffer(data),
     options,
-    source: `loaders.gl@${__VERSION__}`, // Lets worker ignore unrelated messages
+    source: `loaders.gl@${VERSION}`, // Lets worker ignore unrelated messages
     type: 'parse' // For future extension
   });
 }

modules/core/src/worker-utils/worker-pool.js

@@ -85,7 +85,7 @@ export default class WorkerPool {
     // Create fresh worker if we haven't yet created the max amount of worker threads for this worker source
     if (this.count < this.maxConcurrency) {
       this.count++;
-      const name = `${this.name.toLowerCase()}-worker-${this.count}-of-${this.maxConcurrency}`;
+      const name = `${this.name.toLowerCase()}-worker (#${this.count} of ${this.maxConcurrency})`;
       return new WorkerThread({source: this.source, onMessage: this.onMessage, name});
     }
 

modules/core/test/core.bench.js

@@ -1,26 +1,29 @@
-const LENGTH = 10000;
+const LENGTH = 1e3;
 const ARRAY = new Array(LENGTH).fill(0);
 
 // TODO - Async benchmarks pending probe.gl update
-// async function *asyncIterator(array) {
-//   for (const element of array) {
-//     yield Promise.resolve(element);
-//   }
-// }
+async function* asyncIterator(array) {
+  for (const element of array) {
+    yield Promise.resolve(element);
+  }
+}
 
 export default function coreBench(bench) {
-  return bench.group('Core Module - Async Iteration').add('Normal Iteration 10K', () => {
+  bench.group('Core Module - Async Iteration');
+
+  bench.add('Sync Iteration over 1000 elements', () => {
     let sum = 0;
     for (const element of ARRAY) {
       sum += element;
     }
     return sum;
   });
-  // .addAsync('Async Iteration 10K', async () => {
-  //   let sum = 0;
-  //   for await (const element of asyncIterator(ARRAY)) {
-  //     sum += element;
-  //   }
-  //   return sum;
-  // });
+
+  bench.addAsync('Async Iteration over 1000 elements', async () => {
+    let sum = 0;
+    for await (const element of asyncIterator(ARRAY)) {
+      sum += element;
+    }
+    return sum;
+  });
 }

modules/csv/package.json

@@ -32,5 +32,8 @@
   "dependencies": {
     "@loaders.gl/core": "2.0.0-beta.8",
     "@loaders.gl/tables": "2.0.0-beta.8"
+  },
+  "devDependencies": {
+    "d3-dsv": "^1.2.0"
   }
 }

modules/csv/test/csv.bench.js

@@ -3,25 +3,29 @@ import {CSVLoader} from '@loaders.gl/csv';
 
 const SAMPLE_CSV_URL = '@loaders.gl/csv/test/data/sample-very-long.csv';
 
-/*
 // Comparison loader based on D3
 import {csvParseRows} from 'd3-dsv';
 
 const D3CSVLoader = {
-  name: 'CSV',
+  name: 'D3 CSV',
   extensions: ['csv'],
   testText: null,
-  parseTextSync: csvParseRows
+  text: true,
+  parseTextSync: text => csvParseRows(text)
 };
-*/
 
 export default async function csvBench(bench) {
-  const sample = await fetchFile(SAMPLE_CSV_URL);
+  const response = await fetchFile(SAMPLE_CSV_URL);
+  const sample = await response.text();
 
   bench = bench.group('CSV Decode');
 
-  bench = bench.addAsync('CSVLoader#decode', async () => {
-    parse(sample, CSVLoader);
+  bench = bench.addAsync('CSVLoader#parse', async () => {
+    return await parse(sample, CSVLoader);
+  });
+
+  bench = bench.addAsync('d3-dsv#parse', async () => {
+    return await parse(sample, D3CSVLoader);
   });
 
   return bench;

modules/draco/src/draco-writer.js

@@ -6,22 +6,28 @@ export default {
   extensions: ['drc'],
   encode,
   options: {
-    pointcloud: false // Set to true if pointcloud (mode: 0, no indices)
-    // Draco Compression Parameters
-    // method: 'MESH_EDGEBREAKER_ENCODING',
-    // speed: [5, 5],
-    // quantization: {
-    //   POSITION: 10
-    // }
+    draco: {
+      pointcloud: false // Set to true if pointcloud (mode: 0, no indices)
+      // Draco Compression Parameters
+      // method: 'MESH_EDGEBREAKER_ENCODING',
+      // speed: [5, 5],
+      // quantization: {
+      //   POSITION: 10
+      // }
+    }
   }
 };
 
 async function encode(data, options) {
+  // DEPRECATED - remove support for options
+  const dracoOptions = (options && options.draco) || options || {};
+
   // Dynamically load draco
-  const {draco} = await loadDracoEncoderModule(options);
+  const {draco} = await loadDracoEncoderModule(options || {});
   const dracoBuilder = new DRACOBuilder(draco);
+
   try {
-    return dracoBuilder.encodeSync(data, options);
+    return dracoBuilder.encodeSync(data, dracoOptions);
   } finally {
     dracoBuilder.destroy();
   }

modules/draco/test/draco-compression-ratio.spec.js

@@ -23,7 +23,7 @@ test('DracoWriter#compressRawBuffers', async t => {
 
   // Encode mesh
   // TODO - Replace with draco writer
-  const compressedMesh = await encode({attributes}, DracoWriter, {pointcloud: true});
+  const compressedMesh = await encode({attributes}, DracoWriter, {draco: {pointcloud: true}});
   const meshSize = _getMeshSize(attributes);
   const ratio = meshSize / compressedMesh.byteLength;
   t.comment(`Draco compression ${compressedMesh.byteLength} bytes, ratio ${ratio.toFixed(1)}`);

modules/draco/test/draco-writer.spec.js

@@ -9,19 +9,23 @@ const TEST_CASES = [
   {
     title: 'Encoding Draco Mesh: SEQUENTIAL',
     options: {
-      method: 'MESH_SEQUENTIAL_ENCODING'
+      draco: {
+        method: 'MESH_SEQUENTIAL_ENCODING'
+      }
     }
   },
   {
     title: 'Encoding Draco Mesh: EDGEBREAKER',
     options: {
-      method: 'MESH_EDGEBREAKER_ENCODING'
+      draco: {
+        method: 'MESH_EDGEBREAKER_ENCODING'
+      }
     }
   },
   {
     title: 'Encoding Draco PointCloud (no indices)',
     options: {
-      pointcloud: true
+      draco: {pointcloud: true}
     }
   }
 ];
@@ -57,7 +61,7 @@ test('DracoWriter#encode(bunny.drc)', async t => {
   };
 
   for (const tc of TEST_CASES) {
-    const mesh = tc.options.pointcloud ? POINTCLOUD : MESH;
+    const mesh = tc.options.draco.pointcloud ? POINTCLOUD : MESH;
     const meshSize = _getMeshSize(mesh.attributes);
 
     const compressedMesh = await encode(mesh, DracoWriter, tc.options);
@@ -95,7 +99,7 @@ test('DracoParser#encode(bunny.drc)', async t => {
   };
 
   for (const tc of TEST_CASES) {
-    const attributes = tc.options.pointcloud ? pointCloudAttributes : meshAttributes;
+    const attributes = tc.options.draco.pointcloud ? pointCloudAttributes : meshAttributes;
     const meshSize = _getMeshSize(attributes);
 
     const compressedMesh = await encode(attributes, DracoWriter, tc.options);

modules/draco/test/draco.bench.js

@@ -1,6 +1,5 @@
-import {fetchFile, parseSync, encode} from '@loaders.gl/core';
-import {_getMeshSize} from '@loaders.gl/loader-utils';
-import {DracoLoader, DracoWriter} from '@loaders.gl/draco';
+import {fetchFile, encode} from '@loaders.gl/core';
+import {DracoWriter} from '@loaders.gl/draco';
 
 const OPTIONS = [
   {
@@ -29,23 +28,24 @@ export default async function dracoBench(bench) {
     POSITIONS: new Float32Array(POSITIONS),
     COLORS: new Uint8ClampedArray(COLORS)
   };
-  const rawSize = _getMeshSize(attributes);
 
-  OPTIONS.forEach(options => {
-    const dracoEncoder = encode({attributes}, DracoWriter, options);
-    const compressedPointCloud = dracoEncoder.encodePointCloud(attributes);
-    // eslint-disable-next-line
-    console.log(`${options.name} compression rate:
-      ${((compressedPointCloud.byteLength / rawSize) * 100).toFixed(2)}%`);
+  for (const options of OPTIONS) {
+    bench.addAsync(
+      `DracoEncoder#pointcloud ${POSITIONS.byteLength / 12}#${options.name}`,
+      async () => {
+        return await encode(attributes, DracoWriter, {draco: {pointcloud: true}});
+      }
+    );
 
-    bench = bench
-      .add(`DracoEncoder#encode point cloud#${options.name}`, () => {
-        dracoEncoder.encodePointCloud(attributes);
-      })
-      .add(`DracoDecoder#decode point cloud#${options.name}`, () => {
-        parseSync(compressedPointCloud, DracoLoader);
-      });
-  });
+    // TODO - COMMENT OUT until bench.addAsync is fixed (too many invocations)
+    // const compressedPointCloud = await encode(attributes, DracoWriter, {draco: {pointcloud: true}});
+    // bench.addAsync(
+    //   `DracoDecoder#pointcloud ${POSITIONS.byteLength / 12}#${options.name}`,
+    //   async () => {
+    //     return await parse(compressedPointCloud, DracoLoader, {worker: false});
+    //   }
+    // );
+  }
 
   return bench;
 }

modules/gltf/src/lib/deprecated/gltf-builder.js

@@ -171,7 +171,7 @@ export default class GLTFBuilder extends GLBBuilder {
     }
 
     attributes.mode = 0;
-    const compressedData = this.DracoWriter.encodeSync(attributes, {pointcloud: true});
+    const compressedData = this.DracoWriter.encodeSync(attributes, {draco: {pointcloud: true}});
 
     const bufferViewIndex = this.addBufferView(compressedData);
 

modules/gltf/src/lib/extensions/UBER_draco_point_cloud_compression.js

@@ -8,7 +8,7 @@ function addCompressedPointCloud(attributes, options) {
   }
 
   attributes.mode = 0;
-  const compressedData = options.DracoWriter.encodeSync(attributes, {pointcloud: true});
+  const compressedData = options.DracoWriter.encodeSync(attributes, {draco: {pointcloud: true}});
 
   const bufferViewIndex = this.addBufferView(compressedData);
 

modules/json/test/clarinet/clarinet.bench.js

@@ -7,13 +7,13 @@ export default function clarinetBench(bench) {
 
   const parser = new ClarinetParser();
 
-  bench.group('Clarinet - JSON parsing');
+  bench.group('Clarinet - JSON parsing from string');
 
-  bench.add('ClarinetParser(basic.json)', () => {
-    parser.write(STRING);
+  bench.add('JSON.parse (used by parseSync)', () => {
+    JSON.parse(STRING);
   });
 
-  bench.add('JSON.parse(basic.json)', () => {
-    JSON.parse(STRING);
+  bench.add('ClarinetParser (used by parseInBatches)', () => {
+    parser.write(STRING);
   });
 }

modules/json/test/json-loader.bench.js

@@ -0,0 +1,26 @@
+import {JSONLoader} from '@loaders.gl/json';
+import {load, loadInBatches} from '@loaders.gl/core';
+
+import clarinetBench from './clarinet/clarinet.bench';
+
+const GEOJSON_URL = '@loaders.gl/json/test/data/geojson-big.json';
+
+export default async function jsonLoaderBench(suite) {
+  // Test underlying clarinet library
+  await clarinetBench(suite);
+
+  suite.group('JSONLoader - loading from file');
+
+  suite.addAsync('load(JSONLoader) - Uses JSON.parse', async () => {
+    await load(GEOJSON_URL, JSONLoader);
+  });
+
+  suite.addAsync('loadInBatches(JSONLoader) - Uses Clarinet', async () => {
+    const asyncIterator = await loadInBatches(GEOJSON_URL, JSONLoader);
+    // const asyncIterator = await parseInBatches(STRING, JSONLoader);
+    const data = [];
+    for await (const batch of asyncIterator) {
+      data.push(...batch.data);
+    }
+  });
+}

modules/json/test/json-loader.spec.js

@@ -60,6 +60,7 @@ test('JSONLoader#loadInBatches(geojson.json, rows, batchSize = 10)', async t =>
   t.end();
 });
 
+// TODO - columnar table batch support not yet fixed
 /*
 test('JSONLoader#loadInBatches(geojson.json, columns, batchSize = auto)', async t => {
   const iterator = await loadInBatches(GEOJSON_PATH, JSONLoader, {
@@ -72,7 +73,6 @@ test('JSONLoader#loadInBatches(geojson.json, columns, batchSize = auto)', async
   let batch;
   let batchCount = 0;
   let rowCount = 0;
-
   for await (batch of iterator) {
     batchCount++;
     rowCount += batch.length;

package.json

@@ -32,18 +32,14 @@
     "publish": "ocular-publish",
     "prepare": "npm run build",
     "test": "ocular-test",
-    "test-fast": "ocular-test fast",
-    "test-browser": "ocular-test browser",
-    "bench": "ocular-test bench",
-    "bench-browser": "ocular-test bench-browser",
     "metrics": "./scripts/metrics.sh && ocular-metrics"
   },
   "devDependencies": {
     "@babel/register": "^7.4.4",
     "@luma.gl/core": "^8.0.0",
     "@luma.gl/debug": "^8.0.0",
     "@luma.gl/test-utils": "^8.0.0",
-    "@probe.gl/bench": "^3.1.1",
+    "@probe.gl/bench": "^3.2.0",
     "@probe.gl/test-utils": "^3.1.1",
     "arraybuffer-loader": "^1.0.7",
     "babel-loader": "^8.0.6",