Combined loaders (#486)

visgl · Oct 9, 2019 · 956d475 · 956d475
1 parent c1b5a14
commit 956d475
Show file tree

Hide file tree

Showing 17 changed files with 122 additions and 137 deletions.
diff --git a/modules/arrow/src/arrow-loader.js b/modules/arrow/src/arrow-loader.js
@@ -4,29 +4,24 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 import parseSync from './lib/parse-arrow-sync';
 import {parseArrowInBatches, parseArrowInBatchesSync} from './lib/parse-arrow-in-batches';
 
-const ARROW = {
+export const ArrowWorkerLoader = {
   id: 'arrow',
   name: 'Apache Arrow',
   version: VERSION,
   extensions: ['arrow'],
   mimeType: 'application/octet-stream',
-  category: 'table'
+  category: 'table',
+  options: {
+    arrow: {
+      workerUrl: `https://unpkg.com/@loaders.gl/arrow@${VERSION}/dist/arrow-loader.worker.js`
+    }
+  }
 };
 
 export const ArrowLoader = {
-  ...ARROW,
+  ...ArrowWorkerLoader,
   parse: async (arraybuffer, options) => parseSync(arraybuffer, options),
   parseSync,
   parseInBatches: parseArrowInBatches,
-  parseInBatchesSync: parseArrowInBatchesSync,
-  options: {}
-};
-
-export const ArrowWorkerLoader = {
-  ...ARROW,
-  options: {
-    arrow: {
-      workerUrl: `https://unpkg.com/@loaders.gl/arrow@${VERSION}/dist/arrow-loader.worker.js`
-    }
-  }
+  parseInBatchesSync: parseArrowInBatchesSync
 };
diff --git a/modules/arrow/test/arrow-loader.spec.js b/modules/arrow/test/arrow-loader.spec.js
@@ -1,16 +1,9 @@
 import test from 'tape-promise/tape';
 import {validateLoader} from 'test/common/conformance';
 
-import {ArrowLoader} from '@loaders.gl/arrow';
-import {ArrowWorkerLoader} from '@loaders.gl/arrow';
-import {
-  isBrowser,
-  resolvePath,
-  fetchFile,
-  getStreamIterator,
-  parse,
-  parseInBatches
-} from '@loaders.gl/core';
+import {ArrowLoader, ArrowWorkerLoader} from '@loaders.gl/arrow';
+import {isBrowser, getStreamIterator, resolvePath} from '@loaders.gl/core';
+import {setLoaderOptions, fetchFile, parse, parseInBatches} from '@loaders.gl/core';
 
 // Small Arrow Sample Files
 const ARROW_SIMPLE = '@loaders.gl/arrow/test/data/simple.arrow';
@@ -20,13 +13,19 @@ const ARROW_STRUCT = '@loaders.gl/arrow/test/data/struct.arrow';
 // Bigger, batched sample file
 const ARROW_BIOGRID_NODES = '@loaders.gl/arrow/test/data/biogrid-nodes.arrow';
 
+setLoaderOptions({
+  arrow: {
+    workerUrl: 'modules/arrow/dist/arrow-loader.worker.js'
+  }
+});
+
 test('ArrowLoader#loader conformance', t => {
   validateLoader(t, ArrowLoader, 'ArrowLoader');
   t.end();
 });
 
 test('ArrowLoader#parseSync(simple.arrow)', async t => {
-  const columns = await parse(fetchFile(ARROW_SIMPLE), ArrowLoader);
+  const columns = await parse(fetchFile(ARROW_SIMPLE), ArrowLoader, {worker: false});
   // Check loader specific results
   t.ok(columns.bar, 'bar column loaded');
   t.ok(columns.baz, 'baz column loaded');
@@ -56,11 +55,7 @@ test('ArrowLoader#parse (WORKER)', async t => {
     return;
   }
 
-  const data = await parse(fetchFile(ARROW_SIMPLE), ArrowWorkerLoader, {
-    arrow: {
-      workerUrl: 'modules/arrow/dist/arrow-loader.worker.js'
-    }
-  });
+  const data = await parse(fetchFile(ARROW_SIMPLE), ArrowWorkerLoader);
   t.ok(data, 'Data returned');
   t.end();
 });

diff --git a/modules/core/src/lib/loader-utils/parse-with-worker.js b/modules/core/src/lib/loader-utils/parse-with-worker.js
@@ -1,9 +1,13 @@
+/* global __VERSION__ */ // __VERSION__ is injected by babel-plugin-version-inline
 import {toArrayBuffer} from '../../javascript-utils/binary-utils';
 import WorkerFarm from '../../worker-utils/worker-farm';
 import {getTransferList} from '../../worker-utils/get-transfer-list';
 import {parse} from '../parse';
 
 export function canParseWithWorker(loader, data, options, context) {
+  if (!WorkerFarm.isSupported()) {
+    return false;
+  }
   const loaderOptions = options && options[loader.id];
   if (options.worker && loaderOptions && loaderOptions.workerUrl) {
     return loader.useWorker ? loader.useWorker(options) : true;
@@ -31,8 +35,8 @@ export default function parseWithWorker(loader, data, options, context) {
 
   return workerFarm.process(workerSource, `loaders.gl-${workerName}`, {
     arraybuffer: toArrayBuffer(data),
-    options, // __VERSION__ is injected by babel-plugin-version-inline
-    /* global __VERSION__ */ source: `loaders.gl@${__VERSION__}`, // Lets worker ignore unrelated messages
+    options,
+    source: `loaders.gl@${__VERSION__}`, // Lets worker ignore unrelated messages
     type: 'parse' // For future extension
   });
 }

diff --git a/modules/core/src/worker-utils/worker-farm.js b/modules/core/src/worker-utils/worker-farm.js
@@ -6,6 +6,10 @@ const DEFAULT_MAX_CONCURRENCY = 5;
  * Process multiple data messages with a "farm" of different workers (in worker pools)
  */
 export default class WorkerFarm {
+  static isSupported() {
+    return typeof Worker !== 'undefined';
+  }
+
   /**
    * @param processor {function | string} - worker function
    * @param maxConcurrency {number} - max count of workers

diff --git a/modules/draco/src/draco-loader.js b/modules/draco/src/draco-loader.js
@@ -3,32 +3,27 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
 import DracoParser from './lib/draco-parser';
 
-const DRACO = {
+export const DracoWorkerLoader = {
   id: 'draco',
   name: 'DRACO',
   version: VERSION,
   extensions: ['drc'],
   mimeType: 'application/octet-stream',
   binary: true,
-  test: 'DRACO'
-};
-
-export const DracoLoader = {
-  ...DRACO,
-  parse: async (arrayBuffer, options) => parseSync(arrayBuffer, options),
-  parseSync,
-  options: {}
-};
-
-export const DracoWorkerLoader = {
-  ...DRACO,
+  test: 'DRACO',
   options: {
     draco: {
       workerUrl: `https://unpkg.com/@loaders.gl/draco@${VERSION}/dist/draco-loader.worker.js`
     }
   }
 };
 
+export const DracoLoader = {
+  ...DracoWorkerLoader,
+  parse: async (arrayBuffer, options) => parseSync(arrayBuffer, options),
+  parseSync
+};
+
 function parseSync(arrayBuffer, options) {
   const dracoParser = new DracoParser();
   try {

diff --git a/modules/draco/test/draco-loader.spec.js b/modules/draco/test/draco-loader.spec.js
@@ -3,21 +3,26 @@ import test from 'tape-promise/tape';
 import {validateLoader, validatePointCloudCategoryData} from 'test/common/conformance';
 
 import {DracoLoader, DracoWorkerLoader} from '@loaders.gl/draco';
-import {load} from '@loaders.gl/core';
+import {setLoaderOptions, load} from '@loaders.gl/core';
 
 const BUNNY_DRC_URL = '@loaders.gl/draco/test/data/bunny.drc';
 
+setLoaderOptions({
+  draco: {
+    workerUrl: 'modules/draco/dist/draco-loader.worker.js'
+  }
+});
+
 test('DracoLoader#loader conformance', t => {
   validateLoader(t, DracoLoader, 'DracoLoader');
   validateLoader(t, DracoWorkerLoader, 'DracoWorkerLoader');
   t.end();
 });
 
 test('DracoLoader#parse and encode', async t => {
-  const data = await load(BUNNY_DRC_URL, DracoLoader);
+  const data = await load(BUNNY_DRC_URL, DracoLoader, {worker: false});
   validatePointCloudCategoryData(t, data);
   t.equal(data.attributes.POSITION.value.length, 104502, 'POSITION attribute was found');
-
   t.end();
 });
 
@@ -28,11 +33,7 @@ test('DracoWorkerLoader#parse', async t => {
     return;
   }
 
-  const data = await load(BUNNY_DRC_URL, DracoWorkerLoader, {
-    draco: {
-      workerUrl: 'modules/draco/dist/draco-loader.worker.js'
-    }
-  });
+  const data = await load(BUNNY_DRC_URL, DracoWorkerLoader);
   validatePointCloudCategoryData(t, data);
   t.equal(data.attributes.POSITION.value.length, 104502, 'POSITION attribute was found');
 

diff --git a/modules/las/docs/api-reference/las-loader.md b/modules/las/docs/api-reference/las-loader.md
@@ -26,7 +26,7 @@ const data = await load(url, LASWorkerLoader, options);
 
 ## Options
 
-| Option       | Type     | Default | Description                                                               |
-| ------------ | -------- | ------- | ------------------------------------------------------------------------- |
-| `skip`       | Number   | `1`     | Read one from every _n_ points.                                           |
-| `onProgress` | Function | -       | Callback when a new chunk of data is read. Only works on the main thread. |
+| Option               | Type     | Default | Description                                                               |
+| -------------------- | -------- | ------- | ------------------------------------------------------------------------- |
+| `options.las.skip`   | Number   | `1`     | Read one from every _n_ points.                                           |
+| `options.onProgress` | Function | -       | Callback when a new chunk of data is read. Only works on the main thread. |
diff --git a/modules/las/src/las-loader.js b/modules/las/src/las-loader.js
@@ -5,29 +5,24 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
 import parseLAS from './lib/parse-las';
 
-const LAS = {
+export const LASWorkerLoader = {
   id: 'las',
   name: 'LAS',
   version: VERSION,
   extensions: ['las', 'laz'], // LAZ is the "compressed" flavor of LAS,
   mimeType: 'application/octet-stream', // TODO - text version?
   text: true,
   binary: true,
-  test: 'LAS'
-};
-
-export const LASLoader = {
-  ...LAS,
-  parse: async (arrayBuffer, options) => parseLAS(arrayBuffer, options),
-  parseSync: parseLAS,
-  options: {}
-};
-
-export const LASWorkerLoader = {
-  ...LAS,
+  test: 'LAS',
   options: {
     las: {
       workerUrl: `https://unpkg.com/@loaders.gl/las@${VERSION}/dist/las-loader.worker.js`
     }
   }
 };
+
+export const LASLoader = {
+  ...LASWorkerLoader,
+  parse: async (arrayBuffer, options) => parseLAS(arrayBuffer, options),
+  parseSync: parseLAS
+};
diff --git a/modules/las/src/lib/parse-las.js b/modules/las/src/lib/parse-las.js
@@ -12,7 +12,8 @@ export default function parseLAS(arraybuffer, options = {}) {
   let originalHeader;
 
   const result = {};
-  const {skip = 1, onProgress} = options;
+  const {onProgress} = options;
+  const {skip = 1} = options.las || {};
 
   parseLASChunked(arraybuffer, skip, (decoder, header) => {
     if (!originalHeader) {

diff --git a/modules/las/test/las-loader.spec.js b/modules/las/test/las-loader.spec.js
@@ -2,19 +2,25 @@
 import test from 'tape-promise/tape';
 import {validateLoader, validatePointCloudCategoryData} from 'test/common/conformance';
 
-import {fetchFile, parse, load} from '@loaders.gl/core';
 import {LASLoader, LASWorkerLoader} from '@loaders.gl/las';
+import {setLoaderOptions, fetchFile, parse, load} from '@loaders.gl/core';
 
 const LAS_BINARY_URL = '@loaders.gl/las/test/data/indoor.laz';
 
+setLoaderOptions({
+  las: {
+    workerUrl: 'modules/las/dist/las-loader.worker.js'
+  }
+});
+
 test('LASLoader#loader conformance', t => {
   validateLoader(t, LASLoader, 'LASLoader');
   validateLoader(t, LASWorkerLoader, 'LASWorkerLoader');
   t.end();
 });
 
 test('LASLoader#parse(binary)', async t => {
-  const data = await parse(fetchFile(LAS_BINARY_URL), LASLoader, {skip: 10});
+  const data = await parse(fetchFile(LAS_BINARY_URL), LASLoader, {las: {skip: 10}, worker: false});
   validatePointCloudCategoryData(t, data);
 
   t.is(data.header.vertexCount, data.loaderData.header.totalRead, 'Original header was found');
@@ -26,7 +32,7 @@ test('LASLoader#parse(binary)', async t => {
   t.end();
 });
 
-test('LASWorkerLoader#load(binary)', async t => {
+test('LASWorkerLoader#load(worker)', async t => {
   if (typeof Worker === 'undefined') {
     t.comment('Worker is not usable in non-browser environments');
     t.end();
@@ -37,8 +43,7 @@ test('LASWorkerLoader#load(binary)', async t => {
     las: {
       workerUrl: 'modules/las/dist/las-loader.worker.js',
       skip: 10
-    },
-    skip: 10
+    }
   });
   validatePointCloudCategoryData(t, data);
 

diff --git a/modules/loader-utils/src/lib/create-worker.js b/modules/loader-utils/src/lib/create-worker.js
@@ -86,7 +86,7 @@ async function parseData({loader, arraybuffer, byteOffset, byteLength, options,
     throw new Error(`Could not load data with ${loader.name} loader`);
   }
 
-  return await parser(data, options, context, loader);
+  return await parser(data, {...options, worker: false}, context, loader);
 }
 
 // Filter out noise messages sent to workers

diff --git a/modules/obj/src/obj-loader.js b/modules/obj/src/obj-loader.js
@@ -4,30 +4,26 @@ import loadOBJ from './lib/load-obj';
 
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-const OBJ = {
+export const OBJWorkerLoader = {
   id: 'obj',
   name: 'OBJ',
   version: VERSION,
   extensions: ['obj'],
   mimeType: 'text/plain',
-  testText: testOBJFile
-};
-
-export const OBJLoader = {
-  ...OBJ,
-  parse: async (arrayBuffer, options) => loadOBJ(new TextDecoder().decode(arrayBuffer), options),
-  parseTextSync: loadOBJ
-};
-
-export const OBJWorkerLoader = {
-  ...OBJ,
+  testText: testOBJFile,
   options: {
     obj: {
       workerUrl: `https://unpkg.com/@loaders.gl/obj@${VERSION}/dist/obj-loader.worker.js`
     }
   }
 };
 
+export const OBJLoader = {
+  ...OBJWorkerLoader,
+  parse: async (arrayBuffer, options) => loadOBJ(new TextDecoder().decode(arrayBuffer), options),
+  parseTextSync: loadOBJ
+};
+
 function testOBJFile(text) {
   // TODO - There could be comment line first
   return text[0] === 'v';