loader-utils: dynamic library loading (#482)

.nycrc

@@ -3,7 +3,7 @@
   "exclude": [
     "**/bundle.js",
     "**/*.worker.js",
-    "**/*.transpiled.js",
+    "**/src/libs/**",
     "**/text-encoding.js",
     "**/encoding.js",
     "**/laz-perf.js",

docs/developer-guide/dependencies.md

@@ -0,0 +1,41 @@
+# Managing Dependencies
+
+> This section is work in progress, not all options are implemented/finalized
+
+Parsers and encoders for some formats are quite complex and can be quite big in terms of code size.
+
+### Loading Dependencies from Alternate CDN
+
+By default, loaders.gl loads pre-built workers and a number of bigger external libraries from the [https://unpkg.com/](https://unpkg.com/) CDN.
+
+It is possible to specify other CDNs using `options.cdn`.
+
+Keep in mind that it is typically not sufficient to point to a server that just serves the data of the files in question. Browsers do a number of security checks on cross-origin content and requires certain response headers to be properly set, and unfortunately, error messages are not always helpful.
+
+To determine your candidate CDN service is doing what is needed, check with `curl -u <url>` and look for headers like:
+
+```
+content-type: application/javascript; charset=utf-8
+access-control-allow-origin: *
+```
+
+### Loading Dependencies from Your Own Server
+
+By setting `options.cdn: false` and doing some extra setup, you can load dependencies from your own server. This removes the impact of a potentially flaky CDN.
+
+Options:
+
+- Load from `node_modules/@loaders.gl/<module>/dist/libs/...`
+- Load from a modules directory `libs/...`
+- Load from unique locations - `options.modules[<dependency name>]` can be set to url strings.
+
+### Bundling Dependencies
+
+It is also possible to include dependencies in your application bundle
+
+- PRO: Doesn't require copying/configuring/serving supporting modules.
+- CON: Increases the size of your application bundle
+
+`options.modules` will let your application `import` or `require` dependencies (thus bundling them) and supply them to loaders.gl.
+
+See each loader module for information on its dependencies.

docs/table-of-contents.json

@@ -21,6 +21,7 @@
             {"entry": "docs/developer-guide/loader-categories"},
             {"entry": "docs/developer-guide/error-handling"},
             {"entry": "docs/developer-guide/polyfills"},
+            {"entry": "docs/developer-guide/dependencies"},
             {"entry": "docs/developer-guide/creating-loaders-and-writers"},
             {"entry": "docs/roadmap"},
             {"entry": "docs/contributing"}

modules/3d-tiles/src/lib/classes/helpers/tile-3d-accessor-utils.js

@@ -1,5 +1,5 @@
 import {GLType} from '@loaders.gl/math'; // '@math.gl/geometry';
-import assert from '../../utils/assert';
+import {assert} from '@loaders.gl/loader-utils';
 
 const COMPONENTS_PER_ATTRIBUTE = {
   SCALAR: 1,

modules/3d-tiles/src/tile-3d-loader.js

@@ -1,4 +1,5 @@
 /* global __VERSION__ */ // __VERSION__ is injected by babel-plugin-version-inline
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 import {parse3DTile} from './lib/parsers/parse-3d-tile';
 
 async function parse(arrayBuffer, options, context, loader) {
@@ -12,7 +13,7 @@ async function parse(arrayBuffer, options, context, loader) {
 export default {
   id: '3d-tiles',
   name: '3D Tiles',
-  version: __VERSION__,
+  version: VERSION,
   extensions: ['cmpt', 'pnts', 'b3dm', 'i3dm'],
   mimeType: 'application/octet-stream',
   test: ['cmpt', 'pnts', 'b3dm', 'i3dm'],

modules/arrow/src/arrow-loader.js

@@ -7,7 +7,7 @@ import {parseArrowInBatches, parseArrowInBatchesSync} from './lib/parse-arrow-in
 const ARROW = {
   id: 'arrow',
   name: 'Apache Arrow',
-  version: __VERSION__,
+  version: VERSION,
   extensions: ['arrow'],
   mimeType: 'application/octet-stream',
   category: 'table'

modules/basis/src/basis-loader.js

@@ -1,10 +1,13 @@
 /* global __VERSION__ */ // __VERSION__ is injected by babel-plugin-version-inline
+
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
 const EXTENSIONS = ['basis'];
 
 export const BasisLoader = {
   id: 'basis',
   name: 'Basis',
-  version: __VERSION__,
+  version: VERSION,
   extensions: EXTENSIONS,
   parse: data => 'not implemented yet'
 };

modules/core/docs/api-reference/parse.md

@@ -79,9 +79,3 @@ Top-level options
 | ---------------- | ------- | --------- | ------------------------------------------------------------------------------------------------------------------------ |
 | `options.log`    | object  | `console` | By default set to a `console` wrapper. Setting log to `null` will turn off logging.                                      |
 | `options.worker` | boolean | `true`    | If the selected loader is equipped with a worker url (and the runtime environment supports it) parse on a worker thread. |
-
-Per-loader options
-
-| Option                            | Type   | Default | Description                                                                         |
-| --------------------------------- | ------ | ------- | ----------------------------------------------------------------------------------- |
-| `options.<_loader id_>.workerUrl` | string | -       | Enables overriding the loader's workerUrl (e.g. for testing or debugging purposes). |

modules/core/docs/api-reference/set-loader-options.md

@@ -0,0 +1,32 @@
+# setLoaderOptions
+
+## Usage
+
+Bundling the entire `draco3d` library:
+
+```js
+import draco from 'draco3d';
+import {setLoaderOptions} from '@loaders.gl/core';
+setLoaderOptions({
+  modules: {
+    draco3d
+  }
+});
+```
+
+## Functions
+
+### setLoaderOptions(options : Object) : void
+
+Merge the options with the global options
+
+## Options
+
+Top-level options
+
+| Option            | Type    | Default   | Description                                                                                                              |
+| ----------------- | ------- | --------- | ------------------------------------------------------------------------------------------------------------------------ |
+| `options.log`     | object  | `console` | By default set to a `console` wrapper. Setting log to `null` will turn off logging.                                      |
+| `options.worker`  | boolean | `true`    | If the selected loader is equipped with a worker url (and the runtime environment supports it) parse on a worker thread. |
+| `options.cdn`     | boolean | string    | `true`                                                                                                                   | `true` loads from `unpkg.com/@loaders.gl`. `false` load from local urls. `string` alternate CDN url. |
+| `options.modules` | Object  | -         | Supply bundles modules or override local urls.                                                                           |

modules/core/package.json

@@ -27,16 +27,16 @@
     "./src/node/write-file.node.js": false,
     "./src/node/read-file-sync.node.js": false,
     "./dist/es5/node/utils/to-array-buffer.node.js": false,
-    "./dist/es5/node/utils/to-buffer.node.js": false,
-    "./dist/es5/node/write-file.node.js": false,
-    "./dist/es5/node/read-file-sync.node.js": false,
     "./dist/esm/node/utils/to-array-buffer.node.js": false,
-    "./dist/esm/node/utils/to-buffer.node.js": false,
-    "./dist/esm/node/write-file.node.js": false,
-    "./dist/esm/node/read-file-sync.node.js": false,
     "./dist/es6/node/utils/to-array-buffer.node.js": false,
+    "./dist/es5/node/utils/to-buffer.node.js": false,
+    "./dist/esm/node/utils/to-buffer.node.js": false,
     "./dist/es6/node/utils/to-buffer.node.js": false,
+    "./dist/es5/node/write-file.node.js": false,
+    "./dist/esm/node/write-file.node.js": false,
     "./dist/es6/node/write-file.node.js": false,
+    "./dist/es5/node/read-file-sync.node.js": false,
+    "./dist/esm/node/read-file-sync.node.js": false,
     "./dist/es6/node/read-file-sync.node.js": false,
     "fs": false
   },

modules/core/src/index.js

@@ -13,7 +13,8 @@ export {
   getErrorMessageFromResponse as _getErrorMessageFromResponse
 } from './lib/fetch/fetch-error-message';
 
-// FILE PARSING AND ENCODING
+// CONFIGURATION
+export {setLoaderOptions} from './lib/set-loader-options';
 export {registerLoaders} from './lib/register-loaders';
 
 // LOADING (READING + PARSING)
@@ -25,7 +26,7 @@ export {parseInBatchesSync} from './lib/parse-in-batches-sync';
 export {load} from './lib/load';
 export {loadInBatches} from './lib/load-in-batches';
 
-// ENCODING AND SAVING
+// ENCODING (ENCODING AND WRITING)
 export {encode, encodeSync, encodeInBatches} from './lib/encode';
 export {save, saveSync} from './lib/save';
 

modules/core/src/javascript-utils/binary-utils.js

@@ -1,5 +1,5 @@
 /* global ArrayBuffer, TextEncoder */
-import assert from '../utils/assert';
+import {assert} from '@loaders.gl/loader-utils';
 import {toArrayBuffer as bufferToArrayBuffer} from '../node/utils/to-array-buffer.node';
 
 export function toArrayBuffer(data) {

modules/core/src/lib/constants.js

@@ -0,0 +1,9 @@
+import {ConsoleLog} from './loader-utils/loggers';
+
+// TODO - document these options
+export const DEFAULT_LOADER_OPTIONS = {
+  CDN: 'https://unpkg.com/@loaders.gl',
+  worker: true, // By default, use worker if provided by loader
+  log: new ConsoleLog(), // A probe.gl compatible (`log.log()()` syntax) that just logs to console
+  dataType: 'arraybuffer' // TODO - explain why this option is needed for parsing
+};

modules/core/src/lib/fetch/fetch-file.browser.js

@@ -1,5 +1,5 @@
 /* global FileReader, Headers */
-import assert from '../../utils/assert';
+import {assert} from '@loaders.gl/loader-utils';
 
 // File reader fetch "polyfill" for the browser
 class FileReadableResponse {

modules/core/src/lib/fetch/read-file.browser.js

@@ -1,5 +1,5 @@
 // TODO - this file is not tested
-import assert from '../../utils/assert';
+import {assert} from '@loaders.gl/loader-utils';
 
 const DEFAULT_OPTIONS = {
   dataType: 'arraybuffer',

modules/core/src/lib/loader-utils/merge-options.js

@@ -1,41 +1,42 @@
-import {NullLog, ConsoleLog} from './loggers';
-
-const COMMON_DEFAULT_OPTIONS = {
-  worker: true, // By default, use worker if provided by loader
-  log: new ConsoleLog() // A probe.gl compatible (`log.log()()` syntax) that just logs to console
-};
+import {DEFAULT_LOADER_OPTIONS} from '../constants';
+import {NullLog} from './loggers';
 
 const isPureObject = value =>
   value && typeof value === 'object' && value.constructor === {}.constructor;
-// Merges
+
+let globalOptions = {...DEFAULT_LOADER_OPTIONS};
+
+// Set global loader options
+export function setGlobalOptions(options) {
+  globalOptions = mergeOptions(globalOptions, options);
+}
+
+// Merges options with global opts and loader defaults, also injects baseUri
 export function mergeOptions(loader, options, url) {
   const loaderDefaultOptions =
-    loader && (loader.DEFAULT_OPTIONS || loader.defaultOptions || loader.options || {});
+    loader && (loader.DEFAULT_LOADER_OPTIONS || loader.defaultOptions || loader.options || {});
 
   const mergedOptions = {
-    ...COMMON_DEFAULT_OPTIONS,
     ...loaderDefaultOptions,
-    dataType: 'arraybuffer', // TODO - explain why this option is needed for parsing
+    ...globalOptions,
     ...options // Merges any non-nested fields, but clobbers nested fields
   };
 
-  // TODO - remove file component from baseUri
-  if (url && !('baseUri' in mergedOptions)) {
-    mergedOptions.baseUri = url;
-  }
+  addUrlOptions(mergedOptions, url);
 
   // LOGGING: options.log can be set to `null` to defeat logging
   if (mergedOptions.log === null) {
     mergedOptions.log = new NullLog();
   }
 
-  mergeNesteFields(mergedOptions, options, loaderDefaultOptions);
+  mergeNestedFields(mergedOptions, loaderDefaultOptions, globalOptions);
+  mergeNestedFields(mergedOptions, loaderDefaultOptions, options);
 
   return mergedOptions;
 }
 
 // Merge nested options objects
-function mergeNesteFields(mergedOptions, options, loaderDefaultOptions) {
+function mergeNestedFields(mergedOptions, loaderDefaultOptions, options) {
   for (const key in options) {
     const value = options[key];
     // Check for nested options
@@ -53,3 +54,14 @@ function mergeNesteFields(mergedOptions, options, loaderDefaultOptions) {
     // else: No need to merge nested opts, and the initial merge already copied over the nested options
   }
 }
+
+// Harvest information from the url
+// TODO - baseUri should be a directory, i.e. remove file component from baseUri
+// TODO - extract extension?
+// TODO - extract query parameters?
+// TODO - should these be injected on context instead of options?
+function addUrlOptions(options, url) {
+  if (url && !('baseUri' in options)) {
+    options.baseUri = url;
+  }
+}

modules/core/src/lib/loader-utils/normalize-loader.js

@@ -1,4 +1,4 @@
-import assert from '../../utils/assert';
+import {assert} from '@loaders.gl/loader-utils';
 
 export function isLoaderObject(loader) {
   if (!loader) {

modules/core/src/lib/parse.js

@@ -1,11 +1,10 @@
-import assert from '../utils/assert';
+import {assert, validateLoaderVersion} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from './loader-utils/normalize-loader';
 import {mergeOptions} from './loader-utils/merge-options';
 import {getUrlFromData} from './loader-utils/get-data';
 import {getArrayBufferOrStringFromData} from './loader-utils/get-data';
 import {getLoaders, getLoaderContext} from './loader-utils/get-loader-context';
 import parseWithWorker, {canParseWithWorker} from './loader-utils/parse-with-worker';
-import {validateLoaderVersion} from './loader-utils/validate-loader-version.js';
 import {selectLoader} from './select-loader';
 
 export async function parse(data, loaders, options, context) {

modules/core/src/lib/set-loader-options.js

@@ -0,0 +1,6 @@
+import {setGlobalOptions} from './loader-utils/merge-options';
+
+// Set global loader options
+export function setLoaderOptions(options) {
+  setGlobalOptions(options);
+}

modules/core/src/node/utils/to-buffer.node.js

@@ -1,4 +1,4 @@
-import assert from '../../utils/assert';
+import {assert} from '@loaders.gl/loader-utils';
 
 // Convert (copy) ArrayBuffer to Buffer
 export default function toBuffer(binaryData) {

modules/core/src/worker-utils/get-worker-url.js

@@ -1,5 +1,5 @@
 /* global URL, Blob */
-import assert from '../utils/assert';
+import {assert} from '@loaders.gl/loader-utils';
 
 const workerURLCache = new Map();
 

modules/core/test/index.js

@@ -4,13 +4,12 @@ import './lib/fetch';
 import './lib/loader-utils';
 import './lib/progress/fetch-progress.spec';
 
-import './lib/parse.spec';
-import './lib/load.spec';
+import './lib/set-loader-options.spec';
 import './lib/register-loaders.spec';
 import './lib/select-loader.spec';
+import './lib/parse.spec';
+import './lib/load.spec';
 
 // TODO - The worker-utils specs test loading, not just worker farm
 // so we run them after util tests, until loading has been split out
 import './worker-utils';
-
-import './utils';

modules/core/test/lib/register-loaders.spec.js

@@ -1,5 +1,6 @@
 import test from 'tape-promise/tape';
-import {registerLoaders, getRegisteredLoaders} from '@loaders.gl/core/lib/register-loaders';
+import {registerLoaders} from '@loaders.gl/core';
+import {getRegisteredLoaders} from '@loaders.gl/core/lib/register-loaders';
 
 test('registerLoaders', t => {
   const registeredLoadersCount = getRegisteredLoaders().length;