images: worker-loader (for extracting image data in parallel)

docs/upgrade-guide.md

@@ -52,11 +52,9 @@ Some iterator helper functions have been renamed, the old naming is now deprecat
 
 **`@loaders.gl/images`**
 
-The experimental ImageLoaders for individual formats introduced in 2.0 have been removed, use `ImageLoader` for all formats.
-`@loaders.gl/images`
-
 - `getImageData(image)` now returns an object with `{data, width, height}` instead of just the `data` array. This small breaking change ensures that the concept of _image data_ is consistent across the API.
 - `ImageLoader`: `options.image.type`: The `html` and `ndarray` image types are now deprecated and replaced with `image` and `data` respectively.
+- `ImageLoaders`: The experimental loaders for individual formats introduced in 2.0 have been removed, use `ImageLoader` for all formats.
 
 **`@loaders.gl/3d-tiles`**
 

docs/whats-new.md

@@ -135,8 +135,8 @@ The new loaders empowers rendering frameworks to visualize various geospatial da
 
 **@loaders.gl/images**
 
-- Images can now be loaded as data: Using the `ImageLoader` with `options.image.type: 'data'` parameter will return an _image data object_ with width, height and a typed array containing the image data (instead of an opaque `Image` or `ImageBitmap` instance).
-- `ImageBitmap` loading now works reliably, use `ImageLoader` with `options.image.type: 'imagebitmap'`.
+- `ImageLoader`: Images can now be (reliably) loaded as `ImageBitmap`, by specifying `options.image.type: 'imagebitmap'`.
+- `ImageLoader`: Images can now be loaded as data (instead of an opaque `Image` or `ImageBitmap` objects), by specifying `options.image.type: 'data'`. The loader will return an _image data object_ with a `data` field with a typed array containing the image data as well as `width` and `height` fields.
 
 **@loaders.gl/json**
 

modules/core/src/lib/api/parse.js

@@ -64,6 +64,7 @@ async function parseWithLoader(loader, data, options, context) {
 
   // If we have a workerUrl and the loader can parse the given options efficiently in a worker
   if (canParseWithWorker(loader, data, options, context)) {
+    // console.warn('parsing with worker');
     return await parseWithWorker(loader, data, options, context);
   }
 

modules/images/package.json

@@ -28,10 +28,9 @@
     "README.md"
   ],
   "scripts": {
-    "pre-build": "npm run build-bundle && npm run build-bundle -- --env.dev",
-    "build-bundle": "webpack  --display=minimal --config ../../scripts/bundle.config.js",
-    "build-worker": "webpack --entry ./src/image-loader.worker.js --output ./dist/image-loader.worker.js --config ../../scripts/worker-webpack-config.js"
-  },
+  "pre-build": "npm run build-bundle && npm run build-bundle -- --env.dev",
+  "build-bundle": "webpack --display=minimal --config ../../scripts/bundle.config.js"
+},
   "dependencies": {
     "@loaders.gl/loader-utils": "2.2.1"
   }

modules/images/src/image-loader.js

@@ -1,5 +1,6 @@
 import parseImage from './lib/parsers/parse-image';
 import {getBinaryImageMetadata} from './lib/category-api/binary-image-api';
+
 /** @typedef {import('@loaders.gl/loader-utils').LoaderObject} LoaderObject */
 
 // __VERSION__ is injected by babel-plugin-version-inline
@@ -21,19 +22,20 @@ const MIME_TYPES = [
 /** @type {LoaderObject} */
 const ImageLoader = {
   id: 'image',
-  name: 'Images',
+  name: 'Image',
+  category: 'image',
   version: VERSION,
   mimeTypes: MIME_TYPES,
   extensions: EXTENSIONS,
   parse: parseImage,
-  // TODO: byteOffset, byteLength;
+  // TODO: support byteOffset, byteLength;
   test: arrayBuffer => Boolean(getBinaryImageMetadata(new DataView(arrayBuffer))),
   options: {
     image: {
       type: 'auto',
-      decode: true // if format is HTML
+      decode: true // applies only to images of type: 'image' (Image)
     }
-    // imagebitmap: {} - passes (platform dependent) parameters to ImageBitmap constructor
+    // imagebitmap: {} - if supplied, passes platform dependent parameters to `createImageBitmap`
   }
 };
 

modules/images/src/index.js

@@ -12,8 +12,8 @@ export {isImageTypeSupported, getDefaultImageType} from './lib/category-api/imag
 export {
   isImage,
   getImageType,
-  getImageSize,
-  getImageData
+  getImageData,
+  getImageDataAsync
 } from './lib/category-api/parsed-image-api';
 
 // Texture Loading API
@@ -26,6 +26,8 @@ export {loadImageCube} from './lib/texture-api/load-image-cube';
 
 export {default as HTMLImageLoader} from './image-loader';
 
+export {getImageData as getImageSize} from './lib/category-api/parsed-image-api';
+
 import {getDefaultImageType} from './lib/category-api/image-type';
 
 export function getSupportedImageType(imageType = null) {

modules/images/src/lib/category-api/image-type.js

@@ -1,14 +1,17 @@
 /* global ImageBitmap, Image */
-import {global, isBrowser} from '../utils/globals';
+import {global, isBrowser, isWorker} from '../utils/globals';
 
 // @ts-ignore TS2339: Property does not exist on type
 const {_parseImageNode} = global;
 
-const IMAGE_SUPPORTED = typeof Image !== 'undefined'; // NOTE: "false" positives if jsdom is installed
+// NOTE: "false" positives if jsdom is installed
+const IMAGE_SUPPORTED = typeof Image !== 'undefined' && !isWorker; // NOTE: "false" positives if jsdom is installed
 const IMAGE_BITMAP_SUPPORTED = typeof ImageBitmap !== 'undefined';
 const NODE_IMAGE_SUPPORTED = Boolean(_parseImageNode);
 const DATA_SUPPORTED = isBrowser ? true : NODE_IMAGE_SUPPORTED;
 
+const ERR_INSTALL_POLYFILLS = `Install '@loaders.gl/polyfills' to parse images under Node.js`;
+
 // Checks if a loaders.gl image type is supported
 export function isImageTypeSupported(type) {
   switch (type) {
@@ -30,7 +33,7 @@ export function isImageTypeSupported(type) {
       return DATA_SUPPORTED;
 
     default:
-      throw new Error(`@loaders.gl/images: image ${type} not supported in this environment`);
+      throw new Error(`@loaders.gl/images: unknown image type ${type}`);
   }
 }
 
@@ -47,5 +50,5 @@ export function getDefaultImageType() {
   }
 
   // This should only happen in Node.js
-  throw new Error(`Install '@loaders.gl/polyfills' to parse images under Node.js`);
+  throw new Error(ERR_INSTALL_POLYFILLS);
 }

modules/images/src/lib/category-api/parsed-image-api.js

@@ -1,6 +1,4 @@
-/* global Image, ImageBitmap */
-import assert from '../utils/assert';
-
+/* global document, Image, ImageBitmap, Blob, Worker, window */
 export function isImage(image) {
   return Boolean(getImageTypeOrNull(image));
 }
@@ -23,30 +21,75 @@ export function getImageType(image) {
   return format;
 }
 
+let canvas;
+let context2d;
+
 export function getImageData(image) {
   switch (getImageType(image)) {
-    case 'data':
-      return image;
-
     case 'image':
     case 'imagebitmap':
-      // Extract the image data from the image via a canvas
-      /* global document */
-      const canvas = document.createElement('canvas');
-      // TODO - reuse the canvas?
-      const context = canvas.getContext('2d');
+    // @ts-ignore DEPRECATED Backwards compatibility
+    case 'html': // eslint-disable-line no-fallthrough
+      canvas = canvas || document.createElement('canvas');
       canvas.width = image.width;
       canvas.height = image.height;
-      context.drawImage(image, 0, 0);
-      const imageData = context.getImageData(0, 0, image.width, image.height);
-      return imageData;
+      context2d = context2d || canvas.getContext('2d');
+      context2d.drawImage(image, 0, 0);
+      return context2d.getImageData(0, 0, image.width, image.height);
+
+    case 'data':
     default:
-      return assert(false);
+      return image;
   }
 }
 
-// TODO DEPRECATED not needed (use getImageData)
-export {getImageData as getImageSize};
+const WORKER_SCRIPT = `
+let canvas;
+let context2d;
+onmessage = function(event) {
+  try {
+    const {image} = event.data;
+    // TODO - can we reuse and resize instead of creating new canvas for each image?
+    canvas = canvas || new OffscreenCanvas(image.width, image.height);
+    // TODO potentially more efficient, but seems to block 2D context creation?
+    // const bmContext = canvas.getContext('bitmaprenderer');
+    // bmContext.transferFromImageBitmap(image);
+    context2d = context2d || canvas.getContext('2d');
+    context2d.drawImage(image, 0, 0);
+    const imageData = context2d.getImageData(0, 0, image.width, image.height);
+    const {width, height, data} = imageData;
+    postMessage({type: 'done', imageData: {width, height, data: data.buffer, worker: true}}, [data.buffer]);
+  } catch (error) {
+    postMessage({type: 'error', message: error.message});
+  }
+}
+`;
+
+let cachedWorker = null;
+
+function getWorker(script) {
+  if (!cachedWorker) {
+    const blob = new Blob([script]);
+    // Obtain a blob URL reference to our worker 'script'.
+    const blobURL = window.URL.createObjectURL(blob);
+    cachedWorker = new Worker(blobURL);
+  }
+  return cachedWorker;
+}
+
+export async function getImageDataAsync(image) {
+  if (typeof Worker !== 'undefined' && image instanceof ImageBitmap) {
+    const worker = getWorker(WORKER_SCRIPT);
+    return await new Promise((resolve, reject) => {
+      worker.onmessage = function(event) {
+        event.data.imageData.data = new Uint8Array(event.data.imageData.data);
+        resolve(event.data.imageData);
+      };
+      worker.postMessage({type: 'decode', image}); // Start the worker.
+    });
+  }
+  return getImageData(image);
+}
 
 // PRIVATE
 

modules/images/src/lib/parsers/parse-image.js

@@ -37,7 +37,12 @@ export default async function parseImage(arrayBuffer, options, context) {
 
   // Browser: if options.image.type === 'data', we can now extract data from the loaded image
   if (imageType === 'data') {
-    image = getImageData(image);
+    const data = getImageData(image);
+    // imageBitmap has a close method to dispose of graphical resources, test if available
+    if (image.close) {
+      image.close();
+    }
+    return data;
   }
 
   return image;
@@ -53,7 +58,9 @@ function getLoadableImageType(type) {
       return getDefaultImageType();
     default:
       // Throw an error if not supported
-      isImageTypeSupported(type);
+      if (!isImageTypeSupported(type)) {
+        throw new Error(`@loaders.gl/images: image type ${type} not supported in this environment`);
+      }
       return type;
   }
 }

modules/images/src/lib/texture-api/load-image.js

@@ -1,6 +1,6 @@
 import assert from '../utils/assert';
 import parseImage from '../parsers/parse-image';
-import {getImageSize} from '../category-api/parsed-image-api';
+import {getImageData} from '../category-api/parsed-image-api';
 import {generateUrl} from './generate-url';
 import {deepLoad, shallowLoad} from './deep-load';
 
@@ -24,7 +24,7 @@ async function getMipmappedImageUrls(getUrl, mipLevels, options, urlOptions) {
     const url = generateUrl(getUrl, options, {...urlOptions, lod: 0});
     const image = await shallowLoad(url, parseImage, options);
 
-    const {width, height} = getImageSize(image);
+    const {width, height} = getImageData(image);
     mipLevels = getMipLevels({width, height});
 
     // TODO - push image and make `deepLoad` pass through non-url values, avoid loading twice?

modules/images/src/workers/create-worker.js

@@ -0,0 +1,112 @@
+/* eslint-disable no-restricted-globals */
+/* global TextDecoder, self */
+
+import {getTransferList, validateLoaderVersion} from '@loaders.gl/loader-utils/';
+
+export default function createWorker(workerDescriptor) {
+  // TODO - explain when this happens? Just a sanity check? Throw an error or log a warning?
+  if (typeof self === 'undefined') {
+    return;
+  }
+
+  let requestId = 0;
+  const process = (arraybuffer, options = {}, url) =>
+    new Promise((resolve, reject) => {
+      const id = requestId++;
+
+      const onMessage = ({data}) => {
+        if (!data || data.id !== id) {
+          // not ours
+          return;
+        }
+        switch (data.type) {
+          case 'process-done':
+            self.removeEventListener('message', onMessage);
+            resolve(data.result);
+            break;
+
+          case 'process-error':
+            self.removeEventListener('message', onMessage);
+            reject(data.message);
+            break;
+
+          default:
+          // ignore
+        }
+      };
+      self.addEventListener('message', onMessage);
+      // Ask the main thread to decode data
+      // @ts-ignore self is WorkerGlobalScope
+      self.postMessage({type: 'process', id, arraybuffer, options, url}, [arraybuffer]);
+    });
+
+  self.onmessage = async evt => {
+    const {data} = evt;
+
+    try {
+      if (!isKnownMessage(data, workerDescriptor.name)) {
+        return;
+      }
+
+      validateLoaderVersion(workerDescriptor, data.source.split('@')[1]);
+
+      const {arraybuffer, byteOffset = 0, byteLength = 0, options = {}} = data;
+
+      const result = await parseData({
+        workerDescriptor,
+        arraybuffer,
+        byteOffset,
+        byteLength,
+        options,
+        context: {process}
+      });
+      const transferList = getTransferList(result);
+      // @ts-ignore self is WorkerGlobalScope
+      self.postMessage({type: 'done', result}, transferList);
+    } catch (error) {
+      // @ts-ignore self is WorkerGlobalScope
+      self.postMessage({type: 'error', message: error.message});
+    }
+  };
+}
+
+// TODO - Support byteOffset and byteLength (enabling parsing of embedded binaries without copies)
+// TODO - Why not support async workerDescriptor.process* funcs here?
+// TODO - Why not reuse a common function instead of reimplementing workerDescriptor.process* selection logic? Keeping workerDescriptor small?
+// TODO - Lack of appropriate parser functions can be detected when we create worker, no need to wait until process
+async function parseData({
+  workerDescriptor,
+  arraybuffer,
+  byteOffset,
+  byteLength,
+  options,
+  context
+}) {
+  let data;
+  let parser;
+  if (workerDescriptor.parseSync || workerDescriptor.process) {
+    data = arraybuffer;
+    parser = workerDescriptor.parseSync || workerDescriptor.process;
+  } else if (workerDescriptor.parseTextSync) {
+    const textDecoder = new TextDecoder();
+    data = textDecoder.decode(arraybuffer);
+    parser = workerDescriptor.parseTextSync;
+  } else {
+    throw new Error(`Could not load data with ${workerDescriptor.name} workerDescriptor`);
+  }
+
+  // TODO - proper merge in of workerDescriptor options...
+  options = {
+    ...options,
+    modules:
+      (workerDescriptor && workerDescriptor.options && workerDescriptor.options.modules) || {},
+    worker: false
+  };
+
+  return await parser(data, {...options}, context, workerDescriptor);
+}
+
+// Filter out noise messages sent to workers
+function isKnownMessage(data, name) {
+  return data && data.type === 'process' && data.source && data.source.startsWith('loaders.gl');
+}