crypto: fixes and cleanup (#876)

visgl · Jul 31, 2020 · 781ba27 · 781ba27
1 parent b9848e0
commit 781ba27
Show file tree

Hide file tree

Showing 18 changed files with 91 additions and 41 deletions.
diff --git a/docs/table-of-contents.json b/docs/table-of-contents.json
@@ -148,6 +148,15 @@
             {"entry": "modules/compression/docs/api-reference/zstd-inflate-transform"}
           ]
         },
+        {
+          "title": "@loaders.gl/crypto",
+          "entries": [
+            {"entry": "modules/crypto/docs"},
+            {"entry": "modules/crypto/docs/api-reference/crc32-hash-transform"},
+            {"entry": "modules/crypto/docs/api-reference/crypto-hash-transform"},
+            {"entry": "modules/crypto/docs/api-reference/md5-hash-transform"}
+          ]
+        },
         {
           "title": "@loaders.gl/draco",
           "entries": [

diff --git a/docs/whats-new.md b/docs/whats-new.md
@@ -16,7 +16,7 @@ This release brings a new Shapefile loader, compression codecs (Zlib, LZ4, Zstan
 
 **@loaders.gl/crypto** (NEW)
 
-- A new module with "transforms" for calculating cryptographic hashes (MD5, SHA256 etc) incrementally, e.g. on incoming binary chunks while streaming data into `parseInBatches1.
+- A new module with "transforms" for calculating cryptographic hashes (MD5, SHA256 etc) incrementally, e.g. on incoming binary chunks while streaming data into `parseInBatches`.
 
 **@loaders.gl/core**
 

diff --git a/modules/compression/docs/api-reference/lz4-deflate-transform.md b/modules/compression/docs/api-reference/lz4-deflate-transform.md
@@ -9,3 +9,5 @@
 #### `LZ4DeflateTransform.deflate(data: ArrayBuffer, options?: object): Promise<ArrayBuffer>`
 
 #### `LZ4DeflateTransform.deflateSync(data: ArrayBuffer, options?: object): ArrayBuffer`
+
+Compresses (deflates) LZ4 encoded data.
diff --git a/modules/compression/docs/api-reference/lz4-inflate-transform.md b/modules/compression/docs/api-reference/lz4-inflate-transform.md
@@ -10,4 +10,4 @@
 
 #### `LZ4InflateTransform.inflateSync(data: ArrayBuffer, options?: object): ArrayBuffer`
 
-Note: options are passed through to the underlying `pako` library.
+Decompresses (inflates) LZ4 encoded data.
diff --git a/modules/compression/docs/api-reference/zlib-deflate-transform.md b/modules/compression/docs/api-reference/zlib-deflate-transform.md
@@ -10,4 +10,8 @@
 
 #### `ZlibDeflateTransform.deflateSync(data: ArrayBuffer, options?: object): ArrayBuffer`
 
-Note: options are passed through to the underlying `pako` library.
+Compresses (deflates) Zlib encoded data.
+
+## Remarks
+
+- options are passed through to the underlying `pako` library.
diff --git a/modules/compression/docs/api-reference/zlib-inflate-transform.md b/modules/compression/docs/api-reference/zlib-inflate-transform.md
@@ -10,4 +10,8 @@
 
 ### `ZlibInflateTransform.inflateSync(data: ArrayBuffer, options?: object): ArrayBuffer`
 
-Note: options are passed through to the underlying `pako` library.
+Decompresses (inflates) Zlib encoded data.
+
+## Remarks
+
+- options are passed through to the underlying `pako` library.
diff --git a/modules/compression/docs/api-reference/zstd-deflate-transform.md b/modules/compression/docs/api-reference/zstd-deflate-transform.md
@@ -7,3 +7,5 @@
 ## Static Methods
 
 #### `ZstdDeflateTransform.deflate(data: ArrayBuffer, options?: object): Promise<ArrayBuffer>`
+
+Compresses (deflates) Zstandard encoded data.
diff --git a/modules/compression/docs/api-reference/zstd-inflate-transform.md b/modules/compression/docs/api-reference/zstd-inflate-transform.md
@@ -9,3 +9,5 @@
 #### `ZstdInflateTransform.inflate(data: ArrayBuffer, options?: object): Promise<ArrayBuffer>`
 
 #### `ZstdInflateTransform.inflateSync(data: ArrayBuffer, options?: object): ArrayBuffer`
+
+Decompresses (inflates) Zstandard encoded data.
diff --git a/modules/core/docs/api-reference/select-loader.md b/modules/core/docs/api-reference/select-loader.md
@@ -56,7 +56,7 @@ await selectLoader(blob, DracoLoader); // => DracoLoader
 
 ## Functions
 
-### selectLoader(data: Response | ArrayBuffer | String | Blob, ..., loaders?: LoaderObject[], options?: object, context?: object): Promise<boolean>
+### `selectLoader(data: Response | ArrayBuffer | String | Blob, ..., loaders?: LoaderObject[], options?: object, context?: object): Promise<boolean>`
 
 Selects an appropriate loader for a file from a list of candidate loaders by examining the `data` parameter, looking at URL extension, mimeType ('Content-Type') and/or an initial data chunk.
 
@@ -80,7 +80,7 @@ Regarding the `loaders` parameter:
 - a `null` loader list will use the pre-registered list of loaders.
 - A supplied list of loaders will be searched for a matching loader.
 
-### selectLoaderSync(data: Response | ArrayBuffer | String | Blob, ..., loaders?: LoaderObject[], options?: object, context?: object): boolean
+### `selectLoaderSync(data: Response | ArrayBuffer | String | Blob, ..., loaders?: LoaderObject[], options?: object, context?: object): boolean`
 
 ## Supported Formats
 

diff --git a/modules/crypto/docs/api-reference/crc32-hash-transform.md b/modules/crypto/docs/api-reference/crc32-hash-transform.md
@@ -9,3 +9,9 @@
 #### `CRC32HashTransform.hash(data: ArrayBuffer, options?: object): Promise<string>`
 
 #### `CRC32HashTransform.hashSync(data: ArrayBuffer, options?: object): string`
+
+Calculates the CRC32 hash of a byte array.
+
+## Remarks
+
+- This transform supports streaming hashing.
diff --git a/modules/crypto/docs/api-reference/crypto-hash-transform.md b/modules/crypto/docs/api-reference/crypto-hash-transform.md
@@ -10,4 +10,8 @@
 
 #### `CryptoHashTransform.hashSync(data: ArrayBuffer, options?: object): string`
 
-- `options.CryptoJS` needs to be supplied with the CryptoJS library
+- `options.modules.CryptoJS` the CryptoJS library needs to be supplied by the application.
+
+## Remarks
+
+- This transform supports streaming hashing.
diff --git a/modules/crypto/docs/api-reference/md5-hash-transform.md b/modules/crypto/docs/api-reference/md5-hash-transform.md
@@ -4,8 +4,12 @@
   <img src="https://img.shields.io/badge/From-v2.3-blue.svg?style=flat-square" alt="From-v2.3" /> 
 </p>
 
-> Note: This "transform" does not support streaming hashing at the moment. Only the static method is defined.
-
 ## Static Methods
 
 #### `MD5HashTransform.hash(data: ArrayBuffer, options?: object): Promise<string>`
+
+Calculates the MD5 hash of a byte array.
+
+## Remarks
+
+- This transform does not yet support streaming hashing. Only the static method is defined.
diff --git a/modules/crypto/package.json b/modules/crypto/package.json
@@ -35,6 +35,6 @@
     "@babel/runtime": "^7.3.1",
     "@loaders.gl/loader-utils": "2.3.0-alpha.7",
     "@types/crypto-js": "*",
-    "crypto-js": "^4.0.0"
+    "crypto-js": "^3.0.0 || ^4.0.0"
   }
 }
diff --git a/modules/crypto/src/lib/crypto/crypto-hash-transform.js b/modules/crypto/src/lib/crypto/crypto-hash-transform.js
@@ -1,13 +1,17 @@
-// This dependency is too big
+// This dependency is too big, application must provide it
 // import * as CryptoJS from 'crypto-js';
+import {assert} from '@loaders.gl/loader-utils';
+
+const ERR_LIBRARY_NOT_SUPPLIED = 'crypto-js lib must be supplied in options.module.CryptoJS';
 
 export default class CryptoHashTransform {
   static async hash(input, options) {
     return CryptoHashTransform.hashSync(input, options);
   }
 
   static hashSync(input, options = {}) {
-    const {CryptoJS} = options;
+    const {CryptoJS} = options.modules || {};
+    assert(CryptoJS, ERR_LIBRARY_NOT_SUPPLIED);
 
     const transform = new CryptoHashTransform(options);
     const typedWordArray = CryptoJS.lib.WordArray.create(input);
@@ -21,7 +25,9 @@ export default class CryptoHashTransform {
     options = options || {};
     options.crypto = options.crypto || {};
 
-    const {CryptoJS} = options;
+    const {CryptoJS} = options.modules || {};
+    assert(CryptoJS, ERR_LIBRARY_NOT_SUPPLIED);
+
     const {algorithm = CryptoJS.algo.MD5} = options.crypto;
 
     this.options = options;

diff --git a/modules/crypto/test/crypto.bench.js b/modules/crypto/test/crypto.bench.js
@@ -1,5 +1,5 @@
 import {CryptoHashTransform, CRC32HashTransform, MD5HashTransform} from '@loaders.gl/crypto';
-import {getBinaryData} from './lib/crypto-hash-transform.spec';
+import {getBinaryData} from './test-utils/test-utils';
 import * as CryptoJS from 'crypto-js';
 
 export default async function csvBench(bench) {
@@ -22,11 +22,15 @@ export default async function csvBench(bench) {
   bench = bench.add(
     'CryptoHashTransform#hashSync(SHA256)',
     {multiplier: 100000, unit: 'bytes'},
-    () => CryptoHashTransform.hashSync(binaryData, {crypto: {algorithm: CryptoJS.algo.SHA256}})
+    () =>
+      CryptoHashTransform.hashSync(binaryData, {
+        modules: {CryptoJS},
+        crypto: {algorithm: CryptoJS.algo.SHA256}
+      })
   );
 
   bench = bench.addAsync('CryptoHashTransform#hash(MD5)', {multiplier: 100000, unit: 'bytes'}, () =>
-    CryptoHashTransform.hash(binaryData)
+    CryptoHashTransform.hash(binaryData, {modules: {CryptoJS}})
   );
 
   return bench;

diff --git a/modules/crypto/test/lib/crypto-hash-transform.spec.js b/modules/crypto/test/lib/crypto-hash-transform.spec.js
@@ -1,8 +1,5 @@
 import test from 'tape-promise/tape';
-import {
-  generateRandomArrayBuffer,
-  compareArrayBuffers
-} from '@loaders.gl/compression/test/utils/test-utils';
+import {compareArrayBuffers, getBinaryData} from '../test-utils/test-utils';
 import {makeTransformIterator, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
 import {fetchFile, parseInBatches, makeIterator} from '@loaders.gl/core';
 import {CSVLoader} from '@loaders.gl/csv';
@@ -12,13 +9,12 @@ import * as CryptoJS from 'crypto-js';
 const CSV_URL = '@loaders.gl/csv/test/data/sample-very-long.csv';
 /** Externally computed hash: `openssl md5 -binary sample-very-long.json | openssl base64` */
 const CSV_MD5 = 'zmLuuVSkigYR9r5FcsKkCw==';
-const SIZE = 100 * 1000;
 
 test('CryptoHashTransform#hashSync(CSV, against external hash)', async t => {
   const response = await fetchFile(CSV_URL);
   const data = await response.arrayBuffer();
 
-  const hash = CryptoHashTransform.hashSync(data, {CryptoJS});
+  const hash = CryptoHashTransform.hashSync(data, {modules: {CryptoJS}});
   t.equal(hash, CSV_MD5, 'repeated data MD5 hash is correct');
 
   t.end();
@@ -32,7 +28,7 @@ test('CryptoHashTransform#iterator(CSV stream, against external hash)', async t
   let iterator = makeIterator(response);
   // @ts-ignore
   iterator = makeTransformIterator(iterator, CryptoHashTransform, {
-    CryptoJS,
+    modules: {CryptoJS},
     onEnd: result => {
       hash = result.hash;
     }
@@ -54,10 +50,10 @@ test('CryptoHashTransform#iterator(CSV stream, against external hash)', async t
 test('CryptoHashTransform#hashSync(MD5 = default)', t => {
   const {binaryData, repeatedData} = getBinaryData();
 
-  let hash = CryptoHashTransform.hashSync(binaryData, {CryptoJS});
+  let hash = CryptoHashTransform.hashSync(binaryData, {modules: {CryptoJS}});
   t.equal(hash, 'YnxTb+lyen1CsNkpmLv+qA==', 'binary data MD5 hash is correct');
 
-  hash = CryptoHashTransform.hashSync(repeatedData, {CryptoJS});
+  hash = CryptoHashTransform.hashSync(repeatedData, {modules: {CryptoJS}});
   t.equal(hash, '2d4uZUoLXXO/XWJGnrVl5Q==', 'repeated data MD5 hash is correct');
 
   t.end();
@@ -67,7 +63,7 @@ test('CryptoHashTransform#hashSync(SHA256)', t => {
   const {binaryData, repeatedData} = getBinaryData();
 
   let hash = CryptoHashTransform.hashSync(binaryData, {
-    CryptoJS,
+    modules: {CryptoJS},
     crypto: {algorithm: CryptoJS.algo.SHA256}
   });
   t.equal(
@@ -77,7 +73,7 @@ test('CryptoHashTransform#hashSync(SHA256)', t => {
   );
 
   hash = CryptoHashTransform.hashSync(repeatedData, {
-    CryptoJS,
+    modules: {CryptoJS},
     crypto: {algorithm: CryptoJS.algo.SHA256}
   });
   t.equal(
@@ -100,7 +96,7 @@ test('makeTransformIterator#CryptoHashTransform(small chunks)', async t => {
 
   // @ts-ignore
   const transformIterator = makeTransformIterator(inputChunks, CryptoHashTransform, {
-    CryptoJS,
+    modules: {CryptoJS},
     onEnd: result => {
       hash = result.hash;
     }
@@ -130,7 +126,7 @@ test('makeTransformIterator#CryptoHashTransform(100K)', async t => {
 
   // @ts-ignore
   const transformIterator = makeTransformIterator(inputChunks, CryptoHashTransform, {
-    CryptoJS,
+    modules: {CryptoJS},
     onEnd: result => {
       hash = result.hash;
     }
@@ -150,14 +146,3 @@ test('makeTransformIterator#CryptoHashTransform(100K)', async t => {
 
   t.end();
 });
-
-// UTILITIES
-
-const data = {};
-
-export function getBinaryData() {
-  data.binaryData = data.binaryData || generateRandomArrayBuffer({size: SIZE});
-  data.repeatedData =
-    data.repeatedData || generateRandomArrayBuffer({size: SIZE / 10, repetitions: 10});
-  return data;
-}
diff --git a/modules/crypto/test/test-utils/test-utils.js b/modules/crypto/test/test-utils/test-utils.js
@@ -0,0 +1,18 @@
+import {generateRandomArrayBuffer} from '@loaders.gl/compression/test/utils/test-utils';
+
+export {
+  generateRandomArrayBuffer,
+  compareArrayBuffers
+} from '@loaders.gl/compression/test/utils/test-utils';
+
+const SIZE = 100 * 1000;
+
+const data = {};
+
+/** Generate binaryData and repeatedData */
+export function getBinaryData(size = SIZE) {
+  data.binaryData = data.binaryData || generateRandomArrayBuffer({size});
+  data.repeatedData =
+    data.repeatedData || generateRandomArrayBuffer({size: size / 10, repetitions: 10});
+  return data;
+}
diff --git a/website/package.json b/website/package.json
@@ -10,7 +10,7 @@
   "scripts": {
     "start": "yarn develop",
     "develop": "yarn clean-examples && yarn build:gallery && yarn gatsby develop",
-    "build": "yarn clean-examples && yarn build:gallery && gatsby build",
+    "build": "yarn clean && yarn clean-examples && yarn build:gallery && gatsby build",
     "build-staging": "yarn clean && yarn clean-examples && yarn build:gallery && gatsby build --prefix-paths",
     "serve": "gatsby serve",
     "deploy": "NODE_DEBUG=gh-pages gh-pages -d public",