Merge a5aac80 into b5c795c

visgl · Aug 8, 2020 · dc6d890 · dc6d890
2 parents b5c795c + a5aac80
commit dc6d890
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 2 deletions.
diff --git a/modules/crypto/src/lib/crc32/crc32-hash-transform.js b/modules/crypto/src/lib/crc32/crc32-hash-transform.js
@@ -29,7 +29,7 @@ export default class CRC32HashTransform {
     if (this.options.crypto.onEnd) {
       this.options.crypto.onEnd({hash});
     }
-    return hash;
+    return null;
   }
 
   _update(chunk) {

diff --git a/modules/crypto/src/lib/crc32c/crc32c-hash-transform.js b/modules/crypto/src/lib/crc32c/crc32c-hash-transform.js
@@ -29,7 +29,7 @@ export default class CRC32CHashTransform {
     if (this.options.crypto.onEnd) {
       this.options.crypto.onEnd({hash});
     }
-    return hash;
+    return null;
   }
 
   _update(chunk) {

diff --git a/modules/loader-utils/test/lib/iterator-utils/make-transform-iterator.spec.js b/modules/loader-utils/test/lib/iterator-utils/make-transform-iterator.spec.js
@@ -1,5 +1,10 @@
 import test from 'tape-promise/tape';
 import {makeTransformIterator, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
+import {fetchFile, parseInBatches, makeIterator} from '@loaders.gl/core';
+import {ShapefileLoader} from '@loaders.gl/shapefile';
+import {CRC32CHashTransform} from '@loaders.gl/crypto';
+
+const SHAPEFILE_URL = '@loaders.gl/shapefile/test/data/shapefile-js/boolean-property.shp';
 
 class ByteLengthTransform {
   constructor(options = {}) {
@@ -64,3 +69,29 @@ function compareArrayBuffers(a, b) {
   }
   return true;
 }
+
+// Tests that iterator input and non-streaming loader does not crash
+test.only('makeTransformIterator', async t => {
+  // Run a streaming digest on all test cases.
+  let hash;
+
+  const response = await fetchFile(SHAPEFILE_URL);
+  let iterator = makeIterator(response);
+
+  // @ts-ignore
+  iterator = makeTransformIterator(iterator, CRC32CHashTransform, {
+    crypto: {
+      onEnd: result => {
+        hash = result.hash;
+      }
+    }
+  });
+
+  const batchIterator = await parseInBatches(iterator, ShapefileLoader);
+  for await (const batch of batchIterator) {
+    t.ok(batch, 'streaming hash is correct');
+  }
+
+  t.equal(hash, 'sC9tGA==', `Shapefile hash should correct: "${hash}"`);
+  t.end();
+});
diff --git a/modules/shapefile/test/shapefile-loader.spec.js b/modules/shapefile/test/shapefile-loader.spec.js
@@ -81,6 +81,22 @@ test('ShapefileLoader#loadInBatches', async t => {
   t.end();
 });
 
+test('ShapefileLoader#loadInBatches', async t => {
+  // test file load (node) or URL load (browser)
+  for (const testFileName in SHAPEFILE_JS_TEST_FILES) {
+    const filename = `${SHAPEFILE_JS_DATA_FOLDER}/${testFileName}.shp`;
+    const batches = await loadInBatches(filename, ShapefileLoader);
+    let data;
+    for await (const batch of batches) {
+      data = batch;
+      t.comment(`${filename}: ${JSON.stringify(data).slice(0, 70)}`);
+    }
+    await testShapefileData(t, testFileName, data);
+  }
+
+  t.end();
+});
+
 async function getFileList(testFileName) {
   const EXTENSIONS = ['.shp', '.shx', '.dbf', '.cpg', '.prj'];
   const fileList = [];