-
Notifications
You must be signed in to change notification settings - Fork 188
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
17 changed files
with
220 additions
and
76 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| // Breaking big data into iterable chunks, concatenating iterable chunks into big data objects | ||
|
|
||
| import {concatenateArrayBuffers} from '../javascript-utils/memory-copy-utils'; | ||
|
|
||
| /** | ||
| * Concatenates all data chunks yielded by an (async) iterator | ||
| * Supports strings and ArrayBuffers | ||
| * | ||
| * This function can e.g. be used to enable atomic parsers to work on (async) iterator inputs | ||
| */ | ||
| export async function concatenateChunksAsync(asyncIterator) { | ||
| let arrayBuffer = new ArrayBuffer(0); | ||
| let string = ''; | ||
| for await (const chunk of asyncIterator) { | ||
| if (typeof chunk === 'string') { | ||
| string += chunk; | ||
| } else { | ||
| arrayBuffer = concatenateArrayBuffers(arrayBuffer, chunk); | ||
| } | ||
| } | ||
| return string || arrayBuffer; | ||
| } | ||
|
|
||
| /** | ||
| * Returns an iterator that breaks a big `ArrayBuffer` or string into chunks and yields them one-by-one. | ||
| * | ||
| * @param bigArrayBufferOrString | ||
| * @param options | ||
| * @param options.chunkSize | ||
| * @returns iterator that yields chunks of specified size | ||
| * | ||
| * This function can e.g. be used to enable data sources that can only be read atomically | ||
| * (such as `Blob` and `File` via `FileReader`) to still be parsed in batches. | ||
| */ | ||
| export function* makeChunkIterator(bigArrayBufferOrString, options = {}) { | ||
| if (typeof bigArrayBufferOrString === 'string') { | ||
| yield* makeStringChunkIterator(bigArrayBufferOrString, options); | ||
| return; | ||
| } | ||
| if (bigArrayBufferOrString instanceof ArrayBuffer) { | ||
| yield* makeArrayBufferChunkIterator(bigArrayBufferOrString, options); | ||
| return; | ||
| } | ||
| throw new Error('assert'); | ||
| } | ||
|
|
||
| /** | ||
| * Helper: Breaks a big ArrayBuffer into chunks and returns an iterator that yields them one-by-one | ||
| */ | ||
| function* makeArrayBufferChunkIterator(arrayBuffer, options = {}) { | ||
| const {chunkSize = 256 * 1024} = options; | ||
|
|
||
| let byteOffset = 0; | ||
|
|
||
| while (byteOffset < arrayBuffer.byteLength) { | ||
| // Create a chunk of the right size | ||
| const chunkByteLength = Math.min(arrayBuffer.byteLength - byteOffset, chunkSize); | ||
| const chunk = new ArrayBuffer(chunkByteLength); | ||
|
|
||
| // Copy data from the big chunk | ||
| const sourceArray = new Uint8Array(arrayBuffer, byteOffset, chunkByteLength); | ||
| const chunkArray = new Uint8Array(chunk); | ||
| chunkArray.set(sourceArray); | ||
|
|
||
| // yield the chunk | ||
| byteOffset += chunkByteLength; | ||
| yield chunk; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Helper: Breaks a big string into chunks and returns an iterator that yields them one-by-one | ||
| */ | ||
| function* makeStringChunkIterator(string, options = {}) { | ||
| const {chunkSize = 256 * 1024} = options; | ||
|
|
||
| let offset = 0; | ||
|
|
||
| while (offset < string.length) { | ||
| // Create a chunk of the right size | ||
| const chunkLength = Math.min(string.length - offset, chunkSize); | ||
| const chunk = string.slice(offset, offset + chunkLength); | ||
| offset += chunkLength; | ||
|
|
||
| // yield the chunk | ||
| yield chunk; | ||
| } | ||
| } |
2 changes: 1 addition & 1 deletion
2
...core/src/javascript-utils/stream-utils.js → ...re/src/iterator-utils/stream-iteration.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| import test from 'tape-promise/tape'; | ||
|
|
||
| import { | ||
| makeChunkIterator, | ||
| concatenateChunksAsync | ||
| } from '@loaders.gl/core/iterator-utils/chunk-iteration'; | ||
| import {textEncoderAsyncIterator} from '@loaders.gl/core/iterator-utils/async-iteration'; | ||
|
|
||
| /* global setTimeout */ | ||
| const setTimeoutPromise = timeout => new Promise(resolve => setTimeout(resolve, timeout)); | ||
|
|
||
| async function* asyncTexts() { | ||
| await setTimeoutPromise(10); | ||
| yield 'line 1\nline'; | ||
| await setTimeoutPromise(10); | ||
| yield ' 2\nline 3\n'; | ||
| await setTimeoutPromise(10); | ||
| yield 'line 4'; | ||
| } | ||
|
|
||
| function asyncArrayBuffers() { | ||
| return textEncoderAsyncIterator(asyncTexts()); | ||
| } | ||
|
|
||
| test('concatenateChunksAsync', async t => { | ||
| const RESULT = `line 1\nline 2\nline 3\nline 4`; | ||
|
|
||
| const text = await concatenateChunksAsync(asyncTexts()); | ||
| t.is(text, RESULT, 'returns concatenated string'); | ||
|
|
||
| const arraybuffer = await concatenateChunksAsync(asyncArrayBuffers()); | ||
| t.ok(arraybuffer instanceof Uint8Array, 'returns buffer'); | ||
| /* global TextEncoder */ | ||
| t.deepEqual(arraybuffer, new TextEncoder().encode(RESULT), 'returns concatenated arraybuffer'); | ||
|
|
||
| t.end(); | ||
| }); | ||
|
|
||
| test('makeChunkIterator#string', async t => { | ||
| const bigString = '123456'; | ||
| const results = ['12', '34', '56']; | ||
|
|
||
| const iterator = makeChunkIterator(bigString, {chunkSize: 2}); | ||
|
|
||
| for (const chunk of iterator) { | ||
| t.equal(chunk, results.shift()); | ||
| } | ||
|
|
||
| t.end(); | ||
| }); | ||
|
|
||
| test('makeChunkIterator#arrayBuffer', async t => { | ||
| const bigString = new ArrayBuffer(6); | ||
|
|
||
| const iterator = makeChunkIterator(bigString, {chunkSize: 2}); | ||
|
|
||
| for (const chunk of iterator) { | ||
| t.ok(chunk instanceof ArrayBuffer); | ||
| t.equal(chunk.byteLength, 2); | ||
| } | ||
|
|
||
| t.end(); | ||
| }); |
Oops, something went wrong.