-
Notifications
You must be signed in to change notification settings - Fork 187
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(compression): Integrate fflate library
- Loading branch information
Showing
9 changed files
with
493 additions
and
78 deletions.
There are no files selected for viewing
108 changes: 108 additions & 0 deletions
108
modules/compression/src/lib/deflate-compression-pako.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| // loaders.gl, MIT license | ||
| import type {CompressionOptions} from './compression'; | ||
| import {Compression} from './compression'; | ||
| import {getPakoError} from './utils/pako-utils'; | ||
| import pako from 'pako'; // https://bundlephobia.com/package/pako | ||
|
|
||
| export type DeflateCompressionOptions = CompressionOptions & { | ||
| deflate?: pako.InflateOptions & pako.DeflateOptions & {useZlib?: boolean}; | ||
| }; | ||
|
|
||
| /** | ||
| * DEFLATE compression / decompression | ||
| * Using PAKO library | ||
| */ | ||
| export class DeflateCompression extends Compression { | ||
| readonly name: string = 'deflate'; | ||
| readonly extensions: string[] = []; | ||
| readonly contentEncodings = ['deflate']; | ||
| readonly isSupported = true; | ||
|
|
||
| readonly options: DeflateCompressionOptions; | ||
|
|
||
| private _chunks: ArrayBuffer[] = []; | ||
|
|
||
| constructor(options: DeflateCompressionOptions = {}) { | ||
| super(options); | ||
| this.options = options; | ||
| } | ||
|
|
||
| async compress(input: ArrayBuffer): Promise<ArrayBuffer> { | ||
| return this.compressSync(input); | ||
| } | ||
|
|
||
| async decompress(input: ArrayBuffer): Promise<ArrayBuffer> { | ||
| return this.decompressSync(input); | ||
| } | ||
|
|
||
| compressSync(input: ArrayBuffer): ArrayBuffer { | ||
| const pakoOptions: pako.DeflateOptions = this.options?.deflate || {}; | ||
| const inputArray = new Uint8Array(input); | ||
| return pako.deflate(inputArray, pakoOptions).buffer; | ||
| } | ||
|
|
||
| decompressSync(input: ArrayBuffer): ArrayBuffer { | ||
| const pakoOptions: pako.InflateOptions = this.options?.deflate || {}; | ||
| const inputArray = new Uint8Array(input); | ||
| return pako.inflate(inputArray, pakoOptions).buffer; | ||
| } | ||
|
|
||
| async *compressBatches( | ||
| asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer> | ||
| ): AsyncIterable<ArrayBuffer> { | ||
| const pakoOptions: pako.DeflateOptions = this.options?.deflate || {}; | ||
| const pakoProcessor = new pako.Deflate(pakoOptions); | ||
| yield* this.transformBatches(pakoProcessor, asyncIterator); | ||
| } | ||
|
|
||
| async *decompressBatches( | ||
| asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer> | ||
| ): AsyncIterable<ArrayBuffer> { | ||
| const pakoOptions: pako.InflateOptions = this.options?.deflate || {}; | ||
| const pakoProcessor = new pako.Inflate(pakoOptions); | ||
| yield* this.transformBatches(pakoProcessor, asyncIterator); | ||
| } | ||
|
|
||
| async *transformBatches( | ||
| pakoProcessor: pako.Inflate | pako.Deflate, | ||
| asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer> | ||
| ): AsyncIterable<ArrayBuffer> { | ||
| pakoProcessor.onData = this._onData.bind(this); | ||
| pakoProcessor.onEnd = this._onEnd.bind(this); | ||
| for await (const chunk of asyncIterator) { | ||
| const uint8Array = new Uint8Array(chunk); | ||
| const ok = pakoProcessor.push(uint8Array, false); // false -> not last chunk | ||
| if (!ok) { | ||
| throw new Error(`${getPakoError()}write`); | ||
| } | ||
| const chunks = this._getChunks(); | ||
| yield* chunks; | ||
| } | ||
|
|
||
| // End | ||
| const emptyChunk = new Uint8Array(0); | ||
| const ok = pakoProcessor.push(emptyChunk, true); // true -> last chunk | ||
| if (!ok) { | ||
| // For some reason we get error but it still works??? | ||
| // throw new Error(getPakoError() + 'end'); | ||
| } | ||
| const chunks = this._getChunks(); | ||
| yield* chunks; | ||
| } | ||
|
|
||
| _onData(chunk) { | ||
| this._chunks.push(chunk); | ||
| } | ||
|
|
||
| _onEnd(status) { | ||
| if (status !== 0) { | ||
| throw new Error(getPakoError(status) + this._chunks.length); | ||
| } | ||
| } | ||
|
|
||
| _getChunks(): ArrayBuffer[] { | ||
| const chunks = this._chunks; | ||
| this._chunks = []; | ||
| return chunks; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| // loaders.gl, MIT license | ||
| import {isBrowser, toArrayBuffer} from '@loaders.gl/loader-utils'; | ||
| import {promisify2} from '@loaders.gl/loader-utils'; | ||
| import type {CompressionOptions} from './compression'; | ||
| import {Compression} from './compression'; | ||
| import * as zlib from 'zlib'; | ||
| import type {ZlibOptions} from 'zlib'; | ||
|
|
||
| export type DeflateCompressionOptions = CompressionOptions & { | ||
| deflate?: ZlibOptions; | ||
| }; | ||
|
|
||
| /** | ||
| * DEFLATE compression / decompression | ||
| * Using Node.js zlib library (works under Node only) | ||
| */ | ||
| export class DeflateCompression extends Compression { | ||
| readonly name: string = 'deflate'; | ||
| readonly extensions: string[] = []; | ||
| readonly contentEncodings = ['deflate']; | ||
| readonly isSupported = isBrowser; | ||
|
|
||
| readonly options: DeflateCompressionOptions; | ||
|
|
||
| constructor(options: DeflateCompressionOptions = {}) { | ||
| super(options); | ||
| this.options = options; | ||
| if (!isBrowser) { | ||
| throw new Error('zlib only available under Node.js'); | ||
| } | ||
| } | ||
|
|
||
| async compress(input: ArrayBuffer): Promise<ArrayBuffer> { | ||
| const buffer = await promisify2(zlib.deflate)(input, this.options.deflate || {}); | ||
| return toArrayBuffer(buffer); | ||
| } | ||
|
|
||
| async decompress(input: ArrayBuffer): Promise<ArrayBuffer> { | ||
| const buffer = await promisify2(zlib.inflate)(input, this.options.deflate || {}); | ||
| return toArrayBuffer(buffer); | ||
| } | ||
|
|
||
| compressSync(input: ArrayBuffer): ArrayBuffer { | ||
| const buffer = zlib.deflateSync(input, this.options.deflate || {}); | ||
| return toArrayBuffer(buffer); | ||
| } | ||
|
|
||
| decompressSync(input: ArrayBuffer): ArrayBuffer { | ||
| const buffer = zlib.inflateSync(input, this.options.deflate || {}); | ||
| return toArrayBuffer(buffer); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.