Skip to content

Commit

Permalink
Lower the memory consumption
Browse files Browse the repository at this point in the history
A pako object contains a 64k buffer. We create a `FlateWorker` for each zip
entry, meaning a zip file with a lot of entries would take **a lot** of memory.

Lazy-loading the pako object isn't the best solution but it's the quickest.
The best solution is to lazy-load the worker list.

Mitigate the issue Stuk#446.
  • Loading branch information
dduponchel committed Aug 22, 2017
1 parent c978642 commit ed3d333
Showing 1 changed file with 29 additions and 12 deletions.
41 changes: 29 additions & 12 deletions lib/flate.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,12 @@ exports.magic = "\x08\x00";
function FlateWorker(action, options) {
GenericWorker.call(this, "FlateWorker/" + action);

this._pako = new pako[action]({
raw:true,
level : options.level || -1 // default compression
});
this._pako = null;
this._pakoAction = action;
this._pakoOptions = options;
// the `meta` object from the last chunk received
// this allow this worker to pass around metadata
this.meta = {};

var self = this;
this._pako.onData = function(data) {
self.push({
data : data,
meta : self.meta
});
};
}

utils.inherits(FlateWorker, GenericWorker);
Expand All @@ -42,6 +33,9 @@ utils.inherits(FlateWorker, GenericWorker);
*/
FlateWorker.prototype.processChunk = function (chunk) {
this.meta = chunk.meta;
if (this._pako === null) {
this._createPako();
}
this._pako.push(utils.transformTo(ARRAY_TYPE, chunk.data), false);
};

Expand All @@ -50,6 +44,9 @@ FlateWorker.prototype.processChunk = function (chunk) {
*/
FlateWorker.prototype.flush = function () {
GenericWorker.prototype.flush.call(this);
if (this._pako === null) {
this._createPako();
}
this._pako.push([], true);
};
/**
Expand All @@ -60,6 +57,26 @@ FlateWorker.prototype.cleanUp = function () {
this._pako = null;
};

/**
* Create the _pako object.
* TODO: lazy-loading this object isn't the best solution but it's the
* quickest. The best solution is to lazy-load the worker list. See also the
* issue #446.
*/
FlateWorker.prototype._createPako = function () {
this._pako = new pako[this._pakoAction]({
raw: true,
level: this._pakoOptions.level || -1 // default compression
});
var self = this;
this._pako.onData = function(data) {
self.push({
data : data,
meta : self.meta
});
};
};

exports.compressWorker = function (compressionOptions) {
return new FlateWorker("Deflate", compressionOptions);
};
Expand Down

0 comments on commit ed3d333

Please sign in to comment.