Skip to content
Permalink
Browse files

zlib: add brotli support

Refs: #20458

Co-authored-by: Hackzzila <admin@hackzzila.com>

PR-URL: #24938
Reviewed-By: Ruben Bridgewater <ruben@bridgewater.de>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Jan Krems <jan.krems@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Myles Borins <myles.borins@gmail.com>
Reviewed-By: Ali Ijaz Sheikh <ofrobots@google.com>
Reviewed-By: Daniel Bevenius <daniel.bevenius@gmail.com>
  • Loading branch information...
addaleax committed Nov 27, 2018
1 parent ec87b6c commit 7edf8c7e74cf36c27625b2661d884d947a4fa139
Showing with 540 additions and 37 deletions.
  1. +10 −0 doc/api/errors.md
  2. +1 −0 lib/internal/errors.js
  3. +89 −5 lib/zlib.js
  4. +1 −0 node.gyp
  5. +4 −0 node.gypi
  6. +8 −0 src/node_metadata.cc
  7. +1 −0 src/node_metadata.h
  8. +424 −31 src/node_zlib.cc
  9. +2 −1 test/parallel/test-process-versions.js
@@ -624,6 +624,16 @@ An attempt was made to register something that is not a function as an
The type of an asynchronous resource was invalid. Note that users are also able
to define their own types if using the public embedder API.

<a id="ERR_BROTLI_INVALID_PARAM"></a>
### ERR_BROTLI_INVALID_PARAM

An invalid parameter key was passed during construction of a Brotli stream.

<a id="ERR_BROTLI_COMPRESSION_FAILED">
### ERR_BROTLI_COMPRESSION_FAILED

Data passed to a Brotli stream was not successfully compressed.

<a id="ERR_BUFFER_CONTEXT_NOT_AVAILABLE"></a>
### ERR_BUFFER_CONTEXT_NOT_AVAILABLE

@@ -546,6 +546,7 @@ E('ERR_ARG_NOT_ITERABLE', '%s must be iterable', TypeError);
E('ERR_ASSERTION', '%s', Error);
E('ERR_ASYNC_CALLBACK', '%s must be a function', TypeError);
E('ERR_ASYNC_TYPE', 'Invalid name for async "type": %s', TypeError);
E('ERR_BROTLI_INVALID_PARAM', '%s is not a valid Brotli parameter', RangeError);
E('ERR_BUFFER_OUT_OF_BOUNDS',
// Using a default argument here is important so the argument is not counted
// towards `Function#length`.
@@ -22,10 +22,11 @@
'use strict';

const {
ERR_BROTLI_INVALID_PARAM,
ERR_BUFFER_TOO_LARGE,
ERR_INVALID_ARG_TYPE,
ERR_OUT_OF_RANGE,
ERR_ZLIB_INITIALIZATION_FAILED
ERR_ZLIB_INITIALIZATION_FAILED,
} = require('internal/errors').codes;
const Transform = require('_stream_transform');
const {
@@ -47,11 +48,18 @@ const { owner_symbol } = require('internal/async_hooks').symbols;

const constants = internalBinding('constants').zlib;
const {
// Zlib flush levels
Z_NO_FLUSH, Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, Z_FULL_FLUSH, Z_FINISH,
// Zlib option values
Z_MIN_CHUNK, Z_MIN_WINDOWBITS, Z_MAX_WINDOWBITS, Z_MIN_LEVEL, Z_MAX_LEVEL,
Z_MIN_MEMLEVEL, Z_MAX_MEMLEVEL, Z_DEFAULT_CHUNK, Z_DEFAULT_COMPRESSION,
Z_DEFAULT_STRATEGY, Z_DEFAULT_WINDOWBITS, Z_DEFAULT_MEMLEVEL, Z_FIXED,
DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP
// Node's compression stream modes (node_zlib_mode)
DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP,
BROTLI_DECODE, BROTLI_ENCODE,
// Brotli operations (~flush levels)
BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FLUSH,
BROTLI_OPERATION_FINISH
} = constants;

// translation table for return codes.
@@ -212,7 +220,7 @@ function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
// The ZlibBase class is not exported to user land, the mode should only be
// passed in by us.
assert(typeof mode === 'number');
assert(mode >= DEFLATE && mode <= UNZIP);
assert(mode >= DEFLATE && mode <= BROTLI_ENCODE);

if (opts) {
chunkSize = opts.chunkSize;
@@ -481,7 +489,7 @@ function processCallback() {
// important to null out the values once they are no longer needed since
// `_handle` can stay in memory long after the buffer is needed.
var handle = this;
var self = this.jsref;
var self = this[owner_symbol];
var state = self._writeState;

if (self._hadError) {
@@ -622,6 +630,9 @@ function Zlib(opts, mode) {
this._writeState,
processCallback,
dictionary)) {
// TODO(addaleax): Sometimes we generate better error codes in C++ land,
// e.g. ERR_BROTLI_PARAM_SET_FAILED -- it's hard to access them with
// the current bindings setup, though.
throw new ERR_ZLIB_INITIALIZATION_FAILED();
}

@@ -726,6 +737,70 @@ function createConvenienceMethod(ctor, sync) {
}
}

const kMaxBrotliParam = Math.max(...Object.keys(constants).map((key) => {
return key.startsWith('BROTLI_PARAM_') ? constants[key] : 0;
}));

const brotliInitParamsArray = new Uint32Array(kMaxBrotliParam + 1);

const brotliDefaultOpts = {
flush: BROTLI_OPERATION_PROCESS,
finishFlush: BROTLI_OPERATION_FINISH,
fullFlush: BROTLI_OPERATION_FLUSH
};
function Brotli(opts, mode) {
assert(mode === BROTLI_DECODE || mode === BROTLI_ENCODE);

brotliInitParamsArray.fill(-1);
if (opts && opts.params) {
for (const origKey of Object.keys(opts.params)) {
const key = +origKey;
if (Number.isNaN(key) || key < 0 || key > kMaxBrotliParam ||
(brotliInitParamsArray[key] | 0) !== -1) {
throw new ERR_BROTLI_INVALID_PARAM(origKey);
}

const value = opts.params[origKey];
if (typeof value !== 'number' && typeof value !== 'boolean') {
throw new ERR_INVALID_ARG_TYPE('options.params[key]',
'number', opts.params[origKey]);
}
brotliInitParamsArray[key] = value;
}
}

const handle = mode === BROTLI_DECODE ?
new binding.BrotliDecoder(mode) : new binding.BrotliEncoder(mode);

this._writeState = new Uint32Array(2);
if (!handle.init(brotliInitParamsArray,
this._writeState,
processCallback)) {
throw new ERR_ZLIB_INITIALIZATION_FAILED();
}

ZlibBase.call(this, opts, mode, handle, brotliDefaultOpts);
}
Object.setPrototypeOf(Brotli.prototype, Zlib.prototype);
Object.setPrototypeOf(Brotli, Zlib);

function BrotliCompress(opts) {
if (!(this instanceof BrotliCompress))
return new BrotliCompress(opts);
Brotli.call(this, opts, BROTLI_ENCODE);
}
Object.setPrototypeOf(BrotliCompress.prototype, Brotli.prototype);
Object.setPrototypeOf(BrotliCompress, Brotli);

function BrotliDecompress(opts) {
if (!(this instanceof BrotliDecompress))
return new BrotliDecompress(opts);
Brotli.call(this, opts, BROTLI_DECODE);
}
Object.setPrototypeOf(BrotliDecompress.prototype, Brotli.prototype);
Object.setPrototypeOf(BrotliDecompress, Brotli);


function createProperty(ctor) {
return {
configurable: true,
@@ -751,6 +826,8 @@ module.exports = {
DeflateRaw,
InflateRaw,
Unzip,
BrotliCompress,
BrotliDecompress,

// Convenience methods.
// compress/decompress a string or buffer in one step.
@@ -767,7 +844,11 @@ module.exports = {
gunzip: createConvenienceMethod(Gunzip, false),
gunzipSync: createConvenienceMethod(Gunzip, true),
inflateRaw: createConvenienceMethod(InflateRaw, false),
inflateRawSync: createConvenienceMethod(InflateRaw, true)
inflateRawSync: createConvenienceMethod(InflateRaw, true),
brotliCompress: createConvenienceMethod(BrotliCompress, false),
brotliCompressSync: createConvenienceMethod(BrotliCompress, true),
brotliDecompress: createConvenienceMethod(BrotliDecompress, false),
brotliDecompressSync: createConvenienceMethod(BrotliDecompress, true),
};

Object.defineProperties(module.exports, {
@@ -778,6 +859,8 @@ Object.defineProperties(module.exports, {
createGzip: createProperty(Gzip),
createGunzip: createProperty(Gunzip),
createUnzip: createProperty(Unzip),
createBrotliCompress: createProperty(BrotliCompress),
createBrotliDecompress: createProperty(BrotliDecompress),
constants: {
configurable: false,
enumerable: true,
@@ -795,6 +878,7 @@ Object.defineProperties(module.exports, {
const bkeys = Object.keys(constants);
for (var bk = 0; bk < bkeys.length; bk++) {
var bkey = bkeys[bk];
if (bkey.startsWith('BROTLI')) continue;
Object.defineProperty(module.exports, bkey, {
enumerable: true, value: constants[bkey], writable: false
});
@@ -11,6 +11,7 @@
'node_shared%': 'false',
'force_dynamic_crt%': 0,
'node_module_version%': '',
'node_shared_brotli%': 'false',
'node_shared_zlib%': 'false',
'node_experimental_http_parser%': 'false',
'node_shared_http_parser%': 'false',
@@ -212,6 +212,10 @@
'dependencies': [ 'deps/nghttp2/nghttp2.gyp:nghttp2' ],
}],

[ 'node_shared_brotli=="false"', {
'dependencies': [ 'deps/brotli/brotli.gyp:brotli' ],
}],

[ 'OS=="mac"', {
# linking Corefoundation is needed since certain OSX debugging tools
# like Instruments require it for some features
@@ -1,5 +1,6 @@
#include "node_metadata.h"
#include "ares.h"
#include "brotli/encode.h"
#include "nghttp2/nghttp2ver.h"
#include "node.h"
#include "node_internals.h"
@@ -30,6 +31,13 @@ Metadata::Versions::Versions() {
llhttp = llhttp_version;
http_parser = http_parser_version;

brotli =
std::to_string(BrotliEncoderVersion() >> 24) +
"." +
std::to_string((BrotliEncoderVersion() & 0xFFF000) >> 12) +
"." +
std::to_string(BrotliEncoderVersion() & 0xFFF);

#if HAVE_OPENSSL
openssl = crypto::GetOpenSSLVersion();
#endif
@@ -12,6 +12,7 @@ namespace node {
V(v8) \
V(uv) \
V(zlib) \
V(brotli) \
V(ares) \
V(modules) \
V(nghttp2) \

0 comments on commit 7edf8c7

Please sign in to comment.
You can’t perform that action at this time.