From a480e09d941f5a78e17acfa8f1d24760fd18feca Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Fri, 10 Feb 2023 16:10:42 +0100 Subject: [PATCH] stream: bump default highWaterMark This should give a performance boost accross the board. Given that the old limit is a decod old and memory capacity has doubled many times since I think it is appropriate to slightly bump the default limit. --- doc/api/stream.md | 4 ++-- lib/internal/streams/state.js | 2 +- test/parallel/test-https-hwm.js | 2 +- test/parallel/test-stream-duplex-readable-end.js | 2 ++ .../test-stream-pipe-await-drain-push-while-write.js | 2 ++ test/parallel/test-stream-readable-infinite-read.js | 1 + .../parallel/test-stream-transform-split-highwatermark.js | 4 ++-- test/parallel/test-stream-transform-split-objectmode.js | 8 +++++--- test/parallel/test-tls-connect-hwm-option.js | 2 +- 9 files changed, 17 insertions(+), 10 deletions(-) diff --git a/doc/api/stream.md b/doc/api/stream.md index f01b6ad6e1f727..a6eda71844f3f7 100644 --- a/doc/api/stream.md +++ b/doc/api/stream.md @@ -3485,7 +3485,7 @@ changes: * `options` {Object} * `highWaterMark` {number} Buffer level when [`stream.write()`][stream-write] starts returning `false`. **Default:** - `16384` (16 KiB), or `16` for `objectMode` streams. + `65536` (64 KiB), or `16` for `objectMode` streams. * `decodeStrings` {boolean} Whether to encode `string`s passed to [`stream.write()`][stream-write] to `Buffer`s (with the encoding specified in the [`stream.write()`][stream-write] call) before passing @@ -3858,7 +3858,7 @@ changes: * `options` {Object} * `highWaterMark` {number} The maximum [number of bytes][hwm-gotcha] to store in the internal buffer before ceasing to read from the underlying resource. - **Default:** `16384` (16 KiB), or `16` for `objectMode` streams. + **Default:** `65536` (64 KiB), or `16` for `objectMode` streams. * `encoding` {string} If specified, then buffers will be decoded to strings using the specified encoding. **Default:** `null`. * `objectMode` {boolean} Whether this stream should behave diff --git a/lib/internal/streams/state.js b/lib/internal/streams/state.js index 98f8d8a6cc33d5..97affc98bc6a8b 100644 --- a/lib/internal/streams/state.js +++ b/lib/internal/streams/state.js @@ -8,7 +8,7 @@ const { validateInteger } = require('internal/validators'); const { ERR_INVALID_ARG_VALUE } = require('internal/errors').codes; -let defaultHighWaterMarkBytes = 16 * 1024; +let defaultHighWaterMarkBytes = 64 * 1024; let defaultHighWaterMarkObjectMode = 16; function highWaterMarkFrom(options, isDuplex, duplexKey) { diff --git a/test/parallel/test-https-hwm.js b/test/parallel/test-https-hwm.js index 71ee33b7086861..b3f8960195e03d 100644 --- a/test/parallel/test-https-hwm.js +++ b/test/parallel/test-https-hwm.js @@ -62,5 +62,5 @@ const httpsServer = https.createServer({ port: this.address().port, rejectUnauthorized: false, highWaterMark: undefined, - }, loadCallback(16 * 1024)).on('error', common.mustNotCall()).end(); + }, loadCallback(64 * 1024)).on('error', common.mustNotCall()).end(); })); diff --git a/test/parallel/test-stream-duplex-readable-end.js b/test/parallel/test-stream-duplex-readable-end.js index 0e3e62aacb14bc..3b1d4d21ce0a39 100644 --- a/test/parallel/test-stream-duplex-readable-end.js +++ b/test/parallel/test-stream-duplex-readable-end.js @@ -7,6 +7,7 @@ const stream = require('stream'); let loops = 5; const src = new stream.Readable({ + highWaterMark: 16 * 1024, read() { if (loops--) this.push(Buffer.alloc(20000)); @@ -14,6 +15,7 @@ const src = new stream.Readable({ }); const dst = new stream.Transform({ + highWaterMark: 16 * 1024, transform(chunk, output, fn) { this.push(null); fn(); diff --git a/test/parallel/test-stream-pipe-await-drain-push-while-write.js b/test/parallel/test-stream-pipe-await-drain-push-while-write.js index a717291cda2b03..089767166c99b5 100644 --- a/test/parallel/test-stream-pipe-await-drain-push-while-write.js +++ b/test/parallel/test-stream-pipe-await-drain-push-while-write.js @@ -4,6 +4,7 @@ const stream = require('stream'); const assert = require('assert'); const writable = new stream.Writable({ + highWaterMark: 16 * 1024, write: common.mustCall(function(chunk, encoding, cb) { assert.strictEqual( readable._readableState.awaitDrainWriters, @@ -26,6 +27,7 @@ const writable = new stream.Writable({ // A readable stream which produces two buffers. const bufs = [Buffer.alloc(32 * 1024), Buffer.alloc(33 * 1024)]; // above hwm const readable = new stream.Readable({ + highWaterMark: 16 * 1024, read: function() { while (bufs.length > 0) { this.push(bufs.shift()); diff --git a/test/parallel/test-stream-readable-infinite-read.js b/test/parallel/test-stream-readable-infinite-read.js index 3df3e39a73ec36..df88d78b74c36f 100644 --- a/test/parallel/test-stream-readable-infinite-read.js +++ b/test/parallel/test-stream-readable-infinite-read.js @@ -7,6 +7,7 @@ const { Readable } = require('stream'); const buf = Buffer.alloc(8192); const readable = new Readable({ + highWaterMark: 16 * 1024, read: common.mustCall(function() { this.push(buf); }, 31) diff --git a/test/parallel/test-stream-transform-split-highwatermark.js b/test/parallel/test-stream-transform-split-highwatermark.js index b6255c704710ac..290c7d957cc2d7 100644 --- a/test/parallel/test-stream-transform-split-highwatermark.js +++ b/test/parallel/test-stream-transform-split-highwatermark.js @@ -2,9 +2,9 @@ require('../common'); const assert = require('assert'); -const { Transform, Readable, Writable } = require('stream'); +const { Transform, Readable, Writable, getDefaultHighWaterMark } = require('stream'); -const DEFAULT = 16 * 1024; +const DEFAULT = getDefaultHighWaterMark(); function testTransform(expectedReadableHwm, expectedWritableHwm, options) { const t = new Transform(options); diff --git a/test/parallel/test-stream-transform-split-objectmode.js b/test/parallel/test-stream-transform-split-objectmode.js index 55d613573493b9..c637056534ab19 100644 --- a/test/parallel/test-stream-transform-split-objectmode.js +++ b/test/parallel/test-stream-transform-split-objectmode.js @@ -25,12 +25,14 @@ const assert = require('assert'); const Transform = require('stream').Transform; -const parser = new Transform({ readableObjectMode: true }); +const parser = new Transform({ + readableObjectMode: true +}); assert(parser._readableState.objectMode); assert(!parser._writableState.objectMode); assert.strictEqual(parser.readableHighWaterMark, 16); -assert.strictEqual(parser.writableHighWaterMark, 16 * 1024); +assert.strictEqual(parser.writableHighWaterMark, 64 * 1024); assert.strictEqual(parser.readableHighWaterMark, parser._readableState.highWaterMark); assert.strictEqual(parser.writableHighWaterMark, @@ -57,7 +59,7 @@ const serializer = new Transform({ writableObjectMode: true }); assert(!serializer._readableState.objectMode); assert(serializer._writableState.objectMode); -assert.strictEqual(serializer.readableHighWaterMark, 16 * 1024); +assert.strictEqual(serializer.readableHighWaterMark, 64 * 1024); assert.strictEqual(serializer.writableHighWaterMark, 16); assert.strictEqual(parser.readableHighWaterMark, parser._readableState.highWaterMark); diff --git a/test/parallel/test-tls-connect-hwm-option.js b/test/parallel/test-tls-connect-hwm-option.js index e016ccc6cba0dc..e1fb6c187bc783 100644 --- a/test/parallel/test-tls-connect-hwm-option.js +++ b/test/parallel/test-tls-connect-hwm-option.js @@ -37,7 +37,7 @@ server.listen(0, common.mustCall(() => { rejectUnauthorized: false, highWaterMark: undefined, }, common.mustCall(() => { - assert.strictEqual(defaultHighBob.readableHighWaterMark, 16 * 1024); + assert.strictEqual(defaultHighBob.readableHighWaterMark, 64 * 1024); defaultHighBob.end(); }));