From 6ce28e589add26724d645270657953cdefa88c21 Mon Sep 17 00:00:00 2001 From: John Olmsted Date: Fri, 5 Apr 2024 15:23:02 -0400 Subject: [PATCH 1/4] Default reconnect strategy uses exponential backoff and jitter Both are recommended parts of client reconnect strategies to prevent thundering herd problems when many clients lose their connection at once (for example, during a Redis upgrade). --- docs/client-configuration.md | 13 ++++++++++--- packages/client/lib/client/socket.ts | 9 +++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/client-configuration.md b/docs/client-configuration.md index 1854f07158a..770ce644298 100644 --- a/docs/client-configuration.md +++ b/docs/client-configuration.md @@ -12,7 +12,7 @@ | socket.noDelay | `true` | Toggle [`Nagle's algorithm`](https://nodejs.org/api/net.html#net_socket_setnodelay_nodelay) | | socket.keepAlive | `5000` | Toggle [`keep-alive`](https://nodejs.org/api/net.html#net_socket_setkeepalive_enable_initialdelay) functionality | | socket.tls | | See explanation and examples [below](#TLS) | -| socket.reconnectStrategy | `retries => Math.min(retries * 50, 500)` | A function containing the [Reconnect Strategy](#reconnect-strategy) logic | +| socket.reconnectStrategy | `((retries^2) * 50 ms) + 0-200 ms jitter` | A function containing the [Reconnect Strategy](#reconnect-strategy) logic | | username | | ACL username ([see ACL guide](https://redis.io/topics/acl)) | | password | | ACL password or the old "--requirepass" password | | name | | Client name ([see `CLIENT SETNAME`](https://redis.io/commands/client-setname)) | @@ -34,12 +34,19 @@ When the socket closes unexpectedly (without calling `.quit()`/`.disconnect()`), 2. `number` -> wait for `X` milliseconds before reconnecting. 3. `(retries: number, cause: Error) => false | number | Error` -> `number` is the same as configuring a `number` directly, `Error` is the same as `false`, but with a custom error. -By default the strategy is `Math.min(retries * 50, 500)`, but it can be overwritten like so: +By default the strategy uses exponential backoff, but it can be overwritten like so: ```javascript createClient({ socket: { - reconnectStrategy: retries => Math.min(retries * 50, 1000) + reconnectStrategy: retries => { + // Generate a random jitter between 0 – 200 ms: + const jitter = Math.floor(Math.random() * 200); + // Delay is an exponential back off, (times^2) * 50 ms, with a maximum value of 2000 ms: + const delay = Math.min(Math.pow(2, retries) * 50, 2000); + + return delay + jitter; + } } }); ``` diff --git a/packages/client/lib/client/socket.ts b/packages/client/lib/client/socket.ts index b701f6ea979..438e17b52c3 100644 --- a/packages/client/lib/client/socket.ts +++ b/packages/client/lib/client/socket.ts @@ -23,7 +23,7 @@ export interface RedisSocketCommonOptions { * 1. `false` -> do not reconnect, close the client and flush the command queue. * 2. `number` -> wait for `X` milliseconds before reconnecting. * 3. `(retries: number, cause: Error) => false | number | Error` -> `number` is the same as configuring a `number` directly, `Error` is the same as `false`, but with a custom error. - * Defaults to `retries => Math.min(retries * 50, 500)` + * Defaults to `((retries^2) * 50 ms) + 0-200 ms jitter` */ reconnectStrategy?: false | number | ((retries: number, cause: Error) => false | Error | number); } @@ -117,7 +117,12 @@ export default class RedisSocket extends EventEmitter { } } - return Math.min(retries * 50, 500); + // Generate a random jitter between 0 – 200 ms: + const jitter = Math.floor(Math.random() * 200); + // Delay is an exponential back off, (times^2) * 50 ms, with a maximum value of 2000 ms: + const delay = Math.min(Math.pow(2, retries) * 50, 2000); + + return delay + jitter; } #shouldReconnect(retries: number, cause: Error) { From 4a36c6fea9a5ca4acf236179d89374aa6edc67ce Mon Sep 17 00:00:00 2001 From: John Olmsted Date: Thu, 11 Apr 2024 15:33:55 -0400 Subject: [PATCH 2/4] Move default retry strategy to constant --- packages/client/lib/client/socket.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/client/lib/client/socket.ts b/packages/client/lib/client/socket.ts index b85650c6e00..046a3366612 100644 --- a/packages/client/lib/client/socket.ts +++ b/packages/client/lib/client/socket.ts @@ -82,6 +82,15 @@ export default class RedisSocket extends EventEmitter { } #createReconnectStrategy(options?: RedisSocketOptions): ReconnectStrategyFunction { + const defaultStrategy = (retries: number) => { + // Generate a random jitter between 0 – 200 ms: + const jitter = Math.floor(Math.random() * 200); + // Delay is an exponential back off, (times^2) * 50 ms, with a maximum value of 2000 ms: + const delay = Math.min(Math.pow(2, retries) * 50, 2000); + + return delay + jitter; + } + const strategy = options?.reconnectStrategy; if (strategy === false || typeof strategy === 'number') { return () => strategy; @@ -97,19 +106,12 @@ export default class RedisSocket extends EventEmitter { return retryIn; } catch (err) { this.emit('error', err); - return Math.min(retries * 50, 500); + return defaultStrategy(retries); } - - // Generate a random jitter between 0 – 200 ms: - const jitter = Math.floor(Math.random() * 200); - // Delay is an exponential back off, (times^2) * 50 ms, with a maximum value of 2000 ms: - const delay = Math.min(Math.pow(2, retries) * 50, 2000); - - return delay + jitter; }; } - return retries => Math.min(retries * 50, 500); + return defaultStrategy; } #createSocketFactory(options?: RedisSocketOptions) { From 38eb60306b6974c5950432abd0ee542d5980748e Mon Sep 17 00:00:00 2001 From: John Olmsted Date: Mon, 29 Apr 2024 11:03:49 -0400 Subject: [PATCH 3/4] Plain english explanation of default 'socket.reconnectStrategy' --- docs/client-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/client-configuration.md b/docs/client-configuration.md index f44dccc0310..deb68437e16 100644 --- a/docs/client-configuration.md +++ b/docs/client-configuration.md @@ -13,7 +13,7 @@ | socket.keepAlive | `true` | Toggle [`keep-alive`](https://nodejs.org/api/net.html#socketsetkeepaliveenable-initialdelay) functionality | | socket.keepAliveInitialDelay | `5000` | If set to a positive number, it sets the initial delay before the first keepalive probe is sent on an idle socket | | socket.tls | | See explanation and examples [below](#TLS) | -| socket.reconnectStrategy | `((retries^2) * 50 ms) + 0-200 ms` | A function containing the [Reconnect Strategy](#reconnect-strategy) logic | +| socket.reconnectStrategy | Exponential backoff with a maximum of 2000 ms; plus 0-200 ms random jitter. | A function containing the [Reconnect Strategy](#reconnect-strategy) logic | | username | | ACL username ([see ACL guide](https://redis.io/topics/acl)) | | password | | ACL password or the old "--requirepass" password | | name | | Client name ([see `CLIENT SETNAME`](https://redis.io/commands/client-setname)) | From e7304ada8ea4627f744d91c6b025b92131649110 Mon Sep 17 00:00:00 2001 From: John Olmsted Date: Mon, 29 Apr 2024 11:07:59 -0400 Subject: [PATCH 4/4] Extract default connect strategy into helper function --- packages/client/lib/client/socket.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/client/lib/client/socket.ts b/packages/client/lib/client/socket.ts index 046a3366612..3c2666e1067 100644 --- a/packages/client/lib/client/socket.ts +++ b/packages/client/lib/client/socket.ts @@ -82,15 +82,6 @@ export default class RedisSocket extends EventEmitter { } #createReconnectStrategy(options?: RedisSocketOptions): ReconnectStrategyFunction { - const defaultStrategy = (retries: number) => { - // Generate a random jitter between 0 – 200 ms: - const jitter = Math.floor(Math.random() * 200); - // Delay is an exponential back off, (times^2) * 50 ms, with a maximum value of 2000 ms: - const delay = Math.min(Math.pow(2, retries) * 50, 2000); - - return delay + jitter; - } - const strategy = options?.reconnectStrategy; if (strategy === false || typeof strategy === 'number') { return () => strategy; @@ -106,12 +97,12 @@ export default class RedisSocket extends EventEmitter { return retryIn; } catch (err) { this.emit('error', err); - return defaultStrategy(retries); + return this.defaultReconnectStrategy(retries); } }; } - return defaultStrategy; + return this.defaultReconnectStrategy; } #createSocketFactory(options?: RedisSocketOptions) { @@ -342,4 +333,13 @@ export default class RedisSocket extends EventEmitter { this.#isSocketUnrefed = true; this.#socket?.unref(); } + + defaultReconnectStrategy(retries: number) { + // Generate a random jitter between 0 – 200 ms: + const jitter = Math.floor(Math.random() * 200); + // Delay is an exponential back off, (times^2) * 50 ms, with a maximum value of 2000 ms: + const delay = Math.min(Math.pow(2, retries) * 50, 2000); + + return delay + jitter; + } }