diff --git a/packages/javascript-api/src/lib/services/graphql/__tests__/graphql-reconnect-resilience.spec.ts b/packages/javascript-api/src/lib/services/graphql/__tests__/graphql-reconnect-resilience.spec.ts new file mode 100644 index 00000000..69c5812a --- /dev/null +++ b/packages/javascript-api/src/lib/services/graphql/__tests__/graphql-reconnect-resilience.spec.ts @@ -0,0 +1,114 @@ +import { WebSocket } from 'mock-socket'; + +import { GraphQLSubscriptionsFixture } from '../__fixtures__/graphql-subscriptions-fixture'; +import * as backoff from '../../../util/randomized-exponential-backoff/randomized-exponential-backoff'; + +jest.mock('isomorphic-ws', () => WebSocket); +jest.mock('../../../util/sleep-ms/sleep-ms', () => ({ + sleepMs: () => new Promise((resolve) => setTimeout(resolve, 4)), +})); + +describe('GraphQL reconnect resilience', () => { + let fixture: GraphQLSubscriptionsFixture; + let backoffSpy: jest.SpyInstance; + + beforeEach(() => { + backoffSpy = jest.spyOn( + backoff, + 'calculateRandomizedExponentialBackoffTime', + ); + fixture = new GraphQLSubscriptionsFixture(); + }); + + afterEach(async () => { + await fixture.cleanup(); + }); + + it('should escalate reconnect backoff across repeated unstable connections', async () => { + const sub = fixture.triggerSubscription(); + + for (let cycle = 0; cycle < 3; cycle++) { + await fixture.handleConnectionInit(); + await fixture.consumeSubscribeMessage(); + await fixture.closeWithCode(1001); + fixture.openServer(); + } + + const counters = backoffSpy.mock.calls.map((call) => call[0]); + expect(counters).toEqual([0, 1, 2]); + + await fixture.handleConnectionInit(); + sub.unsubscribe(); + }); + + it('should not leak ping intervals when the connection monitor restarts', async () => { + const setIntervalSpy = jest.spyOn(global, 'setInterval'); + const clearIntervalSpy = jest.spyOn(global, 'clearInterval'); + + const sub = fixture.triggerSubscription(); + await fixture.handleConnectionInit(); + await fixture.consumeSubscribeMessage(); + + fixture.sendMessageToClient({ type: 'connection_ack' }); + await fixture.consumeSubscribeMessage(); + + const armedIds = setIntervalSpy.mock.results.map((result) => result.value); + const clearedIds = clearIntervalSpy.mock.calls.map((call) => call[0]); + const liveIntervals = armedIds.filter((id) => !clearedIds.includes(id)); + expect(liveIntervals).toHaveLength(1); + + sub.unsubscribe(); + }); + + it('should reconnect through backoff after a pong timeout, not immediately', async () => { + jest.useFakeTimers(); + const sub = fixture.triggerSubscription(); + await jest.runAllTimersAsync(); + + await fixture.handleConnectionInit(); + await jest.runOnlyPendingTimersAsync(); + await fixture.consumeSubscribeMessage(); + + await jest.advanceTimersToNextTimerAsync(); + await fixture.consumePingMessage(); + + await jest.runOnlyPendingTimersAsync(); + + expect(backoffSpy).toHaveBeenCalled(); + + jest.useRealTimers(); + sub.unsubscribe(); + }); + + it('should log how long the browser was offline when it comes back online', () => { + const service = fixture.graphqlService as any; + const warnSpy = jest.spyOn(service.logger, 'warn'); + let now = 1000; + const nowSpy = jest.spyOn(Date, 'now').mockImplementation(() => now); + + service.handleBrowserOffline(); + now = 6000; + service.handleBrowserOnline(); + + expect(warnSpy).toHaveBeenCalledWith('Browser came back online', { + offlineDurationMs: 5000, + }); + + nowSpy.mockRestore(); + }); + + it('should clear the previous pong timeout before arming a new one', () => { + const service = fixture.graphqlService as any; + service.socket = { send: jest.fn() }; + + const setTimeoutSpy = jest.spyOn(global, 'setTimeout'); + service.sendPing(); + + const clearTimeoutSpy = jest.spyOn(global, 'clearTimeout'); + service.sendPing(); + + expect(clearTimeoutSpy).toHaveBeenCalledWith( + setTimeoutSpy.mock.results[0].value, + ); + }); +}); diff --git a/packages/javascript-api/src/lib/services/graphql/graphql.service.ts b/packages/javascript-api/src/lib/services/graphql/graphql.service.ts index 48e343ec..a45c71e5 100644 --- a/packages/javascript-api/src/lib/services/graphql/graphql.service.ts +++ b/packages/javascript-api/src/lib/services/graphql/graphql.service.ts @@ -184,6 +184,12 @@ export class GraphqlService { private pingPongInterval: any; private readonly sendPingWithThisBound = this.sendPing.bind(this); + private offlineSince: number | null = null; + private readonly handleBrowserOfflineWithThisBound = + this.handleBrowserOffline.bind(this); + private readonly handleBrowserOnlineWithThisBound = + this.handleBrowserOnline.bind(this); + private connectionAttemptsCount = 0; constructor() { @@ -488,8 +494,6 @@ export class GraphqlService { break; case MessageType.GQL_CONNECTION_ACK: { - this.connectionAttemptsCount = 0; - this.clearErroredSubscriptionsTimeouts(); this.retryableErroredSubscriptionsRetryCount = 0; this.retryableErroredSubscriptionsAction$.next({ type: 'clear' }); @@ -553,6 +557,7 @@ export class GraphqlService { case MessageType.GQL_PONG: clearTimeout(this.pongTimeout); + this.connectionAttemptsCount = 0; break; case MessageType.GQL_ERROR: { @@ -659,23 +664,59 @@ export class GraphqlService { private startConnectionMonitoring(): void { this.monitorWithPingPong(); - this.monitorWithOfflineEvent(); + this.monitorWithOnlineOfflineEvents(); } private monitorWithPingPong(): void { + clearInterval(this.pingPongInterval); this.pingPongInterval = setInterval(() => { this.sendPing(); }, PING_PONG_INTERVAL_IN_MS); } - private monitorWithOfflineEvent(): void { + private monitorWithOnlineOfflineEvents(): void { if (typeof window !== 'undefined') { window.removeEventListener('offline', this.sendPingWithThisBound); window.addEventListener('offline', this.sendPingWithThisBound); + + window.removeEventListener( + 'offline', + this.handleBrowserOfflineWithThisBound, + ); + window.addEventListener( + 'offline', + this.handleBrowserOfflineWithThisBound, + ); + + window.removeEventListener( + 'online', + this.handleBrowserOnlineWithThisBound, + ); + window.addEventListener('online', this.handleBrowserOnlineWithThisBound); } } + private handleBrowserOffline(): void { + if (this.offlineSince !== null) { + return; + } + this.offlineSince = Date.now(); + this.logger.warn('Browser went offline'); + } + + private handleBrowserOnline(): void { + if (this.offlineSince === null) { + this.logger.warn('Browser came online'); + return; + } + + const offlineDurationMs = Date.now() - this.offlineSince; + this.offlineSince = null; + this.logger.warn('Browser came back online', { offlineDurationMs }); + } + private sendPing(): void { + clearTimeout(this.pongTimeout); this.pongTimeout = setTimeout(() => { this.handleConnectionDrop().catch((error: Error) => { this.logger.error('Failed to handle pong connection drop: ', error); @@ -698,7 +739,6 @@ export class GraphqlService { this.setConnectionStatus(ConnectionStatus.DISCONNECTED); this.clearPingMonitoring(); - await this.openSocket(); }