-
-
Notifications
You must be signed in to change notification settings - Fork 2.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
When a job fails, respect the
Retry-After
header if applicable
- Loading branch information
1 parent
c7873dd
commit 1f45bce
Showing
12 changed files
with
228 additions
and
30 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
// Copyright 2021 Signal Messenger, LLC | ||
// SPDX-License-Identifier: AGPL-3.0-only | ||
|
||
import type { LoggerType } from '../../logging/log'; | ||
import { sleep } from '../../util/sleep'; | ||
import { parseRetryAfter } from '../../util/parseRetryAfter'; | ||
import { isRecord } from '../../util/isRecord'; | ||
|
||
export async function sleepFor413RetryAfterTimeIfApplicable({ | ||
err, | ||
log, | ||
timeRemaining, | ||
}: Readonly<{ | ||
err: unknown; | ||
log: Pick<LoggerType, 'info'>; | ||
timeRemaining: number; | ||
}>): Promise<void> { | ||
if ( | ||
timeRemaining <= 0 || | ||
!(err instanceof Error) || | ||
err.code !== 413 || | ||
!isRecord(err.responseHeaders) | ||
) { | ||
return; | ||
} | ||
|
||
const retryAfter = Math.min( | ||
parseRetryAfter(err.responseHeaders['retry-after']), | ||
timeRemaining | ||
); | ||
|
||
log.info( | ||
`Got a 413 response code. Sleeping for ${retryAfter} millisecond(s)` | ||
); | ||
|
||
await sleep(retryAfter); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
133 changes: 133 additions & 0 deletions
133
ts/test-node/jobs/helpers/sleepFor413RetryAfterTimeIfApplicable_test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,133 @@ | ||
// Copyright 2021 Signal Messenger, LLC | ||
// SPDX-License-Identifier: AGPL-3.0-only | ||
|
||
import { assert } from 'chai'; | ||
import * as sinon from 'sinon'; | ||
import { HTTPError } from '../../../textsecure/Errors'; | ||
import * as durations from '../../../util/durations'; | ||
|
||
import { sleepFor413RetryAfterTimeIfApplicable } from '../../../jobs/helpers/sleepFor413RetryAfterTimeIfApplicable'; | ||
|
||
describe('sleepFor413RetryAfterTimeIfApplicable', () => { | ||
const createLogger = () => ({ info: sinon.spy() }); | ||
|
||
let sandbox: sinon.SinonSandbox; | ||
let clock: sinon.SinonFakeTimers; | ||
|
||
beforeEach(() => { | ||
sandbox = sinon.createSandbox(); | ||
clock = sandbox.useFakeTimers(); | ||
}); | ||
|
||
afterEach(() => { | ||
sandbox.restore(); | ||
}); | ||
|
||
it('does nothing if not passed a 413 HTTP error', async () => { | ||
const log = createLogger(); | ||
|
||
const errors = [ | ||
undefined, | ||
new Error('Normal error'), | ||
new HTTPError('Uh oh', { code: 422, headers: {}, response: {} }), | ||
]; | ||
await Promise.all( | ||
errors.map(async err => { | ||
await sleepFor413RetryAfterTimeIfApplicable({ | ||
err, | ||
log, | ||
timeRemaining: 1234, | ||
}); | ||
}) | ||
); | ||
|
||
sinon.assert.notCalled(log.info); | ||
}); | ||
|
||
it('waits for 1 second if receiving a 413 HTTP error without a Retry-After header', async () => { | ||
const err = new HTTPError('Slow down', { | ||
code: 413, | ||
headers: {}, | ||
response: {}, | ||
}); | ||
|
||
let done = false; | ||
|
||
(async () => { | ||
await sleepFor413RetryAfterTimeIfApplicable({ | ||
err, | ||
log: createLogger(), | ||
timeRemaining: 1234, | ||
}); | ||
done = true; | ||
})(); | ||
|
||
await clock.tickAsync(999); | ||
assert.isFalse(done); | ||
|
||
await clock.tickAsync(2); | ||
assert.isTrue(done); | ||
}); | ||
|
||
it('waits for Retry-After seconds if receiving a 413', async () => { | ||
const err = new HTTPError('Slow down', { | ||
code: 413, | ||
headers: { 'retry-after': '200' }, | ||
response: {}, | ||
}); | ||
|
||
let done = false; | ||
|
||
(async () => { | ||
await sleepFor413RetryAfterTimeIfApplicable({ | ||
err, | ||
log: createLogger(), | ||
timeRemaining: 123456789, | ||
}); | ||
done = true; | ||
})(); | ||
|
||
await clock.tickAsync(199 * durations.SECOND); | ||
assert.isFalse(done); | ||
|
||
await clock.tickAsync(2 * durations.SECOND); | ||
assert.isTrue(done); | ||
}); | ||
|
||
it("won't wait longer than the remaining time", async () => { | ||
const err = new HTTPError('Slow down', { | ||
code: 413, | ||
headers: { 'retry-after': '99999' }, | ||
response: {}, | ||
}); | ||
|
||
let done = false; | ||
|
||
(async () => { | ||
await sleepFor413RetryAfterTimeIfApplicable({ | ||
err, | ||
log: createLogger(), | ||
timeRemaining: 3 * durations.SECOND, | ||
}); | ||
done = true; | ||
})(); | ||
|
||
await clock.tickAsync(4 * durations.SECOND); | ||
assert.isTrue(done); | ||
}); | ||
|
||
it('logs how long it will wait', async () => { | ||
const log = createLogger(); | ||
const err = new HTTPError('Slow down', { | ||
code: 413, | ||
headers: { 'retry-after': '123' }, | ||
response: {}, | ||
}); | ||
|
||
sleepFor413RetryAfterTimeIfApplicable({ err, log, timeRemaining: 9999999 }); | ||
await clock.nextAsync(); | ||
|
||
sinon.assert.calledOnce(log.info); | ||
sinon.assert.calledWith(log.info, sinon.match(/123000 millisecond\(s\)/)); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.