-
-
Notifications
You must be signed in to change notification settings - Fork 5.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Migrate request to got (part 1 of many) (#6160)
* install new packages * migrate request to got * update dynamic json test This is a behavioural difference between request and got request will send the request, then we'll get a `400 Bad Request` back and re-throw at as invalid got will pick up that the URL is invalid and throw `RequestError: URI malformed` before attempting to send it which we'll re-throw as inaccessible * fix OPM service * fix wordpress querystring Got doesn't natively support assmebling a querystring from nested objects because it uses node's URLSearchParams internally. Use qs and pass qs a string. Wordpress is the only service that needs this, so we could build the string manually in this case if we don't want to take qs as a prod dependency. It is mostly hard-coded values anyway. * fix wercker got overwrites any ?foo=bar in the URL string if searchParams is also passed whereas request appends see https://github.com/sindresorhus/got#url * fix keybase * add tests for got wrapper * bootstrap global agent in server start
- Loading branch information
Showing
15 changed files
with
381 additions
and
96 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
'use strict' | ||
|
||
const got = require('got') | ||
const { Inaccessible, InvalidResponse } = require('./errors') | ||
|
||
function requestOptions2GotOptions(options) { | ||
const requestOptions = Object.assign({}, options) | ||
const gotOptions = {} | ||
const interchangableOptions = ['body', 'form', 'headers', 'method', 'url'] | ||
|
||
interchangableOptions.forEach(function (opt) { | ||
if (opt in requestOptions) { | ||
gotOptions[opt] = requestOptions[opt] | ||
delete requestOptions[opt] | ||
} | ||
}) | ||
|
||
if ('qs' in requestOptions) { | ||
gotOptions.searchParams = requestOptions.qs | ||
delete requestOptions.qs | ||
} | ||
|
||
if ('gzip' in requestOptions) { | ||
gotOptions.decompress = requestOptions.gzip | ||
delete requestOptions.gzip | ||
} | ||
|
||
if ('strictSSL' in requestOptions) { | ||
gotOptions.https = { | ||
rejectUnauthorized: requestOptions.strictSSL, | ||
} | ||
delete requestOptions.strictSSL | ||
} | ||
|
||
if ('auth' in requestOptions) { | ||
gotOptions.username = requestOptions.auth.user | ||
gotOptions.password = requestOptions.auth.pass | ||
delete requestOptions.auth | ||
} | ||
|
||
if (Object.keys(requestOptions).length > 0) { | ||
throw new Error(`Found unrecognised options ${Object.keys(requestOptions)}`) | ||
} | ||
|
||
return gotOptions | ||
} | ||
|
||
async function sendRequest(gotWrapper, url, options) { | ||
const gotOptions = requestOptions2GotOptions(options) | ||
gotOptions.throwHttpErrors = false | ||
gotOptions.retry = 0 | ||
try { | ||
const resp = await gotWrapper(url, gotOptions) | ||
return { res: resp, buffer: resp.body } | ||
} catch (err) { | ||
if (err instanceof got.CancelError) { | ||
throw new InvalidResponse({ | ||
underlyingError: new Error('Maximum response size exceeded'), | ||
}) | ||
} | ||
throw new Inaccessible({ underlyingError: err }) | ||
} | ||
} | ||
|
||
function fetchFactory(fetchLimitBytes) { | ||
const gotWithLimit = got.extend({ | ||
handlers: [ | ||
(options, next) => { | ||
const promiseOrStream = next(options) | ||
promiseOrStream.on('downloadProgress', progress => { | ||
if ( | ||
progress.transferred > fetchLimitBytes && | ||
// just accept the file if we've already finished downloading | ||
// the entire file before we went over the limit | ||
progress.percent !== 1 | ||
) { | ||
/* | ||
TODO: we should be able to pass cancel() a message | ||
https://github.com/sindresorhus/got/blob/main/documentation/advanced-creation.md#examples | ||
but by the time we catch it, err.message is just "Promise was canceled" | ||
*/ | ||
promiseOrStream.cancel('Maximum response size exceeded') | ||
} | ||
}) | ||
|
||
return promiseOrStream | ||
}, | ||
], | ||
}) | ||
|
||
return sendRequest.bind(sendRequest, gotWithLimit) | ||
} | ||
|
||
module.exports = { | ||
requestOptions2GotOptions, | ||
fetchFactory, | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
'use strict' | ||
|
||
const { expect } = require('chai') | ||
const nock = require('nock') | ||
const { requestOptions2GotOptions, fetchFactory } = require('./got') | ||
const { Inaccessible, InvalidResponse } = require('./errors') | ||
|
||
describe('requestOptions2GotOptions function', function () { | ||
it('translates valid options', function () { | ||
expect( | ||
requestOptions2GotOptions({ | ||
body: 'body', | ||
form: 'form', | ||
headers: 'headers', | ||
method: 'method', | ||
url: 'url', | ||
qs: 'qs', | ||
gzip: 'gzip', | ||
strictSSL: 'strictSSL', | ||
auth: { user: 'user', pass: 'pass' }, | ||
}) | ||
).to.deep.equal({ | ||
body: 'body', | ||
form: 'form', | ||
headers: 'headers', | ||
method: 'method', | ||
url: 'url', | ||
searchParams: 'qs', | ||
decompress: 'gzip', | ||
https: { rejectUnauthorized: 'strictSSL' }, | ||
username: 'user', | ||
password: 'pass', | ||
}) | ||
}) | ||
|
||
it('throws if unrecognised options are found', function () { | ||
expect(() => | ||
requestOptions2GotOptions({ body: 'body', foobar: 'foobar' }) | ||
).to.throw(Error, 'Found unrecognised options foobar') | ||
}) | ||
}) | ||
|
||
describe('got wrapper', function () { | ||
it('should not throw an error if the response <= fetchLimitBytes', async function () { | ||
nock('https://www.google.com') | ||
.get('/foo/bar') | ||
.once() | ||
.reply(200, 'x'.repeat(100)) | ||
const sendRequest = fetchFactory(100) | ||
const { res } = await sendRequest('https://www.google.com/foo/bar') | ||
expect(res.statusCode).to.equal(200) | ||
}) | ||
|
||
it('should throw an InvalidResponse error if the response is > fetchLimitBytes', async function () { | ||
nock('https://www.google.com') | ||
.get('/foo/bar') | ||
.once() | ||
.reply(200, 'x'.repeat(101)) | ||
const sendRequest = fetchFactory(100) | ||
return expect( | ||
sendRequest('https://www.google.com/foo/bar') | ||
).to.be.rejectedWith(InvalidResponse, 'Maximum response size exceeded') | ||
}) | ||
|
||
it('should throw an Inaccessible error if the request throws a (non-HTTP) error', async function () { | ||
nock('https://www.google.com').get('/foo/bar').replyWithError('oh no') | ||
const sendRequest = fetchFactory(1024) | ||
return expect( | ||
sendRequest('https://www.google.com/foo/bar') | ||
).to.be.rejectedWith(Inaccessible, 'oh no') | ||
}) | ||
|
||
it('should throw an Inaccessible error if the host can not be accessed', async function () { | ||
this.timeout(5000) | ||
nock.disableNetConnect() | ||
const sendRequest = fetchFactory(1024) | ||
return expect( | ||
sendRequest('https://www.google.com/foo/bar') | ||
).to.be.rejectedWith( | ||
Inaccessible, | ||
'Nock: Disallowed net connect for "www.google.com:443/foo/bar"' | ||
) | ||
}) | ||
|
||
afterEach(function () { | ||
nock.cleanAll() | ||
nock.enableNetConnect() | ||
}) | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.