diff --git a/index.js b/index.js index 09ea52afb..8aee73084 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,6 @@ const Transform = require('stream').Transform; const urlLib = require('url'); const fs = require('fs'); const querystring = require('querystring'); -const CacheableRequest = require('cacheable-request'); const duplexer3 = require('duplexer3'); const intoStream = require('into-stream'); const isStream = require('is-stream'); @@ -94,8 +93,7 @@ function requestAsEventEmitter(opts) { let progressInterval; - const cacheableRequest = new CacheableRequest(fn.request, opts.cache); - const cacheReq = cacheableRequest(opts, res => { + const req = fn.request(opts, res => { clearInterval(progressInterval); ee.emit('uploadProgress', { @@ -180,7 +178,7 @@ function requestAsEventEmitter(opts) { const response = opts.decompress === true && typeof decompressResponse === 'function' && - opts.method !== 'HEAD' ? decompressResponse(progressStream) : progressStream; + req.method !== 'HEAD' ? decompressResponse(progressStream) : progressStream; if (!opts.decompress && ['gzip', 'deflate'].indexOf(res.headers['content-encoding']) !== -1) { opts.encoding = null; @@ -198,66 +196,62 @@ function requestAsEventEmitter(opts) { }); }); - cacheReq.on('error', err => ee.emit('error', new got.CacheError(err, opts))); + req.once('error', err => { + clearInterval(progressInterval); - cacheReq.on('request', req => { - req.once('error', err => { - clearInterval(progressInterval); + const backoff = opts.retries(++retryCount, err); - const backoff = opts.retries(++retryCount, err); + if (backoff) { + setTimeout(get, backoff, opts); + return; + } - if (backoff) { - setTimeout(get, backoff, opts); - return; - } + ee.emit('error', new got.RequestError(err, opts)); + }); - ee.emit('error', new got.RequestError(err, opts)); + ee.on('request', req => { + ee.emit('uploadProgress', { + percent: 0, + transferred: 0, + total: uploadBodySize }); - ee.on('request', req => { - ee.emit('uploadProgress', { - percent: 0, - transferred: 0, - total: uploadBodySize - }); + req.connection.once('connect', () => { + const uploadEventFrequency = 150; - req.connection.once('connect', () => { - const uploadEventFrequency = 150; + progressInterval = setInterval(() => { + const lastUploaded = uploaded; + const headersSize = Buffer.byteLength(req._header); + uploaded = req.connection.bytesWritten - headersSize; - progressInterval = setInterval(() => { - const lastUploaded = uploaded; - const headersSize = Buffer.byteLength(req._header); - uploaded = req.connection.bytesWritten - headersSize; - - // Prevent the known issue of `bytesWritten` being larger than body size - if (uploadBodySize && uploaded > uploadBodySize) { - uploaded = uploadBodySize; - } + // Prevent the known issue of `bytesWritten` being larger than body size + if (uploadBodySize && uploaded > uploadBodySize) { + uploaded = uploadBodySize; + } - // Don't emit events with unchanged progress and - // prevent last event from being emitted, because - // it's emitted when `response` is emitted - if (uploaded === lastUploaded || uploaded === uploadBodySize) { - return; - } + // Don't emit events with unchanged progress and + // prevent last event from being emitted, because + // it's emitted when `response` is emitted + if (uploaded === lastUploaded || uploaded === uploadBodySize) { + return; + } - ee.emit('uploadProgress', { - percent: uploadBodySize ? uploaded / uploadBodySize : 0, - transferred: uploaded, - total: uploadBodySize - }); - }, uploadEventFrequency); - }); + ee.emit('uploadProgress', { + percent: uploadBodySize ? uploaded / uploadBodySize : 0, + transferred: uploaded, + total: uploadBodySize + }); + }, uploadEventFrequency); }); + }); - if (opts.gotTimeout) { - clearInterval(progressInterval); - timedOut(req, opts.gotTimeout); - } + if (opts.gotTimeout) { + clearInterval(progressInterval); + timedOut(req, opts.gotTimeout); + } - setImmediate(() => { - ee.emit('request', req); - }); + setImmediate(() => { + ee.emit('request', req); }); }; @@ -447,7 +441,6 @@ function normalizeArguments(url, opts) { { path: '', retries: 2, - cache: false, decompress: true, useElectronNet: false }, @@ -603,13 +596,6 @@ class StdError extends Error { } } -got.CacheError = class extends StdError { - constructor(error, opts) { - super(error.message, error, opts); - this.name = 'CacheError'; - } -}; - got.RequestError = class extends StdError { constructor(error, opts) { super(error.message, error, opts); diff --git a/package.json b/package.json index 61193777f..07c7ace64 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,6 @@ "electron" ], "dependencies": { - "cacheable-request": "^2.0.0", "decompress-response": "^3.2.0", "duplexer3": "^0.1.4", "get-stream": "^3.0.0", diff --git a/readme.md b/readme.md index 7e070a4f9..3630b5a1b 100644 --- a/readme.md +++ b/readme.md @@ -19,7 +19,6 @@ Created because [`request`](https://github.com/request/request) is bloated *(sev - [Promise & stream API](#api) - [Request cancelation](#aborting-the-request) -- [RFC compliant caching](#cache-adapters) - [Follows redirects](#followredirect) - [Retries on network failure](#retries) - [Progress events](#onuploadprogress-progress) @@ -70,10 +69,6 @@ It's a `GET` request by default, but can be changed in `options`. Returns a Promise for a `response` object with a `body` property, a `url` property with the request URL or the final URL after redirects, and a `requestUrl` property with the original request URL. -The response object will normally be a [Node.js HTTP response stream](https://nodejs.org/api/http.html#http_class_http_incomingmessage), however if returned from the cache it will be a [responselike object](https://github.com/lukechilds/responselike) which behaves in the same way. - -The response will also have a `fromCache` property set with a boolean value. - ##### url Type: `string` `Object` @@ -177,13 +172,6 @@ Decompress the response automatically. If this is disabled, a compressed response is returned as a `Buffer`. This may be useful if you want to handle decompression yourself or stream the raw compressed data. -###### cache - -Type: `Object`
-Default: `false` - -[Cache adapter instance](#cache-adapters) for storing cached data. - ###### useElectronNet Type: `boolean`
@@ -267,10 +255,6 @@ Each error contains (if available) `statusCode`, `statusMessage`, `host`, `hostn In Promise mode, the `response` is attached to the error. -#### got.CacheError - -When a cache method fails, for example if the database goes down, or there's a filesystem error. - #### got.RequestError When a request fails. Contains a `code` property with error class code, like `ECONNREFUSED`. @@ -335,59 +319,6 @@ request.cancel(); ``` - -## Cache - -You can use the JavaScript `Map` type as an in memory cache: - -```js -const got = require('got'); -const map = new Map(); - -(async () => { - let response = await got('sindresorhus.com', {cache: map}); - console.log(response.fromCache); - //=> false - - response = await got('sindresorhus.com', {cache: map}); - console.log(response.fromCache); - //=> true -})(); -``` - -Got uses [Keyv](https://github.com/lukechilds/keyv) internally to support a wide range of storage adapters. For something more scalable you could use an [official Keyv storage adapter](https://github.com/lukechilds/keyv#official-storage-adapters): - -``` -$ npm install @keyv/redis -``` - -```js -const got = require('got'); -const KeyvRedis = require('@keyv/redis'); - -const redis = new KeyvRedis('redis://user:pass@localhost:6379'); - -got('sindresorhus.com', {cache: redis}); -``` - -Got supports anything that follows the Map API, so it's easy to write your own storage adapter or use a third-party solution. - -For example, the following are all valid storage adapters: - -```js -const storageAdapter = new Map(); -// or -const storageAdapter = require('./my-storage-adapter'); -// or -const QuickLRU = require('quick-lru'); -const storageAdapter = new QuickLRU({maxSize: 1000}); - -got('sindresorhus.com', {cache: storageAdapter}); -``` - -View the [Keyv docs](https://github.com/lukechilds/keyv) for more information on how to use storage adapters. - - ## Proxies You can use the [`tunnel`](https://github.com/koichik/node-tunnel) module with the `agent` option to work with proxies: diff --git a/test/cache.js b/test/cache.js deleted file mode 100644 index 04a84236d..000000000 --- a/test/cache.js +++ /dev/null @@ -1,107 +0,0 @@ -import test from 'ava'; -import got from '../'; -import {createServer} from './helpers/server'; - -let s; - -test.before('setup', async () => { - s = await createServer(); - - let noStoreIndex = 0; - s.on('/no-store', (req, res) => { - res.setHeader('Cache-Control', 'public, no-cache, no-store'); - res.end(noStoreIndex.toString()); - noStoreIndex++; - }); - - let cacheIndex = 0; - s.on('/cache', (req, res) => { - res.setHeader('Cache-Control', 'public, max-age=60'); - res.end(cacheIndex.toString()); - cacheIndex++; - }); - - let status301Index = 0; - s.on('/301', (req, res) => { - if (status301Index === 0) { - res.setHeader('Cache-Control', 'public, max-age=60'); - res.setHeader('Location', s.url + '/302'); - res.statusCode = 301; - } - res.end(); - status301Index++; - }); - - let status302Index = 0; - s.on('/302', (req, res) => { - if (status302Index === 0) { - res.setHeader('Cache-Control', 'public, max-age=60'); - res.setHeader('Location', s.url + '/cache'); - res.statusCode = 302; - } - res.end(); - status302Index++; - }); - - await s.listen(s.port); -}); - -test('Non cacheable responses are not cached', async t => { - const endpoint = '/no-store'; - const cache = new Map(); - - const firstResponseInt = Number((await got(s.url + endpoint, {cache})).body); - const secondResponseInt = Number((await got(s.url + endpoint, {cache})).body); - - t.is(cache.size, 0); - t.true(firstResponseInt < secondResponseInt); -}); - -test('Cacheable responses are cached', async t => { - const endpoint = '/cache'; - const cache = new Map(); - - const firstResponse = await got(s.url + endpoint, {cache}); - const secondResponse = await got(s.url + endpoint, {cache}); - - t.is(cache.size, 1); - t.is(firstResponse.body, secondResponse.body); -}); - -test('Cached response is re-encoded to current encoding option', async t => { - const endpoint = '/cache'; - const cache = new Map(); - const firstEncoding = 'base64'; - const secondEncoding = 'hex'; - - const firstResponse = await got(s.url + endpoint, {cache, encoding: firstEncoding}); - const secondResponse = await got(s.url + endpoint, {cache, encoding: secondEncoding}); - - const expectedSecondResponseBody = Buffer.from(firstResponse.body, firstEncoding).toString(secondEncoding); - - t.is(cache.size, 1); - t.is(secondResponse.body, expectedSecondResponseBody); -}); - -test('Redirects are cached and re-used internally', async t => { - const endpoint = '/301'; - const cache = new Map(); - - const firstResponse = await got(s.url + endpoint, {cache}); - const secondResponse = await got(s.url + endpoint, {cache}); - - t.is(cache.size, 3); - t.is(firstResponse.body, secondResponse.body); -}); - -test('Cache error throws got.CacheError', async t => { - const endpoint = '/no-store'; - const cache = {}; - - const err = await t.throws(got(s.url + endpoint, {cache})); - t.is(err.name, 'CacheError'); -}); - -test.after('cleanup', async () => { - await s.close(); -});