-
Notifications
You must be signed in to change notification settings - Fork 606
/
cheerio_crawler.js
635 lines (589 loc) · 28.1 KB
/
cheerio_crawler.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
/* eslint-disable class-methods-use-this */
import _ from 'underscore';
import cheerio from 'cheerio';
import htmlparser from 'htmlparser2';
import * as iconv from 'iconv-lite';
import util from 'util';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
import contentTypeParser from 'content-type';
import { readStreamToString, concatStreamToBuffer } from 'apify-shared/streams_utilities';
import BasicCrawler from './basic_crawler';
import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';
import { getApifyProxyUrl } from '../actor';
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
import * as utilsRequest from '../utils_request';
import { TimeoutError } from '../errors';
/**
* Default mime types, which CheerioScraper supports.
*/
const DEFAULT_MIME_TYPES = ['text/html', 'application/xhtml+xml'];
const DEFAULT_OPTIONS = {
requestTimeoutSecs: 30,
handlePageTimeoutSecs: 60,
handleFailedRequestFunction: ({ request }) => {
const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
log.error('CheerioCrawler: Request failed and reached maximum retries', details);
},
ignoreSslErrors: true,
useApifyProxy: false,
autoscaledPoolOptions: {
snapshotterOptions: {
eventLoopSnapshotIntervalSecs: 2,
maxBlockedMillis: 100,
},
systemStatusOptions: {
maxEventLoopOverloadedRatio: 0.7,
},
},
additionalMimeTypes: [],
useSessionPool: false,
sessionPoolOptions: {},
persistCookiesPerSession: false,
};
/**
* Provides a framework for the parallel crawling of web pages using plain HTTP requests and
* <a href="https://www.npmjs.com/package/cheerio" target="_blank">cheerio</a> HTML parser.
* The URLs to crawl are fed either from a static list of URLs
* or from a dynamic queue of URLs enabling recursive crawling of websites.
*
* Since `CheerioCrawler` uses raw HTTP requests to download web pages,
* it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript
* to display the content, you might need to use {@link PuppeteerCrawler} instead,
* because it loads the pages using full-featured headless Chrome browser.
*
* `CheerioCrawler` downloads each URL using a plain HTTP request,
* parses the HTML content using <a href="https://www.npmjs.com/package/cheerio" target="_blank">Cheerio</a>
* and then invokes the user-provided [`handlePageFunction()`](#new_CheerioCrawler_new) to extract page data
* using a <a href="https://jquery.com/" target="_blank">jQuery</a>-like interface to the parsed HTML DOM.
*
* The source URLs are represented using {@link Request} objects that are fed from
* {@link RequestList} or {@link RequestQueue} instances provided by the [`requestList`](#new_CheerioCrawler_new)
* or [`requestQueue`](#new_CheerioCrawler_new) constructor options, respectively.
*
* If both [`requestList`](#new_CheerioCrawler_new) and [`requestQueue`](#new_CheerioCrawler_new) are used,
* the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
*
* The crawler finishes when there are no more {@link Request} objects to crawl.
*
* By default, `CheerioCrawler` downloads HTML using the
* <a href="https://www.npmjs.com/package/request" target="_blank">request</a> NPM package.
* You can use the `requestOptions` parameter to pass additional options to `request`.
*
* New requests are only dispatched when there is enough free CPU and memory available,
* using the functionality provided by the {@link AutoscaledPool} class.
* All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
* parameter of the `CheerioCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
* {@link AutoscaledPool} options are available directly in the `CheerioCrawler` constructor.
*
* **Example usage:**
*
* ```javascript
* // Prepare a list of URLs to crawl
* const requestList = new Apify.RequestList({
* sources: [
* { url: 'http://www.example.com/page-1' },
* { url: 'http://www.example.com/page-2' },
* ],
* });
* await requestList.initialize();
*
* // Crawl the URLs
* const crawler = new Apify.CheerioCrawler({
* requestList,
* handlePageFunction: async ({ request, response, body, contentType, $ }) => {
* const data = [];
*
* // Do some data extraction from the page with Cheerio.
* $('.some-collection').each((index, el) => {
* data.push({ title: $(el).find('.some-title').text() });
* });
*
* // Save the data to dataset.
* await Apify.pushData({
* url: request.url,
* html: body,
* data,
* })
* },
* });
*
* await crawler.run();
* ```
*
* @param {Object} options All `CheerioCrawler` parameters are passed
* via an options object with the following keys:
* @param {Function} options.handlePageFunction
* User-provided function that performs the logic of the crawler. It is called for each page
* loaded and parsed by the crawler.
*
* The function receives the following object as an argument:
* ```
* {
* $: Cheerio, // the Cheerio object with parsed HTML
* body: String|Buffer // the request body of the web page
* // the parsed object from JSON string
* // if the response contains the content type application/json
* json: Object,
* request: Request,
* contentType: Object, // Parsed Content-Type header: { type, encoding }
* response: Object // An instance of Node's http.IncomingMessage object,
* autoscaledPool: AutoscaledPool,
* session: Session
* }
* ```
* Type of `body` depends on web page `Content-Type` header.
* - String for `text/html`, `application/xhtml+xml`, `application/xml` mime types
* - Buffer for others mime types
*
* Parsed `Content-Type` header using
* <a href="https://www.npmjs.com/package/content-type" target="_blank">content-type package</a>
* is stored in `contentType`.
*
* Cheerio is available only for HTML and XML content types.
*
* With the {@link Request} object representing the URL to crawl.
*
* If the function returns a promise, it is awaited by the crawler.
*
* If the function throws an exception, the crawler will try to re-crawl the
* request later, up to `option.maxRequestRetries` times.
* If all the retries fail, the crawler calls the function
* provided to the `options.handleFailedRequestFunction` parameter.
* To make this work, you should **always**
* let your function throw exceptions rather than catch them.
* The exceptions are logged to the request using the
* [`request.pushErrorMessage`](request#Request+pushErrorMessage) function.
* @param {RequestList} options.requestList
* Static list of URLs to be processed.
* Either `requestList` or `requestQueue` option must be provided (or both).
* @param {RequestQueue} options.requestQueue
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
* Either `requestList` or `requestQueue` option must be provided (or both).
* @param {Object} [options.requestOptions]
* Represents the options passed to
* <a href="https://www.npmjs.com/package/request" target="_blank">request</a> to make the HTTP call.
* Provided `requestOptions` are added to internal defaults that cannot be overridden to ensure
* the operation of `CheerioCrawler` and all its options. Headers will not be merged,
* use {@link RequestList} and/or {@link RequestQueue} to initialize your {@link Request} with the
* correct headers or use `options.prepareRequestFunction` to modify your {@link Request} dynamically.
* If you need more granular control over your requests, use {@link BasicCrawler}.
*
* The mandatory internal defaults that **CANNOT BE OVERRIDDEN** by `requestOptions`:
* ```
* {
* url, // Provided by RequestList and/or RequestQueue
* method, // Provided by RequestList and/or RequestQueue
* headers, // Provided by RequestList and/or RequestQueue
* payload, // Provided by RequestList and/or RequestQueue
* strictSSL, // Use options.ignoreSslErrors
* proxy, // Use options.useApifyProxy or options.proxyUrls
* }
* ```
* @param {Function} [options.prepareRequestFunction]
* A function that executes before the HTTP request is made to the target resource.
* This function is suitable for setting dynamic properties such as cookies to the {@link Request}.
*
* The function receives the following object as an argument:
* ```
* {
* request: Request,
* session: Session
* }
* ```
* where the {@link Request} instance corresponds to the initialized request and the {@link Session} instance corresponds to used session.
*
* The function should modify the properties of the passed {@link Request} instance
* in place because there are already earlier references to it. Making a copy and returning it from
* this function is therefore not supported, because it would create inconsistencies where
* different parts of SDK would have access to a different {@link Request} instance.
*
* @param {Number} [options.handlePageTimeoutSecs=60]
* Timeout in which the function passed as `options.handlePageFunction` needs to finish, given in seconds.
* @param {Number} [options.requestTimeoutSecs=30]
* Timeout in which the HTTP request to the resource needs to finish, given in seconds.
* @param {Boolean} [options.ignoreSslErrors=true]
* If set to true, SSL certificate errors will be ignored.
* @param {Boolean} [options.useApifyProxy=false]
* If set to `true`, `CheerioCrawler` will be configured to use
* <a href="https://my.apify.com/proxy" target="_blank">Apify Proxy</a> for all connections.
* For more information, see the <a href="https://docs.apify.com/proxy" target="_blank">documentation</a>
* @param {String[]} [options.apifyProxyGroups]
* An array of proxy groups to be used
* by the <a href="https://docs.apify.com/proxy" target="_blank">Apify Proxy</a>.
* Only applied if the `useApifyProxy` option is `true`.
* @param {String} [options.apifyProxySession]
* Apify Proxy session identifier to be used with requests made by `CheerioCrawler`.
* All HTTP requests going through the proxy with the same session identifier
* will use the same target proxy server (i.e. the same IP address).
* The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
* Only applied if the `useApifyProxy` option is `true`.
* @param {String[]} [options.proxyUrls]
* An array of custom proxy URLs to be used by the `CheerioCrawler` instance.
* The provided custom proxies' order will be randomized and the resulting list rotated.
* Custom proxies are not compatible with Apify Proxy and an attempt to use both
* configuration options will cause an error to be thrown on startup.
* @param {Function} [options.handleFailedRequestFunction]
* A function to handle requests that failed more than `option.maxRequestRetries` times.
*
* The function receives the following object as an argument:
* ```
* {
* request: Request,
* error: Error,
* }
* ```
* where the {@link Request} instance corresponds to the failed request, and the `Error` instance
* represents the last error thrown during processing of the request.
*
* See <a href="https://github.com/apifytech/apify-js/blob/master/src/crawlers/cheerio_crawler.js#L13">source code</a>
* for the default implementation of this function.
* @param {String[]} [options.additionalMimeTypes]
* An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"
* target="_blank">mime types</a> you want to process.
* By default `text/html`, `application/xhtml+xml` mime types are supported.
* @param {Number} [options.maxRequestRetries=3]
* Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.
* @param {Number} [options.maxRequestsPerCrawl]
* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
* Always set this value in order to prevent infinite loops in misconfigured crawlers.
* Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
* @param {Object} [options.autoscaledPoolOptions]
* Custom options passed to the underlying {@link AutoscaledPool} constructor.
* Note that the `runTaskFunction`, `isTaskReadyFunction` and `isFinishedFunction` options
* are provided by `CheerioCrawler` and cannot be overridden. Reasonable {@link Snapshotter}
* and {@link SystemStatus} defaults are provided to account for the fact that `cheerio`
* parses HTML synchronously and therefore blocks the event loop.
* @param {Number} [options.minConcurrency=1]
* Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
*
* *WARNING:* If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash.
* If you're not sure, just keep the default value and the concurrency will scale up automatically.
* @param {Number} [options.maxConcurrency=1000]
* Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
* @param {Boolean} [options.useSessionPool=false]
* If set to true Crawler will automatically use Session Pool. It will automatically retire sessions on 403, 401 and 429 status codes.
* It also marks Session as bad after a request timeout.
* @param {Object} [options.sessionPoolOptions]
* Custom options passed to the underlying {@link SessionPool} constructor.
* @param {Boolean} [options.persistCookiesPerSession]
* Automatically saves cookies to Session. Works only if Session Pool is used.
*
* It parses cookie from response "set-cookie" header saves or updates cookies for session and once the session is used for next request.
* It passes the "Cookie" header to the request with the session cookies.
*/
class CheerioCrawler {
constructor(options = {}) {
const {
requestOptions,
handlePageFunction,
requestTimeoutSecs,
handlePageTimeoutSecs,
ignoreSslErrors,
useApifyProxy,
apifyProxyGroups,
apifyProxySession,
proxyUrls,
additionalMimeTypes,
// Autoscaled pool shorthands
minConcurrency,
maxConcurrency,
// Basic crawler options
requestList,
requestQueue,
maxRequestRetries,
maxRequestsPerCrawl,
handleFailedRequestFunction,
autoscaledPoolOptions,
prepareRequestFunction,
useSessionPool,
sessionPoolOptions,
persistCookiesPerSession, // @TODO: Better naming
} = _.defaults({}, options, DEFAULT_OPTIONS);
checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');
checkParamOrThrow(requestOptions, 'options.requestOptions', 'Maybe Object');
checkParamOrThrow(requestTimeoutSecs, 'options.requestTimeoutSecs', 'Number');
checkParamOrThrow(handlePageTimeoutSecs, 'options.handlePageTimeoutSecs', 'Number');
checkParamOrThrow(ignoreSslErrors, 'options.ignoreSslErrors', 'Maybe Boolean');
checkParamOrThrow(useApifyProxy, 'options.useApifyProxy', 'Maybe Boolean');
checkParamOrThrow(apifyProxyGroups, 'options.apifyProxyGroups', 'Maybe [String]');
checkParamOrThrow(apifyProxySession, 'options.apifyProxySession', 'Maybe String');
checkParamOrThrow(proxyUrls, 'options.proxyUrls', 'Maybe [String]');
checkParamOrThrow(prepareRequestFunction, 'options.prepareRequestFunction', 'Maybe Function');
checkParamOrThrow(additionalMimeTypes, 'options.additionalMimeTypes', '[String]');
checkParamOrThrow(useSessionPool, 'options.useSessionPool', 'Boolean');
checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');
checkParamOrThrow(persistCookiesPerSession, 'options.persistCookiesPerSession', 'Boolean');
// Enforce valid proxy configuration
if (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');
if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');
if (persistCookiesPerSession && !useSessionPool) {
throw new Error('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');
}
if (apifyProxySession && useSessionPool) {
throw new Error('Cannot use "options.apifyProxySession" with "options.useSessionPool"');
}
this.supportedMimeTypes = new Set(DEFAULT_MIME_TYPES);
if (additionalMimeTypes.length) this._extendSupportedMimeTypes(additionalMimeTypes);
this.requestOptions = requestOptions;
this.handlePageFunction = handlePageFunction;
this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000;
this.requestTimeoutMillis = requestTimeoutSecs * 1000;
this.ignoreSslErrors = ignoreSslErrors;
this.useApifyProxy = useApifyProxy;
this.apifyProxyGroups = apifyProxyGroups;
this.apifyProxySession = apifyProxySession;
this.proxyUrls = _.shuffle(proxyUrls);
this.lastUsedProxyUrlIndex = 0;
this.prepareRequestFunction = prepareRequestFunction;
this.persistCookiesPerSession = persistCookiesPerSession;
this.useSessionPool = useSessionPool;
this.basicCrawler = new BasicCrawler({
// Basic crawler options.
requestList,
requestQueue,
maxRequestRetries,
maxRequestsPerCrawl,
handleRequestFunction: (...args) => this._handleRequestFunction(...args),
handleRequestTimeoutSecs: handlePageTimeoutSecs * BASIC_CRAWLER_TIMEOUT_MULTIPLIER,
handleFailedRequestFunction,
// Autoscaled pool options.
minConcurrency,
maxConcurrency,
autoscaledPoolOptions,
// Session pool options
sessionPoolOptions,
useSessionPool,
});
this.isRunningPromise = null;
}
/**
* Runs the crawler. Returns promise that gets resolved once all the requests got processed.
*
* @return {Promise}
*/
async run() {
if (this.isRunningPromise) return this.isRunningPromise;
this.isRunningPromise = this.basicCrawler.run();
await this.isRunningPromise;
}
/**
* Wrapper around handlePageFunction that opens and closes pages etc.
*
* @ignore
*/
async _handleRequestFunction({ request, autoscaledPool, session }) {
if (this.prepareRequestFunction) await this.prepareRequestFunction({ request, session });
const { dom, isXml, body, contentType, response } = await addTimeoutToPromise(
this._requestFunction({ request, session }),
this.requestTimeoutMillis,
`CheerioCrawler: request timed out after ${this.requestTimeoutMillis / 1000} seconds.`,
);
if (this.useSessionPool) {
this._throwOnBlockedRequest(session, response.statusCode);
}
if (this.persistCookiesPerSession) {
session.setCookiesFromResponse(response);
}
request.loadedUrl = response.url;
const $ = dom ? cheerio.load(dom, { xmlMode: isXml }) : null;
const context = {
$,
// Using a getter here not to break the original API
// and lazy load the HTML only when needed.
get html() {
log.deprecated('CheerioCrawler: The "html" parameter of handlePageFunction is deprecated, use "body" instead.');
return dom && !isXml && $.html({ decodeEntities: false });
},
get json() {
if (contentType.type !== 'application/json') return null;
const jsonString = body.toString(contentType.encoding);
return JSON.parse(jsonString);
},
get body() {
if (dom) {
return isXml ? $.xml() : $.html({ decodeEntities: false });
}
return body;
},
contentType,
request,
response,
autoscaledPool,
session,
};
return addTimeoutToPromise(
this.handlePageFunction(context),
this.handlePageTimeoutMillis,
`CheerioCrawler: handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,
);
}
/**
* Function to make the HTTP request. It performs optimizations
* on the request such as only downloading the request body if the
* received content type matches text/html, application/xml, application/xhtml+xml.
* @ignore
*/
async _requestFunction({ request, session }) {
// Using the streaming API of Request to be able to
// handle the response based on headers receieved.
if (this.persistCookiesPerSession) {
const { headers } = request;
headers.Cookie = session.getCookieString(request.url);
}
const opts = this._getRequestOptions(request, session);
let responseStream;
try {
responseStream = await utilsRequest.requestAsBrowser(opts);
} catch (e) {
if (e instanceof TimeoutError) {
this._handleRequestTimeout(session);
} else {
throw e;
}
}
const { statusCode } = responseStream;
const { type, charset } = parseContentTypeFromResponse(responseStream);
const { response, encoding } = this._encodeResponse(request, responseStream, charset);
const contentType = { type, encoding };
if (statusCode >= 500) {
const body = await readStreamToString(response, encoding);
// Errors are often sent as JSON, so attempt to parse them,
// despite Accept header being set to text/html.
if (type === 'application/json') {
const errorResponse = JSON.parse(body);
let { message } = errorResponse;
if (!message) message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
throw new Error(`${statusCode} - ${message}`);
}
// It's not a JSON so it's probably some text. Get the first 100 chars of it.
throw new Error(`CheerioCrawler: ${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);
} else if (type === 'text/html' || type === 'application/xhtml+xml' || type === 'application/xml') {
const dom = await this._parseHtmlToDom(response);
return ({ dom, isXml: type.includes('xml'), response, contentType });
} else {
const body = await concatStreamToBuffer(response);
return { body, response, contentType };
}
}
/**
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
* @param {Request} request
* @param {Session?} session
* @ignore
*/
_getRequestOptions(request, session) {
const mandatoryRequestOptions = {
url: request.url,
method: request.method,
headers: Object.assign({}, request.headers),
ignoreSslErrors: this.ignoreSslErrors,
proxyUrl: this._getProxyUrl(session),
stream: true,
useCaseSensitiveHeaders: true,
abortFunction: (res) => {
const { statusCode } = res;
const { type } = parseContentTypeFromResponse(res);
if (statusCode === 406) {
request.noRetry = true;
throw new Error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`);
}
if (!this.supportedMimeTypes.has(type) && statusCode < 500) {
request.noRetry = true;
throw new Error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type}, `
+ `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
}
return false;
},
timeoutSecs: this.requestTimeoutMillis / 1000,
};
if (/PATCH|POST|PUT/.test(request.method)) mandatoryRequestOptions.payload = request.payload;
return Object.assign({}, this.requestOptions, mandatoryRequestOptions);
}
/**
* Enables the use of a proxy by returning a proxy URL
* based on configured options or null if no proxy is used.
* @param {Session?} session
* @ignore
*/
_getProxyUrl(session = {}) {
if (this.useApifyProxy) {
return getApifyProxyUrl({
groups: this.apifyProxyGroups,
session: session.id || this.apifyProxySession,
groupsParamName: 'options.apifyProxyGroups',
sessionParamName: 'options.apifyProxySession',
});
}
if (this.proxyUrls) {
return this.proxyUrls[this.lastUsedProxyUrlIndex++ % this.proxyUrls.length];
}
return null;
}
_encodeResponse(request, response, encoding) {
if (!encoding || Buffer.isEncoding(encoding)) return { response, encoding };
if (iconv.encodingExists(encoding)) {
const finalEncoding = 'utf8';
const encodeStream = iconv.encodeStream(finalEncoding);
const decodeStream = iconv.decodeStream(encoding).on('error', err => encodeStream.emit('error', err));
response.on('error', err => decodeStream.emit('error', err));
const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);
encodedResponse.statusCode = response.statusCode;
encodedResponse.headers = response.headers;
encodedResponse.url = response.url;
return {
response: encodedResponse,
encoding: finalEncoding,
};
}
throw new Error(`CheerioCrawler: Resource ${request.url} served with unsupported charset/encoding: ${encoding}`);
}
async _parseHtmlToDom(response) {
return new Promise((resolve, reject) => {
const domHandler = new htmlparser.DomHandler((err, dom) => {
if (err) reject(err);
else resolve(dom);
});
const parser = new htmlparser.Parser(domHandler, { decodeEntities: true });
response.on('error', reject).pipe(parser);
});
}
/**
* Checks and extends supported mime types
* @param additionalMimeTypes
* @ignore
*/
_extendSupportedMimeTypes(additionalMimeTypes) {
additionalMimeTypes.forEach((mimeType) => {
try {
const parsedType = contentTypeParser.parse(mimeType);
this.supportedMimeTypes.add(parsedType.type);
} catch (err) {
throw new Error(`CheerioCrawler: Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
}
});
}
/**
* Handles blocked request
* @param session {Session}
* @param statusCode {Number}
* @private
*/
_throwOnBlockedRequest(session, statusCode) {
const isBlocked = session.retireOnBlockedStatusCodes(statusCode);
if (isBlocked) {
throw new Error(`CheerioCrawler: Request blocked - received ${statusCode} status code`);
}
}
/**
* Handles timeout request
* @param session {Session}
* @private
*/
_handleRequestTimeout(session) {
if (session) session.markBad();
throw new Error(`CheerioCrawler: request timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);
}
}
export default CheerioCrawler;