-
-
Notifications
You must be signed in to change notification settings - Fork 18
/
index.js
66 lines (63 loc) · 2.21 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
"use strict";
const got = require("got");
const solve = require("./core");
const { CookieJar } = require("tough-cookie");
const UserAgent = require("user-agents");
// Got instance to handle cloudflare bypass
const instance = got.extend({
cookieJar: new CookieJar(), // Automatically parse and store cookies
retry: {
limit: 2,
statusCodes: [408, 413, 429, 500, 502, 504, 521, 522, 524]
}, // Do not retry 503, we will handle it
cloudflareRetry: 0, // Prevent cloudflare loop
notFoundRetry: 0, // Handle redirect issue
http2: true, // Use recommended protocol
headers: {
"cache-control": "max-age=0",
"upgrade-insecure-requests": "1",
"user-agent": new UserAgent().toString() // Use random user agent between sessions
}, // Mimic browser environment
hooks: {
beforeRequest: [
options => {
// Add required headers to mimic browser environment
options.headers.origin = options.url.origin;
options.headers.referer = options.url.href;
}
],
afterResponse: [
async response => {
if (
// If site is not hosted on cloudflare skip
response.statusCode === 503 &&
response.headers.server === "cloudflare" &&
response.body.includes("jschl-answer")
) {
// Solve js challange
if (response.request.options.cloudflareRetry < 5) {
const data = await solve(response.url, response.body);
response.request.options.cloudflareRetry++;
return instance({ ...response.request.options, ...data });
}
} else if (
// Handle redirect issue for cloudflare
response.statusCode === 404 &&
response.headers.server === "cloudflare" &&
response.url.includes("?__cf_chl_jschl_tk")
) {
// Do not retry again
if (response.request.options.notFoundRetry === 0) {
return instance({
url: response.url.split("?__cf_chl_jschl_tk")[0],
notFoundRetry: response.request.options.notFoundRetry + 1
});
}
}
return response;
}
]
},
mutableDefaults: true // Defines if config can be changed later
});
module.exports = instance;