From 876ebe417c2371c184c3d44a2f1ae752274c6760 Mon Sep 17 00:00:00 2001 From: pro-src Date: Sat, 13 Apr 2019 03:24:46 -0500 Subject: [PATCH 01/15] Fix UA randomization and header order --- index.js | 49 +++---- lib/browsers.json | 336 ++++++++++++++++++++++++++++++++++++++++++++++ lib/headers.js | 20 +++ test/helper.js | 11 +- 4 files changed, 375 insertions(+), 41 deletions(-) create mode 100644 lib/browsers.json create mode 100644 lib/headers.js diff --git a/index.js b/index.js index 0201c4f..e6d1138 100644 --- a/index.js +++ b/index.js @@ -1,28 +1,14 @@ 'use strict'; -var vm = require('vm'); -var requestModule = require('request-promise'); -var errors = require('./errors'); -var decodeEmails = require('./lib/email-decode.js'); - -var USER_AGENTS = [ - 'Ubuntu Chromium/34.0.1847.116 Chrome/34.0.1847.116 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.21 (KHTML, like Gecko) konqueror/4.14.10 Safari/537.21', - 'Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3', - 'Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)', - 'Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 630) like Gecko', - 'Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch; NOKIA; Lumia 920)', - 'Mozilla/5.0 (Linux; U; Android 2.2; en-us; Sprint APA9292KT Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1', - 'Mozilla/5.0 (X11; Linux x86_64; rv:2.2a1pre) Gecko/20100101 Firefox/4.2a1pre', - 'Mozilla/5.0 (SymbianOS/9.1; U; en-us) AppleWebKit/413 (KHTML, like Gecko) Safari/413 es65', - 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5X Build/MDB08L) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36', - 'Mozilla/5.0 (X11; U; FreeBSD i386; de-CH; rv:1.9.2.8) Gecko/20100729 Firefox/3.6.8' -]; - -var DEFAULT_USER_AGENT = randomUA(); - -var VM_OPTIONS = { +const vm = require('vm'); +const requestModule = require('request-promise'); +const errors = require('./errors'); +const decodeEmails = require('./lib/email-decode.js'); +const getDefaultHeaders = require('./lib/headers'); + +const HOST = Symbol('host'); + +const VM_OPTIONS = { contextOrigin: 'cloudflare:challenge.js', contextCodeGeneration: { strings: true, wasm: false }, timeout: 5000 @@ -38,13 +24,7 @@ function defaults (params) { requester: requestModule, // Cookies should be enabled jar: requestModule.jar(), - headers: { - 'Connection': 'keep-alive', - 'User-Agent': DEFAULT_USER_AGENT, - 'Cache-Control': 'private', - 'Accept': 'application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5', - 'Accept-Language': 'en-US,en;q=0.9' - }, + headers: getDefaultHeaders({ 'Host': HOST }), // Reduce Cloudflare's timeout to cloudflareMaxTimeout if it is excessive cloudflareMaxTimeout: 30000, // followAllRedirects - follow non-GET HTTP 3xx responses as redirects @@ -115,6 +95,11 @@ function performRequest (options, isFirstRequest) { var request = requester(options); + // We must define the host header ourselves to preserve case and order. + if (request.getHeader('host') === HOST) { + request.setHeader('host', request.uri.host); + } + // If the requester is not request-promise, ensure we get a callback. if (typeof request.callback !== 'function') { throw new TypeError('Expected a callback function, got ' + @@ -421,7 +406,3 @@ function createSandbox (options) { document: {} }, options.context); } - -function randomUA () { - return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)]; -} diff --git a/lib/browsers.json b/lib/browsers.json new file mode 100644 index 0000000..c3eab31 --- /dev/null +++ b/lib/browsers.json @@ -0,0 +1,336 @@ +{ + "chrome": [ + { + "User-Agent": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.113 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.8", + "Accept-Encoding": "gzip, deflate, , br" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.8", + "Accept-Encoding": "gzip, deflate, br" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.170 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "User-Agent": null, + "Upgrade-Insecure-Requests": "1", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.40 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.40 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.28 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.28 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.28 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.28 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.28 Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Linux; Android 8.1.0; SM-N960F Build/M1AJQ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; SM-G965F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 Build/OPD1.170816.010) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; Pixel Build/OPR6.170623.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.1.1; SM-A530F Build/NMF26X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.1; Pixel Build/NDE63H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; SM-G955F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; SM-G950F Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.0; SM-T825 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0.1; SM-G930F Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0; Nexus 6 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0; XT1092 Build/MPE24.49-18) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0.1; SM-N910C Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 5.0.2; SM-G920F Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 5.0; Nexus 6 Build/LRX21O) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 9; Pixel 3 XL Build/PD1A.180720.030) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 9; Pixel 3 Build/PD1A.180720.030) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 9; Pixel 2 Build/PPR1.180610.009) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 4.4; Nexus 5 Build/KRT16M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 4.4.2; SM-T530 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Safari/537.36", + "Mozilla/5.0 (Linux; Android 4.4.4; SM-N910C Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 5.1.1; Nexus 9 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Safari/537.36", + "Mozilla/5.0 (Linux; Android 7.1.1; SM-N950F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.90 Mobile Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en-US,en;q=0.9" + } + }, + { + "User-Agent": [ + "Mozilla/5.0 (Linux; Android 8.1.0; SM-T835 Build/M1AJQ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Safari/537.36", + "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36", + "Mozilla/5.0 (Linux; Android 5.0; XT1092 Build/LXE22.46-19) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36" + ], + "headers": { + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "User-Agent": null, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8" + } + } + ] +} diff --git a/lib/headers.js b/lib/headers.js new file mode 100644 index 0000000..e97835a --- /dev/null +++ b/lib/headers.js @@ -0,0 +1,20 @@ +'use strict'; + +const chromeData = require('./browsers').chrome; + +module.exports = function getHeaders (defaults) { + const headers = getChromeHeaders(random(chromeData)); + return Object.assign({}, defaults, headers); +}; + +function random (arr) { + return arr[Math.floor(Math.random() * arr.length)]; +} + +function getChromeHeaders (options) { + const { headers } = options; + + headers['User-Agent'] = random(options['User-Agent']); + + return headers; +} diff --git a/test/helper.js b/test/helper.js index 0b93e7d..f251a55 100644 --- a/test/helper.js +++ b/test/helper.js @@ -5,6 +5,9 @@ var url = require('url'); var path = require('path'); var express = require('express'); +// Clone the default headers for tests +var defaultHeaders = Object.assign({}, require('../').defaultParams.headers); + // Cache fixtures so they're only read from fs but once var cache = {}; @@ -19,13 +22,7 @@ var helper = { requester: sinon.match.func, jar: request.jar(), uri: helper.resolve('/test'), - headers: { - 'Connection': 'keep-alive', - 'User-Agent': sinon.match.string, - 'Cache-Control': 'private', - 'Accept': 'application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5', - 'Accept-Language': 'en-US,en;q=0.9' - }, + headers: Object.assign({}, defaultHeaders), method: 'GET', encoding: null, realEncoding: 'utf8', From 0462147548a7d02e910a3a270baf2c8c11581b08 Mon Sep 17 00:00:00 2001 From: pro-src Date: Sat, 13 Apr 2019 19:12:21 -0500 Subject: [PATCH 02/15] Let & Const --- errors.js | 22 +++---- index.js | 72 ++++++++++----------- lib/email-decode.js | 11 ++-- test/test-emails.js | 46 +++++++------- test/test-errors.js | 101 ++++++++++++++--------------- test/test-index.js | 150 ++++++++++++++++++++++---------------------- test/test-rp.js | 28 ++++----- 7 files changed, 216 insertions(+), 214 deletions(-) diff --git a/errors.js b/errors.js index f2975bf..fee304e 100644 --- a/errors.js +++ b/errors.js @@ -9,17 +9,17 @@ // 1. There is a non-enumerable errorType attribute. // 2. The error constructor is hidden from the stacktrace. -var EOL = require('os').EOL; -var original = require('request-promise-core/errors'); -var http = require('http'); +const EOL = require('os').EOL; +const original = require('request-promise-core/errors'); +const http = require('http'); -var BUG_REPORT = format([ +const BUG_REPORT = format([ '### Cloudflare may have changed their technique, or there may be a bug.', '### Bug Reports: https://github.com/codemanki/cloudscraper/issues', '### Check the detailed exception message that follows for the cause.' ]); -var ERROR_CODES = { +const ERROR_CODES = { // Non-standard 5xx server error HTTP status codes '520': 'Web server is returning an unknown error', '521': 'Web server is down', @@ -48,22 +48,22 @@ ERROR_CODES[1006] = ERROR_CODES[1007] = ERROR_CODES[1008] = 'Access Denied: Your IP address has been banned'; -var OriginalError = original.RequestError; +const OriginalError = original.RequestError; -var RequestError = create('RequestError', 0); -var CaptchaError = create('CaptchaError', 1); +const RequestError = create('RequestError', 0); +const CaptchaError = create('CaptchaError', 1); // errorType 4 is a CloudflareError so this constructor is reused. -var CloudflareError = create('CloudflareError', 2, function (error) { +const CloudflareError = create('CloudflareError', 2, function (error) { if (!isNaN(error.cause)) { - var description = ERROR_CODES[error.cause] || http.STATUS_CODES[error.cause]; + const description = ERROR_CODES[error.cause] || http.STATUS_CODES[error.cause]; if (description) { error.message = error.cause + ', ' + description; } } }); -var ParserError = create('ParserError', 3, function (error) { +const ParserError = create('ParserError', 3, function (error) { error.message = BUG_REPORT + error.message; }); diff --git a/index.js b/index.js index e6d1138..bee01ac 100644 --- a/index.js +++ b/index.js @@ -18,9 +18,9 @@ module.exports = defaults.call(requestModule); function defaults (params) { // isCloudScraper === !isRequestModule - var isRequestModule = this === requestModule; + const isRequestModule = this === requestModule; - var defaultParams = (!isRequestModule && this.defaultParams) || { + let defaultParams = (!isRequestModule && this.defaultParams) || { requester: requestModule, // Cookies should be enabled jar: requestModule.jar(), @@ -38,7 +38,7 @@ function defaults (params) { // Object.assign requires at least nodejs v4, request only test/supports v6+ defaultParams = Object.assign({}, defaultParams, params); - var cloudscraper = requestModule.defaults + const cloudscraper = requestModule.defaults .call(this, defaultParams, function (options) { return performRequest(options, true); }); @@ -86,14 +86,14 @@ function performRequest (options, isFirstRequest) { } // This should be the default export of either request or request-promise. - var requester = options.requester; + const requester = options.requester; if (typeof requester !== 'function') { throw new TypeError('Expected `requester` option to be a function, got ' + typeof (requester) + ' instead.'); } - var request = requester(options); + const request = requester(options); // We must define the host header ourselves to preserve case and order. if (request.getHeader('host') === HOST) { @@ -124,7 +124,7 @@ function performRequest (options, isFirstRequest) { onRequestResponse(options, null, response, body); }); - // Indicate that this is a cloudscraper request, required by test/helper. + // Indicate that this is a cloudscraper request request.cloudscraper = true; return request; } @@ -132,7 +132,7 @@ function performRequest (options, isFirstRequest) { // The argument convention is options first where possible, options // always before response, and body always after response. function onRequestResponse (options, error, response, body) { - var callback = options.callback; + const callback = options.callback; // Encoding is null so body should be a buffer object if (error || !body || !body.toString) { @@ -157,11 +157,11 @@ function onRequestResponse (options, error, response, body) { } function onCloudflareResponse (options, response, body) { - var callback = options.callback; + const callback = options.callback; - var stringBody; - var isChallenge; - var isRedirectChallenge; + let stringBody; + let isChallenge; + let isRedirectChallenge; if (body.length < 1) { // This is a 4xx-5xx Cloudflare response with an empty body. @@ -199,7 +199,7 @@ function onCloudflareResponse (options, response, body) { } function validate (options, response, body) { - var match; + let match; // Finding captcha if (body.indexOf('why_captcha') !== -1 || /cdn-cgi\/l\/chk_captcha/i.test(body)) { @@ -210,7 +210,7 @@ function validate (options, response, body) { match = body.match(/<\w+\s+class="cf-error-code">(.*)<\/\w+>/i); if (match) { - var code = parseInt(match[1]); + let code = parseInt(match[1]); throw new errors.CloudflareError(code, options, response); } @@ -218,9 +218,13 @@ function validate (options, response, body) { } function solveChallenge (options, response, body) { - var callback = options.callback; - var cause; - var error; + const callback = options.callback; + const uri = response.request.uri; + // The query string to send back to Cloudflare + const payload = { /* s, jschl_vc, pass, jschl_answer */ }; + + let cause; + let error; if (options.challengesToSolve === 0) { cause = 'Cloudflare challenge loop'; @@ -230,13 +234,9 @@ function solveChallenge (options, response, body) { return callback(error); } - var timeout = parseInt(options.cloudflareTimeout); - var uri = response.request.uri; - // The query string to send back to Cloudflare - // var payload = { s, jschl_vc, pass, jschl_answer }; - var payload = {}; - var sandbox; - var match; + let timeout = parseInt(options.cloudflareTimeout); + let sandbox; + let match; match = body.match(/name="s" value="(.+?)"/); @@ -321,20 +321,20 @@ function solveChallenge (options, response, body) { } function setCookieAndReload (options, response, body) { - var callback = options.callback; + const callback = options.callback; - var match = body.match(/S='([^']+)'/); + const match = body.match(/S='([^']+)'/); if (!match) { - var cause = 'Cookie code extraction failed'; + const cause = 'Cookie code extraction failed'; return callback(new errors.ParserError(cause, options, response)); } - var base64EncodedCode = match[1]; + const base64EncodedCode = match[1]; response.challenge = Buffer.from(base64EncodedCode, 'base64').toString('ascii'); try { - var sandbox = createSandbox({}); + const sandbox = createSandbox(); // Evaluate cookie setting code vm.runInNewContext(response.challenge, sandbox, VM_OPTIONS); @@ -350,7 +350,7 @@ function setCookieAndReload (options, response, body) { } function processResponseBody (options, response, body) { - var callback = options.callback; + const callback = options.callback; if (typeof options.realEncoding === 'string') { body = body.toString(options.realEncoding); @@ -367,12 +367,12 @@ function processResponseBody (options, response, body) { callback(null, response, body); } -function createSandbox (options) { +function createSandbox (options = {}) { if (options.body) { - var body = options.body; - var href = 'http://' + options.uri.hostname + '/'; - var cache = Object.create(null); - var keys = []; + const body = options.body; + const href = 'http://' + options.uri.hostname + '/'; + const cache = Object.create(null); + const keys = []; // Sandbox for standard IUAM JS challenge return Object.assign({ @@ -385,8 +385,8 @@ function createSandbox (options) { }, getElementById: function (id) { if (keys.indexOf(id) === -1) { - var re = new RegExp(' id=[\'"]?' + id + '[^>]*>([^<]+)'); - var match = body.match(re); + const re = new RegExp(' id=[\'"]?' + id + '[^>]*>([^<]+)'); + const match = body.match(re); keys.push(id); cache[id] = match === null ? match : { innerHTML: match[1] }; diff --git a/lib/email-decode.js b/lib/email-decode.js index 87c2e9c..3a461de 100644 --- a/lib/email-decode.js +++ b/lib/email-decode.js @@ -1,4 +1,4 @@ -var pattern = ( +const pattern = ( // Opening tag // $1 = TAG_NAME '<([a-z]+)(?: [^>]*)?' + '(?:' + @@ -14,10 +14,10 @@ var pattern = ( '(?:[^<]*\\/>|[^<]*?<\\/\\1>)' + ')' ); -var re = new RegExp(pattern, 'gi'); +const re = new RegExp(pattern, 'gi'); module.exports = function (html) { - var match, result; + let match, result; re.lastIndex = 0; @@ -36,9 +36,10 @@ module.exports = function (html) { }; function decode (hexStr) { - var key = parseInt(hexStr.substr(0, 2), 16); - var email = ''; + const key = parseInt(hexStr.substr(0, 2), 16); + let email = ''; + // noinspection ES6ConvertVarToLetConst for (var codePoint, i = 2; i < hexStr.length; i += 2) { codePoint = parseInt(hexStr.substr(i, 2), 16) ^ key; email += String.fromCharCode(codePoint); diff --git a/test/test-emails.js b/test/test-emails.js index 860bfe4..74d0408 100644 --- a/test/test-emails.js +++ b/test/test-emails.js @@ -2,11 +2,11 @@ /* eslint-env node, mocha */ 'use strict'; -var decode = require('../lib/email-decode'); -var expect = require('chai').expect; +const decode = require('../lib/email-decode'); +const expect = require('chai').expect; -var EMAIL = 'cloudscraper@example-site.dev'; -var HEX_STRING = '6506090a10011606170415001725001d040815090048160c11004b010013'; +const EMAIL = 'cloudscraper@example-site.dev'; +const HEX_STRING = '6506090a10011606170415001725001d040815090048160c11004b010013'; function genHTML (body) { return '\n' + @@ -24,73 +24,73 @@ function genHTML (body) { describe('Cloudscraper', function () { it('should not modify unprotected html', function () { - var raw = genHTML(''); + const raw = genHTML(''); expect(decode(raw)).to.equal(raw); }); it('should remove email protection', function () { - var protection = '!@#&*9^%()[]/\\'; + const protection = '!@#&*9^%()[]/\\'; expect(decode(protection)).to.equal(EMAIL); }); it('should replace anchors that have a data-cfemail attribute', function () { - var protection = '[email protected]'; - var raw = genHTML('

The email is ' + EMAIL + '

'); - var enc = genHTML('

The email is ' + protection + '

'); + const raw = genHTML('

The email is ' + EMAIL + '

'); + const enc = genHTML('

The email is ' + protection + '

'); expect(decode(enc)).to.equal(raw); }); it('should replace spans that have a data-cfemail attribute', function () { - var protection = '[email protected]'; - var raw = genHTML('

The email is ' + EMAIL + '

'); - var enc = genHTML('

The email is ' + protection + '

'); + const raw = genHTML('

The email is ' + EMAIL + '

'); + const enc = genHTML('

The email is ' + protection + '

'); expect(decode(enc)).to.equal(raw); }); it('should be space agnostic', function () { - var protection = '\n[email protected]\r\n'; - var raw = genHTML('\r\n

\n The email
is ' + EMAIL + '\r\n

\n'); - var enc = genHTML('\r\n

\n The email
is ' + protection + '\r\n

\n'); + const raw = genHTML('\r\n

\n The email
is ' + EMAIL + '\r\n

\n'); + const enc = genHTML('\r\n

\n The email
is ' + protection + '\r\n

\n'); expect(decode(enc)).to.equal(raw); }); it('should not replace nodes if they have children', function () { - var protection = '[email protected]'; - var enc = genHTML('

The email is ' + protection + '

'); + const enc = genHTML('

The email is ' + protection + '

'); expect(decode(enc)).to.equal(enc); }); it('should not replace malformed html', function () { - var protection = '\n<\n'; - var enc = genHTML('

The email is ' + protection + '

'); + const protection = '\n<\n'; + const enc = genHTML('

The email is ' + protection + '

'); expect(decode(enc)).to.equal(enc); }); it('should account for self-closing nodes', function () { - var protection = 'test'; + const protection = 'test'; expect(decode(protection)).to.equal(EMAIL + 'test'); }); it('should update href attribute values', function () { - var protection = ''; + const protection = ''; - var raw = genHTML(''); - var enc = genHTML(protection); + const raw = genHTML(''); + const enc = genHTML(protection); expect(decode(enc)).to.equal(raw); }); diff --git a/test/test-errors.js b/test/test-errors.js index 2b328da..dad5b7b 100644 --- a/test/test-errors.js +++ b/test/test-errors.js @@ -2,19 +2,19 @@ /* eslint-env node, mocha */ 'use strict'; -var cloudscraper = require('../index'); -var request = require('request-promise'); -var helper = require('./helper'); -var errors = require('../errors'); +const cloudscraper = require('../index'); +const request = require('request-promise'); +const helper = require('./helper'); +const errors = require('../errors'); -var sinon = require('sinon'); -var expect = require('chai').expect; -var assert = require('chai').assert; +const sinon = require('sinon'); +const expect = require('chai').expect; +const assert = require('chai').assert; describe('Cloudscraper', function () { - var sandbox; - var Request; - var uri; + let sandbox; + let Request; + let uri; before(function (done) { helper.listen(function () { @@ -46,7 +46,7 @@ describe('Cloudscraper', function () { res.endAbruptly(); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { expect(error).to.be.instanceOf(errors.RequestError); expect(error.error).to.be.an('error'); expect(error).to.have.property('errorType', 0); @@ -62,7 +62,7 @@ describe('Cloudscraper', function () { res.cloudflare().status(504).end(); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { // errorType 1, means captcha is served expect(error).to.be.instanceOf(errors.CloudflareError); expect(error).to.have.property('error', 504); @@ -82,7 +82,7 @@ describe('Cloudscraper', function () { res.sendChallenge('captcha.html'); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { // errorType 1, means captcha is served expect(error).to.be.instanceOf(errors.CaptchaError); expect(error).to.have.property('error', 'captcha'); @@ -103,7 +103,7 @@ describe('Cloudscraper', function () { res.cloudflare().status(500).sendFixture('access_denied.html'); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { // errorType 2, means inner cloudflare error expect(error).to.be.instanceOf(errors.CloudflareError); expect(error).to.have.property('error', 1006); @@ -117,13 +117,13 @@ describe('Cloudscraper', function () { }); it('should add a description to 5xx range cloudflare errors', function (done) { - var html = helper.getFixture('access_denied.html').toString('utf8'); + const html = helper.getFixture('access_denied.html').toString('utf8'); helper.router.get('/test', function (req, res) { res.cloudflare().status(504).send(html.replace('1006', '504')); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { // errorType 2, means inner cloudflare error expect(error).to.be.instanceOf(errors.CloudflareError); expect(error).to.have.property('error', 504); @@ -137,13 +137,13 @@ describe('Cloudscraper', function () { }); it('should not error if error description is unavailable', function (done) { - var html = helper.getFixture('access_denied.html').toString('utf8'); + const html = helper.getFixture('access_denied.html').toString('utf8'); helper.router.get('/test', function (req, res) { res.cloudflare().status(500).send(html.replace('1006', '5111')); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { // errorType 2, means inner cloudflare error expect(error).to.be.instanceOf(errors.CloudflareError); expect(error).to.have.property('error', 5111); @@ -162,7 +162,7 @@ describe('Cloudscraper', function () { }); // The expected params for all subsequent calls to Request - var expectedParams = helper.extendParams({ + const expectedParams = helper.extendParams({ uri: helper.resolve('/cdn-cgi/l/chk_jschl') }); @@ -172,7 +172,7 @@ describe('Cloudscraper', function () { qs: sinon.match.object }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { expect(error).to.be.instanceOf(errors.CloudflareError); expect(error).to.have.property('error', 'Cloudflare challenge loop'); expect(error).to.have.property('errorType', 4); @@ -180,7 +180,8 @@ describe('Cloudscraper', function () { assert.equal(Request.callCount, 4, 'Request call count'); expect(Request.firstCall).to.be.calledWithExactly(helper.defaultParams); - var total = helper.defaultParams.challengesToSolve + 1; + const total = helper.defaultParams.challengesToSolve + 1; + // noinspection ES6ConvertVarToLetConst for (var i = 1; i < total; i++) { // Decrement the number of challengesToSolve to match actual params expectedParams.challengesToSolve -= 1; @@ -196,10 +197,10 @@ describe('Cloudscraper', function () { res.status(503).end(); }); - var expectedParams = helper.extendParams({ json: true }); - var options = { uri: uri, json: true }; + const expectedParams = helper.extendParams({ json: true }); + const options = { uri: uri, json: true }; - var promise = cloudscraper.get(options, function (error) { + const promise = cloudscraper.get(options, function (error) { expect(error).to.be.instanceOf(errors.RequestError); expect(error).to.have.property('error', null); expect(error).to.have.property('errorType', 0); @@ -218,7 +219,7 @@ describe('Cloudscraper', function () { res.sendChallenge('invalid_js_challenge.html'); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { expect(error).to.be.instanceOf(errors.ParserError); expect(error).to.have.property('error').that.is.ok; expect(error).to.have.property('errorType', 3); @@ -230,14 +231,14 @@ describe('Cloudscraper', function () { }); it('should return error if js challenge has error during evaluation', function (done) { - var html = helper.getFixture('js_challenge_03_12_2018_1.html'); + const html = helper.getFixture('js_challenge_03_12_2018_1.html'); helper.router.get('/test', function (req, res) { // Adds a syntax error near the end of line 37 res.cloudflare().status(503).send(html.replace(/\.toFixed/gm, '..toFixed')); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { expect(error).to.be.instanceOf(errors.ParserError); expect(error).to.have.property('error').that.is.an('error'); expect(error).to.have.property('errorType', 3); @@ -250,13 +251,13 @@ describe('Cloudscraper', function () { }); it('should return error if pass extraction fails', function (done) { - var html = helper.getFixture('js_challenge_03_12_2018_1.html'); + const html = helper.getFixture('js_challenge_03_12_2018_1.html'); helper.router.get('/test', function (req, res) { res.cloudflare().status(503).send(html.replace(/name="pass"/gm, '')); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { expect(error).to.be.instanceOf(errors.ParserError); expect(error).to.have.property('error', 'Attribute (pass) value extraction failed'); expect(error).to.have.property('errorType', 3); @@ -268,13 +269,13 @@ describe('Cloudscraper', function () { }); it('should return error if challengeId extraction fails', function (done) { - var html = helper.getFixture('js_challenge_03_12_2018_1.html'); + const html = helper.getFixture('js_challenge_03_12_2018_1.html'); helper.router.get('/test', function (req, res) { res.cloudflare().status(503).send(html.replace(/name="jschl_vc"/gm, '')); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { expect(error).to.be.instanceOf(errors.ParserError); expect(error).to.have.property('error', 'challengeId (jschl_vc) extraction failed'); expect(error).to.have.property('errorType', 3); @@ -286,14 +287,14 @@ describe('Cloudscraper', function () { }); it('should return error if challenge answer is not a number', function (done) { - var html = helper.getFixture('js_challenge_03_12_2018_1.html'); + const html = helper.getFixture('js_challenge_03_12_2018_1.html'); helper.router.get('/test', function (req, res) { res.cloudflare().status(503) .send(html.replace(/a.value.*/, 'a.value="abc" + t.length')); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { expect(error).to.be.instanceOf(errors.ParserError); expect(error).to.have.property('error', 'Challenge answer is not a number'); expect(error).to.have.property('errorType', 3); @@ -313,7 +314,7 @@ describe('Cloudscraper', function () { res.endAbruptly(); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { // errorType 0, a connection error for example expect(error).to.be.instanceOf(errors.RequestError); expect(error.error).to.be.an('error'); @@ -336,7 +337,7 @@ describe('Cloudscraper', function () { }); // Second call to request.get returns recaptcha - var expectedParams = helper.extendParams({ + const expectedParams = helper.extendParams({ uri: helper.resolve('/cdn-cgi/l/chk_jschl'), challengesToSolve: 2 }); @@ -347,7 +348,7 @@ describe('Cloudscraper', function () { qs: sinon.match.object }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { // errorType 1, means captcha is served expect(error).to.be.instanceOf(errors.CaptchaError); expect(error).to.have.property('error', 'captcha'); @@ -362,14 +363,14 @@ describe('Cloudscraper', function () { }); it('should return error if challenge page cookie extraction fails', function (done) { - var html = helper.getFixture('js_challenge_cookie.html').toString('utf8'); + const html = helper.getFixture('js_challenge_cookie.html').toString('utf8'); helper.router.get('/test', function (req, res) { // The cookie extraction codes looks for the `S` variable assignment res.cloudflare().status(503).send(html.replace(/S=/gm, 'Z=')); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { expect(error).to.be.instanceOf(errors.ParserError); expect(error).to.have.property('error', 'Cookie code extraction failed'); expect(error).to.have.property('errorType', 3); @@ -381,9 +382,9 @@ describe('Cloudscraper', function () { }); it('should throw a TypeError if callback is not a function', function (done) { - var spy = sinon.spy(function () { + const spy = sinon.spy(function () { // request-promise always provides a callback so change requester - var options = { uri: uri, requester: require('request') }; + const options = { uri: uri, requester: require('request') }; cloudscraper.get(options); }); @@ -392,7 +393,7 @@ describe('Cloudscraper', function () { }); it('should throw a TypeError if requester is not a function', function (done) { - var spy = sinon.spy(function () { + const spy = sinon.spy(function () { cloudscraper.get({ requester: null }); }); @@ -401,8 +402,8 @@ describe('Cloudscraper', function () { }); it('should throw a TypeError if challengesToSolve is not a number', function (done) { - var spy = sinon.spy(function () { - var options = { uri: uri, challengesToSolve: 'abc' }; + const spy = sinon.spy(function () { + const options = { uri: uri, challengesToSolve: 'abc' }; cloudscraper.get(options); }); @@ -412,8 +413,8 @@ describe('Cloudscraper', function () { }); it('should throw a TypeError if cloudflareMaxTimeout is not a number', function (done) { - var spy = sinon.spy(function () { - var options = { uri: uri, cloudflareMaxTimeout: 'abc' }; + const spy = sinon.spy(function () { + const options = { uri: uri, cloudflareMaxTimeout: 'abc' }; cloudscraper.get(options, function () {}); }); @@ -424,14 +425,14 @@ describe('Cloudscraper', function () { it('should return error if cookie setting code evaluation fails', function (done) { // Change the cookie setting code so the vm will throw an error - var html = helper.getFixture('js_challenge_cookie.html').toString('utf8'); - var b64 = Buffer.from('throw new Error(\'vm eval failed\');').toString('base64'); + const html = helper.getFixture('js_challenge_cookie.html').toString('utf8'); + const b64 = Buffer.from('throw new Error(\'vm eval failed\');').toString('base64'); helper.router.get('/test', function (req, res) { res.cloudflare().status(503).send(html.replace(/S='([^']+)'/, 'S=\'' + b64 + '\'')); }); - var promise = cloudscraper.get(uri, function (error) { + const promise = cloudscraper.get(uri, function (error) { expect(error).to.be.instanceOf(errors.ParserError); expect(error).to.have.property('error').that.is.an('error'); expect(error).to.have.property('errorType', 3); @@ -444,14 +445,14 @@ describe('Cloudscraper', function () { }); it('should not error if Error.captureStackTrace is undefined', function () { - var desc = Object.getOwnPropertyDescriptor(Error, 'captureStackTrace'); + const desc = Object.getOwnPropertyDescriptor(Error, 'captureStackTrace'); Object.defineProperty(Error, 'captureStackTrace', { configurable: true, value: undefined }); - var spy = sinon.spy(function () { + const spy = sinon.spy(function () { throw new errors.RequestError(); }); diff --git a/test/test-index.js b/test/test-index.js index 7c0178f..cfeca46 100644 --- a/test/test-index.js +++ b/test/test-index.js @@ -2,20 +2,20 @@ /* eslint-env node, mocha */ 'use strict'; -var cloudscraper = require('../index'); -var request = require('request-promise'); -var helper = require('./helper'); -var querystring = require('querystring'); +const cloudscraper = require('../index'); +const request = require('request-promise'); +const helper = require('./helper'); +const querystring = require('querystring'); -var sinon = require('sinon'); -var expect = require('chai').expect; +const sinon = require('sinon'); +const expect = require('chai').expect; describe('Cloudscraper', function () { - var sandbox; - var Request; - var uri; + let sandbox; + let Request; + let uri; - var requestedPage = helper.getFixture('requested_page.html'); + const requestedPage = helper.getFixture('requested_page.html'); before(function (done) { helper.listen(function () { @@ -43,16 +43,16 @@ describe('Cloudscraper', function () { }); it('should return requested page, in the specified encoding', function (done) { - var expectedBody = Buffer.from(requestedPage).toString('utf16le'); + const expectedBody = Buffer.from(requestedPage).toString('utf16le'); helper.router.get('/test', function (req, res) { res.send(requestedPage); }); - var expectedParams = helper.extendParams({ realEncoding: 'utf16le' }); - var options = { uri: uri, encoding: 'utf16le' }; + const expectedParams = helper.extendParams({ realEncoding: 'utf16le' }); + const options = { uri: uri, encoding: 'utf16le' }; - var promise = cloudscraper.get(options, function (error, response, body) { + const promise = cloudscraper.get(options, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledOnceWithExactly(expectedParams); expect(body).to.be.equal(expectedBody); @@ -62,16 +62,16 @@ describe('Cloudscraper', function () { }); it('should return json', function (done) { - var expectedBody = { a: 'test' }; + const expectedBody = { a: 'test' }; helper.router.get('/test', function (req, res) { res.send(expectedBody); }); - var expectedParams = helper.extendParams({ json: true }); - var options = { uri: uri, json: true }; + const expectedParams = helper.extendParams({ json: true }); + const options = { uri: uri, json: true }; - var promise = cloudscraper.get(options, function (error, response, body) { + const promise = cloudscraper.get(options, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledOnceWithExactly(expectedParams); expect(body).to.be.eql(expectedBody); @@ -86,10 +86,10 @@ describe('Cloudscraper', function () { }); // Disable status code checking - var expectedParams = helper.extendParams({ simple: false }); - var options = { uri: uri, simple: false }; + const expectedParams = helper.extendParams({ simple: false }); + const options = { uri: uri, simple: false }; - var promise = cloudscraper.get(options, function (error, response, body) { + const promise = cloudscraper.get(options, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledOnceWithExactly(expectedParams); expect(body).to.be.equal('xyz'); @@ -103,7 +103,7 @@ describe('Cloudscraper', function () { res.send(requestedPage); }); - var promise = cloudscraper.get(uri, function (error, response, body) { + const promise = cloudscraper.get(uri, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); expect(body).to.be.equal(requestedPage); @@ -113,13 +113,13 @@ describe('Cloudscraper', function () { }); it('should not trigger any error if recaptcha is present in page not protected by CF', function (done) { - var expectedBody = helper.getFixture('page_with_recaptcha.html'); + const expectedBody = helper.getFixture('page_with_recaptcha.html'); helper.router.get('/test', function (req, res) { res.send(expectedBody); }); - var promise = cloudscraper.get(uri, function (error, response, body) { + const promise = cloudscraper.get(uri, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); expect(body).to.be.equal(expectedBody); @@ -138,7 +138,7 @@ describe('Cloudscraper', function () { }); // Second call to Request will have challenge solution - var expectedParams = helper.extendParams({ + const expectedParams = helper.extendParams({ uri: helper.resolve('/cdn-cgi/l/chk_jschl'), qs: { 'jschl_vc': '89cdff5eaa25923e0f26e29e5195dce9', @@ -152,7 +152,7 @@ describe('Cloudscraper', function () { challengesToSolve: 2 }); - var promise = cloudscraper.get(uri, function (error, response, body) { + const promise = cloudscraper.get(uri, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledTwice; @@ -176,7 +176,7 @@ describe('Cloudscraper', function () { }); // Second call to Request will have challenge solution - var expectedParams = helper.extendParams({ + const expectedParams = helper.extendParams({ uri: helper.resolve('/cdn-cgi/l/chk_jschl'), qs: { 'jschl_vc': '346b959db0cfa38f9938acc11d6e1e6e', @@ -190,7 +190,7 @@ describe('Cloudscraper', function () { challengesToSolve: 2 }); - var promise = cloudscraper.get(uri, function (error, response, body) { + const promise = cloudscraper.get(uri, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledTwice; @@ -214,7 +214,7 @@ describe('Cloudscraper', function () { }); // Second call to Request will have challenge solution - var expectedParams = helper.extendParams({ + const expectedParams = helper.extendParams({ uri: helper.resolve('/cdn-cgi/l/chk_jschl'), qs: { 'jschl_vc': '18e0eb4e7cc844880cd9822df9d8546e', @@ -228,7 +228,7 @@ describe('Cloudscraper', function () { challengesToSolve: 2 }); - var promise = cloudscraper.get(uri, function (error, response, body) { + const promise = cloudscraper.get(uri, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledTwice; @@ -251,7 +251,7 @@ describe('Cloudscraper', function () { }); // Second call to Request will have challenge solution - var expectedParams = helper.extendParams({ + const expectedParams = helper.extendParams({ uri: helper.resolve('/cdn-cgi/l/chk_jschl'), qs: { 's': '08ee9f79382c9f784ef868f239a0984261a28b2f-1553213547-1800-AXjMT2d0Sx0fifn2gHCBp7sjO3hmbH5Pab9lPE92HxBLetotfG2HQ0U8ioQ2CJwOMGV5pmmBmffUDmmyxIyCuRCBOxecZXzYCBZZReVFCTXgIlpXL8ZcztRhE9Bm3BNGfg==', @@ -265,7 +265,7 @@ describe('Cloudscraper', function () { challengesToSolve: 2 }); - var promise = cloudscraper.get(uri, function (error, response, body) { + const promise = cloudscraper.get(uri, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledTwice; @@ -280,7 +280,7 @@ describe('Cloudscraper', function () { it('should resolve 2 consequent challenges', function (done) { // Cloudflare is enabled for site. It returns a page with JS challenge - var additionalChallenge = true; + let additionalChallenge = true; helper.router .get('/test', function (req, res) { @@ -296,8 +296,8 @@ describe('Cloudscraper', function () { } }); - var firstParams = helper.extendParams({ resolveWithFullResponse: true }); - var secondParams = helper.extendParams({ + const firstParams = helper.extendParams({ resolveWithFullResponse: true }); + const secondParams = helper.extendParams({ resolveWithFullResponse: true, uri: helper.resolve('/cdn-cgi/l/chk_jschl'), qs: { @@ -312,7 +312,7 @@ describe('Cloudscraper', function () { challengesToSolve: 2 }); - var thirdParams = helper.extendParams({ + const thirdParams = helper.extendParams({ resolveWithFullResponse: true, uri: helper.resolve('/cdn-cgi/l/chk_jschl'), qs: { @@ -328,9 +328,9 @@ describe('Cloudscraper', function () { challengesToSolve: 1 }); - var options = { uri: uri, resolveWithFullResponse: true }; + const options = { uri: uri, resolveWithFullResponse: true }; - var promise = cloudscraper.get(options, function (error, response, body) { + const promise = cloudscraper.get(options, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledThrice; @@ -349,16 +349,16 @@ describe('Cloudscraper', function () { res.send(requestedPage); }); - var formData = { some: 'data' }; + const formData = { some: 'data' }; - var expectedParams = helper.extendParams({ + const expectedParams = helper.extendParams({ method: 'POST', formData: formData }); - var options = { uri: uri, formData: formData }; + const options = { uri: uri, formData: formData }; - var promise = cloudscraper.post(options, function (error, response, body) { + const promise = cloudscraper.post(options, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledOnceWithExactly(expectedParams); expect(body).to.be.equal(requestedPage); @@ -372,9 +372,9 @@ describe('Cloudscraper', function () { res.send(requestedPage); }); - var expectedParams = helper.extendParams({ method: 'DELETE' }); + const expectedParams = helper.extendParams({ method: 'DELETE' }); - var promise = cloudscraper.delete(uri, function (error, response, body) { + const promise = cloudscraper.delete(uri, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledOnceWithExactly(expectedParams); expect(body).to.be.equal(requestedPage); @@ -388,12 +388,12 @@ describe('Cloudscraper', function () { res.send(requestedPage); }); - var expectedBody = Buffer.from(requestedPage, 'utf8'); - var expectedParams = helper.extendParams({ realEncoding: null }); + const expectedBody = Buffer.from(requestedPage, 'utf8'); + const expectedParams = helper.extendParams({ realEncoding: null }); - var options = { uri: uri, encoding: null }; + const options = { uri: uri, encoding: null }; - var promise = cloudscraper.get(options, function (error, response, body) { + const promise = cloudscraper.get(options, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledOnceWithExactly(expectedParams); expect(body).to.be.eql(expectedBody); @@ -412,12 +412,12 @@ describe('Cloudscraper', function () { } }); - var expectedParams = helper.extendParams({ challengesToSolve: 2 }); + const expectedParams = helper.extendParams({ challengesToSolve: 2 }); // We need to override cloudscraper's default jar for this test - var options = { uri: uri, jar: helper.defaultParams.jar }; + const options = { uri: uri, jar: helper.defaultParams.jar }; - var promise = cloudscraper.get(options, function (error, response, body) { + const promise = cloudscraper.get(options, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledTwice; @@ -443,12 +443,12 @@ describe('Cloudscraper', function () { } }); - var firstParams = helper.extendParams({ + const firstParams = helper.extendParams({ proxy: helper.uri.href, uri: 'http://example-site.dev/test' }); - var secondParams = helper.extendParams({ + const secondParams = helper.extendParams({ proxy: helper.uri.href, uri: 'http://example-site.dev/cdn-cgi/l/chk_jschl', qs: { @@ -463,12 +463,12 @@ describe('Cloudscraper', function () { challengesToSolve: 2 }); - var options = { + const options = { proxy: helper.uri.href, uri: 'http://example-site.dev/test' }; - var promise = cloudscraper.get(options, function (error, response, body) { + const promise = cloudscraper.get(options, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledTwice; @@ -491,16 +491,16 @@ describe('Cloudscraper', function () { } }); - var customJar = request.jar(); + const customJar = request.jar(); - var firstParams = helper.extendParams({ jar: customJar }); - var secondParams = helper.extendParams({ + const firstParams = helper.extendParams({ jar: customJar }); + const secondParams = helper.extendParams({ jar: customJar, challengesToSolve: 2 }); // We need to override cloudscraper's default jar for this test - var options = { uri: uri, jar: customJar }; + const options = { uri: uri, jar: customJar }; customJar.setCookie('custom cookie', 'http://custom-site.dev/'); @@ -513,7 +513,7 @@ describe('Cloudscraper', function () { expect(body).to.be.equal(requestedPage); - var customCookie = customJar.getCookieString('http://custom-site.dev/'); + let customCookie = customJar.getCookieString('http://custom-site.dev/'); expect(customCookie).to.equal('custom cookie'); cloudscraper.get(options, function (error) { @@ -531,7 +531,7 @@ describe('Cloudscraper', function () { it('should define custom defaults function', function (done) { expect(cloudscraper.defaults).to.not.equal(request.defaults); - var custom = cloudscraper.defaults({ challengesToSolve: 5 }); + const custom = cloudscraper.defaults({ challengesToSolve: 5 }); expect(custom.defaults).to.equal(cloudscraper.defaults); done(); }); @@ -545,11 +545,11 @@ describe('Cloudscraper', function () { res.sendFixture('page_with_emails.html'); }); - var cf = cloudscraper.defaults({ decodeEmails: true }); + const cf = cloudscraper.defaults({ decodeEmails: true }); - var firstParams = helper.extendParams({ decodeEmails: true }); + const firstParams = helper.extendParams({ decodeEmails: true }); - var promise = cf.get(uri, function (error, response, body) { + const promise = cf.get(uri, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledTwice; @@ -570,14 +570,14 @@ describe('Cloudscraper', function () { res.send(requestedPage); }); - var cf = cloudscraper.defaults({ baseUrl: helper.uri.href }); + const cf = cloudscraper.defaults({ baseUrl: helper.uri.href }); - var firstParams = helper.extendParams({ + const firstParams = helper.extendParams({ baseUrl: helper.uri.href, uri: '/test' }); - var promise = cf.get('/test', function (error, response, body) { + const promise = cf.get('/test', function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledTwice; @@ -599,16 +599,16 @@ describe('Cloudscraper', function () { res.send(requestedPage); }); - var expectedParams = helper.extendParams({ cloudflareTimeout: 50 }); + const expectedParams = helper.extendParams({ cloudflareTimeout: 50 }); - var start = Date.now(); - var options = { uri: uri, cloudflareTimeout: 50 }; + const start = Date.now(); + const options = { uri: uri, cloudflareTimeout: 50 }; - var promise = cloudscraper.get(options, function (error) { + const promise = cloudscraper.get(options, function (error) { expect(error).to.be.null; expect(Request.firstCall).to.be.calledWithExactly(expectedParams); - var elapsed = Date.now() - start; + const elapsed = Date.now() - start; // Aiming to be within ~150ms of specified timeout expect(elapsed >= 50 && elapsed <= 200).to.be.ok; }); @@ -617,8 +617,8 @@ describe('Cloudscraper', function () { }); it('sandbox.document.getElementById should not error', function (done) { - var html = helper.getFixture('js_challenge_21_03_2019.html'); - var statements = 'document.getElementById("missing");'.repeat(2); + const html = helper.getFixture('js_challenge_21_03_2019.html'); + const statements = 'document.getElementById("missing");'.repeat(2); helper.router .get('/test', function (req, res) { @@ -629,7 +629,7 @@ describe('Cloudscraper', function () { res.send(requestedPage); }); - var promise = cloudscraper.get(uri, function (error, response, body) { + const promise = cloudscraper.get(uri, function (error, response, body) { expect(error).to.be.null; expect(Request).to.be.calledTwice; }); diff --git a/test/test-rp.js b/test/test-rp.js index f53dcdf..9c6b37b 100644 --- a/test/test-rp.js +++ b/test/test-rp.js @@ -2,19 +2,19 @@ /* eslint-env node, mocha */ 'use strict'; -var cloudscraper = require('../index'); -var request = require('request-promise'); -var helper = require('./helper'); +const cloudscraper = require('../index'); +const request = require('request-promise'); +const helper = require('./helper'); -var sinon = require('sinon'); -var expect = require('chai').expect; +const sinon = require('sinon'); +const expect = require('chai').expect; describe('Cloudscraper', function () { - var sandbox; - var Request; - var uri; + let sandbox; + let Request; + let uri; - var requestedPage = helper.getFixture('requested_page.html'); + const requestedPage = helper.getFixture('requested_page.html'); before(function (done) { helper.listen(function () { @@ -46,7 +46,7 @@ describe('Cloudscraper', function () { res.send(requestedPage); }); - var expectedParams = helper.extendParams({ callback: undefined }); + const expectedParams = helper.extendParams({ callback: undefined }); return cloudscraper.get(uri).then(function (body) { expect(Request).to.be.calledOnceWithExactly(expectedParams); @@ -59,7 +59,7 @@ describe('Cloudscraper', function () { res.send(requestedPage); }); - var expectedParams = helper.extendParams({ + const expectedParams = helper.extendParams({ callback: undefined, resolveWithFullResponse: true }); @@ -67,7 +67,7 @@ describe('Cloudscraper', function () { // The method is implicitly GET delete expectedParams.method; - var options = { + const options = { uri: uri, resolveWithFullResponse: true }; @@ -86,7 +86,7 @@ describe('Cloudscraper', function () { res.endAbruptly(); }); - var caught = false; + let caught = false; cloudscraper(uri) .catch(function () { @@ -102,7 +102,7 @@ describe('Cloudscraper', function () { res.endAbruptly(); }); - var caught = false; + let caught = false; cloudscraper(uri) .then(function () { From 3930c910c283e8e5624ff0ca764b19861849c2e9 Mon Sep 17 00:00:00 2001 From: pro-src Date: Mon, 15 Apr 2019 17:54:19 -0500 Subject: [PATCH 03/15] Adjust document.getElementById to allow empty nodes --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index bee01ac..9ee4e2a 100644 --- a/index.js +++ b/index.js @@ -385,7 +385,7 @@ function createSandbox (options = {}) { }, getElementById: function (id) { if (keys.indexOf(id) === -1) { - const re = new RegExp(' id=[\'"]?' + id + '[^>]*>([^<]+)'); + const re = new RegExp(' id=[\'"]?' + id + '[^>]*>([^<]*)'); const match = body.match(re); keys.push(id); From e32c2852d0de55c6d1ca9ff42efe35e92f2a840e Mon Sep 17 00:00:00 2001 From: pro-src Date: Mon, 15 Apr 2019 17:55:25 -0500 Subject: [PATCH 04/15] Add optional Brotli support --- .travis.yml | 13 ++++++++-- index.js | 15 +++++++++++- lib/brotli.js | 36 +++++++++++++++++++++++++++ lib/headers.js | 6 +++++ package.json | 3 +++ test/helper.js | 3 ++- test/test-brotli.js | 60 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 lib/brotli.js create mode 100644 test/test-brotli.js diff --git a/.travis.yml b/.travis.yml index a585d16..0af7ea6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +sudo: false language: node_js @@ -7,10 +8,18 @@ node_js: - 8 - 6 -sudo: false +matrix: + include: + - node_js: node + env: BROTLI=1 + - node_js: 6 + env: BROTLI=1 + before_install: npm i --save-only request brotli +before_install: npm i --save-only request +install: npm i after_success: npm run coverage notifications: webhooks: https://www.travisbuddy.com/?insertMode=update - on_success: never \ No newline at end of file + on_success: never diff --git a/index.js b/index.js index 9ee4e2a..ce60026 100644 --- a/index.js +++ b/index.js @@ -5,6 +5,7 @@ const requestModule = require('request-promise'); const errors = require('./errors'); const decodeEmails = require('./lib/email-decode.js'); const getDefaultHeaders = require('./lib/headers'); +const brotli = require('./lib/brotli'); const HOST = Symbol('host'); @@ -32,7 +33,9 @@ function defaults (params) { // Support only this max challenges in row. If CF returns more, throw an error challengesToSolve: 3, // Remove Cloudflare's email protection - decodeEmails: false + decodeEmails: false, + // Support gzip encoded responses + gzip: true }; // Object.assign requires at least nodejs v4, request only test/supports v6+ @@ -149,6 +152,16 @@ function onRequestResponse (options, error, response, body) { return callback(null, response, body); } + // Decompress brotli compressed responses + if (/\bbr\b/i.test('' + response.caseless.get('content-encoding'))) { + if (!brotli.isAvailable) { + const cause = 'Received a Brotli compressed response. Please install brotli'; + return callback(new errors.RequestError(cause, options, response)); + } + + response.body = body = brotli.decompress(body); + } + if (response.isCloudflare && response.isHTML) { onCloudflareResponse(options, response, body); } else { diff --git a/lib/brotli.js b/lib/brotli.js new file mode 100644 index 0000000..a1ddcf9 --- /dev/null +++ b/lib/brotli.js @@ -0,0 +1,36 @@ +'use strict'; + +const zlib = require('zlib'); + +const brotli = module.exports; +// Convenience boolean used to check for brotli support +brotli.isAvailable = true; +// Exported for tests +brotli.optional = optional; + +// Check for node's built-in brotli support +if (typeof zlib.brotliDecompress === 'function') { + brotli.decompress = function (buf) { + return zlib.brotliDecompressSync(buf); + }; +} else { + optional(require); +} + +function optional (require) { + try { + // Require the NPM installed brotli + const decompress = require('brotli/decompress'); + + brotli.decompress = function (buf) { + return Buffer.from(decompress(buf)); + }; + } catch (error) { + brotli.isAvailable = false; + + // Don't throw an exception if the module is not installed + if (error.code !== 'MODULE_NOT_FOUND') { + throw error; + } + } +} diff --git a/lib/headers.js b/lib/headers.js index e97835a..c9be0ae 100644 --- a/lib/headers.js +++ b/lib/headers.js @@ -1,6 +1,7 @@ 'use strict'; const chromeData = require('./browsers').chrome; +const useBrotli = require('./brotli').isAvailable; module.exports = function getHeaders (defaults) { const headers = getChromeHeaders(random(chromeData)); @@ -16,5 +17,10 @@ function getChromeHeaders (options) { headers['User-Agent'] = random(options['User-Agent']); + if (!useBrotli && headers['Accept-Encoding']) { + headers['Accept-Encoding'] = + headers['Accept-Encoding'].replace(/,?\s*\bbr\b\s*/i, ''); + } + return headers; } diff --git a/package.json b/package.json index 8883d98..5aa7402 100644 --- a/package.json +++ b/package.json @@ -57,5 +57,8 @@ "nyc": "^13.3.0", "sinon": "^7.2.4", "sinon-chai": "^3.3.0" + }, + "peerDependencies": { + "brotli": "^1.3.2" } } diff --git a/test/helper.js b/test/helper.js index f251a55..146803a 100644 --- a/test/helper.js +++ b/test/helper.js @@ -30,7 +30,8 @@ var helper = { cloudflareTimeout: 1, cloudflareMaxTimeout: 30000, challengesToSolve: 3, - decodeEmails: false + decodeEmails: false, + gzip: true }; }, getFixture: function (fileName) { diff --git a/test/test-brotli.js b/test/test-brotli.js new file mode 100644 index 0000000..5a4bc17 --- /dev/null +++ b/test/test-brotli.js @@ -0,0 +1,60 @@ +/* eslint-disable no-unused-expressions */ +/* eslint-env node, mocha */ +'use strict'; + +const brotli = require('../lib/brotli'); +const helper = require('./helper'); +const zlib = require('zlib'); + +const sinon = require('sinon'); +const expect = require('chai').expect; + +(process.env.BROTLI ? describe : describe.skip)('Brotli', function () { + it('should be available', function () { + expect(brotli.isAvailable).to.be.true; + }); + + it('should have a decompress method', function () { + expect(brotli.decompress).to.be.a('function'); + }); + + it('decompress() should accept exactly 1 argument', function () { + expect(brotli.decompress.length).to.equal(1); + }); + + it('decompress() should accept buffer as input', function () { + const data = Buffer.from([0x0b, 0x01, 0x80, 0x61, 0x62, 0x63, 0x03]); + const result = brotli.decompress(data); + + expect(result).to.be.instanceof(Buffer); + expect(result.toString('utf8')).to.equal('abc'); + }); + + (zlib.brotliCompressSync ? it : it.skip)('[internal] decompress() should produce the expected result', function () { + const input = helper.getFixture('captcha.html'); + const data = zlib.brotliCompressSync(Buffer.from(input, 'utf8')); + const result = brotli.decompress(data); + + expect(result).to.be.instanceof(Buffer); + expect(result.toString('utf8')).to.equal(input); + }); + + (zlib.brotliCompressSync ? it.skip : it)('[external] decompress() should produce the expected result', function () { + const input = helper.getFixture('captcha.html'); + // Try increasing the timeout if this fails on your system. + const data = require('brotli').compress(Buffer.from(input, 'utf8')); + const result = brotli.decompress(Buffer.from(data)); + + expect(result).to.be.instanceof(Buffer); + expect(result.toString('utf8')).to.equal(input); + }); + + it('optional() should throw an error if the module contains an error', function () { + const spy = sinon.spy(function () { + // This method should throw if called without arguments + brotli.optional(); + }); + + expect(spy).to.throw(); + }); +}); From 7e333bea82a6cb6a95fd3bc925169fbc14862254 Mon Sep 17 00:00:00 2001 From: pro-src Date: Sat, 13 Apr 2019 23:38:35 -0500 Subject: [PATCH 05/15] Move request to peerDependencies --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5aa7402..bf33e05 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,6 @@ "license": "MIT", "homepage": "https://github.com/codemanki/cloudscraper", "dependencies": { - "request": "^2.88.0", "request-promise": "^4.2.4" }, "devDependencies": { @@ -59,6 +58,7 @@ "sinon-chai": "^3.3.0" }, "peerDependencies": { + "request": "^2.88.0", "brotli": "^1.3.2" } } From 14eb38190cacdf0205293b3ddb785f28e3ac2313 Mon Sep 17 00:00:00 2001 From: pro-src Date: Sun, 14 Apr 2019 22:30:12 -0500 Subject: [PATCH 06/15] Support reCAPTCHA solving --- index.js | 247 +++++++++++++++++++++++++++++++------------------ lib/sandbox.js | 47 ++++++++++ 2 files changed, 203 insertions(+), 91 deletions(-) create mode 100644 lib/sandbox.js diff --git a/index.js b/index.js index ce60026..51c36b3 100644 --- a/index.js +++ b/index.js @@ -1,19 +1,19 @@ 'use strict'; -const vm = require('vm'); const requestModule = require('request-promise'); -const errors = require('./errors'); +const sandbox = require('./lib/sandbox'); const decodeEmails = require('./lib/email-decode.js'); const getDefaultHeaders = require('./lib/headers'); const brotli = require('./lib/brotli'); -const HOST = Symbol('host'); +const { + RequestError, + CaptchaError, + CloudflareError, + ParserError +} = require('./errors'); -const VM_OPTIONS = { - contextOrigin: 'cloudflare:challenge.js', - contextCodeGeneration: { strings: true, wasm: false }, - timeout: 5000 -}; +const HOST = Symbol('host'); module.exports = defaults.call(requestModule); @@ -43,6 +43,7 @@ function defaults (params) { const cloudscraper = requestModule.defaults .call(this, defaultParams, function (options) { + validateRequest(options); return performRequest(options, true); }); @@ -62,9 +63,7 @@ function defaults (params) { return cloudscraper; } -// This function is wrapped to ensure that we get new options on first call. -// The options object is reused in subsequent calls when calling it directly. -function performRequest (options, isFirstRequest) { +function validateRequest (options) { // Prevent overwriting realEncoding in subsequent calls if (!('realEncoding' in options)) { // Can't just do the normal options.encoding || 'utf8' @@ -88,14 +87,20 @@ function performRequest (options, isFirstRequest) { 'got ' + typeof (options.cloudflareMaxTimeout) + ' instead.'); } - // This should be the default export of either request or request-promise. - const requester = options.requester; - - if (typeof requester !== 'function') { + if (typeof options.requester !== 'function') { throw new TypeError('Expected `requester` option to be a function, got ' + - typeof (requester) + ' instead.'); + typeof (options.requester) + ' instead.'); } +} +// This function is wrapped to ensure that we get new options on first call. +// The options object is reused in subsequent calls when calling it directly. +function performRequest (options, isFirstRequest) { + // This should be the default export of either request or request-promise. + const requester = options.requester; + + // Note that request is always an instanceof ReadableStream, EventEmitter + // If the requester is request-promise, it is also thenable. const request = requester(options); // We must define the host header ourselves to preserve case and order. @@ -140,7 +145,7 @@ function onRequestResponse (options, error, response, body) { // Encoding is null so body should be a buffer object if (error || !body || !body.toString) { // Pure request error (bad connection, wrong url, etc) - return callback(new errors.RequestError(error, options, response)); + return callback(new RequestError(error, options, response)); } response.responseStartTime = Date.now(); @@ -156,7 +161,7 @@ function onRequestResponse (options, error, response, body) { if (/\bbr\b/i.test('' + response.caseless.get('content-encoding'))) { if (!brotli.isAvailable) { const cause = 'Received a Brotli compressed response. Please install brotli'; - return callback(new errors.RequestError(cause, options, response)); + return callback(new RequestError(cause, options, response)); } response.body = body = brotli.decompress(body); @@ -165,7 +170,7 @@ function onRequestResponse (options, error, response, body) { if (response.isCloudflare && response.isHTML) { onCloudflareResponse(options, response, body); } else { - processResponseBody(options, response, body); + onRequestComplete(options, response, body); } } @@ -178,7 +183,7 @@ function onCloudflareResponse (options, response, body) { if (body.length < 1) { // This is a 4xx-5xx Cloudflare response with an empty body. - return callback(new errors.CloudflareError(response.statusCode, options, response)); + return callback(new CloudflareError(response.statusCode, options, response)); } stringBody = body.toString('utf8'); @@ -186,29 +191,34 @@ function onCloudflareResponse (options, response, body) { try { validate(options, response, stringBody); } catch (error) { + if (error instanceof CaptchaError && typeof options.onCaptcha === 'function') { + // Give users a chance to solve the reCAPTCHA via services such as anti-captcha.com + return onCaptcha(options, response, stringBody); + } + return callback(error); } isChallenge = stringBody.indexOf('a = document.getElementById(\'jschl-answer\');') !== -1; if (isChallenge) { - return solveChallenge(options, response, stringBody); + return onChallenge(options, response, stringBody); } isRedirectChallenge = stringBody.indexOf('You are being redirected') !== -1 || stringBody.indexOf('sucuri_cloudproxy_js') !== -1; if (isRedirectChallenge) { - return setCookieAndReload(options, response, stringBody); + return onRedirectChallenge(options, response, stringBody); } // 503 status is always a challenge if (response.statusCode === 503) { - return solveChallenge(options, response, stringBody); + return onChallenge(options, response, stringBody); } // All is good - processResponseBody(options, response, body); + onRequestComplete(options, response, body); } function validate (options, response, body) { @@ -216,7 +226,9 @@ function validate (options, response, body) { // Finding captcha if (body.indexOf('why_captcha') !== -1 || /cdn-cgi\/l\/chk_captcha/i.test(body)) { - throw new errors.CaptchaError('captcha', options, response); + // Convenience boolean + response.isCaptcha = true; + throw new CaptchaError('captcha', options, response); } // Trying to find '1006' @@ -224,13 +236,13 @@ function validate (options, response, body) { if (match) { let code = parseInt(match[1]); - throw new errors.CloudflareError(code, options, response); + throw new CloudflareError(code, options, response); } return false; } -function solveChallenge (options, response, body) { +function onChallenge (options, response, body) { const callback = options.callback; const uri = response.request.uri; // The query string to send back to Cloudflare @@ -241,45 +253,40 @@ function solveChallenge (options, response, body) { if (options.challengesToSolve === 0) { cause = 'Cloudflare challenge loop'; - error = new errors.CloudflareError(cause, options, response); + error = new CloudflareError(cause, options, response); error.errorType = 4; return callback(error); } let timeout = parseInt(options.cloudflareTimeout); - let sandbox; let match; match = body.match(/name="s" value="(.+?)"/); - if (match) { payload.s = match[1]; } match = body.match(/name="jschl_vc" value="(\w+)"/); - if (!match) { cause = 'challengeId (jschl_vc) extraction failed'; - return callback(new errors.ParserError(cause, options, response)); + return callback(new ParserError(cause, options, response)); } payload.jschl_vc = match[1]; match = body.match(/name="pass" value="(.+?)"/); - if (!match) { cause = 'Attribute (pass) value extraction failed'; - return callback(new errors.ParserError(cause, options, response)); + return callback(new ParserError(cause, options, response)); } payload.pass = match[1]; match = body.match(/getElementById\('cf-content'\)[\s\S]+?setTimeout.+?\r?\n([\s\S]+?a\.value\s*=.+?)\r?\n(?:[^{<>]*},\s*(\d{4,}))?/); - if (!match) { cause = 'setTimeout callback extraction failed'; - return callback(new errors.ParserError(cause, options, response)); + return callback(new ParserError(cause, options, response)); } if (isNaN(timeout)) { @@ -295,7 +302,7 @@ function solveChallenge (options, response, body) { } } else { cause = 'Failed to parse challenge timeout'; - return callback(new errors.ParserError(cause, options, response)); + return callback(new ParserError(cause, options, response)); } } @@ -303,16 +310,16 @@ function solveChallenge (options, response, body) { response.challenge = match[1] + '; a.value'; try { - sandbox = createSandbox({ uri: uri, body: body }); - payload.jschl_answer = vm.runInNewContext(response.challenge, sandbox, VM_OPTIONS); + const ctx = new sandbox.Context({ hostname: uri.hostname, body }); + payload.jschl_answer = sandbox.eval(response.challenge, ctx); } catch (error) { error.message = 'Challenge evaluation failed: ' + error.message; - return callback(new errors.ParserError(error, options, response)); + return callback(new ParserError(error, options, response)); } if (isNaN(payload.jschl_answer)) { cause = 'Challenge answer is not a number'; - return callback(new errors.ParserError(cause, options, response)); + return callback(new ParserError(cause, options, response)); } // Prevent reusing the headers object to simplify unit testing. @@ -326,43 +333,141 @@ function solveChallenge (options, response, body) { } // Set the query string and decrement the number of challenges to solve. options.qs = payload; - options.challengesToSolve = options.challengesToSolve - 1; + options.challengesToSolve -= 1; // Make request with answer after delay. timeout -= Date.now() - response.responseStartTime; setTimeout(performRequest, timeout, options, false); } -function setCookieAndReload (options, response, body) { +// Parses the reCAPTCHA form and hands control over to the user +function onCaptcha (options, response, body) { const callback = options.callback; + // UDF that has the responsibility of returning control back to cloudscraper + const handler = options.onCaptcha; + // The form data to send back to Cloudflare + const payload = { /* s, g-re-captcha-response */ }; - const match = body.match(/S='([^']+)'/); + let cause; + let match; + + match = body.match(/]*)? id=["']?challenge-form['"]?(?: [^<>]*)?>([\S\s]*?)<\/form>/); + if (!match) { + cause = 'Challenge form extraction failed'; + return callback(new ParserError(cause, options, response)); + } + + // Defining response.challengeForm for debugging purposes + const form = response.challengeForm = match[1]; + + match = form.match(/\/recaptcha\/api\/fallback\?k=([^\s"'<>]*)/); + if (!match) { + // The site key wasn't inside the form so search the entire document + match = body.match(/data-sitekey=["']?([^\s"'<>]*)/); + if (!match) { + cause = 'Unable to find the reCAPTCHA site key'; + return callback(new ParserError(cause, options, response)); + } + } + + // Everything that is needed to solve the reCAPTCHA + response.captcha = { siteKey: match[1], form: payload }; + + // Adding formData + match = form.match(/]*)? name=[^<>]+(?:\/>|<\/input>)/g); + if (!match) { + cause = 'Challenge form is missing inputs'; + return callback(new ParserError(cause, options, response)); + } + + const inputs = match; + // Only adding inputs that have both a name and value defined + for (let name, value, i = 0; i < inputs.length; i++) { + name = inputs[i].match(/name=["']?([^\s"'<>]*)/); + if (name) { + value = inputs[i].match(/value=["']?([^\s"'<>]*)/); + if (value) { + payload[name[1]] = value[1]; + } + } + } + + // Sanity check + if (!payload['s']) { + cause = 'Challenge form is missing secret input'; + return callback(new ParserError(cause, options, response)); + } + + // The callback used to green light form submission + const submit = function (error) { + if (error) { + // Pass an user defined error back to the original request call + return callback(new CaptchaError(cause, options, response)); + } + + onSubmitCaptcha(options, response, body); + }; + + // This seems like an okay-ish API (fewer arguments to the handler) + response.captcha.submit = submit; + + // We're handing control over to the user now. + const thenable = handler(options, response, body); + // Handle the case where the user returns a promise + if (thenable && typeof thenable.then === 'function') { + // eslint-disable-next-line promise/catch-or-return + thenable.then(submit); + } +} + +function onSubmitCaptcha (options, response) { + const callback = options.callback; + const uri = response.request.uri; + if (!response.captcha.form['g-recaptcha-response']) { + const cause = 'Form submission without g-recaptcha-response'; + return callback(new CaptchaError(cause, options, response)); + } + + options.method = 'GET'; + options.qs = response.captcha.form; + // Use the original uri as the referer and to construct the form action. + options.headers['Referer'] = uri.href; + options.uri = uri.protocol + '//' + uri.host + '/cdn-cgi/l/chk_captcha'; + + performRequest(options, false); +} + +function onRedirectChallenge (options, response, body) { + const callback = options.callback; + const uri = response.request.uri; + + const match = body.match(/S='([^']+)'/); if (!match) { const cause = 'Cookie code extraction failed'; - return callback(new errors.ParserError(cause, options, response)); + return callback(new ParserError(cause, options, response)); } const base64EncodedCode = match[1]; response.challenge = Buffer.from(base64EncodedCode, 'base64').toString('ascii'); try { - const sandbox = createSandbox(); // Evaluate cookie setting code - vm.runInNewContext(response.challenge, sandbox, VM_OPTIONS); + const ctx = new sandbox.Context(); + sandbox.eval(response.challenge, ctx); - options.jar.setCookie(sandbox.document.cookie, response.request.uri.href, { ignoreError: true }); + options.jar.setCookie(ctx.document.cookie, uri.href, { ignoreError: true }); } catch (error) { error.message = 'Cookie code evaluation failed: ' + error.message; - return callback(new errors.ParserError(error, options, response)); + return callback(new ParserError(error, options, response)); } - options.challengesToSolve = options.challengesToSolve - 1; + options.challengesToSolve -= 1; performRequest(options, false); } -function processResponseBody (options, response, body) { +function onRequestComplete (options, response, body) { const callback = options.callback; if (typeof options.realEncoding === 'string') { @@ -379,43 +484,3 @@ function processResponseBody (options, response, body) { callback(null, response, body); } - -function createSandbox (options = {}) { - if (options.body) { - const body = options.body; - const href = 'http://' + options.uri.hostname + '/'; - const cache = Object.create(null); - const keys = []; - - // Sandbox for standard IUAM JS challenge - return Object.assign({ - atob: function (str) { - return Buffer.from(str, 'base64').toString('binary'); - }, - document: { - createElement: function () { - return { firstChild: { href: href } }; - }, - getElementById: function (id) { - if (keys.indexOf(id) === -1) { - const re = new RegExp(' id=[\'"]?' + id + '[^>]*>([^<]*)'); - const match = body.match(re); - - keys.push(id); - cache[id] = match === null ? match : { innerHTML: match[1] }; - } - - return cache[id]; - } - } - }, options.context); - } - - // Sandbox used in setCookieAndReload - return Object.assign({ - location: { - reload: function () {} - }, - document: {} - }, options.context); -} diff --git a/lib/sandbox.js b/lib/sandbox.js new file mode 100644 index 0000000..ed269d2 --- /dev/null +++ b/lib/sandbox.js @@ -0,0 +1,47 @@ +const vm = require('vm'); + +const VM_OPTIONS = { + contextOrigin: 'cloudflare:challenge.js', + contextCodeGeneration: { strings: true, wasm: false }, + timeout: 5000 +}; + +module.exports = { eval: evaluate, Context }; + +function evaluate (code, ctx) { + return vm.runInNewContext(code, ctx, VM_OPTIONS); +} + +// Global context used to evaluate standard IUAM JS challenge +function Context (options) { + if (!options) options = { body: '', hostname: '' }; + + const body = options.body; + const href = 'http://' + options.hostname + '/'; + const cache = Object.create(null); + const keys = []; + + this.atob = function (str) { + return Buffer.from(str, 'base64').toString('binary'); + }; + + // Used for eval during onRedirectChallenge + this.location = { reload: function () {} }; + + this.document = { + createElement: function () { + return { firstChild: { href: href } }; + }, + getElementById: function (id) { + if (keys.indexOf(id) === -1) { + const re = new RegExp(' id=[\'"]?' + id + '[^>]*>([^<]*)'); + const match = body.match(re); + + keys.push(id); + cache[id] = match === null ? match : { innerHTML: match[1] }; + } + + return cache[id]; + } + }; +} From 12bec43beed6046b182ea30d7a08c451de7c4344 Mon Sep 17 00:00:00 2001 From: pro-src Date: Mon, 15 Apr 2019 04:51:02 -0500 Subject: [PATCH 07/15] Add missing use strict statements --- lib/email-decode.js | 2 ++ lib/sandbox.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/email-decode.js b/lib/email-decode.js index 3a461de..900acf1 100644 --- a/lib/email-decode.js +++ b/lib/email-decode.js @@ -1,3 +1,5 @@ +'use strict'; + const pattern = ( // Opening tag // $1 = TAG_NAME diff --git a/lib/sandbox.js b/lib/sandbox.js index ed269d2..17bad8b 100644 --- a/lib/sandbox.js +++ b/lib/sandbox.js @@ -1,3 +1,5 @@ +'use strict'; + const vm = require('vm'); const VM_OPTIONS = { From cc1313a32baf482abdc090df07d910282d28bb01 Mon Sep 17 00:00:00 2001 From: pro-src Date: Mon, 15 Apr 2019 20:00:33 -0500 Subject: [PATCH 08/15] Add test/fixture for latest challenge --- test/fixtures/js_challenge_10_04_2019.html | 96 ++++++++++++++++++++++ test/test-index.js | 36 ++++++++ 2 files changed, 132 insertions(+) create mode 100644 test/fixtures/js_challenge_10_04_2019.html diff --git a/test/fixtures/js_challenge_10_04_2019.html b/test/fixtures/js_challenge_10_04_2019.html new file mode 100644 index 0000000..03d780d --- /dev/null +++ b/test/fixtures/js_challenge_10_04_2019.html @@ -0,0 +1,96 @@ + + + + + + + + + Just a moment... + + + + + + + + + + + + +
+
+ + + +
+ + + + +
+ + + +
+ +
+ + diff --git a/test/test-index.js b/test/test-index.js index cfeca46..26bb747 100644 --- a/test/test-index.js +++ b/test/test-index.js @@ -278,6 +278,42 @@ describe('Cloudscraper', function () { expect(promise).to.eventually.equal(requestedPage).and.notify(done); }); + it('should resolve challenge (version as on 10.04.2019) and then return page', function (done) { + helper.router + .get('/test', function (req, res) { + res.sendChallenge('js_challenge_10_04_2019.html'); + }) + .get('/cdn-cgi/l/chk_jschl', function (req, res) { + res.send(requestedPage); + }); + + const expectedParams = helper.extendParams({ + uri: helper.resolve('/cdn-cgi/l/chk_jschl'), + qs: { + 's': 'f3b4838af97b6cb02b3c8b1e0f149daf27dbee61-1555369946-1800-AakWW8TP/PRVIBQ2t2QmkJFEmb8TAmeIE7/GS7OUCF+d/7LncO0Zwye3YaCZyfhCfRyQogtebFuSWk2ANVV0pDSXqJ/q5qe0URcQQ2NNaGVMuPVrLh/OrUqD2QUPn0dWGA==', + 'jschl_vc': '686d6bea02e6d172aa64f102a684228c', + 'jschl_answer': String(9.8766929385 + helper.uri.hostname.length), + 'pass': '1555369950.717-6S1r4kzOYK' + }, + headers: { + 'Referer': uri + }, + challengesToSolve: 2 + }); + + const promise = cloudscraper.get(uri, function (error, response, body) { + expect(error).to.be.null; + + expect(Request).to.be.calledTwice; + expect(Request.firstCall).to.be.calledWithExactly(helper.defaultParams); + expect(Request.secondCall).to.be.calledWithExactly(expectedParams); + + expect(body).to.be.equal(requestedPage); + }); + + expect(promise).to.eventually.equal(requestedPage).and.notify(done); + }); + it('should resolve 2 consequent challenges', function (done) { // Cloudflare is enabled for site. It returns a page with JS challenge let additionalChallenge = true; From 12809b3b8780acc632343558926b52e72e2d33cc Mon Sep 17 00:00:00 2001 From: pro-src Date: Tue, 16 Apr 2019 05:40:51 -0500 Subject: [PATCH 09/15] Add tests for v4 --- index.js | 15 +- lib/sandbox.js | 3 +- test/fixtures/cf_recaptcha_15_04_2019.html | 162 ++++++++++++++++++ test/helper.js | 4 + test/test-brotli.js | 2 +- test/test-captcha.js | 183 +++++++++++++++++++++ test/test-emails.js | 2 +- test/test-headers.js | 47 ++++++ test/test-sandbox.js | 107 ++++++++++++ test/test-timeout.js | 0 10 files changed, 519 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/cf_recaptcha_15_04_2019.html create mode 100644 test/test-captcha.js create mode 100644 test/test-headers.js create mode 100644 test/test-sandbox.js create mode 100644 test/test-timeout.js diff --git a/index.js b/index.js index 51c36b3..4967dbc 100644 --- a/index.js +++ b/index.js @@ -374,7 +374,7 @@ function onCaptcha (options, response, body) { response.captcha = { siteKey: match[1], form: payload }; // Adding formData - match = form.match(/]*)? name=[^<>]+(?:\/>|<\/input>)/g); + match = form.match(/]*)? name=[^<>]+>/g); if (!match) { cause = 'Challenge form is missing inputs'; return callback(new ParserError(cause, options, response)); @@ -402,7 +402,7 @@ function onCaptcha (options, response, body) { const submit = function (error) { if (error) { // Pass an user defined error back to the original request call - return callback(new CaptchaError(cause, options, response)); + return callback(new CaptchaError(error, options, response)); } onSubmitCaptcha(options, response, body); @@ -416,7 +416,14 @@ function onCaptcha (options, response, body) { // Handle the case where the user returns a promise if (thenable && typeof thenable.then === 'function') { // eslint-disable-next-line promise/catch-or-return - thenable.then(submit); + thenable.then(submit, function (error) { + if (!error) { + // The user broke their promise with a falsy error + submit(new Error('Falsy error')); + } else { + submit(error); + } + }); } } @@ -431,6 +438,8 @@ function onSubmitCaptcha (options, response) { options.method = 'GET'; options.qs = response.captcha.form; + // Prevent reusing the headers object to simplify unit testing. + options.headers = Object.assign({}, options.headers); // Use the original uri as the referer and to construct the form action. options.headers['Referer'] = uri.href; options.uri = uri.protocol + '//' + uri.host + '/cdn-cgi/l/chk_captcha'; diff --git a/lib/sandbox.js b/lib/sandbox.js index 17bad8b..96e3594 100644 --- a/lib/sandbox.js +++ b/lib/sandbox.js @@ -3,7 +3,8 @@ const vm = require('vm'); const VM_OPTIONS = { - contextOrigin: 'cloudflare:challenge.js', + filename: 'iuam-challenge.js', + contextOrigin: 'cloudflare:iuam-challenge.js', contextCodeGeneration: { strings: true, wasm: false }, timeout: 5000 }; diff --git a/test/fixtures/cf_recaptcha_15_04_2019.html b/test/fixtures/cf_recaptcha_15_04_2019.html new file mode 100644 index 0000000..0d9e7a2 --- /dev/null +++ b/test/fixtures/cf_recaptcha_15_04_2019.html @@ -0,0 +1,162 @@ + + + + + + +Attention Required! | Cloudflare + + + + + + + + + + + + + + + + + + + +
+ +
+
+

One more step

+

Please complete the security check to access example-site.dev

+
+ +
+
+
+
+ +
+
+ + +
+ +
+ + + +
+
+ +
+
+ + + +
+
+
+
+
+ +
+
+
+

Why do I have to complete a CAPTCHA?

+ +

Completing the CAPTCHA proves you are a human and gives you temporary access to the web property.

+
+ +
+

What can I do to prevent this in the future?

+ + +

If you are on a personal connection, like at home, you can run an anti-virus scan on your device to make sure it is not infected with malware.

+ +

If you are at an office or shared network, you can ask the network administrator to run a scan across the network looking for misconfigured or infected devices.

+ +
+
+
+ + +
+

+ Cloudflare Ray ID: 0000000000000000 + • + Your IP: 000.00.000.00 + • + Performance & security by Cloudflare + +

+
+ + +
+
+ + + + + diff --git a/test/helper.js b/test/helper.js index 146803a..52e457f 100644 --- a/test/helper.js +++ b/test/helper.js @@ -90,6 +90,10 @@ express.response.sendChallenge = function (fileName) { return this.cloudflare().status(503).sendFixture(fileName); }; +express.response.sendCaptcha = function (fileName) { + return this.cloudflare().status(403).sendFixture(fileName); +}; + express.response.endAbruptly = function () { this.connection.write( 'HTTP/1.1 500\r\n' + diff --git a/test/test-brotli.js b/test/test-brotli.js index 5a4bc17..3608d44 100644 --- a/test/test-brotli.js +++ b/test/test-brotli.js @@ -9,7 +9,7 @@ const zlib = require('zlib'); const sinon = require('sinon'); const expect = require('chai').expect; -(process.env.BROTLI ? describe : describe.skip)('Brotli', function () { +(process.env.BROTLI ? describe : describe.skip)('Brotli (lib)', function () { it('should be available', function () { expect(brotli.isAvailable).to.be.true; }); diff --git a/test/test-captcha.js b/test/test-captcha.js new file mode 100644 index 0000000..d223231 --- /dev/null +++ b/test/test-captcha.js @@ -0,0 +1,183 @@ +/* eslint-disable no-unused-expressions */ +/* eslint-env node, mocha */ +'use strict'; + +const cloudscraper = require('../index'); +const request = require('request-promise'); +const errors = require('../errors'); +const helper = require('./helper'); +const http = require('http'); + +const sinon = require('sinon'); +const expect = require('chai').expect; + +describe('Cloudscraper', function () { + let sandbox; + let Request; + let uri; + + const requestedPage = helper.getFixture('requested_page.html'); + + before(function (done) { + helper.listen(function () { + uri = helper.resolve('/test'); + + // Speed up tests + cloudscraper.defaultParams.cloudflareTimeout = 1; + done(); + }); + }); + + after(function () { + helper.server.close(); + }); + + beforeEach(function () { + // Prepare stubbed Request + sandbox = sinon.createSandbox(); + Request = sandbox.spy(request, 'Request'); + }); + + afterEach(function () { + helper.reset(); + sandbox.restore(); + }); + + it('should handle onCaptcha promise being rejected with a falsy error', function (done) { + helper.router.get('/test', function (req, res) { + res.sendCaptcha('cf_recaptcha_15_04_2019.html'); + }); + + const options = { + uri, + onCaptcha: function () { + // eslint-disable-next-line prefer-promise-reject-errors + return Promise.reject(); + } + }; + + const promise = cloudscraper.get(options, function (error) { + expect(error).to.be.instanceOf(errors.CaptchaError); + expect(error.error).to.be.an('error'); + expect(error).to.have.property('errorType', 1); + expect(error.message).to.include('Falsy error'); + }); + + expect(promise).to.be.rejectedWith(errors.CaptchaError).and.notify(done); + }); + + for (let stage = 0; stage < 4; stage++) { + const desc = { + 0: 'should resolve reCAPTCHA (version as on 10.04.2019) when user calls captcha.submit()', + 1: 'should callback with an error if user calls captcha.submit(error)', + 2: 'should resolve reCAPTCHA (version as on 10.04.2019) when the onCaptcha promise resolves', + 3: 'should callback with an error if the onCaptcha promise is rejected' + }; + + // Run this test 4 times + it(desc[stage], function (done) { + const secret = '6b132d85d185a8255f2451d48fe6a8bee7154ea2-1555377580-1800-AQ1azEkeDOnQP5ByOpwUU/RdbKrmMwHYpkaenRvjPXtB0w8Vbjn/Ceg62tfpp/lT799kjDLEMMuDkEMqQ7iO51kniWCQm00BQvDGl+D0h/WvXDWO96YXOUD3qrqUTuzO7QbUOinc8y8kedvOQkr4c0o='; + const siteKey = '6LfBixYUAAAAABhdHynFUIMA_sa4s-XsJvnjtgB0'; + const expectedError = new Error('anti-captcha failed!'); + + helper.router + .get('/test', function (req, res) { + res.sendCaptcha('cf_recaptcha_15_04_2019.html'); + }) + .get('/cdn-cgi/l/chk_captcha', function (req, res) { + res.send(requestedPage); + }); + + const onCaptcha = sinon.spy(function (options, response, body) { + expect(options).to.be.an('object'); + expect(response).to.be.instanceof(http.IncomingMessage); + expect(body).to.be.a('string'); + + sinon.assert.match(response, { + isCloudflare: true, + isHTML: true, + isCaptcha: true, + challengeForm: sinon.match.string, + captcha: sinon.match.object + }); + + sinon.assert.match(response.captcha, { + form: { s: secret }, + siteKey: siteKey, + submit: sinon.match.func + }); + + // Simulate what the user should do here + response.captcha.form['g-recaptcha-response'] = 'foobar'; + + switch (stage) { + case 0: + // User green lights form submission + response.captcha.submit(); + break; + case 1: + // User reports an error when solving the reCAPTCHA + response.captcha.submit(expectedError); + break; + case 2: + // User green lights form submission by resolving the returned promise + return Promise.resolve(); + case 3: + // User reports an error by rejecting the returned promise + return Promise.reject(expectedError); + } + }); + + const firstParams = helper.extendParams({ onCaptcha, uri }); + const secondParams = helper.extendParams({ + onCaptcha, + method: 'GET', + uri: helper.resolve('/cdn-cgi/l/chk_captcha'), + headers: { + Referer: uri + }, + qs: { + s: secret, + 'g-recaptcha-response': 'foobar' + } + }); + + const options = { onCaptcha, uri }; + + const promise = cloudscraper.get(options, function (error, response, body) { + switch (stage) { + case 0: + case 2: + expect(error).to.be.null; + + expect(onCaptcha).to.be.calledOnce; + + expect(Request).to.be.calledTwice; + expect(Request.firstCall).to.be.calledWithExactly(firstParams); + expect(Request.secondCall).to.be.calledWithExactly(secondParams); + + expect(body).to.be.equal(requestedPage); + break; + case 1: + case 3: + expect(error).to.be.instanceOf(errors.CaptchaError); + expect(error.error).to.be.an('error'); + expect(error).to.have.property('errorType', 1); + expect(error.message).to.include(expectedError.message); + break; + } + }); + + switch (stage) { + case 0: + case 2: + expect(promise).to.eventually.equal(requestedPage).and.notify(done); + break; + case 1: + case 3: + expect(promise).to.be.rejectedWith(errors.CaptchaError).and.notify(done); + break; + } + }); + } +}); diff --git a/test/test-emails.js b/test/test-emails.js index 74d0408..c66afe2 100644 --- a/test/test-emails.js +++ b/test/test-emails.js @@ -22,7 +22,7 @@ function genHTML (body) { '\n'; } -describe('Cloudscraper', function () { +describe('Email (lib)', function () { it('should not modify unprotected html', function () { const raw = genHTML(''); diff --git a/test/test-headers.js b/test/test-headers.js new file mode 100644 index 0000000..dc7beaa --- /dev/null +++ b/test/test-headers.js @@ -0,0 +1,47 @@ +/* eslint-disable no-unused-expressions */ +/* eslint-env node, mocha */ +'use strict'; + +const getHeaders = require('../lib/headers'); + +const sinon = require('sinon'); +const expect = require('chai').expect; + +describe('Headers (lib)', function () { + const browsers = require('../lib/browsers'); + + it('default export should be a function', function () { + expect(getHeaders).to.be.a('function'); + }); + + it('should always return an object with user agent', function () { + for (let i = 0; i < 100; i++) { + sinon.assert.match(getHeaders(), { 'User-Agent': sinon.match.string }); + } + + browsers.chrome.forEach(function (options) { + try { + expect(options['User-Agent']).to.be.an('array'); + expect(options['User-Agent'].length).to.be.above(0); + } catch (error) { + error.message += '\n\n' + JSON.stringify(options, null, 2); + throw error; + } + }); + }); + + it('should always retain insertion order', function () { + for (let keys, i = 0; i < 100; i++) { + keys = Object.keys(getHeaders({ Host: 'foobar' })); + expect(keys[0]).to.equal('Host'); + expect(keys[1]).to.equal('Connection'); + } + + for (let keys, i = 0; i < 100; i++) { + keys = Object.keys(getHeaders({ Host: 'foobar', 'N/A': null })); + expect(keys[0]).to.equal('Host'); + expect(keys[1]).to.equal('N/A'); + expect(keys[2]).to.equal('Connection'); + } + }); +}); diff --git a/test/test-sandbox.js b/test/test-sandbox.js new file mode 100644 index 0000000..faed30a --- /dev/null +++ b/test/test-sandbox.js @@ -0,0 +1,107 @@ +/* eslint-disable no-unused-expressions */ +/* eslint-env node, mocha */ +'use strict'; + +const sandbox = require('../lib/sandbox'); +const expect = require('chai').expect; + +describe('Sandbox (lib)', function () { + it('should export Context', function () { + expect(sandbox.Context).to.be.a('function'); + }); + + it('should export eval', function () { + expect(sandbox.eval).to.be.a('function'); + expect(sandbox.eval('0')).to.equal(0); + expect(sandbox.eval('true')).to.be.true; + expect(sandbox.eval('undefined')).to.equal(void 0); + expect(sandbox.eval('NaN')).to.be.a('number'); + expect(String(sandbox.eval('NaN'))).to.equal('NaN'); + }); + + it('new Context() should return an object', function () { + expect(new sandbox.Context()).to.be.an('object'); + }); + + it('Context() should define atob', function () { + const ctx = new sandbox.Context(); + + expect(ctx.atob).to.be.a('function'); + expect(ctx.atob('YWJj')).to.equal('abc'); + expect(sandbox.eval('atob("YWJj")', ctx)).to.equal('abc'); + }); + + it('Context() should define location.reload', function () { + const ctx = new sandbox.Context(); + + expect(ctx.location).to.be.an('object'); + expect(ctx.location.reload).to.be.an('function'); + + // This is a noop + ctx.location.reload(); + expect(sandbox.eval('location.reload()', ctx)).to.equal(void 0); + }); + + it('Context() should define document.createElement', function () { + let ctx = new sandbox.Context(); + let pseudoElement = { firstChild: { href: 'http:///' } }; + + expect(ctx.document).to.be.an('object'); + + expect(ctx.document.createElement).to.be.an('function'); + expect(ctx.document.createElement('a')).eql(pseudoElement); + expect(sandbox.eval('document.createElement("a")', ctx)).to.eql(pseudoElement); + + ctx = new sandbox.Context({ hostname: 'test.com' }); + pseudoElement = { firstChild: { href: 'http://test.com/' } }; + + expect(ctx.document.createElement('a')).eql(pseudoElement); + expect(sandbox.eval('document.createElement("a")', ctx)).to.eql(pseudoElement); + }); + + it('Context() should define document.geElementById', function () { + let ctx = new sandbox.Context(); + + expect(ctx.document).to.be.an('object'); + + expect(ctx.document.getElementById).to.be.an('function'); + expect(ctx.document.getElementById()).to.be.null; + expect(sandbox.eval('document.getElementById()', ctx)).to.be.null; + expect(ctx.document.getElementById('foobar')).to.be.null; + expect(sandbox.eval('document.getElementById("foobar")', ctx)).to.be.null; + + // Double quotes + ctx = new sandbox.Context({ body: '
foobar
' }); + expect(ctx.document.getElementById('test')).eql({ innerHTML: 'foobar' }); + expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: 'foobar' }); + + // Single quotes + ctx = new sandbox.Context({ body: '
foobar
' }); + expect(ctx.document.getElementById('test')).eql({ innerHTML: 'foobar' }); + expect(sandbox.eval('document.getElementById(\'test\')', ctx)).eql({ innerHTML: 'foobar' }); + + // Empty + ctx = new sandbox.Context({ body: '
' }); + expect(ctx.document.getElementById('test')).eql({ innerHTML: '' }); + expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: '' }); + + // Space agnostic tests + ctx = new sandbox.Context({ body: '
\nabc\n\n
' }); + expect(ctx.document.getElementById('test')).eql({ innerHTML: '\nabc\n\n' }); + expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: '\nabc\n\n' }); + + ctx = new sandbox.Context({ body: '
abc
' }); + expect(ctx.document.getElementById('test')).eql({ innerHTML: ' abc ' }); + expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: ' abc ' }); + + ctx = new sandbox.Context({ body: 'foo="bar" id=\'test\' a=b > abc <' }); + expect(ctx.document.getElementById('test')).eql({ innerHTML: ' abc ' }); + expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: ' abc ' }); + + // Cache test + ctx = new sandbox.Context({ body: '
foobar
' }); + ctx.document.getElementById('test').innerHTML = 'foo'; + expect(ctx.document.getElementById('test')).eql({ innerHTML: 'foo' }); + expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: 'foo' }); + }); +}); diff --git a/test/test-timeout.js b/test/test-timeout.js new file mode 100644 index 0000000..e69de29 From f659287bdf3dca0a2ac17af0eaaf6eb0ca7f4ff5 Mon Sep 17 00:00:00 2001 From: Oleksii Sribnyi Date: Sun, 21 Apr 2019 15:50:33 +0200 Subject: [PATCH 10/15] Test if i can commit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 85c067c..dea2310 100644 --- a/README.md +++ b/README.md @@ -227,3 +227,4 @@ Current Cloudflare implementation requires browser to respect the timeout of 5 s * [request-promise](https://github.com/request/request-promise) + From 16c2659246b366d8ce64132e41b9179585ea1bda Mon Sep 17 00:00:00 2001 From: Oleksii Sribnyi Date: Mon, 22 Apr 2019 11:29:07 +0200 Subject: [PATCH 11/15] Update docs and index.js timeout parsing --- CHANGELOG.md | 8 +++++++- README.md | 6 +++++- index.js | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e178c2..a0aaf7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## Change Log +### v4.0.0 (22/04/2019) +- Randomize `User-Agent` header with random chrome browser +- Recaptcha solving support +- Brotli non-mandatory support +- Various code changes and improvements + ### v3.9.1 (11/04/2019) - Fix for the timeout parsing @@ -11,7 +17,7 @@ ### v3.7.0 (07/04/2019) - [#182](https://github.com/codemanki/cloudscraper/pull/182) Usage examples have been added. -- [#169](https://github.com/codemanki/cloudscraper/pull/169) Cloudscraper now automatically parses out timeout for a CF challenge. `cloudflareTimeout` still can be used, but will be deprecated soon +- [#169](https://github.com/codemanki/cloudscraper/pull/169) Cloudscraper now automatically parses out timeout for a CF challenge. ### v3.6.0 (03/04/2019) - [#180](https://github.com/codemanki/cloudscraper/pull/180) Update code to parse latest CF challenge diff --git a/README.md b/README.md index dea2310..1e85f9d 100644 --- a/README.md +++ b/README.md @@ -151,12 +151,16 @@ var options = { jar: requestModule.jar(), // Custom cookie jar headers: { // User agent, Cache Control and Accept headers are required + // User agent is populated by a random UA. 'User-Agent': 'Ubuntu Chromium/34.0.1847.116 Chrome/34.0.1847.116 Safari/537.36', 'Cache-Control': 'private', 'Accept': 'application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5' }, - // Cloudflare requires a delay of 4 seconds, so wait for at least 5. + // Cloudscraper automatically parses out timeout required by Cloudflare. + // Override cloudflareTimeout to adjust it. cloudflareTimeout: 5000, + // Reduce Cloudflare's timeout to cloudflareMaxTimeout if it is excessive + cloudflareMaxTimeout: 30000, // followAllRedirects - follow non-GET HTTP 3xx responses as redirects followAllRedirects: true, // Support only this max challenges in row. If CF returns more, throw an error diff --git a/index.js b/index.js index 4967dbc..ab7bb5c 100644 --- a/index.js +++ b/index.js @@ -290,7 +290,7 @@ function onChallenge (options, response, body) { } if (isNaN(timeout)) { - if (match.length > 2) { + if (match[2] !== undefined) { timeout = parseInt(match[2]); if (timeout > options.cloudflareMaxTimeout) { From 45ba69592b668ae65f74f9f5b65a39ba1ad15b68 Mon Sep 17 00:00:00 2001 From: Oleksii Sribnyi Date: Mon, 22 Apr 2019 17:05:04 +0200 Subject: [PATCH 12/15] Mention recaptcha --- README.md | 4 ++++ examples/solve-recaptcha.js | 15 +++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 examples/solve-recaptcha.js diff --git a/README.md b/README.md index 1e85f9d..4cfd468 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,10 @@ Cloudscraper wraps request and request-promise, so using cloudscraper is pretty .catch(function (err) { }); ``` + +## Recaptcha +Cloudscraper may help you with the recaptcha page. Take a look at [this example](https://github.com/codemanki/cloudscraper/blob/master/examples/solve-recaptcha.js). + ## Defaults method `cloudscraper.defaults` is a very convenient way of extending the cloudscraper requests with any of your settings. diff --git a/examples/solve-recaptcha.js b/examples/solve-recaptcha.js new file mode 100644 index 0000000..33420f5 --- /dev/null +++ b/examples/solve-recaptcha.js @@ -0,0 +1,15 @@ +#!/usr/bin/env node + +function onCaptcha (options, response, body) { + const captcha = response.captcha; + // solveReCAPTCHA is a method that you should come up with and pass it href and sitekey, in return it will return you a reponse + solveReCAPTCHA(response.request.uri.href, captcha.siteKey, (error, gRes) => { + if (error) return void captcha.submit(error); + captcha.form['g-recaptcha-response'] = gRes; + captcha.submit(); + }); +} + +const cloudscraper = require('..').defaults({ onCaptcha }); +var uri = process.argv[2]; +cloudscraper.get({ uri: uri, headers: { cookie: "captcha=1" } }).catch(console.warn).then(console.log) \ No newline at end of file From d2a047c7d8b53b2c152de3243c1f23a778dab881 Mon Sep 17 00:00:00 2001 From: Oleksii Sribnyi Date: Mon, 22 Apr 2019 17:05:20 +0200 Subject: [PATCH 13/15] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bf33e05..4eaa1c7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cloudscraper", - "version": "3.9.1", + "version": "4.0.0", "description": "Bypasses cloudflare's anti-ddos page", "main": "index.js", "engines": { From 2831e5cda5d914d1b39d3447c0f5bcd6b074e2ef Mon Sep 17 00:00:00 2001 From: Oleksii Sribnyi Date: Mon, 22 Apr 2019 17:14:24 +0200 Subject: [PATCH 14/15] Adding dummy solveReCAPTCHA method --- examples/solve-recaptcha.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/solve-recaptcha.js b/examples/solve-recaptcha.js index 33420f5..33c727b 100644 --- a/examples/solve-recaptcha.js +++ b/examples/solve-recaptcha.js @@ -1,5 +1,8 @@ #!/usr/bin/env node +function solveReCAPTCHA(url, sitekey, callback) { + //.. Here you do some magic with the sitekey provided by cloudscraper +} function onCaptcha (options, response, body) { const captcha = response.captcha; // solveReCAPTCHA is a method that you should come up with and pass it href and sitekey, in return it will return you a reponse From 77e892997a47e8bbf0e532d7624178eb20b4803f Mon Sep 17 00:00:00 2001 From: Oleksii Sribnyi Date: Mon, 22 Apr 2019 17:17:03 +0200 Subject: [PATCH 15/15] Fix lint issues with recaptcha example --- examples/solve-recaptcha.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/solve-recaptcha.js b/examples/solve-recaptcha.js index 33c727b..edb1094 100644 --- a/examples/solve-recaptcha.js +++ b/examples/solve-recaptcha.js @@ -1,8 +1,9 @@ #!/usr/bin/env node -function solveReCAPTCHA(url, sitekey, callback) { - //.. Here you do some magic with the sitekey provided by cloudscraper +function solveReCAPTCHA (url, sitekey, callback) { + // Here you do some magic with the sitekey provided by cloudscraper } + function onCaptcha (options, response, body) { const captcha = response.captcha; // solveReCAPTCHA is a method that you should come up with and pass it href and sitekey, in return it will return you a reponse @@ -15,4 +16,4 @@ function onCaptcha (options, response, body) { const cloudscraper = require('..').defaults({ onCaptcha }); var uri = process.argv[2]; -cloudscraper.get({ uri: uri, headers: { cookie: "captcha=1" } }).catch(console.warn).then(console.log) \ No newline at end of file +cloudscraper.get({ uri: uri, headers: { cookie: 'captcha=1' } }).catch(console.warn).then(console.log); // eslint-disable-line promise/catch-or-return