From 4703b1f76496c29169601c0986fb7dbadba829be Mon Sep 17 00:00:00 2001 From: Oleksii Sribnyi Date: Thu, 28 Nov 2019 11:16:36 +0100 Subject: [PATCH 1/3] Adjusting code to parse the new challenge, adding test for it --- index.js | 26 ++++-- test/fixtures/js_challenge_28_11_2019.html | 103 +++++++++++++++++++++ test/test-index.js | 36 +++++++ 3 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/js_challenge_28_11_2019.html diff --git a/index.js b/index.js index 50100e4..e6dbed4 100644 --- a/index.js +++ b/index.js @@ -289,10 +289,13 @@ function onChallenge (options, response, body) { let timeout = parseInt(options.cloudflareTimeout); let match; + let hiddenInputName; + + match = body.match(/name="(.+?)" value="(.+?)"/); - match = body.match(/name="s" value="(.+?)"/); if (match) { - payload.s = match[1]; + hiddenInputName = match[1]; + payload[hiddenInputName] = match[2]; } match = body.match(/name="jschl_vc" value="(\w+)"/); @@ -354,14 +357,25 @@ function onChallenge (options, response, body) { options.headers = Object.assign({}, options.headers); // Use the original uri as the referer and to construct the answer uri. options.headers.Referer = uri.href; - options.uri = uri.protocol + '//' + uri.host + '/cdn-cgi/l/chk_jschl'; + // Check is form to be submitted via GET or POST + match = body.match(/id="challenge-form" action="(.+?)" method="(.+?)"/); + if(match && match[2] && match[2] === 'POST') { + options.uri = uri.protocol + '//' + uri.host + match[1]; + // Pass the payload using query string + options.form = payload; + options.method = 'POST'; + } else { + // Whatever is there, fallback to GET + options.uri = uri.protocol + '//' + uri.host + '/cdn-cgi/l/chk_jschl'; + // Pass the payload using query string + options.qs = payload; + } + //Decrement the number of challenges to solve. + options.challengesToSolve -= 1; // baseUrl can't be used in conjunction with an absolute uri if (options.baseUrl !== undefined) { options.baseUrl = undefined; } - // Set the query string and decrement the number of challenges to solve. - options.qs = payload; - options.challengesToSolve -= 1; // Make request with answer after delay. timeout -= Date.now() - response.responseStartTime; diff --git a/test/fixtures/js_challenge_28_11_2019.html b/test/fixtures/js_challenge_28_11_2019.html new file mode 100644 index 0000000..917480f --- /dev/null +++ b/test/fixtures/js_challenge_28_11_2019.html @@ -0,0 +1,103 @@ + + + + + + + + + Just a moment... + + + + + + + + + + + + +
+
+ + + +
+ + + + +
+ + + +
+ + +
+ DDoS protection by Cloudflare +
+ Ray ID: 53cb1af29bc6c2d6 +
+
+ + \ No newline at end of file diff --git a/test/test-index.js b/test/test-index.js index 0dc8999..6556e2e 100644 --- a/test/test-index.js +++ b/test/test-index.js @@ -329,6 +329,42 @@ describe('Cloudscraper', function () { }); }); + it('should resolve challenge (version as on 28.11.2019) and then return page', function (done) { + helper.router + .get('/test', function (req, res) { + res.sendChallenge('js_challenge_28_11_2019.html'); + }) + .post('/', function (req, res) { + res.send(requestedPage); + }); + + const expectedParams = helper.extendParams({ + uri: helper.resolve('/?__cf_chl_jschl_tk__=xxxxx'), + method: 'POST', + form: { + r: '9dbc5c7ec65cb42893f2fd063ca80a2185ad6b6b-1574931141', + jschl_vc: '44a73ed828ddcd806a342f7c289cc438', + jschl_answer: String(20.2553376255 + helper.uri.hostname.length), + pass: '1574931145.541-UbeyT63kjo' + }, + headers: { + Referer: uri + }, + challengesToSolve: 2 + }); + + const promise = cloudscraper.get(uri, function (error, response, body) { + expect(error).to.be.null; + + expect(Request).to.be.calledTwice; + expect(Request.firstCall).to.be.calledWithExactly(helper.defaultParams); + expect(Request.secondCall).to.be.calledWithExactly(expectedParams); + + expect(body).to.be.equal(requestedPage); + expect(promise).to.eventually.equal(requestedPage).and.notify(done); + }); + }); + it('should resolve 2 consequent challenges', function (done) { // Cloudflare is enabled for site. It returns a page with JS challenge let additionalChallenge = true; From 6b97077d4c1a3e8ae2ca73f2ff608fd3b447c59b Mon Sep 17 00:00:00 2001 From: Oleksii Sribnyi Date: Thu, 28 Nov 2019 11:20:32 +0100 Subject: [PATCH 2/3] Minor comment changes and move declaration --- index.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index e6dbed4..c27cb08 100644 --- a/index.js +++ b/index.js @@ -289,12 +289,11 @@ function onChallenge (options, response, body) { let timeout = parseInt(options.cloudflareTimeout); let match; - let hiddenInputName; match = body.match(/name="(.+?)" value="(.+?)"/); if (match) { - hiddenInputName = match[1]; + let hiddenInputName = match[1]; payload[hiddenInputName] = match[2]; } @@ -361,7 +360,7 @@ function onChallenge (options, response, body) { match = body.match(/id="challenge-form" action="(.+?)" method="(.+?)"/); if(match && match[2] && match[2] === 'POST') { options.uri = uri.protocol + '//' + uri.host + match[1]; - // Pass the payload using query string + // Pass the payload using body form options.form = payload; options.method = 'POST'; } else { @@ -370,7 +369,7 @@ function onChallenge (options, response, body) { // Pass the payload using query string options.qs = payload; } - //Decrement the number of challenges to solve. + // Decrement the number of challenges to solve. options.challengesToSolve -= 1; // baseUrl can't be used in conjunction with an absolute uri if (options.baseUrl !== undefined) { From d8b400f9914b07238ac20bed1a9079ce737ede93 Mon Sep 17 00:00:00 2001 From: Oleksii Sribnyi Date: Thu, 28 Nov 2019 11:22:24 +0100 Subject: [PATCH 3/3] Fix linting errors --- index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index c27cb08..36c7b5b 100644 --- a/index.js +++ b/index.js @@ -293,7 +293,7 @@ function onChallenge (options, response, body) { match = body.match(/name="(.+?)" value="(.+?)"/); if (match) { - let hiddenInputName = match[1]; + const hiddenInputName = match[1]; payload[hiddenInputName] = match[2]; } @@ -358,7 +358,7 @@ function onChallenge (options, response, body) { options.headers.Referer = uri.href; // Check is form to be submitted via GET or POST match = body.match(/id="challenge-form" action="(.+?)" method="(.+?)"/); - if(match && match[2] && match[2] === 'POST') { + if (match && match[2] && match[2] === 'POST') { options.uri = uri.protocol + '//' + uri.host + match[1]; // Pass the payload using body form options.form = payload; @@ -369,7 +369,7 @@ function onChallenge (options, response, body) { // Pass the payload using query string options.qs = payload; } - // Decrement the number of challenges to solve. + // Decrement the number of challenges to solve. options.challengesToSolve -= 1; // baseUrl can't be used in conjunction with an absolute uri if (options.baseUrl !== undefined) {