From 7a324e8299e7a95e46a6c9be617b705ed5854207 Mon Sep 17 00:00:00 2001 From: jefleponot Date: Fri, 1 Jul 2016 12:42:24 +0200 Subject: [PATCH] add features for asynchronous downloads --- docs/modules/casper.rst | 193 ++++++++++++++++++++++++++++- docs/modules/clientutils.rst | 4 +- modules/casper.js | 220 ++++++++++++++++++++++++++++++--- modules/clientutils.js | 53 +++++--- samples/download.js | 10 +- tests/suites/casper/encode.js | 38 +++++- tests/suites/casper/logging.js | 2 +- 7 files changed, 470 insertions(+), 50 deletions(-) diff --git a/docs/modules/casper.rst b/docs/modules/casper.rst index cd5869b8f..02b476df0 100644 --- a/docs/modules/casper.rst +++ b/docs/modules/casper.rst @@ -386,6 +386,33 @@ Default wait timeout, for ``wait*`` family functions. ``Casper`` prototype ++++++++++++++++++++ +.. _casper_ajaxExists: + +.. index:: HTTP + +``ajaxExists()`` +------------------------------------------------------------------------------- + +**Signature:** ``ajaxExists(String|Function|RegExp test)`` + +Checks if an ajax request has been requested by base64encode or download methods. You can pass either a function, a string or a ``RegExp`` instance to perform the test:: + + casper.start('http://www.google.com/', function() { + base64logo = this.base64encode('http://www.google.fr/images/srpr/logo3w.png'); + if (this.ajaxExists('logo3w.png')) { + this.echo('Google logo loaded'); + } else { + this.echo('Google logo not loaded yet', 'ERROR'); + } + }); + + casper.run(); + +.. note:: + + If you want to wait for a resource to be loaded, use the `waitForAJAX()`_ method. + + ``back()`` ------------------------------------------------------------------------------- @@ -410,7 +437,7 @@ Also have a look at `forward()`_. ``base64encode()`` ------------------------------------------------------------------------------- -**Signature:** ``base64encode(String url [, String method, Object data])`` +**Signature:** ``base64encode(String url [, String method [, Object data, Boolean async ] ] )`` Encodes a resource using the base64 algorithm synchronously using client-side XMLHttpRequest. @@ -445,6 +472,39 @@ encode:: this.echo(base64contents).exit(); }); +.. note:: + + Use this method with waitForAJAX putting async parameter set to 'true' + +.. _casper_base64write: + +.. index:: Base64 + +``base64write()`` +------------------------------------------------------------------------------- + +**Signature:** ``base64write(String str, String targetPath)`` + +Saves a base64 string onto the filesystem after decode it.:: + + casper.start('http://www.google.fr/', function() { + var str = this.evaluate( function() { + function getBase64Image(img) { + var canvas = document.createElement("canvas"); + canvas.width = img.width; + canvas.height = img.height; + var ctx = canvas.getContext("2d"); + ctx.drawImage(img, 0, 0); + return canvas.toDataURL().replace(/[^,]*,/,""); + } + return getBase64Image(document.images[0]); + }); + this.base64write(str, "logo.png'); + }); + casper.run(function() { + this.echo('Done.').exit(); + }); + .. index:: bypass, Step stack ``bypass()`` @@ -699,10 +759,16 @@ Exits phantom with a logged error message and an optional exit status code:: ``download()`` ------------------------------------------------------------------------------- -**Signature:** ``download(String url, String target[, String method, Object data])`` +**Signature:** ``download(String url, String target[, String method, Object data, Boolean async])`` Saves a remote resource onto the filesystem. You can optionally set the HTTP method using the ``method`` argument, and pass request arguments through the ``data`` object (see `base64encode()`_):: +.. note:: + +since phantomJS 2.0.0 synchronous downloads are forbidden : By default async is true / For backward compatibilty async is false with phantomJS 1.x.x:: + +deprecated use:: + casper.start('http://www.google.fr/', function() { var url = 'http://www.google.fr/intl/fr/about/corporate/company/'; this.download(url, 'google_company.html'); @@ -712,10 +778,54 @@ Saves a remote resource onto the filesystem. You can optionally set the HTTP met this.echo('Done.').exit(); }); +better use:: + + var url = 'http://www.google.fr/intl/fr/about/corporate/company/'; + casper.start('http://www.google.fr/', function() { + this.download(url, 'google_company.html'); + }); + + casper.then(function(){ + this.waitForDownload(url){function(){ + this.echo('Done.'); + }); + }); + + casper.run(function() { + this.exit(); + }); + .. note:: If you have some troubles downloading files, try to :ref:`disable web security `. +.. _casper_downloadexists: + +.. index:: download + +``downloadExists()`` +------------------------------------------------------------------------------- + +**Signature:** ``downloadExists(String|Function|RegExp test)`` + +Checks if a download has been done. You can pass either a function, a string or a ``RegExp`` instance to perform the test:: + + casper.start('https://duckduckgo.com/', function() { + if (this.downloadExists('https://duckduckgo.com/assets/logo_homepage.normal.v107.svg')) { + this.echo('DuckDuckGo logo downloaded'); + } else { + this.echo('DuckDuckGo logo not downloaded', 'ERROR'); + } + }); + + casper.run(); + +.. note:: + + If you want to wait for a download to be done, use the `waitForDownload()`_ method. + + + ``each()`` ------------------------------------------------------------------------------- @@ -1073,6 +1183,36 @@ Fills form fields with given values and optionally submits it. While the ``form` .. index:: URL + + +.. _casper_getAjaxBase64: + +.. index:: Base64 + +``getAjaxBase64()`` +------------------------------------------------------------------------------- + +**Signature:** ``getAjaxBase64(String url)`` + +Retrieves an encoded base64 resource after an asynchronous base64encode call using client-side XMLHttpRequest:: + + var base64contents = null; + casper.start("http://www.google.fr/", function() { + var logo = "http://www.google.fr/images/srpr/logo3w.png"; + this.base64encode(logo, "logo.png",'GET',{},true); + this.waitForAJAX(logo,function (){ + base64contents = this.getAjaxBase64(logo); + }); + }); + + casper.run(function() { + this.echo(base64contents).exit(); + }); + +.. note:: + + this method get and drop content. + ``getCurrentUrl()`` ------------------------------------------------------------------------------- @@ -2188,6 +2328,25 @@ This can be used for better error messages or to conditionally ignore some timeo Please note, that all `waitFor` methods are not chainable. Consider wrapping each of them in a `casper.then` in order to acheive this functionality. +.. index:: download, Asynchronicity + +``waitForAJAX()`` +------------------------------------------------------------------------------- + +**Signature:** ``waitForAJAX(String|Function|RegExp testFx[, Function then, Function onTimeout, Number timeout])`` + +Wait until a client-side XMLHttpRequest occurs that matches an ajax matching constraints defined by ``testFx`` are satisfied to process a next step. + +The ``testFx`` argument can be either a string, a function or a ``RegExp`` instance:: + + casper.waitForAJAX("foobar.png", function() { + this.echo('foobar.png has been loaded.'); + }); + +..note:: + + Similar to waitForDownload without file writing check. + .. index:: alert ``waitForAlert()`` @@ -2203,6 +2362,36 @@ Waits until a `JavaScript alert ` instead:: diff --git a/modules/casper.js b/modules/casper.js index 280161faf..771ee1597 100644 --- a/modules/casper.js +++ b/modules/casper.js @@ -163,6 +163,7 @@ var Casper = function Casper(options) { this.pendingWait = false; this.requestUrl = 'about:blank'; this.resources = []; + this.ajax = []; this.result = { log: [], status: "success", @@ -231,6 +232,38 @@ var Casper = function Casper(options) { // Casper class is an EventEmitter utils.inherits(Casper, events.EventEmitter); + +/** + * Checks if an ajax request was done. + * + * @param Function/String/RegExp test A test function, string or regular expression. + * In case a string is passed, url matching will be tested. + * @return Boolean + */ +Casper.prototype.ajaxExists = function ajaxExists(test) { + "use strict"; + this.checkStarted(); + var testFn; + switch (utils.betterTypeOf(test)) { + case "string": + testFn = function _testAjaxExists_String(res) { + return res.url.indexOf(test) !== -1 && !!res.done; + }; + break; + case "regexp": + testFn = function _testAjaxExists_Regexp(res) { + return test.test(res.url) && !!res.done; + }; + break; + case "function": + testFn = test; + break; + default: + throw new CasperError("Invalid type"); + } + return this.ajax.some(testFn); +}; + /** * Go a step back in browser's history * @@ -251,14 +284,51 @@ Casper.prototype.back = function back() { * * NOTE: we cannot use window.btoa() for some strange reasons here. * - * @param String url The url to download - * @param String method The method to use, optional: default GET - * @param String data The data to send, optional - * @return string Base64 encoded result + * @param String url The url to download + * @param String method The method to use, optional: default GET + * @param String data The data to send, optional + * @param Boolean asynchronous Asynchroneous request? (default: false) + * @return string Base64 encoded result + */ +Casper.prototype.base64encode = function base64encode(url, method, data, asynchronous) { + "use strict"; + asynchronous = (typeof asynchronous !== "undefined") ? asynchronous : false; + if ((phantom.version.major > 1) && (!asynchronous)) { + throw new CasperError("asynchronous parameter must be 'true' with base64encode"); + } + var dl = { "url": url }, + dls = this.ajax.filter( function searchDownload( ajax ) { + return (ajax.url === url) && (typeof (ajax.filename) === "undefined"); + }); + if (dls.length === 0) { + this.ajax.push(dl); + } else { + dl = dls[0]; + } + dl.content = this.callUtils("getBase64", url, method, data, asynchronous); + if ((!asynchronous) && (!!dl.filename)) dl.done = true; + return dl.content; +}; + +/** + * Write a base64 resource after a decode process. + * + * NOTE: we cannot use window.atob() for some strange reasons here. + * + * @param String str The base64 encoded contents + * @param String targetPath The destination file path + * @return Casper */ -Casper.prototype.base64encode = function base64encode(url, method, data) { +Casper.prototype.base64write = function base64write(str, targetPath) { "use strict"; - return this.callUtils("getBase64", url, method, data); + var cu = require('clientutils').create(utils.mergeObjects({}, this.options)); + try { + fs.write(targetPath, cu.decode(str), 'wb'); + this.log(f("Saved resource in %s", targetPath)); + } catch (e) { + this.log(f("Error while saving %s: %s", targetPath, e), "error"); + } + return this; }; /** @@ -593,26 +663,69 @@ Casper.prototype.die = function die(message, status) { /** * Downloads a resource and saves it on the filesystem. * - * @param String url The url of the resource to download - * @param String targetPath The destination file path - * @param String method The HTTP method to use (default: GET) - * @param String data Optional data to pass performing the request + * @param String url The url of the resource to download + * @param String targetPath The destination file path + * @param String method The HTTP method to use (default: GET) + * @param String data Optional data to pass performing the request + * @param Boolean asynchronous Asynchroneous request? (default: false) * @return Casper */ -Casper.prototype.download = function download(url, targetPath, method, data) { +Casper.prototype.download = function download(url, targetPath, method, data, asynchronous) { "use strict"; this.checkStarted(); - var cu = require('clientutils').create(utils.mergeObjects({}, this.options)); - try { - fs.write(targetPath, cu.decode(this.base64encode(url, method, data)), 'wb'); + asynchronous = (typeof asynchronous !== "undefined")? asynchronous : false; + if ((phantom.version.major > 1) && (!asynchronous)) { + throw new CasperError("asynchronous parameter must be 'true' with base64encode"); + } + var str, dls = this.ajax.filter( function searchDownload( dow ){ + return (dow.url === url) && (!!dow.filename); + }), dl = {"url": url, "filename" : targetPath}; + + if (dls.length) { + dls[0].filename = targetPath; + } else { + this.ajax.push(dl); + } + str = this.callUtils("getBase64", url, method, data, asynchronous); + if (!asynchronous) { + this.base64write(str, targetPath); + dl.done = true; this.emit('downloaded.file', targetPath); - this.log(f("Downloaded and saved resource in %s", targetPath)); - } catch (e) { - this.log(f("Error while downloading %s to %s: %s", url, targetPath, e), "error"); } return this; }; +/** + * Checks if a given download was done. + * + * @param Function/String/RegExp test A test function, string or regular expression. + * In case a string is passed, url matching will be tested. + * @return Boolean + */ +Casper.prototype.downloadExists = function downloadExists(test) { + "use strict"; + this.checkStarted(); + var testFn; + switch (utils.betterTypeOf(test)) { + case "string": + testFn = function _testDownloadExists_String(res) { + return res.url.indexOf(test) !== -1 && !!res.filename && !!res.done; + }; + break; + case "regexp": + testFn = function _testDownloadExists_Regexp(res) { + return test.test(res.url) && !!res.filename && !!res.done; + }; + break; + case "function": + testFn = test; + break; + default: + throw new CasperError("Invalid type"); + } + return this.ajax.some(testFn); +}; + /** * Iterates over the values of a provided array and execute a callback for each * item. @@ -950,6 +1063,25 @@ Casper.prototype.forward = function forward() { }); }; +/** + * Retrieves an encoded base64 resource after an asynchronous base64encode call + * + * @param String url The url of the resource to download + * @return string Base64 encoded result + */ +Casper.prototype.getAjaxBase64 = function getAjaxBase64(url) { + "use strict"; + this.checkStarted(); + var str, dls = this.ajax.filter( function searchAJAX( ajax ){ + return (ajax.url === url) && (typeof (ajax.filename) === "undefined"); + }); + str = (dls.length>0)? dls[0].content: ""; + this.ajax = this.ajax.filter( function searchAJAX( ajax ){ + return ((ajax.url !== url) || (typeof (ajax.filename) !== "undefined")); + }); + return str; +}; + /** * Creates a new Colorizer instance. Sets `Casper.options.type` to change the * colorizer type name (see the `colorizer` module). @@ -2288,6 +2420,44 @@ Casper.prototype.waitForAlert = function(then, onTimeout, timeout) { }, onTimeout, timeout); }; +/** + * Waits until a given client-side XMLHttpRequest is done + * + * @param String/Function/RegExp test A function to test if the download exists. + * A string will be matched against the download url. + * @param Function then The next step to perform (optional) + * @param Function onTimeout A callback function to call on timeout (optional) + * @param Number timeout The max amount of time to wait, in milliseconds (optional) + * @return Casper + */ +Casper.prototype.waitForAJAX = function waitForAJAX(test, then, onTimeout, timeout) { + "use strict"; + this.checkStarted(); + timeout = timeout ? timeout : this.options.waitTimeout; + return this.waitFor(function _check() { + return this.ajaxExists(test); + }, then, onTimeout, timeout, { resource: test }); +}; + +/** + * Waits until a given download is done + * + * @param String/Function/RegExp test A function to test if the download exists. + * A string will be matched against the download url. + * @param Function then The next step to perform (optional) + * @param Function onTimeout A callback function to call on timeout (optional) + * @param Number timeout The max amount of time to wait, in milliseconds (optional) + * @return Casper + */ +Casper.prototype.waitForDownload = function waitForDownload(test, then, onTimeout, timeout) { + "use strict"; + this.checkStarted(); + timeout = timeout ? timeout : this.options.waitTimeout; + return this.waitFor(function _check() { + return this.downloadExists(test); + }, then, onTimeout, timeout, { resource: test }); +}; + /** * Waits for a popup page having its url matching the provided pattern to be opened * and loaded. @@ -2640,7 +2810,21 @@ function createPage(casper) { }; page.onCallback = function onCallback(data){ - casper.emit('remote.callback',data); + if (data && data.type && (data.type === "casper.sendAJAX") && data.url) { + casper.ajax.forEach( function searchDownload( dow , i){ + if (dow.url === data.url) { + if (!!dow.filename){ + casper.base64write(data.content, dow.filename); + casper.emit('downloaded.file', dow.filename); + } else { + dow.content = data.content; + } + dow.done = true; + } + }); + } else { + casper.emit('remote.callback',data); + } }; page.onError = function onError(msg, trace) { diff --git a/modules/clientutils.js b/modules/clientutils.js index 0ea0e2d0c..fc73e9bf9 100644 --- a/modules/clientutils.js +++ b/modules/clientutils.js @@ -376,27 +376,29 @@ * Downloads a resource behind an url and returns its base64-encoded * contents. * - * @param String url The resource url - * @param String method The request method, optional (default: GET) - * @param Object data The request data, optional - * @return String Base64 contents string + * @param String url The resource url + * @param String method The request method, optional (default: GET) + * @param Object data The request data, optional + * @param Boolean asynchronous Asynchroneous request? (default: false) + * @return String Base64 contents string */ - this.getBase64 = function getBase64(url, method, data) { - return this.encode(this.getBinary(url, method, data)); + this.getBase64 = function getBase64(url, method, data, asynchronous) { + return this.encode(this.getBinary(url, method, data, asynchronous)); }; /** * Retrieves string contents from a binary file behind an url. Silently * fails but log errors. * - * @param String url Url. - * @param String method HTTP method. - * @param Object data Request parameters. + * @param String url Url. + * @param String method HTTP method. + * @param Object data Request parameters. + * @param Boolean asynchronous Asynchroneous request? (default: false) * @return String */ - this.getBinary = function getBinary(url, method, data) { + this.getBinary = function getBinary(url, method, data, asynchronous) { try { - return this.sendAJAX(url, method, data, false, { + return this.sendAJAX(url, method, data, asynchronous, { overrideMimeType: "text/plain; charset=x-user-defined" }); } catch (e) { @@ -846,20 +848,33 @@ /** * Performs an AJAX request. * - * @param String url Url. - * @param String method HTTP method (default: GET). - * @param Object data Request parameters. - * @param Boolean async Asynchroneous request? (default: false) - * @param Object settings Other settings when perform the ajax request - * @return String Response text. + * @param String url Url. + * @param String method HTTP method (default: GET). + * @param Object data Request parameters. + * @param Boolean asynchronous Asynchroneous request? (default: true) + * @param Object settings Other settings when perform the ajax request + * @return String Response text. */ - this.sendAJAX = function sendAJAX(url, method, data, async, settings) { + this.sendAJAX = function sendAJAX(url, method, data, asynchronous, settings) { + var cli = this; var xhr = new XMLHttpRequest(), dataString = "", dataList = []; + asynchronous = (typeof asynchronous !== "undefined") ? asynchronous : false; method = method && method.toUpperCase() || "GET"; var contentType = settings && settings.contentType || "application/x-www-form-urlencoded"; - xhr.open(method, url, !!async); + xhr.open(method, url, asynchronous); + if (asynchronous) { + xhr.responseType = 'arraybuffer'; + xhr.addEventListener('load', function onTransferComplete() { + var content = btoa([].reduce.call(new Uint8Array(xhr.response), function(p, c) { + return p + String.fromCharCode(c); + }, '')); + if (typeof window.callPhantom === 'function') { + window.callPhantom({"type": "casper.sendAJAX", "url": url, "content": content}); + } + }); + } this.log("sendAJAX(): Using HTTP method: '" + method + "'", "debug"); if (settings && settings.overrideMimeType) { xhr.overrideMimeType(settings.overrideMimeType); diff --git a/samples/download.js b/samples/download.js index 55be033a8..3a76e4890 100644 --- a/samples/download.js +++ b/samples/download.js @@ -8,7 +8,13 @@ var casper = require("casper").create(); casper.start("http://www.google.fr/", function() { - this.download("http://www.google.fr/images/srpr/logo3w.png", "logo.png"); + this.download("http://www.google.fr/images/srpr/logo3w.png", "logo.png",'GET',{},true); +}); +casper.then(function(){ + this.waitForDownload("http://www.google.fr/images/srpr/logo3w.png",function(){ + console.log('download sucess'); + }, function(){ + console.log('download failed'); + }); }); - casper.run(); diff --git a/tests/suites/casper/encode.js b/tests/suites/casper/encode.js index 2aeb08f94..d5bdc747a 100644 --- a/tests/suites/casper/encode.js +++ b/tests/suites/casper/encode.js @@ -1,17 +1,43 @@ /*eslint strict:0*/ var fs = require('fs'); -casper.test.begin('base64encode() and download() tests', 2, function(test) { +casper.test.begin('base64encode() and download() tests', 4, function(test) { // FIXME: https://github.com/ariya/phantomjs/pull/364 has been merged, update scheme casper.start('file://' + phantom.casperPath + '/tests/site/index.html', function() { var imageUrl = 'file://' + phantom.casperPath + '/tests/site/images/phantom.png', + image = ""; + if (phantom.version.major < 2 ){ image = this.base64encode(imageUrl); - test.assertEquals(image.length, 6160, 'Casper.base64encode() can retrieve base64 contents'); - this.download(imageUrl, '__test_logo.png'); - test.assert(fs.exists('__test_logo.png'), 'Casper.download() downloads a file'); - if (fs.exists('__test_logo.png')) { - fs.remove('__test_logo.png'); + test.assertEquals(image.length, 6160, 'Casper.base64encode() can retrieve base64 contents'); + } else { + test.assertEquals(image.length, 0 , 'Casper.base64encode() can\'t retrieve base64 contents'); } + image = this.base64encode(imageUrl,'GET',{},true); + this.waitForAJAX(imageUrl,function waitForAjaxTest(){ + image = this.getAjaxBase64(imageUrl); + test.assertEquals(image.length, 6160, 'Casper.base64encode() can retrieve base64 contents'); + }, function onTimeout() { + test.fail("waitForAJAX timeout occured"); + }); + + if (phantom.version.major < 2 ){ + this.download(imageUrl, '__test_logo.png'); + test.assert(fs.exists('__test_logo.png'), 'Casper.download() downloads a file'); + if (fs.exists('__test_logo.png')) { + fs.remove('__test_logo.png'); + } + } else { + test.assert(!fs.exists('__test_logo.png'), 'Casper.download() not downloads a file'); + } + this.download(imageUrl, '__test_logo2.png','GET',{},true); + this.waitForDownload(imageUrl,function waitForDownloadTest(){ + test.assert(fs.exists('__test_logo2.png'), 'Casper.download() downloads a file'); + if (fs.exists('__test_logo2.png')) { + fs.remove('__test_logo2.png'); + } + },function onTimeout(){ + test.fail("waitForDownload timeout occured"); + }); }).run(function() { test.done(); }); diff --git a/tests/suites/casper/logging.js b/tests/suites/casper/logging.js index 12dd0e892..c54af5b6b 100644 --- a/tests/suites/casper/logging.js +++ b/tests/suites/casper/logging.js @@ -6,7 +6,7 @@ casper.test.begin('logging tests', 4, function(test) { casper.then(casper.createStep(function() { oldLevel = casper.options.logLevel; - + casper.result.log = []; casper.options.logLevel = 'info'; casper.options.verbose = false; }, {skipLog: true}));