aisingapore#42 - add upload step for uploading files

Previously casper.page.uploadFile can be used to upload a file.
Uploading files is normally disabled by design due to security reasons.
Otherwise malicious scripts from websites can upload user-private local
data into their server.

PhantomJS allows uploading of files by the uploadFile method which is
explicitly designed to override this standard browser behavior.
However, it is designed without support for XPath selector. Only CSS
selectors can be used.

For integration with Chrome, this method would be not usable. Thus the
upload step is created to use a custom chrome.upload function to do
uploading through Chrome Debugging Protocol’s supported method of
DOM.setFileInputFiles. Only CSS selectors are supported now to be
consistent with PhantomJS behavior.

Also ran into issue when trying to use DOM.performSearch (no results
returned). It seems that method is needed to search by XPath. Can be
investigated further when decision is made to support XPath selector.

src/tagui_chrome.php

@@ -51,6 +51,9 @@
 else if (strpos($tagui_intent,'Target.attachToTarget') !== false) $intent_result_string = trim($client->receive());
 else if (strpos($tagui_intent,'Target.detachFromTarget') !== false) $intent_result_string = trim($client->receive());
 
+// ignore irrelevant DOM.setChildNodes events received when using DOM.querySelector to get NodeId for upload step
+while (strpos($intent_result_string,"DOM.setChildNodes") !== false) {$intent_result_string = trim($client->receive());}
+
 // save intent_result to interface out-file
 echo "[tagui] OUTPUT - \n" . "[" . $tagui_count . "] " . $intent_result_string . "\n\n";
 file_put_contents('tagui_chrome.out',"[" . $tagui_count . "] " . $intent_result_string); usleep($scan_period);} 

src/tagui_header.js

@@ -244,7 +244,19 @@ chrome.capture(filename); var selector_rect = chrome.getRect(selector); // so th
 if (selector_rect.width > 0 && selector_rect.height > 0) // from using other libraries or creating html canvas 
 casper.thenOpen(filename, function() {casper . capture(filename, // spaces around . intentional to avoid replacing 
 {top: selector_rect.top, left: selector_rect.left, width: selector_rect.width, height: selector_rect.height});
-casper . thenOpen('about:blank');});}; // reset phantomjs browser state, spaces intentional to avoid replacing
+casper.thenOpen('about:blank');});}; // reset phantomjs browser state
+
+chrome.upload = function(selector,filename) { // upload function to upload file to provided selector
+if ((selector.toString().length >= 16) && (selector.toString().substr(0,16) == 'xpath selector: '))
+{casper.echo('ERROR - upload step is only implemented for CSS selector and not XPath selector');
+casper.echo('ERROR - for consistency with PhantomJS as it only supports upload with CSS selector');}
+else try {var ws_message = ""; var ws_json = {};
+ws_message = chrome_step('DOM.getDocument',{});
+ws_json = JSON.parse(ws_message);
+ws_message = chrome_step('DOM.querySelector',{nodeId: ws_json.result.root.nodeId, selector: selector});
+ws_json = JSON.parse(ws_message);
+ws_message = chrome_step('DOM.setFileInputFiles',{files: [filename], nodeId: ws_json.result.nodeId});
+} catch(e) {casper.echo('ERROR - unable to upload ' + selector + ' as ' + filename);}};
 
 chrome.download = function(url,filename) { // download function for downloading url resource to file
 // casper download cannot be used for url which requires login as casperjs context is not in chrome
@@ -338,6 +350,7 @@ case 'type': return type_intent(live_line); break;
 case 'select': return select_intent(live_line); break;
 case 'read': return read_intent(live_line); break;
 case 'show': return show_intent(live_line); break;
+case 'upload': return upload_intent(live_line); break;
 case 'down': return down_intent(live_line); break;
 case 'receive': return receive_intent(live_line); break;
 case 'echo': return echo_intent(live_line); break;
@@ -368,7 +381,8 @@ if ((lc_raw_intent.substr(0,5) == 'type ') || (lc_raw_intent.substr(0,6) == 'ent
 if ((lc_raw_intent.substr(0,7) == 'select ') || (lc_raw_intent.substr(0,7) == 'choose ')) return 'select';
 if ((lc_raw_intent.substr(0,5) == 'read ') || (lc_raw_intent.substr(0,6) == 'fetch ')) return 'read';
 if ((lc_raw_intent.substr(0,5) == 'show ') || (lc_raw_intent.substr(0,6) == 'print ')) return 'show';
-if ((lc_raw_intent.substr(0,5) == 'down ') || (lc_raw_intent.substr(4,5) == 'load ')) return 'down';
+if ((lc_raw_intent.substr(0,3) == 'up ') || (lc_raw_intent.substr(0,7) == 'upload ')) return 'upload';
+if ((lc_raw_intent.substr(0,5) == 'down ') || (lc_raw_intent.substr(0,9) == 'download ')) return 'down';
 if (lc_raw_intent.substr(0,8) == 'receive ') return 'receive';
 if (lc_raw_intent.substr(0,5) == 'echo ') return 'echo';
 if (lc_raw_intent.substr(0,5) == 'save ') return 'save';
@@ -392,6 +406,7 @@ if ((lc_raw_intent == 'type') || (lc_raw_intent == 'enter')) return 'type';
 if ((lc_raw_intent == 'select') || (lc_raw_intent == 'choose')) return 'select';
 if ((lc_raw_intent == 'read') || (lc_raw_intent == 'fetch')) return 'read';
 if ((lc_raw_intent == 'show') || (lc_raw_intent == 'print')) return 'show';
+if ((lc_raw_intent == 'up') || (lc_raw_intent == 'upload')) return 'upload';
 if ((lc_raw_intent == 'down') || (lc_raw_intent == 'download')) return 'down';
 if (lc_raw_intent == 'receive') return 'receive';
 if (lc_raw_intent == 'echo') return 'echo';
@@ -528,6 +543,14 @@ if (params.toLowerCase() == 'page') return "this.echo('" + raw_intent + "' + ' -
 if (params == '') return "this.echo('ERROR - target missing for " + raw_intent + "')";
 else if (check_tx(params)) return "this.echo('" + raw_intent + "' + ' - ' + this.fetchText(tx('" + params + "')).trim())";else return "this.echo('ERROR - cannot find " + params + "')";}
 
+function upload_intent(raw_intent) {
+var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim();
+var param1 = (params.substr(0,params.indexOf(' as '))).trim();
+var param2 = (params.substr(4+params.indexOf(' as '))).trim();
+if ((param1 == '') || (param2 == '')) return "this.echo('ERROR - filename missing for " + raw_intent + "')";
+else if (check_tx(param1)) return "this.page.uploadFile(tx('" + param1 + "'),'" + abs_file(param2) + "')";
+else return "this.echo('ERROR - cannot find " + param1 + "')";}
+
 function down_intent(raw_intent) {
 var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim();
 var param1 = (params.substr(0,params.indexOf(' to '))).trim();
@@ -633,6 +656,7 @@ source_code = source_code.replace(/casper\.selectOptionByValue/g,'chrome.selectO
 source_code = source_code.replace(/casper\.fetchText/g,'chrome.fetchText').replace(/this\.fetchText/g,'chrome.fetchText');
 source_code = source_code.replace(/casper\.capture/g,'chrome.capture').replace(/this\.capture/g,'chrome.capture');
 source_code = source_code.replace(/casper\.captureSelector/g,'chrome.captureSelector').replace(/this\.captureSelector/g,'chrome.captureSelector');
+source_code = source_code.replace(/chrome\.page\.uploadFile/g,'chrome.upload').replace(/casper\.page\.uploadFile/g,'chrome.upload').replace(/this\.page\.uploadFile/g,'chrome.upload');
 source_code = source_code.replace(/casper\.download/g,'chrome.download').replace(/this\.download/g,'chrome.download');
 source_code = source_code.replace(/casper\.evaluate/g,'chrome.evaluate').replace(/this\.evaluate/g,'chrome.evaluate');
 source_code = source_code.replace(/casper\.getHTML/g,'chrome.getHTML').replace(/this\.getHTML/g,'chrome.getHTML');

src/tagui_parse.php

@@ -74,6 +74,9 @@
 $script_content = str_replace("this.capture","chrome.capture",$script_content); // change this.capture call as well
 $script_content = str_replace("casper.captureSelector","chrome.captureSelector",$script_content); // capture selector
 $script_content = str_replace("this.captureSelector","chrome.captureSelector",$script_content); // capture selector
+$script_content = str_replace("chrome.page.uploadFile","chrome.upload",$script_content); // change upload method to chrome
+$script_content = str_replace("casper.page.uploadFile","chrome.upload",$script_content); // change upload method to chrome
+$script_content = str_replace("this.page.uploadFile","chrome.upload",$script_content); // change this.upload call as well
 $script_content = str_replace("casper.download","chrome.download",$script_content); // change download method to chrome
 $script_content = str_replace("this.download","chrome.download",$script_content); // change this.download call as well
 $script_content = str_replace("casper.evaluate","chrome.evaluate",$script_content); // change evaluate method to chrome
@@ -155,6 +158,7 @@ function parse_intent($script_line) {$GLOBALS['line_number']++;
 case "select": return select_intent($script_line); break;
 case "read": return read_intent($script_line); break;
 case "show": return show_intent($script_line); break;
+case "upload": return upload_intent($script_line); break;
 case "down": return down_intent($script_line); break;
 case "receive": return receive_intent($script_line); break;
 case "echo": return echo_intent($script_line); break;
@@ -184,7 +188,8 @@ function get_intent($raw_intent) {$lc_raw_intent = strtolower($raw_intent);
 if ((substr($lc_raw_intent,0,7)=="select ") or (substr($lc_raw_intent,0,7)=="choose ")) return "select";
 if ((substr($lc_raw_intent,0,5)=="read ") or (substr($lc_raw_intent,0,6)=="fetch ")) return "read";
 if ((substr($lc_raw_intent,0,5)=="show ") or (substr($lc_raw_intent,0,6)=="print ")) return "show";
-if ((substr($lc_raw_intent,0,5)=="down ") or (substr($lc_raw_intent,4,5)=="load ")) return "down";
+if ((substr($lc_raw_intent,0,3)=="up ") or (substr($lc_raw_intent,0,7)=="upload ")) return "upload";
+if ((substr($lc_raw_intent,0,5)=="down ") or (substr($lc_raw_intent,0,9)=="download ")) return "down";
 if (substr($lc_raw_intent,0,8)=="receive ") return "receive";
 if (substr($lc_raw_intent,0,5)=="echo ") return "echo";
 if (substr($lc_raw_intent,0,5)=="save ") return "save";
@@ -208,6 +213,7 @@ function get_intent($raw_intent) {$lc_raw_intent = strtolower($raw_intent);
 if (($lc_raw_intent=="select") or ($lc_raw_intent=="choose")) return "select";
 if (($lc_raw_intent=="read") or ($lc_raw_intent=="fetch")) return "read";
 if (($lc_raw_intent=="show") or ($lc_raw_intent=="print")) return "show";
+if (($lc_raw_intent=="up") or ($lc_raw_intent=="upload")) return "upload";
 if (($lc_raw_intent=="down") or ($lc_raw_intent=="download")) return "down";
 if ($lc_raw_intent=="receive") return "receive";
 if ($lc_raw_intent=="echo") return "echo";
@@ -380,6 +386,14 @@ function show_intent($raw_intent) {$twb = $GLOBALS['tagui_web_browser'];
 return "{// nothing to do on this line".beg_tx($params).
 "this.echo('".$raw_intent."' + ' - ' + ".$twb.".fetchText(tx('" . $params . "')).trim());".end_tx($params);}
 
+function upload_intent($raw_intent) {$twb = $GLOBALS['tagui_web_browser'];
+$params = trim(substr($raw_intent." ",1+strpos($raw_intent." "," ")));
+$param1 = trim(substr($params,0,strpos($params," as "))); $param2 = trim(substr($params,4+strpos($params," as ")));
+if (($param1 == "") or ($param2 == ""))
+echo "ERROR - " . current_line() . " filename missing for " . $raw_intent . "\n"; else
+return "{techo('".$raw_intent."');".beg_tx($param1).
+$twb.".page.uploadFile(tx('".$param1."'),'".abs_file($param2)."');".end_tx($param1);}
+
 function down_intent($raw_intent) {$twb = $GLOBALS['tagui_web_browser'];
 $params = trim(substr($raw_intent." ",1+strpos($raw_intent." "," ")));
 $param1 = trim(substr($params,0,strpos($params," to "))); $param2 = trim(substr($params,4+strpos($params," to ")));

src/test/positive_test

@@ -1,4 +1,12 @@
 http://www.google.com
+// code to auto-replace local path names in generated js file with /full_path
+var fs = require('fs');
+var js_result = fs.read(fs.workingDirectory + fs.separator + 'test' + fs.separator + 'positive_test.js');
+var local_path = js_result.substring(js_result.indexOf("var flow_path = '") + 17);
+local_path = local_path.substring(0,local_path.indexOf("'"));
+var regex = new RegExp(local_path,"g"); js_result = js_result.replace(regex,"/full_path");
+fs.write(fs.workingDirectory + fs.separator + 'test' + fs.separator + 'positive_test.js', js_result, 'w');
+
 // TEST CONDITIONS
 
 // test contain / not contain
@@ -305,6 +313,10 @@ snap page
 snap page to filename.png
 snap page to /tmp/filename.png
 
+// test upload
+upload #css_selector as abc.png
+upload #css_selector as /tmp/abc.png
+
 // test download
 download http://www.dummytestsite.com/report/month.txt to filename.txt
 download http://www.dummytestsite.com/report/month.txt to /tmp/filename.txt

src/test/positive_test.signature

@@ -265,7 +265,19 @@ chrome.capture(filename); var selector_rect = chrome.getRect(selector); // so th
 if (selector_rect.width > 0 && selector_rect.height > 0) // from using other libraries or creating html canvas 
 casper.thenOpen(filename, function() {casper . capture(filename, // spaces around . intentional to avoid replacing 
 {top: selector_rect.top, left: selector_rect.left, width: selector_rect.width, height: selector_rect.height});
-casper . thenOpen('about:blank');});}; // reset phantomjs browser state, spaces intentional to avoid replacing
+casper.thenOpen('about:blank');});}; // reset phantomjs browser state
+
+chrome.upload = function(selector,filename) { // upload function to upload file to provided selector
+if ((selector.toString().length >= 16) && (selector.toString().substr(0,16) == 'xpath selector: '))
+{casper.echo('ERROR - upload step is only implemented for CSS selector and not XPath selector');
+casper.echo('ERROR - for consistency with PhantomJS as it only supports upload with CSS selector');}
+else try {var ws_message = ""; var ws_json = {};
+ws_message = chrome_step('DOM.getDocument',{});
+ws_json = JSON.parse(ws_message);
+ws_message = chrome_step('DOM.querySelector',{nodeId: ws_json.result.root.nodeId, selector: selector});
+ws_json = JSON.parse(ws_message);
+ws_message = chrome_step('DOM.setFileInputFiles',{files: [filename], nodeId: ws_json.result.nodeId});
+} catch(e) {casper.echo('ERROR - unable to upload ' + selector + ' as ' + filename);}};
 
 chrome.download = function(url,filename) { // download function for downloading url resource to file
 // casper download cannot be used for url which requires login as casperjs context is not in chrome
@@ -359,6 +371,7 @@ case 'type': return type_intent(live_line); break;
 case 'select': return select_intent(live_line); break;
 case 'read': return read_intent(live_line); break;
 case 'show': return show_intent(live_line); break;
+case 'upload': return upload_intent(live_line); break;
 case 'down': return down_intent(live_line); break;
 case 'receive': return receive_intent(live_line); break;
 case 'echo': return echo_intent(live_line); break;
@@ -389,7 +402,8 @@ if ((lc_raw_intent.substr(0,5) == 'type ') || (lc_raw_intent.substr(0,6) == 'ent
 if ((lc_raw_intent.substr(0,7) == 'select ') || (lc_raw_intent.substr(0,7) == 'choose ')) return 'select';
 if ((lc_raw_intent.substr(0,5) == 'read ') || (lc_raw_intent.substr(0,6) == 'fetch ')) return 'read';
 if ((lc_raw_intent.substr(0,5) == 'show ') || (lc_raw_intent.substr(0,6) == 'print ')) return 'show';
-if ((lc_raw_intent.substr(0,5) == 'down ') || (lc_raw_intent.substr(4,5) == 'load ')) return 'down';
+if ((lc_raw_intent.substr(0,3) == 'up ') || (lc_raw_intent.substr(0,7) == 'upload ')) return 'upload';
+if ((lc_raw_intent.substr(0,5) == 'down ') || (lc_raw_intent.substr(0,9) == 'download ')) return 'down';
 if (lc_raw_intent.substr(0,8) == 'receive ') return 'receive';
 if (lc_raw_intent.substr(0,5) == 'echo ') return 'echo';
 if (lc_raw_intent.substr(0,5) == 'save ') return 'save';
@@ -413,6 +427,7 @@ if ((lc_raw_intent == 'type') || (lc_raw_intent == 'enter')) return 'type';
 if ((lc_raw_intent == 'select') || (lc_raw_intent == 'choose')) return 'select';
 if ((lc_raw_intent == 'read') || (lc_raw_intent == 'fetch')) return 'read';
 if ((lc_raw_intent == 'show') || (lc_raw_intent == 'print')) return 'show';
+if ((lc_raw_intent == 'up') || (lc_raw_intent == 'upload')) return 'upload';
 if ((lc_raw_intent == 'down') || (lc_raw_intent == 'download')) return 'down';
 if (lc_raw_intent == 'receive') return 'receive';
 if (lc_raw_intent == 'echo') return 'echo';
@@ -549,6 +564,14 @@ if (params.toLowerCase() == 'page') return "this.echo('" + raw_intent + "' + ' -
 if (params == '') return "this.echo('ERROR - target missing for " + raw_intent + "')";
 else if (check_tx(params)) return "this.echo('" + raw_intent + "' + ' - ' + this.fetchText(tx('" + params + "')).trim())";else return "this.echo('ERROR - cannot find " + params + "')";}
 
+function upload_intent(raw_intent) {
+var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim();
+var param1 = (params.substr(0,params.indexOf(' as '))).trim();
+var param2 = (params.substr(4+params.indexOf(' as '))).trim();
+if ((param1 == '') || (param2 == '')) return "this.echo('ERROR - filename missing for " + raw_intent + "')";
+else if (check_tx(param1)) return "this.page.uploadFile(tx('" + param1 + "'),'" + abs_file(param2) + "')";
+else return "this.echo('ERROR - cannot find " + param1 + "')";}
+
 function down_intent(raw_intent) {
 var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim();
 var param1 = (params.substr(0,params.indexOf(' to '))).trim();
@@ -654,6 +677,7 @@ source_code = source_code.replace(/casper\.selectOptionByValue/g,'chrome.selectO
 source_code = source_code.replace(/casper\.fetchText/g,'chrome.fetchText').replace(/this\.fetchText/g,'chrome.fetchText');
 source_code = source_code.replace(/casper\.capture/g,'chrome.capture').replace(/this\.capture/g,'chrome.capture');
 source_code = source_code.replace(/casper\.captureSelector/g,'chrome.captureSelector').replace(/this\.captureSelector/g,'chrome.captureSelector');
+source_code = source_code.replace(/chrome\.page\.uploadFile/g,'chrome.upload').replace(/casper\.page\.uploadFile/g,'chrome.upload').replace(/this\.page\.uploadFile/g,'chrome.upload');
 source_code = source_code.replace(/casper\.download/g,'chrome.download').replace(/this\.download/g,'chrome.download');
 source_code = source_code.replace(/casper\.evaluate/g,'chrome.evaluate').replace(/this\.evaluate/g,'chrome.evaluate');
 source_code = source_code.replace(/casper\.getHTML/g,'chrome.getHTML').replace(/this\.getHTML/g,'chrome.getHTML');
@@ -692,6 +716,11 @@ casper.start('http://www.google.com', function() {
 techo('http://www.google.com' + ' - ' + this.getTitle() + '\n');});
 
 casper.then(function() {
+var js_result = fs.read(fs.workingDirectory + fs.separator + 'test' + fs.separator + 'positive_test.js');
+var local_path = js_result.substring(js_result.indexOf("var flow_path = '") + 17);
+local_path = local_path.substring(0,local_path.indexOf("'"));
+var regex = new RegExp(local_path,"g"); js_result = js_result.replace(regex,"/full_path");
+fs.write(fs.workingDirectory + fs.separator + 'test' + fs.separator + 'positive_test.js', js_result, 'w');
 // TEST CONDITIONS
 // test contain / not contain
 var variable = "test variable with some text";
@@ -1289,6 +1318,19 @@ this.capture(snap_image());}
 this.capture('/full_path/filename.png');}
 {techo('snap page to /tmp/filename.png');
 this.capture('/tmp/filename.png');}
+// test upload
+{techo('upload #css_selector as abc.png');
+casper.waitFor(function check() {return check_tx('#css_selector');},
+function then() {this.page.uploadFile(tx('#css_selector'),'/full_path/abc.png');},
+function timeout() {this.echo('ERROR - cannot find #css_selector').exit();});}});
+
+casper.then(function() {
+{techo('upload #css_selector as /tmp/abc.png');
+casper.waitFor(function check() {return check_tx('#css_selector');},
+function then() {this.page.uploadFile(tx('#css_selector'),'/tmp/abc.png');},
+function timeout() {this.echo('ERROR - cannot find #css_selector').exit();});}});
+
+casper.then(function() {
 // test download
 {techo('download http://www.dummytestsite.com/report/month.txt to filename.txt');
 this.download('http://www.dummytestsite.com/report/month.txt','/full_path/filename.txt');}