Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[READY] Milestone 3.0 #332

Merged
merged 25 commits into from
Dec 1, 2015
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
013466b
started work on #305
ChrisBAshton Nov 5, 2015
3de2517
Merge branch 'master' into issue305-abstract
ChrisBAshton Nov 30, 2015
97a1a75
got tests passing
ChrisBAshton Nov 30, 2015
c3a24f3
modularising the tests
ChrisBAshton Nov 30, 2015
1595517
expanded browser engine and snap file tests
ChrisBAshton Nov 30, 2015
091b539
made yaml_passed parameter clearer
ChrisBAshton Nov 30, 2015
43816ce
simplified the way the browser engine is configured
ChrisBAshton Nov 30, 2015
2485487
new example configs and fixed #325
ChrisBAshton Nov 30, 2015
605fbb5
Merge pull request #330 from BBC-News/issue305-abstract
ChrisBAshton Nov 30, 2015
fcb75a6
fix #319
ChrisBAshton Nov 30, 2015
1bf709b
Merge pull request #333 from BBC-News/issue319
ChrisBAshton Nov 30, 2015
c58dde3
got the optional efficiency working in casper
ChrisBAshton Nov 30, 2015
914c9c8
made more progress
ChrisBAshton Nov 30, 2015
f27799e
bug fixes
ChrisBAshton Nov 30, 2015
8a63911
set up efficient mode on phantom
ChrisBAshton Nov 30, 2015
b06fb9a
reduced duplication between casperjs and phantomjs snap files
ChrisBAshton Nov 30, 2015
931d357
removed comment, created https://github.com/BBC-News/wraith/issues/33…
ChrisBAshton Nov 30, 2015
70170a0
fixed the tests
ChrisBAshton Nov 30, 2015
c86c63c
added some rudimentary tests
ChrisBAshton Nov 30, 2015
0dabbfd
explicitly pass dimensions object to helper
ChrisBAshton Nov 30, 2015
0b3a55e
keeping scopes absolutely separate
ChrisBAshton Nov 30, 2015
5650dd1
attempting fix found in https://github.com/n1k0/casperjs/issues/1068
ChrisBAshton Dec 1, 2015
6cc609d
expanded efficient/inefficient mode tests
ChrisBAshton Dec 1, 2015
604af55
refactored tests
ChrisBAshton Dec 1, 2015
14c9a44
Merge pull request #335 from BBC-News/efficient_mode
ChrisBAshton Dec 1, 2015
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions lib/wraith/gallery.rb
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,9 @@ def generate_gallery(with_path = "")
generate_html(@location, directories, slideshow_template, dest, with_path)

puts "Gallery generated"
check_failed_shots
failed = check_failed_shots
prompt_user_to_open_gallery dest
exit 1 if failed
end

def check_failed_shots
Expand All @@ -167,12 +169,17 @@ def check_failed_shots
end
end

exit 1
return false
else
true
end
end

def prompt_user_to_open_gallery(dest)
puts "\nView the gallery in your browser:"
puts "\t file://" + `pwd`.chomp + '/' + dest
end

class ErbBinding < OpenStruct
def get_binding
binding
Expand Down
45 changes: 45 additions & 0 deletions lib/wraith/javascript/_helper.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
module.exports = function (commandLineDimensions) {
// remove quotes from dimensions string
commandLineDimensions = commandLineDimensions.replace(/'/gi, '');

function getWidthAndHeight(dimensions) {
dimensions = /(\d*)x?((\d*))?/i.exec(dimensions);
return {
'viewportWidth': parseInt(dimensions[1]),
'viewportHeight': parseInt(dimensions[2] || 1500)
};
}

var multipleDimensions = commandLineDimensions.split(','),
dimensionsToPass;

if (multipleDimensions.length > 1) {
dimensionsToPass = multipleDimensions.map(function (cliDimensions) {
return getWidthAndHeight(cliDimensions);
});
}
else {
dimensionsToPass = getWidthAndHeight(commandLineDimensions);
}

return {
dimensions: dimensionsToPass,
takingMultipleScreenshots: function (dimensions) {
return dimensions.length && dimensions.length > 1;
},
replaceImageNameWithDimensions: function (image_name, currentDimensions) {
// shots/clickable_guide__after_click/MULTI_casperjs_english.png
// ->
// shots/clickable_guide__after_click/1024x359_casperjs_english.png
var dirs = image_name.split('/'),
filename = dirs[dirs.length - 1],
filenameParts = filename.split('_'),
newFilename;

filenameParts[0] = currentDimensions.viewportWidth + 'x' + currentDimensions.viewportHeight;
dirs.pop(); // remove MULTI_casperjs_english.png
newFilename = dirs.join('/') + '/' + filenameParts.join('_');
return newFilename;
}
}
}
112 changes: 112 additions & 0 deletions lib/wraith/javascript/_phantom__common.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
module.exports = function (config) {

// config
var systemArgs = config.systemArgs,
javascriptEnabled = config.javascriptEnabled;

// modules
var page = require('webpage').create(),
helper = require('./_helper.js')(systemArgs[2]);

// command line arguments
var url = systemArgs[1],
dimensions = helper.dimensions,
image_name = systemArgs[3],
selector = systemArgs[4],
globalBeforeCaptureJS = systemArgs[5],
pathBeforeCaptureJS = systemArgs[6],
dimensionsProcessed = 0,
currentDimensions;

globalBeforeCaptureJS = globalBeforeCaptureJS === 'false' ? false : globalBeforeCaptureJS;
pathBeforeCaptureJS = pathBeforeCaptureJS === 'false' ? false : pathBeforeCaptureJS;

var current_requests = 0;
var last_request_timeout;
var final_timeout;

if (helper.takingMultipleScreenshots(dimensions)) {
currentDimensions = dimensions[0];
image_name = helper.replaceImageNameWithDimensions(image_name, currentDimensions);
}
else {
currentDimensions = dimensions;
}

page.viewportSize = { width: currentDimensions.viewportWidth, height: currentDimensions.viewportHeight};
page.settings = { loadImages: true, javascriptEnabled: javascriptEnabled };

// If you want to use additional phantomjs commands, place them here
page.settings.userAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.17';

page.onResourceRequested = function(req) {
current_requests += 1;
};

page.onResourceReceived = function(res) {
if (res.stage === 'end') {
current_requests -= 1;
debounced_render();
}
};

page.open(url, function(status) {
if (status !== 'success') {
console.log('Error with page ' + url);
phantom.exit();
}
});

function runSetupJavaScriptThenCaptureImage() {
if (globalBeforeCaptureJS) {
require(globalBeforeCaptureJS)(page);
}
if (pathBeforeCaptureJS) {
require(pathBeforeCaptureJS)(page);
}
captureImage();
}

function captureImage() {

console.log('Snapping ' + url + ' at: ' + currentDimensions.viewportWidth + 'x' + currentDimensions.viewportHeight);
page.clipRect = {
top: 0,
left: 0,
height: currentDimensions.viewportHeight,
width: currentDimensions.viewportWidth
};
page.render(image_name);

dimensionsProcessed++;
if (helper.takingMultipleScreenshots(dimensions) && dimensionsProcessed < dimensions.length) {
currentDimensions = dimensions[dimensionsProcessed];
image_name = helper.replaceImageNameWithDimensions(image_name, currentDimensions);
setTimeout(captureImage, 1000);
}
else {
// prevent CI from failing from 'Unsafe JavaScript attempt to access frame with URL about:blank from frame with URL' errors
// see https://github.com/n1k0/casperjs/issues/1068
setTimeout(function(){
phantom.exit();
}, 10);
}
}

function debounced_render() {
clearTimeout(last_request_timeout);
clearTimeout(final_timeout);

// If there's no more ongoing resource requests, wait for 1 second before
// rendering, just in case the page kicks off another request
if (current_requests < 1) {
clearTimeout(final_timeout);
last_request_timeout = setTimeout(runSetupJavaScriptThenCaptureImage, 1000);
}

// Sometimes, straggling requests never make it back, in which
// case, timeout after 5 seconds and render the page anyway
final_timeout = setTimeout(runSetupJavaScriptThenCaptureImage, 5000);
}

}
81 changes: 81 additions & 0 deletions lib/wraith/javascript/casper.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// modules
var system = require('system'),
casper = require('casper').create(),
helper = requireRelative('_helper.js')(casper.cli.get(1));

// command line arguments
var url = casper.cli.get(0),
dimensions = helper.dimensions,
image_name = casper.cli.get(2),
selector = casper.cli.get(3),
globalBeforeCaptureJS = casper.cli.get(4),
pathBeforeCaptureJS = casper.cli.get(5),
dimensionsProcessed = 0,
currentDimensions;

// functions
function requireRelative(file) {
// PhantomJS will automatically `require` relatively, but CasperJS needs some extra help. Hence this function.
// 'templates/javascript/casper.js' -> 'templates/javascript'
var currentFilePath = system.args[3].split('/');
currentFilePath.pop();
var fs = require('fs');
currentFilePath = fs.absolute(currentFilePath.join('/'));
return require(currentFilePath + '/' + file);
}
function snap() {
console.log('Snapping ' + url + ' at: ' + currentDimensions.viewportWidth + 'x' + currentDimensions.viewportHeight);

if (!selector) {
this.capture(image_name);
}
else {
this.captureSelector(image_name, selector);
}

dimensionsProcessed++;
if (helper.takingMultipleScreenshots(dimensions) && dimensionsProcessed < dimensions.length) {
currentDimensions = dimensions[dimensionsProcessed];
image_name = helper.replaceImageNameWithDimensions(image_name, currentDimensions);
casper.viewport(currentDimensions.viewportWidth, currentDimensions.viewportHeight);
casper.wait(300, function then () {
snap.bind(this)();
});
}
}

if (helper.takingMultipleScreenshots(dimensions)) {
currentDimensions = dimensions[0];
image_name = helper.replaceImageNameWithDimensions(image_name, currentDimensions);
}
else {
currentDimensions = dimensions;
}

// Casper can now do its magic
casper.start();
casper.open(url);
casper.viewport(currentDimensions.viewportWidth, currentDimensions.viewportHeight);
casper.then(function() {
if (globalBeforeCaptureJS) {
require('./' + globalBeforeCaptureJS)(this);
}
});
casper.then(function() {
if (pathBeforeCaptureJS) {
require('./' + pathBeforeCaptureJS)(this);
}
});
// waits for all images to download before taking screenshots
// (broken images are a big cause of Wraith failures!)
// Credit: http://reff.it/8m3HYP
casper.waitFor(function() {
return this.evaluate(function() {
var images = document.getElementsByTagName('img');
return Array.prototype.every.call(images, function(i) { return i.complete; });
});
}, function then () {
snap.bind(this)();
});

casper.run();
File renamed without changes.
62 changes: 45 additions & 17 deletions lib/wraith/save_images.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
class Wraith::SaveImages
attr_reader :wraith, :history, :meta

def initialize(config, history = false)
@wraith = Wraith::Wraith.new(config)
def initialize(config, history = false, yaml_passed = false)
@wraith = Wraith::Wraith.new(config, yaml_passed)
@history = history
@meta = SaveMetadata.new(@wraith, history)
end
Expand All @@ -21,26 +21,51 @@ def check_paths
end

def save_images
jobs = define_jobs
parallel_task(jobs)
end

def define_jobs
jobs = []
check_paths.each do |label, options|
settings = CaptureOptions.new(options, wraith)

wraith.widths.each do |width|
base_file_name = meta.file_names(width, label, meta.base_label)
compare_file_name = meta.file_names(width, label, meta.compare_label)

jobs << [label, settings.path, width, settings.base_url, base_file_name, settings.selector, wraith.before_capture, settings.before_capture]
jobs << [label, settings.path, width, settings.compare_url, compare_file_name, settings.selector, wraith.before_capture, settings.before_capture] unless settings.compare_url.nil?
if wraith.resize
jobs = jobs + define_individual_job(label, settings, wraith.widths)
else
wraith.widths.each do |width|
jobs = jobs + define_individual_job(label, settings, width)
end
end
end
parallel_task(jobs)
jobs
end

def define_individual_job(label, settings, width)
base_file_name = meta.file_names(width, label, meta.base_label)
compare_file_name = meta.file_names(width, label, meta.compare_label)

jobs = []
jobs << [label, settings.path, prepare_widths_for_cli(width), settings.base_url, base_file_name, settings.selector, wraith.before_capture, settings.before_capture]
jobs << [label, settings.path, prepare_widths_for_cli(width), settings.compare_url, compare_file_name, settings.selector, wraith.before_capture, settings.before_capture] unless settings.compare_url.nil?

jobs
end

def prepare_widths_for_cli(width)
if width.kind_of? Array
# prepare for the command line. [30,40,50] => "'30','40','50'"
width = width.map{ |i| "'#{i}'" }.join(',')
end
width
end

def capture_page_image(browser, url, width, file_name, selector, global_before_capture, path_before_capture)
command = "#{browser} #{wraith.phantomjs_options} '#{wraith.snap_file}' '#{url}' '#{width}' '#{file_name}' '#{selector}' '#{global_before_capture}' '#{path_before_capture}'"

command = "#{browser} #{wraith.phantomjs_options} '#{wraith.snap_file}' '#{url}' \"#{width}\" '#{file_name}' '#{selector}' '#{global_before_capture}' '#{path_before_capture}'"

# @TODO - uncomment the following line when we add a verbose mode
# puts command
#puts command
run_command command
end

Expand Down Expand Up @@ -68,6 +93,10 @@ def attempt_image_capture(width, url, filename, selector, global_before_capture,
max_attempts.times do |i|
capture_page_image meta.engine, url, width, filename, selector, global_before_capture, path_before_capture

if wraith.resize
return # @TODO - need to check if the image was generated, as per the reload example below
end

return if File.exist? filename

puts "Failed to capture image #{filename} on attempt number #{i + 1} of #{max_attempts}"
Expand Down Expand Up @@ -142,12 +171,11 @@ def history_label
history ? "_latest" : ""
end

def engine_label
wraith.engine.key(engine)
end

def file_names(width, label, domain_label)
"#{wraith.directory}/#{label}/#{width}_#{engine_label}_#{domain_label}.png"
if width.kind_of? Array
width = 'MULTI'
end
"#{wraith.directory}/#{label}/#{width}_#{engine}_#{domain_label}.png"
end

def base_label
Expand All @@ -159,6 +187,6 @@ def compare_label
end

def engine
wraith.engine.each { |_label, browser| return browser }
wraith.engine
end
end
Loading