Merge pull request #8163 from piwik/avoid_resemble_comparison_if_poss…

…ible Re-enable Dashboard UI tests
matomo-org · Jun 22, 2015 · 4fd9581 · 4fd9581
2 parents 9a76d1b + 46a79ef
commit 4fd9581
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 29 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,9 @@ This is a changelog for Piwik platform developers. All changes for our HTTP API'
 ### Library updates
 * Updated pChart library from 2.1.3 to 2.1.4. The files were moved from the directory `libs/pChart2.1.3` to `libs/pChart`
 
+### Internal change
+* To execute UI tests "ImageMagick" is now required.
+
 ## Piwik 2.13.0
 
 ### Breaking Changes

diff --git a/tests/UI/screenshot-diffs/diffgenerator.js b/tests/UI/screenshot-diffs/diffgenerator.js
@@ -15,7 +15,8 @@ resemble.outputSettings({
         alpha: 125
     },
     errorType: 'movement',
-    transparency: 0.3
+    transparency: 0.3,
+    largeImageThreshold: 20000
 });
 
 function compareImages(expected, expectedGithub, processed)
@@ -57,6 +58,7 @@ $(function () {
     var expected  = getUrlQueryParam('expected');
     var github    = getUrlQueryParam('github');
     var resembleControl = compareImages(expected, github, processed);
+    resembleControl.ignoreNothing();
 
     $('#toggleAliasing').click(function () {
         resembleControl.ignoreAntialiasing();

diff --git a/tests/UI/specs/Dashboard_spec.js b/tests/UI/specs/Dashboard_spec.js
@@ -74,7 +74,7 @@ describe("Dashboard", function () {
             page.load(url, 5000);
         }, done);
     });
-/*
+
     it("should move a widget when widget is drag & dropped", function (done) {
         expect.screenshot("widget_move").to.be.capture(function (page) {
             page.mousedown('.widgetTop');
@@ -218,5 +218,5 @@ describe("Dashboard", function () {
             page.click('.ui-dialog[aria-describedby=createDashboardConfirm] button>span:contains(Yes)');
         }, done);
     });
-    */
+
 });
diff --git a/tests/lib/resemblejs/resemble.js b/tests/lib/resemblejs/resemble.js
@@ -558,12 +558,12 @@ URL: https://github.com/Huddle/Resemble.js
 			var self = {
 				ignoreNothing: function(){
 
-					tolerance.red = 16;
-					tolerance.green = 16;
-					tolerance.blue = 16;
-					tolerance.alpha = 16;
-					tolerance.minBrightness = 16;
-					tolerance.maxBrightness = 240;
+                    tolerance.red = 0;
+                    tolerance.green = 0;
+                    tolerance.blue = 0;
+                    tolerance.alpha = 0;
+                    tolerance.minBrightness = 0;
+                    tolerance.maxBrightness = 255;
 
 					ignoreAntialiasing = false;
 					ignoreColors = false;

diff --git a/tests/lib/screenshot-testing/run-tests.js b/tests/lib/screenshot-testing/run-tests.js
@@ -51,7 +51,8 @@ resemble.outputSettings({
         alpha: 125
     },
     errorType: 'movement',
-    transparency: 0.3
+    transparency: 0.3,
+    largeImageThreshold: 20000
 });
 
 // run script

diff --git a/tests/lib/screenshot-testing/support/chai-extras.js b/tests/lib/screenshot-testing/support/chai-extras.js
@@ -141,31 +141,66 @@ function capture(screenName, compareAgainst, selector, pageSetupFn, comparisonTh
                 return;
             }
 
-            var expected = fs.read(expectedScreenshotPath),
-                processed = fs.read(processedScreenshotPath);
-
-            if (processed == expected) {
-                pass();
-                return;
-            }
-
-            // if the files are not exact, perform a diff to check if they are truly different
-            resemble("file://" + processedScreenshotPath).compareTo("file://" + expectedScreenshotPath).onComplete(function(data) {
-                if (!screenshotMatches(data.misMatchPercentage)) {
-                    fail("Processed screenshot does not match expected for " + screenshotFileName + ". (mismatch = " + data.misMatchPercentage + ")");
-                    return;
-                }
-
-                pass();
-            });
-
             function screenshotMatches(misMatchPercentage) {
                 if (comparisonThreshold) {
                     return misMatchPercentage <= 100 * (1 - comparisonThreshold);
                 } else {
                     return misMatchPercentage == 0;
                 }
             }
+
+            function compareImages(expected, processed)
+            {
+                var args = ["-metric", "AE", expected, processed, 'null:'];
+                var child = require('child_process').spawn('compare', args);
+
+                var testFailure = '';
+
+                function onCommandResponse (numPxDifference) {
+                    // on success we get numPxDifference = '0' meaning no pixel was different
+                    // on different images we get the number of different pixels
+                    // on any error we get an error message (eg image size different)
+                    numPxDifference = numPxDifference.trim();
+
+                    if (numPxDifference && numPxDifference !== '0') {
+                        if (/^(\d+)$/.test(numPxDifference)) {
+                            testFailure += "(" + numPxDifference + "px difference";
+                        } else {
+                            testFailure += "(image magick error: " + numPxDifference;
+                        }
+
+                        testFailure += ")\n";
+                    }
+                }
+
+                child.stdout.on("data", onCommandResponse);
+                child.stderr.on("data", onCommandResponse);
+
+                child.on("exit", function (code) {
+                    if (testFailure) {
+                        testFailure = 'Processed screenshot does not match expected for ' + screenshotFileName + ' ' + testFailure;
+                    }
+
+                    if (code == 0 && !testFailure) {
+                        pass();
+                    } else if (comparisonThreshold) {
+                        // we use image magick only for exact match comparison, if there is a threshold we now check if this one fails
+                        resemble("file://" + processedScreenshotPath).compareTo("file://" + expectedScreenshotPath).onComplete(function(data) {
+                            if (!screenshotMatches(data.misMatchPercentage)) {
+                                fail(testFailure + ". (mismatch = " + data.misMatchPercentage + ")");
+                                return;
+                            }
+
+                            pass();
+                        });
+                    } else {
+                        fail(testFailure);
+                    }
+                });
+            }
+
+            compareImages(expectedScreenshotPath, processedScreenshotPath);
+
         }, selector);
     } catch (ex) {
         var err = new Error(ex.message);
@@ -369,4 +404,4 @@ chai.Assertion.addChainableMethod('pageContents', function (pageSetupFn, done) {
     var compareAgainst = this.__flags['object'];
 
     compareContents(compareAgainst, pageSetupFn, done);
-});
+});