Moved to RMSE for image comparison to account for slight variations i…

…n golden image tests (flutter#19658) Moved to RMSE for image comparison to account for slight variations in golden image production. (also fixed a flakey test)
mingwandroid · Jul 13, 2020 · 91f80ef · 91f80ef
1 parent ae37971
commit 91f80ef
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 7 deletions.
diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/GoldenImage.m
@@ -6,6 +6,8 @@
 #import <XCTest/XCTest.h>
 #include <sys/sysctl.h>
 
+static const double kRmseThreshold = 0.5;
+
 @interface GoldenImage ()
 
 @end
@@ -67,8 +69,24 @@ - (BOOL)compareGoldenToImage:(UIImage*)image {
   CGContextDrawImage(contextB, CGRectMake(0, 0, widthA, heightA), imageRefB);
   CGContextRelease(contextB);
 
-  BOOL isSame = memcmp(rawA.mutableBytes, rawB.mutableBytes, size) == 0;
-  return isSame;
+  const char* apos = rawA.mutableBytes;
+  const char* bpos = rawB.mutableBytes;
+  double sum = 0.0;
+  for (size_t i = 0; i < size; ++i, ++apos, ++bpos) {
+    // Skip transparent pixels.
+    if (*apos == 0 && *bpos == 0 && i % 4 == 0) {
+      i += 3;
+      apos += 3;
+      bpos += 3;
+    } else {
+      double aval = *apos;
+      double bval = *bpos;
+      double diff = aval - bval;
+      sum += diff * diff;
+    }
+  }
+  double rmse = sqrt(sum / size);
+  return rmse <= kRmseThreshold;
 }
 
 NS_INLINE NSString* _platformName() {

diff --git a/testing/scenario_app/ios/Scenarios/ScenariosUITests/UnobstructedPlatformViewTests.m b/testing/scenario_app/ios/Scenarios/ScenariosUITests/UnobstructedPlatformViewTests.m
@@ -243,12 +243,8 @@ - (void)testPlatformViewsMaxOverlays {
 
   XCUIElement* overlay = app.otherElements[@"platform_view[0].overlay[0]"];
   XCTAssertTrue(overlay.exists);
-  XCTAssertEqual(overlay.frame.origin.x, 75);
-  XCTAssertEqual(overlay.frame.origin.y, 85);
-  XCTAssertEqual(overlay.frame.size.width, 150);
-  XCTAssertEqual(overlay.frame.size.height, 190);
-
   XCTAssertFalse(app.otherElements[@"platform_view[0].overlay[1]"].exists);
+  XCTAssertTrue(CGRectContainsRect(platform_view.frame, overlay.frame));
 }
 
 @end