Backport test suite from master

Author: nyalldawson

src/core/qgsrenderchecker.cpp

@@ -27,6 +27,8 @@
 #include <QDebug>
 #include <QBuffer>
 
+static int renderCounter = 0;
+
 QgsRenderChecker::QgsRenderChecker()
     : mReport( "" )
     , mMatchTarget( 0 )
@@ -35,6 +37,8 @@ QgsRenderChecker::QgsRenderChecker()
     , mExpectedImageFile( "" )
     , mMismatchCount( 0 )
     , mColorTolerance( 0 )
+    , mMaxSizeDifferenceX( 0 )
+    , mMaxSizeDifferenceY( 0 )
     , mElapsedTimeTarget( 0 )
     , mBufferDashMessages( false )
 {
@@ -43,16 +47,14 @@ QgsRenderChecker::QgsRenderChecker()
 QString QgsRenderChecker::controlImagePath() const
 {
   QString myDataDir( TEST_DATA_DIR ); //defined in CmakeLists.txt
-  QString myControlImageDir = myDataDir + QDir::separator() + "control_images" +
-                              QDir::separator() + mControlPathPrefix;
+  QString myControlImageDir = myDataDir + "/control_images/" + mControlPathPrefix;
   return myControlImageDir;
 }
 
 void QgsRenderChecker::setControlName( const QString &theName )
 {
   mControlName = theName;
-  mExpectedImageFile = controlImagePath() + theName + QDir::separator() + mControlPathSuffix
-                       + theName + ".png";
+  mExpectedImageFile = controlImagePath() + theName + "/" + mControlPathSuffix + theName + ".png";
 }
 
 QString QgsRenderChecker::imageToHash( QString theImageFile )
@@ -101,8 +103,7 @@ void QgsRenderChecker::drawBackground( QImage* image )
 
 bool QgsRenderChecker::isKnownAnomaly( QString theDiffImageFile )
 {
-  QString myControlImageDir = controlImagePath() + mControlName
-                              + QDir::separator();
+  QString myControlImageDir = controlImagePath() + mControlName + "/";
   QDir myDirectory = QDir( myControlImageDir );
   QStringList myList;
   QString myFilename = "*";
@@ -121,8 +122,7 @@ bool QgsRenderChecker::isKnownAnomaly( QString theDiffImageFile )
     mReport += "<tr><td colspan=3>"
                "Checking if " + myFile + " is a known anomaly.";
     mReport += "</td></tr>";
-    QString myAnomalyHash = imageToHash( controlImagePath() + mControlName
-                                         + QDir::separator() + myFile );
+    QString myAnomalyHash = imageToHash( controlImagePath() + mControlName + "/" + myFile );
     QString myHashMessage = QString(
                               "Checking if anomaly %1 (hash %2)<br>" )
                             .arg( myFile )
@@ -209,8 +209,7 @@ bool QgsRenderChecker::runTest( QString theTestName,
   // Save the pixmap to disk so the user can make a
   // visual assessment if needed
   //
-  mRenderedImageFile = QDir::tempPath() + QDir::separator() +
-                       theTestName + "_result.png";
+  mRenderedImageFile = QDir::tempPath() + "/" + theTestName + "_result.png";
 
   myImage.setDotsPerMeterX( myExpectedImage.dotsPerMeterX() );
   myImage.setDotsPerMeterY( myExpectedImage.dotsPerMeterY() );
@@ -226,7 +225,7 @@ bool QgsRenderChecker::runTest( QString theTestName,
 
   //create a world file to go with the image...
 
-  QFile wldFile( QDir::tempPath() + QDir::separator() + theTestName + "_result.wld" );
+  QFile wldFile( QDir::tempPath() + "/" + theTestName + "_result.wld" );
   if ( wldFile.open( QIODevice::WriteOnly ) )
   {
     QgsRectangle r = mMapSettings.extent();
@@ -258,9 +257,14 @@ bool QgsRenderChecker::compareImages( QString theTestName,
   }
   if ( ! theRenderedImageFile.isEmpty() )
   {
+#ifndef Q_OS_WIN
     mRenderedImageFile = theRenderedImageFile;
+#else
+    mRenderedImageFile = theRenderedImageFile.replace( "\\", "/" );
+#endif
   }
-  else if ( mRenderedImageFile.isEmpty() )
+
+  if ( mRenderedImageFile.isEmpty() )
   {
     qDebug( "QgsRenderChecker::runTest failed - Rendered Image File not set." );
     mReport = "<table>"
@@ -269,6 +273,7 @@ bool QgsRenderChecker::compareImages( QString theTestName,
               "Image File not set.</td></tr></table>\n";
     return false;
   }
+
   //
   // Load /create the images
   //
@@ -286,9 +291,7 @@ bool QgsRenderChecker::compareImages( QString theTestName,
   QImage myDifferenceImage( myExpectedImage.width(),
                             myExpectedImage.height(),
                             QImage::Format_RGB32 );
-  QString myDiffImageFile = QDir::tempPath() +
-                            QDir::separator() +
-                            theTestName + "_result_diff.png";
+  QString myDiffImageFile = QDir::tempPath() + "/" + theTestName + "_result_diff.png";
   myDifferenceImage.fill( qRgb( 152, 219, 249 ) );
 
   //check for mask
@@ -310,20 +313,22 @@ bool QgsRenderChecker::compareImages( QString theTestName,
   //
   // Set the report with the result
   //
-  mReport = "<table>";
+  mReport = QString( "<script src=\"file://%1/../renderchecker.js\"></script>\n" ).arg( TEST_DATA_DIR );
+  mReport += "<table>";
   mReport += "<tr><td colspan=2>";
-  mReport += "Test image and result image for " + theTestName + "<br>"
-             "Expected size: " + QString::number( myExpectedImage.width() ).toLocal8Bit() + "w x " +
-             QString::number( myExpectedImage.height() ).toLocal8Bit() + "h (" +
-             QString::number( mMatchTarget ).toLocal8Bit() + " pixels)<br>"
-             "Actual   size: " + QString::number( myResultImage.width() ).toLocal8Bit() + "w x " +
-             QString::number( myResultImage.height() ).toLocal8Bit() + "h (" +
-             QString::number( myPixelCount ).toLocal8Bit() + " pixels)";
-  mReport += "</td></tr>";
-  mReport += "<tr><td colspan = 2>\n";
-  mReport += "Expected Duration : <= " + QString::number( mElapsedTimeTarget ) +
-             "ms (0 indicates not specified)<br>";
-  mReport += "Actual Duration :  " + QString::number( mElapsedTime ) + "ms<br>";
+  mReport += QString( "<tr><td colspan=2>"
+                      "Test image and result image for %1<br>"
+                      "Expected size: %2 w x %3 h (%4 pixels)<br>"
+                      "Actual   size: %5 w x %6 h (%7 pixels)"
+                      "</td></tr>" )
+             .arg( theTestName )
+             .arg( myExpectedImage.width() ).arg( myExpectedImage.height() ).arg( mMatchTarget )
+             .arg( myResultImage.width() ).arg( myResultImage.height() ).arg( myPixelCount );
+  mReport += QString( "<tr><td colspan=2>\n"
+                      "Expected Duration : <= %1 (0 indicates not specified)<br>"
+                      "Actual Duration : %2 ms<br></td></tr>" )
+             .arg( mElapsedTimeTarget )
+             .arg( mElapsedTime );
 
   // limit image size in page to something reasonable
   int imgWidth = 420;
@@ -333,21 +338,23 @@ bool QgsRenderChecker::compareImages( QString theTestName,
     imgWidth = qMin( myExpectedImage.width(), imgWidth );
     imgHeight = myExpectedImage.height() * imgWidth / myExpectedImage.width();
   }
-  QString myImagesString = "</td></tr>"
-                           "<tr><td>Test Result:</td><td>Expected Result:</td><td>Difference (all blue is good, any red is bad)</td></tr>\n"
-                           "<tr><td><img width=" + QString::number( imgWidth ) +
-                           " height=" + QString::number( imgHeight ) +
-                           " src=\"file://" +
-                           mRenderedImageFile +
-                           "\"></td>\n<td><img width=" + QString::number( imgWidth ) +
-                           " height=" + QString::number( imgHeight ) +
-                           " src=\"file://" +
-                           mExpectedImageFile +
-                           "\"></td>\n<td><img width=" + QString::number( imgWidth ) +
-                           " height=" + QString::number( imgHeight ) +
-                           " src=\"file://" +
-                           myDiffImageFile  +
-                           "\"></td>\n</tr>\n</table>";
+
+  QString myImagesString = QString(
+                             "<tr>"
+                             "<td colspan=2>Compare actual and expected result</td>"
+                             "<td>Difference (all blue is good, any red is bad)</td>"
+                             "</tr>\n<tr>"
+                             "<td colspan=2 id=\"td-%1-%7\"></td>\n"
+                             "<td align=center><img width=%5 height=%6 src=\"file://%2\"></td>\n"
+                             "</tr>"
+                             "</table>\n"
+                             "<script>\naddComparison(\"td-%1-%7\",\"file://%3\",\"file://%4\",%5,%6);\n</script>\n" )
+                           .arg( theTestName )
+                           .arg( myDiffImageFile )
+                           .arg( mRenderedImageFile )
+                           .arg( mExpectedImageFile )
+                           .arg( imgWidth ).arg( imgHeight )
+                           .arg( renderCounter++ );
 
   QString prefix;
   if ( !mControlPathPrefix.isNull() )
@@ -369,30 +376,44 @@ bool QgsRenderChecker::compareImages( QString theTestName,
 
   if ( mMatchTarget != myPixelCount )
   {
-    qDebug( "Test image and result image for %s are different - FAILING!", theTestName.toLocal8Bit().constData() );
-    mReport += "<tr><td colspan=3>";
-    mReport += "<font color=red>Expected image and result image for " + theTestName + " are different dimensions - FAILING!</font>";
-    mReport += "</td></tr>";
-    mReport += myImagesString;
-    delete maskImage;
-    return false;
+    qDebug( "Test image and result image for %s are different dimensions", theTestName.toLocal8Bit().constData() );
+
+    if ( qAbs( myExpectedImage.width() - myResultImage.width() ) > mMaxSizeDifferenceX ||
+         qAbs( myExpectedImage.height() - myResultImage.height() ) > mMaxSizeDifferenceY )
+    {
+      mReport += "<tr><td colspan=3>";
+      mReport += "<font color=red>Expected image and result image for " + theTestName + " are different dimensions - FAILING!</font>";
+      mReport += "</td></tr>";
+      mReport += myImagesString;
+      delete maskImage;
+      return false;
+    }
+    else
+    {
+      mReport += "<tr><td colspan=3>";
+      mReport += "Expected image and result image for " + theTestName + " are different dimensions, but within tolerance";
+      mReport += "</td></tr>";
+    }
   }
 
   //
   // Now iterate through them counting how many
   // dissimilar pixel values there are
   //
 
+  int maxHeight = qMin( myExpectedImage.height(), myResultImage.height() );
+  int maxWidth = qMin( myExpectedImage.width(), myResultImage.width() );
+
   mMismatchCount = 0;
   int colorTolerance = ( int ) mColorTolerance;
-  for ( int y = 0; y < myExpectedImage.height(); ++y )
+  for ( int y = 0; y < maxHeight; ++y )
   {
     const QRgb* expectedScanline = ( const QRgb* )myExpectedImage.constScanLine( y );
     const QRgb* resultScanline = ( const QRgb* )myResultImage.constScanLine( y );
     const QRgb* maskScanline = hasMask ? ( const QRgb* )maskImage->constScanLine( y ) : 0;
     QRgb* diffScanline = ( QRgb* )myDifferenceImage.scanLine( y );
 
-    for ( int x = 0; x < myExpectedImage.width(); ++x )
+    for ( int x = 0; x < maxWidth; ++x )
     {
       int maskTolerance = hasMask ? qRed( maskScanline[ x ] ) : 0;
       int pixelTolerance = qMax( colorTolerance, maskTolerance );
@@ -440,40 +461,14 @@ bool QgsRenderChecker::compareImages( QString theTestName,
   //
   // Send match result to report
   //
-  mReport += "<tr><td colspan=3>" +
-             QString::number( mMismatchCount ) + "/" +
-             QString::number( mMatchTarget ) +
-             " pixels mismatched (allowed threshold: " +
-             QString::number( theMismatchCount ) +
-             ", allowed color component tolerance: " +
-             QString::number( mColorTolerance ) + ")";
-  mReport += "</td></tr>";
+  mReport += QString( "<tr><td colspan=3>%1/%2 pixels mismatched (allowed threshold: %3, allowed color component tolerance: %4)</td></tr>" )
+             .arg( mMismatchCount ).arg( mMatchTarget ).arg( theMismatchCount ).arg( mColorTolerance );
 
   //
   // And send it to CDash
   //
   emitDashMessage( "Mismatch Count", QgsDartMeasurement::Integer, QString( "%1/%2" ).arg( mMismatchCount ).arg( mMatchTarget ) );
 
-  bool myAnomalyMatchFlag = isKnownAnomaly( myDiffImageFile );
-
-  if ( myAnomalyMatchFlag )
-  {
-    mReport += "<tr><td colspan=3>"
-               "Difference image matched a known anomaly - passing test! "
-               "</td></tr>";
-    return true;
-  }
-  else
-  {
-    mReport += "<tr><td colspan=3>"
-               "</td></tr>";
-    emitDashMessage( "No Anomalies Match", QgsDartMeasurement::Text, "Difference image did not match any known anomaly."
-                     " If you feel the difference image should be considered an anomaly "
-                     "you can do something like this\n"
-                     "cp " + myDiffImageFile  + " ../tests/testdata/control_images/" + theTestName +
-                     "/<imagename>.{wld,png}" );
-  }
-
   if ( mMismatchCount <= theMismatchCount )
   {
     mReport += "<tr><td colspan = 3>\n";
@@ -495,12 +490,27 @@ bool QgsRenderChecker::compareImages( QString theTestName,
       return true;
     }
   }
-  else
+
+  bool myAnomalyMatchFlag = isKnownAnomaly( myDiffImageFile );
+  if ( myAnomalyMatchFlag )
   {
-    mReport += "<tr><td colspan = 3>\n";
-    mReport += "<font color=red>Test image and result image for " + theTestName + " are mismatched</font><br>";
-    mReport += "</td></tr>";
-    mReport += myImagesString;
-    return false;
+    mReport += "<tr><td colspan=3>"
+               "Difference image matched a known anomaly - passing test! "
+               "</td></tr>";
+    return true;
   }
+
+  mReport += "<tr><td colspan=3></td></tr>";
+  emitDashMessage( "Image mismatch", QgsDartMeasurement::Text, "Difference image did not match any known anomaly or mask."
+                   " If you feel the difference image should be considered an anomaly "
+                   "you can do something like this\n"
+                   "cp '" + myDiffImageFile + "' " + controlImagePath() + mControlName +
+                   "/\nIf it should be included in the mask run\n"
+                   "scripts/generate_test_mask_image.py '" + mExpectedImageFile + "' '" + mRenderedImageFile + "'\n" );
+
+  mReport += "<tr><td colspan = 3>\n";
+  mReport += "<font color=red>Test image and result image for " + theTestName + " are mismatched</font><br>";
+  mReport += "</td></tr>";
+  mReport += myImagesString;
+  return false;
 }

src/core/qgsrenderchecker.h

@@ -41,11 +41,12 @@ class CORE_EXPORT QgsRenderChecker
     QgsRenderChecker();
 
     //! Destructor
-    ~QgsRenderChecker() {};
+    ~QgsRenderChecker() {}
 
     QString controlImagePath() const;
 
-    QString report() { return mReport; };
+    QString report() { return mReport; }
+
     float matchPercent()
     {
       return static_cast<float>( mMismatchCount ) /
@@ -55,7 +56,7 @@ class CORE_EXPORT QgsRenderChecker
     unsigned int matchTarget() { return mMatchTarget; }
     //only records time for actual render part
     int elapsedTime() { return mElapsedTime; }
-    void setElapsedTimeTarget( int theTarget ) { mElapsedTimeTarget = theTarget; };
+    void setElapsedTimeTarget( int theTarget ) { mElapsedTimeTarget = theTarget; }
 
     /** Base directory name for the control image (with control image path
       * suffixed) the path to the image will be constructed like this:
@@ -66,9 +67,9 @@ class CORE_EXPORT QgsRenderChecker
     /** Prefix where the control images are kept.
      * This will be appended to controlImagePath
       */
-    void setControlPathPrefix( const QString &theName ) { mControlPathPrefix = theName + QDir::separator(); }
+    void setControlPathPrefix( const QString &theName ) { mControlPathPrefix = theName + "/"; }
 
-    void setControlPathSuffix( const QString& theName ) { mControlPathSuffix = theName + QDir::separator(); }
+    void setControlPathSuffix( const QString& theName ) { mControlPathSuffix = theName + "/"; }
 
     /** Get an md5 hash that uniquely identifies an image */
     QString imageToHash( QString theImageFile );
@@ -96,6 +97,14 @@ class CORE_EXPORT QgsRenderChecker
      * @note added in 2.1
      */
     void setColorTolerance( unsigned int theColorTolerance ) { mColorTolerance = theColorTolerance; }
+
+    /** Sets the largest allowable difference in size between the rendered and the expected image.
+     * @param xTolerance x tolerance in pixels
+     * @param yTolerance y tolerance in pixels
+     * @note added in QGIS 2.12
+     */
+    void setSizeTolerance( int xTolerance, int yTolerance ) { mMaxSizeDifferenceX = xTolerance; mMaxSizeDifferenceY = yTolerance; }
+
     /**
      * Test using renderer to generate the image to be compared.
      * @param theTestName - to be used as the basis for writing a file to
@@ -129,7 +138,7 @@ class CORE_EXPORT QgsRenderChecker
     */
     bool isKnownAnomaly( QString theDiffImageFile );
 
-    /**Draws a checkboard pattern for image backgrounds, so that transparency is visible
+    /** Draws a checkboard pattern for image backgrounds, so that transparency is visible
      * without requiring a transparent background for the image
      */
     static void drawBackground( QImage* image );
@@ -173,6 +182,8 @@ class CORE_EXPORT QgsRenderChecker
     QString mControlName;
     unsigned int mMismatchCount;
     unsigned int mColorTolerance;
+    int mMaxSizeDifferenceX;
+    int mMaxSizeDifferenceY;
     int mElapsedTimeTarget;
     QgsMapSettings mMapSettings;
     QString mControlPathPrefix;