[unit tests] Add multirenderchecker

The multirenderchecker allows to have several images, each with its own set of
anomalies distributed in several subdirectories.
With the help of multiple reference images, it is possible to apply a color
tolerance to each of these

Author: m-kuhn

python/core/core.sip

@@ -66,6 +66,7 @@
 %Include qgsmessagelog.sip
 %Include qgsmessageoutput.sip
 %Include qgsmimedatautils.sip
+%Include qgsmultirenderchecker.sip
 %Include qgsnetworkaccessmanager.sip
 %Include qgsnetworkcontentfetcher.sip
 %Include qgsofflineediting.sip

python/core/qgsmultirenderchecker.sip

@@ -0,0 +1,91 @@
+/***************************************************************************
+    qgsmultirenderchecker.sip
+     --------------------------------------
+    Date                 : 6.11.2014
+    Copyright            : (C) 2014 Matthias Kuhn
+    Email                : matthias dot kuhn at gmx dot ch
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+/**
+ *
+ * @note added in 2.8
+ */
+
+class QgsMultiRenderChecker
+{
+%TypeHeaderCode
+#include <qgsmultirenderchecker.h>
+%End
+  public:
+    QgsMultiRenderChecker();
+
+    /**
+     * Base directory name for the control image (with control image path
+     * suffixed) the path to the image will be constructed like this:
+     * controlImagePath + '/' + mControlName + '/' + mControlName + '.png'
+     */
+    void setControlName( const QString& theName );
+
+    void setControlPathPrefix( const QString& prefix );
+
+    /**
+     * Set the path to the rendered image. If this is not set or set to QString::Null, an image
+     * will be rendered based on the provided mapsettings
+     *
+     * @param renderedImagePath A path to the rendered image with which control images will be compared
+     */
+    void setRenderedImage( const QString& renderedImagePath );
+
+    /**
+     * Set the map settings to use to render the image
+     *
+     * @param mapSettings The map settings
+     */
+    void setMapSettings( const QgsMapSettings& mapSettings );
+
+    /**
+     * Set tolerance for color components used by runTest()
+     * Default value is 0.
+     *
+     * @param theColorTolerance The maximum difference for each color component
+     *                          including alpha to be considered correct.
+     */
+    void setColorTolerance( unsigned int theColorTolerance );
+
+    /**
+     * Test using renderer to generate the image to be compared.
+     *
+     * @param theTestName - to be used as the basis for writing a file to
+     * e.g. /tmp/theTestName.png
+     *
+     * @param theMismatchCount - defaults to 0 - the number of pixels that
+     * are allowed to be different from the control image. In some cases
+     * rendering may be non-deterministic. This parameter allows you to account
+     * for that by providing a tolerance.
+     *
+     * @note make sure to call setExpectedImage and setMapSettings first
+     */
+    bool runTest( const QString& theTestName, unsigned int theMismatchCount = 0 );
+
+    /**
+     * Returns a report for this test
+     *
+     * @return A report
+     */
+    const QString& report() const;
+
+    /**
+     * @brief controlImagePath
+     * @return
+     */
+    const QString controlImagePath() const;
+
+};
+

src/core/CMakeLists.txt

@@ -114,12 +114,13 @@ SET(QGIS_CORE_SRCS
   qgsmapsettings.cpp
   qgsmaptopixel.cpp
   qgsmaptopixelgeometrysimplifier.cpp
+  qgsmessagelog.cpp
   qgsmessageoutput.cpp
   qgsmimedatautils.cpp
-  qgsmessagelog.cpp
+  qgsmultirenderchecker.cpp
   qgsnetworkaccessmanager.cpp
-  qgsnetworkreplyparser.cpp
   qgsnetworkcontentfetcher.cpp
+  qgsnetworkreplyparser.cpp
   qgsobjectcustomproperties.cpp
   qgsofflineediting.cpp
   qgsogcutils.cpp
@@ -131,17 +132,17 @@ SET(QGIS_CORE_SRCS
   qgspoint.cpp
   qgsproject.cpp
   qgsprojectfiletransform.cpp
-  qgsprojectversion.cpp
   qgsprojectproperty.cpp
+  qgsprojectversion.cpp
   qgsprovidercountcalcevent.cpp
   qgsproviderextentcalcevent.cpp
   qgsprovidermetadata.cpp
   qgsproviderregistry.cpp
   qgspythonrunner.cpp
   qgsrelation.cpp
   qgsrelationmanager.cpp
-  qgsrendercontext.cpp
   qgsrenderchecker.cpp
+  qgsrendercontext.cpp
   qgsrectangle.cpp
   qgsrunprocess.cpp
   qgsscalecalculator.cpp
@@ -513,8 +514,9 @@ SET(QGIS_CORE_HDRS
   qgsmapunitscale.h
   qgsmessageoutput.h
   qgsmimedatautils.h
-  qgsnetworkreplyparser.h
+  qgsmultirenderchecker.h
   qgsnetworkcontentfetcher.h
+  qgsnetworkreplyparser.h
   qgsobjectcustomproperties.h
   qgsofflineediting.h
   qgsogcutils.h
@@ -534,10 +536,10 @@ SET(QGIS_CORE_HDRS
   qgsproviderregistry.h
   qgspythonrunner.h
   qgsrectangle.h
-  qgsrendercontext.h
-  qgsrenderchecker.h
   qgsrelation.h
   qgsrelationmanager.h
+  qgsrenderchecker.h
+  qgsrendercontext.h
   qgsrunprocess.h
   qgsscalecalculator.h
   qgsscaleutils.h

src/core/qgsmultirenderchecker.cpp

@@ -0,0 +1,92 @@
+/***************************************************************************
+    qgsmultirenderchecker.cpp
+     --------------------------------------
+    Date                 : 6.11.2014
+    Copyright            : (C) 2014 Matthias Kuhn
+    Email                : matthias dot kuhn at gmx dot ch
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsmultirenderchecker.h"
+
+#include <QDebug>
+
+QgsMultiRenderChecker::QgsMultiRenderChecker()
+{
+}
+
+void QgsMultiRenderChecker::setControlName( const QString& theName )
+{
+  mControlName = theName;
+}
+
+void QgsMultiRenderChecker::setControlPathPrefix( const QString& prefix )
+{
+  mControlPathPrefix = prefix;
+}
+
+void QgsMultiRenderChecker::setMapSettings( const QgsMapSettings& mapSettings )
+{
+  mMapSettings = mapSettings;
+}
+
+bool QgsMultiRenderChecker::runTest( const QString& theTestName, unsigned int theMismatchCount )
+{
+  bool successful = false;
+
+  const QString baseDir = controlImagePath();
+
+  QStringList subDirs = QDir( baseDir ).entryList( QDir::Dirs | QDir::NoDotAndDotDot );
+
+  if ( subDirs.count() == 0 )
+  {
+    subDirs << "";
+  }
+
+  Q_FOREACH( const QString& suffix, subDirs )
+  {
+    qDebug() << "Checking subdir " << suffix;
+    bool result;
+    QgsRenderChecker checker;
+    checker.setColorTolerance( mColorTolerance );
+    checker.setControlPathPrefix( mControlPathPrefix );
+    checker.setControlPathSuffix( suffix );
+    checker.setControlName( mControlName );
+    checker.setMapSettings( mMapSettings );
+
+    if ( !mRenderedImage.isNull() )
+    {
+      checker.setRenderedImage( mRenderedImage );
+      result = checker.compareImages( theTestName, theMismatchCount, mRenderedImage );
+    }
+    else
+    {
+      result = checker.runTest( theTestName, theMismatchCount );
+      mRenderedImage = checker.renderedImage();
+    }
+
+    qDebug() << " * Subdir check " << suffix << ": " << result;
+    successful |= result;
+
+    mReport += checker.report();
+  }
+
+  if ( !successful )
+    qDebug() << "No matching image found. If you think that this result should be considered ok, please copy it into a new subdirectory inside " << baseDir;
+
+  return successful;
+}
+
+const QString QgsMultiRenderChecker::controlImagePath() const
+{
+  QString myDataDir( TEST_DATA_DIR ); //defined in CmakeLists.txt
+  QString myControlImageDir = myDataDir + QDir::separator() + "control_images" +
+                              QDir::separator() + mControlPathPrefix + QDir::separator() + mControlName + QDir::separator();
+  return myControlImageDir;
+}

src/core/qgsmultirenderchecker.h

@@ -0,0 +1,126 @@
+/***************************************************************************
+    qgsmultirenderchecker.h
+     --------------------------------------
+    Date                 : 6.11.2014
+    Copyright            : (C) 2014 Matthias Kuhn
+    Email                : matthias dot kuhn at gmx dot ch
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSMULTIRENDERCHECKER_H
+#define QGSMULTIRENDERCHECKER_H
+
+#include "qgsrenderchecker.h"
+
+/**
+ * This class allows to check rendered images against comparison images.
+ * Its main purpose is for the unit testing framework.
+ *
+ * It will either
+ * <ul>
+ *   <li>take an externally rendered image (setRenderedImage())</li>
+ *   <li>render the image based on provided mapSettings (setMapSettings())</li>
+ * </ul>
+ *
+ * This image will then be compared against one or several images in a folder inside
+ * the control directory (tests/testdata/control_images/{controlName}).
+ *
+ * There are modes for single and for multiple reference images.
+ * <ul>
+ *   <li>If there are no subfolders in the control directory, it will assume an image
+ *       with the name {controlImage}.png in the control directory itself.</li>
+ *
+ *   <li>If there are subfolders inside the control directory, it will search for images
+ *       with the name {controlImage}.png in every subfolder.</li>
+ * </ul>
+ *
+ * For every control image there may be one or several randomly named anomaly images defining
+ * allowed anomalies.
+ * For every control image, the allowed mismatch and color tolerance values will be calculated
+ * individually.
+ *
+ * @note added in 2.8
+ */
+
+class CORE_EXPORT QgsMultiRenderChecker
+{
+  public:
+    QgsMultiRenderChecker();
+
+    /**
+     * Base directory name for the control image (with control image path
+     * suffixed) the path to the image will be constructed like this:
+     * controlImagePath + '/' + mControlName + '/' + mControlName + '.png'
+     */
+    void setControlName( const QString& theName );
+
+    void setControlPathPrefix( const QString& prefix );
+
+    /**
+     * Set the path to the rendered image. If this is not set or set to QString::Null, an image
+     * will be rendered based on the provided mapsettings
+     *
+     * @param renderedImagePath A path to the rendered image with which control images will be compared
+     */
+    void setRenderedImage( const QString& renderedImagePath ) { mRenderedImage = renderedImagePath; }
+
+    /**
+     * Set the map settings to use to render the image
+     *
+     * @param mapSettings The map settings
+     */
+    void setMapSettings( const QgsMapSettings& mapSettings );
+
+    /**
+     * Set tolerance for color components used by runTest()
+     * Default value is 0.
+     *
+     * @param theColorTolerance The maximum difference for each color component
+     *                          including alpha to be considered correct.
+     */
+    void setColorTolerance( unsigned int theColorTolerance ) { mColorTolerance = theColorTolerance; }
+
+    /**
+     * Test using renderer to generate the image to be compared.
+     *
+     * @param theTestName - to be used as the basis for writing a file to
+     * e.g. /tmp/theTestName.png
+     *
+     * @param theMismatchCount - defaults to 0 - the number of pixels that
+     * are allowed to be different from the control image. In some cases
+     * rendering may be non-deterministic. This parameter allows you to account
+     * for that by providing a tolerance.
+     *
+     * @note make sure to call setExpectedImage and setMapSettings first
+     */
+    bool runTest( const QString& theTestName, unsigned int theMismatchCount = 0 );
+
+    /**
+     * Returns a report for this test
+     *
+     * @return A report
+     */
+    const QString& report() const { return mReport; }
+
+    /**
+     * @brief controlImagePath
+     * @return
+     */
+    const QString controlImagePath() const;
+
+  private:
+    QString mReport;
+    QString mRenderedImage;
+    QString mControlName;
+    QString mControlPathPrefix;
+    unsigned int mColorTolerance;
+    QgsMapSettings mMapSettings;
+};
+
+#endif // QGSMULTIRENDERCHECKER_H

src/core/qgsrenderchecker.cpp

@@ -51,7 +51,7 @@ QString QgsRenderChecker::controlImagePath() const
 void QgsRenderChecker::setControlName( const QString theName )
 {
   mControlName = theName;
-  mExpectedImageFile = controlImagePath() + theName + QDir::separator()
+  mExpectedImageFile = controlImagePath() + theName + QDir::separator() + mControlPathSuffix
                        + theName + ".png";
 }