Skip to content

Commit

Permalink
Improve image comparison in chart unit tests
Browse files Browse the repository at this point in the history 
Number of tests in /testsuites/org.eclipse.birt.report.tests.chart and
/chart/org.eclipse.birt.chart.tests packages may fail on Linux due to
actual-to-golden image (JPG, PNG, SVG) comparison is done using byte
comparison. The files may look similar visually, but different
internally. This is solved as:
* New image comparison utility was implemented to compare image files by
comparing average weighted brightness in corresponding blocks. It can
also compare images of different sizes. Settings are parameterized. The
default settings were confirmed to work in most tests. If the images are
found to be different, a difference image is returned with blocks failed
validation marked in red.
* Updated failed regression and unit test cases to use the new image
comparison where appropriate.
* File comparison utility was updated to (optionally) ignore line breaks
for comparing text files created on different platforms.

Signed-off-by: Serguei Krivtsov <skrivtsov@actuate.com>
  • Loading branch information
@serguei-actuate
serguei-actuate committed Jun 13, 2016
1 parent dabcd14 commit bcac728
Show file tree
Hide file tree
Showing 63 changed files with 334 additions and 82 deletions.
5 changes: 4 additions & 1 deletion chart/org.eclipse.birt.chart.tests/META-INF/MANIFEST.MF
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,10 @@ Require-Bundle: org.eclipse.core.runtime,
org.junit,
org.eclipse.birt.chart.device.pdf,
org.eclipse.birt.report.model,
org.eclipse.birt.chart.reportitem
org.eclipse.birt.chart.reportitem,
org.eclipse.birt.tests.core,
org.apache.batik.transcoder,
org.apache.xerces;bundle-version="[2.8.0,3.0.0)";resolution:=optional
Eclipse-LazyStart: true
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Import-Package: com.ibm.icu.text;version="3.4.4",
Expand Down
13 changes: 13 additions & 0 deletions chart/org.eclipse.birt.chart.tests/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,17 @@
<artifactId>org.eclipse.birt.chart.tests</artifactId>
<version>4.6.0-SNAPSHOT</version>
<packaging>eclipse-test-plugin</packaging>
<build>
<plugins>
<plugin>
<groupId>org.eclipse.tycho</groupId>
<artifactId>tycho-surefire-plugin</artifactId>
<version>${tycho.version}</version>
<configuration>
<testSuite>org.eclipse.birt.chart.tests</testSuite>
<testClass>org.eclipse.birt.chart.tests.AllTests</testClass>
</configuration>
</plugin>
</plugins>
</build>
</project>
78 changes: 61 additions & 17 deletions ....birt.chart.tests/src/org/eclipse/birt/chart/tests/device/render/ImageOutputBaseTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,30 @@

package org.eclipse.birt.chart.tests.device.render;

import java.awt.Image;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;

import junit.framework.TestCase;
import org.apache.batik.transcoder.TranscoderInput;
import org.apache.batik.transcoder.TranscoderOutput;
import org.apache.batik.transcoder.image.PNGTranscoder;

import org.eclipse.birt.chart.tests.util.FileUtil;
import junit.framework.TestCase;
import utility.ImageUtil;
import utility.ImageUtil.ImageCompParam;

/**
* BaseTest is a JUnit test case that will generate image files based on
* a set of primitive drawing events defined in drawing script files,
* and then compare the generated files with the expected files.
*/
public class ImageOutputBaseTest extends TestCase {

protected File file;
protected String dirName;
protected String filename;
Expand All @@ -45,22 +55,56 @@ public ImageOutputBaseTest(File file, String dirName, String filename, String wo
}

public void runTest( ) throws Throwable {


Png24PrimitiveGen generator2 = new Png24PrimitiveGen(new FileInputStream(workspaceDir+File.separator+ImageRenderTest.INDIR+dirName+File.separator+filename+ImageRenderTest.DRAWEXT), workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".png");//$NON-NLS-1$
generator2.generate();
generator2.flush();
assertTrue(FileUtil.compareFiles(new FileInputStream(workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".png"), new FileInputStream(workspaceDir+File.separator+ImageRenderTest.CONTROLDIR+dirName+File.separator+filename+".png")));//$NON-NLS-1$//$NON-NLS-2$

SvgPrimitiveGen generator3 = new SvgPrimitiveGen(new FileInputStream(workspaceDir+File.separator+ImageRenderTest.INDIR+dirName+File.separator+filename+ImageRenderTest.DRAWEXT), workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".svg");//$NON-NLS-1$
generator3.generate();
generator3.flush();
assertTrue(FileUtil.compareFiles(new FileInputStream(workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".svg"), new FileInputStream(workspaceDir+File.separator+ImageRenderTest.CONTROLDIR+dirName+File.separator+filename+".svg")));//$NON-NLS-1$//$NON-NLS-2$
Map<ImageCompParam, Integer> params = new HashMap<ImageCompParam, Integer>();
params.put(ImageCompParam.TOLERANCE, 4);

// 1: verify PNG
Png24PrimitiveGen generator2 = new Png24PrimitiveGen(new FileInputStream(workspaceDir+File.separator+ImageRenderTest.INDIR+dirName+File.separator+filename+ImageRenderTest.DRAWEXT), workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".png");//$NON-NLS-1$
generator2.generate();
generator2.flush();
Image result =
ImageUtil.compare(
workspaceDir+File.separator+ImageRenderTest.CONTROLDIR+dirName+File.separator+filename+".png", //$NON-NLS-1$
workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".png"); //$NON-NLS-1$
if(result != null) {
ImageUtil.savePNG(result, workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".diff.png");
fail();
}

// 2: verify SVG
SvgPrimitiveGen generator3 = new SvgPrimitiveGen(new FileInputStream(workspaceDir+File.separator+ImageRenderTest.INDIR+dirName+File.separator+filename+ImageRenderTest.DRAWEXT), workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".svg"); //$NON-NLS-1$
generator3.generate();
generator3.flush();

// convert golden and actual SVG to PNG
String[] files = {
workspaceDir+File.separator+ImageRenderTest.CONTROLDIR+dirName+File.separator+filename+".svg", //$NON-NLS-1$
workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".svg" //$NON-NLS-1$
};
for(String file: files) {
InputStream inStream = new FileInputStream(file);
TranscoderInput input = new TranscoderInput( inStream );
OutputStream outStream = new FileOutputStream(file + ".png"); //$NON-NLS-1$
TranscoderOutput output = new TranscoderOutput(outStream);
PNGTranscoder converter = new PNGTranscoder();
converter.transcode(input, output);
outStream.flush();
outStream.close();
}
result =
ImageUtil.compare(
files[0] + ".png", //$NON-NLS-1$
files[1] + ".png"); //$NON-NLS-1$
if(result != null) {
ImageUtil.savePNG(result, files[1] + ".diff.png"); //$NON-NLS-1$
fail();
}

// PdfPrimitiveGen generator4 = new PdfPrimitiveGen(new FileInputStream(workspaceDir+File.separator+ImageRenderTest.INDIR+dirName+File.separator+filename+ImageRenderTest.DRAWEXT), workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".pdf");//$NON-NLS-1$
// generator4.generate();
// generator4.flush();
// assertTrue(FileUtil.compareFiles(new FileInputStream(workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".pdf"), new FileInputStream(workspaceDir+File.separator+ImageRenderTest.CONTROLDIR+dirName+File.separator+filename+".pdf")));//$NON-NLS-1$//$NON-NLS-2$
// PdfPrimitiveGen generator4 = new PdfPrimitiveGen(new FileInputStream(workspaceDir+File.separator+ImageRenderTest.INDIR+dirName+File.separator+filename+ImageRenderTest.DRAWEXT), workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".pdf");//$NON-NLS-1$
// generator4.generate();
// generator4.flush();
// assertTrue(FileUtil.compareFiles(new FileInputStream(workspaceDir+File.separator+ImageRenderTest.OUTDIR+dirName+File.separator+filename+".pdf"), new FileInputStream(workspaceDir+File.separator+ImageRenderTest.CONTROLDIR+dirName+File.separator+filename+".pdf")));//$NON-NLS-1$//$NON-NLS-2$
}
}

Expand Down
32 changes: 26 additions & 6 deletions chart/org.eclipse.birt.chart.tests/src/org/eclipse/birt/chart/tests/util/FileUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,15 @@

package org.eclipse.birt.chart.tests.util;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;

/**
*
Expand All @@ -27,14 +35,26 @@ public class FileUtil {
* @return true if the contents match; otherwise, false is returned.
* @throws Exception thrown if io errors occur
*/
public static boolean compareFiles(InputStream left, InputStream right)
public static boolean compareFiles(InputStream left, InputStream right, boolean ignoreLineBreaks)
throws Exception {
int leftChar = -1;
while ((leftChar = left.read()) != -1){
if (leftChar != right.read())

int leftChar, rightChar;
do {
do {
leftChar = left.read();
} while(ignoreLineBreaks && (leftChar == 0x0D || leftChar == 0x0A));
do {
rightChar = right.read();
} while(ignoreLineBreaks && (rightChar == 0x0D || rightChar == 0x0A));
if(leftChar != rightChar) {
return false;

}
}
} while(leftChar != -1 && rightChar != -1);
return true;
}

public static boolean compareFiles(InputStream left, InputStream right)
throws Exception {
return compareFiles(left, right, false);
}
}
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_101039.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ public Regression_101039( )
public void test_regression_101039( ) throws Exception
{
Regression_101039 st = new Regression_101039( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_101855.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public Regression_101855( )
public void test_regression_101855( ) throws Exception
{
Regression_101855 st = new Regression_101855( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_103787.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public Regression_103787( )
public void test_regression_103787( ) throws Exception
{
Regression_103787 st = new Regression_103787( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_104472.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ public Regression_104472( )
public void test_regression_104472( ) throws Exception
{
Regression_104472 st = new Regression_104472( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_104627.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ public Regression_104627( )
public void test_regression_104627( ) throws Exception
{
Regression_104627 st = new Regression_104627( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_106126.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public Regression_106126( )
public void test_regression_106126( ) throws Exception
{
Regression_106126 st = new Regression_106126( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...t.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_109622_1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public Regression_109622_1( )
public void test_regression_109622_1( ) throws Exception
{
Regression_109622_1 st = new Regression_109622_1( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

private void bindGroupingData( Chart chart )
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_109641.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public Regression_109641( )
public void test_regression_109641( ) throws Exception
{
Regression_109641 st = new Regression_109641( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

private void bindGroupingData( Chart chart )
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_113536.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ public Regression_113536( )
public void test_regression_113536( ) throws Exception
{
new Regression_113536( );
assertTrue( this.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( this.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_115433.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ public Regression_115433( )
public void test_regression_115433( ) throws Exception
{
Regression_115433 st = new Regression_115433( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_119411.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ public Regression_119411( )
public void test_regression_119411( ) throws Exception
{
Regression_119411 st = new Regression_119411( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_119805.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ public Regression_119805( )
public void test_regression_119805( ) throws Exception
{
Regression_119805 st = new Regression_119805( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_120557.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ public Regression_120557( )
public void test_regression_120557( ) throws Exception
{
Regression_120557 st = new Regression_120557( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_120919.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public Regression_120919( )
public void test_regression_120919( ) throws Exception
{
Regression_120919 st = new Regression_120919( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

private void bindGroupingData( Chart chart )
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_121383.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public Regression_121383( )
public void test_regression_121383( ) throws Exception
{
Regression_121383 st = new Regression_121383( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_121813.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ public Regression_121813( )
public void test_regression_121813( ) throws Exception
{
Regression_121813 st = new Regression_121813( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_121828.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ public Regression_121828( )
public void test_regression_121828( ) throws Exception
{
Regression_121828 st = new Regression_121828( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_121831.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ public Regression_121831( )
public void test_regression_121831( ) throws Exception
{
Regression_121831 st = new Regression_121831( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_121836.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public Regression_121836( )
public void test_regression_121836( ) throws Exception
{
Regression_121836 st = new Regression_121836( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_122396.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ public Regression_122396( )
public void test_regression_122396( ) throws Exception
{
Regression_122396 st = new Regression_122396( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_122807.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ public Regression_122807( )
public void test_regression_122807( ) throws Exception
{
Regression_122807 st = new Regression_122807( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_123202.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public Regression_123202( )
public void test_regression_123202( ) throws Exception
{
Regression_123202 st = new Regression_123202( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
2 changes: 1 addition & 1 deletion ...ort.tests.chart/src/org/eclipse/birt/report/tests/chart/regression/Regression_123208.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public Regression_123208( )
public void test_regression_123208( ) throws Exception
{
Regression_123208 st = new Regression_123208( );
assertTrue( st.compareBytes( GOLDEN, OUTPUT ) );
assertTrue( st.compareImages( GOLDEN, OUTPUT ) );
}

/**
Expand Down
Loading

0 comments on commit bcac728

Please sign in to comment.