From c1039652b510ea4a45fd74808b3169121ed2494b Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 8 Apr 2016 12:06:43 +0200 Subject: [PATCH] Add new metric for tag combinations The metric measures how well combinations of tags (pairs only) are covered by the TCK. The output is an HTML table and a heatmap (PNG image). --- .../cypher/compatibility-suite/LICENSES.txt | 1 + .../cypher/compatibility-suite/NOTICE.txt | 1 + community/cypher/compatibility-suite/pom.xml | 9 +- .../reporting/CombinationChartWriter.java | 157 ++++++++++++++++++ ...rtWriter.java => CoverageChartWriter.java} | 4 +- .../reporting/CypherResultReporter.scala | 14 +- .../feature/reporting/OutputProducer.scala | 28 +++- .../CombinationChartWriterTest.scala | 103 ++++++++++++ 8 files changed, 307 insertions(+), 10 deletions(-) create mode 100644 community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/CombinationChartWriter.java rename community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/{ChartWriter.java => CoverageChartWriter.java} (97%) create mode 100644 community/cypher/compatibility-suite/src/test/scala/cypher/feature/parser/reporting/CombinationChartWriterTest.scala diff --git a/community/cypher/compatibility-suite/LICENSES.txt b/community/cypher/compatibility-suite/LICENSES.txt index 986d1feb4a416..20719dc70cee2 100644 --- a/community/cypher/compatibility-suite/LICENSES.txt +++ b/community/cypher/compatibility-suite/LICENSES.txt @@ -21,6 +21,7 @@ Apache Software License, Version 2.0 Lucene Memory MongoDB Java Driver nscala-time + openCypher Grammar Developer Tools openCypher TCK Developer Tools parboiled-core parboiled-scala diff --git a/community/cypher/compatibility-suite/NOTICE.txt b/community/cypher/compatibility-suite/NOTICE.txt index dbabccda893c7..47a3ffdbd5e32 100644 --- a/community/cypher/compatibility-suite/NOTICE.txt +++ b/community/cypher/compatibility-suite/NOTICE.txt @@ -44,6 +44,7 @@ Apache Software License, Version 2.0 Lucene Memory MongoDB Java Driver nscala-time + openCypher Grammar Developer Tools openCypher TCK Developer Tools parboiled-core parboiled-scala diff --git a/community/cypher/compatibility-suite/pom.xml b/community/cypher/compatibility-suite/pom.xml index 294bf87f4a416..eecd6a49debab 100644 --- a/community/cypher/compatibility-suite/pom.xml +++ b/community/cypher/compatibility-suite/pom.xml @@ -48,6 +48,7 @@ cypher.internal 2.11.8 2.11 + 1.2016-04-15 @@ -89,7 +90,13 @@ org.opencypher tck - 1.2016-04-11 + ${opencypher.version} + + + + org.opencypher + grammar + ${opencypher.version} diff --git a/community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/CombinationChartWriter.java b/community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/CombinationChartWriter.java new file mode 100644 index 0000000000000..7693fae0dc4a7 --- /dev/null +++ b/community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/CombinationChartWriter.java @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cypher.feature.parser.reporting; + +import java.awt.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; +import javax.imageio.ImageIO; + +import org.jfree.chart.ChartColor; +import org.jfree.chart.renderer.LookupPaintScale; +import org.jfree.data.general.DefaultHeatMapDataset; +import org.jfree.data.general.HeatMapDataset; +import org.jfree.data.general.HeatMapUtilities; +import org.opencypher.tools.io.HtmlTag; + +import static org.opencypher.tools.io.HtmlTag.attr; + + +public class CombinationChartWriter +{ + private final File outDirectory; + private final String filename; + private final LookupPaintScale paintScale; + + static final int UPPER_BOUND = 10000; + + public CombinationChartWriter( File outDirectory, String filename ) + { + this.outDirectory = outDirectory; + this.filename = filename; + paintScale = new LookupPaintScale( -1, UPPER_BOUND, Color.WHITE ); + buildPaintScale(); + } + + private void buildPaintScale() + { + paintScale.add( -1, ChartColor.BLACK ); + paintScale.add( 0, ChartColor.VERY_DARK_RED ); + paintScale.add( 10, ChartColor.DARK_RED ); + paintScale.add( 25, ChartColor.RED ); + paintScale.add( 50, ChartColor.LIGHT_RED ); + paintScale.add( 75, ChartColor.VERY_LIGHT_RED ); + paintScale.add( 100, ChartColor.VERY_LIGHT_GREEN ); + paintScale.add( 150, ChartColor.LIGHT_GREEN ); + paintScale.add( 200, ChartColor.GREEN ); + paintScale.add( 300, ChartColor.DARK_GREEN ); + paintScale.add( 500, ChartColor.VERY_DARK_GREEN ); + } + + void dumpPNG( List> data ) + { + try ( FileOutputStream output = new FileOutputStream( new File( outDirectory, filename + ".png" ) ) ) + { + ImageIO.write( HeatMapUtilities.createHeatMapImage( createHeatMapDataset( data ), paintScale ), + "png", output ); + } + catch ( IOException e ) + { + throw new RuntimeException( "Unexpected error during PNG file creation", e ); + } + } + + public void dumpHTML( List> data, List tags ) + { + dumpPNG( data ); + try ( HtmlTag.Html html = HtmlTag.html( new File( outDirectory, filename + ".html" ).toPath() ) ) + { + try ( HtmlTag body = html.body() ) + { + buildTable( body, data, tags ); + body.tag( "img", attr( "src", filename + ".png" ) ); + } + } + } + + private void buildTable( HtmlTag body, List> data, List tags ) + { + try ( HtmlTag table = body.tag( "table", attr( "border", "1" ) ) ) + { + for ( int i = tags.size() - 1; i > -1; --i ) + { + try ( HtmlTag row = table.tag( "tr" ) ) + { + for ( int j = 0; j < data.get( i ).size(); ++j ) + { + if ( j == i ) + { + final String columns = String.valueOf( j + 1 ); + row.tag( "td", attr( "colspan", columns ), attr( "align", "right" ) ).text( tags.get( i ) ); + } + else if ( j > i ) + { + row.tag( "td", attr( "align", "center" ), attr( "width", "25px" ), + attr( "height", "25px" ) ).text( data.get( i ).get( j ).toString() ); + } + } + } + } + } + } + + private HeatMapDataset createHeatMapDataset( List> data ) + { + int magnification = 10; + DefaultHeatMapDataset dataset = new DefaultHeatMapDataset( data.size() * magnification, + data.size() * magnification, 0, data.size() * magnification, 0, data.size() * magnification ); + + for ( int i = data.size() - 1; i > -1; --i ) + { + for ( int j = 0; j < data.get( i ).size(); ++j ) + { + double z; + if ( j < i ) + { + z = UPPER_BOUND + 1; + } + else if ( j == i ) + { + z = -1; + } + else + { + z = data.get( i ).get( j ); + } + // magnify each point to a 10x10 pixel square + for ( int xi = 0; xi < magnification; ++xi ) + { + for ( int yi = 0; yi < magnification; ++yi ) + { + dataset.setZValue( j * magnification + yi, i * magnification + xi, z ); + } + } + } + } + return dataset; + } +} diff --git a/community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/ChartWriter.java b/community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/CoverageChartWriter.java similarity index 97% rename from community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/ChartWriter.java rename to community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/CoverageChartWriter.java index 5974ad443bb89..269326eb8b44e 100644 --- a/community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/ChartWriter.java +++ b/community/cypher/compatibility-suite/src/main/java/cypher/feature/parser/reporting/CoverageChartWriter.java @@ -43,12 +43,12 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -public class ChartWriter +public class CoverageChartWriter { private final File outDirectory; private final String filename; - public ChartWriter( File outDirectory, String filename ) + public CoverageChartWriter( File outDirectory, String filename ) { this.outDirectory = outDirectory; this.filename = filename; diff --git a/community/cypher/compatibility-suite/src/main/scala/cypher/feature/reporting/CypherResultReporter.scala b/community/cypher/compatibility-suite/src/main/scala/cypher/feature/reporting/CypherResultReporter.scala index 33a50c1024dd3..c30be12c953fb 100644 --- a/community/cypher/compatibility-suite/src/main/scala/cypher/feature/reporting/CypherResultReporter.scala +++ b/community/cypher/compatibility-suite/src/main/scala/cypher/feature/reporting/CypherResultReporter.scala @@ -22,7 +22,7 @@ package cypher.feature.reporting import java.io.{File, PrintStream} import cypher.cucumber.CucumberAdapter -import cypher.feature.parser.reporting.ChartWriter +import cypher.feature.parser.reporting.{CombinationChartWriter, CoverageChartWriter} import gherkin.formatter.model.{Match, Result, Step} import org.opencypher.tools.tck.constants.TCKStepDefinitions @@ -36,13 +36,15 @@ object CypherResultReporter { } } -class CypherResultReporter(producer: OutputProducer, jsonWriter: PrintStream, chartWriter: ChartWriter) +class CypherResultReporter(producer: OutputProducer, jsonWriter: PrintStream, chartWriter: CoverageChartWriter, + combinationChartWriter: CombinationChartWriter) extends CucumberAdapter { def this(reportDir: File) = { this(producer = JsonProducer, jsonWriter = CypherResultReporter.createPrintStream(reportDir, "compact.json"), - chartWriter = new ChartWriter(reportDir, "tags")) + chartWriter = new CoverageChartWriter(reportDir, "tags"), + combinationChartWriter = new CombinationChartWriter(reportDir, "tagCombinations")) } private var query: String = null @@ -52,8 +54,10 @@ class CypherResultReporter(producer: OutputProducer, jsonWriter: PrintStream, ch override def done(): Unit = { jsonWriter.println(producer.dump()) - chartWriter.dumpSVG(producer.dumpTagStats()) - chartWriter.dumpPNG(producer.dumpTagStats()) + chartWriter.dumpSVG(producer.dumpTagStats) + chartWriter.dumpPNG(producer.dumpTagStats) + val stats = producer.dumpTagCombinationStats + combinationChartWriter.dumpHTML(stats._1, stats._2) } override def close(): Unit = { diff --git a/community/cypher/compatibility-suite/src/main/scala/cypher/feature/reporting/OutputProducer.scala b/community/cypher/compatibility-suite/src/main/scala/cypher/feature/reporting/OutputProducer.scala index ce53c117b0072..ee373dd97bfa3 100644 --- a/community/cypher/compatibility-suite/src/main/scala/cypher/feature/reporting/OutputProducer.scala +++ b/community/cypher/compatibility-suite/src/main/scala/cypher/feature/reporting/OutputProducer.scala @@ -19,6 +19,8 @@ */ package cypher.feature.reporting +import java.util + import org.neo4j.cypher.internal.compiler.v3_1.ast.{QueryTagger, QueryTags} import scala.collection.JavaConverters._ @@ -29,7 +31,8 @@ trait OutputProducer { def complete(query: String, outcome: Outcome): Unit def dump(): String - def dumpTagStats(): java.util.Map[String, Integer] + def dumpTagStats: java.util.Map[String, Integer] + def dumpTagCombinationStats: (java.util.List[java.util.List[java.lang.Integer]], java.util.List[String]) } object JsonProducer extends JsonProducer(tagger = QueryTagger) @@ -49,7 +52,7 @@ class JsonProducer(tagger: QueryTagger[String]) extends OutputProducer { grater[JsonResult].toPrettyJSONArray(results.toList) } - override def dumpTagStats(): java.util.Map[String, Integer] = { + override def dumpTagStats: java.util.Map[String, Integer] = { val tagCounts = results.map(result => result.prettyTags).foldLeft(Map.empty[String, Integer]) { case (map, tags) => tags.foldLeft(map) { @@ -66,4 +69,25 @@ class JsonProducer(tagger: QueryTagger[String]) extends OutputProducer { private def sortByValue(map: Map[String, Integer]) = ListMap(map.toList.sortBy(_._2): _*) + override def dumpTagCombinationStats: (java.util.List[java.util.List[java.lang.Integer]], java.util.List[String]) = { + val tags = QueryTags.all.toList.map(_.toString) + val indexMap = tags.zipWithIndex.toMap + + val innerList = tags.indices.map(_ => Int.box(0)) + + val list = new util.ArrayList[java.util.List[java.lang.Integer]]() + tags.indices.foreach(_ => list.add(new util.ArrayList[java.lang.Integer](innerList.asJava))) + + results.map(result => result.prettyTags).foreach { tags => + // compile combinations + val map: Set[(String, String)] = tags.flatMap(tag => tags.filterNot(s => s == tag).map(s => (tag, s))) + map.foreach { + case (t1, t2) if t1 != t2 => + val ints = list.get(indexMap(t1)) + ints.set(indexMap(t2), ints.get(indexMap(t2)) + 1) + case _ => + } + } + (list, tags.asJava) + } } diff --git a/community/cypher/compatibility-suite/src/test/scala/cypher/feature/parser/reporting/CombinationChartWriterTest.scala b/community/cypher/compatibility-suite/src/test/scala/cypher/feature/parser/reporting/CombinationChartWriterTest.scala new file mode 100644 index 0000000000000..b31fb8269e1b7 --- /dev/null +++ b/community/cypher/compatibility-suite/src/test/scala/cypher/feature/parser/reporting/CombinationChartWriterTest.scala @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cypher.feature.parser.reporting + +import java.io.File +import javax.imageio.ImageIO + +import cypher.feature.parser.reporting.CombinationChartWriter.UPPER_BOUND +import org.neo4j.cypher.internal.compiler.v3_1.ast.{QueryTag, QueryTags} +import org.scalatest.FunSuite +import org.scalatest.Matchers._ + +import scala.collection.JavaConverters._ + +class CombinationChartWriterTest extends FunSuite { + + test("should generate simple png image") { + val writer = new CombinationChartWriter(new File("target"), "test") + + val tags = List("a", "b", "c") + + val data = List(List(Int.box(-1), Int.box(25), Int.box(50)).asJava, + List(Int.box(25), Int.box(-1), Int.box(75)).asJava, + List(Int.box(50), Int.box(75), Int.box(-1)).asJava).asJava + + writer.dumpPNG(data) + + val file = new File("target", "test.png") + file.exists() shouldBe true + ImageIO.read(file).getHeight shouldBe tags.length * 10 + } + + test("should generate png from tags") { + val writer = new CombinationChartWriter(new File("target"), "tags") + + val tags: List[QueryTag] = QueryTags.all.toList + + val data = tags.map { tag => + tags.map { otherTag => + Integer.valueOf(Math.abs((tag.toString + otherTag.toString).toSet.hashCode) % UPPER_BOUND) + }.asJava + }.asJava + + writer.dumpPNG(data) + + val file = new File("target", "tags.png") + file.exists() shouldBe true + ImageIO.read(file).getHeight shouldBe QueryTags.all.size * 10 + } + + test("should generate simple html table") { + val writer = new CombinationChartWriter(new File("target"), "table") + + val tags = List("a", "b", "c") + + val data = tags.map { tag => + tags.map { otherTag => + if (tag == otherTag) Int.box(-1) + else Integer.valueOf(Math.abs((tag + otherTag).toSet.hashCode) % UPPER_BOUND) + }.asJava + }.asJava + + writer.dumpHTML(data, tags.asJava) + + val file = new File("target", "table.html") + file.exists() shouldBe true + } + + test("should generate html from tags") { + val writer = new CombinationChartWriter(new File("target"), "tags") + + val tags = QueryTags.all.toList + + val data = tags.map { tag => + tags.map { otherTag => + Integer.valueOf(Math.abs((tag.toString + otherTag.toString).toSet.hashCode) % UPPER_BOUND) + }.asJava + }.asJava + + writer.dumpHTML(data, tags.map(_.toString).asJava) + + val file = new File("target", "tags.html") + file.exists() shouldBe true + } + +}