From 5246cf933237664aba303a3950d066aa3ff45fb3 Mon Sep 17 00:00:00 2001 From: Igal Tabachnik Date: Wed, 12 Jun 2019 15:21:49 +0300 Subject: [PATCH 1/3] Specs2 now will create its JUnit Description tree with filtered child items --- .../specs2/Specs2RunnerBuilder.scala | 40 +++++++++++-------- .../io/bazel/rulesscala/specs2/package.scala | 6 ++- .../test/junit/specs2/Specs2Tests.scala | 8 ++++ test_rules_scala.sh | 34 ++++++++++++++++ 4 files changed, 69 insertions(+), 19 deletions(-) diff --git a/src/java/io/bazel/rulesscala/specs2/Specs2RunnerBuilder.scala b/src/java/io/bazel/rulesscala/specs2/Specs2RunnerBuilder.scala index d783956ee..b9ca19f49 100644 --- a/src/java/io/bazel/rulesscala/specs2/Specs2RunnerBuilder.scala +++ b/src/java/io/bazel/rulesscala/specs2/Specs2RunnerBuilder.scala @@ -3,8 +3,8 @@ package io.bazel.rulesscala.specs2 import java.util import java.util.regex.Pattern -import io.bazel.rulesscala.test_discovery._ import io.bazel.rulesscala.test_discovery.FilteredRunnerBuilder.FilteringRunnerBuilder +import io.bazel.rulesscala.test_discovery._ import org.junit.runner.notification.RunNotifier import org.junit.runner.{Description, RunWith, Runner} import org.junit.runners.Suite @@ -12,12 +12,13 @@ import org.junit.runners.model.RunnerBuilder import org.specs2.concurrent.ExecutionEnv import org.specs2.control.Action import org.specs2.main.{Arguments, CommandLine, Select} -import org.specs2.specification.core.Env +import org.specs2.specification.core.{Env, Fragment, SpecStructure} import org.specs2.specification.process.Stats -import scala.language.reflectiveCalls import scala.collection.JavaConverters._ +import scala.language.reflectiveCalls import scala.util.Try +import scala.util.control.NonFatal @RunWith(classOf[Specs2PrefixSuffixTestDiscoveringSuite]) class Specs2DiscoveredTestSuite @@ -54,12 +55,15 @@ class FilteredSpecs2ClassRunner(testClass: Class[_], testFilter: Pattern) extends org.specs2.runner.JUnitRunner(testClass) { override def getDescription(env: Env): Description = { - val root = super.getDescription(env) - val filtered = flatten(root).filter(matchingFilter) + try createFilteredDescription(specStructure, env.specs2ExecutionEnv) + catch { case NonFatal(t) => env.shutdown; throw t; } + } - val flattenedRoot = root.childlessCopy() - filtered.foreach(flattenedRoot.addChild) - flattenedRoot + private def createFilteredDescription(specStructure: SpecStructure, ee: ExecutionEnv): Description = { + val tree = allFragmentDescriptions(ee) + val desc = Description.createSuiteDescription(testClass) + tree.values.filter(matchingFilter).foreach(desc.addChild) + desc } def matchesFilter: Boolean = { @@ -75,6 +79,11 @@ class FilteredSpecs2ClassRunner(testClass: Class[_], testFilter: Pattern) else sanitized } + private def allFragmentDescriptions(implicit ee: ExecutionEnv): Map[Fragment, Description] = + Try(allDescriptions[specs2_v4].fragmentDescriptions(specStructure)(ee)) + .orElse(Try(allDescriptions[specs2_v3].fragmentDescriptions(specStructure))) + .getOrElse(Map.empty) + /** * Retrieves an original (un-sanitized) text of an example fragment, * used later as a regex string for specs2 matching. @@ -82,13 +91,13 @@ class FilteredSpecs2ClassRunner(testClass: Class[_], testFilter: Pattern) * This is done by matching the actual (sanitized) string with the sanitized version * of the original example text. */ - private def specs2Description(desc: String)(implicit ee: ExecutionEnv): String = - Try { allDescriptions[specs2_v4].fragmentDescriptions(specStructure)(ee) } - .getOrElse(allDescriptions[specs2_v3].fragmentDescriptions(specStructure)) + private def specs2Description(desc: String)(implicit ee: ExecutionEnv): String = { + allFragmentDescriptions .keys .map(fragment => fragment.description.show) .find(sanitize(_) == desc) .getOrElse(desc) + } private def toDisplayName(description: Description)(implicit ee: ExecutionEnv): Option[String] = for { name <- Option(description.getMethodName) @@ -115,11 +124,8 @@ class FilteredSpecs2ClassRunner(testClass: Class[_], testFilter: Pattern) * * This function returns a flat list of the descriptions and their children, starting with the root. */ - def flatten(root: Description): List[Description] = { - def flatten0(desc: Description, xs: List[Description]): List[Description] = - desc.childlessCopy() :: desc.getChildren.asScala.foldLeft(xs)((acc, x) => flatten0(x, acc)) - - flatten0(root, Nil) + def flattenChildren(root: Description): List[Description] = { + root.getChildren.asScala.toList.flatMap(d => d :: flattenChildren(d)) } private def matchingFilter(desc: Description): Boolean = { @@ -128,7 +134,7 @@ class FilteredSpecs2ClassRunner(testClass: Class[_], testFilter: Pattern) } private def specs2Examples(implicit ee: ExecutionEnv): List[String] = - flatten(getDescription).flatMap(toDisplayName(_)) + flattenChildren(getDescription).flatMap(toDisplayName(_)) override def runWithEnv(n: RunNotifier, env: Env): Action[Stats] = { implicit val ee = env.executionEnv diff --git a/src/java/io/bazel/rulesscala/specs2/package.scala b/src/java/io/bazel/rulesscala/specs2/package.scala index 341cb1649..af486236b 100644 --- a/src/java/io/bazel/rulesscala/specs2/package.scala +++ b/src/java/io/bazel/rulesscala/specs2/package.scala @@ -7,11 +7,13 @@ import org.specs2.specification.core.{Fragment, SpecStructure} package object specs2 { type specs2_v4 = { + //noinspection ScalaUnusedSymbol def fragmentDescriptions(spec: SpecStructure)(ee: ExecutionEnv): Map[Fragment, Description] } type specs2_v3 = { + //noinspection ScalaUnusedSymbol def fragmentDescriptions(spec: SpecStructure): Map[Fragment, Description] } - def allDescriptions[T] = JUnitDescriptions.asInstanceOf[T] -} + def allDescriptions[T]: T = JUnitDescriptions.asInstanceOf[T] +} \ No newline at end of file diff --git a/test/src/main/scala/scalarules/test/junit/specs2/Specs2Tests.scala b/test/src/main/scala/scalarules/test/junit/specs2/Specs2Tests.scala index d7b078f05..95fa48eba 100644 --- a/test/src/main/scala/scalarules/test/junit/specs2/Specs2Tests.scala +++ b/test/src/main/scala/scalarules/test/junit/specs2/Specs2Tests.scala @@ -17,6 +17,14 @@ class JunitSpecs2Test extends SpecWithJUnit { } } +class FailingSpecs2Test extends SpecWithJUnit { + + "specs2 tests" should { + "succeed" >> success + "fail" >> failure + } +} + class JunitSpecs2AnotherTest extends SpecWithJUnit { "other specs2 tests" should { diff --git a/test_rules_scala.sh b/test_rules_scala.sh index 038d5d220..b3b35a5f4 100755 --- a/test_rules_scala.sh +++ b/test_rules_scala.sh @@ -528,6 +528,38 @@ scala_specs2_only_filtered_test_shows_in_the_xml(){ test -e } +scala_specs2_all_tests_show_in_the_xml(){ + bazel test \ + --nocache_test_results \ + --test_output=streamed \ + '--test_filter=scalarules.test.junit.specs2.JunitSpecs2Test#' \ + test:Specs2Tests + matches=$(grep -c -e "testcase name='specs2 tests should::run smoothly in bazel'" -e "testcase name='specs2 tests should::not run smoothly in bazel'" ./bazel-testlogs/test/Specs2Tests/test.xml) + if [ $matches -eq 2 ]; then + return 0 + else + echo "Expecting two results, found a different number ($matches). Please check 'bazel-testlogs/test/Specs2Tests/test.xml'" + return 1 + fi + test -e +} + +scala_specs2_only_failed_test_shows_in_the_xml(){ + bazel test \ + --nocache_test_results \ + --test_output=streamed \ + '--test_filter=scalarules.test.junit.specs2.FailingSpecs2Test#specs2 tests should::fail$' \ + test:Specs2Tests + matches=$(grep -c -e "testcase name='specs2 tests should::fail'" -e "testcase name='specs2 tests should::succeed'" ./bazel-testlogs/test/Specs2Tests/test.xml) + if [ $matches -eq 1 ]; then + return 0 + else + echo "Expecting only one result, found more than one. Please check 'bazel-testlogs/test/Specs2Tests/test.xml'" + return 1 + fi + test -e +} + scala_specs2_junit_test_test_filter_exact_match(){ local output=$(bazel test \ --nocache_test_results \ @@ -1002,7 +1034,9 @@ $runner scala_specs2_junit_test_test_filter_exact_match_escaped_and_sanitized $runner scala_specs2_junit_test_test_filter_match_multiple_methods $runner scala_specs2_exception_in_initializer_without_filter $runner scala_specs2_exception_in_initializer_terminates_without_timeout +$runner scala_specs2_all_tests_show_in_the_xml $runner scala_specs2_only_filtered_test_shows_in_the_xml +$runner scala_specs2_only_failed_test_shows_in_the_xml $runner scalac_jvm_flags_are_configured $runner javac_jvm_flags_are_configured $runner javac_jvm_flags_via_javacopts_are_configured From 032ff50fe5814a5cd36458e0359f36d1509a8970 Mon Sep 17 00:00:00 2001 From: Igal Tabachnik Date: Thu, 13 Jun 2019 17:06:10 +0300 Subject: [PATCH 2/3] Creating a filtered description tree from Specs2 utilities - keeps ordering and hashCodes intact --- .../specs2/Specs2RunnerBuilder.scala | 30 +++++++++++++------ .../io/bazel/rulesscala/specs2/package.scala | 5 ++-- .../test/junit/specs2/Specs2Tests.scala | 8 ----- test_expect_failure/scala_junit_test/BUILD | 2 +- .../specs2/SuiteWithOneFailingTest.scala | 15 ++++++++++ test_rules_scala.sh | 8 ++--- 6 files changed, 44 insertions(+), 24 deletions(-) create mode 100644 test_expect_failure/scala_junit_test/specs2/SuiteWithOneFailingTest.scala diff --git a/src/java/io/bazel/rulesscala/specs2/Specs2RunnerBuilder.scala b/src/java/io/bazel/rulesscala/specs2/Specs2RunnerBuilder.scala index b9ca19f49..c26c63a13 100644 --- a/src/java/io/bazel/rulesscala/specs2/Specs2RunnerBuilder.scala +++ b/src/java/io/bazel/rulesscala/specs2/Specs2RunnerBuilder.scala @@ -11,6 +11,8 @@ import org.junit.runners.Suite import org.junit.runners.model.RunnerBuilder import org.specs2.concurrent.ExecutionEnv import org.specs2.control.Action +import org.specs2.data.Trees._ +import org.specs2.fp.TreeLoc import org.specs2.main.{Arguments, CommandLine, Select} import org.specs2.specification.core.{Env, Fragment, SpecStructure} import org.specs2.specification.process.Stats @@ -60,10 +62,15 @@ class FilteredSpecs2ClassRunner(testClass: Class[_], testFilter: Pattern) } private def createFilteredDescription(specStructure: SpecStructure, ee: ExecutionEnv): Description = { - val tree = allFragmentDescriptions(ee) - val desc = Description.createSuiteDescription(testClass) - tree.values.filter(matchingFilter).foreach(desc.addChild) - desc + val descTree = createDescriptionTree(ee).map(_._2) + descTree.toTree.bottomUp { + (description: Description, children: Stream[Description]) => + children.filter(matchingFilter).foreach { + child => description.addChild(child) + } + description + }.rootLabel + } def matchesFilter: Boolean = { @@ -79,10 +86,12 @@ class FilteredSpecs2ClassRunner(testClass: Class[_], testFilter: Pattern) else sanitized } + private def createDescriptionTree(implicit ee: ExecutionEnv): TreeLoc[(Fragment, Description)] = + Try(allDescriptions[specs2_v4].createDescriptionTree(specStructure)(ee)) + .getOrElse(allDescriptions[specs2_v3].createDescriptionTree(specStructure)) + private def allFragmentDescriptions(implicit ee: ExecutionEnv): Map[Fragment, Description] = - Try(allDescriptions[specs2_v4].fragmentDescriptions(specStructure)(ee)) - .orElse(Try(allDescriptions[specs2_v3].fragmentDescriptions(specStructure))) - .getOrElse(Map.empty) + createDescriptionTree(ee).toTree.flattenLeft.toMap /** * Retrieves an original (un-sanitized) text of an example fragment, @@ -129,8 +138,11 @@ class FilteredSpecs2ClassRunner(testClass: Class[_], testFilter: Pattern) } private def matchingFilter(desc: Description): Boolean = { - val testCase = desc.getClassName + "#" + desc.getMethodName - testFilter.toString.r.findAllIn(testCase).nonEmpty + if (desc.isSuite) true + else { + val testCase = desc.getClassName + "#" + Option(desc.getMethodName).mkString + testFilter.toString.r.findFirstIn(testCase).nonEmpty + } } private def specs2Examples(implicit ee: ExecutionEnv): List[String] = diff --git a/src/java/io/bazel/rulesscala/specs2/package.scala b/src/java/io/bazel/rulesscala/specs2/package.scala index af486236b..030842a3b 100644 --- a/src/java/io/bazel/rulesscala/specs2/package.scala +++ b/src/java/io/bazel/rulesscala/specs2/package.scala @@ -2,17 +2,18 @@ package io.bazel.rulesscala import org.junit.runner.Description import org.specs2.concurrent.ExecutionEnv +import org.specs2.fp.TreeLoc import org.specs2.reporter.JUnitDescriptions import org.specs2.specification.core.{Fragment, SpecStructure} package object specs2 { type specs2_v4 = { //noinspection ScalaUnusedSymbol - def fragmentDescriptions(spec: SpecStructure)(ee: ExecutionEnv): Map[Fragment, Description] + def createDescriptionTree(spec: SpecStructure)(ee: ExecutionEnv): TreeLoc[(Fragment, Description)] } type specs2_v3 = { //noinspection ScalaUnusedSymbol - def fragmentDescriptions(spec: SpecStructure): Map[Fragment, Description] + def createDescriptionTree(spec: SpecStructure): TreeLoc[(Fragment, Description)] } def allDescriptions[T]: T = JUnitDescriptions.asInstanceOf[T] diff --git a/test/src/main/scala/scalarules/test/junit/specs2/Specs2Tests.scala b/test/src/main/scala/scalarules/test/junit/specs2/Specs2Tests.scala index 95fa48eba..d7b078f05 100644 --- a/test/src/main/scala/scalarules/test/junit/specs2/Specs2Tests.scala +++ b/test/src/main/scala/scalarules/test/junit/specs2/Specs2Tests.scala @@ -17,14 +17,6 @@ class JunitSpecs2Test extends SpecWithJUnit { } } -class FailingSpecs2Test extends SpecWithJUnit { - - "specs2 tests" should { - "succeed" >> success - "fail" >> failure - } -} - class JunitSpecs2AnotherTest extends SpecWithJUnit { "other specs2 tests" should { diff --git a/test_expect_failure/scala_junit_test/BUILD b/test_expect_failure/scala_junit_test/BUILD index 2f91acad5..e6fb14702 100644 --- a/test_expect_failure/scala_junit_test/BUILD +++ b/test_expect_failure/scala_junit_test/BUILD @@ -23,6 +23,6 @@ scala_junit_test( scala_specs2_junit_test( name = "specs2_failing_test", size = "small", - srcs = ["specs2/FailingTest.scala"], + srcs = ["specs2/FailingTest.scala", "specs2/SuiteWithOneFailingTest.scala"], suffixes = ["Test"], ) diff --git a/test_expect_failure/scala_junit_test/specs2/SuiteWithOneFailingTest.scala b/test_expect_failure/scala_junit_test/specs2/SuiteWithOneFailingTest.scala new file mode 100644 index 000000000..bef083903 --- /dev/null +++ b/test_expect_failure/scala_junit_test/specs2/SuiteWithOneFailingTest.scala @@ -0,0 +1,15 @@ +package scalarules.test.junit.specs2 + +import org.specs2.mutable.SpecWithJUnit + +class SuiteWithOneFailingTest extends SpecWithJUnit { + + "specs2 tests" should { + "succeed" >> success + "fail" >> failure("boom") + } + + "some other suite" should { + "do stuff" >> success + } +} diff --git a/test_rules_scala.sh b/test_rules_scala.sh index b3b35a5f4..a03df7f0c 100755 --- a/test_rules_scala.sh +++ b/test_rules_scala.sh @@ -548,13 +548,13 @@ scala_specs2_only_failed_test_shows_in_the_xml(){ bazel test \ --nocache_test_results \ --test_output=streamed \ - '--test_filter=scalarules.test.junit.specs2.FailingSpecs2Test#specs2 tests should::fail$' \ - test:Specs2Tests - matches=$(grep -c -e "testcase name='specs2 tests should::fail'" -e "testcase name='specs2 tests should::succeed'" ./bazel-testlogs/test/Specs2Tests/test.xml) + '--test_filter=scalarules.test.junit.specs2.SuiteWithOneFailingTest#specs2 tests should::fail$' \ + test_expect_failure/scala_junit_test:specs2_failing_test + matches=$(grep -c -e "testcase name='specs2 tests should::fail'" -e "testcase name='specs2 tests should::succeed'" ./bazel-testlogs/test_expect_failure/scala_junit_test/specs2_failing_test/test.xml) if [ $matches -eq 1 ]; then return 0 else - echo "Expecting only one result, found more than one. Please check 'bazel-testlogs/test/Specs2Tests/test.xml'" + echo "Expecting only one result, found more than one. Please check './bazel-testlogs/test_expect_failure/scala_junit_test/specs2_failing_test/test.xml'" return 1 fi test -e From 29d86633236153c172a883ac1de8f35576b4ad5c Mon Sep 17 00:00:00 2001 From: Igal Tabachnik Date: Wed, 19 Jun 2019 16:12:05 +0300 Subject: [PATCH 3/3] Redirecting test error output --- test_rules_scala.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test_rules_scala.sh b/test_rules_scala.sh index a03df7f0c..b68d92083 100755 --- a/test_rules_scala.sh +++ b/test_rules_scala.sh @@ -545,19 +545,20 @@ scala_specs2_all_tests_show_in_the_xml(){ } scala_specs2_only_failed_test_shows_in_the_xml(){ + set +e bazel test \ - --nocache_test_results \ - --test_output=streamed \ - '--test_filter=scalarules.test.junit.specs2.SuiteWithOneFailingTest#specs2 tests should::fail$' \ - test_expect_failure/scala_junit_test:specs2_failing_test + --nocache_test_results \ + --test_output=streamed \ + '--test_filter=scalarules.test.junit.specs2.SuiteWithOneFailingTest#specs2 tests should::fail$' \ + test_expect_failure/scala_junit_test:specs2_failing_test + echo "got results" matches=$(grep -c -e "testcase name='specs2 tests should::fail'" -e "testcase name='specs2 tests should::succeed'" ./bazel-testlogs/test_expect_failure/scala_junit_test/specs2_failing_test/test.xml) if [ $matches -eq 1 ]; then return 0 else echo "Expecting only one result, found more than one. Please check './bazel-testlogs/test_expect_failure/scala_junit_test/specs2_failing_test/test.xml'" - return 1 + return 1 fi - test -e } scala_specs2_junit_test_test_filter_exact_match(){