Permalink
Browse files

Added Maven profile ``retryFlakyTests`` to reduce spurious test failures

Use the Maven profile ``retryFlakyTests`` for being robust against tests that
are known to be flaky. Inside Eclipse, you can achieve the same by passing the
system property ``-DretryFlakyTests=true``.

This profile should only be used by the Scala/IDE integration script to avoid
failing because of flakiness in our regression suite.
  • Loading branch information...
1 parent cfc7ee7 commit c17e58a76500c5b10d8c9dab610b4e10db951763 @dotta dotta committed Aug 19, 2013
Showing with 15 additions and 4 deletions.
  1. +7 −3 org.scala-ide.sdt.core.tests/src/scala/tools/eclipse/FlakyTest.scala
  2. +8 −1 pom.xml
@@ -3,6 +3,10 @@ package scala.tools.eclipse
import scala.tools.eclipse.logging.HasLogger
object FlakyTest extends HasLogger {
+ private final val RetryFlakyTests = "retryFlakyTests"
+ private lazy val retryFlakyTests =
+ "true".equalsIgnoreCase(System.getProperty(RetryFlakyTests))
+
/** Retry to execute the passed `test` up to N `times`.
*
* @param methodName The test's method name.
@@ -17,8 +21,7 @@ object FlakyTest extends HasLogger {
logger.debug(s"Test run number: ${attempt})")
test
logger.debug(s"Test `%{testName}` was successful!")
- }
- catch {
+ } catch {
case _: AssertionError if attempt < times => loop(attempt + 1)
case e: AssertionError =>
logger.debug(s"Bailing out after ${attempt} attempts. The test is failing consistenly, this may actually be a real regression!?")
@@ -30,6 +33,7 @@ object FlakyTest extends HasLogger {
if(errorMsg.nonEmpty)
logger.debug(s"""When the test fails, it usually reports the following error: \"${errorMsg}\"""")
- loop(1)
+ if(retryFlakyTests) loop(1)
+ else test
}
}
View
@@ -13,6 +13,7 @@
<properties>
<encoding>UTF-8</encoding>
+ <retryFlakyTests>false</retryFlakyTests>
<!-- p2 repositories location -->
<repo.eclipse.indigo>http://download.eclipse.org/releases/indigo/</repo.eclipse.indigo>
<repo.eclipse.juno>http://download.eclipse.org/releases/juno</repo.eclipse.juno>
@@ -45,7 +46,7 @@
<tycho.test.OSspecific></tycho.test.OSspecific>
<!-- Partial workaround against JDT Weaving deadlocks. See #1000317 and the original ticket on https://issuetracker.springsource.com/browse/STS-1445 -->
<tycho.test.weaving>-XX:+UnlockDiagnosticVMOptions -XX:+UnsyncloadClass -Dosgi.classloader.lock=classname</tycho.test.weaving>
- <tycho.test.jvmArgs>-Xmx1024m -XX:MaxPermSize=256m -Dsdtcore.headless -Dsdtcore.notimeouts ${tycho.test.weaving} ${tycho.test.OSspecific}</tycho.test.jvmArgs>
+ <tycho.test.jvmArgs>-Xmx1024m -XX:MaxPermSize=256m -Dsdtcore.headless -Dsdtcore.notimeouts -DretryFlakyTests=${retryFlakyTests} ${tycho.test.weaving} ${tycho.test.OSspecific}</tycho.test.jvmArgs>
<!-- dependencies repos, can be overwritten by profiles -->
<eclipse.codename>Select a profile</eclipse.codename>
@@ -67,6 +68,12 @@
</properties>
<profiles>
+ <profile>
+ <id>retryFlakyTests</id>
+ <properties>
+ <retryFlakyTests>true</retryFlakyTests>
+ </properties>
+ </profile>
<profile>
<id>scala-2.10.x</id>

0 comments on commit c17e58a

Please sign in to comment.