Skip to content
Browse files

SI-7409 Par-Test: A crash is not a DNC for neg tests

A compiler crash does not count as Does Not Compile for
purposes of a negative test. Changing the test kind from
"neg" to "dnc" is out of scope for this PR.

Even if the user asks to update the check file with the
crash result, we must prevent him from doing so. Any
further attempts to update the check file with a crash
will dispatch the Scala SWAT squad which will race to
his location and physically restrain the user. Only
Martin holds the code which will allow the squad to
stand down. So make sure his cell is on next time you
want to --update-check.

A neg test will stop trying to compile after the first
failed round, which is all that matters.

By popular request, a new test outcome is emitted when
the check file is updated. It is called "Updated". It
even has its own short status, the double-plus that is
reminiscent of diff output.
  • Loading branch information...
1 parent 0c6bcc9 commit 6227837f54e07a538f33358da9160c0bc5192525 @som-snytt som-snytt committed with paulp Apr 24, 2013
View
19 src/partest/scala/tools/partest/TestState.scala
@@ -12,13 +12,15 @@ sealed abstract class TestState {
def isOk = false
def isSkipped = false
def testIdent = testFile.testIdent
- def transcriptString = transcript.mkString("\n")
+ def transcriptString = transcript mkString EOL
def identAndReason = testIdent + reasonString
def status = s"$what - $identAndReason"
def longStatus = status + transcriptString
def reasonString = if (reason == "") "" else s" [$reason]"
+ def shortStatus = if (isOk) "ok" else "!!"
+
override def toString = status
}
@@ -27,18 +29,27 @@ object TestState {
def what = "uninitialized"
def reason = what
def transcript = Nil
+ override def shortStatus = "??"
}
case class Pass(testFile: File) extends TestState {
- final override def isOk = true
def what = "pass"
+ override def isOk = true
def transcript: List[String] = Nil
def reason = ""
}
- case class Skip(testFile: File, reason: String) extends TestState {
+ case class Updated(testFile: File) extends TestState {
+ def what = "updated"
override def isOk = true
- final override def isSkipped = true
def transcript: List[String] = Nil
+ def reason = "updated check file"
+ override def shortStatus = "++"
+ }
+ case class Skip(testFile: File, reason: String) extends TestState {
def what = "skip"
+ override def isOk = true
+ override def isSkipped = true
+ def transcript: List[String] = Nil
+ override def shortStatus = "--"
}
case class Fail(testFile: File, reason: String, transcript: List[String]) extends TestState {
def what = "fail"
View
13 src/partest/scala/tools/partest/nest/NestUI.scala
@@ -62,11 +62,14 @@ object NestUI {
def statusLine(state: TestState) = {
import state._
- val word = bold(
- if (isSkipped) yellow("--")
- else if (isOk) green("ok")
- else red("!!")
- )
+ import TestState._
+ val colorizer = state match {
+ case _: Skip => yellow
+ case _: Updated => cyan
+ case s if s.isOk => green
+ case _ => red
+ }
+ val word = bold(colorizer(state.shortStatus))
f"$word $testNumber - $testIdent%-40s$reasonString"
}
View
57 src/partest/scala/tools/partest/nest/Runner.scala
@@ -24,6 +24,7 @@ import scala.tools.scalap.scalax.rules.scalasig.ByteCode
import scala.util.{ Try, Success, Failure }
import ClassPath.{ join, split }
import PartestDefaults.{ javaCmd, javacCmd }
+import TestState.{ Pass, Fail, Crash, Uninitialized, Updated }
trait PartestRunSettings {
def gitPath: Path
@@ -65,7 +66,7 @@ class Runner(val testFile: File, fileManager: FileManager, val testRunParams: Te
private var _lastState: TestState = null
private var _transcript = new TestTranscript
- def lastState = if (_lastState == null) TestState.Uninitialized(testFile) else _lastState
+ def lastState = if (_lastState == null) Uninitialized(testFile) else _lastState
def setLastState(s: TestState) = _lastState = s
def transcript: List[String] = _transcript.fail ++ logFile.fileLines
def pushTranscript(msg: String) = _transcript add msg
@@ -97,10 +98,11 @@ class Runner(val testFile: File, fileManager: FileManager, val testRunParams: Te
genCrash(t)
}
- def genPass() = TestState.Pass(testFile)
- def genFail(reason: String) = TestState.Fail(testFile, reason, _transcript.fail)
- def genTimeout() = TestState.Fail(testFile, "timed out", _transcript.fail)
- def genCrash(caught: Throwable) = TestState.Crash(testFile, caught, _transcript.fail)
+ def genPass() = Pass(testFile)
+ def genFail(reason: String) = Fail(testFile, reason, _transcript.fail)
+ def genTimeout() = Fail(testFile, "timed out", _transcript.fail)
+ def genCrash(caught: Throwable) = Crash(testFile, caught, _transcript.fail)
+ def genUpdated() = Updated(testFile)
def speclib = PathSettings.srcSpecLib.toString // specialization lib
def codelib = PathSettings.srcCodeLib.toString // reify lib
@@ -154,14 +156,18 @@ class Runner(val testFile: File, fileManager: FileManager, val testRunParams: Te
case _ => "% "
}
+ /** Evaluate an action body and update the test state.
+ * @param failFn optionally map a result to a test state.
+ */
def nextTestAction[T](body: => T)(failFn: PartialFunction[T, TestState]): T = {
val result = body
setLastState( if (failFn isDefinedAt result) failFn(result) else genPass() )
result
}
- def nextTestActionExpectTrue[T](reason: String, body: => Boolean): Boolean = {
+ def nextTestActionExpectTrue(reason: String, body: => Boolean): Boolean = (
nextTestAction(body) { case false => genFail(reason) }
- }
+ )
+ def nextTestActionFailing(reason: String): Boolean = nextTestActionExpectTrue(reason, false)
private def assembleTestCommand(outDir: File, logFile: File): List[String] = {
// check whether there is a ".javaopts" file
@@ -324,18 +330,20 @@ class Runner(val testFile: File, fileManager: FileManager, val testRunParams: Te
def diffIsOk: Boolean = {
val diff = currentDiff
- val ok: Boolean = (diff == "") || {
- fileManager.updateCheck && {
+ // if diff is not empty, is update needed?
+ val updating: Option[Boolean] = (
+ if (diff == "") None
+ else Some(fileManager.updateCheck)
+ )
+ pushTranscript(s"diff $logFile $checkFile")
+ nextTestAction(updating) {
+ case Some(true) =>
NestUI.verbose("Updating checkfile " + checkFile)
checkFile writeAll file2String(logFile)
- true
- }
- }
- pushTranscript(s"diff $logFile $checkFile")
- nextTestAction(ok) {
- case false =>
+ genUpdated()
+ case Some(false) =>
// Get a word-highlighted diff from git if we can find it
- val bestDiff = if (ok) "" else {
+ val bestDiff = if (updating.isEmpty) "" else {
if (checkFile.canRead)
gitDiff(logFile, checkFile) getOrElse {
s"diff $logFile $checkFile\n$diff"
@@ -347,7 +355,8 @@ class Runner(val testFile: File, fileManager: FileManager, val testRunParams: Te
// TestState.fail("output differs", "output differs",
// genFail("output differs")
// TestState.Fail("output differs", bestDiff)
- }
+ case None => genPass() // redundant default case
+ } getOrElse true
}
/** 1. Creates log file and output directory.
@@ -437,12 +446,16 @@ class Runner(val testFile: File, fileManager: FileManager, val testRunParams: Te
def runNegTest() = runInContext {
val rounds = compilationRounds(testFile)
- if (rounds forall (x => nextTestActionExpectTrue("compilation failed", x.isOk)))
- nextTestActionExpectTrue("expected compilation failure", false)
- else {
- normalizeLog // put errors in a normal form
- diffIsOk
+ // failing means Does Not Compile
+ val failing = rounds find (x => nextTestActionExpectTrue("compilation failed", x.isOk) == false)
+
+ // which means passing if it checks and didn't crash the compiler
+ def checked(r: CompileRound) = r.result match {
+ case f: Crash => false
+ case f => normalizeLog(); diffIsOk
}
+
+ failing map (checked) getOrElse nextTestActionFailing("expected compilation failure")
}
def runTestCommon(andAlso: => Boolean): (Boolean, LogContext) = runInContext {

0 comments on commit 6227837

Please sign in to comment.
Something went wrong with that request. Please try again.