Improved multiprocess test harness so that if the scxml process crash…

…es, errors will be reported and testing will continue.

src/main/coffeescript/test-harness/multi-process/client.coffee

@@ -71,7 +71,11 @@ define ["util/BufferedStream","util/set/ArraySet","util/utils",'util/memory','sc
 			#start up a new statechart process
 			currentScxmlProcess = child_process.spawn "bash",["#{projectDir}/src/test-scripts/run-module.sh",SCXML_MODULE,currentTest.interpreter]
 
-			unexpectedExitListener = -> console.error "statechart process ended unexpectedly"
+			unexpectedExitListener = ->
+				msg = "statechart process ended unexpectedly"
+				finishTest "error",msg
+				console.error msg
+
 			currentScxmlProcess.on "exit",unexpectedExitListener
 
 			scxmlWL = utils.wrapLine currentScxmlProcess.stdin.write,currentScxmlProcess.stdin
@@ -117,30 +121,44 @@ define ["util/BufferedStream","util/set/ArraySet","util/utils",'util/memory','sc
 				#set timeout to do it again
 				setTimeout sendRandomEvents,eventDensity
 
-		finishTest = (pass,msg) ->
-			ns = if pass then nextStep else failBack
+		#FIXME: we may want to add better, more explicit error handling (e.g. using errback)
+		finishTest = (returnStatus,msg) ->
+
+			ns = switch returnStatus
+				when "pass" then nextStep
+				when "fail" then failBack
+				when "error" then errBack
 
 			#prevent sending further events
-			if not pass then eventsToSend = []
+			if returnStatus isnt "pass" then eventsToSend = []
 
-			#check memory usage
-			memory.push memoryUtil.getMemory currentScxmlProcess.pid
-			finishTime = new Date()
+			if returnStatus is "error"
+				#just call the next step
+				ns()
 
-			pushAggregateStats pass,msg
+				finishTime = new Date()
+				pushAggregateStats returnStatus,msg
+			else
 
-			#remove the unexpected exit listener
-			currentScxmlProcess.removeListener 'exit',unexpectedExitListener
+				#check memory usage
+				memory.push memoryUtil.getMemory currentScxmlProcess.pid
 
-			#we're done, post results and send signal to fetch next test
-			currentScxmlProcess.on 'exit',->
-				#remove all other event listeners
-				currentScxmlProcess.removeAllListeners()
-				#call next step or failBack
-				ns()
+				finishTime = new Date()
+				pushAggregateStats returnStatus,msg
+
+				#otherwise, child process should still be alive
+				#remove the unexpected exit listener
+				currentScxmlProcess.removeListener 'exit',unexpectedExitListener
 
-			#close the pipe, which will terminate the process
-			currentScxmlProcess.stdin.end()
+				#we're done, post results and send signal to fetch next test
+				currentScxmlProcess.on 'exit',->
+					#remove all other event listeners
+					currentScxmlProcess.removeAllListeners()
+					#call next step or failBack
+					ns()
+
+				#close the pipe, which will terminate the process
+				currentScxmlProcess.stdin.end()
 
 		processClientMessage = (jsonMessage) ->
 			switch jsonMessage.method
@@ -174,7 +192,7 @@ define ["util/BufferedStream","util/set/ArraySet","util/utils",'util/memory','sc
 						#statechart has executed all events, so wrap up
 						#+1 because we will receive initial configuration as well
 						if receivedConfigurationCounter == (numberOfEventsToSendInPerformanceTestMode+1)
-							finishTest true
+							finishTest "pass"
 					else
 						expectedConfiguration = expectedConfigurations.shift()
 
@@ -185,14 +203,14 @@ define ["util/BufferedStream","util/set/ArraySet","util/utils",'util/memory','sc
 							console.error "Matched expected configuration."
 
 							#if we're out of tests, then we're done and we report that we succeeded
-							if not expectedConfigurations.length then finishTest true
+							if not expectedConfigurations.length then finishTest "pass"
 
 
 						else
 							#test has failed
 							msg = "Did not match expected configuration. Received: #{JSON.stringify(configuration)}. Expected:#{JSON.stringify(expectedConfiguration)}."
 
-							finishTest false,msg
+							finishTest "fail",msg
 
 
 				when "set-timeout"
@@ -214,7 +232,7 @@ define ["util/BufferedStream","util/set/ArraySet","util/utils",'util/memory','sc
 				eventSet = (k for own k of currentTest.scxmlJson.events)
 				if not eventSet.length
 					console.error "No need to run performance test on statechart that does not have transitions other than default transitions. Posting results as trivial success."
-					postTestResults currentTest.id,true
+					postTestResults currentTest.id,"pass"
 					return
 
 			#TODO: accumulate aggregate results in some data structure so we can post them here
@@ -228,9 +246,14 @@ define ["util/BufferedStream","util/set/ArraySet","util/utils",'util/memory','sc
 				),
 				(->
 					console.error "Done. Posting results."
-					postTestResults currentTest.id,true),
-				(->),
-				(-> postTestResults currentTest.id,false)
+					postTestResults currentTest.id,"pass"),
+				(->
+					console.error "Done (test errored). Posting results."
+					postTestResults currentTest.id,"error"),
+				(->
+
+					console.error "Done (test failed). Posting results."
+					postTestResults currentTest.id,"fail")
 			)
 
 		process.stdin.resume()

src/main/coffeescript/test-harness/multi-process/server.coffee

@@ -163,13 +163,16 @@ define ['test-harness/multi-process/json-tests','util/BufferedStream',"test-harn
 
 			testMap[jsonResults.testId].stats = jsonResults.stats
 
-			if jsonResults.pass
+			if jsonResults.pass is "pass"
 				results.testsPassed.push testMap[jsonResults.testId].test
 			else
-				results.testsFailed.push testMap[jsonResults.testId].test
+				if jsonResults.pass is "fail"
+					results.testsFailed.push testMap[jsonResults.testId].test
+				else
+					results.testsErrored.push testMap[jsonResults.testId].test
 
 				#error message will be in there somewhere. TODO: maybe filter it?
-				console.log jsonResults.stats
+				console.log "Test errored or failed:", jsonResults
 
 				#if stopOnFail is set, then wrap up
 				if stopOnFail

src/test-scripts/run-multi-process-testing-framework.sh

@@ -9,7 +9,7 @@ basedir=`dirname $t`
 #spidermonkey rhino jsc v8 
 interpreters=${@-spidermonkey}
 
-$basedir/src/test-scripts/run-module.sh test-harness/multi-process/server node -projectDir $basedir -local -numLocalProcesses 8 -interpreters $interpreters -verbose -logFile out.txt
+$basedir/src/test-scripts/run-module.sh test-harness/multi-process/server node -performanceTestMode -projectDir $basedir -local -numLocalProcesses 8 -interpreters $interpreters -verbose -logFile out.txt
 
 #./bin/run-multi-process-testing-framework.sh -projectDir `pwd` -clientAddresses node-0{1..9} node-{10..19} node-{21..33} -interpreters v8 spidermonkey rhino -verbose -logFile out.txt -stopOnFail -clientModulePath `pwd`/src/main/coffeescript/test-harness/multi-process/client.coffee