Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Improve test report parsing

  • Loading branch information...
commit e7e9dc486566f188331846a183124839e0e4ab96 1 parent 0e710ae
@rayh authored
View
5 lib/xcode/builder.rb
@@ -54,10 +54,11 @@ def test
parser << line
end
rescue => e
+ parser.flush
# Let the failure bubble up unless parser has got an error from the output
- raise e unless parser.exit_code!=0
+ raise e unless parser.failed?
end
- exit parser.exit_code if parser.exit_code!=0
+ exit 1 if parser.failed?
@siuying
siuying added a note

it is probably not a good idea to exit VM when a parser fail. Say i run test in a GUI, if the test actually failed, the whole process are aborted.

I think probably we want this function to return parser here, so that we can check for test result, output ... etc.

Or it is also reasonable to return a boolean to indicate test success or fail.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
self
end
View
16 lib/xcode/test/formatters/stdout_formatter.rb
@@ -18,12 +18,20 @@ def after(report)
e.errors.each do |error|
puts " #{error[:message]}"
puts " at #{error[:location]}"
- puts "\n Test Output:"
- puts " > #{error[:data].join(" > ")}\n\n"
- end
+ if error[:data].count>0
+ puts "\n Test Output:"
+ puts " > #{error[:data].join(" > ")}\n\n"
+ end
+ end
+
+ # if there is left over data in the test report, show that
+ if e.data.count>0
+ puts "\n There was this trailing output after the above failures"
+ puts " > #{e.data.join(" > ")}\n\n"
+ end
end
- puts "End tests (exit code #{report.exit_code})"
+ puts "End tests (#{report.failed? ? 'FAILED' : 'PASSED'})"
end
def before_suite(suite)
View
95 lib/xcode/test/ocunit_report_parser.rb
@@ -11,7 +11,7 @@ module Formatters
class OCUnitReportParser
- attr_reader :exit_code, :reports
+ attr_reader :reports
attr_accessor :debug, :formatters
def initialize
@@ -19,21 +19,46 @@ def initialize
@exit_code = 0
@reports = []
@formatters = []
+ @failed = false
+ @finished = false
add_formatter :junit, 'test-reports'
add_formatter :stdout
end
+ def failed?
+ @failed
+ end
+
+ def finished?
+ @finished
+ end
+
def add_formatter(format, *args)
require "xcode/test/formatters/#{format.to_s}_formatter"
formatter = Formatters.const_get("#{format.to_s.capitalize}Formatter").new(*args)
@formatters << formatter
end
- def notify_formatters(event, obj=nil)
- @formatters.each do |f|
- f.send event, obj if f.respond_to? event
+ def flush
+ return if finished?
+
+ # if there is a current, unfinished test - fail it
+ unless current_test.nil? or current_test.passed?
+ fail_current_test(0)
+ @failed = true
end
+
+ # if there is a current suite which isnt finished - finish it
+ unless current_suite.nil? or current_suite.finished?
+ @failed = true # It may not have failed, but we want to indicate an unexpected end
+ current_suite.finish
+ notify_formatters(:after_suite, current_suite)
+ end
+
+ # finish all tests
+ @finished = true
+ notify_formatters(:after, self)
end
def <<(piped_row)
@@ -49,7 +74,7 @@ def <<(piped_row)
notify_formatters(:before, self)
else
@reports << SuiteResult.new(name, time)
- notify_formatters(:before_suite, @reports.last)
+ notify_formatters(:before_suite, current_suite)
end
when /Test Suite '(\S+)'.*finished at\s+(.*)./
@@ -57,47 +82,77 @@ def <<(piped_row)
name = $1
if name=~/\//
# all tests ended
+ @finished = true
notify_formatters(:after, self)
else
@reports.last.finish(time)
- notify_formatters(:after_suite, @reports.last)
+ notify_formatters(:after_suite, current_suite)
end
when /Test Case '-\[\S+\s+(\S+)\]' started./
- test = TestResult.new($1, @reports.last)
+ test = TestResult.new($1, current_suite)
@reports.last.tests << test
notify_formatters(:before_test, test)
when /Test Case '-\[\S+\s+(\S+)\]' passed \((.*) seconds\)/
@reports.last.tests.last.passed($2.to_f)
- notify_formatters(:after_test, @reports.last.tests.last)
+ notify_formatters(:after_test, current_test)
when /(.*): error: -\[(\S+) (\S+)\] : (.*)/
- @reports.last.tests.last.add_error($4,$1)
- @exit_code = 1 # should terminate
+ current_test.add_error($4,$1)
+ @failed = true
# notify_formatters(:after_test, @reports.last.tests.last)
when /Test Case '-\[\S+ (\S+)\]' failed \((\S+) seconds\)/
- @reports.last.tests.last.failed($2.to_f)
- @exit_code = 1 # should terminate
- notify_formatters(:after_test, @reports.last.tests.last)
-
- when /failed with exit code (\d+)/
- @exit_code = $1.to_i
-
- when /BUILD FAILED/
- @exit_code = -1;
+ fail_current_test($2.to_f)
+ @failed = true
+
+ # when /failed with exit code (\d+)/,
+ when /BUILD FAILED/
+ flush
+ when /Run test case (\w+)/
+ # ignore
when /Run test suite (\w+)/
# ignore
when /Executed (\d+) test, with (\d+) failures \((\d+) unexpected\) in (\S+) \((\S+)\) seconds/
# ignore
else
- @reports.last.tests.last.data << piped_row unless @reports.last.nil? or @reports.last.tests.last.nil?
+ append_line_to_current_test piped_row
end # case
end # <<
+ private
+
+ def notify_formatters(event, obj=nil)
+ @formatters.each do |f|
+ f.send event, obj if f.respond_to? event
+ end
+ end
+
+ def current_suite
+ @reports.last
+ end
+
+ def current_test
+ @reports.last.tests.last
+ end
+
+ def fail_current_test(duration=0)
+ return if current_test.nil?
+
+ current_test.failed(duration)
+ notify_formatters(:after_test, current_test)
+ end
+
+ def append_line_to_current_test(line)
+ return if current_suite.nil? or !current_suite.end_time.nil?
+ return if current_test.nil?
+ current_test << line
+ end
+
+
end # OCUnitReportParser
end # Test
end # Xcode
View
6 lib/xcode/test/suite_result.rb
@@ -9,11 +9,15 @@ def initialize(name, start_time)
@tests = []
end
- def finish(time)
+ def finish(time=Time.now)
raise "Time is nil" if time.nil?
@end_time = time
end
+ def finished?
+ !@end_time.nil?
+ end
+
def total_errors
errors = 0
@tests.each do |t|
View
6 lib/xcode/test/test_result.rb
@@ -27,6 +27,12 @@ def failed(time)
@passed = false
@time = time
end
+
+ def << (line)
+ # puts "[#{@suite.name} #{@name}] << #{line}"
+ return if @data.count==0 and line.strip.empty?
+ @data << line
+ end
def add_error(error_message,error_location)
@errors << {:message => error_message, :location => error_location, :data => @data}
View
30 spec/test_report_spec.rb
@@ -75,14 +75,14 @@ def example_failing_report
t.reports.first.end_time.should==Time.parse("2012-02-10 00:37:04 +0000")
end
- it "should set the exist status to 0" do
+ it "should detect a passing report" do
t = example_report
- t.exit_code.should==0
+ t.failed?.should==false
end
- it "should set the exit status to non 0" do
+ it "should detect a failing report" do
t = example_failing_report
- t.exit_code.should_not==0
+ t.failed?.should==true
end
it "should record a failure" do
@@ -103,6 +103,28 @@ def example_failing_report
t.reports.first.tests[1].time.should==0.003
t.reports.first.tests[1].passed?.should==true
end
+
+ it "should capture failed build" do
+ t = Xcode::Test::OCUnitReportParser.new
+ t << "Run test suite AnExampleTestSuite"
+ t << "Test Suite 'AnExampleTestSuite' started at 2012-02-10 00:37:04 +0000"
+ t << "Run test case anExampleTest1"
+ t << "Test Case '-[AnExampleTestSuite anExampleTest1]' started."
+ t << "/Path/To/Project/Tests/YPKeywordSuggestHandlerTest.m:45: error: -[AnExampleTestSuite anExampleTest1] : 'An example test spec' [FAILED], mock received unexpected message -setSuspended: 1 "
+ t << "/Developer/Tools/RunPlatformUnitTests.include: line 415: 32225 Bus error: 10 \"${THIN_TEST_RIG}\" \"${OTHER_TEST_FLAGS}\" \"${TEST_BUNDLE_PATH}\""
+ t << "/Developer/Tools/RunPlatformUnitTests.include:451: error: Test rig '/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator5.0.sdk/Developer/usr/bin/otest' exited abnormally with code 138 (it may have crashed)."
+ t << "** BUILD FAILED **"
+ t << ""
+ t << "The following build commands failed:"
+ t << " PhaseScriptExecution \"Run Script\" build/Project.build/Debug-iphonesimulator/Tests.build/Script-750943CE1474BB7200ECF882.sh"
+ t << "(1 failure)"
+
+ failure = t.reports.first.tests[0]
+ failure.passed?.should==false
+ failure.data.count.should==2
+ failure.data[0].should=~/32225 Bus error: 10/
+ failure.data[1].should=~/Test rig/
+ end
context "Junit output" do

1 comment on commit e7e9dc4

@siuying

it is probably not a good idea to exit VM when a parser fail. Say i run test in a GUI, if the test actually failed, the whole process are aborted.

I think probably we want this function to return parser here, so that we can check for test result, output ... etc.

Or it is also reasonable to return a boolean to indicate test success or fail.

@rayh
Owner

I've resolved this in a recent series of commits, please check to see if this makes sense

Please sign in to comment.
Something went wrong with that request. Please try again.