diff --git a/.gitignore b/.gitignore index 58c941992..7e4d1ab27 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,37 @@ lastlog-1.* *.test_info.*.json .testsharedjobslots.yml xxx + +!Build/ +.last_cover_stats +/MYMETA.* +*.o +*.pm.tdy +*.bs +*.ERR +*.bak + +# Devel::NYTProf +nytprof.out + +# Dizt::Zilla +/.build/ + +# Module::Build +_build/ +Build +Build.bat + +# Module::Install +inc/ + +# ExtUtils::MakeMaker +/blib/ +/_eumm/ +/*.gz +/Makefile +/Makefile.old +/MANIFEST.bak +/pm_to_blib +/*.zip +/Test2-Harness-Renderer-JUnit-* diff --git a/Makefile.PL b/Makefile.PL index 00fe2aca8..ed75145e0 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -61,11 +61,14 @@ my %WriteMakefileArgs = ( "IPC::Open3" => 0, "Import::Into" => 0, "Importer" => "0.025", + "JSON::PP" => 0, "Linux::Inotify2" => "2.3", "List::Util" => "1.56", + "Long::Jump" => "0.000001", "POSIX" => 0, "Scalar::Util" => 0, "Scope::Guard" => 0, + "Storable" => 0, "Symbol" => 0, "Sys::Hostname" => 0, "Term::Table" => "0.015", @@ -91,6 +94,7 @@ my %WriteMakefileArgs = ( "Test::More" => "1.302198", "Text::ParseWords" => 0, "Time::HiRes" => 0, + "XML::Generator" => 0, "YAML::Tiny" => 0, "base" => 0, "constant" => 0, @@ -99,11 +103,17 @@ my %WriteMakefileArgs = ( }, "TEST_REQUIRES" => { "Child" => 0, - "File::Copy" => 0 + "Data::Dumper" => 0, + "File::Copy" => 0, + "File::Temp" => 0, + "Test2::Plugin::NoWarnings" => 0, + "Test2::Tools::Explain" => 0, + "Test::More" => "1.302198", + "XML::Simple" => 0 }, "VERSION" => "2.000000", "test" => { - "TESTS" => "t/*.t t/acceptence/*.t t/integration/*.t t/integration/signals/*.t t/unit/App/*.t t/unit/App/Yath/*.t t/unit/App/Yath/Command/*.t t/unit/App/Yath/Options/*.t t/unit/App/Yath/Plugin/*.t t/unit/App/Yath/Renderer/*.t t/unit/App/Yath/Renderer/Default/*.t t/unit/App/Yath/Resource/*.t t/unit/App/Yath/Resource/SharedJobSlots/*.t t/unit/Getopt/*.t t/unit/Getopt/Yath/*.t t/unit/Getopt/Yath/Option/*.t t/unit/Getopt/Yath/Settings/*.t t/unit/Test2/*.t t/unit/Test2/Formatter/*.t t/unit/Test2/Harness/*.t t/unit/Test2/Harness/Auditor/*.t t/unit/Test2/Harness/Collector/*.t t/unit/Test2/Harness/Collector/Auditor/*.t t/unit/Test2/Harness/Collector/IOParser/*.t t/unit/Test2/Harness/IPC/*.t t/unit/Test2/Harness/IPC/Protocol/*.t t/unit/Test2/Harness/IPC/Protocol/AtomicPipe/*.t t/unit/Test2/Harness/Instance/*.t t/unit/Test2/Harness/Log/*.t t/unit/Test2/Harness/Log/CoverageAggregator/*.t t/unit/Test2/Harness/Preload/*.t t/unit/Test2/Harness/Reloader/*.t t/unit/Test2/Harness/Renderer/*.t t/unit/Test2/Harness/Resource/*.t t/unit/Test2/Harness/Run/*.t t/unit/Test2/Harness/Runner/*.t t/unit/Test2/Harness/Runner/Preload/*.t t/unit/Test2/Harness/Runner/Preloader/*.t t/unit/Test2/Harness/Runner/Preloading/*.t t/unit/Test2/Harness/Runner/Resource/*.t t/unit/Test2/Harness/Runner/Resource/SharedJobSlots/*.t t/unit/Test2/Harness/Scheduler/*.t t/unit/Test2/Harness/Settings/*.t t/unit/Test2/Harness/Util/*.t t/unit/Test2/Harness/Util/File/*.t t/unit/Test2/Tools/*.t" + "TESTS" => "t/*.t t/JUnit/*.t t/acceptence/*.t t/integration/*.t t/integration/signals/*.t t/unit/App/*.t t/unit/App/Yath/*.t t/unit/App/Yath/Command/*.t t/unit/App/Yath/Options/*.t t/unit/App/Yath/Plugin/*.t t/unit/App/Yath/Renderer/*.t t/unit/App/Yath/Renderer/Default/*.t t/unit/App/Yath/Resource/*.t t/unit/App/Yath/Resource/SharedJobSlots/*.t t/unit/Getopt/*.t t/unit/Getopt/Yath/*.t t/unit/Getopt/Yath/Option/*.t t/unit/Getopt/Yath/Settings/*.t t/unit/Test2/*.t t/unit/Test2/Formatter/*.t t/unit/Test2/Harness/*.t t/unit/Test2/Harness/Auditor/*.t t/unit/Test2/Harness/Collector/*.t t/unit/Test2/Harness/Collector/Auditor/*.t t/unit/Test2/Harness/Collector/IOParser/*.t t/unit/Test2/Harness/IPC/*.t t/unit/Test2/Harness/IPC/Protocol/*.t t/unit/Test2/Harness/IPC/Protocol/AtomicPipe/*.t t/unit/Test2/Harness/Instance/*.t t/unit/Test2/Harness/Log/*.t t/unit/Test2/Harness/Log/CoverageAggregator/*.t t/unit/Test2/Harness/Preload/*.t t/unit/Test2/Harness/Reloader/*.t t/unit/Test2/Harness/Renderer/*.t t/unit/Test2/Harness/Resource/*.t t/unit/Test2/Harness/Run/*.t t/unit/Test2/Harness/Runner/*.t t/unit/Test2/Harness/Runner/Preload/*.t t/unit/Test2/Harness/Runner/Preloader/*.t t/unit/Test2/Harness/Runner/Preloading/*.t t/unit/Test2/Harness/Runner/Resource/*.t t/unit/Test2/Harness/Runner/Resource/SharedJobSlots/*.t t/unit/Test2/Harness/Scheduler/*.t t/unit/Test2/Harness/Settings/*.t t/unit/Test2/Harness/Util/*.t t/unit/Test2/Harness/Util/File/*.t t/unit/Test2/Tools/*.t" } ); @@ -135,11 +145,14 @@ my %FallbackPrereqs = ( "IPC::Open3" => 0, "Import::Into" => 0, "Importer" => "0.025", + "JSON::PP" => 0, "Linux::Inotify2" => "2.3", "List::Util" => "1.56", + "Long::Jump" => "0.000001", "POSIX" => 0, "Scalar::Util" => 0, "Scope::Guard" => 0, + "Storable" => 0, "Symbol" => 0, "Sys::Hostname" => 0, "Term::Table" => "0.015", @@ -150,10 +163,12 @@ my %FallbackPrereqs = ( "Test2::Event::V2" => "1.302198", "Test2::Formatter" => "1.302198", "Test2::Plugin::MemUsage" => "0.002003", + "Test2::Plugin::NoWarnings" => 0, "Test2::Plugin::UUID" => "0.002001", "Test2::Tools::AsyncSubtest" => "0.000159", "Test2::Tools::Basic" => 0, "Test2::Tools::Compare" => 0, + "Test2::Tools::Explain" => 0, "Test2::Tools::Subtest" => "0.000159", "Test2::Util" => "1.302198", "Test2::Util::Table" => 0, @@ -165,6 +180,8 @@ my %FallbackPrereqs = ( "Test::More" => "1.302198", "Text::ParseWords" => 0, "Time::HiRes" => 0, + "XML::Generator" => 0, + "XML::Simple" => 0, "YAML::Tiny" => 0, "base" => 0, "constant" => 0, diff --git a/cpanfile b/cpanfile index 045391f14..501b5bdea 100644 --- a/cpanfile +++ b/cpanfile @@ -26,11 +26,14 @@ requires "IPC::Cmd" => "0"; requires "IPC::Open3" => "0"; requires "Import::Into" => "0"; requires "Importer" => "0.025"; +requires "JSON::PP" => "0"; requires "Linux::Inotify2" => "2.3"; requires "List::Util" => "1.56"; +requires "Long::Jump" => "0.000001"; requires "POSIX" => "0"; requires "Scalar::Util" => "0"; requires "Scope::Guard" => "0"; +requires "Storable" => "0"; requires "Symbol" => "0"; requires "Sys::Hostname" => "0"; requires "Term::Table" => "0.015"; @@ -56,6 +59,7 @@ requires "Test::Builder::Formatter" => "1.302198"; requires "Test::More" => "1.302198"; requires "Text::ParseWords" => "0"; requires "Time::HiRes" => "0"; +requires "XML::Generator" => "0"; requires "YAML::Tiny" => "0"; requires "base" => "0"; requires "constant" => "0"; @@ -76,7 +80,13 @@ suggests "Win32::Console::ANSI" => "0"; on 'test' => sub { requires "Child" => "0"; + requires "Data::Dumper" => "0"; requires "File::Copy" => "0"; + requires "File::Temp" => "0"; + requires "Test2::Plugin::NoWarnings" => "0"; + requires "Test2::Tools::Explain" => "0"; + requires "Test::More" => "1.302198"; + requires "XML::Simple" => "0"; }; on 'configure' => sub { @@ -85,7 +95,9 @@ on 'configure' => sub { on 'develop' => sub { requires "Test2::Require::Module" => "0.000127"; + requires "Test::CheckManifest" => "0"; requires "Test::Perl::Critic" => "0"; requires "Test::Pod" => "1.41"; + requires "Test::Pod::Coverage" => "0"; requires "Test::Spelling" => "0.12"; }; diff --git a/dist.ini b/dist.ini index 8d6297bde..579b511c4 100644 --- a/dist.ini +++ b/dist.ini @@ -86,11 +86,14 @@ IPC::Cmd = 0 IPC::Open3 = 0 Import::Into = 0 Importer = 0.025 +JSON::PP = 0 Linux::Inotify2 = 2.3 List::Util = 1.56 +Long::Jump = 0.000001 POSIX = 0 Scalar::Util = 0 Scope::Guard = 0 +Storable = 0 Symbol = 0 Sys::Hostname = 0 Term::Table = 0.015 @@ -116,16 +119,29 @@ Test::Builder::Formatter = 1.302198 Test::More = 1.302198 Text::ParseWords = 0 Time::HiRes = 0 +XML::Generator = 0 YAML::Tiny = 0 [Prereqs / TestRequires] -File::Copy = 0 -Child = 0 +Child = 0 +Data::Dumper = 0 +File::Copy = 0 +File::Temp = 0 +Test2::Plugin::NoWarnings = 0 +Test2::Tools::Explain = 0 +Test::More = 0 +XML::Simple = 0 + +[Prereqs / ConfigureRequires] +ExtUtils::MakeMaker = 0 [Prereqs / DevelopRequires] -Test::Spelling = 0.12 ; for xt/author/pod-spell.t -Test::Perl::Critic = 0 Test2::Require::Module = 0.000127 +Test::CheckManifest = 0 +Test::Perl::Critic = 0 +Test::Pod = 1.41 +Test::Pod::Coverage = 0 +Test::Spelling = 0.12 ; for xt/author/pod-spell.t [Prereqs / RuntimeSuggests] Class::XSAccessor = 1.19 diff --git a/lib/App/Yath/Renderer/JUnit.pm b/lib/App/Yath/Renderer/JUnit.pm new file mode 100644 index 000000000..f050f9ebb --- /dev/null +++ b/lib/App/Yath/Renderer/JUnit.pm @@ -0,0 +1,479 @@ +package App::Yath::Renderer::JUnit; + +use 5.010000; +use strict; +use warnings; + +our $VERSION = '2.000000'; + +# This is used frequently during development to determine what different events look like so we can determine how to capture test data. +use Data::Dumper; +$Data::Dumper::Sortkeys = 1; + +use File::Spec; +use POSIX (); +use Storable qw/dclone/; +use XML::Generator (); +use Carp (); + +BEGIN { require App::Yath::Renderer; our @ISA = ('App::Yath::Renderer') } +use Test2::Harness::Util::HashBase qw{ + -io -io_err + -formatter + -show_run_info + -show_job_info + -show_job_launch + -show_job_end + -times +}; + +sub init { + my $self = shift; + + my $settings = $self->{ +SETTINGS }; + + $self->{'xml'} = XML::Generator->new( ':pretty', ':std', 'escape' => 'always,high-bit,even-entities', 'encoding' => 'UTF-8' ); + + $self->{'xml_content'} = []; + + $self->{'allow_passing_todos'} = $ENV{'ALLOW_PASSING_TODOS'} ? 1 : 0; + $self->{'junit_file'} //= $ENV{'JUNIT_TEST_FILE'} || 'junit.xml'; + + $self->{'tests'} = {}; # We need a pointer to each test so we know where to go for each event. +} + +# This sub is called for every Harness event. We capture the data we need so we can emit the appropriate junit file. + +sub render_event { + my $self = shift; + my ($event) = @_; + + # We modify the event, which would be bad if there were multiple renderers, + # so we deep clone it. + $event = dclone($event); + my $f = $event->{facet_data}; + my $job = $f->{harness_job}; + my $job_id = $f->{'harness'}->{'job_id'} or return; + my $job_try = $f->{'harness'}->{'job_try'} // 0; + my $stamp = $event->{'stamp'}; + + if ( !defined $stamp ) { + $f //= 'unknown facet_data'; + die "No time stamp found for event '$f' ?!?!?!? ...\n" . "Event:\n" . Dumper($event) . "\n" . Carp::longmess(); + } + + # Throw out job events if they are for a previous run and we've already started collecting job + # information for a successive run. + return if $self->{'tests'}->{$job_id} && $job_try < ( $self->{'tests'}->{$job_id}->{'job_try'} // 0 ); + + # At job launch we need to start collecting a new junit testdata section. + # We throw out anything we've collected to date on a previous run. + if ( $f->{'harness_job_launch'} ) { + my $full_test_name = $job->{'file'}; + my $test_file = File::Spec->abs2rel($full_test_name); + + $self->{'tests'}->{$job_id} = { + 'name' => $job->{'file'}, + 'file' => _squeaky_clean($test_file), + 'job_id' => $job_id, + 'job_try' => $job_try, + 'job_name' => $f->{'harness_job'}->{'job_name'}, + 'testcase' => [], + 'system-out' => '', + 'system-err' => '', + 'start' => $stamp, + 'last_job_start' => $stamp, + 'testsuite' => { + 'errors' => 0, + 'failures' => 0, + 'tests' => 0, + 'name' => _get_testsuite_name($test_file), + 'id' => $job_id, # add a UID in the XML output + }, + }; + + return; + } + + my $test = $self->{'tests'}->{$job_id}; + + # We have all the data. Print the XML. + if ( $f->{'harness_job_end'} ) { + $self->close_open_failure_testcase( $test, -1 ); + $test->{'stop'} = $event->{'stamp'}; + $test->{'testsuite'}->{'time'} = $test->{'stop'} - $test->{'start'}; + $test->{'testsuite'}->{'timestamp'} = _timestamp( $test->{'start'} ); + + if ( $f->{'errors'} ) { + my $test_error_messages = ''; + my $alternative_error = ''; + foreach my $msg ( @{ $f->{'errors'} } ) { + next unless $msg->{'from_harness'}; + next unless $msg->{'tag'} // '' eq 'REASON'; + + my $details = $msg->{details}; + if ( $details =~ m/^Planned for ([0-9]+) assertions?, but saw ([0-9]+)/ ) { + $test->{'testsuite'}->{'errors'} += abs( $1 - $2 ); + } + if ( $details =~ m/Test script returned error|Assertion failures were encountered|Subtest failures were encountered/ ) { + $alternative_error .= "$details\n"; + } + else { + $test_error_messages .= "$details\n"; + } + } + + if ($test_error_messages) { + push @{ $test->{'testcase'} }, $self->xml->testcase( + { 'name' => "Test Plan Failure", 'time' => $stamp - $test->{'last_job_start'}, 'classname' => $test->{'testsuite'}->{'name'} }, + $self->xml->failure($test_error_messages) + ); + } + + # We only want to show this alternative error if all of the tests passed but the program still exited non-zero. + elsif ( !$test->{'testsuite'}->{'errors'} && $alternative_error ) { + $test->{'testsuite'}->{'errors'}++; + push @{ $test->{'testcase'} }, $self->xml->testcase( + { 'name' => "Program Ended Unexpectedly", 'time' => $stamp - $test->{'last_job_start'}, 'classname' => $test->{'testsuite'}->{'name'} }, + $self->xml->failure($alternative_error) + ); + } + } + + push @{ $test->{'testcase'} }, $self->xml->testcase( + { 'name' => "Tear down.", 'time' => $stamp - $test->{'last_job_start'}, 'classname' => $test->{'testsuite'}->{'name'} }, + ); + + return; + } + + if ( $f->{'plan'} ) { + if ( $f->{'plan'}->{'skip'} ) { + my $skip = $f->{'plan'}->{'details'}; + $test->{'system-out'} .= "# SKIP $skip\n"; + } + if ( $f->{'plan'}->{'count'} ) { + $test->{'plan'} = $f->{'plan'}->{'count'}; + } + + return; + } + + if ( $f->{'harness_job_exit'} ) { + return unless $f->{'harness_job_exit'}->{'exit'}; + + # If we don't see + $test->{'testsuite'}->{'errors'}++; + $test->{'error-msg'} //= $f->{'harness_job_exit'}->{'details'} . "\n"; + + return; + } + + # We just hit an ok/not ok line. + if ( $f->{'assert'} ) { + + # Ignore subtests + return if ( $f->{'hubs'} && $f->{'hubs'}->[0]->{'nested'} ); + + my $test_num = $event->{'assert_count'} || $f->{'assert'}->{'number'}; + $test_num = sprintf "%04d", $test_num if defined $test_num; + my $test_name = _squeaky_clean( $f->{'assert'}->{'details'} // 'UNKNOWN_TEST?' ); + $test_name = join " - ", grep { defined } $test_num, $test_name; + $test->{'testsuite'}->{'tests'}++; + + $self->close_open_failure_testcase( $test, $test_num ); + + warn Dumper $event unless $stamp; + + my $run_time = $stamp - $test->{'last_job_start'}; + $test->{'last_job_start'} = $stamp; + + if ( $f->{'amnesty'} && grep { ( $_->{'tag'} // '' ) eq 'TODO' } @{ $f->{'amnesty'} } ) { # All TODO Tests + if ( !$f->{'assert'}->{'pass'} ) { # Failing TODO + push @{ $test->{'testcase'} }, $self->xml->testcase( { 'name' => "$test_name (TODO)", 'time' => $run_time, 'classname' => $test->{'testsuite'}->{'name'} }, "" ); + } + elsif ( $self->{'allow_passing_todos'} ) { # junit parsers don't like passing TODO tests. Let's just not tell them about it if $ENV{ALLOW_PASSING_TODOS} is set. + push @{ $test->{'testcase'} }, $self->xml->testcase( { 'name' => "$test_name (PASSING TODO)", 'time' => $run_time, 'classname' => $test->{'testsuite'}->{'name'} }, "" ); + } + else { # Passing TODO (Failure) when not allowed. + + $test->{'testsuite'}->{'failures'}++; + $test->{'testsuite'}->{'errors'}++; + + # Grab the first amnesty description that's a TODO message. + my ($todo_message) = map { $_->{'details'} } grep { $_->{'tag'} // '' eq 'TODO' } @{ $f->{'amnesty'} }; + + push @{ $test->{'testcase'} }, $self->xml->testcase( + { 'name' => "$test_name (TODO)", 'time' => $run_time, 'classname' => $test->{'testsuite'}->{'name'} }, + $self->xml->error( + { 'message' => $todo_message, 'type' => "TodoTestSucceeded" }, + $self->_cdata("ok $test_name") + ) + ); + + } + } + elsif ( $f->{'assert'}->{'pass'} ) { # Passing test + push @{ $test->{'testcase'} }, $self->xml->testcase( + { 'name' => $test_name, 'time' => $run_time, 'classname' => $test->{'testsuite'}->{'name'} }, + "" + ); + } + else { # Failing Test. + $test->{'testsuite'}->{'failures'}++; + $test->{'testsuite'}->{'errors'}++; + + my $message = "not ok" . ( $test_name ? " $test_name" : "" ); + + # Trap the test information. We can't generate the XML for this test until we get all the diag information. + $test->{'last_failure'} = { + 'test_num' => $test_num, + 'test_name' => $test_name, + 'time' => $run_time, + 'message' => $message, + 'full_message' => "$message\n", + }; + } + + return; + } + + # This is diag information. Append it to the last failure. + if ( $f->{'info'} && $test->{'last_failure'} ) { + foreach my $line ( @{ $f->{'info'} } ) { + next unless $line->{'details'}; + chomp $line->{'details'}; + $test->{'last_failure'}->{'full_message'} .= "# $line->{details}\n"; + } + return; + } + +} + +# This is called when the last run is complete and we're ready to emit the junit file. + +sub finish { + my $self = shift; + + open( my $fh, '>:encoding(UTF-8)', $self->{'junit_file'} ) or die("Can't open '$self->{junit_file}' ($!)"); + + my $xml = $self->xml; + + # These are method calls but you can't do methods with a dash in them so we have to store them as a SV and call it. + my $out_method = 'system-out'; + my $err_method = 'system-err'; + + print {$fh} "\n"; + my @jobs = sort { $a->{'job_name'} <=> $b->{'job_name'} } values %{ $self->{'tests'} }; + foreach my $job (@jobs) { + print {$fh} $xml->testsuite( + $job->{'testsuite'}, + @{ $job->{'testcase'} }, + $xml->$out_method( $self->_cdata( $job->{$out_method} ) ), + $xml->$err_method( $self->_cdata( $job->{$err_method} ) ), + ) . "\n"; + } + + print {$fh} "\n"; + close $fh; + + return; +} + +# Because we want to test diag messages after a failed test, we delay closing failures +# until we see the end of the testcase or until we see a new test number. + +sub close_open_failure_testcase { + my ( $self, $test, $new_test_number ) = @_; + + # Need to handle failed TODOs + + # The last test wasn't a fail. + return unless $test->{'last_failure'}; + + my $fail = $test->{'last_failure'}; + + # This causes the entire suite to choke. We don't want this. + # If we're here already, we've already failed the test. let's just make sure the person reviewing + # it knows the test count was messed up. + if ( defined $fail->{'test_num'} + && defined $new_test_number + && $fail->{'test_num'} == $new_test_number ) + { + $fail->{'message'} + .= "# WARNING This test number has already been seen. Duplicate TEST # in output!\n"; + } + + my $xml = $self->xml; + push @{ $test->{'testcase'} }, $xml->testcase( + { 'name' => $fail->{'test_name'}, 'time' => $fail->{'time'}, 'classname' => $test->{'testsuite'}->{'name'} }, + $xml->failure( + { 'message' => $fail->{message}, 'type' => 'TestFailed' }, + $self->_cdata( $fail->{'full_message'} ) ) + ); + + delete $test->{'last_failure'}; + return; +} + +sub xml { + my $self = shift; + return $self->{'xml'}; +} + +# These helpers were borrowed from https://metacpan.org/pod/TAP::Formatter::JUnit. Thanks! + +############################################################################### +# Generates the name for the entire test suite. +sub _get_testsuite_name { + my $name = shift; + $name =~ s{^\./}{}; + $name =~ s{^t/}{}; + return _clean_to_java_class_name($name); +} + +############################################################################### +# Cleans up the given string, removing any characters that aren't suitable for +# use in a Java class name. +sub _clean_to_java_class_name { + my $str = shift; + $str =~ s/[^-:_A-Za-z0-9]+/_/gs; + return $str; +} + +############################################################################### +# Creates a CDATA block for the given data (which is made squeaky clean first, +# so that JUnit parsers like Hudson's don't choke). +sub _cdata { + my ( $self, $data ) = @_; + + # When I first added this conditional, I returned $data and at one point it was returning ^A and breaking the xml parser. + return '' if ( !$data or $data !~ m/\S/ms ); + + return $self->xml->xmlcdata( _squeaky_clean($data) ); +} + +############################################################################### +# Clean a string to the point that JUnit can't possibly have a problem with it. +sub _squeaky_clean { + my $string = shift; + + # control characters (except CR and LF) + $string =~ s/([\x00-\x09\x0b\x0c\x0e-\x1f])/"^".chr(ord($1)+64)/ge; + + # high-byte characters + $string =~ s/([\x7f-\xff])/'[\\x'.sprintf('%02x',ord($1)).']'/ge; + return $string; +} + +sub _timestamp { + my $time = shift; + return POSIX::strftime( '%Y-%m-%dT%H:%M:%S', localtime( int($time) ) ); +} + +1; + +__END__ + +=pod + +=encoding UTF-8 + +=head1 NAME + +App::Yath::Renderer::JUnit - Captures Test2::Harness results and emits a junit xml file. + +=head1 SYNOPSIS + +On the command line, with F: + + JUNIT_TEST_FILE="/tmp/test-output.xml" ALLOW_PASSING_TODOS=1 yath test --renderer=Formatter --renderer=JUnit -j4 t/*.t + +=head1 DESCRIPTION + +C provides JUnit output formatting sufficient +to be parsed by Jenkins and hopefully other junit parsers. + +This code borrows many ideas from C but unlike that module +does not provide a method to emit a different xml file for every testcase. +Instead, it defaults to emitting to a single B to whatever the directory +was you were in when you ran yath. This can be overridden by setting the +C environment variable + +Timing information is included in the JUnit XML since this is native to C + +In standard use, "passing TODOs" are treated as failure conditions (and are +reported as such in the generated JUnit). If you wish to treat these as a +"pass" and not a "fail" condition, setting C in your +environment will turn these into pass conditions. + +The JUnit output generated was developed to be used by Jenkins +(L). That's the build tool we use at the +moment and needed to be able to generate JUnit output for. + +=head1 METHODS + +=over + +=item B + +This is the only method (other than finish) that is called by Test2::Harness in order to +gather the data needed to emit the needed xml. + +=item B + +This method is called whenever a new test result or the end of a run is seen. Because +we want to capture test diag messages after a failed test, we delay emitting a failure +until we see the end of the testcase or until we see a new test number. + +=item B + +This method is called by Test2::Harness when all runs are complete. It takes what has +been gathered to that point and creates the junit xml file. + +=item xml + +An C instance, to be used to generate XML output. + +=item init + +This subroutine is called during object initialization for Test2::Hanress objects. +We do basic setup here. + +=back + +=head1 SOURCE + +The source code repository for Test2-Harness-Renderer-JUnit can be found at +F. + +=head1 MAINTAINERS + +=over 4 + +=item Todd Rinaldo, C<< >> + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 AUTHORS + +=over 4 + +=item Todd Rinaldo, C<< >> + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 COPYRIGHT + +Copyright Todd Rinaldo Etoddr@cpanel.netE. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +See F + +=cut diff --git a/lib/Test2/Harness/Renderer/JUnit.pm b/lib/Test2/Harness/Renderer/JUnit.pm new file mode 100644 index 000000000..5ed03db80 --- /dev/null +++ b/lib/Test2/Harness/Renderer/JUnit.pm @@ -0,0 +1,58 @@ +package Test2::Harness::Renderer::JUnit; +use strict; +use warnings; + +our $VERSION = '2.000000'; + +use Test2::Harness::Util::Deprecated( + delegate => 'App::Yath::Renderer::JUnit', +); + +1; + +__END__ + +=pod + +=encoding UTF-8 + +=head1 NAME + +Test2::Harness::Renderer::JUnit - Deprecated, use L. + +=head1 SOURCE + +The source code repository for Test2-Harness can be found at +L. + +=head1 MAINTAINERS + +=over 4 + +=item Todd Rinaldo, C<< >> + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 AUTHORS + +=over 4 + +=item Todd Rinaldo, C<< >> + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 COPYRIGHT + +Copyright Chad Granum Eexodist7@gmail.comE. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +See L + +=cut + diff --git a/t/JUnit/00-load.t b/t/JUnit/00-load.t new file mode 100644 index 000000000..52d1a645f --- /dev/null +++ b/t/JUnit/00-load.t @@ -0,0 +1,12 @@ +use 5.010000; +use strict; +use warnings; +use Test::More; + +plan tests => 1; + +BEGIN { + use_ok('App::Yath::Renderer::JUnit') || print "Bail out!\n"; +} + +diag("Testing App::Yath::Renderer::JUnit $App::Yath::Renderer::JUnit::VERSION, Perl $], $^X"); diff --git a/t/JUnit/test-die/die.tx b/t/JUnit/test-die/die.tx new file mode 100644 index 000000000..c5d5a06d9 --- /dev/null +++ b/t/JUnit/test-die/die.tx @@ -0,0 +1,8 @@ +use Test2::V0; + +plan 2; + +pass("pass"); +pass("another success"); + +die "plan 2 with 2 success exit non 0..."; diff --git a/t/JUnit/test-fail/fail.tx b/t/JUnit/test-fail/fail.tx new file mode 100644 index 000000000..3d82c604a --- /dev/null +++ b/t/JUnit/test-fail/fail.tx @@ -0,0 +1,8 @@ +use Test2::V0; + +# plan ok one test fail + +pass("pass"); +fail("this is a failure"); + +done_testing; diff --git a/t/JUnit/test-no-numbers/pass-1.tx b/t/JUnit/test-no-numbers/pass-1.tx new file mode 100644 index 000000000..adc923ca3 --- /dev/null +++ b/t/JUnit/test-no-numbers/pass-1.tx @@ -0,0 +1,14 @@ +use Test2::V0; + +sub pass_ok { + my $ctx = context(); + $ctx->hub->format->set_no_numbers(1); + + pass("pass"); + + $ctx->release; +} + +pass_ok(); + +done_testing; diff --git a/t/JUnit/test-ok/pass-1.tx b/t/JUnit/test-ok/pass-1.tx new file mode 100644 index 000000000..96a016cd5 --- /dev/null +++ b/t/JUnit/test-ok/pass-1.tx @@ -0,0 +1,5 @@ +use Test2::V0; + +pass("pass"); + +done_testing; diff --git a/t/JUnit/test-ok/pass-2.tx b/t/JUnit/test-ok/pass-2.tx new file mode 100644 index 000000000..96a016cd5 --- /dev/null +++ b/t/JUnit/test-ok/pass-2.tx @@ -0,0 +1,5 @@ +use Test2::V0; + +pass("pass"); + +done_testing; diff --git a/t/JUnit/test-plan/plan.tx b/t/JUnit/test-plan/plan.tx new file mode 100644 index 000000000..858b49ab0 --- /dev/null +++ b/t/JUnit/test-plan/plan.tx @@ -0,0 +1,9 @@ +use Test2::V0; + +plan 4; + +# plan ok one test fail + +pass("pass"); +pass("another success"); + diff --git a/t/JUnit/test-retry/retry.tx b/t/JUnit/test-retry/retry.tx new file mode 100644 index 000000000..38d50061f --- /dev/null +++ b/t/JUnit/test-retry/retry.tx @@ -0,0 +1,30 @@ +# HARNESS-DURATION-SHORT +use strict; +use warnings; + +use Test2::V0; +use Test2::API qw/test2_formatter/; + +ok(1, "Minimal result"); + +sub { + my $ctx = context(); + + diag "Formatter: " . test2_formatter(); + + $ctx->release; +}->(); + + +$ENV{T2_HARNESS_JOB_IS_TRY} //= 0; +$ENV{FAIL_ONCE} //= 0; +$ENV{FAIL_ALWAYS} //= 0; + +diag "JOB_IS_TRY = $ENV{T2_HARNESS_JOB_IS_TRY}"; +diag "FAIL_ONCE = $ENV{FAIL_ONCE}"; +diag "FAIL_ALWAYS = $ENV{FAIL_ALWAYS}"; + +ok(0, "Should fail once") if $ENV{FAIL_ONCE} && $ENV{T2_HARNESS_JOB_IS_TRY} < 1; +ok(0, "Should fail always") if $ENV{FAIL_ALWAYS}; + +done_testing(); diff --git a/t/JUnit/test.t b/t/JUnit/test.t new file mode 100644 index 000000000..2017a2acf --- /dev/null +++ b/t/JUnit/test.t @@ -0,0 +1,612 @@ +use Test2::V0; +use Test2::Plugin::BailOnFail; + +use FindBin; + +use File::Temp (); +use File::Spec; + +use App::Yath::Tester qw/yath/; + +use Test2::Harness::Util::File::JSONL (); +use App::Yath::Renderer::JUnit (); + +use Test2::Harness::Util qw/clean_path/; +use Test2::Harness::Util::JSON qw/decode_json/; + +#use Test2::Bundle::Extended; +use Test2::Tools::Explain; +use Test2::Plugin::NoWarnings; + +use XML::Simple (); + +my $tmpdir = File::Temp->newdir(); + +my $dir = __FILE__; +$dir =~ s{\.t$}{}g; + +my @renderers = qw{--renderer=JUnit}; + +delete $ENV{JUNIT_TEST_FILE}; # make sure it's not defined + +my $env = { # env passed to yath + PERL5LIB => "$FindBin::Bin/../../lib", +}; + +{ + note "all tests are ok - JUNIT_TEST_FILE not set ; renderer define"; + + my $sdir = $dir . '-ok'; + delete $env->{JUNIT_TEST_FILE}; + + my $default_junit_xml = "$FindBin::Bin/../../junit.xml"; + + unlink $default_junit_xml if -e $default_junit_xml; + + yath( + command => 'test', + args => [ $sdir, '--ext=tx', @renderers, '-v' ], + exit => 0, + env => $env, + no_app_path => 1, + test => sub { + my $out = shift; + + like( + $out->{output}, + qr{\Q( PASSED )\E.*\Qt/JUnit/test-ok/pass-1.tx\E}, + "t/JUnit/test-ok/pass-1.tx" + ); + like( + $out->{output}, + qr{\Q( PASSED )\E.*\Qt/JUnit/test-ok/pass-2.tx\E}, + "t/JUnit/test-ok/pass-2.tx" + ); + + like( $out->{output}, qr/Result: PASSED/, "Result: PASSED" ); + }, + ); + + ok -e $default_junit_xml, "use default junit.xml location"; + + unlink $default_junit_xml if -e $default_junit_xml; + +} + +{ + note "all tests are ok - JUNIT_TEST_FILE not reachable - No such file or directory"; + + my $sdir = $dir . '-ok'; + + $env->{JUNIT_TEST_FILE} = "$tmpdir/x/y/z/ok.xml"; + + yath( + command => 'test', + args => [ $sdir, '--ext=tx', @renderers, '-v' ], + exit => T(), + env => $env, + no_app_path => 1, + test => sub { + my $out = shift; + + like( + $out->{output}, + qr{\Q( PASSED )\E.*\Qt/JUnit/test-ok/pass-1.tx\E}, + "t/JUnit/test-ok/pass-1.tx" + ); + like( + $out->{output}, + qr{\Q( PASSED )\E.*\Qt/JUnit/test-ok/pass-2.tx\E}, + "t/JUnit/test-ok/pass-2.tx" + ); + + like( + $out->{output}, qr{\QNo such file or directory\E}, + "No such file or directory." + ); + }, + ); + +} + +{ + note "all tests are ok"; + + my $sdir = $dir . '-ok'; + + $env->{JUNIT_TEST_FILE} = "$tmpdir/ok.xml"; + + yath( + command => 'test', + args => [$sdir, '--ext=tx', @renderers, '-v'], + exit => 0, + env => $env, + no_app_path => 1, + test => sub { + my $out = shift; + + like( + $out->{output}, + qr{\Q( PASSED )\E.*\Qt/JUnit/test-ok/pass-1.tx\E}, + "t/JUnit/test-ok/pass-1.tx" + ); + like( + $out->{output}, + qr{\Q( PASSED )\E.*\Qt/JUnit/test-ok/pass-2.tx\E}, + "t/JUnit/test-ok/pass-2.tx" + ); + + like($out->{output}, qr/Result: PASSED/, "Result: PASSED"); + }, + ); + + # checking xml file + ok -e $env->{JUNIT_TEST_FILE}, 'junit file exists'; + + my $junit = XML::Simple::XMLin($env->{JUNIT_TEST_FILE}); + like( + $junit => hash { + field testsuite => hash { + field 'JUnit_test-ok_pass-1_tx' => hash { + field errors => 0; + field failures => 0; + field id => D(); + + field 'system-err' => hash { end; }; + field 'system-out' => hash { end; }; + + field 'testcase' => hash { + field '0001 - pass' => hash { + field classname => 'JUnit_test-ok_pass-1_tx'; + field time => D(); + end; + }; + field 'Tear down.' => hash { + field classname => 'JUnit_test-ok_pass-1_tx'; + field time => D(); + end; + }; + + end; + }; + + field tests => 1; + field time => D(); + field timestamp => D(); + + end; + }; + + field 'JUnit_test-ok_pass-2_tx' => hash { + field errors => 0; + field failures => 0; + field id => D(); + + field 'system-err' => hash { end; }; + field 'system-out' => hash { end; }; + + field 'testcase' => hash { + field '0001 - pass' => hash { + field classname => 'JUnit_test-ok_pass-2_tx'; + field time => D(); + end; + }; + field 'Tear down.' => hash { + field classname => 'JUnit_test-ok_pass-2_tx'; + field time => D(); + end; + }; + + end; + }; + + field tests => 1; + field time => D(); + field timestamp => D(); + + end; + }; + + end; + }; + + end; + + }, + 'junit output', + explain($junit), + ); +} + +{ + note "all tests are ok without numbers"; + + my $sdir = $dir . '-no-numbers'; + + $env->{JUNIT_TEST_FILE} = "$tmpdir/no-numbers.xml"; + + yath( + command => 'test', + args => [ $sdir, '--ext=tx', @renderers, '-v' ], + exit => 0, + env => $env, + no_app_path => 1, + test => sub { + my $out = shift; + + like( + $out->{output}, + qr{\Q( PASSED )\E.*\Qt/JUnit/test-no-numbers/pass-1.tx\E}, + "t/JUnit/test-no-numbers/pass-1.tx" + ); + + like( $out->{output}, qr/Result: PASSED/, "Result: PASSED" ); + }, + ); + + # checking xml file + ok -e $env->{JUNIT_TEST_FILE}, 'junit file exists'; + + my $junit = XML::Simple::XMLin($env->{JUNIT_TEST_FILE}); + like( + $junit => hash { + field testsuite => hash { + field errors => 0; + field failures => 0; + field id => D(); + field name => 'JUnit_test-no-numbers_pass-1_tx'; + + field 'system-err' => hash { end; }; + field 'system-out' => hash { end; }; + + field 'testcase' => hash { + field 'pass' => hash { + field classname => 'JUnit_test-no-numbers_pass-1_tx'; + field time => D(); + end; + }; + field 'Tear down.' => hash { + field classname => 'JUnit_test-no-numbers_pass-1_tx'; + field time => D(); + end; + }; + + end; + }; + + field tests => 1; + field time => D(); + field timestamp => D(); + + end; + }; + + end; + + }, + 'junit output', + explain($junit) + ); +} + +{ + note "plan ok - one failure"; + + my $sdir = $dir . '-fail'; + + $env->{JUNIT_TEST_FILE} = "$tmpdir/failure.xml"; + + yath( + command => 'test', + args => [$sdir, '--ext=tx', @renderers, '-v'], + exit => T(), + env => $env, + no_app_path => 1, + test => sub { + my $out = shift; + + like( + $out->{output}, + qr{\Q( FAILED )\E.*\Qt/JUnit/test-fail/fail.tx\E}, + "t/JUnit/test-fail/fail.tx" + ); + + like($out->{output}, qr/Result: FAILED/, "Result: FAILED"); + }, + ); + + ok -e $env->{JUNIT_TEST_FILE}, 'junit file exists'; + + my $junit = XML::Simple::XMLin($env->{JUNIT_TEST_FILE}); + + like( + $junit => hash { + field testsuite => hash { + field errors => 1; + field failures => 1; + field id => D(); + field name => 'JUnit_test-fail_fail_tx'; + + field 'system-err' => hash { end; }; + field 'system-out' => hash { end; }; + + field 'testcase' => hash { + field '0001 - pass' => hash { + field classname => 'JUnit_test-fail_fail_tx'; + field time => D(); + end; + }; + field '0002 - this is a failure' => hash { + field classname => 'JUnit_test-fail_fail_tx'; + field time => D(); + field failure => hash { + field content => match qr{not ok 0002 - this is a failure}; + field message => 'not ok 0002 - this is a failure'; + field type => 'TestFailed'; + }; + end; + }; + field 'Tear down.' => hash { + field classname => 'JUnit_test-fail_fail_tx'; + field time => D(); + end; + }; + + end; + }; + + field tests => 2; + field time => D(); + field timestamp => D(); + + end; + }; + + end; + + }, + 'junit output', + explain($junit), + ); +} + +{ + note "plan failure - test exit 0"; + + my $sdir = $dir . '-plan'; + + $env->{JUNIT_TEST_FILE} = "$tmpdir/plan.xml"; + + yath( + command => 'test', + args => [$sdir, '--ext=tx', @renderers, '-v'], + exit => T(), + env => $env, + no_app_path => 1, + test => sub { + my $out = shift; + + like( + $out->{output}, + qr{\Q( FAILED )\E.*\Qt/JUnit/test-plan/plan.tx\E}, + "t/JUnit/test-plan/plan.tx" + ); + + like($out->{output}, qr/Result: FAILED/, "Result: FAILED"); + }, + ); + + ok -e $env->{JUNIT_TEST_FILE}, 'junit file exists'; + + my $junit = XML::Simple::XMLin($env->{JUNIT_TEST_FILE}); + + like( + $junit => hash { + field testsuite => hash { + field errors => 2; + field failures => 0; + field id => D(); + field name => 'JUnit_test-plan_plan_tx'; + + field 'system-err' => hash { end; }; + field 'system-out' => hash { end; }; + + field 'testcase' => hash { + field '0001 - pass' => hash { + field classname => 'JUnit_test-plan_plan_tx'; + field time => D(); + end; + }; + field '0002 - another success' => hash { + field classname => 'JUnit_test-plan_plan_tx'; + field time => D(); + end; + }; + field 'Tear down.' => hash { + field classname => 'JUnit_test-plan_plan_tx'; + field time => D(); + end; + }; + + field 'Test Plan Failure' => hash { + field classname => 'JUnit_test-plan_plan_tx'; + field time => D(); + field failure => match q[Planned for 4 assertions, but saw 2]; + end; + }; + + end; + }; + + field tests => 2; + field time => D(); + field timestamp => D(); + + end; + }; + + end; + + }, + 'junit output', + explain($junit), + ); +} + +{ + note "plan ok - test exit non 0"; + + my $sdir = $dir . '-die'; + + $env->{JUNIT_TEST_FILE} = "$tmpdir/die.xml"; + + yath( + command => 'test', + args => [ $sdir, '--ext=tx', @renderers, '-v' ], + exit => T(), + env => $env, + no_app_path => 1, + test => sub { + my $out = shift; + + like( + $out->{output}, + qr{\Q( FAILED )\E.*\Qt/JUnit/test-die/die.tx\E}, + "t/JUnit/test-die/die.tx" + ); + + like( $out->{output}, qr/Result: FAILED/, "Result: FAILED" ); + }, + ); + + ok -e $env->{JUNIT_TEST_FILE}, 'junit file exists'; + + my $junit = XML::Simple::XMLin( $env->{JUNIT_TEST_FILE} ); + like( + $junit => hash { + field testsuite => hash { + field errors => 1; + field failures => 0; + field id => D(); + field name => 'JUnit_test-die_die_tx'; + + field 'system-err' => hash { end; }; + field 'system-out' => hash { end; }; + + field 'testcase' => hash { + field '0001 - pass' => hash { + field classname => 'JUnit_test-die_die_tx'; + field time => D(); + end; + }; + field '0002 - another success' => hash { + field classname => 'JUnit_test-die_die_tx'; + field time => D(); + end; + }; + field 'Tear down.' => hash { + field classname => 'JUnit_test-die_die_tx'; + field time => D(); + end; + }; + + field 'Program Ended Unexpectedly' => hash { + field classname => 'JUnit_test-die_die_tx'; + field time => D(); + field failure => match q[Test script returned error]; + end; + }; + + end; + }; + + field tests => 2; + field time => D(); + field timestamp => D(); + + end; + }; + + end; + + }, + 'junit output', + explain($junit) + ); +} + +{ + note "retry test - all succeed"; + + my $sdir = $dir . '-retry'; + + $env->{JUNIT_TEST_FILE} = "$tmpdir/retry.xml"; + $env->{FAIL_ONCE} = 1; + + yath( + command => 'test', + args => [$sdir, '--ext=tx', @renderers, '-v', '--retry=1',], + exit => 0, + env => $env, + no_app_path => 1, + test => sub { + my $out = shift; + + like($out->{output}, qr{FAIL.*Should fail once}, "one failure"); + like( + $out->{output}, + qr{\Q(TO RETRY)\E.*\Qt/JUnit/test-retry/retry.tx\E}, + "TO RETRY t/JUnit/test-retry/retry.tx" + ); + like( + $out->{output}, + qr{\Q( PASSED )\E.*\Qt/JUnit/test-retry/retry.tx\E}, + "PASSED t/JUnit/test-retry/retry.tx" + ); + + like($out->{output}, qr/Result: PASSED/, "Result: PASSED"); + }, + ); + + my $junit = XML::Simple::XMLin($env->{JUNIT_TEST_FILE}); + like( + $junit => hash { + field testsuite => hash { + field errors => 0; + field failures => 0; + field id => D(); + field name => 'JUnit_test-retry_retry_tx'; + + field 'system-err' => hash { end; }; + field 'system-out' => hash { end; }; + + field 'testcase' => hash { + field '0001 - Minimal result' => hash { + field classname => 'JUnit_test-retry_retry_tx'; + field time => D(); + end; + }; + field 'Tear down.' => hash { + field classname => 'JUnit_test-retry_retry_tx'; + field time => D(); + end; + }; + end; + }; + + field tests => 1; + field time => D(); + field timestamp => D(); + + end; + }; + + end; + + }, + 'junit output', + explain($junit), + ); +} + +done_testing;