Skip to content

Commit

Permalink
MDEV-22176 Add JUnit support to MTR to generate XML test result
Browse files Browse the repository at this point in the history 
A new parameter has been added called xml-report, with which the
filename of the XML file is given to which the XML result is
written. There is also xml-package for adding a package value in
the XML output. Example usage:
./mysql-test-run.pl main.events_bugs innodb.count_distinct
main.explain_json innodb.file_format_defaults json.json_no_table
--suite=main,innodb,json --force --xml-report=build123456789.xml
--xml-package=simpletestrun
  • Loading branch information
@rasmushoj
rasmushoj committed Apr 7, 2020
1 parent 241ac3f commit 5720db2
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 3 deletions.
2 changes: 1 addition & 1 deletion mysql-test/lib/My/Tee.pm
View file
Expand Up @@ -10,7 +10,7 @@ sub PUSHED
open($copyfh, '>', "$::opt_vardir/log/stdout.log")
or die "open(>$::opt_vardir/log/stdout.log): $!"
unless $copyfh;
bless { }, shift;
bless { }, shift;
}

sub WRITE
Expand Down
90 changes: 90 additions & 0 deletions mysql-test/lib/mtr_report.pm
View file
Expand Up @@ -20,7 +20,9 @@
# same name.

package mtr_report;

use strict;
use Sys::Hostname;

use base qw(Exporter);
our @EXPORT= qw(report_option mtr_print_line mtr_print_thick_line
Expand Down Expand Up @@ -253,6 +255,7 @@ sub mtr_report_stats ($$$$) {
# Find out how we where doing
# ----------------------------------------------------------------------

my $tot_disabled = 0;
my $tot_skipped= 0;
my $tot_skipdetect= 0;
my $tot_passed= 0;
Expand All @@ -273,6 +276,7 @@ sub mtr_report_stats ($$$$) {
{
# Test was skipped (disabled not counted)
$tot_skipped++ unless $tinfo->{'disable'};
$tot_disabled++ if $tinfo->{'disable'};
$tot_skipdetect++ if $tinfo->{'skip_detected_by_test'};
}
elsif ( $tinfo->{'result'} eq 'MTR_RES_PASSED' )
Expand Down Expand Up @@ -402,6 +406,92 @@ sub mtr_report_stats ($$$$) {
print "All $tot_tests tests were successful.\n\n";
}

if ($::opt_xml_report) {
my $xml_report = "";
my @sorted_tests = sort {$a->{'name'} cmp $b->{'name'}} @$tests;
my $last_suite = "";
my $current_suite = "";
my $timest = isotime(time);
my %suite_totals;
my %suite_time;
my %suite_tests;
my %suite_failed;
my %suite_disabled;
my %suite_skipped;
my $host = hostname;
my $suiteNo = 0;

# loop through test results to count totals
foreach my $test ( @sorted_tests ) {
$current_suite = $test->{'suite'}->{'name'};

if ($test->{'timer'} eq "") {
$test->{'timer'} = 0;
}

$suite_time{$current_suite} = $suite_time{$current_suite} + $test->{'timer'};
$suite_tests{$current_suite} = $suite_tests{$current_suite} + 1;

if ($test->{'result'} eq "MTR_RES_FAILED") {
$suite_failed{$current_suite} = $suite_failed{$current_suite} + 1;
} elsif ($test->{'result'} eq "MTR_RES_SKIPPED" && $test->{'disable'}) {
$suite_disabled{$current_suite} = $suite_disabled{$current_suite} + 1;
} elsif ($test->{'result'} eq "MTR_RES_SKIPPED") {
$suite_skipped{$current_suite} = $suite_skipped{$current_suite} + 1;
}

$suite_totals{"all_time"} = $suite_totals{"all_time"} + $test->{'timer'};
}

my $all_time = sprintf("%.3f", $suite_totals{"all_time"} / 1000);
my $suite_time = 0;
my $test_time = 0;

# generate xml
$xml_report = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
$xml_report .= qq(<testsuites disabled="$tot_disabled" errors="" failures="$tot_failed" name="" tests="$tot_tests" time="$all_time">\n);

foreach my $test ( @sorted_tests ) {
$current_suite = $test->{'suite'}->{'name'};

if ($current_suite ne $last_suite) {
if ($last_suite ne "") {
$xml_report .= "\t</testsuite>\n";
$suiteNo++;
}

$suite_time = sprintf("%.3f", $suite_time{$current_suite} / 1000);
$xml_report .= qq(\t<testsuite disabled="$suite_disabled{$current_suite}" errors="" failures="$suite_failed{$current_suite}" hostname="$host" id="$suiteNo" name="$current_suite" package="" skipped="$suite_skipped{$current_suite}" tests="$suite_tests{$current_suite}" time="$suite_time" timestamp="$timest">\n);
$last_suite = $current_suite;
}

$test_time = sprintf("%.3f", $test->{timer} / 1000);
$xml_report .= qq(\t\t<testcase assertions="" classname="$current_suite" name="$test->{'name'}" status="$test->{'result'}" time="$test_time");

my $comment = $test->{'comment'};
$comment =~ s/[\"]//g;

if ($test->{'result'} eq "MTR_RES_FAILED") {
$xml_report .= qq(>\n\t\t\t<failure message="" type="$test->{'result'}">\n<![CDATA[$test->{'logfile'}]]>\n\t\t\t</failure>\n\t\t</testcase>\n);
} elsif ($test->{'result'} eq "MTR_RES_SKIPPED" && $test->{'disable'}) {
$xml_report .= qq(>\n\t\t\t<disabled message="$comment" type="$test->{'result'}"/>\n\t\t</testcase>\n);
} elsif ($test->{'result'} eq "MTR_RES_SKIPPED") {
$xml_report .= qq(>\n\t\t\t<skipped message="$comment" type="$test->{'result'}"/>\n\t\t</testcase>\n);
} else {
$xml_report .= " />\n";
}
}

$xml_report .= "\t</testsuite>\n</testsuites>\n";

# save to file
my $xml_file = $::opt_xml_report;

open XML_FILE, ">", $xml_file or die "Cannot create file $xml_file: $!";
print XML_FILE $xml_report;
close XML_FILE;
}

if (@$extra_warnings)
{
print <<MSG;
Expand Down
8 changes: 6 additions & 2 deletions mysql-test/mysql-test-run.pl
View file
Expand Up @@ -129,6 +129,8 @@ BEGIN
our $default_vardir;
our $opt_vardir; # Path to use for var/ dir
our $plugindir;
our $opt_xml_report; # XML output

my $path_vardir_trace; # unix formatted opt_vardir for trace files
my $opt_tmpdir; # Path to use for tmp/ dir
my $opt_tmpdir_pid;
Expand Down Expand Up @@ -563,7 +565,6 @@ sub main {
}

print_total_times($opt_parallel) if $opt_report_times;

mtr_report_stats($prefix, $fail, $completed, $extra_warnings);

if ( @$completed != $num_tests)
Expand Down Expand Up @@ -1059,6 +1060,7 @@ sub print_global_resfile {
resfile_global("warnings", $opt_warnings ? 1 : 0);
resfile_global("max-connections", $opt_max_connections);
resfile_global("product", "MySQL");
resfile_global("xml-report", $opt_xml_report);
# Somewhat hacky code to convert numeric version back to dot notation
my $v1= int($mysql_version_id / 10000);
my $v2= int(($mysql_version_id % 10000)/100);
Expand Down Expand Up @@ -1225,7 +1227,8 @@ sub command_line_setup {
'help|h' => \$opt_usage,
# list-options is internal, not listed in help
'list-options' => \$opt_list_options,
'skip-test-list=s' => \@opt_skip_test_list
'skip-test-list=s' => \@opt_skip_test_list,
'xml-report=s' => \$opt_xml_report
);

# fix options (that take an optional argument and *only* after = sign
Expand Down Expand Up @@ -6281,6 +6284,7 @@ ($)
phases of test execution.
stress=ARGS Run stress test, providing options to
mysql-stress-test.pl. Options are separated by comma.
xml-report=<file> Output jUnit xml file of the results.
Some options that control enabling a feature for normal test runs,
can be turned off by prepending 'no' to the option, e.g. --notimer.
Expand Down

0 comments on commit 5720db2

Please sign in to comment.