Permalink
Browse files

test: offer "automake" output and check for perl better

runtests.pl -am now uses the "PASS/FAIL: [desc]" output for each
executed test. You can run 'make test-am' in the root build directory to
invoke that. The reason for this output style is to better allow generic
test suite parsers to also grok our test output.

The test Makefile now also tests that perl was indeed found and that the
PERL variable points to an executable before it tries to run the main
test perl script runtests.pl,
  • Loading branch information...
1 parent ef6f040 commit 14dc679ce0b9018ebe1bc7d0356b7869747ecd96 @bagder bagder committed Nov 23, 2012
Showing with 57 additions and 38 deletions.
  1. +3 −0 Makefile.am
  2. +12 −7 tests/Makefile.am
  3. +42 −31 tests/runtests.pl
View
3 Makefile.am
@@ -81,6 +81,9 @@ test-full:
test-torture:
@(cd tests; $(MAKE) all torture-test)
+test-am:
+ @(cd tests; $(MAKE) all am-test)
+
endif
examples:
View
19 tests/Makefile.am
@@ -55,26 +55,31 @@ curl:
if CROSSCOMPILING
TEST = @echo "NOTICE: we can't run the tests when cross-compiling!"
-TEST_Q =
-TEST_F =
-TEST_T =
else # if not cross-compiling:
TEST = srcdir=$(srcdir) $(PERL) $(PERLFLAGS) $(srcdir)/runtests.pl
TEST_Q = -a -s
+TEST_AM = -a -am
TEST_F = -a -p -r
TEST_T = -a -t
endif
-test: all
+# make sure that PERL is pointing to an executable
+perlcheck:
+ @if ! test -x "$(PERL)"; then echo "No perl!"; exit 2; fi
+
+test: perhcheck all
$(TEST)
-quiet-test: all
+quiet-test: perlcheck all
$(TEST) $(TEST_Q)
-full-test: all
+am-test: perlcheck all
+ $(TEST) $(TEST_AM)
+
+full-test: perlcheck all
$(TEST) $(TEST_F)
-torture-test: all
+torture-test: perlcheck all
$(TEST) $(TEST_T)
.1.html:
View
73 tests/runtests.pl
@@ -262,6 +262,7 @@ BEGIN
#
my $short;
+my $automakestyle;
my $verbose;
my $debugprotocol;
my $anyway;
@@ -2148,18 +2149,25 @@ sub filteroff {
#
sub compare {
# filter off patterns _before_ this comparison!
- my ($subject, $firstref, $secondref)=@_;
+ my ($testnum, $testname, $subject, $firstref, $secondref)=@_;
my $result = compareparts($firstref, $secondref);
if($result) {
+ # timestamp test result verification end
+ $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
+
if(!$short) {
- logmsg "\n $subject FAILED:\n";
+ logmsg "\n $testnum: $subject FAILED:\n";
logmsg showdiff($LOGDIR, $firstref, $secondref);
}
- else {
+ elsif(!$automakestyle) {
logmsg "FAILED\n";
}
+ else {
+ # automakestyle
+ logmsg "FAIL: $testnum - $testname - $subject\n";
+ }
}
return $result;
}
@@ -2874,7 +2882,7 @@ sub singletest {
timestampskippedevents($testnum);
return -1;
}
- logmsg sprintf("test %03d...", $testnum);
+ logmsg sprintf("test %03d...", $testnum) if(!$automakestyle);
# extract the reply data
my @reply = getpart("reply", "data");
@@ -2916,12 +2924,9 @@ sub singletest {
# name of the test
my @testname= getpart("client", "name");
-
- if(!$short) {
- my $name = $testname[0];
- $name =~ s/\n//g;
- logmsg "[$name]\n";
- }
+ my $testname = $testname[0];
+ $testname =~ s/\n//g;
+ logmsg "[$testname]\n" if(!$short);
if($listonly) {
timestampskippedevents($testnum);
@@ -3333,10 +3338,8 @@ sub singletest {
chomp($validstdout[$#validstdout]);
}
- $res = compare("stdout", \@actual, \@validstdout);
+ $res = compare($testnum, $testname, "stdout", \@actual, \@validstdout);
if($res) {
- # timestamp test result verification end
- $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
return 1;
}
$ok .= "s";
@@ -3357,10 +3360,8 @@ sub singletest {
map s/\r\n/\n/g, @out;
}
- $res = compare("data", \@out, \@reply);
+ $res = compare($testnum, $testname, "data", \@out, \@reply);
if ($res) {
- # timestamp test result verification end
- $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
return 1;
}
$ok .= "d";
@@ -3372,10 +3373,8 @@ sub singletest {
if(@upload) {
# verify uploaded data
my @out = loadarray("$LOGDIR/upload.$testnum");
- $res = compare("upload", \@out, \@upload);
+ $res = compare($testnum, $testname, "upload", \@out, \@upload);
if ($res) {
- # timestamp test result verification end
- $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
return 1;
}
$ok .= "u";
@@ -3419,10 +3418,8 @@ sub singletest {
}
}
- $res = compare("protocol", \@out, \@protstrip);
+ $res = compare($testnum, $testname, "protocol", \@out, \@protstrip);
if($res) {
- # timestamp test result verification end
- $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
return 1;
}
@@ -3469,10 +3466,8 @@ sub singletest {
}
}
- $res = compare("proxy", \@out, \@protstrip);
+ $res = compare($testnum, $testname, "proxy", \@out, \@protstrip);
if($res) {
- # timestamp test result verification end
- $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
return 1;
}
@@ -3521,10 +3516,9 @@ sub singletest {
@outfile = fixarray(@outfile);
- $res = compare("output ($filename)", \@generated, \@outfile);
+ $res = compare($testnum, $testname, "output ($filename)",
+ \@generated, \@outfile);
if($res) {
- # timestamp test result verification end
- $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
return 1;
}
@@ -3614,8 +3608,13 @@ sub singletest {
}
my @e = valgrindparse($srcdir, $feature{'SSL'}, "$LOGDIR/$vgfile");
if(@e && $e[0]) {
- logmsg " valgrind ERROR ";
- logmsg @e;
+ if($automakestyle) {
+ logmsg "FAIL: $testnum - $testname - valgrind\n";
+ }
+ else {
+ logmsg " valgrind ERROR ";
+ logmsg @e;
+ }
# timestamp test result verification end
$timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
return 1;
@@ -3641,7 +3640,13 @@ sub singletest {
my $left=sprintf("remaining: %02d:%02d",
$estleft/60,
$estleft%60);
- logmsg sprintf("OK (%-3d out of %-3d, %s)\n", $count, $total, $left);
+
+ if(!$automakestyle) {
+ logmsg sprintf("OK (%-3d out of %-3d, %s)\n", $count, $total, $left);
+ }
+ else {
+ logmsg "PASS: $testnum - $testname\n";
+ }
# the test succeeded, remove all log files
if(!$keepoutfiles) {
@@ -4284,6 +4289,11 @@ sub runtimestats {
# short output
$short=1;
}
+ elsif($ARGV[0] eq "-am") {
+ # automake-style output
+ $short=1;
+ $automakestyle=1;
+ }
elsif($ARGV[0] eq "-n") {
# no valgrind
undef $valgrind;
@@ -4360,6 +4370,7 @@ sub runtimestats {
-r run time statistics
-rf full run time statistics
-s short output
+ -am automake style output PASS/FAIL: [number] [name]
-t[N] torture (simulate memory alloc failures); N means fail Nth alloc
-v verbose output
[num] like "5 6 9" or " 5 to 22 " to run those tests only

0 comments on commit 14dc679

Please sign in to comment.