From 2ab2f837ccdde6464782d38f89a6dc4009a40cc0 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Tue, 1 Dec 2009 21:59:39 -0800 Subject: [PATCH 1/2] Make tests executable which allows using /bin/sh -c to exec a test and allow us to not care which interpreter to run it with --- t/00-sanity.t | 0 t/01-parse_plan.t | 0 t/02-parse_tapstream.t | 0 t/03-parse_tapstream_error.t | 0 t/harness.pir | 5 ++++- 5 files changed, 4 insertions(+), 1 deletion(-) mode change 100644 => 100755 t/00-sanity.t mode change 100644 => 100755 t/01-parse_plan.t mode change 100644 => 100755 t/02-parse_tapstream.t mode change 100644 => 100755 t/03-parse_tapstream_error.t diff --git a/t/00-sanity.t b/t/00-sanity.t old mode 100644 new mode 100755 diff --git a/t/01-parse_plan.t b/t/01-parse_plan.t old mode 100644 new mode 100755 diff --git a/t/02-parse_tapstream.t b/t/02-parse_tapstream.t old mode 100644 new mode 100755 diff --git a/t/03-parse_tapstream_error.t b/t/03-parse_tapstream_error.t old mode 100644 new mode 100755 diff --git a/t/harness.pir b/t/harness.pir index b6f7093..de26c7f 100644 --- a/t/harness.pir +++ b/t/harness.pir @@ -30,8 +30,11 @@ unless file goto done inc total_files print file + # these should be normalized to make the output format 'pretty' print ".........." - output = qx('parrot',file) + + # how to make this platform-independent? + output = qx('/bin/sh','-c',file) stream = tapir.'parse_tapstream'(output) success = stream.'is_pass'() unless success goto fail From 1c4c87337a72d60282ad07f7f3e6e066824c83fc Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Tue, 1 Dec 2009 23:14:28 -0800 Subject: [PATCH 2/2] Fix some embarrassing bugs, add the --exec option and now Tapir can run Plumage's test suite! Various bugs in parsing unrecognized lines in TAP were fixed, and now we are able to run the Plumage test suite with Tapir. It appears that for the current Plumage test suite of 64 tests in 4 files, Tapir is the same speed as nqpTAP (which was somewhat unexpected), while having more features, fewer bugs and lots of tests. t/harness in the Plumage repo has no tests. --- TODO | 5 +++ lib/Tapir/Parser.pir | 44 ++++++++++++------------- t/03-parse_tapstream_error.t | 63 +++++++++++++++++++++++++++++++++++- t/harness.pir | 22 +++++++++++-- 4 files changed, 108 insertions(+), 26 deletions(-) diff --git a/TODO b/TODO index 4573d3e..5b98a9a 100644 --- a/TODO +++ b/TODO @@ -5,8 +5,13 @@ Please let me know if you start hacking on one of these or if you have any amazi * Be able to run any test file, not just PIR tests Run /bin/sh -c or ./file, and then default to parrot if neither work ? + + Turns out /bin/sh -c foo.t means foo.t must be executable. + We also have the option of using proc_exec from the global parrot config +* Use Getopt to have some nice command line options, like --help and --version + * Parse todo/skips/bailout correctly http://cpansearch.perl.org/src/ANDYA/Test-Harness-3.17/lib/TAP/Parser/Grammar.pm diff --git a/lib/Tapir/Parser.pir b/lib/Tapir/Parser.pir index a480f5c..50740f0 100644 --- a/lib/Tapir/Parser.pir +++ b/lib/Tapir/Parser.pir @@ -10,22 +10,22 @@ Written and maintained by Jonathan "Duke" Leto C<< jonathan@leto.net >>. .sub parse_tapstream :method .param string tap - .local string curr_line, delim + .local string curr_line .local pmc plan, pass, fail, skip, todo - .local int i, curr_test + .local int i, curr_test, reported_test .local pmc tap_lines, parts, klass, stream - i = 0 - fail = new 'Integer' - skip = new 'Integer' - todo = new 'Integer' - pass = new 'Integer' - plan = new 'Integer' - + i = 0 + curr_test = 1 + fail = new 'Integer' + skip = new 'Integer' + todo = new 'Integer' + pass = new 'Integer' + plan = new 'Integer' tap_lines = new 'ResizablePMCArray' parts = new 'ResizablePMCArray' - delim = "\n" - split tap_lines, delim, tap + + split tap_lines, "\n", tap $I0 = tap_lines .local string plan_line @@ -36,23 +36,24 @@ Written and maintained by Jonathan "Duke" Leto C<< jonathan@leto.net >>. loop: if i >= $I0 goto done curr_line = tap_lines[i] - unless curr_line goto done - delim = "ok " - split parts, delim, curr_line + split parts, "ok ", curr_line - prefix = parts[0] - curr_test = parts[1] + prefix = parts[0] + reported_test = parts[1] if prefix == 'not ' goto fail_or_todo - if curr_test == i goto got_pass + if reported_test == curr_test goto got_pass + + # it was an unrecognized line inc i goto loop got_pass: # check curr_test for comments inc pass inc i + inc curr_test goto loop fail_or_todo: split parts, "# ", curr_line @@ -62,17 +63,16 @@ Written and maintained by Jonathan "Duke" Leto C<< jonathan@leto.net >>. if $S0 != "todo" goto failz # it is a TODO test! inc todo + inc curr_test inc i goto loop failz: inc fail + inc curr_test inc i goto loop done: - # XXX hack - dec pass # passes get mis-counted by 1 - stream = new [ 'Tapir'; 'Stream' ] stream.'set_pass'(pass) stream.'set_fail'(fail) @@ -88,11 +88,9 @@ Written and maintained by Jonathan "Duke" Leto C<< jonathan@leto.net >>. .param string plan_line .local pmc plan_parts .local int num_expected_tests - .local string delim - delim = ".." plan_parts = new 'ResizablePMCArray' - split plan_parts, delim, plan_line + split plan_parts, "..", plan_line unless plan_parts goto plan_error num_expected_tests = plan_parts[1] diff --git a/t/03-parse_tapstream_error.t b/t/03-parse_tapstream_error.t index 9d358a3..85226e3 100755 --- a/t/03-parse_tapstream_error.t +++ b/t/03-parse_tapstream_error.t @@ -7,7 +7,7 @@ .include 'test_more.pir' .local pmc tapir, klass - plan(12) + plan(19) # setup test data klass = newclass [ 'Tapir'; 'Parser' ] @@ -15,7 +15,68 @@ test_parse_death(tapir) test_parse_death_with_passing_tests(tapir) + test_plumage_sanity(tapir) +.end +.sub test_plumage_sanity + .param pmc tapir + .local string tap_error + .local pmc stream + tap_error = <<"TAP" +1..18 +invalidjunkdoesnotexist +ok 1 - do_run()ing invalidjunk returns false +./plumage asdfversion +I don't know how to 'asdfversion'! +ok 2 - plumage returns failure for invalid commands +ok 3 +ok 4 +ok 5 +ok 6 +ok 7 +ok 8 - no args give usage +ok 9 - no args give usage +ok 10 - plumage fetch no args +ok 11 +ok 12 +./plumage version +This is Parrot Plumage, version 0. + +Copyright (C) 2009, Parrot Foundation. + +This code is distributed under the terms of the Artistic License 2.0. +For more details, see the full text of the license in the LICENSE file +included in the Parrot Plumage source tree. +ok 13 - plumage version returns success +ok 14 - plumage version knows its name +ok 15 - version mentions Parrot Foundation +ok 16 - version mentions Artistic License +ok 17 - info rakudo +ok 18 - info rakudo +.end +TAP + stream = tapir.'parse_tapstream'(tap_error) + + $I0 = stream.'get_plan'() + is($I0,18,"plan is correct") + + $I0 = stream.'is_pass'() + is($I0,1,"parse_tapstream passes Plumage's sanity test") + + $I0 = stream.'get_pass'() + is($I0,18,"parse_tapstream gets 18 tests") + + $I0 = stream.'get_todo'() + is($I0,0,"parse_tapstream gets no todo tests") + + $I0 = stream.'get_skip'() + is($I0,0,"parse_tapstream gets no skip tests") + + $I0 = stream.'get_fail'() + is($I0,0,"parse_tapstream gets no fails") + + $I0 = stream.'total'() + is($I0,18,"parse_tapstream gets 18 tests in total") .end .sub test_parse_death diff --git a/t/harness.pir b/t/harness.pir index de26c7f..25feca1 100644 --- a/t/harness.pir +++ b/t/harness.pir @@ -5,10 +5,22 @@ .sub main :main .param pmc argv + + load_bytecode "Getopt/Obj.pbc" $S0 = shift argv # get rid of harness.pir in the args list $I0 = argv dec $I0 + # parse command line args + .local pmc getopts, opts + getopts = new "Getopt::Obj" + getopts."notOptStop"(1) + push getopts, "exec|e:s" + opts = getopts."get_options"(argv) + + .local string exec + exec = opts["exec"] + .local pmc tapir, klass klass = newclass [ 'Tapir'; 'Parser' ] tapir = klass.'new'() @@ -33,8 +45,14 @@ # these should be normalized to make the output format 'pretty' print ".........." - # how to make this platform-independent? - output = qx('/bin/sh','-c',file) + # we assume the test is PIR unless given an --exec flag + # how to do proper shebang-line detection? + .local string exec_cmd + exec_cmd = 'parrot' + unless exec goto run_cmd + exec_cmd = exec + run_cmd: + output = qx(exec,file) stream = tapir.'parse_tapstream'(output) success = stream.'is_pass'() unless success goto fail