2011-05-19 Dmitry Lomov <dslomov@google.com>

        Reviewed by Adam Roben.

        run-api-tests should run one test per process
        https://bugs.webkit.org/show_bug.cgi?id=61088

        * Scripts/run-api-tests: Resurrecting the previous revison of this file, with fixes to system call under Windows,
        return code, and parsing GTest output format.

Canonical link: https://commits.webkit.org/76519@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@86907 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog

@@ -1,3 +1,13 @@
+2011-05-19  Dmitry Lomov  <dslomov@google.com>
+
+        Reviewed by Adam Roben.
+
+        run-api-tests should run one test per process
+        https://bugs.webkit.org/show_bug.cgi?id=61088
+
+        * Scripts/run-api-tests: Resurrecting the previous revison of this file, with fixes to system call under Windows, 
+        return code, and parsing GTest output format.
+
 2011-05-15  Robert Hogan  <robert@webkit.org>
 
         Reviewed by Antonio Gomes.

Tools/Scripts/run-api-tests

@@ -36,9 +36,14 @@ use Getopt::Long qw(:config pass_through);
 use IPC::Open3;
 use lib $FindBin::Bin;
 use webkitdirs;
+use Term::ANSIColor qw(:constants);
 
-sub runTestTool(@);
 sub buildTestTool();
+sub dumpAllTests();
+sub populateTests();
+sub runAllTests();
+sub runAllTestsInSuite($);
+sub runTest($$);
 
 my $showHelp = 0;
 my $verbose = 0;
@@ -66,18 +71,64 @@ if ($showHelp) {
 setConfiguration();
 buildTestTool();
 setPathForRunningWebKitApp(\%ENV);
+my %testsToRun = populateTests();
 
 if ($dump) {
-    my @dumpArguments = ("--gtest_list_tests");
-    runTestTool(@dumpArguments);
-    exit(0);
+    dumpAllTests();
+    exit 0;
 }
 
-runTestTool();
+if (runAllTests()) {
+    exit 1;
+}
+
+sub dumpAllTests()
+{
+    print "Dumping test cases\n";
+    print "------------------\n";
+    for my $suite (keys %testsToRun) {
+        print $suite . ":\n";
+        print map { "   " . $_ . "\n" } @{ $testsToRun{$suite} };
+    }
+    print "------------------\n";
+}
+
+sub runAllTests()
+{
+    my $anyFailures = 0;
+    for my $suite (keys %testsToRun) {
+        my $failed = runAllTestsInSuite($suite);
+        if ($failed) {
+            $anyFailures = 1;
+        }
+    }
+    return $anyFailures;
+}
 
-sub runTestTool(@)
+sub runAllTestsInSuite($)
 {
-    my (@arguments) = @_;
+    my ($suite) = @_;
+    print "Suite: $suite\n";
+
+    my $anyFailures = 0;
+    for my $test (@{$testsToRun{$suite}}) {
+        my $failed = runTest($suite, $test);
+        if ($failed) {
+            $anyFailures = 1;
+        }
+    }
+
+    return $anyFailures;
+}
+
+sub runTest($$)
+{
+    my ($suite, $testName) = @_;
+    my $test = $suite . "." . $testName;
+
+    my $gtestArg = "--gtest_filter=" . $test;
+
+    print "    Test: $testName -> ";
 
     my $result = 0;
     if (isAppleMacWebKit()) {
@@ -88,26 +139,26 @@ sub runTestTool(@)
 
         local *DEVNULL;
         my ($childIn, $childOut, $childErr);
-
-        $childOut = ">&STDOUT";
-        unless ($verbose) {
+        if ($verbose) {
+            $childOut = ">&STDOUT";
+            $childErr = ">&STDERR";
+        } else {
             open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
+            $childOut = ">&DEVNULL";
             $childErr = ">&DEVNULL";
-        } else {
-            $childErr = ">&STDERR";
         }
 
         my $pid;
         if (architecture()) {
-            $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), $apiTesterPath, @arguments, @ARGV) or die "Failed to run test tool.";
+            $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), $apiTesterPath, $gtestArg, @ARGV) or die "Failed to run test: $test.";
         } else {
-            $pid = open3($childIn, $childOut, $childErr, $apiTesterPath, @arguments, @ARGV) or die "Failed to run test tool.";
+            $pid = open3($childIn, $childOut, $childErr, $apiTesterPath, $gtestArg, @ARGV) or die "Failed to run test: $test.";
         }
 
         close($childIn);
         close($childOut);
         close($childErr);
-        close(DEVNULL);
+        close(DEVNULL) unless ($verbose);
 
         waitpid($pid, 0);
         $result = $?;
@@ -119,14 +170,91 @@ sub runTestTool(@)
             $apiTesterNameSuffix = "_debug";
         }
         my $apiTesterPath = File::Spec->catfile(productDir(), "TestWebKitAPI$apiTesterNameSuffix.exe");
-        $result = system { $apiTesterPath } $apiTesterPath, @ARGV;
+        $result = system { $apiTesterPath } $apiTesterPath, $gtestArg, @ARGV;
     } else {
         die "run-api-tests is not supported on this platform.\n"
     }
-
+
+    if (!$result) {
+        print BOLD GREEN, "Passed", RESET, "\n";
+    } else {
+        print BOLD RED, "Failed", RESET, "\n";
+    }
     return $result;
 }
 
+sub populateTests()
+{
+    my @tests;
+    my $timedOut;
+
+    if (isAppleMacWebKit()) {
+        my $productDir = productDir();
+        $ENV{DYLD_FRAMEWORK_PATH} = $productDir;
+        $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES";
+        my $apiTesterPath = "$productDir/TestWebKitAPI";
+
+        local *DEVNULL;
+        my ($childIn, $childOut, $childErr);
+        if ($verbose) {
+            $childErr = ">&STDERR";
+        } else {
+            open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
+            $childErr = ">&DEVNULL";
+        }
+
+        my $pid;
+        if (architecture()) {
+            $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), $apiTesterPath, "--gtest_list_tests") or die "Failed to build list of tests!";
+        } else {
+            $pid = open3($childIn, $childOut, $childErr, $apiTesterPath, "--gtest_list_tests") or die "Failed to build list of tests!";
+        }
+
+        close($childIn);
+        @tests = <$childOut>;
+        close($childOut);
+        close($childErr);
+        close(DEVNULL) unless ($verbose);
+
+        waitpid($pid, 0);
+        my $result = $?;
+
+        if ($result) {
+            print STDERR "Failed to build list of tests!\n";
+            exit exitStatus($result);
+        }
+    } elsif (isAppleWinWebKit()) {
+        my $apiTesterNameSuffix;
+        if (configurationForVisualStudio() ne "Debug_All") {
+            $apiTesterNameSuffix = "";
+        } else {
+            $apiTesterNameSuffix = "_debug";
+        }
+        my $apiTesterPath = File::Spec->catfile(productDir(), "TestWebKitAPI$apiTesterNameSuffix.exe");
+        open(TESTS, "-|", $apiTesterPath, "--dump-tests") or die $!;
+        @tests = <TESTS>;
+        close(TESTS) or die $!;
+    } else {
+        die "run-api-tests is not supported on this platform.\n"
+    }
+
+    my %keyedTests = ();
+    my $suite;
+    for my $test (@tests) {
+       $test =~ s/[\r\n]*$//;
+       if ($test =~ m/\.$/) {
+          $test =~ s/\.$//;
+          $suite = $test;
+       } else {
+          $test =~ s/^\s*//;
+          push @{$keyedTests{$suite}}, $test;
+        }
+    }
+
+    return %keyedTests;
+}
+
+
 sub buildTestTool()
 {
     my $originalCwd = getcwd();
@@ -138,14 +266,14 @@ sub buildTestTool()
 
     local *DEVNULL;
     my ($childIn, $childOut, $childErr);
-    unless ($verbose) {
-        open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
-        $childOut = ">&DEVNULL";
-        $childErr = ">&DEVNULL";
-    } else {
+    if ($verbose) {
         # When not quiet, let the child use our stdout/stderr.
         $childOut = ">&STDOUT";
         $childErr = ">&STDERR";
+    } else {
+        open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
+        $childOut = ">&DEVNULL";
+        $childErr = ">&DEVNULL";
     }
 
     my @args = argumentsForConfiguration();