2010-02-25 Dirk Pranke <dpranke@chromium.org>

        Reviewed by Eric Seidel.

        Add a 'passing' port implementation to new-run-webkit-tests that
        acts as a wrapper around an existing implementation but stubs out
        the actual test invocations (instead, the expected results are echoed
        back to the harness). This is useful for coverage and perf testing
        of the harness (especially perf testing as it essentially provides
        a lower bound on how fast the harness can run).

        Also added a --nostart-helper flag to new-run-webkit-tests so that
        you can skip starting the layout_test_helper and actually run the
        harness even if you don't have a build of that port.

        Also fix a bug in the 'test' port implementation to actually
        create the results directory under /tmp instead of /.

        https://bugs.webkit.org/show_bug.cgi?id=35370

        * Scripts/webkitpy/layout_tests/port/factory.py: Modified.
        * Scripts/webkitpy/layout_tests/port/passing.py: Added.
        * Scripts/webkitpy/layout_tests/port/test.py: Added.
        * Scripts/webkitpy/layout_tests/run_webkit_tests.py: Modified.

Canonical link: https://commits.webkit.org/46569@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@55268 268f45cc-cd09-0410-ab3c-d52691b4dbfc

WebKitTools/ChangeLog

@@ -1,3 +1,28 @@
+2010-02-25  Dirk Pranke  <dpranke@chromium.org>
+
+        Reviewed by Eric Seidel.
+
+        Add a 'passing' port implementation to new-run-webkit-tests that
+        acts as a wrapper around an existing implementation but stubs out
+        the actual test invocations (instead, the expected results are echoed
+        back to the harness). This is useful for coverage and perf testing
+        of the harness (especially perf testing as it essentially provides
+        a lower bound on how fast the harness can run).
+
+        Also added a --nostart-helper flag to new-run-webkit-tests so that
+        you can skip starting the layout_test_helper and actually run the
+        harness even if you don't have a build of that port.
+
+        Also fix a bug in the 'test' port implementation to actually
+        create the results directory under /tmp instead of /.
+
+        https://bugs.webkit.org/show_bug.cgi?id=35370
+
+        * Scripts/webkitpy/layout_tests/port/factory.py: Modified.
+        * Scripts/webkitpy/layout_tests/port/passing.py: Added.
+        * Scripts/webkitpy/layout_tests/port/test.py: Added.
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py: Modified.
+
 2010-02-25  Eric Seidel  <eric@webkit.org>
 
         Fix typo in my last change.  No review.

WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py

@@ -49,6 +49,9 @@ def get(port_name=None, options=None):
     if port_to_use == 'test':
         import test
         return test.TestPort(port_name, options)
+    elif port_to_use.startswith('passing'):
+        import passing
+        return passing.PassingPort(port_name, options)
     elif port_to_use.startswith('mac'):
         import mac
         return mac.MacPort(port_name, options)

WebKitTools/Scripts/webkitpy/layout_tests/port/passing.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""This is a test implementation of the Port interface that generates the
+   correct output for every test. It can be used for perf testing, because
+   it is pretty much a lower limit on how fast a port can possibly run.
+
+   This implementation acts as a wrapper around a real port (the real port
+   is held as a delegate object). To specify which port, use the port name
+   'passing-XXX' (e.g., 'passing-chromium-mac-leopard'). If you use just
+   'passing', it uses the default port.
+
+   Note that because this is really acting as a wrapper around the underlying
+   port, you must be able to run the underlying port as well
+   (check_sys_deps() must pass and the layout_test_helper must work). You
+   may be able to get around this with --nocheck-sys-deps and --nostart-helper.
+
+   This implementation also modifies the test expectations so that all
+   tests are either SKIPPED or expected to PASS."""
+
+import base
+import factory
+
+class PassingPort(object):
+    """Passing implementation of the Port interface."""
+
+    def __init__(self, port_name=None, options=None):
+        pfx = 'passing-'
+        if port_name.startswith(pfx):
+            port_name = port_name[len(pfx):]
+        else:
+            port_name = None
+        self.__delegate = factory.get(port_name, options)
+
+    def __getattr__(self, name):
+        return getattr(self.__delegate, name)
+
+    def start_driver(self, image_path, options):
+        return PassingDriver(self, image_path, options)
+
+    def test_expectations(self):
+        exps = self.__delegate.test_expectations()
+        skips = []
+        for line in exps.split('\n'):
+            if line.find("SKIP") != -1:
+                skips.append(line)
+        return '\n'.join(skips)
+
+class PassingDriver(base.Driver):
+    """Passing implementation of the DumpRenderTree / Driver interface."""
+
+    def __init__(self, port, image_path, test_driver_options):
+        self._port = port
+        self._driver_options = test_driver_options
+        self._image_path = image_path
+        self._layout_tests_dir = None
+
+    def poll(self):
+        return None
+
+    def returncode(self):
+        return 0
+
+    def run_test(self, uri, timeoutms, image_hash):
+        test_name = self._uri_to_test(uri)
+
+        text_filename = self._port.expected_filename(test_name, '.txt')
+        try:
+            text_output = open(text_filename, 'r').read()
+        except IOError:
+            text_output = ''
+
+        if image_hash:
+            image_filename = self._port.expected_filename(test_name, '.png')
+            image = file(image_filename, 'rb').read()
+            output_file = file(self._image_path, 'w')
+            output_file.write(image)
+            output_file.close()
+            hash_filename = self._port.expected_filename(test_name,
+                '.checksum')
+            hash = file(hash_filename, 'r').read()
+        else:
+            hash = None
+        return (False, False, hash, text_output, None)
+
+    def stop(self):
+        pass
+
+    def _uri_to_test(self, uri):
+        if not self._layout_tests_dir:
+            self._layout_tests_dir = self._port.layout_tests_dir()
+        test = uri
+
+        if uri.startswith("file:///"):
+            test = test.replace('file://', '')
+            return test
+        elif uri.startswith("http://127.0.0.1:8880/"):
+            # websocket tests
+            test = test.replace('http://127.0.0.1:8880/',
+                                self._layout_tests_dir + '/')
+            return test
+        elif uri.startswith("http://"):
+            # regular HTTP test
+            test = test.replace('http://127.0.0.1:8000/',
+                                self._layout_tests_dir + '/http/tests/')
+            return test
+        elif uri.startswith("https://"):
+            test = test.replace('https://127.0.0.1:8443/',
+                                self._layout_tests_dir + '/http/tests/')
+            return test
+        else:
+            raise NotImplementedError('unknown url type: %s' % uri)
+

WebKitTools/Scripts/webkitpy/layout_tests/port/test.py

@@ -76,7 +76,7 @@ def options(self):
         return self._options
 
     def results_directory(self):
-        return '/tmp' + self._options.results_directory
+        return '/tmp/' + self._options.results_directory
 
     def setup_test_run(self):
         pass

WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@ -1468,7 +1468,8 @@ def main(options, args):
     test_runner.parse_expectations(port_obj.test_platform_name(),
                                    options.target == 'Debug')
 
-    port_obj.start_helper()
+    if not options.nostart_helper:
+        port_obj.start_helper()
 
     # Check that the system dependencies (themes, fonts, ...) are correct.
     if (not options.nocheck_sys_deps and
@@ -1490,7 +1491,8 @@ def main(options, args):
     meter.update("Starting ...")
     has_new_failures = test_runner.run(result_summary)
 
-    port_obj.stop_helper()
+    if not options.nostart_helper:
+        port_obj.stop_helper()
 
     logging.debug("Exit status: %d" % has_new_failures)
     sys.exit(has_new_failures)
@@ -1619,6 +1621,9 @@ def parse_args(args=None):
     option_parser.add_option("", "--experimental-fully-parallel",
                              action="store_true", default=False,
                              help="run all tests in parallel")
+    option_parser.add_option("", "--nostart-helper",
+                             action="store_true", default=False,
+                             help="don't run layout_test_helper")
     return option_parser.parse_args(args)
 
 if '__main__' == __name__: