--site-isolation should not run every test as a reference test

https://bugs.webkit.org/show_bug.cgi?id=271298
rdar://125064104

Reviewed by Pascoe and Alex Christensen.

`-—site-isolation` uses `--self-compare-with-header` to compare the output of a test in a cross-origin
iframe with and without site isolation enabled. However, `-—self-compare-with-header` will force each
test to run as a reference test. For testing site isolation, tests that intend to output and compare
text should still do so.

* LayoutTests/platform/mac-site-isolation/TestExpectations:
* Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
(SingleTestRunner):
(SingleTestRunner._run_self_comparison_without_reference_test):
* Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py:
(_set_up_derived_options):

Canonical link: https://commits.webkit.org/276416@main

LayoutTests/platform/mac-site-isolation/TestExpectations

@@ -137,6 +137,7 @@ memory [ Skip ]
 mhtml [ Skip ]
 model-element [ Skip ]
 pageoverlay [ Skip ]
+pdf [ Skip ]
 perf [ Skip ]
 performance-api [ Skip ]
 platform [ Skip ]
@@ -185,37 +186,68 @@ http/tests/site-isolation/window-open-with-name.html [ Skip ]
 # Tests that time out #
 #######################
 http/tests/loading/basic-auth-remove-credentials.html [ Skip ]
+http/tests/loading/cross-origin-XHR-willLoadRequest.html [ Skip ]
 http/tests/loading/main-resource-delegates-on-back-navigation.html [ Skip ]
-http/tests/loading/preload-img-srcset-sizes.html [ Skip ]
-http/tests/loading/preload-script-srcset-ignored.html [ Skip ]
 http/tests/loading/sizes/preload-image-sizes-2x.html [ Skip ]
 http/tests/loading/unfinished-load-back-to-cached-page-callbacks.html [ Skip ]
 http/tests/loading/unfinished-main-resource-back-to-cached-page-callbacks.html [ Skip ]
 http/tests/site-isolation/key-events.html [ Skip ]
-http/tests/site-isolation/mouse-events/context-menu-main-frame-event.html [ Skip ]
-http/tests/site-isolation/mouse-events/mainframe-prevent-default.html [ Skip ]
-http/tests/site-isolation/mouse-events/scrolled-iframe.html [ Skip ]
-http/tests/site-isolation/mouse-events/scrolled-mainframe.html [ Skip ]
+http/tests/site-isolation/mouse-events [ Skip ]
 http/tests/site-isolation/scrolling/basic-scrolling-tree.html [ Skip ]
+http/tests/site-isolation/window-properties.html [ Skip ]
 http/tests/site-isolation/wheel-event-coordinates.html [ Skip ]
 
 ###################
 # Tests that fail #
 ###################
+http/tests/loading/307-after-303-after-post.html [ Skip ]
 http/tests/loading/authentication-after-redirect-stores-wrong-credentials/authentication-after-redirect-stores-wrong-credentials.html [ Skip ]
+http/tests/loading/bad-scheme-subframe.html [ Skip ]
 http/tests/loading/bad-server-subframe.html [ Skip ]
 http/tests/loading/basic-auth-load-URL-with-consecutive-slashes.html [ Skip ]
 http/tests/loading/basic-auth-resend-wrong-credentials.html [ Skip ]
-http/tests/loading/sizes/preload-image-sizes.html [ Skip ]
+http/tests/loading/basic-credentials-sent-automatically.html [ Skip ]
+http/tests/loading/basic.html [ Skip ]
+http/tests/loading/deleted-host-in-resource-load-delegate-callback.html [ Skip ]
+http/tests/loading/do-not-preload-css-blocked-by-csp.html [ Skip ]
+http/tests/loading/do-not-preload-script-src-blocked-by-csp.html [ Skip ]
+http/tests/loading/empty-subframe.html [ Skip ]
 http/tests/loading/fire-error-event-empty-404-script.html [ Skip ]
 http/tests/loading/fire-error-event-script-no-content-type.html [ Skip ]
+http/tests/loading/hidpi-preload-picture-sizes.html [ Skip ]
+http/tests/loading/load-javascript-after-many-xhrs.html [ Skip ]
+http/tests/loading/location-hash-reload-cycle.html [ Skip ]
+http/tests/loading/multiple-prioritization.html [ Skip ]
 http/tests/loading/nested_bad_objects.py [ Skip ]
+http/tests/loading/oauth.html [ Skip ]
+http/tests/loading/onload-vs-immediate-refresh.pl [ Skip ]
+http/tests/loading/pdf-commit-load-callbacks.html [ Skip ]
 http/tests/loading/post-in-iframe-with-back-navigation.html [ Skip ]
+http/tests/loading/preload-append-scan.py [ Skip ]
+http/tests/loading/preload-css-import-layer.html [ Skip ]
 http/tests/loading/preload-css-test.html [ Skip ]
+http/tests/loading/preload-css-with-csp-nonce.html [ Skip ]
+http/tests/loading/preload-ignore-invalid-base.html [ Skip ]
+http/tests/loading/preload-img-src.html [ Skip ]
+http/tests/loading/preload-img-srcset-sizes.html [ Skip ]
+http/tests/loading/preload-img-srcset.html [ Skip ]
+http/tests/loading/preload-img-test.html [ Skip ]
+http/tests/loading/preload-picture-invalid.html [ Skip ]
+http/tests/loading/preload-picture-nested.html [ Skip ]
+http/tests/loading/preload-picture-sizes.html [ Skip ]
+http/tests/loading/preload-script-src-with-csp-nonce.html [ Skip ]
+http/tests/loading/preload-script-src.html [ Skip ]
+http/tests/loading/preload-script-srcset-ignored.html [ Skip ]
+http/tests/loading/preload-slow-loading.py [ Skip ]
+http/tests/loading/progress-finished-callback.html [ Skip ]
+http/tests/loading/redirect-methods.html [ Skip ]
 http/tests/loading/redirect-with-no-location-crash.html [ Skip ]
+http/tests/loading/remove-child-triggers-parser.html [ Skip ]
 http/tests/loading/reusing-cached-stylesheet-from-different-domain.html [ Skip ]
 http/tests/loading/server-redirect-for-provisional-load-caching.html [ Skip ]
 http/tests/loading/simple-subframe.html [ Skip ]
+http/tests/loading/sizes/preload-image-sizes.html [ Skip ]
 http/tests/loading/slow-parsing-subframe.html [ Skip ]
+http/tests/loading/state-object-security-exception.html [ Skip ]
+http/tests/loading/test.html [ Skip ]
 http/tests/loading/text-content-type-with-binary-extension.html [ Skip ]
-http/tests/site-isolation/iframe-process-termination.html [ Skip ]

Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

@@ -114,6 +114,11 @@ def run(self):
         self_comparison_header = self._port.get_option('self_compare_with_header')
         if self_comparison_header:
             return self._run_self_comparison_test(self_comparison_header)
+        if self._options.site_isolation:
+            comparison_header = 'SiteIsolationEnabled=true runInCrossOriginFrame=true'
+            if self._reference_files:
+                return self._run_self_comparison_test(comparison_header)
+            return self._run_self_comparison_without_reference_test(comparison_header)
         if self._reference_files:
             if self._port.get_option('no_ref_tests') or self._options.reset_results:
                 reftest_type = set([reference_file[0] for reference_file in self._reference_files])
@@ -394,6 +399,23 @@ def _run_self_comparison_test(self, header):
         test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory, self._test_name, test_output, reference_output, test_result.failures)
         return TestResult(self._test_input, test_result.failures, test_result.test_run_time, test_result.has_stderr, pid=test_result.pid)
 
+    def _run_self_comparison_without_reference_test(self, header):
+        driver_input = self._driver_input()
+        expected_driver_output = self._driver.run_test(driver_input, self._stop_when_done)
+        driver_input.self_comparison_header = header
+        driver_output = self._driver.run_test(driver_input, self._stop_when_done)
+
+        for output in (expected_driver_output, driver_output):
+            if self._options.ignore_metrics:
+                output.strip_metrics()
+            output.strip_patterns(self._port.logging_patterns_to_strip())
+            output.strip_text_start_if_needed(self._port.logging_detectors_to_strip_text_start(driver_input.test_name))
+            output.strip_stderror_patterns(self._port.stderr_patterns_to_strip())
+
+        test_result = self._compare_output(expected_driver_output, driver_output)
+        test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory, self._test_name, driver_output, expected_driver_output, test_result.failures)
+        return test_result
+
     @staticmethod
     def _relative_reference_path(test_full_path, reference_full_path):
         test_dir = os.path.split(test_full_path)[0]

Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@ -479,7 +479,6 @@ def _set_up_derived_options(port, options):
 
     if port.port_name == "mac" and options.site_isolation:
         options.additional_header = 'runInCrossOriginFrame=true'
-        options.self_compare_with_header = 'SiteIsolationEnabled=true runInCrossOriginFrame=true'
         host = Host()
         host.initialize_scm()
         options.additional_expectations.insert(0, port.host.filesystem.join(host.scm().checkout_root, 'LayoutTests/platform/mac-site-isolation/TestExpectations'))