[import-w3c-tests] Get rid of all --src-dir differences

https://bugs.webkit.org/show_bug.cgi?id=260349
rdar://problem/114027584

Reviewed by Jonathan Bedard.

At this point, there are almost no differences left between passing
--src-dir and not: the only difference left is that we implicitly skip
several directories with --src-dir.

If we drop this difference, we can avoid the entire first copying step
when running without --src-dir, as all the downloader has to do is
create the WPT clone to import from, the same as if a WPT clone had been
specified via --src-dir.

* Tools/Scripts/webkitpy/w3c/test_downloader.py:
(TestDownloader.update_import_expectations):
(TestDownloader.download_tests):
(TestDownloader._add_test_suite_paths): Deleted.
(TestDownloader._empty_directory): Deleted.
(TestDownloader.copy_tests): Deleted.
(TestDownloader.copy_tests.longest_path): Deleted.
(TestDownloader.copy_tests.should_copy_dir): Deleted.
(TestDownloader.copy_tests.should_copy_file): Deleted.
* Tools/Scripts/webkitpy/w3c/test_importer.py:
(TestImporter.__init__):
(TestImporter.do_import):
(TestImporter.find_importable_tests):
(TestImporter.find_importable_tests.should_keep_subdir): Deleted.
* Tools/Scripts/webkitpy/w3c/test_importer_unittest.py:

Canonical link: https://commits.webkit.org/267091@main

Tools/Scripts/webkitpy/w3c/test_downloader.py

@@ -110,67 +110,6 @@ def update_import_expectations(self, test_paths):
             import_lines[path.rstrip(self._filesystem.sep)] = 'import'
         self._filesystem.write_text_file(self.import_expectations_path, json.dumps(import_lines, sort_keys=True, indent=4, separators=(',', ': ')))
 
-    def _add_test_suite_paths(self, test_paths, directory, webkit_path):
-        for name in self._filesystem.listdir(directory):
-            original_path = self._filesystem.join(webkit_path, name)
-            if not name.startswith('.'):
-                test_paths.append(original_path)
-
-    def _empty_directory(self, directory):
-        if self._filesystem.exists(directory):
-            self._filesystem.rmtree(directory)
-        self._filesystem.maybe_make_directory(directory)
-
-    def copy_tests(self, destination_directory, test_paths):
-        for test_repository in self.test_repositories:
-            self._empty_directory(self._filesystem.join(destination_directory, test_repository['name']))
-
-        copy_paths = []
-        if test_paths:
-            for path in test_paths:
-                copy_paths.append(path)
-            for path in self.paths_to_import:
-                copy_paths.append(path)
-        else:
-            for test_repository in self.test_repositories:
-                self._add_test_suite_paths(copy_paths, self._filesystem.join(self.repository_directory, test_repository['name']), test_repository['name'])
-            # Handling of tests marked as [ Pass ] in expectations file.
-            for path in self.paths_to_import:
-                if not path in copy_paths:
-                    copy_paths.append(path)
-
-        def longest_path(filesystem, paths):
-            longest_matching_path = ""
-            for path in paths:
-                if path.startswith(longest_matching_path):
-                    longest_matching_path = path
-            return longest_matching_path
-
-        def should_copy_dir(filesystem, directory):
-            relative_path = self._filesystem.relpath(directory, self.repository_directory)
-            if relative_path == ".":
-                return True
-
-            if any(relative_path.startswith(copy_directory) for copy_directory in copy_paths):
-                return True
-
-            return False
-
-        # Compute directories for which we should copy direct children
-        directories_to_copy = self._filesystem.dirs_under(self.repository_directory, should_copy_dir)
-
-        def should_copy_file(filesystem, dirname, filename):
-            full_path = filesystem.join(dirname, filename)
-            relative_path = self._filesystem.relpath(full_path, self.repository_directory)
-            if relative_path in copy_paths:
-                return True
-            return dirname in directories_to_copy
-
-        for source_path in self._filesystem.files_under(self.repository_directory, file_filter=should_copy_file):
-            destination_path = self._filesystem.join(destination_directory, self._filesystem.relpath(source_path, self.repository_directory))
-            self._filesystem.maybe_make_directory(self._filesystem.dirname(destination_path))
-            self._filesystem.copyfile(source_path, destination_path)
-
     def _git_submodules_description(self, test_repository):
         directory = self._filesystem.join(self.repository_directory, test_repository['name'])
 
@@ -214,6 +153,5 @@ def clone_tests(self, use_tip_of_tree=False):
         for test_repository in self.test_repositories:
             self.checkout_test_repository(test_repository['revision'] if not use_tip_of_tree else 'origin/master', test_repository['url'], self._filesystem.join(self.repository_directory, test_repository['name']))
 
-    def download_tests(self, destination_directory, test_paths=[], use_tip_of_tree=False):
+    def download_tests(self, use_tip_of_tree=False):
         self.clone_tests(use_tip_of_tree)
-        self.copy_tests(destination_directory, test_paths)

Tools/Scripts/webkitpy/w3c/test_importer.py

@@ -181,7 +181,6 @@ def __init__(self, host, test_paths, options):
         self._potential_test_resource_files = []
 
         self.import_list = []
-        self._importing_downloaded_tests = self.source_directory is None
 
         self._test_resource_files_json_path = self.filesystem.join(self.layout_tests_w3c_path, "resources", "resource-files.json")
         self._test_resource_files = json.loads(self.filesystem.read_text_file(self._test_resource_files_json_path)) if self.filesystem.exists(self._test_resource_files_json_path) else None
@@ -202,10 +201,9 @@ def __init__(self, host, test_paths, options):
     def do_import(self):
         if not self.source_directory:
             _log.info('Downloading W3C test repositories')
-            self.source_directory = self.filesystem.join(self.tests_download_path, 'to-be-imported')
             self.filesystem.maybe_make_directory(self.tests_download_path)
-            self.filesystem.maybe_make_directory(self.source_directory)
-            self.test_downloader().download_tests(self.source_directory, self.test_paths, self.options.use_tip_of_tree)
+            self.test_downloader().download_tests(self.options.use_tip_of_tree)
+            self.source_directory = self.tests_download_path
 
         for test_path in self.test_paths:
             if test_path != "web-platform-tests" and not test_path.startswith(
@@ -243,8 +241,7 @@ def do_import(self):
         for test_path in test_paths:
             self.remove_dangling_expectations(test_path)
 
-        if self._importing_downloaded_tests:
-            self.generate_git_submodules_description_for_all_repositories()
+        self.generate_git_submodules_description_for_all_repositories()
 
         self.test_downloader().update_import_expectations(self.test_paths)
 
@@ -323,16 +320,8 @@ def _source_root_directory_for_path(self, path):
                 return source_directory
 
     def find_importable_tests(self, directory):
-        def should_keep_subdir(filesystem, path):
-            if self._importing_downloaded_tests:
-                return True
-            subdir = path[len(directory):]
-            DIRS_TO_SKIP = ('work-in-progress', 'tools', 'support')
-            should_skip = filesystem.basename(subdir).startswith('.') or (subdir in DIRS_TO_SKIP)
-            return not should_skip
-
         source_root_directory = self._source_root_directory_for_path(directory)
-        directories = self.filesystem.dirs_under(directory, should_keep_subdir)
+        directories = self.filesystem.dirs_under(directory)
         for root in directories:
             _log.info('Scanning ' + root + '...')
             total_tests = 0

Tools/Scripts/webkitpy/w3c/test_importer_unittest.py

@@ -276,6 +276,32 @@ def test_skip_test_import_source(self):
         self.assertFalse(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/dir-to-skip/dir-to-not-import/test-to-not-import.html'))
         self.assertFalse(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/dir-to-skip/test-to-skip.html'))
 
+    def test_no_implicit_skip_with_download(self):
+        FAKE_FILES = {
+            '/mock-checkout/WebKitBuild/w3c-tests/web-platform-tests/css/css-images/tools/test1.html': MINIMAL_TESTHARNESS,
+            '/mock-checkout/WebKitBuild/w3c-tests/web-platform-tests/css/css-images/support/test2.html': MINIMAL_TESTHARNESS,
+            '/mock-checkout/WebKitBuild/w3c-tests/web-platform-tests/css/css-images/work-in-progress/test3.html': MINIMAL_TESTHARNESS,
+        }
+        FAKE_FILES.update(FAKE_REPOSITORIES)
+
+        fs = self.import_directory(['-d', '/mock-checkout/LayoutTests/w3c/web-platform-tests'], FAKE_FILES, ['web-platform-tests/css/css-images'])
+        self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/web-platform-tests/css/css-images/tools/test1.html'))
+        self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/web-platform-tests/css/css-images/support/test2.html'))
+        self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/web-platform-tests/css/css-images/work-in-progress/test3.html'))
+
+    def test_no_implicit_skip_with_source(self):
+        FAKE_FILES = {
+            '/home/user/wpt/web-platform-tests/css/css-images/tools/test1.html': MINIMAL_TESTHARNESS,
+            '/home/user/wpt/web-platform-tests/css/css-images/support/test2.html': MINIMAL_TESTHARNESS,
+            '/home/user/wpt/web-platform-tests/css/css-images/work-in-progress/test3.html': MINIMAL_TESTHARNESS,
+        }
+        FAKE_FILES.update(FAKE_REPOSITORIES)
+
+        fs = self.import_directory(['-s', '/home/user/wpt', '-d', '/mock-checkout/LayoutTests/w3c/web-platform-tests'], FAKE_FILES, ['web-platform-tests/css/css-images'])
+        self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/web-platform-tests/css/css-images/tools/test1.html'))
+        self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/web-platform-tests/css/css-images/support/test2.html'))
+        self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/web-platform-tests/css/css-images/work-in-progress/test3.html'))
+
     def test_checkout_directory(self):
         FAKE_FILES = {
             '/mock-checkout/WebKitBuild2/w3c-tests/web-platform-tests/existing-test.html': '',
@@ -578,28 +604,18 @@ def test_webkit_test_runner_import_reftests_with_absolute_paths_download(self):
 
         fs = self.import_downloaded_tests(['--no-fetch', '--import-all', '-d', 'w3c'], FAKE_FILES)
         # test1
-        self.assertTrue(fs.exists('/mock-checkout/WebKitBuild/w3c-tests/to-be-imported/csswg-tests/t/test1.html'))
-        self.assertTrue(fs.exists('/mock-checkout/WebKitBuild/w3c-tests/to-be-imported/csswg-tests/t/test1-ref.html'))
         self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/csswg-tests/t/test1.html'))
         self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/csswg-tests/t/test1-expected.html'))
         # test2
-        self.assertTrue(fs.exists('/mock-checkout/WebKitBuild/w3c-tests/to-be-imported/csswg-tests/t/test2.html'))
-        self.assertTrue(fs.exists('/mock-checkout/WebKitBuild/w3c-tests/to-be-imported/csswg-tests/some/directory/in/csswg-root/test2-ref.html'))
         self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/csswg-tests/t/test2.html'))
         self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/csswg-tests/t/test2-expected.html'))
         # test3
-        self.assertTrue(fs.exists('/mock-checkout/WebKitBuild/w3c-tests/to-be-imported/web-platform-tests/css/css-images/test3.html'))
-        self.assertTrue(fs.exists('/mock-checkout/WebKitBuild/w3c-tests/to-be-imported/web-platform-tests/css/css-images/test3-ref.html'))
         self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/css/css-images/test3.html'))
         self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/css/css-images/test3-expected.html'))
         # test4
-        self.assertTrue(fs.exists('/mock-checkout/WebKitBuild/w3c-tests/to-be-imported/web-platform-tests/css/css-images/test4.html'))
-        self.assertTrue(fs.exists('/mock-checkout/WebKitBuild/w3c-tests/to-be-imported/web-platform-tests/some/directory/in/wpt-root/test4-ref.html'))
         self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/css/css-images/test4.html'))
         self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/css/css-images/test4-expected.html'))
         # test5
-        self.assertTrue(fs.exists('/mock-checkout/WebKitBuild/w3c-tests/to-be-imported/web-platform-tests/css/css-images/test5.html'))
-        self.assertTrue(fs.exists('/mock-checkout/WebKitBuild/w3c-tests/to-be-imported/web-platform-tests/some/directory/in/wpt-root/test5-ref.html'))
         self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/css/css-images/test5.html'))
         self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/web-platform-tests/css/css-images/test5-expected.html'))