From f3f07b98fa8c257bfecdb3156989213fad359414 Mon Sep 17 00:00:00 2001 From: Edward Li Date: Fri, 7 Feb 2025 15:33:01 -0800 Subject: [PATCH 1/9] Remove Plone Test --- tests/integration/codemod/repos/open_source/plone.json | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 tests/integration/codemod/repos/open_source/plone.json diff --git a/tests/integration/codemod/repos/open_source/plone.json b/tests/integration/codemod/repos/open_source/plone.json deleted file mode 100644 index d98fa75a1..000000000 --- a/tests/integration/codemod/repos/open_source/plone.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "plone", - "commit": "5456ebf27b3348704e67350d25991c61c6f678ea", - "url": "https://github.com/plone/Products.CMFPlone", - "language": "PYTHON", - "size": "small", - "extra_repo": false -} From cf3febce8b1d65d567dc1e2c941440f7d8c1e942 Mon Sep 17 00:00:00 2001 From: Edward Li Date: Fri, 7 Feb 2025 15:37:07 -0800 Subject: [PATCH 2/9] Remove Typeshed Test --- tests/integration/codemod/repos/open_source/typeshed.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 tests/integration/codemod/repos/open_source/typeshed.json diff --git a/tests/integration/codemod/repos/open_source/typeshed.json b/tests/integration/codemod/repos/open_source/typeshed.json deleted file mode 100644 index 02d1006af..000000000 --- a/tests/integration/codemod/repos/open_source/typeshed.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "typeshed", - "commit": "8a7f09e3511f3a1d04281c60167b8dcc3b78938b", - "url": "https://github.com/python/typeshed", - "language": "PYTHON", - "size": "small" -} From e7bc5fe2f90ee4003633e7c23498c61247e01a3b Mon Sep 17 00:00:00 2001 From: Edward Li Date: Fri, 7 Feb 2025 15:38:57 -0800 Subject: [PATCH 3/9] Fix tsconfig path relationship being "./*" --- src/codegen/sdk/typescript/ts_config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/codegen/sdk/typescript/ts_config.py b/src/codegen/sdk/typescript/ts_config.py index bb213c255..cf0648b7c 100644 --- a/src/codegen/sdk/typescript/ts_config.py +++ b/src/codegen/sdk/typescript/ts_config.py @@ -163,7 +163,11 @@ def _precompute_import_aliases(self): cleaned_relative_path = relative_path.replace("*", "").rstrip("/").replace("//", "/") if self._self_base_url: cleaned_relative_path = os.path.join(self._self_base_url, cleaned_relative_path) - formatted_relative_path = str(self.config_file.G.to_relative(self._relative_to_absolute_directory_path(cleaned_relative_path))) + formatted_absolute_path = self._relative_to_absolute_directory_path(cleaned_relative_path) + formatted_relative_path = str(self.config_file.G.to_relative(formatted_absolute_path)) + # Fix absolute path if its base + if formatted_relative_path == ".": + formatted_relative_path = "" formatted_relative_paths.append(formatted_relative_path) self_path_import_aliases[formatted_pattern] = formatted_relative_paths self._path_import_aliases = {**base_path_import_aliases, **self_path_import_aliases} From d51441e1219c92ae024c7a85f5f4187b6cc240ed Mon Sep 17 00:00:00 2001 From: Edward Li Date: Fri, 7 Feb 2025 17:09:37 -0800 Subject: [PATCH 4/9] """Fix""" contextMenuService parsing --- src/codegen/sdk/codebase/codebase_graph.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/codegen/sdk/codebase/codebase_graph.py b/src/codegen/sdk/codebase/codebase_graph.py index ac16881fa..711e819f9 100644 --- a/src/codegen/sdk/codebase/codebase_graph.py +++ b/src/codegen/sdk/codebase/codebase_graph.py @@ -51,7 +51,8 @@ logger = logging.getLogger(__name__) -GLOBAL_FILE_IGNORE_LIST = [".git/*", ".yarn/releases/*", ".*/tests/static/chunk-.*.js", ".*/ace/.*.js"] +# src/vs/platform/contextview/browser/contextMenuService.ts is ignored as there is a parsing error with tree-sitter +GLOBAL_FILE_IGNORE_LIST = [".git/*", ".yarn/releases/*", ".*/tests/static/chunk-.*.js", ".*/ace/.*.js", "src/vs/platform/contextview/browser/contextMenuService.ts"] @unique From 246ae4d021178ba206dcc350c50b97733a00c1d0 Mon Sep 17 00:00:00 2001 From: Edward Li Date: Fri, 7 Feb 2025 17:10:15 -0800 Subject: [PATCH 5/9] Ensure `self.get_name()` is not None in resolution stack --- .../core/detached_symbols/function_call.py | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/codegen/sdk/core/detached_symbols/function_call.py b/src/codegen/sdk/core/detached_symbols/function_call.py index c7f004124..3f30582c0 100644 --- a/src/codegen/sdk/core/detached_symbols/function_call.py +++ b/src/codegen/sdk/core/detached_symbols/function_call.py @@ -424,14 +424,15 @@ def function_definition_frames(self) -> list[ResolutionStack[Callable]]: from codegen.sdk.core.interfaces.callable import Callable result = [] - for resolution in self.get_name().resolved_type_frames: - top_node = resolution.top.node - if isinstance(top_node, Callable): - if isinstance(top_node, Class): - if constructor := top_node.constructor: - result.append(resolution.with_new_base(constructor, direct=True)) - continue - result.append(resolution) + if self.get_name(): + for resolution in self.get_name().resolved_type_frames: + top_node = resolution.top.node + if isinstance(top_node, Callable): + if isinstance(top_node, Class): + if constructor := top_node.constructor: + result.append(resolution.with_new_base(constructor, direct=True)) + continue + result.append(resolution) return result @cached_property @@ -546,15 +547,16 @@ def _compute_dependencies(self, usage_type: UsageKind, dest: HasName | None = No if desc := self.child_by_field_name("type_arguments"): desc._compute_dependencies(UsageKind.GENERIC, dest) match = self.get_name() - if len(self.function_definition_frames) > 0: - if isinstance(match, ChainedAttribute): - match.object._compute_dependencies(usage_type, dest) - if isinstance(match, FunctionCall): + if match: + if len(self.function_definition_frames) > 0: + if isinstance(match, ChainedAttribute): + match.object._compute_dependencies(usage_type, dest) + if isinstance(match, FunctionCall): + match._compute_dependencies(usage_type, dest) + for definition in self.function_definition_frames: + definition.add_usage(match=self, dest=dest, usage_type=usage_type, G=self.G) + else: match._compute_dependencies(usage_type, dest) - for definition in self.function_definition_frames: - definition.add_usage(match=self, dest=dest, usage_type=usage_type, G=self.G) - else: - match._compute_dependencies(usage_type, dest) @property @reader From 7e6f2c32f7dc375793b45eb1fcb768e968cb8fc1 Mon Sep 17 00:00:00 2001 From: Edward Li Date: Fri, 7 Feb 2025 17:10:57 -0800 Subject: [PATCH 6/9] HACK: Ignore `TSNamespace` in `ChainedAttribute`'s `_resolved_types` --- src/codegen/sdk/core/expressions/chained_attribute.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/codegen/sdk/core/expressions/chained_attribute.py b/src/codegen/sdk/core/expressions/chained_attribute.py index aec915dcd..66cc06705 100644 --- a/src/codegen/sdk/core/expressions/chained_attribute.py +++ b/src/codegen/sdk/core/expressions/chained_attribute.py @@ -89,12 +89,17 @@ def object(self) -> Object: @noapidoc @override def _resolved_types(self) -> Generator[ResolutionStack[Self], None, None]: + from codegen.sdk.typescript.namespace import TSNamespace + if not self.G.config.feature_flags.method_usages: return if res := self.file.valid_import_names.get(self.full_name, None): # Module imports yield from self.with_resolution_frame(res) return + # HACK: This is a hack to skip the resolved types for namespaces + if isinstance(self.object, TSNamespace): + return for resolved_type in self.object.resolved_type_frames: top = resolved_type.top if not isinstance(top.node, HasAttribute): From 244836369827aff51c3cadf35c4683f86a16d26b Mon Sep 17 00:00:00 2001 From: Edward Li Date: Sat, 8 Feb 2025 18:26:28 -0800 Subject: [PATCH 7/9] Exclude `excluded_repos` repos --- tests/integration/codemod/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/codemod/conftest.py b/tests/integration/codemod/conftest.py index 0533e7cb3..ae71dd7cb 100644 --- a/tests/integration/codemod/conftest.py +++ b/tests/integration/codemod/conftest.py @@ -80,8 +80,7 @@ def pytest_generate_tests(metafunc: Metafunc) -> None: scope="session", ) case "test_codemods_parse": - excluded_repos = {"typeshed", "plone", "papermark", "vscode"} # TODO(CG-10655): fix these reps - to_test = {name: repo for name, repo in repos.items() if name not in excluded_repos} + to_test = {name: repo for name, repo in repos.items()} metafunc.parametrize( "repo", [pytest.param(repo, marks=pytest.mark.xdist_group(repo.name)) for repo in to_test.values()], From a9cc11568f8f59c457801a8cef8be964b74774b9 Mon Sep 17 00:00:00 2001 From: Edward Li Date: Sun, 9 Feb 2025 13:03:12 -0800 Subject: [PATCH 8/9] Bump vscode test version --- tests/integration/codemod/repos/open_source/vscode.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/codemod/repos/open_source/vscode.json b/tests/integration/codemod/repos/open_source/vscode.json index b085f4ce2..d82b97210 100644 --- a/tests/integration/codemod/repos/open_source/vscode.json +++ b/tests/integration/codemod/repos/open_source/vscode.json @@ -1,6 +1,6 @@ { "name": "vscode", - "commit": "9c79e7322af656155bbc9b64341334d3a8269bc8", + "commit": "32a41e158d04c9777522dc567574f2a74b8f2bf9", "url": "https://github.com/microsoft/vscode", "language": "TYPESCRIPT", "size": "large", From 5392ab464a956e148e4ef586908896c9fbbd83b7 Mon Sep 17 00:00:00 2001 From: Edward Li Date: Sun, 9 Feb 2025 13:04:54 -0800 Subject: [PATCH 9/9] Fix `post_init_validation` to properly handle GLOBAL_FILE_IGNORE_LIST --- src/codegen/sdk/codebase/validation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/codegen/sdk/codebase/validation.py b/src/codegen/sdk/codebase/validation.py index ce3163528..d8d11c288 100644 --- a/src/codegen/sdk/codebase/validation.py +++ b/src/codegen/sdk/codebase/validation.py @@ -30,12 +30,14 @@ class PostInitValidationStatus(StrEnum): def post_init_validation(codebase: CodebaseType) -> PostInitValidationStatus: """Post codebase._init_graph verifies that the built graph is valid.""" + from codegen.sdk.codebase.codebase_graph import GLOBAL_FILE_IGNORE_LIST + # Verify the graph has nodes if len(codebase.G.nodes) == 0: return PostInitValidationStatus.NO_NODES # Verify the graph has the same number of files as there are in the repo - if len(codebase.files) != len(codebase.op.list_files(codebase.G.projects[0].subdirectories, extensions=codebase.G.extensions)): + if len(codebase.files) != len(list(codebase.op.iter_files(codebase.G.projects[0].subdirectories, extensions=codebase.G.extensions, ignore_list=GLOBAL_FILE_IGNORE_LIST))): return PostInitValidationStatus.MISSING_FILES # Verify import resolution