-
Notifications
You must be signed in to change notification settings - Fork 171
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[RemoveUnusedImports] Support string type annotations #353
Changes from 6 commits
187f249
520abd0
f436658
a763d7c
6d23154
6e6037e
ee6c812
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
# Copyright (c) Facebook, Inc. and its affiliates. | ||
# | ||
# This source code is licensed under the MIT license found in the | ||
# LICENSE file in the root directory of this source tree. | ||
|
||
import re | ||
from typing import Dict, Pattern, Union | ||
|
||
import libcst as cst | ||
import libcst.matchers as m | ||
from libcst.codemod._context import CodemodContext | ||
from libcst.codemod._visitor import ContextAwareVisitor | ||
from libcst.metadata import PositionProvider | ||
|
||
|
||
class GatherCommentsVisitor(ContextAwareVisitor): | ||
METADATA_DEPENDENCIES = (PositionProvider,) | ||
|
||
def __init__(self, context: CodemodContext, comment_regex: str) -> None: | ||
super().__init__(context) | ||
|
||
self.comments: Dict[int, cst.Comment] = {} | ||
|
||
self._comment_matcher: Pattern[str] = re.compile(comment_regex) | ||
|
||
@m.visit(m.EmptyLine(comment=m.DoesNotMatch(None))) | ||
@m.visit(m.TrailingWhitespace(comment=m.DoesNotMatch(None))) | ||
def visit_comment(self, node: Union[cst.EmptyLine, cst.TrailingWhitespace]) -> None: | ||
comment = node.comment | ||
assert comment is not None # hello, type checker | ||
if not self._comment_matcher.match(comment.value): | ||
return | ||
line = self.get_metadata(PositionProvider, comment).start.line | ||
if isinstance(node, cst.EmptyLine): | ||
# Standalone comments refer to the next line | ||
line += 1 | ||
self.comments[line] = comment |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
# Copyright (c) Facebook, Inc. and its affiliates. | ||
# | ||
# This source code is licensed under the MIT license found in the | ||
# LICENSE file in the root directory of this source tree. | ||
|
||
from typing import Set, Union, cast | ||
|
||
import libcst as cst | ||
import libcst.matchers as m | ||
from libcst.codemod._context import CodemodContext | ||
from libcst.codemod._visitor import ContextAwareVisitor | ||
from libcst.metadata import MetadataWrapper, QualifiedNameProvider | ||
|
||
|
||
FUNCS_CONSIDERED_AS_STRING_ANNOTATIONS = {"typing.TypeVar"} | ||
ANNOTATION_MATCHER: m.BaseMatcherNode = m.Annotation() | m.Call( | ||
metadata=m.MatchMetadataIfTrue( | ||
QualifiedNameProvider, | ||
lambda qualnames: any( | ||
qn.name in FUNCS_CONSIDERED_AS_STRING_ANNOTATIONS for qn in qualnames | ||
), | ||
) | ||
) | ||
|
||
|
||
class GatherNamesFromStringAnnotationsVisitor(ContextAwareVisitor): | ||
METADATA_DEPENDENCIES = (QualifiedNameProvider,) | ||
|
||
def __init__(self, context: CodemodContext) -> None: | ||
super().__init__(context) | ||
|
||
self.names: Set[str] = set() | ||
|
||
@m.call_if_inside(ANNOTATION_MATCHER) | ||
@m.visit(m.ConcatenatedString()) | ||
def handle_any_string( | ||
self, node: Union[cst.SimpleString, cst.ConcatenatedString] | ||
) -> None: | ||
value = node.evaluated_value | ||
if value is None: | ||
return | ||
mod = cst.parse_module(value) | ||
extracted_nodes = m.extractall( | ||
mod, | ||
m.Name( | ||
value=m.SaveMatchedNode(m.DoNotCare(), "name"), | ||
metadata=m.MatchMetadataIfTrue( | ||
cst.metadata.ParentNodeProvider, | ||
lambda parent: not isinstance(parent, cst.Attribute), | ||
), | ||
) | ||
| m.SaveMatchedNode(m.Attribute(), "attribute"), | ||
metadata_resolver=MetadataWrapper(mod, unsafe_skip_copy=True), | ||
) | ||
names = { | ||
cast(str, values["name"]) for values in extracted_nodes if "name" in values | ||
} | { | ||
name | ||
for values in extracted_nodes | ||
if "attribute" in values | ||
for name, _ in cst.metadata.scope_provider._gen_dotted_names( | ||
cast(cst.Attribute, values["attribute"]) | ||
) | ||
} | ||
self.names.update(names) | ||
|
||
@m.call_if_inside(ANNOTATION_MATCHER) | ||
@m.call_if_not_inside(m.ConcatenatedString()) | ||
@m.visit(m.SimpleString()) | ||
def handle_simple_string(self, node: cst.SimpleString) -> None: | ||
self.handle_any_string(node) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,96 @@ | ||
# Copyright (c) Facebook, Inc. and its affiliates. | ||
# | ||
# This source code is licensed under the MIT license found in the | ||
# LICENSE file in the root directory of this source tree. | ||
# | ||
|
||
from typing import Iterable, Set, Tuple, Union | ||
|
||
import libcst as cst | ||
import libcst.matchers as m | ||
from libcst.codemod._context import CodemodContext | ||
from libcst.codemod._visitor import ContextAwareVisitor | ||
from libcst.codemod.visitors._gather_exports import GatherExportsVisitor | ||
from libcst.codemod.visitors._gather_string_annotation_names import ( | ||
GatherNamesFromStringAnnotationsVisitor, | ||
) | ||
from libcst.metadata import ProviderT, ScopeProvider | ||
from libcst.metadata.scope_provider import _gen_dotted_names | ||
|
||
|
||
class GatherUnusedImportsVisitor(ContextAwareVisitor): | ||
|
||
METADATA_DEPENDENCIES: Tuple[ProviderT] = ( | ||
*GatherNamesFromStringAnnotationsVisitor.METADATA_DEPENDENCIES, | ||
ScopeProvider, | ||
) | ||
|
||
def __init__(self, context: CodemodContext) -> None: | ||
super().__init__(context) | ||
|
||
self._string_annotation_names: Set[str] = set() | ||
self._exported_names: Set[str] = set() | ||
self.unused_imports: Set[ | ||
Tuple[cst.ImportAlias, Union[cst.Import, cst.ImportFrom]] | ||
] = set() | ||
|
||
def visit_Module(self, node: cst.Module) -> bool: | ||
export_collector = GatherExportsVisitor(self.context) | ||
node.visit(export_collector) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @jimmylai, just wondering... won't calling I was thinking maybe using multiple inheritance instead of this, so we make it that it only traverses the tree once. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes it will traverse the tree multiple times. I don't think that's a big deal for a codemod like this. Running the codemod on a bunch of large files is still not visibly slower than before. I don't have benchmarks for you though :) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If we use this in a linter, which will run over and over, while the user types, we want this as fast as possible, otherwise linting times will hit the experience. Although I’m not sure how expensive traversing the tree is, processing time would be exponential, potentially noticeable on big files. What do you think @jimmylai ? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Performance is a concern for lint but not for codemod. In lint, we try to avoid creating a visitor and calling There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The |
||
self._exported_names = export_collector.explicit_exported_objects | ||
annotation_visitor = GatherNamesFromStringAnnotationsVisitor(self.context) | ||
node.visit(annotation_visitor) | ||
self._string_annotation_names = annotation_visitor.names | ||
return True | ||
|
||
@m.visit( | ||
m.Import() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How could we add other exceptions to this reusable class, like if we wanted to ignore There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it's best to apply filtering on the results of this visitor (just like how you pointed out that the suppression filtering belongs outside of this class). So I would filter for special module names after running There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, I agree :) |
||
| m.ImportFrom( | ||
module=m.DoesNotMatch(m.Name("__future__")), | ||
names=m.DoesNotMatch(m.ImportStar()), | ||
) | ||
) | ||
def handle_import(self, node: Union[cst.Import, cst.ImportFrom]) -> None: | ||
names = node.names | ||
assert not isinstance(names, cst.ImportStar) # hello, type checker | ||
|
||
for alias in names: | ||
self.unused_imports.add((alias, node)) | ||
|
||
def leave_Module(self, original_node: cst.Module) -> None: | ||
self.unused_imports = self.filter_unused_imports(self.unused_imports) | ||
|
||
def filter_unused_imports( | ||
self, | ||
candidates: Iterable[Tuple[cst.ImportAlias, Union[cst.Import, cst.ImportFrom]]], | ||
) -> Set[Tuple[cst.ImportAlias, Union[cst.Import, cst.ImportFrom]]]: | ||
unused_imports = set() | ||
for (alias, parent) in candidates: | ||
scope = self.get_metadata(ScopeProvider, parent) | ||
if scope is None: | ||
continue | ||
if not self.is_in_use(scope, alias): | ||
unused_imports.add((alias, parent)) | ||
return unused_imports | ||
|
||
def is_in_use(self, scope: cst.metadata.Scope, alias: cst.ImportAlias) -> bool: | ||
asname = alias.asname | ||
names = _gen_dotted_names( | ||
cst.ensure_type(asname.name, cst.Name) if asname is not None else alias.name | ||
) | ||
|
||
for name_or_alias, _ in names: | ||
if ( | ||
name_or_alias in self._exported_names | ||
or name_or_alias in self._string_annotation_names | ||
): | ||
return True | ||
|
||
for assignment in scope[name_or_alias]: | ||
if ( | ||
isinstance(assignment, cst.metadata.Assignment) | ||
and isinstance(assignment.node, (cst.ImportFrom, cst.Import)) | ||
and len(assignment.references) > 0 | ||
): | ||
return True | ||
return False |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Also, don't forget about this one as well, I think it's better than the current one:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This doesn't match
# noqa
though, which is the most common caseThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You are right!
What do you think of:
r".*\W(noqa|lint-ignore)(: ?(F401|unused-import)|(?= )|$)(\W.*)?$"