Skip to content

Commit

Permalink
Make the type inference optional, as its quite slow and CPU intensive.
Browse files Browse the repository at this point in the history
  • Loading branch information
domdfcoding committed Jun 5, 2021
1 parent 76b62cc commit 9497fe4
Show file tree
Hide file tree
Showing 13 changed files with 219 additions and 49 deletions.
2 changes: 1 addition & 1 deletion __pkginfo__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@
]

__version__ = "0.3.5"
extras_require = {}
extras_require = {"classes": ["jedi>=0.18.0"], "all": ["jedi>=0.18.0"]}
13 changes: 13 additions & 0 deletions doc-source/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,19 @@ Installation

.. end installation
.. latex:vspace:: 20px
In version 0.4.0 and above the functionality for checking classes
(:class:`~configparser.ConfigParser` and :class:`~.pathlib.Path` for now)
requires the ``classes`` extra to be installed:

.. prompt:: bash

python3 -m pip install flake8-encodings[classes]

The checks for classes are slower and CPU intensive,
so only enable them if you use the classes in question.


Motivation
-------------
Expand Down
5 changes: 5 additions & 0 deletions doc-source/usage.rst
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ Flake8 codes
ENC012

.. versionadded:: 0.2.0
.. versionchanged:: 0.4.0 These codes now require the classes extra to be installed [1]_.

**ENC02X**: checks for :meth:`pathlib.Path.open`, :meth:`read_text() <pathlib.Path.read_text>` and :meth:`write_text() <pathlib.Path.write_text>`.

Expand All @@ -40,6 +41,10 @@ Flake8 codes
ENC026

.. versionadded:: 0.3.0
.. versionchanged:: 0.4.0 These codes now require the classes extra to be installed [1]_.

.. [1] Install using ``python3 -m pip install flake8-encodings[classes]``
Examples
^^^^^^^^^^
Expand Down
136 changes: 102 additions & 34 deletions flake8_encodings/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,22 +39,26 @@
import configparser
import pathlib
import tempfile
from typing import Callable, Iterator, List, Optional, Tuple, Type
from typing import TYPE_CHECKING, Callable, Iterator, List, Optional, Tuple, Type

# 3rd party
import flake8_helper
import jedi # type: ignore
from astatine import get_attribute_name, kwargs_from_node
from domdf_python_tools.paths import PathPlus
from domdf_python_tools.typing import PathLike

if TYPE_CHECKING:
# 3rd party
from jedi import Script # type: ignore
from jedi.api.classes import Name # type: ignore

__author__: str = "Dominic Davis-Foster"
__copyright__: str = "2020-2021 Dominic Davis-Foster"
__license__: str = "MIT License"
__version__: str = "0.3.5"
__email__: str = "dominic@davis-foster.co.uk"

__all__ = ["Visitor", "Plugin"]
__all__ = ["Visitor", "ClassVisitor", "Plugin"]

ENC001 = "ENC001 no encoding specified for 'open'."
ENC002 = "ENC002 'encoding=None' used for 'open'."
Expand All @@ -71,8 +75,6 @@
ENC025 = "ENC025 no encoding specified for 'pathlib.Path.write_text'."
ENC026 = "ENC026 'encoding=None' used for 'pathlib.Path.write_text'."

jedi.settings.fast_parser = False

_configparser_read = configparser.ConfigParser().read
_pathlib_open = pathlib.Path().open
_pathlib_read_text = pathlib.Path().read_text
Expand All @@ -99,36 +101,19 @@ def mode_is_binary(mode: ast.AST) -> Optional[bool]:
class Visitor(flake8_helper.Visitor):
"""
AST visitor to identify incorrect use of encodings.
"""

def __init__(self):
super().__init__()
self.filename = PathPlus("<unknown>")
self.jedi_script = jedi.Script('')

def first_visit(self, node: ast.AST, filename: PathPlus):
"""
Like :meth:`ast.NodeVisitor.visit`, but configures type inference.
.. versionadded:: 0.2.0
:param node:
:param filename: The path to Python source file the AST node was generated from.
"""
.. versionchanged:: 0.4.0
self.filename = PathPlus(filename)
self.jedi_script = jedi.Script(self.filename.read_text(), path=self.filename)
self.visit(node)
The functionality for checking classes has moved to the :class:`~.ClassVisitor` subclass.
"""

def check_open_encoding(self, node: ast.Call):
"""
Check the call represented by the given AST node is using encodings correctly.
This function checks :func:`open`, :func:`builtins.open <open>` and :func:`io.open`.
.. versionchanged:: 0.2.0
Renamed from ``check_encoding``
.. versionchanged:: 0.2.0 Renamed from ``check_encoding``
"""

kwargs = kwargs_from_node(node, open)
Expand Down Expand Up @@ -156,6 +141,77 @@ def check_open_encoding(self, node: ast.Call):

check_encoding = check_open_encoding # deprecated

def visit_Call(self, node: ast.Call): # noqa: D102

if isinstance(node.func, ast.Name):

if node.func.id == "open":
# print(node.func.id)
self.check_encoding(node)
return

elif isinstance(node.func, ast.Attribute):
if isinstance(node.func.value, ast.Name):

if node.func.value.id in {"builtins", "io"} and node.func.attr == "open":
self.check_open_encoding(node)
return

if isinstance(node.func.value, ast.Str): # pragma: no cover
# Attribute on a string
return self.generic_visit(node)

elif isinstance(node.func.value, ast.BinOp): # pragma: no cover
# TODO
# Expressions such as (tmp_pathplus / "code.py").write_text(example_source)
return self.generic_visit(node)

elif isinstance(node.func.value, ast.Subscript): # pragma: no cover
# TODO
# Expressions such as my_list[0].run()
return self.generic_visit(node)

self.generic_visit(node)


class ClassVisitor(Visitor):
"""
AST visitor to identify incorrect use of encodings,
with support for :class:`pathlib.Path` and :class:`configparser.ConfigParser`.
.. versionadded:: 0.4.0
""" # noqa: D400

def __init__(self):
try:
# 3rd party
import jedi
except ImportError as e:
exc = e.__class__("This class requires 'jedi' to be installed but it could not be imported.")
exc.__traceback__ = e.__traceback__
raise exc from None

super().__init__()
self.filename = PathPlus("<unknown>")
self.jedi_script = jedi.Script('')

def first_visit(self, node: ast.AST, filename: PathPlus):
"""
Like :meth:`ast.NodeVisitor.visit`, but configures type inference.
.. versionadded:: 0.2.0
:param node:
:param filename: The path to Python source file the AST node was generated from.
"""

# 3rd party
import jedi # nodep

self.filename = PathPlus(filename)
self.jedi_script = jedi.Script(self.filename.read_text(), path=self.filename)
self.visit(node)

def check_configparser_encoding(self, node: ast.Call):
"""
Check the call represented by the given AST node is using encodings correctly.
Expand Down Expand Up @@ -292,19 +348,31 @@ def __init__(self, tree: ast.AST, filename: PathLike):

def run(self) -> Iterator[Tuple[int, int, str, Type["Plugin"]]]: # noqa: D102

original_cache_dir = jedi.settings.cache_directory
try:
# 3rd party
import jedi
jedi.settings.fast_parser = False

with tempfile.TemporaryDirectory() as cache_directory:
jedi.settings.cache_directory = cache_directory
original_cache_dir = jedi.settings.cache_directory

with tempfile.TemporaryDirectory() as cache_directory:
jedi.settings.cache_directory = cache_directory

class_visitor = ClassVisitor()
class_visitor.first_visit(self._tree, self.filename)

for line, col, msg in class_visitor.errors:
yield line, col, msg, type(self)

jedi.settings.cache_directory = original_cache_dir

except ImportError:
visitor = Visitor()
visitor.first_visit(self._tree, self.filename)
visitor.visit(self._tree)

for line, col, msg in visitor.errors:
yield line, col, msg, type(self)

jedi.settings.cache_directory = original_cache_dir


def is_configparser_read(class_name: str, method_name: str) -> bool:
"""
Expand Down Expand Up @@ -346,7 +414,7 @@ def is_pathlib_method(class_name: str, method_name: str) -> bool:
return True


def get_inferred_types(jedi_script: jedi.Script, node: ast.Call) -> List[str]:
def get_inferred_types(jedi_script: "Script", node: ast.Call) -> List[str]:
"""
Returns a list of types inferred by ``jedi`` for the given call node.
Expand All @@ -357,7 +425,7 @@ def get_inferred_types(jedi_script: jedi.Script, node: ast.Call) -> List[str]:
attr_names = tuple(get_attribute_name(node.func))
inferred_types = set()

inferred_name: jedi.api.classes.Name
inferred_name: "Name"
for inferred_name in jedi_script.infer(node.lineno, node.func.col_offset):
inferred_types.add(inferred_name.full_name)

Expand Down
4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ Homepage = "https://github.com/domdfcoding/flake8-encodings"
"Source Code" = "https://github.com/domdfcoding/flake8-encodings"
Documentation = "https://flake8-encodings.readthedocs.io/en/latest"

[project.optional-dependencies]
classes = [ "jedi>=0.18.0",]
all = [ "jedi>=0.18.0",]

[tool.whey]
base-classifiers = [
"Development Status :: 4 - Beta",
Expand Down
9 changes: 8 additions & 1 deletion repo_helper.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ version: "0.3.5"
username: "domdfcoding"
license: 'MIT'
short_desc: "A Flake8 plugin to identify incorrect use of encodings."
min_coverage: 100
min_coverage: 94

use_whey: true
python_deploy_version: 3.6
Expand Down Expand Up @@ -53,3 +53,10 @@ keywords:

sphinx_conf_epilogue:
- nitpicky = True

extras_require:
classes:
- jedi>=0.18.0

tox_unmanaged:
- testenv
1 change: 0 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,3 @@ astatine>=0.3.1
domdf-python-tools>=2.8.1
flake8>=3.7
flake8-helper>=0.1.1
jedi>=0.18.0
19 changes: 18 additions & 1 deletion tests/test_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,32 @@
import ast

# 3rd party
import pytest
from coincidence.regressions import AdvancedDataRegressionFixture
from domdf_python_tools.paths import PathPlus

# this package
from flake8_encodings import Plugin
from tests.example_source import example_source

try:
# 3rd party
import jedi # type: ignore
has_jedi = True
except ImportError:
has_jedi = False

def test_plugin(tmp_pathplus: PathPlus, advanced_data_regression: AdvancedDataRegressionFixture):
skip_reason = "Output differs depending on jedi availability"


@pytest.mark.parametrize(
"has_jedi",
[
pytest.param(True, id="has_jedi", marks=pytest.mark.skipif(not has_jedi, reason=skip_reason)),
pytest.param(False, id="no_jedi", marks=pytest.mark.skipif(has_jedi, reason=skip_reason)),
]
)
def test_plugin(tmp_pathplus: PathPlus, advanced_data_regression: AdvancedDataRegressionFixture, has_jedi):
(tmp_pathplus / "code.py").write_text(example_source)

plugin = Plugin(ast.parse(example_source), filename=str(tmp_pathplus / "code.py"))
Expand Down
File renamed without changes.
5 changes: 5 additions & 0 deletions tests/test_plugin_/test_plugin_no_jedi_.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
- '6:9: ENC001 no encoding specified for ''open''.'
- '11:10: ENC001 no encoding specified for ''open''.'
- '12:10: ENC001 no encoding specified for ''open''.'
- '13:10: ENC002 ''encoding=None'' used for ''open''.'
- '23:11: ENC003 no encoding specified for ''open'' with unknown mode.'
37 changes: 35 additions & 2 deletions tests/test_visitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,21 @@
import ast

# 3rd party
import pytest
from coincidence.regressions import AdvancedDataRegressionFixture
from domdf_python_tools.paths import PathPlus

# this package
from flake8_encodings import Visitor
from flake8_encodings import ClassVisitor, Visitor
from tests.example_source import example_source

try:
# 3rd party
import jedi # type: ignore
has_jedi = True
except ImportError:
has_jedi = False


def test_visitor(advanced_data_regression: AdvancedDataRegressionFixture):
visitor = Visitor()
Expand All @@ -17,9 +25,34 @@ def test_visitor(advanced_data_regression: AdvancedDataRegressionFixture):


def test_visitor_with_jedi(tmp_pathplus: PathPlus, advanced_data_regression: AdvancedDataRegressionFixture):
visitor = Visitor()
pytest.importorskip("jedi")

visitor = ClassVisitor()

(tmp_pathplus / "code.py").write_text(example_source)

visitor.first_visit(ast.parse(example_source), filename=tmp_pathplus / "code.py")
advanced_data_regression.check(visitor.errors)


def test_visitor_with_jedi_visit_method(
tmp_pathplus: PathPlus, advanced_data_regression: AdvancedDataRegressionFixture
):
pytest.importorskip("jedi")

visitor = ClassVisitor()

(tmp_pathplus / "code.py").write_text(example_source)

visitor.visit(ast.parse(example_source))
advanced_data_regression.check(visitor.errors)


def test_classvisitor_importerror():
if has_jedi:
pytest.skip(msg="Requires that jedi isn't installed")

with pytest.raises(
ImportError, match="This class requires 'jedi' to be installed but it could not be imported."
):
ClassVisitor()
Loading

0 comments on commit 9497fe4

Please sign in to comment.