Merge pull request #159 from jamescooke/type-checking

Tighten type-checking: check 3rd party imports

Makefile

@@ -23,12 +23,21 @@ tox:
 
 # --- Tox recipes ---
 
+# Location in `.tox/{envdir}/lib/` of site-packages
+lib_dir = python$$(python --version | grep '.\..' -o)
+
+# Turn on checking for pytest. Extracted as its own recipe for use only when
+# running in tox. E.g. `make lint` works from outside of tox invocation.
+.PHONY: pytyped
+pytyped:
+	touch $$TOXDIR/lib/$(lib_dir)/site-packages/pytest/py.typed $$TOXDIR/lib/$(lib_dir)/site-packages/_pytest/py.typed
+
 .PHONY: lint
 lint:
 	@echo "=== flake8 ==="
 	flake8 $(lint_files)
 	@echo "=== mypy ==="
-	mypy src/flake8_aaa tests --ignore-missing-imports
+	MYPYPATH=stubs mypy src/flake8_aaa tests
 	@echo "=== isort ==="
 	isort --quiet --recursive --diff $(lint_files) > isort.out
 	if [ "$$(wc -l isort.out)" != "0 isort.out" ]; then cat isort.out; exit 1; fi
@@ -39,7 +48,6 @@ lint:
 	@echo "=== setup.py ==="
 	python setup.py check --metadata --strict
 
-
 .PHONY: fixlint
 fixlint:
 	@echo "=== fixing isort ==="

requirements/base.txt

@@ -6,7 +6,7 @@
 #
 asttokens==2.0.4          # via -r base.in
 flake8==3.8.3             # via -r base.in
-importlib-metadata==1.6.1  # via flake8
+importlib-metadata==1.7.0  # via flake8
 mccabe==0.6.1             # via flake8
 pycodestyle==2.6.0        # via flake8
 pyflakes==2.2.0           # via flake8

requirements/dev.txt

@@ -5,14 +5,15 @@
 #    pip-compile --output-file=dev.txt dev.in
 #
 bleach==3.1.5             # via readme-renderer
-certifi==2020.4.5.2       # via -c test.txt, requests
+certifi==2020.6.20        # via -c test.txt, requests
 cffi==1.14.0              # via cryptography
 chardet==3.0.4            # via -c test.txt, requests
 click==7.1.2              # via pip-tools
+colorama==0.4.3           # via twine
 cryptography==2.9.2       # via secretstorage
 docutils==0.16            # via -c test.txt, readme-renderer
-idna==2.9                 # via -c test.txt, requests
-importlib-metadata==1.6.1  # via -c base.txt, -c test.txt, keyring, twine
+idna==2.10                # via -c test.txt, requests
+importlib-metadata==1.7.0  # via -c base.txt, -c test.txt, keyring, twine
 jeepney==0.4.3            # via keyring, secretstorage
 keyring==21.2.1           # via twine
 packaging==20.4           # via -c test.txt, bleach
@@ -23,11 +24,12 @@ pygments==2.6.1           # via -c test.txt, readme-renderer
 pyparsing==2.4.7          # via -c test.txt, packaging
 readme-renderer==26.0     # via twine
 requests-toolbelt==0.9.1  # via twine
-requests==2.23.0          # via -c test.txt, requests-toolbelt, twine
+requests==2.24.0          # via -c test.txt, requests-toolbelt, twine
+rfc3986==1.4.0            # via twine
 secretstorage==3.1.2      # via keyring
 six==1.15.0               # via -c base.txt, -c test.txt, bleach, cryptography, packaging, pip-tools, readme-renderer
 tqdm==4.46.1              # via twine
-twine==3.1.1              # via -r dev.in
+twine==3.2.0              # via -r dev.in
 urllib3==1.25.9           # via -c test.txt, requests
 webencodings==0.5.1       # via bleach
 zipp==3.1.0               # via -c base.txt, -c test.txt, importlib-metadata

requirements/examples.txt

@@ -9,15 +9,15 @@ attrs==19.3.0             # via black, pytest
 black==19.10b0            # via -r examples.in
 click==7.1.2              # via black
 flake8==3.8.3             # via -r examples.in
-importlib-metadata==1.6.1  # via flake8, pluggy, pytest
+importlib-metadata==1.7.0  # via flake8, pluggy, pytest
 mccabe==0.6.1             # via flake8
-more-itertools==8.3.0     # via pytest
+more-itertools==8.4.0     # via pytest
 mypy-extensions==0.4.3    # via mypy
-mypy==0.780               # via -r examples.in
+mypy==0.782               # via -r examples.in
 packaging==20.4           # via pytest
 pathspec==0.8.0           # via black
 pluggy==0.13.1            # via pytest
-py==1.8.1                 # via pytest
+py==1.9.0                 # via pytest
 pycodestyle==2.6.0        # via flake8
 pyflakes==2.2.0           # via flake8
 pyparsing==2.4.7          # via packaging
@@ -27,5 +27,5 @@ six==1.15.0               # via packaging
 toml==0.10.1              # via black
 typed-ast==1.4.1          # via black, mypy
 typing-extensions==3.7.4.2  # via mypy
-wcwidth==0.2.4            # via pytest
+wcwidth==0.2.5            # via pytest
 zipp==3.1.0               # via importlib-metadata

requirements/test.in

@@ -5,7 +5,8 @@ flake8
 isort
 mypy
 pygments
-pytest
+# use pytest master for typing
+git+https://github.com/pytest-dev/pytest@7450b6dd95aa4dfb9c64c2442a173ca65143b442#egg=pytest
 restructuredtext-lint
 six
 sphinx-rtd-theme

requirements/test.txt

@@ -8,51 +8,51 @@ alabaster==0.7.12         # via sphinx
 appdirs==1.4.4            # via virtualenv
 attrs==19.3.0             # via pytest
 babel==2.8.0              # via sphinx
-certifi==2020.4.5.2       # via requests
+certifi==2020.6.20        # via requests
 chardet==3.0.4            # via requests
-distlib==0.3.0            # via virtualenv
+distlib==0.3.1            # via virtualenv
 docutils==0.16            # via restructuredtext-lint, sphinx
 filelock==3.0.12          # via tox, virtualenv
 flake8==3.8.3             # via -c base.txt, -r test.in
-idna==2.9                 # via requests
+idna==2.10                # via requests
 imagesize==1.2.0          # via sphinx
-importlib-metadata==1.6.1  # via -c base.txt, flake8, importlib-resources, pluggy, pytest, tox, virtualenv
-importlib-resources==1.5.0  # via virtualenv
+importlib-metadata==1.7.0  # via -c base.txt, flake8, importlib-resources, pluggy, pytest, tox, virtualenv
+importlib-resources==2.0.1  # via virtualenv
+iniconfig==1.0.0          # via pytest
 isort==4.3.21             # via -r test.in
 jinja2==2.11.2            # via sphinx
 markupsafe==1.1.1         # via jinja2
 mccabe==0.6.1             # via -c base.txt, flake8
-more-itertools==8.3.0     # via pytest
+more-itertools==8.4.0     # via pytest
 mypy-extensions==0.4.3    # via mypy
-mypy==0.780               # via -r test.in
+mypy==0.782               # via -r test.in
 packaging==20.4           # via pytest, sphinx, tox
 pluggy==0.13.1            # via pytest, tox
-py==1.8.1                 # via pytest, tox
+py==1.9.0                 # via pytest, tox
 pycodestyle==2.6.0        # via -c base.txt, flake8
 pyflakes==2.2.0           # via -c base.txt, flake8
 pygments==2.6.1           # via -r test.in, sphinx
 pyparsing==2.4.7          # via packaging
-pytest==5.4.3             # via -r test.in
+git+https://github.com/pytest-dev/pytest@7450b6dd95aa4dfb9c64c2442a173ca65143b442#egg=pytest  # via -r test.in
 pytz==2020.1              # via babel
-requests==2.23.0          # via sphinx
+requests==2.24.0          # via sphinx
 restructuredtext-lint==1.3.1  # via -r test.in
 six==1.15.0               # via -c base.txt, -r test.in, packaging, tox, virtualenv
 snowballstemmer==2.0.0    # via sphinx
-sphinx-rtd-theme==0.4.3   # via -r test.in
-sphinx==3.1.0             # via -r test.in, sphinx-rtd-theme
+sphinx-rtd-theme==0.5.0   # via -r test.in
+sphinx==3.1.1             # via -r test.in, sphinx-rtd-theme
 sphinxcontrib-applehelp==1.0.2  # via sphinx
 sphinxcontrib-devhelp==1.0.2  # via sphinx
 sphinxcontrib-htmlhelp==1.0.3  # via sphinx
 sphinxcontrib-jsmath==1.0.1  # via sphinx
 sphinxcontrib-qthelp==1.0.3  # via sphinx
 sphinxcontrib-serializinghtml==1.1.4  # via sphinx
-toml==0.10.1              # via tox
-tox==3.15.2               # via -r test.in
+toml==0.10.1              # via pytest, tox
+tox==3.16.0               # via -r test.in
 typed-ast==1.4.1          # via mypy
 typing-extensions==3.7.4.2  # via mypy
 urllib3==1.25.9           # via requests
-virtualenv==20.0.21       # via tox
-wcwidth==0.2.4            # via pytest
+virtualenv==20.0.25       # via tox
 yapf==0.30.0              # via -r test.in
 zipp==3.1.0               # via -c base.txt, importlib-metadata, importlib-resources
 

src/flake8_aaa/checker.py

@@ -45,7 +45,7 @@ def all_funcs(self, skip_noqa: bool = False) -> Generator[Function, None, None]:
         if self.ast_tokens is None:
             raise TokensNotLoaded("ast_tokens is `None`")
         for f in find_test_functions(self.tree, skip_noqa=skip_noqa):
-            yield Function(f, self.lines, self.ast_tokens.get_tokens(f, include_extra=True))
+            yield Function(f, self.lines, list(self.ast_tokens.get_tokens(f, include_extra=True)))
 
     def run(self) -> Generator[Tuple[int, int, str, type], None, None]:
         """

src/flake8_aaa/function.py

@@ -2,6 +2,8 @@
 import tokenize
 from typing import Generator, List, Optional
 
+from asttokens.util import Token as ASTToken
+
 from .act_node import ActNode
 from .block import Block
 from .exceptions import AAAError, EmptyBlock, ValidationError
@@ -37,7 +39,7 @@ class Function:
         to the test definition.
     """
 
-    def __init__(self, node: ast.FunctionDef, file_lines: List[str], file_tokens: List[tokenize.TokenInfo]):
+    def __init__(self, node: ast.FunctionDef, file_lines: List[str], file_tokens: List[ASTToken]):
         """
         Args:
             node

stubs/asttokens/__init__.py

@@ -0,0 +1,19 @@
+import ast
+from typing import Generator, Optional
+
+from .util import Token
+
+
+class ASTTokens:
+
+    def __init__(
+        self,
+        source_text: str,
+        parse: bool = False,
+        tree: Optional[ast.AST] = None,
+        filename: str = '<unknown>',
+    ):
+        ...
+
+    def get_tokens(self, node: ast.AST, include_extra: bool = False) -> Generator[Token, None, None]:
+        ...

stubs/asttokens/util.py

@@ -0,0 +1,5 @@
+import collections
+
+
+class Token(collections.namedtuple('Token', 'type string start end line index startpos endpos')):
+    ...

tests/conftest.py

@@ -4,8 +4,8 @@
 
 import asttokens
 import pytest
-from flake8.defaults import MAX_LINE_LENGTH
-from flake8.processor import FileProcessor
+from flake8.defaults import MAX_LINE_LENGTH  # type: ignore
+from flake8.processor import FileProcessor  # type: ignore
 
 
 class FakeOptions:

tox.ini

@@ -47,13 +47,14 @@ commands =
     py38-examples: {[py38-examples]commands}
     examples_aaa: {[examples_aaa]commands}
     install: {[install]commands}
-    lint: make lint
+    lint: make pytyped lint
     test: pytest {posargs:tests}
 skip_install =
     lint: true
     examples: true
 setenv =
     PYTHONWARNINGS = default
+    TOXDIR = {envdir}
 whitelist_externals =
     bash
     diff