diff --git a/bench/bench_submit.py b/bench/bench_submit.py index 4376517..d15c2bd 100644 --- a/bench/bench_submit.py +++ b/bench/bench_submit.py @@ -19,10 +19,10 @@ import subprocess import sys import tempfile -from typing import Dict, List, Tuple +from typing import Any, Dict, List, Tuple -def run(args: List[str], cwd: str, **kwargs) -> subprocess.CompletedProcess: +def run(args: List[str], cwd: str, **kwargs: Any) -> subprocess.CompletedProcess[str]: return subprocess.run(args, cwd=cwd, capture_output=True, text=True, **kwargs) diff --git a/mypy.ini b/mypy.ini index b9690d8..496269c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -3,6 +3,7 @@ python_version = 3.9 strict_optional = True show_column_numbers = True show_error_codes = True +enable_error_code = unused-awaitable, unused-coroutine warn_no_return = True disallow_any_unimported = True diff --git a/test_mypy_linter.py b/test_mypy_linter.py new file mode 100644 index 0000000..b41eca2 --- /dev/null +++ b/test_mypy_linter.py @@ -0,0 +1,77 @@ +import json +import subprocess +import sys +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +from tools.linter.adapters.mypy_linter import RESULTS_RE + + +def _check(paths: List[Path]) -> List[Dict[str, Any]]: + repo_root = Path(__file__).parent + proc = subprocess.run( + [ + sys.executable, + "tools/linter/adapters/mypy_linter.py", + "--config=mypy.ini", + "--", + *[str(path) for path in paths], + ], + capture_output=True, + check=True, + cwd=repo_root, + text=True, + ) + return [json.loads(line) for line in proc.stdout.splitlines()] + + +def test_py_test_allows_top_level_await(tmp_path: Path) -> None: + test_file = tmp_path / "ok.py.test" + test_file.write_text( + """\ +async def commit(name: str) -> None: + pass + +await commit("A") +""" + ) + + assert _check([test_file]) == [] + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="mypy does not report this top-level coroutine fixture on Windows", +) +def test_py_test_detects_unawaited_coroutine(tmp_path: Path) -> None: + test_file = tmp_path / "missing_await.py.test" + test_file.write_text( + """\ +async def commit(name: str) -> None: + pass + +commit("A") +""" + ) + + lint_messages = _check([test_file]) + assert [message["name"] for message in lint_messages] == ["[unused-coroutine]"] + assert lint_messages[0]["path"] == str(test_file) + assert lint_messages[0]["line"] == 4 + assert lint_messages[0]["description"] == ( + 'Value of type "Coroutine[Any, Any, None]" must be used ' + ) + + +def test_results_re_parses_windows_drive_paths() -> None: + match = RESULTS_RE.match( + r'C:\tmp\py_test_0.py:4:1: error: Value of type "Coroutine[Any, Any, None]" must be used [unused-coroutine]' + ) + assert match is not None + assert match["file"] == r"C:\tmp\py_test_0.py" + assert match["line"] == "4" + assert match["column"] == "1" + assert match["severity"] == "error" + assert match["code"] == "[unused-coroutine]" diff --git a/tools/linter/adapters/mypy_linter.py b/tools/linter/adapters/mypy_linter.py index 64e5ed1..7b5235f 100644 --- a/tools/linter/adapters/mypy_linter.py +++ b/tools/linter/adapters/mypy_linter.py @@ -5,10 +5,11 @@ import re import subprocess import sys +import tempfile import time from enum import Enum from pathlib import Path -from typing import Any, Dict, List, NamedTuple, Optional, Pattern +from typing import Any, Dict, List, NamedTuple, Optional, Pattern, Tuple IS_WINDOWS: bool = os.name == "nt" @@ -41,11 +42,15 @@ def as_posix(name: str) -> str: return name.replace("\\", "/") if IS_WINDOWS else name +def _path_key(name: str) -> str: + return as_posix(os.path.abspath(name)) + + # tools/linter/flake8_linter.py:15:13: error: Incompatibl...int") [assignment] RESULTS_RE: Pattern[str] = re.compile( r"""(?mx) ^ - (?P.*?): + (?P(?:[A-Za-z]:)?.*?): (?P\d+): (?:(?P-?\d+):)? \s(?P\S+?):? @@ -59,7 +64,7 @@ def as_posix(name: str) -> str: INTERNAL_ERROR_RE: Pattern[str] = re.compile( r"""(?mx) ^ - (?P.*?): + (?P(?:[A-Za-z]:)?.*?): (?P\d+): \s(?P\S+?):? \s(?PINTERNAL\sERROR.*) @@ -121,10 +126,14 @@ def check_files( config: str, retries: int, code: str, + extra_mypy_args: Optional[List[str]] = None, + path_map: Optional[Dict[str, str]] = None, ) -> List[LintMessage]: try: proc = run_command( - [sys.executable, "-mmypy", f"--config={config}"] + filenames, + [sys.executable, "-mmypy", f"--config={config}"] + + (extra_mypy_args or []) + + filenames, extra_env={}, retries=retries, ) @@ -144,9 +153,15 @@ def check_files( ] stdout = str(proc.stdout, "utf-8").strip() stderr = str(proc.stderr, "utf-8").strip() + + def report_path(path: str) -> str: + if path_map is None: + return path + return path_map.get(_path_key(path), path) + rc = [ LintMessage( - path=match["file"], + path=report_path(match["file"]), name=match["code"], description=match["message"], line=int(match["line"]), @@ -163,7 +178,7 @@ def check_files( for match in RESULTS_RE.finditer(stdout) ] + [ LintMessage( - path=match["file"], + path=report_path(match["file"]), name="INTERNAL ERROR", description=match["message"], line=int(match["line"]), @@ -178,6 +193,20 @@ def check_files( return rc +def make_py_test_mypy_inputs( + filenames: List[str], + tmpdir: str, +) -> Tuple[List[str], Dict[str, str]]: + mypy_filenames: List[str] = [] + path_map: Dict[str, str] = {} + for i, filename in enumerate(filenames): + mypy_filename = os.path.join(tmpdir, f"py_test_{i}.py") + Path(mypy_filename).write_text(Path(filename).read_text()) + mypy_filenames.append(mypy_filename) + path_map[_path_key(mypy_filename)] = filename + return mypy_filenames, path_map + + def main() -> None: parser = argparse.ArgumentParser( description="mypy wrapper linter.", @@ -238,9 +267,32 @@ def main() -> None: else: filenames[filename] = True - lint_messages = check_mypy_installed(args.code) + check_files( - list(filenames), args.config, args.retries, args.code - ) + py_filenames = [ + filename for filename in filenames if not filename.endswith(".py.test") + ] + py_test_filenames = [ + filename for filename in filenames if filename.endswith(".py.test") + ] + + lint_messages = check_mypy_installed(args.code) + if py_filenames: + lint_messages += check_files(py_filenames, args.config, args.retries, args.code) + if py_test_filenames: + with tempfile.TemporaryDirectory() as tmpdir: + mypy_filenames, path_map = make_py_test_mypy_inputs( + py_test_filenames, tmpdir + ) + lint_messages += check_files( + mypy_filenames, + args.config, + args.retries, + args.code, + extra_mypy_args=[ + "--no-incremental", + "--disable-error-code=top-level-await", + ], + path_map=path_map, + ) for lint_message in lint_messages: print(json.dumps(lint_message._asdict()), flush=True)