From 371219347a6d17e16924bbabf3e693c6874e7138 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 1 Nov 2023 15:33:45 +0000 Subject: [PATCH] Fix file reloading in dmypy with --export-types (#16359) Fixes https://github.com/python/mypy/issues/15794 Unfortunately, this requires to pass `--export-types` to `dmypy run` if one wants to inspect a file that was previously kicked out of the build. --- mypy/dmypy_server.py | 52 +++++++++++++++++++++++++++++++----- mypy/test/testfinegrained.py | 3 ++- test-data/unit/daemon.test | 27 +++++++++++++++++++ 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 0db349b5bf82..42236497f275 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -393,15 +393,21 @@ def cmd_recheck( t1 = time.time() manager = self.fine_grained_manager.manager manager.log(f"fine-grained increment: cmd_recheck: {t1 - t0:.3f}s") - self.options.export_types = export_types + old_export_types = self.options.export_types + self.options.export_types = self.options.export_types or export_types if not self.following_imports(): - messages = self.fine_grained_increment(sources, remove, update) + messages = self.fine_grained_increment( + sources, remove, update, explicit_export_types=export_types + ) else: assert remove is None and update is None - messages = self.fine_grained_increment_follow_imports(sources) + messages = self.fine_grained_increment_follow_imports( + sources, explicit_export_types=export_types + ) res = self.increment_output(messages, sources, is_tty, terminal_width) self.flush_caches() self.update_stats(res) + self.options.export_types = old_export_types return res def check( @@ -412,17 +418,21 @@ def check( If is_tty is True format the output nicely with colors and summary line (unless disabled in self.options). Also pass the terminal_width to formatter. """ - self.options.export_types = export_types + old_export_types = self.options.export_types + self.options.export_types = self.options.export_types or export_types if not self.fine_grained_manager: res = self.initialize_fine_grained(sources, is_tty, terminal_width) else: if not self.following_imports(): - messages = self.fine_grained_increment(sources) + messages = self.fine_grained_increment(sources, explicit_export_types=export_types) else: - messages = self.fine_grained_increment_follow_imports(sources) + messages = self.fine_grained_increment_follow_imports( + sources, explicit_export_types=export_types + ) res = self.increment_output(messages, sources, is_tty, terminal_width) self.flush_caches() self.update_stats(res) + self.options.export_types = old_export_types return res def flush_caches(self) -> None: @@ -535,6 +545,7 @@ def fine_grained_increment( sources: list[BuildSource], remove: list[str] | None = None, update: list[str] | None = None, + explicit_export_types: bool = False, ) -> list[str]: """Perform a fine-grained type checking increment. @@ -545,6 +556,8 @@ def fine_grained_increment( sources: sources passed on the command line remove: paths of files that have been removed update: paths of files that have been changed or created + explicit_export_types: --export-type was passed in a check command + (as opposite to being set in dmypy start) """ assert self.fine_grained_manager is not None manager = self.fine_grained_manager.manager @@ -559,6 +572,10 @@ def fine_grained_increment( # Use the remove/update lists to update fswatcher. # This avoids calling stat() for unchanged files. changed, removed = self.update_changed(sources, remove or [], update or []) + if explicit_export_types: + # If --export-types is given, we need to force full re-checking of all + # explicitly passed files, since we need to visit each expression. + add_all_sources_to_changed(sources, changed) changed += self.find_added_suppressed( self.fine_grained_manager.graph, set(), manager.search_paths ) @@ -577,7 +594,9 @@ def fine_grained_increment( self.previous_sources = sources return messages - def fine_grained_increment_follow_imports(self, sources: list[BuildSource]) -> list[str]: + def fine_grained_increment_follow_imports( + self, sources: list[BuildSource], explicit_export_types: bool = False + ) -> list[str]: """Like fine_grained_increment, but follow imports.""" t0 = time.time() @@ -603,6 +622,9 @@ def fine_grained_increment_follow_imports(self, sources: list[BuildSource]) -> l changed, new_files = self.find_reachable_changed_modules( sources, graph, seen, changed_paths ) + if explicit_export_types: + # Same as in fine_grained_increment(). + add_all_sources_to_changed(sources, changed) sources.extend(new_files) # Process changes directly reachable from roots. @@ -1011,6 +1033,22 @@ def find_all_sources_in_build( return result +def add_all_sources_to_changed(sources: list[BuildSource], changed: list[tuple[str, str]]) -> None: + """Add all (explicit) sources to the list changed files in place. + + Use this when re-processing of unchanged files is needed (e.g. for + the purpose of exporting types for inspections). + """ + changed_set = set(changed) + changed.extend( + [ + (bs.module, bs.path) + for bs in sources + if bs.path and (bs.module, bs.path) not in changed_set + ] + ) + + def fix_module_deps(graph: mypy.build.Graph) -> None: """After an incremental update, update module dependencies to reflect the new state. diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index 953f91a60df7..f61a58c425fc 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -149,6 +149,7 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo options.use_fine_grained_cache = self.use_cache and not build_cache options.cache_fine_grained = self.use_cache options.local_partial_types = True + options.export_types = "inspect" in testcase.file # Treat empty bodies safely for these test cases. options.allow_empty_bodies = not testcase.name.endswith("_no_empty") if re.search("flags:.*--follow-imports", source) is None: @@ -163,7 +164,7 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo return options def run_check(self, server: Server, sources: list[BuildSource]) -> list[str]: - response = server.check(sources, export_types=True, is_tty=False, terminal_width=-1) + response = server.check(sources, export_types=False, is_tty=False, terminal_width=-1) out = response["out"] or response["err"] assert isinstance(out, str) return out.splitlines() diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 77367eb02bfe..ca2c969d2f5e 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -360,6 +360,33 @@ def bar() -> None: x = foo('abc') # type: str foo(arg='xyz') +[case testDaemonInspectCheck] +$ dmypy start +Daemon started +$ dmypy check foo.py +Success: no issues found in 1 source file +$ dmypy check foo.py --export-types +Success: no issues found in 1 source file +$ dmypy inspect foo.py:1:1 +"int" +[file foo.py] +x = 1 + +[case testDaemonInspectRun] +$ dmypy run test1.py +Daemon started +Success: no issues found in 1 source file +$ dmypy run test2.py +Success: no issues found in 1 source file +$ dmypy run test1.py --export-types +Success: no issues found in 1 source file +$ dmypy inspect test1.py:1:1 +"int" +[file test1.py] +a: int +[file test2.py] +a: str + [case testDaemonGetType] $ dmypy start --log-file log.txt -- --follow-imports=error --no-error-summary --python-version 3.8 Daemon started