From 24f78ebed2a14e6912b07e2bf2e46a3c277847c1 Mon Sep 17 00:00:00 2001 From: Yassine El Khadiri Date: Wed, 2 Jan 2019 18:37:45 +0100 Subject: [PATCH 1/3] Fix sys.stdout overriding in mypy.api Overriding sys.stdout and sys.stderr in mypy.api is not threadsafe. This causes problems sometimes when using the api in pyls for example. --- mypy/api.py | 25 ++++++--------- mypy/build.py | 46 ++++++++++++++++----------- mypy/errors.py | 16 +++++----- mypy/main.py | 72 +++++++++++++++++++++++++----------------- mypy/mypyc_hacks.py | 22 +++++++------ mypy/test/testgraph.py | 6 ++++ mypy/typeshed | 2 +- 7 files changed, 107 insertions(+), 82 deletions(-) diff --git a/mypy/api.py b/mypy/api.py index 7222f943076e..c6664b26fd0c 100644 --- a/mypy/api.py +++ b/mypy/api.py @@ -44,36 +44,31 @@ import sys from io import StringIO -from typing import List, Tuple, Callable +from typing import List, Tuple, Union, TextIO, Callable +from mypy_extensions import DefaultArg -def _run(f: Callable[[], None]) -> Tuple[str, str, int]: - old_stdout = sys.stdout - new_stdout = StringIO() - sys.stdout = new_stdout +def _run(f: Callable[[TextIO, TextIO], None]) -> Tuple[str, str, int]: - old_stderr = sys.stderr - new_stderr = StringIO() - sys.stderr = new_stderr + stdout = StringIO() + stderr = StringIO() try: - f() + f(stdout, stderr) exit_status = 0 except SystemExit as system_exit: exit_status = system_exit.code - finally: - sys.stdout = old_stdout - sys.stderr = old_stderr - return new_stdout.getvalue(), new_stderr.getvalue(), exit_status + return stdout.getvalue(), stderr.getvalue(), exit_status def run(args: List[str]) -> Tuple[str, str, int]: # Lazy import to avoid needing to import all of mypy to call run_dmypy from mypy.main import main - return _run(lambda: main(None, args=args)) + return _run(lambda stdout, stderr: main(None, args=args, + stdout=stdout, stderr=stderr)) def run_dmypy(args: List[str]) -> Tuple[str, str, int]: from mypy.dmypy import main - return _run(lambda: main(args)) + return _run(lambda stdout, stderr: main(args)) diff --git a/mypy/build.py b/mypy/build.py index cfab61320993..a4ce1fe3ba67 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -25,7 +25,7 @@ import types from typing import (AbstractSet, Any, Dict, Iterable, Iterator, List, - Mapping, NamedTuple, Optional, Set, Tuple, Union, Callable) + Mapping, NamedTuple, Optional, Set, Tuple, Union, Callable, TextIO) MYPY = False if MYPY: from typing import ClassVar @@ -128,6 +128,8 @@ def build(sources: List[BuildSource], alt_lib_path: Optional[str] = None, flush_errors: Optional[Callable[[List[str], bool], None]] = None, fscache: Optional[FileSystemCache] = None, + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr, ) -> BuildResult: """Analyze a program. @@ -161,7 +163,7 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None: flush_errors = flush_errors or default_flush_errors try: - result = _build(sources, options, alt_lib_path, flush_errors, fscache) + result = _build(sources, options, alt_lib_path, flush_errors, fscache, stdout, stderr) result.errors = messages return result except CompileError as e: @@ -180,6 +182,8 @@ def _build(sources: List[BuildSource], alt_lib_path: Optional[str], flush_errors: Callable[[List[str], bool], None], fscache: Optional[FileSystemCache], + stdout: TextIO, + stderr: TextIO, ) -> BuildResult: # This seems the most reasonable place to tune garbage collection. gc.set_threshold(150 * 1000) @@ -197,7 +201,7 @@ def _build(sources: List[BuildSource], source_set = BuildSourceSet(sources) errors = Errors(options.show_error_context, options.show_column_numbers) - plugin, snapshot = load_plugins(options, errors) + plugin, snapshot = load_plugins(options, errors, stdout) # Construct a build manager object to hold state during the build. # @@ -212,12 +216,14 @@ def _build(sources: List[BuildSource], plugins_snapshot=snapshot, errors=errors, flush_errors=flush_errors, - fscache=fscache) + fscache=fscache, + stdout=stdout, + stderr=stderr) manager.trace(repr(options)) reset_global_state() try: - graph = dispatch(sources, manager) + graph = dispatch(sources, manager, stdout) if not options.fine_grained_incremental: TypeState.reset_all_subtype_caches() return BuildResult(manager, graph) @@ -319,7 +325,7 @@ def import_priority(imp: ImportBase, toplevel_priority: int) -> int: return toplevel_priority -def load_plugins(options: Options, errors: Errors) -> Tuple[Plugin, Dict[str, str]]: +def load_plugins(options: Options, errors: Errors, stdout: TextIO = sys.stdout) -> Tuple[Plugin, Dict[str, str]]: """Load all configured plugins. Return a plugin that encapsulates all plugins chained together. Always @@ -383,7 +389,7 @@ def plugin_error(message: str) -> None: try: plugin_type = getattr(module, func_name)(__version__) except Exception: - print('Error calling the plugin(version) entry point of {}\n'.format(plugin_path)) + print('Error calling the plugin(version) entry point of {}\n'.format(plugin_path), file=stdout) raise # Propagate to display traceback if not isinstance(plugin_type, type): @@ -398,7 +404,7 @@ def plugin_error(message: str) -> None: custom_plugins.append(plugin_type(options)) snapshot[module_name] = take_module_snapshot(module) except Exception: - print('Error constructing plugin instance of {}\n'.format(plugin_type.__name__)) + print('Error constructing plugin instance of {}\n'.format(plugin_type.__name__), file=stdout) raise # Propagate to display traceback # Custom plugins take precedence over the default plugin. return ChainedPlugin(options, custom_plugins + [default_plugin]), snapshot @@ -496,8 +502,10 @@ def __init__(self, data_dir: str, errors: Errors, flush_errors: Callable[[List[str], bool], None], fscache: FileSystemCache, + stdout: TextIO, + stderr: TextIO, ) -> None: - super().__init__() + super().__init__(stdout, stderr) self.start_time = time.time() self.data_dir = data_dir self.errors = errors @@ -558,7 +566,7 @@ def __init__(self, data_dir: str, self.plugin = plugin self.plugins_snapshot = plugins_snapshot self.old_plugins_snapshot = read_plugins_snapshot(self) - self.quickstart_state = read_quickstart_file(options) + self.quickstart_state = read_quickstart_file(options, self.stdout) def dump_stats(self) -> None: self.log("Stats:") @@ -904,7 +912,7 @@ def read_plugins_snapshot(manager: BuildManager) -> Optional[Dict[str, str]]: return snapshot -def read_quickstart_file(options: Options) -> Optional[Dict[str, Tuple[float, int, str]]]: +def read_quickstart_file(options: Options, stdout: TextIO = sys.stdout) -> Optional[Dict[str, Tuple[float, int, str]]]: quickstart = None # type: Optional[Dict[str, Tuple[float, int, str]]] if options.quickstart_file: # This is very "best effort". If the file is missing or malformed, @@ -918,7 +926,7 @@ def read_quickstart_file(options: Options) -> Optional[Dict[str, Tuple[float, in for file, (x, y, z) in raw_quickstart.items(): quickstart[file] = (x, y, z) except Exception as e: - print("Warning: Failed to load quickstart file: {}\n".format(str(e))) + print("Warning: Failed to load quickstart file: {}\n".format(str(e)), file=stdout) return quickstart @@ -1769,7 +1777,7 @@ def wrap_context(self, check_blockers: bool = True) -> Iterator[None]: except CompileError: raise except Exception as err: - report_internal_error(err, self.path, 0, self.manager.errors, self.options) + report_internal_error(err, self.path, 0, self.manager.errors, self.options, self.manager.stdout, self.manager.stderr) self.manager.errors.set_import_context(save_import_context) # TODO: Move this away once we've removed the old semantic analyzer? if check_blockers: @@ -2429,7 +2437,7 @@ def log_configuration(manager: BuildManager) -> None: # The driver -def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph: +def dispatch(sources: List[BuildSource], manager: BuildManager, stdout: TextIO = sys.stdout) -> Graph: log_configuration(manager) t0 = time.time() @@ -2454,11 +2462,11 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph: fm_cache_size=len(manager.find_module_cache.results), ) if not graph: - print("Nothing to do?!") + print("Nothing to do?!", file=stdout) return graph manager.log("Loaded graph with %d nodes (%.3f sec)" % (len(graph), t1 - t0)) if manager.options.dump_graph: - dump_graph(graph) + dump_graph(graph, stdout) return graph # Fine grained dependencies that didn't have an associated module in the build @@ -2480,7 +2488,7 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph: manager.log("Error reading fine-grained dependencies cache -- aborting cache load") manager.cache_enabled = False manager.log("Falling back to full run -- reloading graph...") - return dispatch(sources, manager) + return dispatch(sources, manager, stdout) # If we are loading a fine-grained incremental mode cache, we # don't want to do a real incremental reprocess of the @@ -2528,7 +2536,7 @@ def dumps(self) -> str: json.dumps(self.deps)) -def dump_graph(graph: Graph) -> None: +def dump_graph(graph: Graph, stdout: TextIO = sys.stdout) -> None: """Dump the graph as a JSON string to stdout. This copies some of the work by process_graph() @@ -2562,7 +2570,7 @@ def dump_graph(graph: Graph) -> None: if (dep_id != node.node_id and (dep_id not in node.deps or pri < node.deps[dep_id])): node.deps[dep_id] = pri - print("[" + ",\n ".join(node.dumps() for node in nodes) + "\n]") + print("[" + ",\n ".join(node.dumps() for node in nodes) + "\n]", file=stdout) def load_graph(sources: List[BuildSource], manager: BuildManager, diff --git a/mypy/errors.py b/mypy/errors.py index 3c4420bf0928..80f5e59470f0 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -3,7 +3,7 @@ import traceback from collections import OrderedDict, defaultdict -from typing import Tuple, List, TypeVar, Set, Dict, Optional +from typing import Tuple, List, TypeVar, Set, Dict, Optional, TextIO from mypy.scope import Scope from mypy.options import Options @@ -586,7 +586,7 @@ def remove_path_prefix(path: str, prefix: Optional[str]) -> str: def report_internal_error(err: Exception, file: Optional[str], line: int, - errors: Errors, options: Options) -> None: + errors: Errors, options: Options, stdout: TextIO = sys.stdout, stderr: TextIO = sys.stderr) -> None: """Report internal error and exit. This optionally starts pdb or shows a traceback. @@ -597,7 +597,7 @@ def report_internal_error(err: Exception, file: Optional[str], line: int, for msg in errors.new_messages(): print(msg) except Exception as e: - print("Failed to dump errors:", repr(e), file=sys.stderr) + print("Failed to dump errors:", repr(e), file=stderr) # Compute file:line prefix for official-looking error messages. if file: @@ -612,11 +612,11 @@ def report_internal_error(err: Exception, file: Optional[str], line: int, print('{}error: INTERNAL ERROR --'.format(prefix), 'please report a bug at https://github.com/python/mypy/issues', 'version: {}'.format(mypy_version), - file=sys.stderr) + file=stderr) # If requested, drop into pdb. This overrides show_tb. if options.pdb: - print('Dropping into pdb', file=sys.stderr) + print('Dropping into pdb', file=stderr) import pdb pdb.post_mortem(sys.exc_info()[2]) @@ -627,15 +627,15 @@ def report_internal_error(err: Exception, file: Optional[str], line: int, if not options.pdb: print('{}: note: please use --show-traceback to print a traceback ' 'when reporting a bug'.format(prefix), - file=sys.stderr) + file=stderr) else: tb = traceback.extract_stack()[:-2] tb2 = traceback.extract_tb(sys.exc_info()[2]) print('Traceback (most recent call last):') for s in traceback.format_list(tb + tb2): print(s.rstrip('\n')) - print('{}: {}'.format(type(err).__name__, err)) - print('{}: note: use --pdb to drop into pdb'.format(prefix), file=sys.stderr) + print('{}: {}'.format(type(err).__name__, err), file=stdout) + print('{}: note: use --pdb to drop into pdb'.format(prefix), file=stderr) # Exit. The caller has nothing more to say. # We use exit code 2 to signal that this is no ordinary error. diff --git a/mypy/main.py b/mypy/main.py index 97abbc483616..5430c5fe39ef 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -10,7 +10,8 @@ import sys import time -from typing import Any, Dict, List, Mapping, Optional, Tuple +from typing import Any, Dict, List, Mapping, Optional, Tuple, TextIO +from io import StringIO from mypy import build from mypy import defaults @@ -46,7 +47,9 @@ def stat_proxy(path: str) -> os.stat_result: return st -def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: +def main(script_path: Optional[str], args: Optional[List[str]] = None, + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr) -> None: """Main entry point to the type checker. Args: @@ -62,13 +65,14 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: args = sys.argv[1:] fscache = FileSystemCache() - sources, options = process_options(args, fscache=fscache) + sources, options = process_options(args, stdout=stdout, stderr=stderr, + fscache=fscache) messages = [] def flush_errors(new_messages: List[str], serious: bool) -> None: messages.extend(new_messages) - f = sys.stderr if serious else sys.stdout + f = stderr if serious else stdout try: for msg in new_messages: f.write(msg + '\n') @@ -82,7 +86,7 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: try: # Keep a dummy reference (res) for memory profiling below, as otherwise # the result could be freed. - res = build.build(sources, options, None, flush_errors, fscache) + res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) except CompileError as e: blockers = True if not e.use_stdout: @@ -92,7 +96,7 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: (options.config_file, ", ".join("[mypy-%s]" % glob for glob in options.per_module_options.keys() if glob in options.unused_configs)), - file=sys.stderr) + file=stderr) if options.junit_xml: t1 = time.time() py_version = '{}_{}'.format(options.python_version[0], options.python_version[1]) @@ -280,6 +284,8 @@ def infer_python_executable(options: Options, def process_options(args: List[str], + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr, require_targets: bool = True, server_options: bool = False, fscache: Optional[FileSystemCache] = None, @@ -708,7 +714,7 @@ def add_invertible_flag(flag: str, # Parse config file first, so command line can override. options = Options() - parse_config_file(options, config_file) + parse_config_file(options, config_file, stdout, stderr) # Set strict flags before parsing (if strict mode enabled), so other command # line options can override. @@ -795,10 +801,11 @@ def add_invertible_flag(flag: str, cache = FindModuleCache(search_paths, fscache) for p in special_opts.packages: if os.sep in p or os.altsep and os.altsep in p: - fail("Package name '{}' cannot have a slash in it.".format(p)) + fail("Package name '{}' cannot have a slash in it.".format(p), + stderr) p_targets = cache.find_modules_recursive(p) if not p_targets: - fail("Can't find package '{}'".format(p)) + fail("Can't find package '{}'".format(p), stderr) targets.extend(p_targets) for m in special_opts.modules: targets.append(BuildSource(None, m, None)) @@ -811,7 +818,7 @@ def add_invertible_flag(flag: str, try: targets = create_source_list(special_opts.files, options, fscache) except InvalidSourceList as e: - fail(str(e)) + fail(str(e), stderr) return targets, options @@ -916,7 +923,9 @@ def split_and_match_files(paths: str) -> List[str]: } # type: Final -def parse_config_file(options: Options, filename: Optional[str]) -> None: +def parse_config_file(options: Options, filename: Optional[str], + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr) -> None: """Parse a config file into an Options object. Errors are written to stderr but are not fatal. @@ -936,7 +945,7 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None: try: parser.read(config_file) except configparser.Error as err: - print("%s: %s" % (config_file, err), file=sys.stderr) + print("%s: %s" % (config_file, err), file=stderr) else: file_read = config_file options.config_file = file_read @@ -946,11 +955,12 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None: if 'mypy' not in parser: if filename or file_read not in defaults.SHARED_CONFIG_FILES: - print("%s: No [mypy] section in config file" % file_read, file=sys.stderr) + print("%s: No [mypy] section in config file" % file_read, file=stderr) else: section = parser['mypy'] prefix = '%s: [%s]' % (file_read, 'mypy') - updates, report_dirs = parse_section(prefix, options, section) + updates, report_dirs = parse_section(prefix, options, section, + stdout, stderr) for k, v in updates.items(): setattr(options, k, v) options.report_dirs.update(report_dirs) @@ -958,15 +968,16 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None: for name, section in parser.items(): if name.startswith('mypy-'): prefix = '%s: [%s]' % (file_read, name) - updates, report_dirs = parse_section(prefix, options, section) + updates, report_dirs = parse_section(prefix, options, section, + stdout, stderr) if report_dirs: print("%s: Per-module sections should not specify reports (%s)" % (prefix, ', '.join(s + '_report' for s in sorted(report_dirs))), - file=sys.stderr) + file=stderr) if set(updates) - PER_MODULE_OPTIONS: print("%s: Per-module sections should only specify per-module flags (%s)" % (prefix, ', '.join(sorted(set(updates) - PER_MODULE_OPTIONS))), - file=sys.stderr) + file=stderr) updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS} globs = name[5:] for glob in globs.split(','): @@ -980,13 +991,16 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None: print("%s: Patterns must be fully-qualified module names, optionally " "with '*' in some components (e.g spam.*.eggs.*)" % prefix, - file=sys.stderr) + file=stderr) else: options.per_module_options[glob] = updates def parse_section(prefix: str, template: Options, - section: Mapping[str, str]) -> Tuple[Dict[str, object], Dict[str, str]]: + section: Mapping[str, str], + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr + ) -> Tuple[Dict[str, object], Dict[str, str]]: """Parse one section of a config file. Returns a dict of option values encountered, and a dict of report directories. @@ -1005,17 +1019,17 @@ def parse_section(prefix: str, template: Options, report_dirs[report_type] = section[key] else: print("%s: Unrecognized report type: %s" % (prefix, key), - file=sys.stderr) + file=stderr) continue if key.startswith('x_'): continue # Don't complain about `x_blah` flags elif key == 'strict': print("%s: Strict mode is not supported in configuration files: specify " "individual flags instead (see 'mypy -h' for the list of flags enabled " - "in strict mode)" % prefix, file=sys.stderr) + "in strict mode)" % prefix, file=stderr) else: print("%s: Unrecognized option: %s = %s" % (prefix, key, section[key]), - file=sys.stderr) + file=stderr) continue ct = type(dv) v = None # type: Any @@ -1026,19 +1040,19 @@ def parse_section(prefix: str, template: Options, try: v = ct(section.get(key)) except argparse.ArgumentTypeError as err: - print("%s: %s: %s" % (prefix, key, err), file=sys.stderr) + print("%s: %s: %s" % (prefix, key, err), file=stderr) continue else: - print("%s: Don't know what type %s should have" % (prefix, key), file=sys.stderr) + print("%s: Don't know what type %s should have" % (prefix, key), file=stderr) continue except ValueError as err: - print("%s: %s: %s" % (prefix, key, err), file=sys.stderr) + print("%s: %s: %s" % (prefix, key, err), file=stderr) continue if key == 'cache_dir': v = os.path.expanduser(v) if key == 'silent_imports': print("%s: silent_imports has been replaced by " - "ignore_missing_imports=True; follow_imports=skip" % prefix, file=sys.stderr) + "ignore_missing_imports=True; follow_imports=skip" % prefix, file=stderr) if v: if 'ignore_missing_imports' not in results: results['ignore_missing_imports'] = True @@ -1046,7 +1060,7 @@ def parse_section(prefix: str, template: Options, results['follow_imports'] = 'skip' if key == 'almost_silent': print("%s: almost_silent has been replaced by " - "follow_imports=error" % prefix, file=sys.stderr) + "follow_imports=error" % prefix, file=stderr) if v: if 'follow_imports' not in results: results['follow_imports'] = 'error' @@ -1054,6 +1068,6 @@ def parse_section(prefix: str, template: Options, return results, report_dirs -def fail(msg: str) -> None: - sys.stderr.write('%s\n' % msg) +def fail(msg: str, stderr: TextIO) -> None: + stderr.write('%s\n' % msg) sys.exit(2) diff --git a/mypy/mypyc_hacks.py b/mypy/mypyc_hacks.py index 7cd12035daff..f709ee9d4da6 100644 --- a/mypy/mypyc_hacks.py +++ b/mypy/mypyc_hacks.py @@ -1,13 +1,15 @@ """Stuff that we had to move out of its right place because of mypyc limitations.""" -from typing import Dict, Any +from typing import Dict, Any, TextIO import sys # Extracted from build.py because we can't handle *args righit class BuildManagerBase: - def __init__(self) -> None: + def __init__(self, stdout: TextIO, stderr: TextIO) -> None: self.stats = {} # type: Dict[str, Any] # Values are ints or floats + self.stdout = stdout + self.stderr = stderr def verbosity(self) -> int: return self.options.verbosity # type: ignore @@ -15,10 +17,10 @@ def verbosity(self) -> int: def log(self, *message: str) -> None: if self.verbosity() >= 1: if message: - print('LOG: ', *message, file=sys.stderr) + print('LOG: ', *message, file=self.stderr) else: - print(file=sys.stderr) - sys.stderr.flush() + print(file=self.stderr) + self.stderr.flush() def log_fine_grained(self, *message: str) -> None: import mypy.build @@ -27,15 +29,15 @@ def log_fine_grained(self, *message: str) -> None: elif mypy.build.DEBUG_FINE_GRAINED: # Output log in a simplified format that is quick to browse. if message: - print(*message, file=sys.stderr) + print(*message, file=self.stderr) else: - print(file=sys.stderr) - sys.stderr.flush() + print(file=self.stderr) + self.stderr.flush() def trace(self, *message: str) -> None: if self.verbosity() >= 2: - print('TRACE:', *message, file=sys.stderr) - sys.stderr.flush() + print('TRACE:', *message, file=self.stderr) + self.stderr.flush() def add_stats(self, **kwds: Any) -> None: for key, value in kwds.items(): diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index 8cff9c1c1bf7..18263ef73356 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -2,6 +2,8 @@ from typing import AbstractSet, Dict, Set, List +from io import StringIO + from mypy.test.helpers import assert_equal, Suite from mypy.build import BuildManager, State, BuildSourceSet from mypy.modulefinder import SearchPaths @@ -42,6 +44,8 @@ def _make_manager(self) -> BuildManager: options = Options() fscache = FileSystemCache() search_paths = SearchPaths((), (), (), ()) + stdout = StringIO() + stderr = StringIO() manager = BuildManager( data_dir='', search_paths=search_paths, @@ -55,6 +59,8 @@ def _make_manager(self) -> BuildManager: errors=errors, flush_errors=lambda msgs, serious: None, fscache=fscache, + stdout=stdout, + stderr=stderr, ) return manager diff --git a/mypy/typeshed b/mypy/typeshed index 36b28e5b97ef..f7c00b8b3300 160000 --- a/mypy/typeshed +++ b/mypy/typeshed @@ -1 +1 @@ -Subproject commit 36b28e5b97efd0f2573aafcec4836f287bf5be5f +Subproject commit f7c00b8b33008de74d291f4a5084b129eb2eece6 From 51de94d8037dd84a09936fd9e75d52638ffd52d6 Mon Sep 17 00:00:00 2001 From: Amanda Walker Date: Tue, 14 May 2019 10:44:25 -0400 Subject: [PATCH 2/3] Reformat to keep files within line length limits --- mypy/build.py | 24 ++++++++++++++++++------ mypy/errors.py | 10 ++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index a4ce1fe3ba67..0f6d729e0888 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -325,7 +325,11 @@ def import_priority(imp: ImportBase, toplevel_priority: int) -> int: return toplevel_priority -def load_plugins(options: Options, errors: Errors, stdout: TextIO = sys.stdout) -> Tuple[Plugin, Dict[str, str]]: +def load_plugins( + options: Options, + errors: Errors, + stdout: TextIO = sys.stdout +) -> Tuple[Plugin, Dict[str, str]]: """Load all configured plugins. Return a plugin that encapsulates all plugins chained together. Always @@ -389,7 +393,8 @@ def plugin_error(message: str) -> None: try: plugin_type = getattr(module, func_name)(__version__) except Exception: - print('Error calling the plugin(version) entry point of {}\n'.format(plugin_path), file=stdout) + print('Error calling the plugin(version) entry point of {}\n'.format(plugin_path), + file=stdout) raise # Propagate to display traceback if not isinstance(plugin_type, type): @@ -404,7 +409,8 @@ def plugin_error(message: str) -> None: custom_plugins.append(plugin_type(options)) snapshot[module_name] = take_module_snapshot(module) except Exception: - print('Error constructing plugin instance of {}\n'.format(plugin_type.__name__), file=stdout) + print('Error constructing plugin instance of {}\n'.format(plugin_type.__name__), + file=stdout) raise # Propagate to display traceback # Custom plugins take precedence over the default plugin. return ChainedPlugin(options, custom_plugins + [default_plugin]), snapshot @@ -912,7 +918,9 @@ def read_plugins_snapshot(manager: BuildManager) -> Optional[Dict[str, str]]: return snapshot -def read_quickstart_file(options: Options, stdout: TextIO = sys.stdout) -> Optional[Dict[str, Tuple[float, int, str]]]: +def read_quickstart_file(options: Options, + stdout: TextIO = sys.stdout + ) -> Optional[Dict[str, Tuple[float, int, str]]]: quickstart = None # type: Optional[Dict[str, Tuple[float, int, str]]] if options.quickstart_file: # This is very "best effort". If the file is missing or malformed, @@ -1777,7 +1785,8 @@ def wrap_context(self, check_blockers: bool = True) -> Iterator[None]: except CompileError: raise except Exception as err: - report_internal_error(err, self.path, 0, self.manager.errors, self.options, self.manager.stdout, self.manager.stderr) + report_internal_error(err, self.path, 0, self.manager.errors, + self.options, self.manager.stdout, self.manager.stderr) self.manager.errors.set_import_context(save_import_context) # TODO: Move this away once we've removed the old semantic analyzer? if check_blockers: @@ -2437,7 +2446,10 @@ def log_configuration(manager: BuildManager) -> None: # The driver -def dispatch(sources: List[BuildSource], manager: BuildManager, stdout: TextIO = sys.stdout) -> Graph: +def dispatch(sources: List[BuildSource], + manager: BuildManager, + stdout: TextIO = sys.stdout + ) -> Graph: log_configuration(manager) t0 = time.time() diff --git a/mypy/errors.py b/mypy/errors.py index 80f5e59470f0..02bba1d217d1 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -585,8 +585,14 @@ def remove_path_prefix(path: str, prefix: Optional[str]) -> str: return path -def report_internal_error(err: Exception, file: Optional[str], line: int, - errors: Errors, options: Options, stdout: TextIO = sys.stdout, stderr: TextIO = sys.stderr) -> None: +def report_internal_error(err: Exception, + file: Optional[str], + line: int, + errors: Errors, + options: Options, + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr + ) -> None: """Report internal error and exit. This optionally starts pdb or shows a traceback. From 3a5432ce6cd93a7aa2435a0aaf288cb385ddda27 Mon Sep 17 00:00:00 2001 From: Amanda Walker Date: Tue, 14 May 2019 20:49:04 -0400 Subject: [PATCH 3/3] Stop setting stdout and stderr at module load time --- mypy/__main__.py | 5 +++-- mypy/api.py | 4 ++-- mypy/build.py | 19 ++++++++++--------- mypy/dmypy_server.py | 4 +++- mypy/errors.py | 6 ++++-- mypy/main.py | 8 +++++--- mypy/test/testgraph.py | 7 +++---- 7 files changed, 30 insertions(+), 23 deletions(-) diff --git a/mypy/__main__.py b/mypy/__main__.py index 625242d100be..c5e42c63a633 100644 --- a/mypy/__main__.py +++ b/mypy/__main__.py @@ -1,11 +1,12 @@ """Mypy type checker command line tool.""" +import sys from mypy.main import main def console_entry() -> None: - main(None) + main(None, sys.stdout, sys.stderr) if __name__ == '__main__': - main(None) + main(None, sys.stdout, sys.stderr) diff --git a/mypy/api.py b/mypy/api.py index c6664b26fd0c..5f7866a0adc2 100644 --- a/mypy/api.py +++ b/mypy/api.py @@ -48,13 +48,13 @@ from mypy_extensions import DefaultArg -def _run(f: Callable[[TextIO, TextIO], None]) -> Tuple[str, str, int]: +def _run(main_wrapper: Callable[[TextIO, TextIO], None]) -> Tuple[str, str, int]: stdout = StringIO() stderr = StringIO() try: - f(stdout, stderr) + main_wrapper(stdout, stderr) exit_status = 0 except SystemExit as system_exit: exit_status = system_exit.code diff --git a/mypy/build.py b/mypy/build.py index 0f6d729e0888..8de60680f862 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -128,8 +128,8 @@ def build(sources: List[BuildSource], alt_lib_path: Optional[str] = None, flush_errors: Optional[Callable[[List[str], bool], None]] = None, fscache: Optional[FileSystemCache] = None, - stdout: TextIO = sys.stdout, - stderr: TextIO = sys.stderr, + stdout: Optional[TextIO] = None, + stderr: Optional[TextIO] = None, ) -> BuildResult: """Analyze a program. @@ -161,6 +161,8 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None: messages.extend(new_messages) flush_errors = flush_errors or default_flush_errors + stdout = stdout or sys.stdout + stderr = stderr or sys.stderr try: result = _build(sources, options, alt_lib_path, flush_errors, fscache, stdout, stderr) @@ -325,11 +327,10 @@ def import_priority(imp: ImportBase, toplevel_priority: int) -> int: return toplevel_priority -def load_plugins( - options: Options, - errors: Errors, - stdout: TextIO = sys.stdout -) -> Tuple[Plugin, Dict[str, str]]: +def load_plugins(options: Options, + errors: Errors, + stdout: TextIO, + ) -> Tuple[Plugin, Dict[str, str]]: """Load all configured plugins. Return a plugin that encapsulates all plugins chained together. Always @@ -919,7 +920,7 @@ def read_plugins_snapshot(manager: BuildManager) -> Optional[Dict[str, str]]: def read_quickstart_file(options: Options, - stdout: TextIO = sys.stdout + stdout: TextIO, ) -> Optional[Dict[str, Tuple[float, int, str]]]: quickstart = None # type: Optional[Dict[str, Tuple[float, int, str]]] if options.quickstart_file: @@ -2448,7 +2449,7 @@ def log_configuration(manager: BuildManager) -> None: def dispatch(sources: List[BuildSource], manager: BuildManager, - stdout: TextIO = sys.stdout + stdout: TextIO, ) -> Graph: log_configuration(manager) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 54f85f3bfc28..d07df55767e7 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -308,7 +308,9 @@ def cmd_run(self, version: str, args: Sequence[str]) -> Dict[str, object]: if self.fine_grained_manager: manager = self.fine_grained_manager.manager start_plugins_snapshot = manager.plugins_snapshot - _, current_plugins_snapshot = mypy.build.load_plugins(options, manager.errors) + _, current_plugins_snapshot = mypy.build.load_plugins(options, + manager.errors, + sys.stdout) if current_plugins_snapshot != start_plugins_snapshot: return {'restart': 'plugins changed'} except InvalidSourceList as err: diff --git a/mypy/errors.py b/mypy/errors.py index 02bba1d217d1..5937c2fe9e40 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -590,13 +590,15 @@ def report_internal_error(err: Exception, line: int, errors: Errors, options: Options, - stdout: TextIO = sys.stdout, - stderr: TextIO = sys.stderr + stdout: Optional[TextIO] = None, + stderr: Optional[TextIO] = None, ) -> None: """Report internal error and exit. This optionally starts pdb or shows a traceback. """ + stdout = (stdout or sys.stdout) + stderr = (stderr or sys.stderr) # Dump out errors so far, they often provide a clue. # But catch unexpected errors rendering them. try: diff --git a/mypy/main.py b/mypy/main.py index 5430c5fe39ef..f3a02ed5caf6 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -47,9 +47,11 @@ def stat_proxy(path: str) -> os.stat_result: return st -def main(script_path: Optional[str], args: Optional[List[str]] = None, - stdout: TextIO = sys.stdout, - stderr: TextIO = sys.stderr) -> None: +def main(script_path: Optional[str], + stdout: TextIO, + stderr: TextIO, + args: Optional[List[str]] = None, + ) -> None: """Main entry point to the type checker. Args: diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index 18263ef73356..ca005f5fd928 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -1,5 +1,6 @@ """Test cases for graph processing code in build.py.""" +import sys from typing import AbstractSet, Dict, Set, List from io import StringIO @@ -44,8 +45,6 @@ def _make_manager(self) -> BuildManager: options = Options() fscache = FileSystemCache() search_paths = SearchPaths((), (), (), ()) - stdout = StringIO() - stderr = StringIO() manager = BuildManager( data_dir='', search_paths=search_paths, @@ -59,8 +58,8 @@ def _make_manager(self) -> BuildManager: errors=errors, flush_errors=lambda msgs, serious: None, fscache=fscache, - stdout=stdout, - stderr=stderr, + stdout=sys.stdout, + stderr=sys.stderr, ) return manager