Skip to content

Commit

Permalink
Merge e41a71d into b4b6fd1
Browse files Browse the repository at this point in the history
  • Loading branch information
@AWhetter
AWhetter committed Jun 25, 2016
2 parents b4b6fd1 + e41a71d commit 1b40746
Show file tree
Hide file tree
Showing 2 changed files with 176 additions and 128 deletions.
251 changes: 128 additions & 123 deletions pylint/lint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ def _patch_sysmodules():


if multiprocessing is not None:
class ChildLinter(multiprocessing.Process):
class ChildRunner(multiprocessing.Process):
def run(self):
# pylint: disable=no-member, unbalanced-tuple-unpacking
tasks_queue, results_queue, self._config = self._args
Expand Down Expand Up @@ -712,116 +712,23 @@ def should_analyze_file(self, modname, path): # pylint: disable=unused-argument,
"""
return path.endswith('.py')

def _init_msg_states(self):
for msg in self.msgs_store.messages:
if not msg.may_be_emitted():
self._msgs_state[msg.msgid] = False

def check(self, files_or_modules):
"""main checking entry: check a list of files or modules from their
name.
"""
# initialize msgs_state now that all messages have been registered into
# the store
for msg in self.msgs_store.messages:
if not msg.may_be_emitted():
self._msgs_state[msg.msgid] = False
self._init_msg_states()

if not isinstance(files_or_modules, (list, tuple)):
files_or_modules = (files_or_modules,)

if self.config.jobs == 1:
self._do_check(files_or_modules)
else:
with _patch_sysmodules():
self._parallel_check(files_or_modules)

def _get_jobs_config(self):
child_config = collections.OrderedDict()
filter_options = {'long-help'}
filter_options.update((opt_name for opt_name, _ in self._external_opts))
for opt_providers in six.itervalues(self._all_options):
for optname, optdict, val in opt_providers.options_and_values():
if optdict.get('deprecated'):
continue

if optname not in filter_options:
child_config[optname] = utils._format_option_value(
optdict, val)
child_config['python3_porting_mode'] = self._python3_porting_mode
child_config['plugins'] = self._dynamic_plugins
return child_config

def _parallel_task(self, files_or_modules):
# Prepare configuration for child linters.
child_config = self._get_jobs_config()

children = []
manager = multiprocessing.Manager()
tasks_queue = manager.Queue()
results_queue = manager.Queue()

for _ in range(self.config.jobs):
child_linter = ChildLinter(args=(tasks_queue, results_queue,
child_config))
child_linter.start()
children.append(child_linter)

# Send files to child linters.
expanded_files = self.expand_files(files_or_modules)
for files_or_module in expanded_files:
path = files_or_module['path']
tasks_queue.put([path])

# collect results from child linters
failed = False
for _ in expanded_files:
try:
result = results_queue.get()
except Exception as ex:
print("internal error while receiving results from child linter",
file=sys.stderr)
print(ex, file=sys.stderr)
failed = True
break
yield result

# Stop child linters and wait for their completion.
for _ in range(self.config.jobs):
tasks_queue.put('STOP')
for child in children:
child.join()

if failed:
print("Error occured, stopping the linter.", file=sys.stderr)
sys.exit(32)

def _parallel_check(self, files_or_modules):
# Reset stats.
self.open()

all_stats = []
module = None
for result in self._parallel_task(files_or_modules):
(
_,
self.file_state.base_name,
module,
messages,
stats,
msg_status
) = result

for msg in messages:
msg = utils.Message(*msg)
self.set_current_module(module)
self.reporter.handle_message(msg)

all_stats.append(stats)
self.msg_status |= msg_status

self.stats = _merge_stats(all_stats)
self.current_name = module

# Insert stats data to local checkers.
for checker in self.get_checkers():
if checker is not self:
checker.stats = self.stats
self._do_check(files_or_modules)

def _do_check(self, files_or_modules):
walker = utils.PyLintASTWalker(self)
Expand All @@ -837,9 +744,16 @@ def _do_check(self, files_or_modules):
if interfaces.implements(checker, interfaces.IAstroidChecker):
walker.add_checker(checker)
# build ast and check modules or packages
for descr in self.expand_files(files_or_modules):
modname, filepath = descr['name'], descr['path']
if not descr['isarg'] and not self.should_analyze_file(modname, filepath):
expanded_files = utils.expand_files(
files_or_modules,
self,
self.config.black_list,
self.config.black_list_re
)
for module_desc in expanded_files:
modname = module_desc.name
filepath = module_desc.path
if not module_desc.isarg and not self.should_analyze_file(modname, filepath):
continue
if self.config.files_output:
reportfile = 'pylint_%s.%s' % (modname, self.reporter.extension)
Expand All @@ -852,7 +766,7 @@ def _do_check(self, files_or_modules):
# XXX to be correct we need to keep module_msgs_state for every
# analyzed module (the problem stands with localized messages which
# are only detected in the .close step)
self.file_state = utils.FileState(descr['basename'])
self.file_state = utils.FileState(module_desc.basename)
self._ignore_file = False
# fix the current file (if the source file was not available or
# if it's actually a c extension)
Expand All @@ -867,20 +781,6 @@ def _do_check(self, files_or_modules):
for checker in reversed(_checkers):
checker.close()

def expand_files(self, modules):
"""get modules and errors from a list of modules and handle errors
"""
result, errors = utils.expand_modules(modules, self.config.black_list,
self.config.black_list_re)
for error in errors:
message = modname = error["mod"]
key = error["key"]
self.set_current_module(modname)
if key == "fatal":
message = str(error["ex"]).replace(os.getcwd() + os.sep, '')
self.add_message(key, args=message)
return result

def set_current_module(self, modname, filepath=None):
"""set the name of the currently analyzed module and
init statistics for it
Expand Down Expand Up @@ -1299,18 +1199,123 @@ def __init__(self, args, reporter=None, exit=True):
print("Multiprocessing library is missing, "
"fallback to single process", file=sys.stderr)
linter.set_option("jobs", 1)
else:
if linter.config.jobs == 0:
linter.config.jobs = multiprocessing.cpu_count()
elif linter.config.jobs == 0:
linter.config.jobs = multiprocessing.cpu_count()

# insert current working directory to the python path to have a correct
# behaviour
with fix_import_path(args):
linter.check(args)
if linter.config.jobs == 1:
linter.check(args)
else:
self._parallel_run(args)

linter.generate_reports()

if exit:
sys.exit(self.linter.msg_status)

def _parallel_run(self, files_or_modules):
with _patch_sysmodules():
self.linter._init_msg_states()
self._parallel_check(files_or_modules)

def _parallel_task(self, files_or_modules):
# Prepare configuration for child linters.
child_config = self._get_jobs_config()

children = []
manager = multiprocessing.Manager()
tasks_queue = manager.Queue()
results_queue = manager.Queue()

for _ in range(self.linter.config.jobs):
child_linter = ChildRunner(args=(tasks_queue, results_queue,
child_config))
child_linter.start()
children.append(child_linter)

# Send files to child linters.
expanded_files = utils.expand_files(
files_or_modules,
self.linter,
self.linter.config.black_list,
self.linter.config.black_list_re
)
for module_desc in expanded_files:
tasks_queue.put([module_desc.path])

# collect results from child linters
failed = False
for _ in expanded_files:
try:
result = results_queue.get()
except Exception as ex:
print("internal error while receiving results from child linter",
file=sys.stderr)
print(ex, file=sys.stderr)
failed = True
break
yield result

# Stop child linters and wait for their completion.
for _ in range(self.linter.config.jobs):
tasks_queue.put('STOP')
for child in children:
child.join()

if failed:
print("Error occured, stopping the linter.", file=sys.stderr)
sys.exit(32)

def _parallel_check(self, files_or_modules):
# Reset stats.
self.linter.open()

all_stats = []
module = None
for result in self._parallel_task(files_or_modules):
(
_,
self.linter.file_state.base_name,
module,
messages,
stats,
msg_status
) = result

for msg in messages:
msg = utils.Message(*msg)
self.linter.set_current_module(module)
self.linter.reporter.handle_message(msg)

all_stats.append(stats)
self.linter.msg_status |= msg_status

self.linter.stats = _merge_stats(all_stats)
self.linter.current_name = module

# Insert stats data to local checkers.
for checker in self.linter.get_checkers():
if checker is not self.linter:
checker.stats = self.linter.stats

def _get_jobs_config(self):
child_config = collections.OrderedDict()
filter_options = {'long-help'}
filter_options.update((opt_name for opt_name, _ in self.linter._external_opts))
for opt_providers in six.itervalues(self.linter._all_options):
for optname, optdict, val in opt_providers.options_and_values():
if optdict.get('deprecated'):
continue

if optname not in filter_options:
child_config[optname] = utils._format_option_value(
optdict, val)
child_config['python3_porting_mode'] = self.linter._python3_porting_mode
child_config['plugins'] = self.linter._dynamic_plugins
return child_config

def cb_set_rcfile(self, name, value):
"""callback for option preprocessing (i.e. before option parsing)"""
self._rcfile = value
Expand Down
53 changes: 48 additions & 5 deletions pylint/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -797,6 +797,52 @@ def _is_package_cb(path, parts):
return modutils.modpath_from_file_with_callback(filename, is_package_cb=_is_package_cb)


class ModuleDescription(object):
def __init__(self, path, name, isarg, basepath, basename):
self.path = path
self.name = name
self.isarg = isarg
self.basepath = basepath
self.basename = basename


# TODO: Replace linter with a message store as part of #938
def expand_files(files_or_modules, linter, black_list, black_list_re):
"""
Run :func:`expand_modules` and turn errors into messages.
:param files_or_modules: A list of files, directories, modules, or packages
to turn into :class:`ModuleDescription`s.
:type files_or_modules: :obj:`list` of :obj:`str`
:param linter: A linter to store messages onto.
:type linter: :class:`pylint.PyLinter`
:param black_list: A list of files or directories to ignore.
:type black_list: :obj:`list` or :obj:`tuple`
:param black_list_re: A list of regex patterns. Any files, directories,
modules, or packages matching against any of these patterns will be
ignored.
:type black_list_re: :obj:`list`: of :obj:`re.RegexObject`
:returns: The :class:`ModuleDescription`s for each found file, directory,
module, or package.
:rtype: :obj:`list` of :class:`ModuleDescription`
"""
result, errors = expand_modules(files_or_modules, black_list, black_list_re)

for error in errors:
message = modname = error["mod"]
key = error["key"]
linter.set_current_module(modname)
if key == "fatal":
message = str(error["ex"]).replace(os.getcwd() + os.sep, '')
linter.add_message(key, args=message)

return result


def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
Expand Down Expand Up @@ -841,8 +887,7 @@ def expand_modules(files_or_modules, black_list, black_list_re):
is_directory = spec.type == modutils.ModuleType.PKG_DIRECTORY

if not is_namespace:
result.append({'path': filepath, 'name': modname, 'isarg': True,
'basepath': filepath, 'basename': modname})
result.append(ModuleDescription(filepath, modname, True, filepath, modname))

has_init = (not (modname.endswith('.__init__') or modname == '__init__')
and '__init__.py' in filepath)
Expand All @@ -857,9 +902,7 @@ def expand_modules(files_or_modules, black_list, black_list_re):

modpath = _modpath_from_file(subfilepath, is_namespace)
submodname = '.'.join(modpath)
result.append({'path': subfilepath, 'name': submodname,
'isarg': False,
'basepath': filepath, 'basename': modname})
result.append(ModuleDescription(subfilepath, submodname, False, filepath, modname))
return result, errors


Expand Down

0 comments on commit 1b40746

Please sign in to comment.