From 494b1478fc1eea6b6ca3bdb7fecc483fce1bbed0 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 2 Dec 2018 21:29:10 +0000 Subject: [PATCH] implement parallel invocation of tox environments #439 --- docs/changelog/439.feature.rst | 2 ++ src/tox/config.py | 17 +++++++-- src/tox/session.py | 54 +++++++++++++++++++++++++++-- tests/unit/session/test_parallel.py | 33 ++++++++++++++++++ 4 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 docs/changelog/439.feature.rst create mode 100644 tests/unit/session/test_parallel.py diff --git a/docs/changelog/439.feature.rst b/docs/changelog/439.feature.rst new file mode 100644 index 000000000..8bdcd1ac7 --- /dev/null +++ b/docs/changelog/439.feature.rst @@ -0,0 +1,2 @@ +``tox --parallel`` now allows running tox environments in parallel. ``--parallel-live`` allows showing the live output of the standard output and error. Note that parallel evaluation + disables standard input. Use non parallel invocation if you need standard input. By :user:`gaborbernat`. diff --git a/src/tox/config.py b/src/tox/config.py index aad17ebdc..f7816edeb 100644 --- a/src/tox/config.py +++ b/src/tox/config.py @@ -274,7 +274,9 @@ def parse_cli(args, pm): print(get_version_info(pm)) raise SystemExit(0) interpreters = Interpreters(hook=pm.hook) - config = Config(pluginmanager=pm, option=option, interpreters=interpreters, parser=parser) + config = Config( + pluginmanager=pm, option=option, interpreters=interpreters, parser=parser, args=args + ) return config, option @@ -413,6 +415,15 @@ def tox_addoption(parser): dest="sdistonly", help="only perform the sdist packaging activity.", ) + parser.add_argument( + "--parallel", action="store_true", dest="parallel", help="run tox environments in parallel" + ) + parser.add_argument( + "--parallel-live", + action="store_true", + dest="parallel_live", + help="connect to stdout while running environments", + ) parser.add_argument( "--parallel--safe-build", action="store_true", @@ -822,7 +833,7 @@ def __call__(self, parser, namespace, values, option_string=None): class Config(object): """Global Tox config object.""" - def __init__(self, pluginmanager, option, interpreters, parser): + def __init__(self, pluginmanager, option, interpreters, parser, args): self.envconfigs = OrderedDict() """Mapping envname -> envconfig""" self.invocationcwd = py.path.local() @@ -831,6 +842,7 @@ def __init__(self, pluginmanager, option, interpreters, parser): self.option = option self._parser = parser self._testenv_attr = parser._testenv_attr + self.args = args """option namespace containing all parsed command line options""" @@ -1133,6 +1145,7 @@ def make_envconfig(self, name, section, subs, config, replace=True): def _getenvdata(self, reader, config): candidates = ( + os.environ.get("_PARALLEL_TOXENV"), self.config.option.env, os.environ.get("TOXENV"), reader.getstring("envlist", replace=False), diff --git a/src/tox/session.py b/src/tox/session.py index fe54ee433..b53e0d8da 100644 --- a/src/tox/session.py +++ b/src/tox/session.py @@ -568,6 +568,14 @@ def subcommand_test(self): venv.envconfig.setenv[str("TOX_PACKAGE")] = str(venv.package) if self.config.option.sdistonly: return + if self.config.option.parallel: + self.run_parallel() + else: + self.run_sequential() + retcode = self._summary() + return retcode + + def run_sequential(self): for venv in self.venvlist: if self.setupenv(venv): if venv.envconfig.skip_install: @@ -581,9 +589,49 @@ def subcommand_test(self): self.installpkg(venv, venv.package) self.runenvreport(venv) - self.runtestenv(venv) - retcode = self._summary() - return retcode + self.runtestenv(venv) + + def run_parallel(self): + """here we'll just start parallel sub-processes""" + live_out = self.config.option.parallel_live + args = [sys.executable, "-m", "tox"] + self.config.args + try: + position = args.index("--") + except ValueError: + position = len(args) + try: + parallel_at = args[0:position].index("--parallel") + del args[parallel_at] + position -= 1 + except ValueError: + pass + + tox_runs = {} + for venv in self.venvlist: + env = os.environ.copy() + env["_PARALLEL_TOXENV"] = venv.envconfig.envname + args_sub = list(args) + if hasattr(venv, "package"): + args_sub.insert(position, str(venv.package)) + args_sub.insert(position, "--installpkg") + stdout, stderr = (None, None) if live_out else (subprocess.PIPE, subprocess.PIPE) + run = subprocess.Popen(args_sub, env=env, stdout=stdout, stderr=stderr) + tox_runs[venv.name] = (venv, run) + + to_finish = set(tox_runs.keys()) + while to_finish: + for name in set(to_finish): + venv, run = tox_runs[name] + res = run.poll() + if res is not None: + venv.status = "skipped tests" if self.config.option.notest else res + if not live_out: + out, err = run.communicate() + venv.out = out + venv.err = err + to_finish.remove(name) + if to_finish: + time.sleep(0.1) def runenvreport(self, venv): """ diff --git a/tests/unit/session/test_parallel.py b/tests/unit/session/test_parallel.py new file mode 100644 index 000000000..5726d31d7 --- /dev/null +++ b/tests/unit/session/test_parallel.py @@ -0,0 +1,33 @@ +import os + + +def test_parallel_live(cmd, initproj): + initproj( + "pkg123-0.7", + filedefs={ + "tox.ini": """ + [tox] + envlist = a, b + [testenv] + commands=python -c "import sys; print(sys.executable)" + """ + }, + ) + result = cmd("--parallel", "--parallel-live") + assert result.ret == 0, "{}{}{}".format(result.err, os.linesep, result.out) + + +def test_parallel(cmd, initproj): + initproj( + "pkg123-0.7", + filedefs={ + "tox.ini": """ + [tox] + envlist = a, b + [testenv] + commands=python -c "import sys; print(sys.executable)" + """ + }, + ) + result = cmd("--parallel") + assert result.ret == 0, "{}{}{}".format(result.err, os.linesep, result.out)