From 934119cf01acf5ed5bed06b433abda5de94faaf8 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 27 Sep 2015 15:14:28 +0300 Subject: [PATCH 1/3] Fix argument handling in Find --- asv/commands/find.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asv/commands/find.py b/asv/commands/find.py index 3892d388a..8a46808f7 100644 --- a/asv/commands/find.py +++ b/asv/commands/find.py @@ -60,7 +60,7 @@ def setup_arguments(cls, subparsers): @classmethod def run_from_conf_args(cls, conf, args): return cls.run( - conf, args.range[0], args.bench[0], + conf, args.range, args.bench, invert=args.invert, show_stderr=args.show_stderr, machine=args.machine ) From d958e92205a5c3328a571a7b49c6a4316b40d61f Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 27 Sep 2015 16:04:06 +0300 Subject: [PATCH 2/3] Use command line parsing to run asv from tests This ensures better test coverage of the whole stack, ensuring the command line parsing stays in sync with the run methods. --- asv/commands/continuous.py | 5 ++-- asv/commands/dev.py | 5 ++-- asv/commands/find.py | 4 +-- asv/commands/profiling.py | 4 +-- asv/commands/run.py | 5 ++-- test/test_compare.py | 5 ++-- test/test_dev.py | 12 +++------ test/test_publish.py | 4 +-- test/test_quickstart.py | 5 ++-- test/test_rm.py | 7 +++--- test/test_web.py | 16 ++++++------ test/test_workflow.py | 50 +++++++++++++++++++------------------- test/tools.py | 26 ++++++++++++++++++-- 13 files changed, 84 insertions(+), 64 deletions(-) diff --git a/asv/commands/continuous.py b/asv/commands/continuous.py index 2210c061a..680d1b030 100644 --- a/asv/commands/continuous.py +++ b/asv/commands/continuous.py @@ -42,10 +42,11 @@ def setup_arguments(cls, subparsers): return parser @classmethod - def run_from_conf_args(cls, conf, args): + def run_from_conf_args(cls, conf, args, **kwargs): return cls.run( conf=conf, branch=args.branch, base=args.base, factor=args.factor, - show_stderr=args.show_stderr, bench=args.bench, machine=args.machine + show_stderr=args.show_stderr, bench=args.bench, machine=args.machine, + **kwargs ) @classmethod diff --git a/asv/commands/dev.py b/asv/commands/dev.py index bf3e8f4df..35b4fd790 100644 --- a/asv/commands/dev.py +++ b/asv/commands/dev.py @@ -27,8 +27,9 @@ def setup_arguments(cls, subparsers): return parser @classmethod - def run_from_conf_args(cls, conf, args): - return cls.run(conf, bench=args.bench, machine=args.machine, python=args.python) + def run_from_conf_args(cls, conf, args, **kwargs): + return cls.run(conf, bench=args.bench, machine=args.machine, python=args.python, + **kwargs) @classmethod def run(cls, conf, bench=None, python="same", machine=None, _machine_file=None): diff --git a/asv/commands/find.py b/asv/commands/find.py index 8a46808f7..4c572c698 100644 --- a/asv/commands/find.py +++ b/asv/commands/find.py @@ -58,11 +58,11 @@ def setup_arguments(cls, subparsers): return parser @classmethod - def run_from_conf_args(cls, conf, args): + def run_from_conf_args(cls, conf, args, **kwargs): return cls.run( conf, args.range, args.bench, invert=args.invert, show_stderr=args.show_stderr, - machine=args.machine + machine=args.machine, **kwargs ) @classmethod diff --git a/asv/commands/profiling.py b/asv/commands/profiling.py index 905659399..ef1cd5f0c 100644 --- a/asv/commands/profiling.py +++ b/asv/commands/profiling.py @@ -90,11 +90,11 @@ def find_guis(cls): cls.guis[x.name] = x @classmethod - def run_from_conf_args(cls, conf, args): + def run_from_conf_args(cls, conf, args, **kwargs): return cls.run( conf=conf, benchmark=args.benchmark, revision=args.revision, gui=args.gui, output=args.output, force=args.force, - environment=args.environment, python=args.python) + environment=args.environment, python=args.python, **kwargs) @classmethod def run(cls, conf, benchmark, revision=None, gui=None, output=None, diff --git a/asv/commands/run.py b/asv/commands/run.py index 39fea5794..ea67d2866 100644 --- a/asv/commands/run.py +++ b/asv/commands/run.py @@ -110,7 +110,7 @@ def setup_arguments(cls, subparsers): return parser @classmethod - def run_from_conf_args(cls, conf, args): + def run_from_conf_args(cls, conf, args, **kwargs): return cls.run( conf=conf, range_spec=args.range, steps=args.steps, bench=args.bench, parallel=args.parallel, @@ -119,7 +119,8 @@ def run_from_conf_args(cls, conf, args): dry_run=args.dry_run, machine=args.machine, skip_successful=args.skip_existing_successful or args.skip_existing, skip_failed=args.skip_existing_failed or args.skip_existing, - skip_existing_commits=args.skip_existing_commits + skip_existing_commits=args.skip_existing_commits, + **kwargs ) @classmethod diff --git a/test/test_compare.py b/test/test_compare.py index 5c95a74de..d9c0b0773 100644 --- a/test/test_compare.py +++ b/test/test_compare.py @@ -68,9 +68,8 @@ def test_compare(capsys, tmpdir): 'repo': tools.generate_test_repo(tmpdir).path, 'project': 'asv'}) - mock_args = Namespace(revision1='22b920c6', revision2='fcf8c079', machine='cheetah', - factor=2, split=False) - Compare.run_from_conf_args(conf, mock_args) + tools.run_asv_with_conf(conf, 'compare', '22b920c6', 'fcf8c079', '--machine=cheetah', + '--factor=2') text, err = capsys.readouterr() assert text.strip() == REFERENCE.strip() diff --git a/test/test_dev.py b/test/test_dev.py index 138e800f0..ff9dff1ad 100644 --- a/test/test_dev.py +++ b/test/test_dev.py @@ -14,9 +14,6 @@ import six from asv import config -from asv.commands.dev import Dev -from asv.commands.profiling import Profile -from asv.commands.run import Run from asv.commands import make_argparser from . import tools @@ -56,7 +53,7 @@ def test_dev(capsys, basic_conf): tmpdir, local, conf = basic_conf # Test Dev runs - Dev.run(conf, _machine_file=join(tmpdir, 'asv-machine.json')) + tools.run_asv_with_conf(conf, 'dev', _machine_file=join(tmpdir, 'asv-machine.json')) text, err = capsys.readouterr() # time_with_warnings failure case @@ -72,7 +69,7 @@ def test_run_python_same(capsys, basic_conf): tmpdir, local, conf = basic_conf # Test Run runs with python=same - Run.run(conf, _machine_file=join(tmpdir, 'asv-machine.json'), python="same") + tools.run_asv_with_conf(conf, 'run', '--python=same', _machine_file=join(tmpdir, 'asv-machine.json')) text, err = capsys.readouterr() assert re.search("time_exception.*failed", text, re.S) @@ -87,9 +84,8 @@ def test_profile_python_same(capsys, basic_conf): tmpdir, local, conf = basic_conf # Test Profile can run with python=same - Profile.run(conf, "time_secondary.track_value", - _machine_file=join(tmpdir, 'asv-machine.json'), - python="same") + tools.run_asv_with_conf(conf, 'profile', '--python=same', "time_secondary.track_value", + _machine_file=join(tmpdir, 'asv-machine.json')) text, err = capsys.readouterr() # time_with_warnings failure case diff --git a/test/test_publish.py b/test/test_publish.py index 316fa1dc0..180949520 100644 --- a/test/test_publish.py +++ b/test/test_publish.py @@ -69,7 +69,7 @@ def test_publish(tmpdir): 'repo': dvcs.path, 'project': 'asv'}) - Publish.run(conf) + tools.run_asv_with_conf(conf, 'publish') # Check output assert isfile(join(tmpdir, 'html', 'index.html')) @@ -98,7 +98,7 @@ def check_file(branch): # Publish with branches set in the config conf.branches = ['master', 'some-branch'] - Publish.run(conf) + tools.run_asv_with_conf(conf, 'publish') # Check output check_file("master") diff --git a/test/test_quickstart.py b/test/test_quickstart.py index 34148414d..7880bfdfd 100644 --- a/test/test_quickstart.py +++ b/test/test_quickstart.py @@ -6,15 +6,16 @@ from os.path import isfile, join +import os import six -from asv.commands.quickstart import Quickstart +from . import tools def test_quickstart(tmpdir): tmpdir = six.text_type(tmpdir) - Quickstart.run(tmpdir) + tools.run_asv('quickstart', '--dest', tmpdir) assert isfile(join(tmpdir, 'asv.conf.json')) assert isfile(join(tmpdir, 'benchmarks', 'benchmarks.py')) diff --git a/test/test_rm.py b/test/test_rm.py index 5695f5150..0755c64fc 100644 --- a/test/test_rm.py +++ b/test/test_rm.py @@ -11,7 +11,8 @@ from asv import config from asv import results -from asv.commands.rm import Rm + +from . import tools def test_rm(tmpdir): @@ -26,14 +27,14 @@ def test_rm(tmpdir): 'repo': "### IGNORED, BUT REQUIRED ###" }) - Rm.run(conf, ['benchmark=time_quantity*']) + tools.run_asv_with_conf(conf, 'rm', '-y', 'benchmark=time_quantity*') results_a = list(results.iter_results(tmpdir)) for result in results_a: for key in six.iterkeys(result.results): assert not key.startswith('time_quantity') - Rm.run(conf, ['commit_hash=05d283b9']) + tools.run_asv_with_conf(conf, 'rm', '-y', 'commit_hash=05d283b9') results_b = list(results.iter_results(tmpdir)) assert len(results_b) == len(results_a) - 1 diff --git a/test/test_web.py b/test/test_web.py index b8bcb4f9d..ff353d6c5 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -15,12 +15,9 @@ import pytest from asv import config -from asv.commands.run import Run -from asv.commands.publish import Publish from . import tools from .tools import browser, get_with_retry -from .test_workflow import basic_conf @pytest.fixture(scope="session") @@ -55,9 +52,9 @@ def basic_html(request): } }) - Run.run(conf, range_spec="master~5..master", steps=3, - _machine_file=machine_file, quick=True) - Publish.run(conf) + tools.run_asv_with_conf(conf, 'run', "master~5..master", '--steps=3', + '--quick', _machine_file=machine_file) + tools.run_asv_with_conf(conf, 'publish') finally: os.chdir(cwd) @@ -131,9 +128,10 @@ def test_web_regressions(browser, tmpdir): }, }) - Run.run(conf, range_spec="ALL", bench='params_examples.track_find_test', - _machine_file=machine_file, show_stderr=True, quick=True) - Publish.run(conf) + tools.run_asv_with_conf(conf, 'run', 'ALL', '--bench=params_examples.track_find_test', + '--show-stderr', '--quick', + _machine_file=machine_file) + tools.run_asv_with_conf(conf, 'publish') finally: os.chdir(cwd) diff --git a/test/test_workflow.py b/test/test_workflow.py index 8c7e7abeb..73d8d459f 100644 --- a/test/test_workflow.py +++ b/test/test_workflow.py @@ -15,10 +15,6 @@ import pytest from asv import config -from asv.commands.run import Run -from asv.commands.publish import Publish -from asv.commands.find import Find -from asv.commands.continuous import Continuous from asv.util import check_output, which from . import tools @@ -80,15 +76,16 @@ def test_run_publish(capfd, basic_conf): tmpdir, local, conf, machine_file = basic_conf # Tests a typical complete run/publish workflow - Run.run(conf, range_spec="master~5..master", steps=2, - _machine_file=machine_file, quick=True, show_stderr=True) + tools.run_asv_with_conf(conf, 'run', "master~5..master", '--steps=2', + '--quick', '--show-stderr', + _machine_file=machine_file) text, err = capfd.readouterr() assert len(os.listdir(join(tmpdir, 'results_workflow', 'orangutan'))) == 5 assert len(os.listdir(join(tmpdir, 'results_workflow'))) == 2 assert 'asv: benchmark timed out (timeout 0.1s)' in text - Publish.run(conf) + tools.run_asv_with_conf(conf, 'publish') assert isfile(join(tmpdir, 'html', 'index.html')) assert isfile(join(tmpdir, 'html', 'index.json')) @@ -112,32 +109,34 @@ def test_run_publish(capfd, basic_conf): # Check that the skip options work capfd.readouterr() - Run.run(conf, range_spec="master~5..master", steps=2, - _machine_file=join(tmpdir, 'asv-machine.json'), quick=True, - skip_successful=True, skip_failed=True) - Run.run(conf, range_spec="master~5..master", steps=2, - _machine_file=join(tmpdir, 'asv-machine.json'), quick=True, - skip_existing_commits=True) + tools.run_asv_with_conf(conf, 'run', "master~5..master", '--steps=2', + '--quick', '--skip-existing-successful', + '--skip-existing-failed', + _machine_file=join(tmpdir, 'asv-machine.json')) + tools.run_asv_with_conf(conf, 'run', "master~5..master", '--steps=2', + '--quick', '--skip-existing-commits', + _machine_file=join(tmpdir, 'asv-machine.json')) text, err = capfd.readouterr() assert 'Running benchmarks.' not in text # Check EXISTING works - Run.run(conf, range_spec="EXISTING", - _machine_file=machine_file, quick=True) + tools.run_asv_with_conf(conf, 'run', "EXISTING", '--quick', + _machine_file=machine_file) # Remove the benchmarks.json file to make sure publish can # regenerate it os.remove(join(tmpdir, "results_workflow", "benchmarks.json")) - Publish.run(conf) + tools.run_asv_with_conf(conf, 'publish') def test_continuous(capfd, basic_conf): tmpdir, local, conf, machine_file = basic_conf # Check that asv continuous runs - Continuous.run(conf, branch="master^", _machine_file=machine_file, show_stderr=True) + tools.run_asv_with_conf(conf, 'continuous', "master^", '--show-stderr', + _machine_file=machine_file) text, err = capfd.readouterr() assert "SOME BENCHMARKS HAVE CHANGED SIGNIFICANTLY" in text @@ -155,8 +154,8 @@ def test_find(capfd, basic_conf): "from py.test runner.") # Test find at least runs - Find.run(conf, "master~5..master", "params_examples.track_find_test", - _machine_file=machine_file) + tools.run_asv_with_conf(conf, 'find', "master~5..master", "params_examples.track_find_test", + _machine_file=machine_file) # Check it found the first commit after the initially tested one output, err = capfd.readouterr() @@ -175,8 +174,8 @@ def _test_run_branches(tmpdir, dvcs, conf, machine_file, range_spec, commits.append(dvcs.get_hash(branch)) # Run tests - Run.run(conf, range_spec=range_spec, - _machine_file=machine_file, quick=True) + tools.run_asv_with_conf(conf, 'run', range_spec, '--quick', + _machine_file=machine_file) # Check that files for all commits expected were generated expected = set(['machine.json']) @@ -211,10 +210,11 @@ def init_results(): results_dir = os.path.join(tmpdir, 'results_workflow') if os.path.isdir(results_dir): shutil.rmtree(results_dir) - Run.run(conf, range_spec=initial_commit+"^!", - bench=["time_secondary.track_value"], - _machine_file=join(tmpdir, 'asv-machine.json'), quick=True, - skip_successful=True, skip_failed=True) + tools.run_asv_with_conf(conf, 'run', initial_commit+"^!", + '--bench=time_secondary.track_value', + '--quick', '--skip-existing-successful', + '--skip-existing-failed', + _machine_file=join(tmpdir, 'asv-machine.json')) # Without branches in config, should just use master init_results() diff --git a/test/tools.py b/test/tools.py index 47764e535..70bbe180e 100644 --- a/test/tools.py +++ b/test/tools.py @@ -13,11 +13,11 @@ import threading import time import six +import sys from os.path import abspath, join, dirname, relpath, isdir from contextlib import contextmanager from distutils.spawn import find_executable -from distutils.version import LooseVersion -from six.moves import SimpleHTTPServer, socketserver +from six.moves import SimpleHTTPServer import pytest @@ -27,6 +27,8 @@ hglib = None from asv import util +from asv import commands +from asv import config from asv.commands.preview import create_httpd @@ -50,6 +52,26 @@ FIREFOX = ['firefox'] +def run_asv(*argv): + parser, subparsers = commands.make_argparser() + args = parser.parse_args(argv) + return args.func(args) + + +def run_asv_with_conf(conf, *argv, **kwargs): + assert isinstance(conf, config.Config) + + parser, subparsers = commands.make_argparser() + args = parser.parse_args(argv) + + if sys.version_info[0] >= 3: + cls = args.func.__self__ + else: + cls = args.func.im_self + + return cls.run_from_conf_args(conf, args, **kwargs) + + # These classes are defined here, rather than using asv/plugins/git.py # and asv/plugins/mercurial.py since here we need to perform write # operations to the repository, and the others should be read-only for From 77339dc8aa8b94ba4603c1ff821590779db3113c Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 27 Sep 2015 17:10:41 +0300 Subject: [PATCH 3/3] Deal with too long directory names in write_json on Windows --- asv/util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/asv/util.py b/asv/util.py index 6e56e024e..1ca2fc2a2 100644 --- a/asv/util.py +++ b/asv/util.py @@ -524,8 +524,10 @@ def write_json(path, data, api_version=None): Writes JSON to the given path, including indentation and sorting. """ path = os.path.abspath(path) - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) + + dirname = long_path(os.path.dirname(path)) + if not os.path.exists(dirname): + os.makedirs(dirname) if api_version is not None: data['version'] = api_version