From 94e975fec14c25cee4090edbaf7094cff281295e Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 25 Jun 2015 08:14:41 -0400 Subject: [PATCH 01/27] Add Appveyor support --- .../appveyor/install-miniconda.ps1 | 71 +++++++++++++++++++ .../appveyor/windows_sdk.cmd | 47 ++++++++++++ appveyor.yml | 47 ++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 .continuous-integration/appveyor/install-miniconda.ps1 create mode 100644 .continuous-integration/appveyor/windows_sdk.cmd create mode 100644 appveyor.yml diff --git a/.continuous-integration/appveyor/install-miniconda.ps1 b/.continuous-integration/appveyor/install-miniconda.ps1 new file mode 100644 index 000000000..aeccc3b1c --- /dev/null +++ b/.continuous-integration/appveyor/install-miniconda.ps1 @@ -0,0 +1,71 @@ +# Sample script to install anaconda under windows +# Authors: Stuart Mumford +# Borrwed from: Olivier Grisel and Kyle Kastner +# License: BSD 3 clause + +$MINICONDA_URL = "http://repo.continuum.io/miniconda/" + +function DownloadMiniconda ($version, $platform_suffix) { + $webclient = New-Object System.Net.WebClient + $filename = "Miniconda-" + $version + "-Windows-" + $platform_suffix + ".exe" + + $url = $MINICONDA_URL + $filename + + $basedir = $pwd.Path + "\" + $filepath = $basedir + $filename + if (Test-Path $filename) { + Write-Host "Reusing" $filepath + return $filepath + } + + # Download and retry up to 3 times in case of network transient errors. + Write-Host "Downloading" $filename "from" $url + $retry_attempts = 2 + for($i=0; $i -lt $retry_attempts; $i++){ + try { + $webclient.DownloadFile($url, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + if (Test-Path $filepath) { + Write-Host "File saved at" $filepath + } else { + # Retry once to get the error message if any at the last try + $webclient.DownloadFile($url, $filepath) + } + return $filepath +} + +function InstallMiniconda ($python_version, $architecture, $python_home) { + Write-Host "Installing miniconda" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "x86") { + $platform_suffix = "x86" + } else { + $platform_suffix = "x86_64" + } + $filepath = DownloadMiniconda $python_version $platform_suffix + Write-Host "Installing" $filepath "to" $python_home + $args = "/InstallationType=AllUsers /S /AddToPath=1 /RegisterPython=1 /D=" + $python_home + Write-Host $filepath $args + Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru + #Start-Sleep -s 15 + if (Test-Path C:\conda) { + Write-Host "Miniconda $python_version ($architecture) installation complete" + } else { + Write-Host "Failed to install Python in $python_home" + Exit 1 + } +} + +function main () { + InstallMiniconda $env:MINICONDA_VERSION $env:PLATFORM $env:PYTHON +} + +main diff --git a/.continuous-integration/appveyor/windows_sdk.cmd b/.continuous-integration/appveyor/windows_sdk.cmd new file mode 100644 index 000000000..3a472bc83 --- /dev/null +++ b/.continuous-integration/appveyor/windows_sdk.cmd @@ -0,0 +1,47 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds do not require specific environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows + +SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" +IF %MAJOR_PYTHON_VERSION% == "2" ( + SET WINDOWS_SDK_VERSION="v7.0" +) ELSE IF %MAJOR_PYTHON_VERSION% == "3" ( + SET WINDOWS_SDK_VERSION="v7.1" +) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 +) + +IF "%PYTHON_ARCH%"=="64" ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..7848f7e75 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,47 @@ +# AppVeyor.com is a Continuous Integration service to build and run tests under +# Windows + +environment: + + global: + PYTHON: "C:\\conda" + MINICONDA_VERSION: "latest" + CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\.continuous-integration\\appveyor\\windows_sdk.cmd" + + matrix: + + - PYTHON_VERSION: "2.6" + platform: x64 + PYTHON_ARCH: "64" + - PYTHON_VERSION: "2.7" + platform: x64 + PYTHON_ARCH: "64" + - PYTHON_VERSION: "2.7" + platform: x86 + PYTHON_ARCH: "32" + - PYTHON_VERSION: "3.4" + platform: x64 + PYTHON_ARCH: "64" + +install: + # Install miniconda using a powershell script. + - "powershell .continuous-integration/appveyor/install-miniconda.ps1" + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" + + # Install the build and runtime dependencies of the project. + - "conda update --yes conda" + # Create a conda environment + - "conda create -q --yes -n test python=%PYTHON_VERSION%" + - "activate test" + + # Check that we have the expected version of Python + - "python --version" + + # Install specified version of dependencies + - "conda install -q --yes six pytest" + +# Not a .NET project +build: false + +test_script: + - "%CMD_IN_ENV% python setup.py test" \ No newline at end of file From d9450cf882ce170cd2311395604a69a5efc0f257 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 17:05:05 +0300 Subject: [PATCH 02/27] Fix up color printing on Windows Based on Robert McGibbon's patch --- asv/console.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/asv/console.py b/asv/console.py index c86f36f7d..d72d9ca3a 100644 --- a/asv/console.py +++ b/asv/console.py @@ -8,6 +8,8 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import os +import warnings import codecs import contextlib import locale @@ -18,6 +20,16 @@ import six from six.moves import xrange, input +WIN = (os.name == "nt") + + +if WIN: + try: + import colorama + except ImportError: + colorama = None + warnings.warn('the colorama package is required for terminal color on Windows') + def isatty(file): """ @@ -168,11 +180,14 @@ def color_print(*args, **kwargs): """ file = kwargs.get('file', sys.stdout) - end = kwargs.get('end', '') write = file.write - if isatty(file): + if isatty(file) and (not WIN or colorama): + if WIN: + file = colorama.AnsiToWin32(file).stream + write = file.write + for i in xrange(0, len(args), 2): msg = args[i] if i + 1 == len(args): From 624009d2ec6d172b810902c533ee51b07c1a6990 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 17:05:23 +0300 Subject: [PATCH 03/27] Allow multi-argument range spec for Git Based on Robert McGibbon's patch --- asv/plugins/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asv/plugins/git.py b/asv/plugins/git.py index 305f9787d..f31be5efc 100644 --- a/asv/plugins/git.py +++ b/asv/plugins/git.py @@ -98,7 +98,7 @@ def get_date(self, hash): def get_hashes_from_range(self, range_spec): args = ['log', '--quiet', '--first-parent', '--format=format:%H'] if range_spec != "": - args += [range_spec] + args += range_spec.split() output = self._run_git(args, valid_return_codes=(0, 1), dots=False) return output.strip().split() From 913851b726e88109d96a367eb493ce74dbf6936f Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 17:05:54 +0300 Subject: [PATCH 04/27] Fix up util.check_output + util.which on Windows Based on Robert McGibbon's patch --- asv/util.py | 181 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 109 insertions(+), 72 deletions(-) diff --git a/asv/util.py b/asv/util.py index 64b01b774..c219c6862 100644 --- a/asv/util.py +++ b/asv/util.py @@ -19,12 +19,7 @@ import sys import time import errno - -try: - from select import PIPE_BUF -except ImportError: - # PIPE_BUF is not available on Python 2.6 - PIPE_BUF = os.pathconf('.', os.pathconf_names['PC_PIPE_BUF']) +import threading import six from six.moves import xrange @@ -33,6 +28,16 @@ from .extern import minify_json +WIN = (os.name == 'nt') + +if not WIN: + try: + from select import PIPE_BUF + except ImportError: + # PIPE_BUF is not available on Python 2.6 + PIPE_BUF = os.pathconf('.', os.pathconf_names['PC_PIPE_BUF']) + + TIMEOUT_RETCODE = -256 @@ -209,6 +214,10 @@ def which(filename): Raises an IOError if no result is found. """ + if WIN: + if not filename.endswith('.exe'): + filename = filename + '.exe' + locations = os.environ.get("PATH").split(os.pathsep) candidates = [] for location in locations: @@ -334,7 +343,7 @@ def get_content(header=None): proc = subprocess.Popen( args, - close_fds=True, + close_fds=(not WIN), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -346,75 +355,103 @@ def get_content(header=None): stdout_chunks = [] stderr_chunks = [] is_timeout = False - try: - if posix: - # Forward signals related to Ctrl-Z handling; the child - # process is in a separate process group so it won't receive - # these automatically from the terminal - def sig_forward(signum, frame): - os.killpg(proc.pid, signum) - if signum == signal.SIGTSTP: - os.kill(os.getpid(), signal.SIGSTOP) - signal.signal(signal.SIGTSTP, sig_forward) - signal.signal(signal.SIGCONT, sig_forward) - - fds = { - proc.stdout.fileno(): stdout_chunks, - proc.stderr.fileno(): stderr_chunks - } - - while proc.poll() is None: - try: - rlist, wlist, xlist = select.select( - list(fds.keys()), [], [], timeout) - except select.error as err: - if err.args[0] == errno.EINTR: - # interrupted by signal handler; try again - continue - - if len(rlist) == 0: - # We got a timeout - is_timeout = True - break - for f in rlist: - output = os.read(f, PIPE_BUF) - fds[f].append(output) - if dots and time.time() - last_dot_time > 0.5: - if dots is True: - log.dot() - elif dots: - dots() - last_dot_time = time.time() - finally: - if posix: - # Restore signal handlers - signal.signal(signal.SIGTSTP, signal.SIG_DFL) - signal.signal(signal.SIGCONT, signal.SIG_DFL) - - if proc.returncode is None: - # Timeout or another exceptional condition occurred, and - # the program is still running. + + if WIN: + start_time = time.time() + was_timeout = [False] + + def watcher_run(): + while proc.returncode is None: + time.sleep(0.1) + if time.time() - start_time > timeout: + was_timeout[0] = True + proc.terminate() + + watcher = threading.Thread(target=watcher_run) + watcher.start() + try: + stdout, stderr = proc.communicate() + finally: + if proc.returncode is None: + proc.terminate() + proc.wait() + watcher.join() + + is_timeout = was_timeout[0] + else: + try: if posix: - # Terminate the whole process group - os.killpg(proc.pid, signal.SIGTERM) - for j in range(10): - time.sleep(0.1) - if proc.poll() is not None: - break - else: - # Didn't terminate within 1 sec, so kill it + # Forward signals related to Ctrl-Z handling; the child + # process is in a separate process group so it won't receive + # these automatically from the terminal + def sig_forward(signum, frame): + os.killpg(proc.pid, signum) + if signum == signal.SIGTSTP: + os.kill(os.getpid(), signal.SIGSTOP) + signal.signal(signal.SIGTSTP, sig_forward) + signal.signal(signal.SIGCONT, sig_forward) + + fds = { + proc.stdout.fileno(): stdout_chunks, + proc.stderr.fileno(): stderr_chunks + } + + while proc.poll() is None: + try: + rlist, wlist, xlist = select.select( + list(fds.keys()), [], [], timeout) + except select.error as err: + if err.args[0] == errno.EINTR: + # interrupted by signal handler; try again + continue + raise + + if len(rlist) == 0: + # We got a timeout + is_timeout = True + break + for f in rlist: + output = os.read(f, PIPE_BUF) + fds[f].append(output) + if dots and time.time() - last_dot_time > 0.5: + if dots is True: + log.dot() + elif dots: + dots() + last_dot_time = time.time() + finally: + if posix: + # Restore signal handlers + signal.signal(signal.SIGTSTP, signal.SIG_DFL) + signal.signal(signal.SIGCONT, signal.SIG_DFL) + + if proc.returncode is None: + # Timeout or another exceptional condition occurred, and + # the program is still running. + if posix: + # Terminate the whole process group os.killpg(proc.pid, signal.SIGTERM) - else: - proc.terminate() - proc.wait() + for j in range(10): + time.sleep(0.1) + if proc.poll() is not None: + break + else: + # Didn't terminate within 1 sec, so kill it + os.killpg(proc.pid, signal.SIGTERM) + else: + proc.terminate() + proc.wait() + + proc.stdout.flush() + proc.stderr.flush() - proc.stdout.flush() - proc.stderr.flush() - stdout_chunks.append(proc.stdout.read()) - stderr_chunks.append(proc.stderr.read()) + stdout_chunks.append(proc.stdout.read()) + stderr_chunks.append(proc.stderr.read()) + stdout = b''.join(stdout_chunks) + stderr = b''.join(stderr_chunks) - stdout = b''.join(stdout_chunks).decode('utf-8', 'replace') - stderr = b''.join(stderr_chunks).decode('utf-8', 'replace') + stdout = stdout.decode('utf-8', 'replace') + stderr = stderr.decode('utf-8', 'replace') if is_timeout: retcode = TIMEOUT_RETCODE From 65ac5711a55fe40c4f427f7c985b977dc4bd47d4 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 16:01:07 +0300 Subject: [PATCH 05/27] Fix up executable discovery for virtualenv & conda on Windows --- asv/plugins/conda.py | 33 ++++++++++++++++++----- asv/plugins/virtualenv.py | 55 ++++++++++++++++++++++++++++++--------- test/test_environment.py | 2 +- 3 files changed, 69 insertions(+), 21 deletions(-) diff --git a/asv/plugins/conda.py b/asv/plugins/conda.py index d7ee3a973..40a100e07 100644 --- a/asv/plugins/conda.py +++ b/asv/plugins/conda.py @@ -14,6 +14,9 @@ from .. import util +WIN = (os.name == "nt") + + class Conda(environment.Environment): """ Manage an environment using conda. @@ -75,8 +78,9 @@ def get_environments(cls, conf, python): def check_presence(self): if not super(Conda, self).check_presence(): return False - for fn in ['pip', 'python']: - if not os.path.isfile(os.path.join(self._path, 'bin', fn)): + for executable in ['pip', 'python']: + exe = self._find_executable(executable) + if not os.path.isfile(exe): return False try: self._run_executable('python', ['-c', 'pass']) @@ -102,9 +106,9 @@ def _setup(self): 'pip']) log.info("Installing requirements for {0}".format(self.name)) - self._install_requirements() + self._install_requirements(conda) - def _install_requirements(self): + def _install_requirements(self, conda): self.install('wheel') if self._requirements: @@ -117,11 +121,26 @@ def _install_requirements(self): args.append("{0}={1}".format(key, val)) else: args.append(key) - self._run_executable('conda', args) + + util.check_output([conda] + args) + + def _find_executable(self, executable): + """Find an executable in the environment""" + if WIN: + executable += ".exe" + + exe = os.path.join(self._path, 'Scripts', executable) + if os.path.isfile(exe): + return exe + exe = os.path.join(self._path, executable) + if os.path.isfile(exe): + return exe + + return os.path.join(self._path, 'bin', executable) def _run_executable(self, executable, args, **kwargs): - return util.check_output([ - os.path.join(self._path, 'bin', executable)] + args, **kwargs) + exe = self._find_executable(executable) + return util.check_output([exe] + args, **kwargs) def install(self, package): log.info("Installing into {0}".format(self.name)) diff --git a/asv/plugins/virtualenv.py b/asv/plugins/virtualenv.py index 801048514..a626722a1 100644 --- a/asv/plugins/virtualenv.py +++ b/asv/plugins/virtualenv.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, unicode_literals, print_function from distutils.version import LooseVersion +import sys import inspect import os import subprocess @@ -15,6 +16,9 @@ from .. import util +WIN = (os.name == "nt") + + class Virtualenv(environment.Environment): """ Manage an environment using virtualenv. @@ -54,11 +58,25 @@ def __init__(self, conf, python, executable, requirements): self._virtualenv_path = os.path.abspath( inspect.getsourcefile(virtualenv)) - @classmethod - def get_environments(cls, conf, python): + @staticmethod + def _find_python(python): + """Find Python executable for the given Python version""" + # Find Python executable on path try: - executable = util.which('python{0}'.format(python)) + return util.which('python{0}'.format(python)) except IOError: + pass + + # Maybe the current one is correct + if '{0[0]}.{0[1]}'.format(sys.version_info) == python: + return sys.executable + + return None + + @classmethod + def get_environments(cls, conf, python): + executable = Virtualenv._find_python(python) + if executable is None: log.warn("No executable found for python {0}".format(python)) else: for configuration in environment.iter_configuration_matrix(conf.matrix): @@ -79,18 +97,15 @@ def matches(self, python): log.warn( "If using virtualenv, it much be at least version 1.10") - try: - util.which('python{0}'.format(python)) - except IOError: - return False - else: - return True + executable = Virtualenv._find_python(python) + return executable is not None def check_presence(self): if not super(Virtualenv, self).check_presence(): return False - for fn in ['pip', 'python']: - if not os.path.isfile(os.path.join(self._path, 'bin', fn)): + for executable in ['pip', 'python']: + exe = self._find_executable(executable) + if not os.path.isfile(exe): return False try: self._run_executable('python', ['-c', 'pass']) @@ -126,9 +141,23 @@ def _install_requirements(self): args.append(key) self._run_executable('pip', args) + def _find_executable(self, executable): + """Find an executable in the environment""" + if WIN: + executable += ".exe" + + exe = os.path.join(self._path, 'Scripts', executable) + if os.path.isfile(exe): + return exe + exe = os.path.join(self._path, executable) + if os.path.isfile(exe): + return exe + + return os.path.join(self._path, 'bin', executable) + def _run_executable(self, executable, args, **kwargs): - return util.check_output([ - os.path.join(self._path, 'bin', executable)] + args, **kwargs) + exe = self._find_executable(executable) + return util.check_output([exe] + args, **kwargs) def install(self, package): log.info("Installing into {0}".format(self.name)) diff --git a/test/test_environment.py b/test/test_environment.py index f98d2c34d..e64660486 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -79,7 +79,7 @@ def test_large_environment_matrix(tmpdir): # this test run a long time, we only set up the environment, # but don't actually install dependencies into it. This is # enough to trigger the bug in #169. - env._install_requirements = lambda: None + env._install_requirements = lambda *a: None env.create() From efbde6f275e9f4496d417a6b1f86e6a015322642 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 16:03:45 +0300 Subject: [PATCH 06/27] Fix up memory benchmark test on Windows --- test/test_benchmarks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_benchmarks.py b/test/test_benchmarks.py index d5957e1e7..79ff5c46d 100644 --- a/test/test_benchmarks.py +++ b/test/test_benchmarks.py @@ -68,7 +68,7 @@ def test_find_benchmarks(tmpdir): assert times[ 'subdir.time_subdir.time_foo']['result'] is not None assert times[ - 'mem_examples.mem_list']['result'] > 2000 + 'mem_examples.mem_list']['result'] > 1000 assert times[ 'time_secondary.track_value']['result'] == 42.0 assert 'profile' in times[ From 7e5f51e67b41d1f07a3d05fc48f780d02eb7d9c7 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 16:24:09 +0300 Subject: [PATCH 07/27] Fix issues in test_environment on windows --- test/test_environment.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/test/test_environment.py b/test/test_environment.py index e64660486..b89cb84a9 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -5,6 +5,7 @@ unicode_literals) import os +import sys import six import pytest @@ -13,18 +14,21 @@ from asv import util +WIN = (os.name == "nt") + + try: util.which('python2.7') HAS_PYTHON_27 = True -except RuntimeError: - HAS_PYTHON_27 = False +except (RuntimeError, IOError): + HAS_PYTHON_27 = (sys.version_info[:2] == (2, 7)) try: util.which('python3.4') HAS_PYTHON_34 = True -except RuntimeError: - HAS_PYTHON_34 = False +except (RuntimeError, IOError): + HAS_PYTHON_34 = (sys.version_info[:2] == (3, 4)) @pytest.mark.xfail(not HAS_PYTHON_27 or not HAS_PYTHON_34, @@ -109,8 +113,23 @@ def test_presence_checks(tmpdir): env.run(['-c', 'import os']) # Check env is recreated if crucial things are missing - pip_fn = os.path.join(env._path, 'bin', 'pip') - os.remove(pip_fn) + pip_fns = [ + os.path.join(env._path, 'bin', 'pip') + ] + if WIN: + pip_fns += [ + os.path.join(env._path, 'bin', 'pip.exe'), + os.path.join(env._path, 'Scripts', 'pip'), + os.path.join(env._path, 'Scripts', 'pip.exe') + ] + + some_removed = False + for pip_fn in pip_fns: + if os.path.isfile(pip_fn): + some_removed = True + os.remove(pip_fn) + assert some_removed + env._is_setup = False env.create() assert os.path.isfile(pip_fn) From 9344e17c131e5b42fa4e145685818b9d3d1ec6d1 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 16:24:54 +0300 Subject: [PATCH 08/27] Use colorama instead of psutil for tests, because psutil is not pure-Python --- test/test_dev.py | 2 +- test/test_environment.py | 6 +++--- test/test_web.py | 2 +- test/test_workflow.py | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/test_dev.py b/test/test_dev.py index fbf8486f8..5458755c0 100644 --- a/test/test_dev.py +++ b/test/test_dev.py @@ -40,7 +40,7 @@ def basic_conf(tmpdir): 'project': 'asv', 'matrix': { "six": [None], - "psutil": ["1.2", "2.1"] + "colorama": ["0.3.1", "0.3.3"] } }) diff --git a/test/test_environment.py b/test/test_environment.py index b89cb84a9..594287931 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -41,7 +41,7 @@ def test_matrix_environments(tmpdir): conf.pythons = ["2.7", "3.4"] conf.matrix = { "six": ["1.4", None], - "psutil": ["1.2", "2.1"] + "colorama": ["0.3.1", "0.3.3"] } environments = list(environment.get_environments(conf)) @@ -58,8 +58,8 @@ def test_matrix_environments(tmpdir): assert output.startswith(six.text_type(env._requirements['six'])) output = env.run( - ['-c', 'import psutil, sys; sys.stdout.write(psutil.__version__)']) - assert output.startswith(six.text_type(env._requirements['psutil'])) + ['-c', 'import colorama, sys; sys.stdout.write(colorama.__version__)']) + assert output.startswith(six.text_type(env._requirements['colorama'])) @pytest.mark.xfail(not HAS_PYTHON_27, diff --git a/test/test_web.py b/test/test_web.py index 38e155e9e..238a84b8f 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -50,7 +50,7 @@ def basic_html(request): 'project': 'asv', 'matrix': { "six": [None], - "psutil": ["1.2", "2.1"] + "colorama": ["0.3.1", "0.3.3"] } }) diff --git a/test/test_workflow.py b/test/test_workflow.py index ef6ebeb96..d53ed2a30 100644 --- a/test/test_workflow.py +++ b/test/test_workflow.py @@ -61,7 +61,7 @@ def basic_conf(tmpdir): 'project': 'asv', 'matrix': { "six": [None], - "psutil": ["1.2", "2.1"] + "colorama": ["0.3.1", "0.3.3"] } }) @@ -89,8 +89,8 @@ def test_run_publish(capfd, basic_conf): # Check parameterized test json data format filename = glob.glob(join(tmpdir, 'html', 'graphs', 'arch-x86_64', 'branch-master', - 'cpu-Blazingly fast', 'machine-orangutan', 'os-GNU', - 'Linux', 'psutil-2.1', 'python-*', 'ram-128GB', + 'colorama-0.3.3', 'cpu-Blazingly fast', 'machine-orangutan', + 'os-GNU', 'Linux', 'python-*', 'ram-128GB', 'six', 'params_examples.time_skip.json'))[0] with open(filename, 'r') as fp: data = json.load(fp) @@ -167,8 +167,8 @@ def _test_run_branches(tmpdir, dvcs, conf, machine_file, range_spec, # Check that files for all commits expected were generated expected = set(['machine.json']) for commit in commits: - for psver in ['1.2', '2.1']: - expected.add('{0}-py{1[0]}.{1[1]}-psutil{2}-six.json'.format( + for psver in ['0.3.1', '0.3.3']: + expected.add('{0}-py{1[0]}.{1[1]}-colorama{2}-six.json'.format( commit[:8], sys.version_info, psver)) result_files = os.listdir(join(tmpdir, 'results_workflow', 'orangutan')) From 7fa11ffc1eeba1f72b7cfa7999321b474ff9f725 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 16:45:06 +0300 Subject: [PATCH 09/27] Work around the 260 char PATH_MAX on Windows The place where this easily goes over is in graph data files, which are stored in a deep directory tree. --- asv/commands/publish.py | 2 +- asv/util.py | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/asv/commands/publish.py b/asv/commands/publish.py index eed8c53c2..0fb5cd899 100644 --- a/asv/commands/publish.py +++ b/asv/commands/publish.py @@ -125,7 +125,7 @@ def run(cls, conf): log.set_nitems(5) if os.path.exists(conf.html_dir): - shutil.rmtree(conf.html_dir) + util.long_path_rmtree(conf.html_dir) benchmarks = Benchmarks.load(conf) diff --git a/asv/util.py b/asv/util.py index c219c6862..10c6dd413 100644 --- a/asv/util.py +++ b/asv/util.py @@ -20,6 +20,7 @@ import time import errno import threading +import shutil import six from six.moves import xrange @@ -486,7 +487,7 @@ def write_json(path, data, api_version=None): if api_version is not None: data['version'] = api_version - with open(path, 'w') as fd: + with long_path_open(path, 'w') as fd: json.dump(data, fd, indent=4, sort_keys=True) @@ -496,7 +497,7 @@ def load_json(path, api_version=None, cleanup=True): """ path = os.path.abspath(path) - with open(path, 'r') as fd: + with long_path_open(path, 'r') as fd: content = fd.read() if cleanup: @@ -786,3 +787,19 @@ def is_nan(x): if isinstance(x, float): return x != x return False + + +if not WIN: + long_path_open = open + long_path_rmtree = shutil.rmtree +else: + def _long_path_prefix(path): + if path.startswith("\\\\"): + return path + return "\\\\?\\" + os.path.abspath(path) + + def long_path_open(filename, *a, **kw): + return open(_long_path_prefix(filename), *a, **kw) + + def long_path_rmtree(path, *a, **kw): + shutil.rmtree(_long_path_prefix(path), *a, **kw) From 1b96efc88752c55f7171a047fe0df7b1b7817592 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 17:18:22 +0300 Subject: [PATCH 10/27] Fix up test failure in test_workflow on 32-bit Python 2.x --- test/test_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_workflow.py b/test/test_workflow.py index d53ed2a30..b4f9d0c16 100644 --- a/test/test_workflow.py +++ b/test/test_workflow.py @@ -95,7 +95,7 @@ def test_run_publish(capfd, basic_conf): with open(filename, 'r') as fp: data = json.load(fp) assert len(data) == 2 - assert isinstance(data[0][0], int) # date + assert isinstance(data[0][0], six.integer_types) # date assert len(data[0][1]) == 3 assert len(data[1][1]) == 3 assert isinstance(data[0][1][0], float) From 2f85a696a75c70d8815f86ab9cb1239ea5bec758 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 17:32:25 +0300 Subject: [PATCH 11/27] Fix right-aligning of benchmark results; Windows cmd terminal wraps one character early --- asv/benchmarks.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/asv/benchmarks.py b/asv/benchmarks.py index 935c49f7a..26c16daf6 100644 --- a/asv/benchmarks.py +++ b/asv/benchmarks.py @@ -20,6 +20,9 @@ from . import util +WIN = (os.name == "nt") + + # Can't use benchmark.__file__, because that points to the compiled # file, so it can't be run by another version of Python. BENCHMARK_RUN_SCRIPT = os.path.join( @@ -75,7 +78,10 @@ def run_benchmark(benchmark, root, env, show_stderr=False, quick=False, log.info(initial_message) def log_result(msg): - padding = " "*(util.get_terminal_width() - len(initial_message) - 14 - 1 - len(msg)) + padding_length = util.get_terminal_width() - len(initial_message) - 14 - 1 - len(msg) + if WIN: + padding_length -= 1 + padding = " "*padding_length log.add(" {0}{1}".format(padding, msg)) with log.indent(): From 13628813e6eb4bdc890a3e23d47dad71a061d921 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 17:48:30 +0300 Subject: [PATCH 12/27] Fix multiprocessing test failures, when run via python setup.py test on spawn/forkserver mode (on Windows) --- setup.py | 138 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 70 insertions(+), 68 deletions(-) diff --git a/setup.py b/setup.py index 469feae89..ed076900a 100755 --- a/setup.py +++ b/setup.py @@ -89,71 +89,73 @@ def write_version_file(filename, version, revision): version = '0.2.dev' -git_hash = get_git_hash() - -# Indicates if this version is a release version -release = 'dev' not in version - -if not release: - version = '{0}{1}+{2}'.format( - version, get_git_revision(), git_hash[:8]) - -write_version_file( - os.path.join(basedir, 'asv', '_version.py'), version, git_hash) - - -# Install entry points for making releases with zest.releaser -entry_points = {} -for hook in [('releaser', 'middle'), ('postreleaser', 'before')]: - hook_ep = 'zest.releaser.' + '.'.join(hook) - hook_name = 'asv.release.' + '.'.join(hook) - hook_func = 'asv._release:' + '_'.join(hook) - entry_points[hook_ep] = ['%s = %s' % (hook_name, hook_func)] - -entry_points['console_scripts'] = ['asv = asv.main:main'] - - -setup( - name="asv", - version=version, - packages=['asv', - 'asv.commands', - 'asv.plugins', - 'asv.extern', - 'asv._release'], - entry_points=entry_points, - - install_requires=[ - str('six>=1.4') - ], - - extras_require={ - str('hg'): ["python-hglib>=1.5"] - }, - - package_data={ - str('asv'): [ - 'www/*.html', - 'www/*.js', - 'www/*.css', - 'www/*.png', - 'www/*.ico', - 'www/flot/*.js', - 'template/__init__.py', - 'template/asv.conf.json', - 'template/benchmarks/*.py' - ] - }, - - zip_safe=False, - - # py.test testing - tests_require=['pytest'], - cmdclass={'test': PyTest}, - - author="Michael Droettboom", - author_email="mdroe@stsci.edu", - description="Airspeed Velocity: A simple Python history benchmarking tool", - license="BSD", - url="http://github.com/spacetelescope/asv" -) + +if __name__ == "__main__": + git_hash = get_git_hash() + + # Indicates if this version is a release version + release = 'dev' not in version + + if not release: + version = '{0}{1}+{2}'.format( + version, get_git_revision(), git_hash[:8]) + + write_version_file( + os.path.join(basedir, 'asv', '_version.py'), version, git_hash) + + + # Install entry points for making releases with zest.releaser + entry_points = {} + for hook in [('releaser', 'middle'), ('postreleaser', 'before')]: + hook_ep = 'zest.releaser.' + '.'.join(hook) + hook_name = 'asv.release.' + '.'.join(hook) + hook_func = 'asv._release:' + '_'.join(hook) + entry_points[hook_ep] = ['%s = %s' % (hook_name, hook_func)] + + entry_points['console_scripts'] = ['asv = asv.main:main'] + + + setup( + name="asv", + version=version, + packages=['asv', + 'asv.commands', + 'asv.plugins', + 'asv.extern', + 'asv._release'], + entry_points=entry_points, + + install_requires=[ + str('six>=1.4') + ], + + extras_require={ + str('hg'): ["python-hglib>=1.5"] + }, + + package_data={ + str('asv'): [ + 'www/*.html', + 'www/*.js', + 'www/*.css', + 'www/*.png', + 'www/*.ico', + 'www/flot/*.js', + 'template/__init__.py', + 'template/asv.conf.json', + 'template/benchmarks/*.py' + ] + }, + + zip_safe=False, + + # py.test testing + tests_require=['pytest'], + cmdclass={'test': PyTest}, + + author="Michael Droettboom", + author_email="mdroe@stsci.edu", + description="Airspeed Velocity: A simple Python history benchmarking tool", + license="BSD", + url="http://github.com/spacetelescope/asv" + ) From 5fa41ed81c885f9b3d50ac0b8e455c5f2ec2c3d0 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 18:32:16 +0300 Subject: [PATCH 13/27] Ensure setup.py test passes the given extra arguments to pytest --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ed076900a..cede3266a 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ class PyTest(TestCommand): def initialize_options(self): TestCommand.initialize_options(self) - self.pytest_args = [] + self.pytest_args = '' def finalize_options(self): TestCommand.finalize_options(self) @@ -26,6 +26,8 @@ def finalize_options(self): def run_tests(self): import pytest + if self.pytest_args: + self.test_args += self.pytest_args.split() errno = pytest.main(self.test_args) sys.exit(errno) From 6d4859444404049c15acd81c018f7a64faca2f27 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 18:32:45 +0300 Subject: [PATCH 14/27] Run pytest in appveyor with --tb=native to avoid pytest bug Traceback parsing has bugs on some pytest versions, cf. https://bitbucket.org/pytest-dev/py/issue/55/broken-comment-parsin --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 7848f7e75..ca3873166 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -44,4 +44,4 @@ install: build: false test_script: - - "%CMD_IN_ENV% python setup.py test" \ No newline at end of file + - "%CMD_IN_ENV% python setup.py test -a --tb=native" From 1d84c836d368f244d2cc1ea11ca47416d8b2fd21 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 22:31:45 +0300 Subject: [PATCH 15/27] Try harder to remove directories on Windows --- asv/environment.py | 4 ++-- asv/util.py | 17 +++++++++++++++-- asv/wheel_cache.py | 4 ++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/asv/environment.py b/asv/environment.py index a03894ec3..ffda4c1ac 100644 --- a/asv/environment.py +++ b/asv/environment.py @@ -259,7 +259,7 @@ def create(self): if not self.check_presence(): if os.path.exists(self._path): - shutil.rmtree(self._path) + util.long_path_rmtree(self._path) if not os.path.exists(self._env_dir): try: @@ -279,7 +279,7 @@ def create(self): except: log.error("Failure creating environment for {0}".format(self.name)) if os.path.exists(self._path): - shutil.rmtree(self._path) + util.long_path_rmtree(self._path) raise self.save_info_file(self._path) diff --git a/asv/util.py b/asv/util.py index 10c6dd413..0cd507674 100644 --- a/asv/util.py +++ b/asv/util.py @@ -21,6 +21,7 @@ import errno import threading import shutil +import stat import six from six.moves import xrange @@ -798,8 +799,20 @@ def _long_path_prefix(path): return path return "\\\\?\\" + os.path.abspath(path) + def _remove_readonly(func, path, exc_info): + """Clear the readonly bit and reattempt the removal; + Windows rmtree doesn't do this by default""" + os.chmod(path, stat.S_IWRITE) + func(path) + def long_path_open(filename, *a, **kw): return open(_long_path_prefix(filename), *a, **kw) - def long_path_rmtree(path, *a, **kw): - shutil.rmtree(_long_path_prefix(path), *a, **kw) + def long_path_rmtree(path, ignore_errors=False): + if ignore_errors: + onerror = None + else: + onerror = _remove_readonly + shutil.rmtree(_long_path_prefix(path), + ignore_errors=ignore_errors, + onerror=onerror) diff --git a/asv/wheel_cache.py b/asv/wheel_cache.py index ce97e7741..f59d1a3d0 100644 --- a/asv/wheel_cache.py +++ b/asv/wheel_cache.py @@ -69,7 +69,7 @@ def _cleanup_wheel_cache(self): for name in names[self._wheel_cache_size:]: path = os.path.join(self._path, name) if os.path.isdir(path): - shutil.rmtree(path) + util.long_path_rmtree(path) def build_project_cached(self, env, package, commit_hash): if self._wheel_cache_size == 0: @@ -90,7 +90,7 @@ def build_project_cached(self, env, package, commit_hash): '--no-deps', '--no-index', build_root]) except util.ProcessError: # failed -- clean up - shutil.rmtree(cache_path) + util.long_path_rmtree(cache_path) raise return self._get_wheel(commit_hash) From c74a5588a65aca731094ce958f6b5ffcc9c32caa Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sat, 27 Jun 2015 22:37:24 +0300 Subject: [PATCH 16/27] Fix division by zero in asv continuous --- asv/commands/continuous.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/asv/commands/continuous.py b/asv/commands/continuous.py index 2b7691d5c..b9016a90b 100644 --- a/asv/commands/continuous.py +++ b/asv/commands/continuous.py @@ -107,7 +107,14 @@ def run(cls, conf, branch=None, base=None, factor=2.0, show_stderr=False, bench= table = [] slowed_down = False for name, benchmark in six.iteritems(all_benchmarks): - change = after[name] / before[name] + if before[name] == 0: + if after[name] == 0: + change = 1.0 + else: + change = float('inf') + else: + change = after[name] / before[name] + if change > factor or change < 1.0 / factor: table.append( (change, before[name], after[name], name, benchmark)) From 413fe9df923aff211d41e0b138af068d79713d47 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 28 Jun 2015 01:31:34 +0300 Subject: [PATCH 17/27] Disable colorama colors on Windows -- too many failure modes in practice --- asv/console.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/asv/console.py b/asv/console.py index d72d9ca3a..5f0b7af6a 100644 --- a/asv/console.py +++ b/asv/console.py @@ -23,14 +23,6 @@ WIN = (os.name == "nt") -if WIN: - try: - import colorama - except ImportError: - colorama = None - warnings.warn('the colorama package is required for terminal color on Windows') - - def isatty(file): """ Returns `True` if `file` is a tty. @@ -183,11 +175,7 @@ def color_print(*args, **kwargs): end = kwargs.get('end', '') write = file.write - if isatty(file) and (not WIN or colorama): - if WIN: - file = colorama.AnsiToWin32(file).stream - write = file.write - + if isatty(file) and not WIN: for i in xrange(0, len(args), 2): msg = args[i] if i + 1 == len(args): From b685890e2a06414b3b8b0a5a7e41ee14e962035f Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 28 Jun 2015 19:36:19 +0300 Subject: [PATCH 18/27] Mark a failing test on Windows/Appveyor as xfail It appears the Python DLLs in created (Conda) environments stay in use when run on Windows/Appveyor, so that removing the environments fails and their re-creation cannot be tested. Does not occur on all Windows platoforms, however, and adding sleeps does not seem to resolve it. --- test/test_environment.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_environment.py b/test/test_environment.py index 594287931..900a79c80 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -89,6 +89,11 @@ def test_large_environment_matrix(tmpdir): @pytest.mark.xfail(not HAS_PYTHON_27, reason="Requires Python 2.7") +@pytest.mark.xfail(WIN, + reason=("Fails on some Windows installations; the Python DLLs " + "in the created environments are apparently not unloaded " + "properly so that removing the environments fails. This is " + "likely not a very common occurrence in real use cases.")) def test_presence_checks(tmpdir): conf = config.Config() From f353c7b1e933000abd9f23e0c3ac53e0a46e0d61 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 28 Jun 2015 20:33:49 +0300 Subject: [PATCH 19/27] Skip multiprocessing using tests when run from py.test on Windows --- test/test_util.py | 9 +++++++++ test/test_workflow.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/test/test_util.py b/test/test_util.py index 4d7a39192..0078c2370 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -11,10 +11,14 @@ import multiprocessing import subprocess import traceback +import pytest from asv import util +WIN = (os.name == 'nt') + + def _multiprocessing_raise_processerror(arg): try: raise util.ProcessError(["a"], 1, "aa", "bb") @@ -34,6 +38,11 @@ def _multiprocessing_raise_usererror(arg): def test_parallelfailure(): # Check the workaround for https://bugs.python.org/issue9400 works + if WIN and os.path.basename(sys.argv[0]).lower().startswith('py.test'): + # Multiprocessing in spawn mode can result to problems with py.test + pytest.skip("Multiprocessing spawn mode on Windows not safe to run " + "from py.test runner.") + # The exception class must be pickleable exc = util.ParallelFailure("test", Exception, "something") exc2 = pickle.loads(pickle.dumps(exc)) diff --git a/test/test_workflow.py b/test/test_workflow.py index b4f9d0c16..617226cda 100644 --- a/test/test_workflow.py +++ b/test/test_workflow.py @@ -24,6 +24,9 @@ from . import tools +WIN = (os.name == 'nt') + + dummy_values = [ (None, None), (1, 1), @@ -140,6 +143,12 @@ def test_continuous(capfd, basic_conf): def test_find(capfd, basic_conf): tmpdir, local, conf, machine_file = basic_conf + if WIN and os.path.basename(sys.argv[0]).lower().startswith('py.test'): + # Multiprocessing in spawn mode can result to problems with py.test + # Find.run calls Setup.run in parallel mode by default + pytest.skip("Multiprocessing spawn mode on Windows not safe to run " + "from py.test runner.") + # Test find at least runs Find.run(conf, "master~5..master", "params_examples.track_find_test", _machine_file=machine_file) From cc6accaee3dbaace0b0d60285dccf883f291be34 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 28 Jun 2015 23:10:28 +0300 Subject: [PATCH 20/27] Convert xrange to while loop, because xrange on 32-bit Python 2.x cannot handle long ints --- asv/graph.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/asv/graph.py b/asv/graph.py index 99fcdeba0..7dc92da18 100644 --- a/asv/graph.py +++ b/asv/graph.py @@ -118,13 +118,15 @@ def resample_data(self, val): new_val = [] j = 0 - for i in xrange(min_time + step_size, max_time + step_size, step_size): + i = min_time + step_size + while i < max_time + step_size: chunk = [] while j < len(val) and val[j][0] < i: chunk.append(val[j][1]) j += 1 if len(chunk): new_val.append((i, _mean_with_none(chunk))) + i += step_size return new_val def get_data(self): From 35d8c99d95e88c340f2bb7f05675fdd9da523e11 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 28 Jun 2015 23:20:29 +0300 Subject: [PATCH 21/27] Tell jquery to parse incoming data as json, for json files (asv preview server on windows doesn't seem to serve them right) --- asv/www/asv.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/asv/www/asv.js b/asv/www/asv.js index c4624bfb1..e8bad4e1b 100644 --- a/asv/www/asv.js +++ b/asv/www/asv.js @@ -208,6 +208,7 @@ $(function() { based on it. */ $.ajax({ url: "index.json", + dataType: "json", cache: false }).done(function (index) { master_json = index; @@ -527,6 +528,7 @@ $(function() { callback_in_view(plot_div, function() { $.ajax({ url: 'graphs/summary/' + bm_name + '.json', + dataType: "json", cache: false }).done(function(data) { var options = { @@ -1028,6 +1030,7 @@ $(function() { $.each(to_load, function(i, item) { $.ajax({ url: item[0], + dataType: "json", cache: false }).done(function(data) { $.each(item[1], function(j, graph_content) { @@ -1052,6 +1055,7 @@ $(function() { is probably down. */ $.ajax({ url: "swallow.ico", + dataType: "text", cache: false }).done(function (index) { update_graphs(); From 9eda5f5f2eb9bcca6d2b5c0d7d03fe4957132a93 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Sun, 28 Jun 2015 23:25:57 +0300 Subject: [PATCH 22/27] Fix up serving long path names on Windows for asv preview --- asv/commands/preview.py | 7 ++++++- asv/util.py | 8 +++++--- test/tools.py | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/asv/commands/preview.py b/asv/commands/preview.py index dce46806a..3ecac03f2 100644 --- a/asv/commands/preview.py +++ b/asv/commands/preview.py @@ -81,7 +81,12 @@ def run_from_conf_args(cls, conf, args): def run(cls, conf, port=0, browser=False): os.chdir(conf.html_dir) - Handler = SimpleHTTPServer.SimpleHTTPRequestHandler + class Handler(SimpleHTTPServer.SimpleHTTPRequestHandler): + def translate_path(self, path): + path = SimpleHTTPServer.SimpleHTTPRequestHandler.translate_path( + self, path) + return util.long_path(path) + httpd, base_url = create_httpd(Handler, port=port) log.info("Serving at {0}".format(base_url)) diff --git a/asv/util.py b/asv/util.py index 0cd507674..d6b1a535f 100644 --- a/asv/util.py +++ b/asv/util.py @@ -793,8 +793,10 @@ def is_nan(x): if not WIN: long_path_open = open long_path_rmtree = shutil.rmtree + def long_path(path): + return path else: - def _long_path_prefix(path): + def long_path(path): if path.startswith("\\\\"): return path return "\\\\?\\" + os.path.abspath(path) @@ -806,13 +808,13 @@ def _remove_readonly(func, path, exc_info): func(path) def long_path_open(filename, *a, **kw): - return open(_long_path_prefix(filename), *a, **kw) + return open(long_path(filename), *a, **kw) def long_path_rmtree(path, ignore_errors=False): if ignore_errors: onerror = None else: onerror = _remove_readonly - shutil.rmtree(_long_path_prefix(path), + shutil.rmtree(long_path(path), ignore_errors=ignore_errors, onerror=onerror) diff --git a/test/tools.py b/test/tools.py index 47f500ba7..503aa481e 100644 --- a/test/tools.py +++ b/test/tools.py @@ -316,7 +316,7 @@ def translate_path(self, path): # Don't serve from cwd, but from a different directory path = SimpleHTTPServer.SimpleHTTPRequestHandler.translate_path(self, path) path = os.path.join(base_path, os.path.relpath(path, os.getcwd())) - return path + return util.long_path(path) httpd, base_url = create_httpd(Handler) From aae7a03c8f5d6301211253adad4b5870e6c5453f Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Mon, 29 Jun 2015 00:03:59 +0300 Subject: [PATCH 23/27] Refactor Conda/Virtualenv by moving common helper methods to Environment base class --- asv/environment.py | 46 ++++++++++++++++++++++++++++++++++++--- asv/plugins/conda.py | 41 +++++----------------------------- asv/plugins/virtualenv.py | 43 +++++------------------------------- asv/wheel_cache.py | 2 +- 4 files changed, 55 insertions(+), 77 deletions(-) diff --git a/asv/environment.py b/asv/environment.py index ffda4c1ac..282993e85 100644 --- a/asv/environment.py +++ b/asv/environment.py @@ -13,6 +13,7 @@ import os import shutil import sys +import subprocess import six @@ -22,6 +23,9 @@ from . import wheel_cache +WIN = (os.name == "nt") + + def iter_configuration_matrix(matrix): """ Iterate through all combinations of the given configuration @@ -244,9 +248,20 @@ def check_presence(self): 'python': self._python, 'requirements': self._requirements } + if info != expected_info: return False + for executable in ['pip', 'python']: + exe = self.find_executable(executable) + if not os.path.isfile(exe): + return False + + try: + self.run_executable('python', ['-c', 'pass']) + except (subprocess.CalledProcessError, OSError): + return False + return True def create(self): @@ -275,7 +290,7 @@ def create(self): pass try: - self._setup() + self.setup() except: log.error("Failure creating environment for {0}".format(self.name)) if os.path.exists(self._path): @@ -286,7 +301,7 @@ def create(self): self._is_setup = True - def _setup(self): + def setup(self): """ Implementation for setting up the environment. """ @@ -353,6 +368,31 @@ def can_install_project(self): """ return True + def find_executable(self, executable): + """ + Find an executable (eg. python, pip) in the environment. + """ + + # Assume standard virtualenv/Conda layout + if WIN: + executable += ".exe" + + exe = os.path.join(self._path, 'Scripts', executable) + if os.path.isfile(exe): + return exe + exe = os.path.join(self._path, executable) + if os.path.isfile(exe): + return exe + + return os.path.join(self._path, 'bin', executable) + + def run_executable(self, executable, args, **kwargs): + """ + Run a given executable (eg. python, pip) in the environment. + """ + exe = self.find_executable(executable) + return util.check_output([exe] + args, **kwargs) + def load_info_file(self, path): path = os.path.join(path, 'asv-env-info.json') return util.load_json(path) @@ -414,7 +454,7 @@ def check_presence(self): def create(self): pass - def _setup(self): + def setup(self): pass def install_project(self, conf, commit_hash=None): diff --git a/asv/plugins/conda.py b/asv/plugins/conda.py index 40a100e07..17125d417 100644 --- a/asv/plugins/conda.py +++ b/asv/plugins/conda.py @@ -75,20 +75,7 @@ def get_environments(cls, conf, python): for configuration in environment.iter_configuration_matrix(conf.matrix): yield cls(conf, python, configuration) - def check_presence(self): - if not super(Conda, self).check_presence(): - return False - for executable in ['pip', 'python']: - exe = self._find_executable(executable) - if not os.path.isfile(exe): - return False - try: - self._run_executable('python', ['-c', 'pass']) - except (subprocess.CalledProcessError, OSError): - return False - return True - - def _setup(self): + def setup(self): try: conda = util.which('conda') except IOError as e: @@ -124,33 +111,15 @@ def _install_requirements(self, conda): util.check_output([conda] + args) - def _find_executable(self, executable): - """Find an executable in the environment""" - if WIN: - executable += ".exe" - - exe = os.path.join(self._path, 'Scripts', executable) - if os.path.isfile(exe): - return exe - exe = os.path.join(self._path, executable) - if os.path.isfile(exe): - return exe - - return os.path.join(self._path, 'bin', executable) - - def _run_executable(self, executable, args, **kwargs): - exe = self._find_executable(executable) - return util.check_output([exe] + args, **kwargs) - def install(self, package): log.info("Installing into {0}".format(self.name)) - self._run_executable('pip', ['install', package]) + self.run_executable('pip', ['install', package]) def uninstall(self, package): log.info("Uninstalling from {0}".format(self.name)) - self._run_executable('pip', ['uninstall', '-y', package], - valid_return_codes=None) + self.run_executable('pip', ['uninstall', '-y', package], + valid_return_codes=None) def run(self, args, **kwargs): log.debug("Running '{0}' in {1}".format(' '.join(args), self.name)) - return self._run_executable('python', args, **kwargs) + return self.run_executable('python', args, **kwargs) diff --git a/asv/plugins/virtualenv.py b/asv/plugins/virtualenv.py index a626722a1..a1263b376 100644 --- a/asv/plugins/virtualenv.py +++ b/asv/plugins/virtualenv.py @@ -100,20 +100,7 @@ def matches(self, python): executable = Virtualenv._find_python(python) return executable is not None - def check_presence(self): - if not super(Virtualenv, self).check_presence(): - return False - for executable in ['pip', 'python']: - exe = self._find_executable(executable) - if not os.path.isfile(exe): - return False - try: - self._run_executable('python', ['-c', 'pass']) - except (subprocess.CalledProcessError, OSError): - return False - return True - - def _setup(self): + def setup(self): """ Setup the environment on disk using virtualenv. Then, all of the requirements are installed into @@ -130,7 +117,7 @@ def _setup(self): self._install_requirements() def _install_requirements(self): - self._run_executable('pip', ['install', 'wheel']) + self.run_executable('pip', ['install', 'wheel']) if self._requirements: args = ['install', '--upgrade'] @@ -139,35 +126,17 @@ def _install_requirements(self): args.append("{0}=={1}".format(key, val)) else: args.append(key) - self._run_executable('pip', args) - - def _find_executable(self, executable): - """Find an executable in the environment""" - if WIN: - executable += ".exe" - - exe = os.path.join(self._path, 'Scripts', executable) - if os.path.isfile(exe): - return exe - exe = os.path.join(self._path, executable) - if os.path.isfile(exe): - return exe - - return os.path.join(self._path, 'bin', executable) - - def _run_executable(self, executable, args, **kwargs): - exe = self._find_executable(executable) - return util.check_output([exe] + args, **kwargs) + self.run_executable('pip', args) def install(self, package): log.info("Installing into {0}".format(self.name)) - self._run_executable('pip', ['install', package]) + self.run_executable('pip', ['install', package]) def uninstall(self, package): log.info("Uninstalling from {0}".format(self.name)) - self._run_executable('pip', ['uninstall', '-y', package], + self.run_executable('pip', ['uninstall', '-y', package], valid_return_codes=None) def run(self, args, **kwargs): log.debug("Running '{0}' in {1}".format(' '.join(args), self.name)) - return self._run_executable('python', args, **kwargs) + return self.run_executable('python', args, **kwargs) diff --git a/asv/wheel_cache.py b/asv/wheel_cache.py index f59d1a3d0..be9783d35 100644 --- a/asv/wheel_cache.py +++ b/asv/wheel_cache.py @@ -85,7 +85,7 @@ def build_project_cached(self, env, package, commit_hash): cache_path = self._create_wheel_cache_path(commit_hash) try: - env._run_executable( + env.run_executable( 'pip', ['wheel', '--wheel-dir', cache_path, '--no-deps', '--no-index', build_root]) except util.ProcessError: From 0d2a125e8bc6525dd5e3c455c58e73989aa79f11 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Mon, 29 Jun 2015 00:06:22 +0300 Subject: [PATCH 24/27] Enable running all test_environment tests if conda is available Mark the tests as skipped rather than xfail, since we expect them to pass if the environment is set up. --- test/test_environment.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/test/test_environment.py b/test/test_environment.py index 900a79c80..e37111b4d 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -31,8 +31,16 @@ HAS_PYTHON_34 = (sys.version_info[:2] == (3, 4)) -@pytest.mark.xfail(not HAS_PYTHON_27 or not HAS_PYTHON_34, - reason="Requires Python 2.7 and 3.4") +try: + # Conda can install Python 2.7 and 3.4 on demand + util.which('conda') + HAS_CONDA = True +except (RuntimeError, IOError): + HAS_CONDA = False + + +@pytest.mark.skipif(not ((HAS_PYTHON_27 and HAS_PYTHON_34) or HAS_CONDA), + reason="Requires Python 2.7 and 3.4") def test_matrix_environments(tmpdir): conf = config.Config() @@ -62,8 +70,8 @@ def test_matrix_environments(tmpdir): assert output.startswith(six.text_type(env._requirements['colorama'])) -@pytest.mark.xfail(not HAS_PYTHON_27, - reason="Requires Python 2.7") +@pytest.mark.skipif(not (HAS_PYTHON_27 or HAS_CONDA), + reason="Requires Python 2.7") def test_large_environment_matrix(tmpdir): # As seen in issue #169, conda can't handle using really long # directory names in its environment. This creates an environment @@ -87,8 +95,8 @@ def test_large_environment_matrix(tmpdir): env.create() -@pytest.mark.xfail(not HAS_PYTHON_27, - reason="Requires Python 2.7") +@pytest.mark.skipif(not (HAS_PYTHON_27 or HAS_CONDA), + reason="Requires Python 2.7") @pytest.mark.xfail(WIN, reason=("Fails on some Windows installations; the Python DLLs " "in the created environments are apparently not unloaded " From 0f0ef02e227c6115683662d2b0090d08ffb0432a Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Mon, 29 Jun 2015 00:18:27 +0300 Subject: [PATCH 25/27] Mark hglib requiring test as skipif rather than xfail --- test/test_repo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_repo.py b/test/test_repo.py index 06d8a4920..a556e9a1b 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -81,8 +81,8 @@ def test_repo_git(tmpdir): _test_branches(conf, branch_commits) -@pytest.mark.xfail(hglib is None, - reason="needs hglib") +@pytest.mark.skipif(hglib is None, + reason="needs hglib") def test_repo_hg(tmpdir): tmpdir = six.text_type(tmpdir) From 53457ec6728d6bb5a9f22c0a2de33bfdcc3ee353 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Mon, 29 Jun 2015 00:49:42 +0300 Subject: [PATCH 26/27] Add test checking check_presence doesn't return obvious false negatives --- test/test_environment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_environment.py b/test/test_environment.py index e37111b4d..0dcc3c7cc 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -113,6 +113,7 @@ def test_presence_checks(tmpdir): for env in environments: env.create() + assert env.check_presence() # Check env is recreated when info file is clobbered info_fn = os.path.join(env._path, 'asv-env-info.json') From d8809532404944a80a1a4de070678518ee55f28e Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Mon, 6 Jul 2015 10:54:51 +0300 Subject: [PATCH 27/27] Add comment explaining not using xrange + make Environment.setup private again --- asv/environment.py | 6 +++--- asv/graph.py | 4 ++++ asv/plugins/conda.py | 2 +- asv/plugins/virtualenv.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/asv/environment.py b/asv/environment.py index 282993e85..e89de9f80 100644 --- a/asv/environment.py +++ b/asv/environment.py @@ -290,7 +290,7 @@ def create(self): pass try: - self.setup() + self._setup() except: log.error("Failure creating environment for {0}".format(self.name)) if os.path.exists(self._path): @@ -301,7 +301,7 @@ def create(self): self._is_setup = True - def setup(self): + def _setup(self): """ Implementation for setting up the environment. """ @@ -454,7 +454,7 @@ def check_presence(self): def create(self): pass - def setup(self): + def _setup(self): pass def install_project(self, conf, commit_hash=None): diff --git a/asv/graph.py b/asv/graph.py index 7dc92da18..296c82709 100644 --- a/asv/graph.py +++ b/asv/graph.py @@ -116,6 +116,10 @@ def resample_data(self, val): max_time = max(x[0] for x in val) step_size = int((max_time - min_time) / RESAMPLED_POINTS) + # This loop cannot use xrange, because xrange on Python2 on + # 32-bit systems can only deal with 32-bit integers, and + # Javascript timestamps (1000*unix_timestamp) handled here + # overflow this range new_val = [] j = 0 i = min_time + step_size diff --git a/asv/plugins/conda.py b/asv/plugins/conda.py index 17125d417..a8205ffd8 100644 --- a/asv/plugins/conda.py +++ b/asv/plugins/conda.py @@ -75,7 +75,7 @@ def get_environments(cls, conf, python): for configuration in environment.iter_configuration_matrix(conf.matrix): yield cls(conf, python, configuration) - def setup(self): + def _setup(self): try: conda = util.which('conda') except IOError as e: diff --git a/asv/plugins/virtualenv.py b/asv/plugins/virtualenv.py index a1263b376..3834570ff 100644 --- a/asv/plugins/virtualenv.py +++ b/asv/plugins/virtualenv.py @@ -100,7 +100,7 @@ def matches(self, python): executable = Virtualenv._find_python(python) return executable is not None - def setup(self): + def _setup(self): """ Setup the environment on disk using virtualenv. Then, all of the requirements are installed into