Skip to content

Commit

Permalink
Merge pull request #893 from qwcode/fast_tests
Browse files Browse the repository at this point in the history 
use "fast" test environment for setuptools and distribute
  • Loading branch information
@qwcode
qwcode committed Apr 16, 2013
2 parents 2198b5d + 979412c commit 7176cd2
Show file tree
Hide file tree
Showing 3 changed files with 159 additions and 173 deletions.
2 changes: 1 addition & 1 deletion tests/test_find_links.py
View file
Expand Up @@ -24,7 +24,7 @@ def test_find_links_requirements_file_relative_path():
e = reset_env()
write_file('test-req.txt', textwrap.dedent("""
--no-index
--find-links=../../../packages/
--find-links=../../../../packages/
parent==0.1
"""))
result = run_pip(
Expand Down
242 changes: 85 additions & 157 deletions tests/test_pip.py
View file
@@ -1,4 +1,5 @@
#!/usr/bin/env python
import imp
import os
import sys
import re
Expand All @@ -12,7 +13,7 @@
from scripttest import TestFileEnvironment, FoundDir
from tests.path import Path, curdir, u
from pip.util import rmtree
from pip.backwardcompat import ssl
from pip.backwardcompat import ssl, uses_pycache

#allow py25 unit tests to work
if sys.version_info[:2] == (2, 5) and not ssl:
Expand Down Expand Up @@ -107,9 +108,8 @@ def install_setuptools(env):

env = None


def reset_env(environ=None,
use_distribute=None,
use_distribute=False,
system_site_packages=False,
sitecustomize=None,
insecure=True):
Expand All @@ -123,19 +123,26 @@ def reset_env(environ=None,
insecure: how to set the --insecure option for py25 tests.
"""

if sys.version_info >= (3,):
use_distribute = True

global env
# FastTestPipEnv reuses env, not safe if use_distribute specified
if use_distribute is None and not system_site_packages:
env = FastTestPipEnvironment(environ, sitecustomize=sitecustomize)

if use_distribute:
test_class = TestPipEnvironmentD
else:
env = TestPipEnvironment(environ, use_distribute=use_distribute, sitecustomize=sitecustomize)
test_class = TestPipEnvironment

env = test_class(environ, sitecustomize=sitecustomize)
test_class.rebuild_venv = False

if system_site_packages:
#testing often occurs starting from a private virtualenv (e.g. with tox)
#from that context, you can't successfully use virtualenv.create_environment
#to create a 'system-site-packages' virtualenv
#hence, this workaround
(env.lib_path/'no-global-site-packages.txt').rm()
test_class.rebuild_venv = True

if sys.version_info[:2] == (2, 5) and insecure:
#allow py25 tests to work
Expand Down Expand Up @@ -269,6 +276,10 @@ def assert_installed(self, pkg_name, editable=True, with_files=[], without_files
'unexpected content %f' % (pkg_dir, f))


fast_test_env_root = here / 'tests_cache' / 'test_ws'
fast_test_env_backup = here / 'tests_cache' / 'test_ws_backup'


class TestPipEnvironment(TestFileEnvironment):
"""A specialized TestFileEnvironment for testing pip"""

Expand All @@ -295,16 +306,20 @@ class TestPipEnvironment(TestFileEnvironment):
scratch = Path('scratch')

exe = sys.platform == 'win32' and '.exe' or ''

verbose = False
use_distribute = False
setuptools = 'setuptools'
rebuild_venv = True

def __init__(self, environ=None, use_distribute=None, sitecustomize=None):
def __init__(self, environ=None, sitecustomize=None):
import virtualenv

self.root_path = Path(tempfile.mkdtemp('-piptest'))
self.root_path = fast_test_env_root / self.setuptools
self.backup_path = fast_test_env_backup / self.setuptools

# We will set up a virtual environment at root_path.
self.scratch_path = self.root_path / self.scratch

# We will set up a virtual environment at root_path.
self.venv_path = self.root_path / self.venv

if not environ:
Expand All @@ -315,22 +330,12 @@ def __init__(self, environ=None, use_distribute=None, sitecustomize=None):
environ['PIP_NO_INPUT'] = '1'
environ['PIP_LOG_FILE'] = str(self.root_path/'pip-log.txt')

super(TestPipEnvironment, self).__init__(
TestFileEnvironment.__init__(self,
self.root_path, ignore_hidden=False,
environ=environ, split_cmd=False, start_clear=False,
cwd=self.scratch_path, capture_temp=True, assert_no_temp=True)

demand_dirs(self.venv_path)
demand_dirs(self.scratch_path)

if use_distribute is None:
use_distribute = os.environ.get('PIP_TEST_USE_DISTRIBUTE', False)
self.use_distribute = use_distribute

# Create a virtualenv and remember where it's putting things.
virtualenv_paths = create_virtualenv(self.venv_path, distribute=self.use_distribute)

assert self.venv_path == virtualenv_paths[0] # sanity check
virtualenv_paths = virtualenv.path_locations(self.venv_path)

for id, path in zip(('venv', 'lib', 'include', 'bin'), virtualenv_paths):
#fix for virtualenv issue #306
Expand All @@ -346,51 +351,72 @@ def __init__(self, environ=None, use_distribute=None, sitecustomize=None):
else:
self.site_packages = self.lib/'site-packages'
self.user_base_path = self.venv_path/'user'
self.user_site_path = self.venv_path/'user'/site_packages_suffix
self.user_site_path = self.venv_path/'user'/'lib'/self.lib.name/'site-packages'

self.user_site = relpath(self.root_path, self.user_site_path)
demand_dirs(self.user_site_path)
self.environ["PYTHONUSERBASE"] = self.user_base_path

# create easy-install.pth in user_site, so we always have it updated instead of created
open(self.user_site_path/'easy-install.pth', 'w').close()
self.environ["PYTHONUSERBASE"] = self.user_base_path

# put the test-scratch virtualenv's bin dir first on the PATH
self.environ['PATH'] = Path.pathsep.join((self.bin_path, self.environ['PATH']))

# test that test-scratch virtualenv creation produced sensible venv python
result = self.run('python', '-c', 'import sys; print(sys.executable)')
pythonbin = result.stdout.strip()
if self.root_path.exists:
rmtree(self.root_path)
if self.backup_path.exists and not self.rebuild_venv:
shutil.copytree(self.backup_path, self.root_path, True)
else:
demand_dirs(self.venv_path)
demand_dirs(self.scratch_path)

if Path(pythonbin).noext != self.bin_path/'python':
raise RuntimeError(
"Oops! 'python' in our test environment runs %r"
" rather than expected %r" % (pythonbin, self.bin_path/'python'))
# Create a virtualenv and remember where it's putting things.
create_virtualenv(self.venv_path, distribute=self.use_distribute)

demand_dirs(self.user_site_path)

# make sure we have current setuptools to avoid svn incompatibilities
if not self.use_distribute:
install_setuptools(self)
# create easy-install.pth in user_site, so we always have it updated instead of created
open(self.user_site_path/'easy-install.pth', 'w').close()

# Uninstall whatever version of pip came with the virtualenv.
# Earlier versions of pip were incapable of
# self-uninstallation on Windows, so we use the one we're testing.
self.run('python', '-c',
'"import sys; sys.path.insert(0, %r); import pip; sys.exit(pip.main());"' % os.path.dirname(here),
'uninstall', '-vvv', '-y', 'pip')
# test that test-scratch virtualenv creation produced sensible venv python
result = self.run('python', '-c', 'import sys; print(sys.executable)')
pythonbin = result.stdout.strip()

# Install this version instead
self.run('python', 'setup.py', 'install', cwd=src_folder, expect_stderr=True)
if Path(pythonbin).noext != self.bin_path/'python':
raise RuntimeError(
"Oops! 'python' in our test environment runs %r"
" rather than expected %r" % (pythonbin, self.bin_path/'python'))

# make sure we have current setuptools to avoid svn incompatibilities
if not self.use_distribute:
install_setuptools(self)

# Uninstall whatever version of pip came with the virtualenv.
# Earlier versions of pip were incapable of
# self-uninstallation on Windows, so we use the one we're testing.
self.run('python', '-c',
'"import sys; sys.path.insert(0, %r); import pip; sys.exit(pip.main());"' % os.path.dirname(here),
'uninstall', '-vvv', '-y', 'pip')

# Install this version instead
self.run('python', 'setup.py', 'install', cwd=src_folder, expect_stderr=True)

# make the backup (remove previous backup if exists)
if self.backup_path.exists:
rmtree(self.backup_path)
shutil.copytree(self.root_path, self.backup_path, True)

#create sitecustomize.py and add patches
self._create_empty_sitecustomize()
self._use_cached_pypi_server()
if sitecustomize:
self._add_to_sitecustomize(sitecustomize)

# Ensure that $TMPDIR exists (because we use start_clear=False, it's not created for us)
assert self.root_path.exists

# Ensure that $TMPDIR exists (because we use start_clear=False, it's not created for us)
if self.temp_path and not os.path.exists(self.temp_path):
os.makedirs(self.temp_path)


def _ignore_file(self, fn):
if fn.endswith('__pycache__') or fn.endswith(".pyc"):
result = True
Expand All @@ -408,9 +434,6 @@ def run(self, *args, **kw):
assert not isinstance(cwd, Path)
return TestPipResult(super(TestPipEnvironment, self).run(cwd=cwd, *args, **kw), verbose=self.verbose)

def __del__(self):
rmtree(str(self.root_path), ignore_errors=True)

def _use_cached_pypi_server(self):
# previously, this was handled in a pth file, and not in sitecustomize.py
# pth processing happens during the construction of sys.path.
Expand Down Expand Up @@ -440,117 +463,22 @@ def _add_to_sitecustomize(self, snippet):
%s
''' %snippet))
sitecustomize.close()
# caught py32 with an outdated __pycache__ file after a sitecustomize update (after python should have updated it)
# https://github.com/pypa/pip/pull/893#issuecomment-16426701
# will delete the cache file to be sure
if uses_pycache:
cache_path = imp.cache_from_source(sitecustomize_path)
if os.path.isfile(cache_path):
os.remove(cache_path)

fast_test_env_root = here / 'tests_cache' / 'test_ws'
fast_test_env_backup = here / 'tests_cache' / 'test_ws_backup'


class FastTestPipEnvironment(TestPipEnvironment):
def __init__(self, environ=None, sitecustomize=None):
import virtualenv

self.root_path = fast_test_env_root
self.backup_path = fast_test_env_backup

self.scratch_path = self.root_path / self.scratch

# We will set up a virtual environment at root_path.
self.venv_path = self.root_path / self.venv

if not environ:
environ = os.environ.copy()
environ = clear_environ(environ)
environ['PIP_DOWNLOAD_CACHE'] = str(download_cache)

environ['PIP_NO_INPUT'] = '1'
environ['PIP_LOG_FILE'] = str(self.root_path/'pip-log.txt')

TestFileEnvironment.__init__(self,
self.root_path, ignore_hidden=False,
environ=environ, split_cmd=False, start_clear=False,
cwd=self.scratch_path, capture_temp=True, assert_no_temp=True)

virtualenv_paths = virtualenv.path_locations(self.venv_path)

for id, path in zip(('venv', 'lib', 'include', 'bin'), virtualenv_paths):
#fix for virtualenv issue #306
if hasattr(sys, "pypy_version_info") and id == 'lib':
path = os.path.join(self.venv_path, 'lib-python', pyversion)
setattr(self, id+'_path', Path(path))
setattr(self, id, relpath(self.root_path, path))

assert self.venv == TestPipEnvironment.venv # sanity check

if hasattr(sys, "pypy_version_info"):
self.site_packages = self.venv/'site-packages'
else:
self.site_packages = self.lib/'site-packages'
self.user_base_path = self.venv_path/'user'
self.user_site_path = self.venv_path/'user'/'lib'/self.lib.name/'site-packages'

self.user_site = relpath(self.root_path, self.user_site_path)

self.environ["PYTHONUSERBASE"] = self.user_base_path

# put the test-scratch virtualenv's bin dir first on the PATH
self.environ['PATH'] = Path.pathsep.join((self.bin_path, self.environ['PATH']))

self.use_distribute = os.environ.get('PIP_TEST_USE_DISTRIBUTE', False)

if self.root_path.exists:
rmtree(self.root_path)
if self.backup_path.exists:
shutil.copytree(self.backup_path, self.root_path, True)
else:
demand_dirs(self.venv_path)
demand_dirs(self.scratch_path)

# Create a virtualenv and remember where it's putting things.
create_virtualenv(self.venv_path, distribute=self.use_distribute)

demand_dirs(self.user_site_path)

# create easy-install.pth in user_site, so we always have it updated instead of created
open(self.user_site_path/'easy-install.pth', 'w').close()

# test that test-scratch virtualenv creation produced sensible venv python
result = self.run('python', '-c', 'import sys; print(sys.executable)')
pythonbin = result.stdout.strip()

if Path(pythonbin).noext != self.bin_path/'python':
raise RuntimeError(
"Oops! 'python' in our test environment runs %r"
" rather than expected %r" % (pythonbin, self.bin_path/'python'))

# make sure we have current setuptools to avoid svn incompatibilities
if not self.use_distribute:
install_setuptools(self)

# Uninstall whatever version of pip came with the virtualenv.
# Earlier versions of pip were incapable of
# self-uninstallation on Windows, so we use the one we're testing.
self.run('python', '-c',
'"import sys; sys.path.insert(0, %r); import pip; sys.exit(pip.main());"' % os.path.dirname(here),
'uninstall', '-vvv', '-y', 'pip')

# Install this version instead
self.run('python', 'setup.py', 'install', cwd=src_folder, expect_stderr=True)
shutil.copytree(self.root_path, self.backup_path, True)

#create sitecustomize.py and add patches
self._create_empty_sitecustomize()
self._use_cached_pypi_server()
if sitecustomize:
self._add_to_sitecustomize(sitecustomize)

assert self.root_path.exists

# Ensure that $TMPDIR exists (because we use start_clear=False, it's not created for us)
if self.temp_path and not os.path.exists(self.temp_path):
os.makedirs(self.temp_path)
class TestPipEnvironmentD(TestPipEnvironment):
"""A specialized TestFileEnvironment that contains distribute"""

def __del__(self):
pass # shutil.rmtree(str(self.root_path), ignore_errors=True)
use_distribute = True
setuptools = 'distribute'
rebuild_venv = True


def run_pip(*args, **kw):
Expand Down

0 comments on commit 7176cd2

Please sign in to comment.