Merge #60 and #61: Add Windows support! (fixes #53)

.travis.yml

@@ -1,3 +1,6 @@
+# .travis.yml: Configuration for continuous integration (automated tests)
+# hosted on Travis CI, see https://travis-ci.org/paylogic/pip-accel.
+
 sudo: required
 language: python
 python:
@@ -10,7 +13,7 @@ env:
 before_install:
   - scripts/retry-command sudo apt-get update
 install:
-  - scripts/retry-command pip install coveralls --editable "file://${PWD}[s3]"
+  - LC_ALL=C scripts/retry-command pip install coveralls --editable "file://${PWD}[s3]"
   - scripts/retry-command gem install fakes3
 script:
   - scripts/collect-full-coverage

README.rst

@@ -30,8 +30,9 @@ detects missing system packages when a build fails and prompts the user whether
 to install the missing dependencies and retry the build.
 
 The pip-accel program is currently tested on cPython 2.6, 2.7 and 3.4 and PyPy
-(2.7). The automated test suite regularly runs on Ubuntu Linux but other Linux
-variants (also those not based on Debian Linux) should work fine.
+(2.7). The automated test suite regularly runs on Ubuntu Linux (`Travis CI`_)
+as well as Microsoft Windows (AppVeyor_). In addition to these platforms
+pip-accel should work fine on most UNIX systems (e.g. Mac OS X).
 
 .. contents::
 

appveyor.yml

@@ -0,0 +1,15 @@
+# appveyor.yml: Configuration for continuous integration (automated tests)
+# hosted on AppVeyor, see https://ci.appveyor.com/project/xolox/pip-accel.
+
+version: 1.0.{build}
+environment:
+  PYTHON: C:\Python27
+  COVERALLS_REPO_TOKEN:
+    secure: DCxZQaYFWVR0zWqjTPXhhlRlLdmKNMS2qDUwIR8jRar13clunOqJIaXn+vKInS7g
+install:
+  - cmd: '"%PYTHON%\Scripts\pip.exe" install coveralls .'
+build: off
+test_script:
+  - cmd: '"%PYTHON%\Scripts\coverage.exe" run setup.py test'
+on_success:
+  - cmd: '"%PYTHON%\Scripts\coveralls.exe"'

pip_accel/__init__.py

@@ -1,7 +1,7 @@
 # Accelerator for pip, the Python package manager.
 #
 # Author: Peter Odding <peter.odding@paylogic.com>
-# Last Change: September 22, 2015
+# Last Change: October 30, 2015
 # URL: https://github.com/paylogic/pip-accel
 #
 # TODO Permanently store logs in the pip-accel directory (think about log rotation).
@@ -44,7 +44,7 @@
 """
 
 # Semi-standard module versioning.
-__version__ = '0.32.1'
+__version__ = '0.33'
 
 # Standard library modules.
 import logging
@@ -58,7 +58,7 @@
 from pip_accel.bdist import BinaryDistributionManager
 from pip_accel.exceptions import EnvironmentMismatchError, NothingToDoError
 from pip_accel.req import Requirement
-from pip_accel.utils import is_installed, makedirs, match_option_with_value, uninstall
+from pip_accel.utils import is_installed, makedirs, match_option_with_value, same_directories, uninstall
 
 # External dependencies.
 from humanfriendly import concatenate, Timer, pluralize
@@ -118,12 +118,7 @@ def validate_environment(self):
         """
         environment = os.environ.get('VIRTUAL_ENV')
         if environment:
-            try:
-                # Because os.path.samefile() itself can raise exceptions, e.g.
-                # when $VIRTUAL_ENV points to a non-existing directory, we use
-                # an assertion to allow us to use a single code path :-)
-                assert os.path.samefile(sys.prefix, environment)
-            except Exception:
+            if not same_directories(sys.prefix, environment):
                 raise EnvironmentMismatchError("""
                     You are trying to install packages in environment #1 which
                     is different from environment #2 where pip-accel is
@@ -133,8 +128,7 @@ def validate_environment(self):
                     Environment #1: {environment} (defined by $VIRTUAL_ENV)
 
                     Environment #2: {prefix} (Python's installation prefix)
-                """, environment=environment,
-                     prefix=sys.prefix)
+                """, environment=environment, prefix=sys.prefix)
 
     def initialize_directories(self):
         """Automatically create the local source distribution index directory."""

pip_accel/bdist.py

@@ -1,7 +1,7 @@
 # Functions to manipulate Python binary distribution archives.
 #
 # Author: Peter Odding <peter.odding@paylogic.com>
-# Last Change: September 9, 2015
+# Last Change: October 28, 2015
 # URL: https://github.com/paylogic/pip-accel
 
 """
@@ -32,6 +32,7 @@
 
 # Modules included in our package.
 from pip_accel.caches import CacheManager
+from pip_accel.compat import WINDOWS
 from pip_accel.deps import SystemPackageManager
 from pip_accel.exceptions import BuildFailed, InvalidSourceDistribution, NoBuildOutput
 from pip_accel.utils import compact, makedirs
@@ -89,13 +90,19 @@ def get_binary_dist(self, requirement):
             fd, transformed_file = tempfile.mkstemp(prefix='pip-accel-bdist-', suffix='.tar.gz')
             try:
                 archive = tarfile.open(transformed_file, 'w:gz')
-                for member, from_handle in self.transform_binary_dist(raw_file):
-                    archive.addfile(member, from_handle)
-                archive.close()
+                try:
+                    for member, from_handle in self.transform_binary_dist(raw_file):
+                        archive.addfile(member, from_handle)
+                finally:
+                    archive.close()
                 # Push the binary distribution archive to all available backends.
                 with open(transformed_file, 'rb') as handle:
                     self.cache.put(requirement, handle)
             finally:
+                # Close file descriptor before removing the temporary file.
+                # Without closing Windows is complaining that the file cannot
+                # be removed because it is used by another process.
+                os.close(fd)
                 # Cleanup the temporary file.
                 os.remove(transformed_file)
             # Get the absolute pathname of the file in the local cache.
@@ -145,7 +152,7 @@ def build_binary_dist(self, requirement):
             return self.build_binary_dist_helper(requirement, ['bdist_dumb', '--format=tar'])
         except (BuildFailed, NoBuildOutput):
             logger.warning("Build of %s failed, falling back to alternative method ..", requirement)
-            return self.build_binary_dist_helper(requirement, ['bdist'])
+            return self.build_binary_dist_helper(requirement, ['bdist', '--formats=gztar'])
 
     def build_binary_dist_helper(self, requirement, setup_command):
         """
@@ -239,6 +246,10 @@ def build_binary_dist_helper(self, requirement, setup_command):
             logger.info("Finished building %s in %s.", requirement.name, build_timer)
             return os.path.join(dist_directory, filenames[0])
         finally:
+            # Close file descriptor before removing the temporary file.
+            # Without closing Windows is complaining that the file cannot
+            # be removed because it is used by another process.
+            os.close(fd)
             os.unlink(temporary_file)
 
     def transform_binary_dist(self, archive_path):

pip_accel/caches/__init__.py

@@ -1,7 +1,7 @@
 # Accelerator for pip, the Python package manager.
 #
 # Author: Peter Odding <peter.odding@paylogic.com>
-# Last Change: April 11, 2015
+# Last Change: October 28, 2015
 # URL: https://github.com/paylogic/pip-accel
 
 """
@@ -22,6 +22,7 @@
 import logging
 
 # Modules included in our package.
+from pip_accel.compat import WINDOWS
 from pip_accel.exceptions import CacheBackendDisabledError
 from pip_accel.utils import get_python_version
 
@@ -35,6 +36,9 @@
 # Initialize the registry of cache backends.
 registered_backends = set()
 
+# On Windows it is not allowed to have colons in filenames so we use a dollar sign instead.
+FILENAME_PATTERN = 'v%i\\%s$%s$%s.tar.gz' if WINDOWS else 'v%i/%s:%s:%s.tar.gz'
+
 class CacheBackendMeta(type):
 
     """Metaclass to intercept cache backend definitions."""
@@ -193,5 +197,6 @@ def generate_filename(self, requirement):
                   including a single leading directory component to indicate
                   the cache format revision.
         """
-        return 'v%i/%s:%s:%s.tar.gz' % (self.config.cache_format_revision,
-                                        requirement.name, requirement.version, get_python_version())
+        return FILENAME_PATTERN % (self.config.cache_format_revision,
+                                   requirement.name, requirement.version,
+                                   get_python_version())

pip_accel/caches/local.py

@@ -1,7 +1,7 @@
 # Accelerator for pip, the Python package manager.
 #
 # Author: Peter Odding <peter.odding@paylogic.com>
-# Last Change: November 16, 2014
+# Last Change: October 29, 2015
 # URL: https://github.com/paylogic/pip-accel
 
 """
@@ -26,7 +26,7 @@
 
 # Modules included in our package.
 from pip_accel.caches import AbstractCacheBackend
-from pip_accel.utils import makedirs
+from pip_accel.utils import makedirs, replace_file
 
 # Initialize a logger for this module.
 logger = logging.getLogger(__name__)
@@ -71,8 +71,8 @@ def put(self, filename, handle):
         logger.debug("Using temporary file to avoid partial reads: %s", temporary_file)
         with open(temporary_file, 'wb') as temporary_file_handle:
             shutil.copyfileobj(handle, temporary_file_handle)
+        logger.debug("Moving temporary file into place ..")
         # Atomically move the distribution archive into its final place
         # (again, to avoid race conditions between multiple processes).
-        logger.debug("Moving temporary file into place ..")
-        os.rename(temporary_file, file_in_cache)
+        replace_file(temporary_file, file_in_cache)
         logger.debug("Finished caching distribution archive in local cache.")

pip_accel/compat.py

@@ -1,10 +1,32 @@
+# Accelerator for pip, the Python package manager.
+#
+# Author: Peter Odding <peter.odding@paylogic.com>
+# Last Change: October 27, 2015
+# URL: https://github.com/paylogic/pip-accel
+
+# Standard library modules.
+import sys
+
+# Inform static code analysis tools about our intention to expose the
+# following variables. This avoids 'imported but unused' warnings.
+__all__ = (
+    'WINDOWS',
+    'StringIO',
+    'configparser',
+    'urlparse',
+)
+
+# Detect whether we're running on Microsoft Windows.
+WINDOWS = sys.platform.startswith('win')
+
+# Compatibility between Python 2 and 3.
 try:
-    # Python 2.x.
+    # Python 2.
     from StringIO import StringIO
     from urlparse import urlparse
     import ConfigParser as configparser
 except ImportError:
-    # Python 3.x.
+    # Python 3.
     from io import StringIO
     from urllib.parse import urlparse
     import configparser

pip_accel/config.py

@@ -1,7 +1,7 @@
 # Configuration defaults for the pip accelerator.
 #
 # Author: Peter Odding <peter.odding@paylogic.com>
-# Last Change: April 11, 2015
+# Last Change: October 27, 2015
 # URL: https://github.com/paylogic/pip-accel
 
 """
@@ -77,6 +77,7 @@
 
 # Modules included in our package.
 from pip_accel.compat import configparser
+from pip_accel.utils import is_root, expand_path
 
 # External dependencies.
 from cached_property import cached_property
@@ -217,10 +218,10 @@ def data_directory(self):
         - Configuration option: ``data-directory``
         - Default: ``/var/cache/pip-accel`` if running as ``root``, ``~/.pip-accel`` otherwise
         """
-        return parse_path(self.get(property_name='data_directory',
-                                   environment_variable='PIP_ACCEL_CACHE',
-                                   configuration_option='data-directory',
-                                   default='/var/cache/pip-accel' if os.getuid() == 0 else '~/.pip-accel'))
+        return expand_path(self.get(property_name='data_directory',
+                                    environment_variable='PIP_ACCEL_CACHE',
+                                    configuration_option='data-directory',
+                                    default='/var/cache/pip-accel' if is_root() else '~/.pip-accel'))
 
     @cached_property
     def on_debian(self):

pip_accel/deps/__init__.py

@@ -1,7 +1,7 @@
 # Extension of pip-accel that deals with dependencies on system packages.
 #
 # Author: Peter Odding <peter.odding@paylogic.com>
-# Last Change: November 22, 2014
+# Last Change: October 28, 2015
 # URL: https://github.com/paylogic/pip-accel
 
 """
@@ -26,8 +26,9 @@
 import sys
 
 # Modules included in our package.
-from pip_accel.compat import configparser
+from pip_accel.compat import WINDOWS, configparser
 from pip_accel.exceptions import DependencyInstallationFailed, DependencyInstallationRefused, SystemDependencyError
+from pip_accel.utils import is_root
 
 # External dependencies.
 from humanfriendly import Timer, concatenate, pluralize
@@ -63,19 +64,20 @@ def __init__(self, config):
                 # Check if the package manager is supported.
                 supported_command = parser.get('commands', 'supported')
                 logger.debug("Checking if configuration is supported: %s", supported_command)
-                if subprocess.call(supported_command, shell=True) == 0:
-                    logger.debug("System package manager configuration is supported!")
-                    # Get the commands to list and install system packages.
-                    self.list_command = parser.get('commands', 'list')
-                    self.install_command = parser.get('commands', 'install')
-                    # Get the known dependencies.
-                    self.dependencies = dict((n.lower(), v.split()) for n, v
-                                             in parser.items('dependencies'))
-                    logger.debug("Loaded dependencies of %s: %s",
-                                 pluralize(len(self.dependencies), "Python package"),
-                                 concatenate(sorted(self.dependencies)))
-                else:
-                    logger.debug("Command failed, assuming configuration doesn't apply ..")
+                with open(os.devnull, 'wb') as null_device:
+                    if subprocess.call(supported_command, shell=True, stdout=null_device, stderr=subprocess.STDOUT) == 0:
+                        logger.debug("System package manager configuration is supported!")
+                        # Get the commands to list and install system packages.
+                        self.list_command = parser.get('commands', 'list')
+                        self.install_command = parser.get('commands', 'install')
+                        # Get the known dependencies.
+                        self.dependencies = dict((n.lower(), v.split()) for n, v
+                                                 in parser.items('dependencies'))
+                        logger.debug("Loaded dependencies of %s: %s",
+                                     pluralize(len(self.dependencies), "Python package"),
+                                     concatenate(sorted(self.dependencies)))
+                    else:
+                        logger.debug("Command failed, assuming configuration doesn't apply ..")
 
     def install_dependencies(self, requirement):
         """
@@ -97,8 +99,10 @@ def install_dependencies(self, requirement):
         if missing_dependencies:
             # Compose the command line for the install command.
             install_command = shlex.split(self.install_command) + missing_dependencies
-            if os.getuid() != 0:
-                # Prepend `sudo' to the command line.
+            # Prepend `sudo' to the command line?
+            if not WINDOWS and not is_root():
+                # FIXME Ideally this should properly detect the presence of `sudo'.
+                #       Or maybe this should just be embedded in the *.ini files?
                 install_command.insert(0, 'sudo')
             # Always suggest the installation command to the operator.
             logger.info("You seem to be missing %s: %s",
@@ -208,6 +212,7 @@ def confirm_installation(self, requirement, missing_dependencies, install_comman
         """
         terminal = "\n"
         try:
+            # FIXME raw_input() doesn't exist on Python 3. Switch to humanfriendly.prompts.prompt_for_confirmation()?
             prompt = "\n  Do you want me to install %s %s? [Y/n] "
             choice = raw_input(prompt % ("this" if len(missing_dependencies) == 1 else "these",
                                          "dependency" if len(missing_dependencies) == 1 else "dependencies"))