better handling of complex version specs to fix #8

pur/__init__.py

@@ -10,6 +10,7 @@
 
 import click
 import os
+import re
 import sys
 try:
     from StringIO import StringIO
@@ -81,16 +82,17 @@ def pur(**options):
         updated = 0
         for line, req, spec_ver, latest_ver in requirements:
             if req and req.name.lower() not in options['skip']:
-                if spec_ver and latest_ver and (spec_ver == 'Unknown'
-                                                or spec_ver < latest_ver):
-                    if spec_ver == 'Unknown':
+                if should_update(req, spec_ver, latest_ver,
+                                 force=options['force']):
+                    if not spec_ver[0]:
                         new_line = '{0}=={1}'.format(line, latest_ver)
                     else:
-                        new_line = line.replace(str(spec_ver), str(latest_ver), 1)
+                        new_line = update_requirement(req, line, spec_ver,
+                                                      latest_ver)
                     buf.write(new_line)
                     click.echo('Updated {package}: {old} -> {new}'.format(
                         package=req.name,
-                        old=spec_ver,
+                        old=spec_ver[1] if spec_ver[0] else 'Unknown',
                         new=latest_ver,
                     ))
                     updated += 1
@@ -116,11 +118,21 @@ def pur(**options):
         raise ExitCodeException(10)
 
 
+def patch_pip():
+    """Patch pip to prevent parsing nested requirements files."""
+
+    old_fn = req_file.parse_requirements
+    def patched_parse_requirements(*args, **kwargs):
+        return []
+    req_file.parse_requirements = patched_parse_requirements
+    return old_fn
+
+
 def get_requirements_and_latest(filename, force=False):
     """Parse a requirements file and get latest version for each requirement.
 
     Yields a tuple of (original line, InstallRequirement instance,
-    spec_version, latest_version).
+    spec_versions, latest_version).
 
     :param filename:  Path to a requirements.txt file.
     :param force:     Force getting latest version even for packages without
@@ -129,12 +141,12 @@ def get_requirements_and_latest(filename, force=False):
     session = PipSession()
 
     url, content = get_file_content(filename, session=session)
-    for orig_line, line_number, line in yield_lines(content):
+    for line_number, line, orig_line in yield_lines(content):
         line = req_file.COMMENT_RE.sub('', line)
         line = line.strip()
         req = parse_requirement(line, filename, line_number, session)
-        spec_ver = current_version(req, force=force)
-        if spec_ver:
+        spec_ver = current_version(req)
+        if spec_ver or force:
             latest_ver = latest_version(req, session)
             yield (orig_line, req, spec_ver, latest_ver)
         else:
@@ -158,32 +170,99 @@ def parse_requirement(line, filename, line_number, session):
     return reqs[0] if len(reqs) > 0 else None
 
 
-def current_version(req, force=False):
+def current_version(req):
     """Get the current version from an InstallRequirement instance.
 
+    Returns a tuple (found, eq_ver, gt_ver, gte_ver, lt_ver, lte_ver, not_ver).
+    The versions in the returned tuple will be either a
+    pip.req.req_install.Version instance or None.
+
     :param req:    Instance of pip.req.req_install.InstallRequirement.
-    :param force:  Force getting latest version even for packages without
     """
 
     if not req or not req.req:
         return None
 
-    ver = None
-    try:
-        ver = Version(req.req.specs[0][1])
-    except IndexError:
-        pass
+    eq_ver = None
+    gt_ver = None
+    gte_ver = None
+    lt_ver = None
+    lte_ver = None
+    not_ver = None
+    for spec in req.req.specs:
+        try:
+            ver = Version(spec[1])
+            if spec[0] == '==':
+                eq_ver = ver
+            elif spec[0] == '>':
+                if not gt_ver or ver > gt_ver:
+                    gt_ver = ver
+            elif spec[0] == '>=':
+                if not gte_ver or ver > gte_ver:
+                    gte_ver = ver
+            elif spec[0] == '<':
+                if not lt_ver or ver < lt_ver:
+                    lt_ver = ver
+            elif spec[0] == '<=':
+                if not lte_ver or ver < lte_ver:
+                    lte_ver = ver
+            elif spec[0] == '!=':
+                not_ver = ver
+        except IndexError:
+            pass
+
+    found = (eq_ver is not None or gt_ver is not None or gte_ver is not None or
+             lt_ver is not None or lte_ver is not None or not_ver is not None)
+
+    return found, eq_ver, gt_ver, gte_ver, lt_ver, lte_ver, not_ver
 
-    if not ver and force and req.link is None:
-        ver = 'Unknown'
 
-    return ver
+def yield_lines(content):
+    """Yields a tuple of each line in a requirements file string.
 
+    The tuple contains (lineno, joined_line, original_line).
 
-def yield_lines(content):
+    :param content:  Text content of a requirements.txt file.
+    """
     lines = content.splitlines()
-    for line_number, line in req_file.join_lines(enumerate(lines)):
-        yield (lines[line_number], line_number + 1, line)
+    for lineno, joined, orig in join_lines(enumerate(lines, start=1)):
+        yield lineno, joined, orig
+
+
+def join_lines(lines_enum):
+    """Joins a line ending in '\' with the previous line.
+
+    (except when following comments). The joined line takes on the index of the
+    first line.
+    """
+    COMMENT_RE = re.compile(r'(^|\s)+#.*$')
+    primary_line_number = None
+    new_line = []
+    orig_lines = []
+    for line_number, orig_line in lines_enum:
+        line = orig_line
+        if not line.endswith('\\') or COMMENT_RE.match(line):
+            if COMMENT_RE.match(line):
+                # this ensures comments are always matched later
+                line = ' ' + line
+            if new_line:
+                new_line.append(line)
+                orig_lines.append(orig_line)
+                yield (primary_line_number, ''.join(new_line),
+                       "\n".join(orig_lines))
+                new_line = []
+                orig_lines = []
+            else:
+                yield line_number, line, orig_line
+        else:
+            if not new_line:
+                primary_line_number = line_number
+            new_line.append(line.rstrip('\\'))
+            orig_lines.append(orig_line)
+
+    # last line contains \
+    if new_line:
+        yield primary_line_number, ''.join(new_line), "\n".join(orig_lines)
 
 
 def latest_version(req, session, include_prereleases=False):
@@ -216,12 +295,66 @@ def latest_version(req, session, include_prereleases=False):
     return remote_version
 
 
-def patch_pip():
-    old_fn = req_file.parse_requirements
-    def patched_parse_requirements(*args, **kwargs):
-        return []
-    req_file.parse_requirements = patched_parse_requirements
-    return old_fn
+def should_update(req, spec_ver, latest_ver, force=False):
+    """Returns True if this requirement should be updated, False otherwise.
+
+    :param req:         Instance of pip.req.req_install.InstallRequirement.
+    :param spec_ver:    Tuple of current versions from the requirements file.
+    :param latest_ver:  Latest version from pypi.
+    :param force:       Force getting latest version even for packages without
+                        a version specified.
+    """
+
+    found = spec_ver[0]
+    eq_ver = spec_ver[1]
+    lt_ver = spec_ver[4]
+    lte_ver = spec_ver[5]
+    not_ver = spec_ver[6]
+
+    if not found and (not force or req.link is not None):
+        return False
+
+    if eq_ver is not None and latest_ver <= eq_ver:
+        return False
+
+    if not_ver is not None and latest_ver == not_ver:
+        return False
+
+    if lt_ver is not None and not latest_ver < lt_ver:
+        return False
+
+    if lte_ver is not None and not latest_ver <= lte_ver:
+        return False
+
+    return True
+
+
+def update_requirement(req, line, spec_ver, latest_ver):
+    """Updates the version of a requirement line.
+
+    Returns a new requirement line with the package version updated.
+
+    :param req:         Instance of pip.req.req_install.InstallRequirement.
+    :param line:        The requirement line string.
+    :param spec_ver:    Tuple of current versions from the requirements file.
+    :param latest_ver:  Latest version from pypi.
+    """
+
+    start_of_spec = (line.index(']') + 1 if ']' in line.split('#')[0]
+                     else len(req.name))
+    package_part = line[:start_of_spec]
+    spec_part = line[start_of_spec:]
+
+    pattern = r'(==\s*){0}'.format(re.sub(r'(\W)', r'\\\1', str(spec_ver[1])))
+    match = re.search(pattern, spec_part)
+    pre_part = match.group(1)
+    old = '{0}{1}'.format(pre_part, str(spec_ver[1]))
+    new = '{0}{1}'.format(pre_part, str(latest_ver))
+    new_line = '{package_part}{spec_part}'.format(
+        package_part=package_part,
+        spec_part=spec_part.replace(old, new, 1),
+    )
+    return new_line
 
 
 class ExitCodeException(click.ClickException):

tests/samples/requirements-version-in-name.txt

@@ -0,0 +1 @@
+package1==1

tests/samples/requirements-with-extras.txt

@@ -0,0 +1 @@
+firstpackage1[secondpackage1] == 1 # this is a comment

tests/samples/requirements-with-max-version-spec.txt

@@ -0,0 +1,2 @@
+afakepackage == 0.9, < 1.0
+afakepackage == 0.9, <= 1.0

tests/samples/results/test_updates_package_with_extras

@@ -0,0 +1 @@
+firstpackage1[secondpackage1] == 2.0 # this is a comment

tests/samples/results/test_updates_package_with_max_version_spec

@@ -0,0 +1,2 @@
+afakepackage == 0.10.1, < 1.0
+afakepackage == 0.10.1, <= 1.0

tests/samples/results/test_updates_package_with_version_in_name

@@ -0,0 +1 @@
+package1==2.0

tests/test_pur.py

@@ -161,10 +161,11 @@ def test_skip_package(self):
             self.assertEquals(open(requirements).read(), expected_requirements)
 
     def test_skip_multiple_packages(self):
+        requirements = 'tests/samples/requirements-multiple.txt'
         tempdir = tempfile.mkdtemp()
-        requirements = os.path.join(tempdir, 'requirements.txt')
-        shutil.copy('tests/samples/requirements-multiple.txt', requirements)
-        args = ['-r', requirements, '-s', 'flask, alembic , SQLAlchemy']
+        tmpfile = os.path.join(tempdir, 'requirements.txt')
+        shutil.copy(requirements, tmpfile)
+        args = ['-r', tmpfile, '-s', 'flask, alembic , SQLAlchemy']
 
         with utils.mock.patch('pip.index.PackageFinder.find_all_candidates') as mock_find_all_candidates:
             project = 'flask'
@@ -178,8 +179,8 @@ def test_skip_multiple_packages(self):
             expected_output = "All requirements up-to-date.\n"
             self.assertEquals(u(result.output), u(expected_output))
             self.assertEquals(result.exit_code, 0)
-            expected_requirements = open('tests/samples/results/test_skip_multiple_packages').read()
-            self.assertEquals(open(requirements).read(), expected_requirements)
+            expected_requirements = open(requirements).read()
+            self.assertEquals(open(tmpfile).read(), expected_requirements)
 
     def test_updates_package_with_no_version_specified(self):
         tempdir = tempfile.mkdtemp()
@@ -260,3 +261,88 @@ def test_no_arguments_and_no_requirements_file(self):
             expected_output = "Error: Could not open requirements file: [Errno 2] No such file or directory: 'requirements.txt'\n"
             self.assertEquals(u(result.output), u(expected_output))
             self.assertEquals(result.exit_code, 1)
+
+    def test_updates_package_with_number_in_name(self):
+        tempdir = tempfile.mkdtemp()
+        requirements = os.path.join(tempdir, 'requirements.txt')
+        shutil.copy('tests/samples/requirements-version-in-name.txt', requirements)
+        args = ['-r', requirements]
+
+        with utils.mock.patch('pip.index.PackageFinder.find_all_candidates') as mock_find_all_candidates:
+            project = 'package1'
+            version = '2.0'
+            link = Link('')
+            candidate = InstallationCandidate(project, version, link)
+            mock_find_all_candidates.return_value = [candidate]
+
+            result = self.runner.invoke(pur, args)
+            self.assertIsNone(result.exception)
+            expected_output = "Updated package1: 1 -> 2.0\nAll requirements up-to-date.\n"
+            self.assertEquals(u(result.output), u(expected_output))
+            self.assertEquals(result.exit_code, 0)
+            expected_requirements = open('tests/samples/results/test_updates_package_with_version_in_name').read()
+            self.assertEquals(open(requirements).read(), expected_requirements)
+
+    def test_updates_package_with_extras(self):
+        tempdir = tempfile.mkdtemp()
+        requirements = os.path.join(tempdir, 'requirements.txt')
+        shutil.copy('tests/samples/requirements-with-extras.txt', requirements)
+        args = ['-r', requirements]
+
+        with utils.mock.patch('pip.index.PackageFinder.find_all_candidates') as mock_find_all_candidates:
+            project = 'firstpackage'
+            version = '2.0'
+            link = Link('')
+            candidate = InstallationCandidate(project, version, link)
+            mock_find_all_candidates.return_value = [candidate]
+
+            result = self.runner.invoke(pur, args)
+            expected_output = "Updated firstpackage1: 1 -> 2.0\nAll requirements up-to-date.\n"
+            self.assertEquals(u(result.output), u(expected_output))
+            self.assertIsNone(result.exception)
+            self.assertEquals(result.exit_code, 0)
+            expected_requirements = open('tests/samples/results/test_updates_package_with_extras').read()
+            self.assertEquals(open(requirements).read(), expected_requirements)
+
+    def test_updates_package_with_max_version_spec(self):
+        tempdir = tempfile.mkdtemp()
+        requirements = os.path.join(tempdir, 'requirements.txt')
+        shutil.copy('tests/samples/requirements-with-max-version-spec.txt', requirements)
+        args = ['-r', requirements]
+
+        with utils.mock.patch('pip.index.PackageFinder.find_all_candidates') as mock_find_all_candidates:
+            project = 'afakepackage'
+            version = '0.10.1'
+            link = Link('')
+            candidate = InstallationCandidate(project, version, link)
+            mock_find_all_candidates.return_value = [candidate]
+
+            result = self.runner.invoke(pur, args)
+            expected_output = "Updated afakepackage: 0.9 -> 0.10.1\nUpdated afakepackage: 0.9 -> 0.10.1\nAll requirements up-to-date.\n"
+            self.assertEquals(u(result.output), u(expected_output))
+            self.assertIsNone(result.exception)
+            self.assertEquals(result.exit_code, 0)
+            expected_requirements = open('tests/samples/results/test_updates_package_with_max_version_spec').read()
+            self.assertEquals(open(requirements).read(), expected_requirements)
+
+    def test_max_version_spec_prevents_updating_package(self):
+        requirements = 'tests/samples/requirements-with-max-version-spec.txt'
+        tempdir = tempfile.mkdtemp()
+        tmpfile = os.path.join(tempdir, 'requirements.txt')
+        shutil.copy(requirements, tmpfile)
+        args = ['-r', tmpfile]
+
+        with utils.mock.patch('pip.index.PackageFinder.find_all_candidates') as mock_find_all_candidates:
+            project = 'afakepackage'
+            version = '2.0'
+            link = Link('')
+            candidate = InstallationCandidate(project, version, link)
+            mock_find_all_candidates.return_value = [candidate]
+
+            result = self.runner.invoke(pur, args)
+            self.assertIsNone(result.exception)
+            expected_output = "All requirements up-to-date.\n"
+            self.assertEquals(u(result.output), u(expected_output))
+            self.assertEquals(result.exit_code, 0)
+            expected_requirements = open(tmpfile).read()
+            self.assertEquals(open(tmpfile).read(), expected_requirements)