diff --git a/docs/reference/pip_install.rst b/docs/reference/pip_install.rst index 7eb37e6a73f..85217b7bf76 100644 --- a/docs/reference/pip_install.rst +++ b/docs/reference/pip_install.rst @@ -25,6 +25,7 @@ Each line of the requirements file indicates something to be installed, and like arguments to :ref:`pip install`, the following forms are supported:: + [--install-options="..."] [--global-options="..."] [-e] [-e] @@ -41,6 +42,9 @@ A line that begins with ``#`` is treated as a comment and ignored. Whitespace followed by a ``#`` causes the ``#`` and the remainder of the line to be treated as a comment. +A line ending in an unescaped ``\`` is treated as a line continuation +and the newline following it is effectively ignored. + Additionally, the following Package Index Options are supported: * :ref:`-i, --index-url <--index-url>` @@ -89,6 +93,25 @@ Some Examples: Don't use single or double quotes in a ``requirements.txt`` file. +.. _`Per-requirement Overrides`: + +Per-requirement Overrides ++++++++++++++++++++++++++ + +It is possible to set ``--install-options`` and ``--global-options`` +for each requirement in the requirements file: + + :: + + FooProject >= 1.2 --install-options="--prefix='/usr/local'" \ + --global-options="--no-user-cfg" + +The above translates roughly into running FooProject's ``setup.py`` +script as: + + :: + + python setup.py --no-user-cfg install --prefix='/usr/local' .. _`Pre Release Versions`: @@ -506,5 +529,3 @@ Examples :: $ pip install --pre SomePackage - - diff --git a/pip/exceptions.py b/pip/exceptions.py index 1c8f6b1a370..df6f3aeda28 100644 --- a/pip/exceptions.py +++ b/pip/exceptions.py @@ -18,9 +18,14 @@ class DistributionNotFound(InstallationError): """Raised when a distribution cannot be found to satisfy a requirement""" +class RequirementsFileParseError(PipError): + """Raised when an invalid state is encountered during requirement file + parsing.""" + + class BestVersionAlreadyInstalled(PipError): """Raised when the most up-to-date version of a package is already - installed. """ + installed.""" class BadCommand(PipError): diff --git a/pip/req/req_file.py b/pip/req/req_file.py index 12a7bc8cfe5..051ff49c1a1 100644 --- a/pip/req/req_file.py +++ b/pip/req/req_file.py @@ -1,137 +1,284 @@ + +""" +Routines for parsing requirements files (i.e. requirements.txt). +""" + from __future__ import absolute_import import os import re +import shlex +import getopt from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves import filterfalse from pip.download import get_file_content from pip.req.req_install import InstallRequirement +from pip.exceptions import RequirementsFileParseError from pip.utils import normalize_name + +# Flags that don't take any options. +parser_flags = set([ + '--no-index', + '--allow-all-external', + '--no-use-wheel', +]) + +# Flags that take options. +parser_options = set([ + '-i', '--index-url', + '-f', '--find-links', + '--extra-index-url', + '--allow-external', + '--allow-unverified', +]) + +# Encountering any of these is a no-op. +parser_compat = set([ + '-Z', '--always-unzip', + '--use-wheel', # Default in 1.5 + '--no-allow-external', # Remove in 7.0 + '--no-allow-insecure', # Remove in 7.0 +]) + +# The following options and flags can be used on requirement lines. +# For example: INITools==0.2 --install-options="--prefix=/opt" +parser_requirement_flags = set() +parser_requirement_options = set([ + '--install-options', + '--global-options', +]) + +# The types of lines understood by the requirements file parser. +REQUIREMENT = 0 +REQUIREMENT_FILE = 1 +REQUIREMENT_EDITABLE = 2 +FLAG = 3 +OPTION = 4 +IGNORE = 5 + _scheme_re = re.compile(r'^(http|https|file):', re.I) +_comment_re = re.compile(r'(^|\s)#.*$') -def parse_requirements(filename, finder=None, comes_from=None, options=None, - session=None): +def parse_requirements(filename, finder=None, comes_from=None, options=None, session=None): + """ + Parse a requirements file and yield InstallRequirement instances. + + :param filename: Path or url of requirements file. + :param finder: Instance of pip.index.PackageFinder. + :param comes_from: Origin summary for yield requirements. + :param options: Global options. + :param session: Instance of pip.download.PipSession. + """ + if session is None: raise TypeError( "parse_requirements() missing 1 required keyword argument: " "'session'" ) - skip_match = None + _, content = get_file_content(filename, comes_from=comes_from, session=session) + parser = parse_content(filename, content, finder, comes_from, options, session) + for item in parser: + yield item + + +def parse_content(filename, content, finder=None, comes_from=None, options=None, session=None): + # Split, sanitize and join lines with continuations. + content = content.splitlines() + content = ignore_comments(content) + content = join_lines(content) + + # Optionally exclude lines that match '--skip-requirements-regex'. skip_regex = options.skip_requirements_regex if options else None if skip_regex: - skip_match = re.compile(skip_regex) - reqs_file_dir = os.path.dirname(os.path.abspath(filename)) - filename, content = get_file_content( - filename, - comes_from=comes_from, - session=session, - ) - for line_number, line in enumerate(content.splitlines(), 1): - line = line.strip() + content = filterfalse(re.compile(skip_regex).search, content) - # Remove comments from file - line = re.sub(r"(^|\s)#.*$", "", line) + for line_number, line in enumerate(content, 1): + # The returned value depends on the type of line that was parsed. + linetype, value = parse_line(line) - if not line or line.startswith('#'): - continue - if skip_match and skip_match.search(line): - continue - if line.startswith('-r') or line.startswith('--requirement'): - if line.startswith('-r'): - req_url = line[2:].strip() - else: - req_url = line[len('--requirement'):].strip().strip('=') + # --------------------------------------------------------------------- + if linetype == REQUIREMENT: + req, opts = value + + # InstallRequirement.install() expects these options to be lists. + if opts: + for opt in '--global-options', '--install-options': + opts[opt] = shlex.split(opts[opt]) if opt in opts else [] + + comes_from = '-r %s (line %s)' % (filename, line_number) + isolated = options.isolated_mode if options else False + yield InstallRequirement.from_line( + req, comes_from, isolated=isolated, options=opts + ) + + # --------------------------------------------------------------------- + elif linetype == REQUIREMENT_EDITABLE: + comes_from = '-r %s (line %s)' % (filename, line_number) + isolated = options.isolated_mode if options else False, + default_vcs = options.default_vcs if options else None, + yield InstallRequirement.from_editable( + value, comes_from=comes_from, default_vcs=default_vcs, isolated=isolated + ) + + # --------------------------------------------------------------------- + elif linetype == REQUIREMENT_FILE: if _scheme_re.search(filename): - # Relative to a URL - req_url = urllib_parse.urljoin(filename, req_url) - elif not _scheme_re.search(req_url): - req_url = os.path.join(os.path.dirname(filename), req_url) - for item in parse_requirements( - req_url, finder, - comes_from=filename, - options=options, - session=session): - yield item - elif line.startswith('-Z') or line.startswith('--always-unzip'): - # No longer used, but previously these were used in - # requirement files, so we'll ignore. - pass - elif line.startswith('-f') or line.startswith('--find-links'): - if line.startswith('-f'): - line = line[2:].strip() - else: - line = line[len('--find-links'):].strip().lstrip('=') - # FIXME: it would be nice to keep track of the source of - # the find_links: - # support a find-links local path relative to a requirements file - relative_to_reqs_file = os.path.join(reqs_file_dir, line) - if os.path.exists(relative_to_reqs_file): - line = relative_to_reqs_file - if finder: - finder.find_links.append(line) - elif line.startswith('-i') or line.startswith('--index-url'): - if line.startswith('-i'): - line = line[2:].strip() - else: - line = line[len('--index-url'):].strip().lstrip('=') - if finder: - finder.index_urls = [line] - elif line.startswith('--extra-index-url'): - line = line[len('--extra-index-url'):].strip().lstrip('=') - if finder: - finder.index_urls.append(line) - elif line.startswith('--use-wheel'): - # Default in 1.5 - pass - elif line.startswith('--no-use-wheel'): - if finder: + # Relative to an URL. + req_url = urllib_parse.urljoin(filename, value) + elif not _scheme_re.search(value): + req_dir = os.path.dirname(filename) + req_url = os.path.join(os.path.dirname(filename), value) + # TODO: Why not use `comes_from='-r {} (line {})'` here as well? + parser = parse_requirements(req_url, finder, comes_from, options, session) + for req in parser: + yield req + + # --------------------------------------------------------------------- + elif linetype == FLAG: + if not finder: + continue + + if finder and value == '--no-use-wheel': finder.use_wheel = False - elif line.startswith('--no-index'): - if finder: + elif value == '--no-index': finder.index_urls = [] - elif line.startswith("--allow-external"): - line = line[len("--allow-external"):].strip().lstrip("=") - if finder: - finder.allow_external |= set([normalize_name(line).lower()]) - elif line.startswith("--allow-all-external"): - if finder: + elif value == '--allow-all-external': finder.allow_all_external = True - # Remove in 7.0 - elif line.startswith("--no-allow-external"): - pass - # Remove in 7.0 - elif line.startswith("--no-allow-insecure"): - pass - # Remove after 7.0 - elif line.startswith("--allow-insecure"): - line = line[len("--allow-insecure"):].strip().lstrip("=") - if finder: - finder.allow_unverified |= set([normalize_name(line).lower()]) - elif line.startswith("--allow-unverified"): - line = line[len("--allow-unverified"):].strip().lstrip("=") - if finder: + + # --------------------------------------------------------------------- + elif linetype == OPTION: + if not finder: + continue + + opt, value = value + if opt == '-i' or opt == '--index-url': + finder.index_urls = [value] + elif opt == '--extra-index-url': + finder.index_urls.append(value) + elif opt == '--allow-external': + finder.allow_external |= set([normalize_name(value).lower()]) + elif opt == '--allow-insecure': + # Remove after 7.0 finder.allow_unverified |= set([normalize_name(line).lower()]) + elif opt == '-f' or opt == '--find-links': + # FIXME: it would be nice to keep track of the source + # of the find_links: support a find-links local path + # relative to a requirements file. + req_dir = os.path.dirname(os.path.abspath(filename)) + relative_to_reqs_file = os.path.join(req_dir, value) + if os.path.exists(relative_to_reqs_file): + value = relative_to_reqs_file + finder.find_links.append(value) + + # --------------------------------------------------------------------- + elif linetype == IGNORE: + pass + + +def parse_line(line): + if not line.startswith('-'): + if ' --' in line: + req, opts = line.split(' --', 1) + opts = parse_requirement_options( + '--%s' % opts, + parser_requirement_flags, + parser_requirement_options + ) else: - comes_from = '-r %s (line %s)' % (filename, line_number) - if line.startswith('-e') or line.startswith('--editable'): - if line.startswith('-e'): - line = line[2:].strip() - else: - line = line[len('--editable'):].strip().lstrip('=') - req = InstallRequirement.from_editable( - line, - comes_from=comes_from, - default_vcs=options.default_vcs if options else None, - isolated=options.isolated_mode if options else False, - ) + req = line + opts = {} + + return REQUIREMENT, (req, opts) + + firstword, rest = partition_line(line) + # ------------------------------------------------------------------------- + if firstword == '-e' or firstword == '--editable': + return REQUIREMENT_EDITABLE, rest + + # ------------------------------------------------------------------------- + if firstword == '-r' or firstword == '--requirement': + return REQUIREMENT_FILE, rest + + # ------------------------------------------------------------------------- + if firstword in parser_flags: + if rest: + msg = 'Option %r does not accept values.' % firstword + raise RequirementsFileParseError(msg) + return FLAG, firstword + + # ------------------------------------------------------------------------- + if firstword in parser_options: + if not rest: + msg = 'Option %r requires value.' % firstword + raise RequirementsFileParseError(msg) + return OPTION, (firstword, rest) + + # ------------------------------------------------------------------------- + if firstword in parser_compat: + return IGNORE, line + + +def parse_requirement_options(req_line, flags=None, options=None): + long_opts = [] + if options: + long_opts += ['%s=' % i.lstrip('-') for i in options] + if flags: + long_opts += [i.lstrip('-') for i in flags] + + opts, _ = getopt.getopt(shlex.split(req_line), '', long_opts) + return dict(opts) + + +# ----------------------------------------------------------------------------- +# Utility functions related to requirements file parsing. +def join_lines(iterator): + """ + Joins a line ending in '\' with the previous line. + """ + + lines = [] + for line in iterator: + if not line.endswith('\\'): + if lines: + lines.append(line) + yield ''.join(lines) + lines = [] else: - req = InstallRequirement.from_line( - line, - comes_from, - isolated=options.isolated_mode if options else False, - ) - yield req + yield line + else: + lines.append(line.strip('\\')) + + # TODO: handle space after '\'. + # TODO: handle '\' on last line. + + +def ignore_comments(iterator): + """ + Strips and filters empty or commented lines. + """ + + for line in iterator: + line = _comment_re.sub('', line) + line = line.strip() + if line: + yield line + + +def partition_line(line): + firstword, _, rest = line.partition('=') + firstword = firstword.strip() + + if ' ' in firstword: + firstword, _, rest = line.partition(' ') + firstword = firstword.strip() + + rest = rest.strip() + return firstword, rest + + +__all__ = 'parse_requirements' diff --git a/pip/req/req_install.py b/pip/req/req_install.py index bd53920ae2e..769f8d9d53c 100644 --- a/pip/req/req_install.py +++ b/pip/req/req_install.py @@ -48,7 +48,7 @@ class InstallRequirement(object): def __init__(self, req, comes_from, source_dir=None, editable=False, link=None, as_egg=False, update=True, editable_options=None, - pycompile=True, markers=None, isolated=False): + pycompile=True, markers=None, isolated=False, options=None): self.extras = () if isinstance(req, six.string_types): req = pkg_resources.Requirement.parse(req) @@ -82,6 +82,7 @@ def __init__(self, req, comes_from, source_dir=None, editable=False, self.uninstalled = None self.use_user_site = False self.target_dir = None + self.options = options if options else {} self.pycompile = pycompile @@ -111,7 +112,7 @@ def from_editable(cls, editable_req, comes_from=None, default_vcs=None, return res @classmethod - def from_line(cls, name, comes_from=None, isolated=False): + def from_line(cls, name, comes_from=None, isolated=False, options=None): """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ @@ -177,7 +178,7 @@ def from_line(cls, name, comes_from=None, isolated=False): req = name return cls(req, comes_from, link=link, markers=markers, - isolated=isolated) + isolated=isolated, options=options) def __str__(self): if self.req: @@ -755,7 +756,7 @@ def match_markers(self): else: return True - def install(self, install_options, global_options=(), root=None): + def install(self, install_options, global_options=[], root=None): if self.editable: self.install_editable(install_options, global_options) return @@ -767,6 +768,14 @@ def install(self, install_options, global_options=(), root=None): self.install_succeeded = True return + # Extend the list of global and install options passed on to + # the setup.py call with the ones from the requirements file. + # Options specified in requirements file override those + # specified on the command line, since the last option given + # to setup.py is the one that is used. + global_options += self.options.get('--global-options', []) + install_options += self.options.get('--install-options', []) + if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 35d7cd11ac2..362c4eedac4 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -2,15 +2,24 @@ import shutil import sys import tempfile +import subprocess import pytest from mock import Mock, patch, mock_open +from textwrap import dedent from pip.exceptions import ( PreviousBuildDirError, InvalidWheelFilename, UnsupportedWheel, + RequirementsFileParseError, ) from pip.download import PipSession from pip.index import PackageFinder +from pip.req.req_file import (parse_requirement_options, + parse_content, + parse_line, + join_lines, + REQUIREMENT_EDITABLE, + REQUIREMENT, FLAG, OPTION) from pip.req import (InstallRequirement, RequirementSet, Requirements, parse_requirements) from pip.req.req_install import parse_editable @@ -357,3 +366,132 @@ def test_req_file_no_finder(tmpdir): """) parse_requirements(tmpdir.join("req.txt"), session=PipSession()) + + +def test_join_line_continuations(): + """ + Test joining of line continuations + """ + + lines = dedent('''\ + line 1 + line 2:1 \\ + line 2:2 + line 3:1 \\ + line 3:2 \\ + line 3:3 + line 4 + ''').splitlines() + + expect = [ + 'line 1', + 'line 2:1 line 2:2', + 'line 3:1 line 3:2 line 3:3', + 'line 4', + ] + + assert expect == list(join_lines(lines)) + + +def test_parse_editable_from_requirements(): + lines = [ + '--editable svn+https://foo#egg=foo', + '--editable=svn+https://foo#egg=foo', + '-e svn+https://foo#egg=foo' + ] + + res = [parse_line(i) for i in lines] + assert res == [(REQUIREMENT_EDITABLE, 'svn+https://foo#egg=foo')] * 3 + + req = next(parse_content('fn', lines[0])) + assert req.name == 'foo' + assert req.link.url == 'svn+https://foo#egg=foo' + + +@pytest.fixture +def session(): + return PipSession() + + +@pytest.fixture +def finder(session): + return PackageFinder([], [], session=session) + + +def test_parse_options_from_requirements(finder): + pl = parse_line + + pl('-i abc') == (OPTION, ('-i', 'abc')) + pl('-i=abc') == (OPTION, ('-i', 'abc')) + pl('-i = abc') == (OPTION, ('--index-url', 'abc')) + + pl('--index-url abc') == (OPTION, ('--index-url', 'abc')) + pl('--index-url=abc') == (OPTION, ('--index-url', 'abc')) + pl('--index-url = abc') == (OPTION, ('--index-url', 'abc')) + + with pytest.raises(RequirementsFileParseError): + parse_line('--allow-external') + + res = parse_line('--extra-index-url 123') + assert res == (OPTION, ('--extra-index-url', '123')) + + next(parse_content('fn', '-i abc', finder=finder), None) + assert finder.index_urls == ['abc'] + + +def test_parse_flags_from_requirements(finder): + assert parse_line('--no-index') == (FLAG, ('--no-index')) + assert parse_line('--no-use-wheel') == (FLAG, ('--no-use-wheel')) + + with pytest.raises(RequirementsFileParseError): + parse_line('--no-use-wheel true') + + next(parse_content('fn', '--no-index', finder=finder), None) + assert finder.index_urls == [] + + +def test_get_requirement_options(): + pro = parse_requirement_options + + res = pro('--aflag --bflag', ['--aflag', '--bflag']) + assert res == {'--aflag': '', '--bflag': ''} + + res = pro('--install-options="--abc --zxc"', [], ['--install-options']) + assert res == {'--install-options': '--abc --zxc'} + + res = pro('--aflag --global-options="--abc" --install-options="--aflag"', + ['--aflag'], ['--install-options', '--global-options']) + assert res == {'--aflag': '', '--global-options': '--abc', '--install-options': '--aflag'} + + line = 'INITools==2.0 --global-options="--one --two -3" --install-options="--prefix=/opt"' + assert parse_line(line) == (REQUIREMENT, ( + 'INITools==2.0', { + '--global-options': '--one --two -3', + '--install-options': '--prefix=/opt' + })) + + +def test_install_requirements_with_options(tmpdir, finder, session): + content = ''' + INITools == 2.0 --global-options="--one --two -3" \ + --install-options="--prefix=/opt" + ''' + + req_path = tmpdir.join('requirements.txt') + with open(req_path, 'w') as fh: + fh.write(content) + + req = next(parse_requirements(req_path, finder=finder, session=session)) + + req.source_dir = os.curdir + with patch.object(subprocess, 'Popen') as popen: + try: + req.install([]) + except: + pass + + call = popen.call_args_list[0][0][0] + for i in '--one', '--two', '-3', '--prefix=/opt': + assert i in call + + # TODO: assert that --global-options come before --install-options.