From 81976f752af0a6fe2e18c3e779951362c3a4fe38 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 13 Apr 2018 19:55:16 -0400 Subject: [PATCH] Handle indexes, extra indexes, uncached sources - Handle extra-index-urls when resolving - Handle extra-index-url when using `--skip-lock` - Parse index arguments when installing individual packages - Translate index aliases to urls - Always include extra indexes when installing a packages - `get_source()` falls back to `parsed_pipfile['source']` for sources when not present in the lockfile (#1994) - Include index and extra-index-url arguments in `pipenv lock -r` output - Fixes #1973, #1974, #1852, #1977, #1994 Signed-off-by: Dan Ryan --- pipenv/core.py | 50 +++++++++++++++++++++++++++++++++-------------- pipenv/project.py | 38 +++++++++++++++++++++++++++++------ pipenv/utils.py | 3 +++ 3 files changed, 70 insertions(+), 21 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 4944f538da..a29bbf733a 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -20,12 +20,12 @@ import pipdeptree from .vendor.pipreqs import pipreqs from blindspin import spinner -from first import first from requests.packages import urllib3 from requests.packages.urllib3.exceptions import InsecureRequestWarning +import six from .cmdparse import ScriptEmptyError -from .project import Project +from .project import Project, SourceNotFound from .utils import ( convert_deps_from_pip, convert_deps_to_pip, @@ -778,7 +778,10 @@ def cleanup_procs(procs, concurrent): l[i] = list(l[i]) if '--hash' in l[i][0]: l[i][0] = (l[i][0].split('--hash')[0].strip()) + index_args = prepare_pip_source_args(project.sources) + index_args = ' '.join(index_args).replace(' -', '\n-') # Output only default dependencies + click.echo(index_args) if not dev: click.echo('\n'.join(d[0] for d in sorted(deps_list))) sys.exit(0) @@ -836,7 +839,7 @@ def cleanup_procs(procs, concurrent): verbose=verbose, index=index, requirements_dir=requirements_dir, - extra_index=extra_index, + extra_indexes=extra_index, ) # The Installation failed... if c.return_code != 0: @@ -1417,20 +1420,25 @@ def pip_install( # Try installing for each source in project.sources. if index: - valid_indexes = [] - extra_indexes = [] if not extra_indexes else [extra_indexes] - if 'source' in project.parsed_pipfile: - valid_indexes = [p['name'] for p in project.parsed_pipfile['source']] - if not is_valid_url(index) and index in valid_indexes: - index = first([p['url'] for p in project.parsed_pipfile['source'] if p['name'] == index]) + if not is_valid_url(index): + index = project.find_source(index).get('url') sources = [{'url': index}] if extra_indexes: - extra_indexes = [{'url': extra_src} for extra_src in extra_indexes if extra_src != index] - elif 'source' in project.parsed_pipfile and len(project.parsed_pipfile['source']) > 1: - extra_indexes = [{'url': s['url']} for s in project.parsed_pipfile['source'] if s['url'] != index] - sources = sources.extend(extra_indexes) + if isinstance(extra_indexes, six.string_types): + extra_indexes = [extra_indexes,] + for idx in extra_indexes: + try: + extra_src = project.find_source(idx).get('url') + except SourceNotFound: + extra_src = idx + if extra_src != index: + sources.append({'url': extra_src}) + else: + for idx in project.pipfile_sources: + if idx['url'] != sources[0]['url']: + sources.append({'url': idx['url']}) else: - sources = project.sources + sources = project.pipfile_sources if package_name.startswith('-e '): install_reqs = ' -e "{0}"'.format(package_name.split('-e ')[1]) elif r: @@ -1452,7 +1460,6 @@ def pip_install( pre = '--pre' if pre else '' quoted_python = which('python', allow_global=allow_global) quoted_python = escape_grouped_arguments(quoted_python) - sources = [] upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else '' pip_command = '{0} -m pipenv.vendor.pip9 install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format( quoted_python, @@ -1848,6 +1855,17 @@ def do_install( more_packages = list(more_packages) if package_name == '-e': package_name = ' '.join([package_name, more_packages.pop(0)]) + # capture indexes and extra indexes + line = [package_name] + more_packages + index_indicators = ['-i', '--index', '--extra-index-url'] + index, extra_indexes = None, None + if more_packages and any(more_packages[0].startswith(s) for s in index_indicators): + line, index = split_index(' '.join(line)) + line, extra_indexes = split_extra_index(line) + package_names = line.split() + package_name = package_names[0] + if len(package_names) > 1: + more_packages = package_names[1:] # Capture . argument and assign it to nothing if package_name == '.': package_name = False @@ -1927,6 +1945,8 @@ def do_install( verbose=verbose, pre=pre, requirements_dir=requirements_directory.name, + index=index, + extra_indexes=extra_indexes, ) # Warn if --editable wasn't passed. try: diff --git a/pipenv/project.py b/pipenv/project.py index 14b204cdbc..50522b0c51 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -614,6 +614,12 @@ def write_toml(self, data, path=None): # pipfile is mutated! self.clear_pipfile_cache() + @property + def pipfile_sources(self): + if 'source' in self.parsed_pipfile: + return self.parsed_pipfile['source'] + return [DEFAULT_SOURCE] + @property def sources(self): if self.lockfile_exists: @@ -627,15 +633,35 @@ def sources(self): else: return [DEFAULT_SOURCE] + def find_source(self, source): + """given a source, find it. + + source can be a url or an index name. + """ + if not is_valid_url(source): + try: + source = self.get_source(name=source) + except SourceNotFound: + source = self.get_source(url=source) + else: + source = self.get_source(url=source) + return source + def get_source(self, name=None, url=None): - for source in self.sources: + def find_source(sources, name=None, url=None): if name: - if source.get('name') == name: - return source - + source = [s for s in sources if s.get('name') == name] elif url: - if source.get('url') in url: - return source + source = [s for s in sources if s.get('url') in url] + if source: + return source[0] + + found_source = find_source(self.sources, name=name, url=url) + if found_source: + return found_source + found_source = find_source(self.pipfile_sources, name=name, url=url) + if found_source: + return found_source raise SourceNotFound(name or url) def destroy_lockfile(self): diff --git a/pipenv/utils.py b/pipenv/utils.py index d5f97ca251..8db1b43dff 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -675,6 +675,9 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False): pip_src_args = [] if 'index' in deps[dep]: pip_src_args = [project.get_source(deps[dep]['index'])] + for idx in project.sources: + if idx['url'] != pip_src_args[0]['url']: + pip_src_args.append(idx) else: pip_src_args = project.sources pip_args = prepare_pip_source_args(pip_src_args)