Merge pull request #215 from dephell/fix-repositories-propagation

Preserve repositories for empty dependencies

dephell/controllers/_dependency.py

@@ -54,10 +54,14 @@ def from_requirement(cls, source, req, *, url=None, envs=None, marker: Union[Mar
             envs = {'main'}
         envs.update(marker.extract('extra'))
 
+        default_repo = None
+        if source.repo and source.repo.propagate:
+            default_repo = source.repo
+
         base_dep = cls.dep_class(
             raw_name=req.name,
             constraint=constraint,
-            repo=get_repo(link),
+            repo=get_repo(link, default=default_repo),
             link=link,
             marker=marker,
             editable=editable,
@@ -71,7 +75,7 @@ def from_requirement(cls, source, req, *, url=None, envs=None, marker: Union[Mar
 
     @classmethod
     def from_params(cls, *, raw_name: str, constraint,
-                    url: Optional[str] = None, source: Optional['Dependency'] = None,
+                    source: Dependency, url: Optional[str] = None,
                     repo=None, marker: Union[Markers, str] = None,
                     extras: Optional[List[str]] = None, envs=None,
                     **kwargs) -> List[Union[Dependency, ExtraDependency]]:
@@ -82,14 +86,17 @@ def from_params(cls, *, raw_name: str, constraint,
             raw_name = link.name
 
         # make constraint
-        if source:
+        if isinstance(constraint, str):
             constraint = Constraint(source, constraint)
             if isinstance(link, VCSLink) and link.rev:
                 constraint._specs[source.name] = GitSpecifier()
 
         # make repo
         if repo is None:
-            repo = get_repo(link)
+            default_repo = None
+            if source.repo and source.repo.propagate:
+                default_repo = source.repo
+            repo = get_repo(link, default=default_repo)
 
         # make marker
         if isinstance(marker, Markers):

dephell/controllers/_repos.py

@@ -38,6 +38,8 @@ class RepositoriesRegistry(WarehouseBaseRepo):
     _urls = attr.ib(factory=set)
     _names = attr.ib(factory=set)
 
+    propagate = True
+
     def add_repo(self, *, url: str, name: str = None, from_config: bool = False) -> bool:
         # try to interpret URL as local path
         if url in self._urls:

dephell/converters/base.py

@@ -54,8 +54,11 @@ def dumps(self, reqs, *, project: RootDependency, content: str = None) -> str:
         raise NotImplementedError
 
     def dump(self, reqs, *, path, project: RootDependency) -> None:
+        if isinstance(path, str):
+            path = Path(path)
+        path = self._make_source_path_absolute(path)
+
         # read
-        path = Path(str(path))
         if path.exists():
             with path.open('r', encoding='utf8') as stream:
                 content = stream.read()

dephell/converters/pipfile.py

@@ -165,6 +165,7 @@ def _make_deps(root, name: str, content) -> List[Dependency]:
             raw_name=name,
             # https://github.com/sarugaku/requirementslib/blob/master/src/requirementslib/models/utils.py
             constraint=Constraint(root, content.get('version', '')),
+            source=root,
             extras=set(content.get('extras', [])),
             marker=content.get('markers'),
             url=url,

dephell/converters/poetry.py

@@ -11,7 +11,7 @@
 # app
 from ..controllers import DependencyMaker, Readme, RepositoriesRegistry
 from ..models import Author, Constraint, Dependency, EntryPoint, RootDependency
-from ..repositories import WarehouseBaseRepo, WarehouseLocalRepo, get_repo
+from ..repositories import WarehouseLocalRepo
 from .base import BaseConverter
 
 
@@ -82,6 +82,13 @@ def loads(self, content) -> RootDependency:
                 root.entrypoints.append(EntryPoint(name=name, path=path, group=group_name))
         root.entrypoints = tuple(root.entrypoints)
 
+        # update repository
+        root.repo = RepositoriesRegistry()
+        if section.get('source'):
+            for source in section['source']:
+                root.repo.add_repo(url=source['url'], name=source['name'])
+        root.repo.attach_config()
+
         # get envs for deps
         envs = defaultdict(set)
         for extra, deps in section.get('extras', {}).items():
@@ -106,16 +113,6 @@ def loads(self, content) -> RootDependency:
                     envs=envs.get(name),
                 ))
 
-        # update repository
-        if section.get('source'):
-            repo = RepositoriesRegistry()
-            for source in section['source']:
-                repo.add_repo(url=source['url'], name=source['name'])
-            repo.attach_config()
-            for dep in deps:
-                if isinstance(dep.repo, WarehouseBaseRepo):
-                    dep.repo = repo
-
         root.attach_dependencies(deps)
         return root
 
@@ -208,15 +205,17 @@ def dumps(self, reqs, project: RootDependency, content=None) -> str:
             # deop all old extras if there are no new extras
             del section['extras']
 
-        self._add_repositories(section=section, reqs=reqs)
+        if not project.dependencies:
+            project.attach_dependencies([req.dep for req in reqs])
+        self._add_repositories(section=section, root=project)
         return tomlkit.dumps(doc).rstrip() + '\n'
 
     @staticmethod
     def _add_entrypoints(section, entrypoints):
         # drop old console_scripts
         if 'scripts' in section:
             scripts = {e.name for e in entrypoints if e.group == 'console_scripts'}
-            for script_name in section['scripts']:
+            for script_name in list(section['scripts']):
                 if script_name not in scripts:
                     del section['scripts'][script_name]
 
@@ -259,34 +258,26 @@ def _add_entrypoints(section, entrypoints):
             section['plugins'][entrypoint.group][entrypoint.name] = entrypoint.path
 
     @staticmethod
-    def _add_repositories(section, reqs):
+    def _add_repositories(section, root: RootDependency):
         # get repositories
         urls = dict()
-        for req in reqs:
-            if not isinstance(req.dep.repo, WarehouseBaseRepo):
+        for repo in root.warehouses:
+            if isinstance(repo, WarehouseLocalRepo):
                 continue
-            for repo in req.dep.repo.repos:
-                if repo.from_config:
-                    continue
-                if isinstance(repo, WarehouseLocalRepo):
-                    continue
-                urls[repo.name] = repo.pretty_url
+            urls[repo.name] = repo.pretty_url
 
         # remove or update old repositories
         added = []
+        sources = tomlkit.aot()
         if section.get('source'):
             old = list(section['source'])
-            section['source'] = tomlkit.aot()
             added = []
             for source in old:
                 if source['name'] in urls:
                     if source['url'] != urls[source['name']]:
                         source['url'] = urls[source['name']]
-                    section['source'].append(source)
+                    sources.append(source)
                     added.append(source['name'])
-            sources = section['source']
-        else:
-            sources = tomlkit.aot()
 
         # add new repositories
         for name, url in sorted(urls.items()):
@@ -298,14 +289,18 @@ def _add_repositories(section, reqs):
 
         section['source'] = sources
 
+        # remove section if empty
+        if not section['source']:
+            del section['source']
+
     # https://github.com/sdispater/tomlkit/blob/master/pyproject.toml
     @staticmethod
     def _make_deps(root, name: str, content, envs: set) -> List[Dependency]:
         if isinstance(content, str):
             deps = [Dependency(
                 raw_name=name,
                 constraint=Constraint(root, content),
-                repo=get_repo(),
+                repo=root.repo,
                 envs=envs,
             )]
             return deps
@@ -330,6 +325,7 @@ def _make_deps(root, name: str, content, envs: set) -> List[Dependency]:
         deps = DependencyMaker.from_params(
             raw_name=name,
             constraint=Constraint(root, content.get('version', '')),
+            source=root,
             extras=set(content.get('extras', [])),
             marker=markers or None,
             url=url,

dephell/converters/poetrylock.py

@@ -40,11 +40,11 @@ def loads(self, content) -> RootDependency:
         root.python = RangeSpecifier(doc.get('metadata', {}).get('python-versions', '*'))
 
         # get repositories
-        repo = RepositoriesRegistry()
+        root.repo = RepositoriesRegistry()
         if doc.get('source'):
             for source in doc['source']:
-                repo.add_repo(url=source['url'], name=source['name'])
-        repo.attach_config()
+                root.repo.add_repo(url=source['url'], name=source['name'])
+        root.repo.attach_config()
 
         envs = defaultdict(set)
         for extra, deps in doc.get('extras', {}).items():
@@ -60,7 +60,7 @@ def loads(self, content) -> RootDependency:
                 root=root,
                 content=content,
                 envs=envs[content['name']],
-                repo=repo,
+                repo=root.repo,
             ))
         root.attach_dependencies(deps)
         return root
@@ -83,23 +83,15 @@ def dumps(self, reqs, project: RootDependency, content=None) -> str:
 
         # add repositories
         sources = tomlkit.aot()
-        added = set()
-        for req in reqs:
-            if not isinstance(req.dep.repo, RepositoriesRegistry):
+        if not project.dependencies:
+            project.attach_dependencies([req.dep for req in reqs])
+        for repo in project.warehouses:
+            if isinstance(repo, WarehouseLocalRepo):
                 continue
-            for repo in req.dep.repo.repos:
-                if repo.from_config:
-                    continue
-                if repo.name in added:
-                    continue
-                if isinstance(repo, WarehouseLocalRepo):
-                    continue
-                added.add(repo.name)
-
-                source = tomlkit.table()
-                source['name'] = repo.name
-                source['url'] = repo.pretty_url
-                sources.append(source)
+            source = tomlkit.table()
+            source['name'] = repo.name
+            source['url'] = repo.pretty_url
+            sources.append(source)
         if sources:
             doc['source'] = sources
 
@@ -150,6 +142,7 @@ def _make_deps(cls, root, content, envs, repo) -> List[Dependency]:
             raw_name=content['name'],
             description=content.get('description', ''),
             constraint=Constraint(root, version),
+            source=root,
             marker=marker,
             url=url,
             editable=False,
@@ -173,6 +166,7 @@ def _make_deps(cls, root, content, envs, repo) -> List[Dependency]:
                     raw_name=subname,
                     constraint=Constraint(root, '==' + subcontent),
                     envs=envs,
+                    source=root,
                 ))
                 continue
 
@@ -186,6 +180,7 @@ def _make_deps(cls, root, content, envs, repo) -> List[Dependency]:
             subdeps.extend(DependencyMaker.from_params(
                 raw_name=subname,
                 constraint=Constraint(root, subcontent['version']),
+                source=root,
                 marker=marker,
                 envs=envs,
             ))

dephell/models/root.py

@@ -1,5 +1,6 @@
 # built-in
 from contextlib import suppress
+from itertools import chain
 from pathlib import Path
 from typing import Tuple
 
@@ -38,6 +39,7 @@ def __str__(self):
 class RootDependency:
     raw_name = attr.ib(default='root')
     dependencies = attr.ib(factory=list, repr=False)
+    repo = attr.ib(default=None, repr=False)
 
     # additional info strings
     version = attr.ib(default='0.0.0', repr=False)      # Version
@@ -57,7 +59,6 @@ class RootDependency:
     python = attr.ib(default=RangeSpecifier(), repr=False)  # Requires-Python
     readme = attr.ib(default=None, repr=False)              # Description
 
-    repo = None
     applied = False
     locked = False
     compat = True
@@ -95,6 +96,22 @@ def groups(self) -> Tuple[Group]:
     def python_compat(self) -> bool:
         return True
 
+    @property
+    def warehouses(self) -> tuple:
+        from ..repositories import WarehouseBaseRepo
+
+        repos = dict()
+        for dep in chain(self.dependencies, [self]):
+            if dep.repo is None:
+                continue
+            if not isinstance(dep.repo, WarehouseBaseRepo):
+                continue
+            for repo in dep.repo.repos:
+                if repo.from_config:
+                    continue
+                repos[repo.name] = repo
+        return tuple(repos.values())
+
     def attach_dependencies(self, dependencies) -> None:
         self.dependencies.extend(dependencies)
 

dephell/repositories/_getter.py

@@ -18,13 +18,15 @@
 )
 
 
-def get_repo(link=None, name: str = None):
+def get_repo(link=None, *, name: str = None, default=None):
+    from ..controllers import RepositoriesRegistry
+
     if name is not None:
         return _repos[name]
 
     if link is None:
-        from ..controllers import RepositoriesRegistry
-
+        if default is not None:
+            return default
         repo = RepositoriesRegistry()
         repo.attach_config()
         return repo

tests/test_commands/test_project_bump.py

@@ -1,9 +1,10 @@
 # built-in
 from pathlib import Path
+from textwrap import dedent
 
 # external
-# third-party
 import pytest
+import tomlkit
 
 # project
 from dephell.actions._git import _run
@@ -26,6 +27,37 @@ def test_bump_command(temp_path: Path):
     assert init_path.read_text() == '__version__ = "1.2.4"'
 
 
+def test_bump_pyproject(temp_path: Path):
+    from_path = temp_path / 'pyproject.toml'
+    from_path.write_text(dedent("""
+        [tool.poetry]
+        name = "check-me"
+        version = "1.2.3"
+
+        [tool.poetry.dependencies]
+        python = "*"
+
+        [[tool.poetry.source]]
+        name = "pypi"
+        url = "https://pypi.org/pypi"
+    """))
+    before_toml = tomlkit.loads(from_path.read_text())
+    config = Config()
+    config.attach({
+        'project': str(temp_path),
+        'from': {'format': 'poetry', 'path': 'pyproject.toml'},
+    })
+
+    command = ProjectBumpCommand(argv=['fix'], config=config)
+    result = command()
+
+    assert result is True
+    after_toml = tomlkit.loads(from_path.read_text())
+    assert after_toml['tool']['poetry']['version'] == '1.2.4', 'Version was not bumped properly'
+    after_toml['tool']['poetry']['version'] = '1.2.3'
+    assert after_toml == before_toml, 'Bump command altered attributes other than version'
+
+
 @pytest.mark.parametrize('tag_template, expected_tag', [
     ('prefix.', 'prefix.1.2.4'),
     ('with.{version}.placeholder', 'with.1.2.4.placeholder'),