Merge pull request #232 from dephell/lazy-overwriting

Lazy dependencies overwriting

dephell/converters/pipfile.py

@@ -7,6 +7,7 @@
 from dephell_discover import Root as PackageRoot
 from dephell_pythons import Pythons
 from dephell_specifier import RangeSpecifier
+from packaging.utils import canonicalize_name
 
 # app
 from ..controllers import DependencyMaker, RepositoriesRegistry
@@ -119,22 +120,41 @@ def dumps(self, reqs, project: RootDependency, content=None) -> str:
             doc['requires']['python_version'] = str(python.get_short_version())
 
         # dependencies
+        names_mapping = dict()
         for section, is_dev in [('packages', False), ('dev-packages', True)]:
             # create section if doesn't exist
             if section not in doc:
                 doc[section] = tomlkit.table()
                 continue
 
-            # clean packages from old packages
-            req_names = {req.name for req in reqs if is_dev is req.is_dev}
-            for old_req in {name for name in doc[section] if name not in req_names}:
-                del doc[section][old_req]
+            # clean file from outdated dependencies
+            names = {req.name for req in reqs if is_dev is req.is_dev}
+            for name in dict(doc[section]):
+                normalized_name = canonicalize_name(name)
+                names_mapping[normalized_name] = name
+                if normalized_name not in names:
+                    del doc[section][name]
 
         # write new packages
         for section, is_dev in [('packages', False), ('dev-packages', True)]:
             for req in reqs:
-                if is_dev is req.is_dev:
-                    doc[section][req.raw_name] = self._format_req(req=req)
+                if is_dev is not req.is_dev:
+                    continue
+                raw_name = names_mapping.get(req.name, req.raw_name)
+                old_spec = doc[section].get(raw_name)
+
+                # do not overwrite dep if nothing is changed
+                if old_spec:
+                    old_dep = self._make_deps(
+                        root=RootDependency(),
+                        name=raw_name,
+                        content=old_spec,
+                    )[0]
+                    if req.same_dep(old_dep):
+                        continue
+
+                # overwrite
+                doc[section][raw_name] = self._format_req(req=req)
 
         return tomlkit.dumps(doc).rstrip() + '\n'
 

dephell/converters/poetry.py

@@ -7,6 +7,7 @@
 import tomlkit
 from dephell_discover import Root as PackageRoot
 from dephell_specifier import RangeSpecifier
+from packaging.utils import canonicalize_name
 
 # app
 from ..controllers import DependencyMaker, Readme, RepositoriesRegistry
@@ -159,24 +160,47 @@ def dumps(self, reqs, project: RootDependency, content=None) -> str:
         self._add_entrypoints(section=section, entrypoints=project.entrypoints)
 
         # dependencies
+        names_mapping = dict()
         for section_name, is_dev in [('dependencies', False), ('dev-dependencies', True)]:
             if section_name not in section:
                 section[section_name] = tomlkit.table()
                 continue
-            # clean dependencies from old dependencies
+
+            # clean file from outdated dependencies
             names = {req.name for req in reqs if is_dev is req.is_dev} | {'python'}
             for name in dict(section[section_name]):
-                if name not in names:
+                normalized_name = canonicalize_name(name)
+                names_mapping[normalized_name] = name
+                if normalized_name not in names:
                     del section[section_name][name]
 
         # python version
-        section['dependencies']['python'] = str(project.python) or '*'
+        if section['dependencies'].get('python', '') != (project.python or '*'):
+            section['dependencies']['python'] = str(project.python) or '*'
 
         # write dependencies
         for section_name, is_dev in [('dependencies', False), ('dev-dependencies', True)]:
             for req in reqs:
-                if is_dev is req.is_dev:
-                    section[section_name][req.raw_name] = self._format_req(req=req)
+                if is_dev is not req.is_dev:
+                    continue
+                raw_name = names_mapping.get(req.name, req.raw_name)
+                old_spec = section[section_name].get(raw_name)
+
+                # do not overwrite dep if nothing is changed
+                if old_spec:
+                    old_dep = self._make_deps(
+                        root=RootDependency(),
+                        name=raw_name,
+                        content=old_spec,
+                        envs={'main'},
+                    )[0]
+                    if req.same_dep(old_dep):
+                        continue
+
+                # overwrite
+                section[section_name][raw_name] = self._format_req(req=req)
+
+            # remove empty section
             if not section[section_name].value:
                 del section[section_name]
 

dephell/models/dependency.py

@@ -21,6 +21,7 @@ class Dependency:
     constraint = attr.ib(type=Constraint)
     repo = attr.ib(repr=False)
     link = attr.ib(default=None, repr=False)
+    marker = None
 
     # flags
     applied = attr.ib(type=bool, default=False, repr=False)
@@ -43,7 +44,7 @@ class Dependency:
 
     extra = None
 
-    # properties
+    # prlicenseoperties
 
     @cached_property
     def name(self) -> str:

dephell/models/requirement.py

@@ -3,7 +3,8 @@
 from typing import Iterable, Optional, Set, Tuple
 
 # external
-from dephell_links import DirLink, FileLink
+import attr
+from dephell_links import DirLink, FileLink, VCSLink
 
 # app
 from ..cached_property import cached_property
@@ -217,6 +218,30 @@ def optional(self) -> bool:
             return False
         return bool(self.dep.envs - {'dev', 'main'})
 
+    # methods
+
+    @staticmethod
+    def _get_comparable_dict(dep) -> dict:
+        excluded = {'constraint', 'repo', 'link', 'marker', 'license', 'inherited_envs', 'locations'}
+        result = attr.asdict(dep, recurse=True, filter=lambda x, _: x.name not in excluded)
+        result['constraint'] = str(dep.constraint)
+        if dep.marker:
+            result['marker'] = str(dep.marker)
+        if isinstance(dep.link, VCSLink):
+            result['link'] = attr.asdict(dep.link, recurse=True)
+        result['license'] = str(dep.license)
+        return result
+
+    def same_dep(self, other_dep) -> bool:
+        """Is given dependency the same as dependency inside of this Requirement?
+
+        It's used in converters to not overwrite unchanged requirements.
+        It looks in this way to make comparation as easy as possible:
+        converters produce dependencies, but get requirements.
+        So, it's easier to compare requirement with dependency.
+        """
+        return self._get_comparable_dict(self.dep) == self._get_comparable_dict(other_dep)
+
     # magic methods
 
     def __iter__(self):

setup.py

@@ -23,7 +23,7 @@
 setup(
     long_description=readme,
     name='dephell',
-    version='0.7.5',
+    version='0.7.6',
     description='Dependency resolution for Python',
     python_requires='>=3.5',
     project_urls={
@@ -33,8 +33,7 @@
     author='Gram',
     author_email='master_fess@mail.ru',
     license='MIT',
-    keywords=
-    'dephell packaging dependency dependencies venv licenses pip poetry pipfile pipenv setuptools',
+    keywords='dephell packaging dependency dependencies venv licenses pip poetry pipfile pipenv setuptools',
     classifiers=[
         'Development Status :: 4 - Beta', 'Environment :: Console',
         'Framework :: Setuptools Plugin', 'Intended Audience :: Developers',
@@ -66,12 +65,12 @@
     ],
     extras_require={
         'full': ['aiofiles', 'autopep8', 'colorama', 'graphviz', 'yapf'],
+        'tests': ['aioresponses', 'pytest', 'requests-mock'],
         'dev': [
             'aioresponses', 'flake8-isort', 'isort[pyproject]',
             'pygments-github-lexers', 'pytest', 'recommonmark', 'requests-mock',
             'sphinx', 'sphinx-rtd-theme'
         ],
-        'tests': ['aioresponses', 'pytest', 'requests-mock'],
         'docs': [
             'pygments-github-lexers', 'recommonmark', 'sphinx',
             'sphinx-rtd-theme'

tests/test_commands/test_project_bump.py

@@ -36,6 +36,9 @@ def test_bump_pyproject(temp_path: Path):
 
         [tool.poetry.dependencies]
         python = "*"
+        sentry_sdk = ">=0.9.0"
+        npm = "^0.9.0"
+        reponame = { git = "ssh://git@our-git-server:port/group/reponame.git", branch = "v3_2" }
 
         [[tool.poetry.source]]
         name = "pypi"

tests/test_converters/test_poetry.py

@@ -3,6 +3,7 @@
 from textwrap import dedent
 
 # external
+import pytest
 import tomlkit
 
 # project
@@ -84,3 +85,33 @@ def test_preserve_repositories():
     new_parsed = tomlkit.parse(new_content)['tool']['poetry']
     assert parsed['source'] == new_parsed['source']
     assert parsed == new_parsed
+
+
+@pytest.mark.parametrize('req', [
+    'a = "*"',
+    'a = "^9.5"',
+    'strangE_nAm.e = ">=9.5"',
+    'reponame = { git = "ssh://git@our-git-server:port/group/reponame.git", branch = "v3_2" }',
+    'a = {version = "*", extras = ["nani"] }',
+    'a = "*"  # god bless comments',
+])
+def test_preserve_reqs_format(req, temp_path: Path):
+    content = dedent("""
+        [tool.poetry]
+        name = "test"
+        version = "1.2.3"
+
+        [tool.poetry.dependencies]
+        python = "*"
+        {req}
+    """).format(req=req)
+
+    converter = PoetryConverter(project_path=temp_path)
+    resolver = converter.loads_resolver(content)
+    reqs = Requirement.from_graph(graph=resolver.graph, lock=False)
+    new_content = converter.dumps(
+        reqs=reqs,
+        project=resolver.graph.metainfo,
+        content=content,
+    )
+    assert content == new_content

tests/test_models/test_requirement.py

@@ -0,0 +1,40 @@
+# external
+import pytest
+
+# project
+from dephell.converters import PIPConverter
+from dephell.models import Requirement
+
+
+@pytest.mark.parametrize('left, right, ok', [
+    ('a>=3',    'a>=3',     True),
+    ('a >= 3',  'a>=3',     True),
+    ('a>=3',    'a>3',      False),
+    (
+        '-e git+git@github.com:dephell/dephell.git#egg=dephell',
+        '-e git+git@github.com:dephell/dephell.git#egg=dephell',
+        True,
+    ),
+    (
+        '-e git+git@github.com:dephell/dephell.git#egg=dephell',
+        '-e git+ssh://git@github.com:dephell/dephell.git#egg=dephell',
+        True,
+    ),
+    (
+        '-e git+git@github.com:dephell/dephell.git#egg=dephell',
+        '-e git+git@github.com:dephell/dephell_discovery.git#egg=dephell',
+        False,
+    ),
+])
+def test_equal(left, right, ok):
+    converter = PIPConverter(lock=False)
+    dep_left = converter.loads(left).dependencies[0]
+    dep_right = converter.loads(right).dependencies[0]
+
+    dict_left = Requirement._get_comparable_dict(dep_left)
+    dict_right = Requirement._get_comparable_dict(dep_right)
+    assert (dict_left == dict_right) is ok
+
+    req_left = Requirement(dep=dep_left, lock=False)
+    equal = req_left.same_dep(dep_right)
+    assert equal is ok