Merge pull request #420 from dephell/fixes

Small fixes in requesting warehouse (simple and API)

dephell/cache.py

@@ -16,7 +16,7 @@ class BaseCache:
     def __init__(self, *keys, ttl: int = -1):
         self.path = Path(config['cache']['path'], *keys)
         if self.ext:
-            self.path = self.path.with_suffix(self.ext)
+            self.path = self.path.with_name(self.path.name + self.ext)
         self.ttl = ttl
         self._check_ttl()
 

dephell/commands/deps_tree.py

@@ -63,12 +63,18 @@ def __call__(self) -> bool:
 
     @classmethod
     def _make_tree(cls, dep, *, level: int = 0) -> List[str]:
-        lines = ['{level}- {name} [required: {constraint}, locked: {best}, latest: {latest}]'.format(
+        best = dep.group.best_release.version
+        latest = dep.group.best_release.version
+        if best == latest:
+            template = '{level}- {name} [required: {constraint}, latest: {latest}]'
+        else:
+            template = '{level}- {name} [required: {constraint}, locked: {best}, latest: {latest}]'
+        lines = [template.format(
             level='  ' * level,
             name=dep.name,
             constraint=str(dep.constraint) or '*',
-            best=str(dep.group.best_release.version),
-            latest=str(dep.groups.releases[0].version),
+            best=str(best),
+            latest=str(latest),
         )]
         deps = {dep.name: dep for dep in dep.dependencies}.values()  # drop duplicates
         for subdep in sorted(deps):

dephell/converters/egginfo.py

@@ -2,14 +2,15 @@
 from collections import defaultdict
 from email.parser import Parser
 from itertools import chain
+from logging import getLogger
 from pathlib import Path
 from typing import Dict, Optional
 
 # external
 from dephell_discover import Root as PackageRoot
 from dephell_links import parse_link
 from dephell_markers import Markers
-from packaging.requirements import Requirement as PackagingRequirement
+from packaging.requirements import InvalidRequirement, Requirement as PackagingRequirement
 
 # app
 from ..constants import DOWNLOAD_FIELD, HOMEPAGE_FIELD
@@ -20,6 +21,7 @@
 
 
 class _Reader:
+    logger = getLogger('dephell.converters.egginfo')
 
     def can_parse(self, path: Path, content: Optional[str] = None) -> bool:
         if isinstance(path, str):
@@ -151,7 +153,15 @@ def parse_info(cls, content: str, root=None, urls: Dict[str, str] = None) -> Roo
         # dependencies
         deps = []
         for req in cls._get_list(info, 'Requires-Dist'):
-            req = PackagingRequirement(req)
+            try:
+                req = PackagingRequirement(req)
+            except InvalidRequirement:
+                cls.logger.warning('invalid requirement', extra=dict(
+                    requirement=req,
+                    package_name=root.name,
+                    package_version=root.version,
+                ))
+                continue
             deps.extend(DependencyMaker.from_requirement(
                 source=root,
                 req=req,

dephell/models/dependency.py

@@ -193,7 +193,7 @@ def __str__(self) -> str:
 
         marker = Markers(str(self.marker))
         if self.envs - {'main'}:
-            extra_markers = {'extra == "{}"'.format(env) for env in self.envs - {'main'}}
+            extra_markers = {'extra == {!r}'.format(env) for env in self.envs - {'main'}}
             marker &= Markers(' or '.join(extra_markers))
         if marker:
             result += '; ' + str(marker)

dephell/networking.py

@@ -1,16 +1,20 @@
 # built-in
+from functools import partial, update_wrapper
+from logging import getLogger
 from ssl import create_default_context
+from time import sleep
 
 # external
 import certifi
 import requests
-from aiohttp import ClientSession, TCPConnector
+from aiohttp import ClientError, ClientSession, TCPConnector
 
 # app
 from . import __version__
 
 
 USER_AGENT = 'DepHell/{version}'.format(version=__version__)
+logger = getLogger('dephell.networking')
 
 
 def aiohttp_session(*, auth=None, **kwargs):
@@ -36,3 +40,22 @@ def requests_session(*, auth=None, headers=None, **kwargs):
     if kwargs:
         session.__dict__.update(kwargs)
     return session
+
+
+def aiohttp_repeat(func=None, *, count: int = 4):
+    if func is None:
+        return partial(func, count=count)
+
+    async def wrapper(*args, **kwargs):
+        for pause in range(1, count + 1):
+            try:
+                return await func(*args, **kwargs)
+            except ClientError:
+                if pause == count:
+                    raise
+                logger.debug('aiohttp payload error, repeating...', exc_info=True)
+                sleep(pause)
+        raise RuntimeError('unreachable')
+
+    wrapper = update_wrapper(wrapper=wrapper, wrapped=func)
+    return wrapper

dephell/repositories/_warehouse/_base.py

@@ -13,7 +13,7 @@
 # app
 from ...cached_property import cached_property
 from ...constants import WAREHOUSE_DOMAINS
-from ...networking import aiohttp_session
+from ...networking import aiohttp_repeat, aiohttp_session
 from ..base import Interface
 
 
@@ -99,22 +99,23 @@ def _convert_deps(*, deps, name, version, extra):
                     ))
 
             try:
-                dep_extra = req.marker and Markers(req.marker).extra
+                dep_extras = req.marker and Markers(req.marker).get_strings('extra')
             except ValueError:  # unsupported operation for version marker python_version: in
-                dep_extra = None
+                dep_extras = set()
 
             # it's not extra and we want not extra too
-            if dep_extra is None and extra is None:
+            if not dep_extras and extra is None:
                 result.append(req)
                 continue
             # it's extra, but we want not the extra
             # or it's not the extra, but we want extra.
-            if dep_extra is None or extra is None:
+            if not dep_extras or extra is None:
                 continue
             # it's extra and we want this extra
-            elif dep_extra == extra:
-                result.append(req)
-                continue
+            for dep_extra in dep_extras:
+                if dep_extra == extra:
+                    result.append(req)
+                    break
 
         return tuple(result)
 
@@ -136,6 +137,7 @@ async def _download_and_parse(self, *, url: str, converter) -> Tuple[str, ...]:
                         deps.append(str(dep))
             return tuple(deps)
 
+    @aiohttp_repeat
     async def _download(self, *, url: str, path: Path) -> None:
         async with aiohttp_session(auth=self.auth) as session:
             async with session.get(url) as response:

dephell/repositories/_warehouse/_simple.py

@@ -53,9 +53,19 @@ def get_releases(self, dep) -> tuple:
         releases_info = dict()
         for link in links:
             name, version = self._parse_name(link['name'])
-            if canonicalize_name(name) != dep.name:
+            if canonicalize_name(name) != canonicalize_name(dep.base_name):
+                logger.warning('bad dist name', extra=dict(
+                    dist_name=link['name'],
+                    package_name=dep.base_name,
+                    reason='package name does not match',
+                ))
                 continue
             if not version:
+                logger.warning('bad dist name', extra=dict(
+                    dist_name=link['name'],
+                    package_name=dep.base_name,
+                    reason='no version specified',
+                ))
                 continue
 
             if version not in releases_info:
@@ -142,11 +152,13 @@ def _get_links(self, name: str) -> List[Dict[str, str]]:
             ttl=config['cache']['ttl'],
         )
         links = cache.load()
-        if links:
-            return links
+        if links is not None:
+            yield from links
+            return
 
         dep_url = posixpath.join(self.url, quote(name)) + '/'
         with requests_session() as session:
+            logger.debug('getting dep info from simple repo', extra=dict(url=dep_url))
             response = session.get(dep_url, auth=self.auth)
         if response.status_code == 404:
             raise PackageNotFoundError(package=name, url=dep_url)
@@ -164,17 +176,19 @@ def _get_links(self, name: str) -> List[Dict[str, str]]:
 
             python = tag.get('data-requires-python')
             fragment = parse_qs(parsed.fragment)
-            yield dict(
+            link = dict(
                 url=urljoin(dep_url, link),
                 name=parsed.path.strip('/').split('/')[-1],
                 python=html.unescape(python) if python else '*',
                 digest=fragment['sha256'][0] if 'sha256' in fragment else None,
             )
+            links.append(link)
+            yield link
 
         cache.dump(links)
         return links
 
-    async def _get_deps_from_links(self, name, version):
+    async def _get_deps_from_links(self, name: str, version):
         from ...converters import SDistConverter, WheelConverter
 
         links = self._get_links(name=name)
@@ -183,7 +197,7 @@ async def _get_deps_from_links(self, name, version):
             link_name, link_version = self._parse_name(link['name'])
             if canonicalize_name(link_name) != name:
                 continue
-            if link_version != version:
+            if link_version != str(version):
                 continue
             good_links.append(link)
 

docs/conf.py

@@ -7,7 +7,6 @@
 
 # external
 import alabaster
-from recommonmark.parser import CommonMarkParser
 from recommonmark.transform import AutoStructify
 
 
@@ -20,12 +19,10 @@
     'sphinx.ext.coverage',
     'sphinx.ext.viewcode',
     'sphinx.ext.githubpages',
+    'recommonmark',
 ]
 
 templates_path = ['_templates']
-source_parsers = {
-    '.md': CommonMarkParser,
-}
 source_suffix = ['.rst', '.md']
 master_doc = 'index'
 

setup.cfg

@@ -11,3 +11,8 @@ exclude=
     .pytest_cache
     build
     setup.py
+
+[tool:pytest]
+addopts = --strict-markers
+markers =
+    allow_hosts: allow network requests to specified hostnames.

tests/test_controllers/test_safety.py

@@ -9,5 +9,5 @@
 def test_safety():
     safety = Safety()
     vulns = safety.get('django', '1.11.0')
-    assert len(vulns) == 5
+    assert len(vulns) == 13
     assert {vuln.name for vuln in vulns} == {'django'}