Merge pull request #240 from kammala/fix/support-port-in-url

Support warehouse urls with port inside

dephell/repositories/_warehouse/_api.py

@@ -44,6 +44,7 @@
 class WarehouseAPIRepo(WarehouseBaseRepo):
     name = attr.ib(type=str)
     url = attr.ib(type=str)
+    pretty_url = attr.ib(type=str, default='')
     auth = attr.ib(default=None)
 
     prereleases = attr.ib(type=bool, factory=lambda: config['prereleases'])  # allow prereleases
@@ -57,24 +58,9 @@ def __attrs_post_init__(self):
         # make name canonical
         if self.name in ('pypi.org', 'pypi.python.org'):
             self.name = 'pypi'
-
-        # replace link on simple index by link on pypi api
-        parsed = urlparse(self.url)
-        if parsed.path in ('', '/', '/simple', '/simple/'):
-            path = '/pypi/'
-        else:
-            path = parsed.path
-        if parsed.hostname == 'pypi.python.org':
-            hostname = 'pypi.org'
-        else:
-            hostname = parsed.hostname
-        self.url = parsed.scheme + '://' + hostname + path
-
-    @property
-    def pretty_url(self):
-        parsed = urlparse(self.url)
-        path = '/simple/' if parsed.path == '/pypi/' else parsed.path
-        return parsed.scheme + '://' + parsed.hostname + path
+        if not self.pretty_url:
+            self.pretty_url = self.url
+        self.url = self._get_url(self.url, default_path='/pypi/')
 
     def get_releases(self, dep) -> tuple:
         # retrieve data

dephell/repositories/_warehouse/_base.py

@@ -4,14 +4,15 @@
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Tuple
-from urllib.parse import urlparse
+from urllib.parse import urlparse, urlunparse
 
 # external
 from dephell_markers import Markers
 from packaging.requirements import InvalidRequirement, Requirement
 
 # app
 from ...cached_property import cached_property
+from ...constants import WAREHOUSE_DOMAINS
 from ...networking import aiohttp_session
 from ..base import Interface
 
@@ -43,6 +44,38 @@ def repos(self):
     async def download(self, name: str, version: str, path: Path) -> bool:
         raise NotImplementedError
 
+    @staticmethod
+    def _get_url(url: str, default_path: str) -> str:
+        # replace link on pypi api by link on simple index
+        parsed = urlparse(url)
+        if not parsed.hostname:
+            parsed = urlparse('http://' + url)
+
+        if parsed.hostname == 'pypi.python.org':
+            hostname = 'pypi.org'
+        else:
+            hostname = parsed.netloc
+
+        if hostname in WAREHOUSE_DOMAINS:
+            path = default_path
+        else:
+            path = parsed.path
+
+        scheme = parsed.scheme
+        if hostname in WAREHOUSE_DOMAINS:
+            scheme = 'https'
+        if not scheme:
+            scheme = 'http'
+
+        return urlunparse((
+            scheme,
+            hostname,
+            path,
+            parsed.params,
+            parsed.query,
+            parsed.fragment,
+        ))
+
     @staticmethod
     def _convert_deps(*, deps, name, version, extra):
 

dephell/repositories/_warehouse/_simple.py

@@ -32,6 +32,7 @@
 class WarehouseSimpleRepo(WarehouseBaseRepo):
     name = attr.ib(type=str)
     url = attr.ib(type=str)
+    pretty_url = attr.ib(type=str, default='')
     auth = attr.ib(default=None)
 
     prereleases = attr.ib(type=bool, factory=lambda: config['prereleases'])  # allow prereleases
@@ -42,22 +43,9 @@ def __attrs_post_init__(self):
         # make name canonical
         if self.name in ('pypi.org', 'pypi.python.org'):
             self.name = 'pypi'
-
-        # replace link on pypi api by link on simple index
-        parsed = urlparse(self.url)
-        if parsed.hostname == 'pypi.python.org':
-            hostname = 'pypi.org'
-        else:
-            hostname = parsed.hostname
-        if hostname in ('pypi.org', 'test.pypi.org'):
-            path = '/simple/'
-        else:
-            path = parsed.path
-        self.url = parsed.scheme + '://' + hostname + path
-
-    @property
-    def pretty_url(self) -> str:
-        return self.url
+        if not self.pretty_url:
+            self.pretty_url = self.url
+        self.url = self._get_url(self.url, default_path='/simple/')
 
     def get_releases(self, dep) -> tuple:
         links = self._get_links(name=dep.base_name)

dephell/repositories/base.py

@@ -20,7 +20,11 @@ async def get_dependencies(self, name: str, version: str, extra: Optional[str] =
 
     @property
     def pretty_url(self):
-        return self.url
+        return self.__dict__.get('pretty_url', self.url)
+
+    @pretty_url.setter
+    def pretty_url(self, url):
+        self.__dict__['pretty_url'] = url
 
     def search(self, query: Iterable[str]) -> List[Dict[str, str]]:
         raise NotImplementedError('search is unsupported by this repo')

tests/test_repositories/test_warehouse_simple.py

@@ -148,3 +148,24 @@ def test_download(requests_mock, asyncio_mock, temp_cache, fixtures_path: Path,
     assert result is True
     assert (temp_path / file_name).exists()
     assert (temp_path / file_name).read_bytes() == file_content
+
+
+@pytest.mark.parametrize('user_input, expected', [
+    # test pypi
+    ('pypi.org', 'https://pypi.org/simple/'),
+    ('pypi.python.org', 'https://pypi.org/simple/'),
+    ('https://pypi.org', 'https://pypi.org/simple/'),
+    ('https://pypi.python.org', 'https://pypi.org/simple/'),
+    ('https://pypi.org/simple/', 'https://pypi.org/simple/'),
+
+    # test custom
+    ('https://not-pypi.org/not/simple/', 'https://not-pypi.org/not/simple/'),
+    ('not-pypi.org/not/simple/', 'http://not-pypi.org/not/simple/'),
+    ('not-pypi.org', 'http://not-pypi.org'),
+    ('http://pypi.example.com:8000/simple/', 'http://pypi.example.com:8000/simple/'),
+    ('pypi.example.com:8000/simple/', 'http://pypi.example.com:8000/simple/'),
+    ('localhost:8001', 'http://localhost:8001'),
+])
+def test_get_url(user_input, expected):
+    repo = WarehouseSimpleRepo(name='pypi', url=user_input)
+    assert repo.url == expected