Merge pull request #243 from hugovk/add-more-user-agents

Add User-Agent to requests

dephell/actions/_downloads.py

@@ -6,18 +6,16 @@
 
 # external
 import attr
-import requests
 
 # app
-from ..constants import USER_AGENT
+from ..networking import requests_session
 
 
 RECENT_URL = 'https://pypistats.org/api/packages/{}/recent'
 CATEGORIES_URLS = dict(
     pythons='https://pypistats.org/api/packages/{}/python_minor',
     systems='https://pypistats.org/api/packages/{}/system',
 )
-HEADERS = {'User-Agent': USER_AGENT}
 
 
 @attr.s()
@@ -53,7 +51,8 @@ def make_chart(values: Iterable[int], group: int = None, ticks: str = '_▁▂
 
 def get_total_downloads(name: str) -> Dict[str, int]:
     url = RECENT_URL.format(name)
-    response = requests.get(url, headers=HEADERS)
+    with requests_session() as session:
+        response = session.get(url)
     response.raise_for_status()
     body = response.json()['data']
     return dict(
@@ -65,7 +64,8 @@ def get_total_downloads(name: str) -> Dict[str, int]:
 
 def get_downloads_by_category(*, category: str, name: str) -> List[Dict[str, int]]:
     url = CATEGORIES_URLS[category].format(name)
-    response = requests.get(url, headers=HEADERS)
+    with requests_session() as session:
+        response = session.get(url)
     response.raise_for_status()
     body = response.json()['data']
 

dephell/cacher.py

@@ -11,13 +11,13 @@
 from itertools import islice
 
 # external
-import requests
 from packaging.requirements import Requirement
 
 # project
 from dephell.constants import DEFAULT_WAREHOUSE
 from dephell.controllers import DependencyMaker
 from dephell.models import RootDependency
+from dephell.networking import requests_session
 from dephell.repositories import WarehouseAPIRepo
 
 
@@ -27,7 +27,8 @@
 
 def get_deps():
     root = RootDependency()
-    response = requests.get(URL)
+    with requests_session() as session:
+        response = session.get(URL)
     for info in response.json()['rows']:
         yield from DependencyMaker.from_requirement(
             source=root,

dephell/constants.py

@@ -5,9 +5,6 @@
 from enum import Enum, unique
 from types import MappingProxyType
 
-# app
-from . import __version__
-
 
 @unique
 class ReturnCodes(Enum):
@@ -112,5 +109,3 @@ class ReturnCodes(Enum):
     'dephell_venvs',
     'dephell_versioning',
 )
-
-USER_AGENT = 'DepHell/{version}'.format(version=__version__)

dephell/controllers/_docker.py

@@ -7,11 +7,11 @@
 import attr
 import docker
 import dockerpty
-import requests
 from dephell_venvs import VEnvs
 
 # app
 from ..cached_property import cached_property
+from ..networking import requests_session
 
 
 DOCKER_PREFIX = 'dephell-'
@@ -66,7 +66,8 @@ def tags(self):
         url = 'https://hub.docker.com/v2/repositories/{}/tags/'.format(repository)
         tags = dict()
         while url is not None:
-            response = requests.get(url)
+            with requests_session() as session:
+                response = session.get(url)
             response.raise_for_status()
             content = response.json()
             url = content['next']

dephell/controllers/_safety.py

@@ -4,13 +4,13 @@
 
 # external
 import attr
-import requests
 from dephell_specifier import RangeSpecifier
 from packaging.version import Version
 
 # app
 from ..cache import JSONCache
 from ..cached_property import cached_property
+from ..networking import requests_session
 
 
 DUMP_URL = 'https://github.com/pyupio/safety-db/raw/master/data/insecure_full.json'
@@ -38,7 +38,8 @@ def vulns(self) -> Dict[str, Tuple[SafetyVulnInfo, ...]]:
         cache = JSONCache('pyup.io', ttl=3600 * 24)
         records = cache.load()
         if records is None:
-            response = requests.get(self.url)
+            with requests_session() as session:
+                response = session.get(self.url)
             response.raise_for_status()
             records = response.json()
             cache.dump(records)

dephell/controllers/_snyk.py

@@ -6,12 +6,12 @@
 
 # external
 import attr
-import requests
 from dephell_specifier import RangeSpecifier
 from packaging.version import VERSION_PATTERN, Version
 
 # app
 from ..cached_property import cached_property
+from ..networking import requests_session
 
 
 RSS_URL = 'https://snyk.io/vuln/feed.xml?type=pip'
@@ -44,7 +44,8 @@ class Snyk:
 
     @cached_property
     def vulns(self) -> Dict[str, List[SnykVulnInfo]]:
-        response = requests.get(self.url)
+        with requests_session() as session:
+            response = session.get(self.url)
         response.raise_for_status()
         root = ElementTree.fromstring(response.content)
 

dephell/converters/imports.py

@@ -4,14 +4,14 @@
 from typing import Dict, List, Set
 
 # external
-import requests
 from dephell_discover import Root as PackageRoot
 
 # app
 from ..cache import TextCache
 from ..cached_property import cached_property
 from ..controllers import DependencyMaker
 from ..models import RootDependency
+from ..networking import requests_session
 from .base import BaseConverter
 
 
@@ -97,7 +97,8 @@ def aliases(self) -> Dict[str, str]:
         cache = TextCache('imports', 'aliases', ttl=CACHE_TTL)
         lines = cache.load()
         if not lines:
-            response = requests.get(MAPPING_URLS[0])
+            with requests_session() as session:
+                response = session.get(MAPPING_URLS[0])
             lines = response.text.splitlines()
             cache.dump(lines)
 
@@ -116,7 +117,8 @@ def stdlib(self) -> List[str]:
         if lines:
             return lines
 
-        response = requests.get(STDLIB_URL)
+        with requests_session() as session:
+            response = session.get(STDLIB_URL)
         lines = response.text.splitlines()
         cache.dump(lines)
         return lines

dephell/networking.py

@@ -3,8 +3,15 @@
 
 # external
 import certifi
+import requests
 from aiohttp import ClientSession, TCPConnector
 
+# app
+from . import __version__
+
+
+USER_AGENT = 'DepHell/{version}'.format(version=__version__)
+
 
 def aiohttp_session(*, auth=None, **kwargs):
     headers = dict()
@@ -16,3 +23,16 @@ def aiohttp_session(*, auth=None, **kwargs):
     except TypeError:
         connector = TCPConnector(ssl_context=ssl_context)
     return ClientSession(headers=headers, connector=connector, **kwargs)
+
+
+def requests_session(*, auth=None, headers=None, **kwargs):
+    session = requests.Session()
+    if auth:
+        session.auth = auth
+    if headers is None:
+        headers = dict()
+    headers.setdefault('User-Agent', USER_AGENT)
+    session.headers = headers
+    if kwargs:
+        session.__dict__.update(kwargs)
+    return session

dephell/repositories/_conda/_cloud.py

@@ -10,7 +10,6 @@
 
 # external
 import attr
-import requests
 from dephell_specifier import RangeSpecifier
 from packaging.utils import canonicalize_name
 from packaging.version import parse
@@ -21,6 +20,7 @@
 from ...config import config
 from ...models.release import Release
 from ...models.simple_dependency import SimpleDependency
+from ...networking import requests_session
 from ._base import CondaBaseRepo
 
 
@@ -129,7 +129,8 @@ def search(self, query: Iterable[str]) -> List[Dict[str, str]]:
                     allowed=', '.join(self._allowed_values[field]),
                 ))
 
-        response = requests.get(self._search_url, params=fields)
+        with requests_session() as session:
+            response = session.get(self._search_url, params=fields)
         response.raise_for_status()
 
         results = []
@@ -203,7 +204,8 @@ def _packages(self) -> Dict[str, Dict[str, Any]]:
                 continue
 
             url = self._get_chan_url(channel=channel)
-            response = requests.get(url)
+            with requests_session() as session:
+                response = session.get(url)
             response.raise_for_status()
             channel_packages = dict()
             for name, info in response.json()['packages'].items():
@@ -246,7 +248,8 @@ def _releases(self) -> Dict[str, Dict[str, Dict[str, Any]]]:
 
             channel_deps = defaultdict(dict)
             for url in self._get_urls(channel=channel):
-                response = requests.get(url)
+                with requests_session() as session:
+                    response = session.get(url)
                 response.raise_for_status()
                 content = BZ2Decompressor().decompress(response.content).decode('utf-8')
                 base_url = url.rsplit('/', 1)[0]

dephell/repositories/_conda/_git.py

@@ -12,7 +12,6 @@
 
 # external
 import attr
-import requests
 from dephell_specifier import RangeSpecifier
 from jinja2 import Environment
 
@@ -22,7 +21,7 @@
 from ...config import config
 from ...models.release import Release
 from ...models.simple_dependency import SimpleDependency
-from ...networking import aiohttp_session
+from ...networking import aiohttp_session, requests_session
 from ...yaml import yaml_load
 from ._base import CondaBaseRepo
 
@@ -170,7 +169,8 @@ def _get_revs(self, name: str) -> List[Dict[str, str]]:
         revs = []
         for cookbook in cookbooks:
             url = HISTORY_URL.format(**cookbook)
-            response = requests.get(url)
+            with requests_session() as session:
+                response = session.get(url)
             if response.status_code != 200:
                 continue
             for commit in response.json():

dephell/repositories/_git/bitbucket.py

@@ -1,12 +1,10 @@
 # built-in
 import re
 
-# external
-import requests
-
 # app
 from ...cached_property import cached_property
 from ...config import config
+from ...networking import requests_session
 from .base import BaseRepo
 
 
@@ -30,7 +28,8 @@ def _get_tags(self):
             author=self.author,
             name=self.name,
         )
-        response = requests.get(url)
+        with requests_session() as session:
+            response = session.get(url)
 
         tags = []
         for tag in response.json()['values']:

dephell/repositories/_git/github.py

@@ -1,10 +1,8 @@
 # built-in
 import re
 
-# external
-import requests
-
 # app
+from ...networking import requests_session
 from ..cached_property import cached_property
 from .base import BaseRepo
 
@@ -27,7 +25,8 @@ def _get_tags(self):
             author=self.author,
             name=self.name,
         )
-        response = requests.get(url)
+        with requests_session() as session:
+            response = session.get(url)
 
         tags = []
         for release in response.json():

dephell/repositories/_git/gitlab.py

@@ -2,10 +2,8 @@
 import re
 from urllib.parse import urlencode
 
-# external
-import requests
-
 # app
+from ...networking import requests_session
 from ..cached_property import cached_property
 from .base import BaseRepo
 
@@ -28,7 +26,8 @@ def _get_tags(self):
         url = 'https://gitlab.com/api/v4/projects/{id}/repository/tags'.format(
             id=urlencode(self.author + '/' + self.name),
         )
-        response = requests.get(url)
+        with requests_session() as session:
+            response = session.get(url)
 
         tags = []
         for tag in response.json():

dephell/repositories/_warehouse/_api.py

@@ -9,7 +9,6 @@
 
 # external
 import attr
-import requests
 from dephell_licenses import License, licenses
 from packaging.requirements import Requirement
 
@@ -19,7 +18,7 @@
 from ...exceptions import InvalidFieldsError, PackageNotFoundError
 from ...models.author import Author
 from ...models.release import Release
-from ...networking import aiohttp_session
+from ...networking import aiohttp_session, requests_session
 from ._base import WarehouseBaseRepo
 
 
@@ -86,7 +85,8 @@ def get_releases(self, dep) -> tuple:
         data = cache.load()
         if data is None:
             url = '{url}{name}/json'.format(url=self.url, name=dep.base_name)
-            response = requests.get(url, auth=self.auth)
+            with requests_session() as session:
+                response = session.get(url, auth=self.auth)
             if response.status_code == 404:
                 raise PackageNotFoundError(package=dep.base_name, url=url)
             data = response.json()