From 61cc836a8c736d2f44cf68b3ecbc13c70e553655 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Tue, 14 Jul 2020 15:04:18 -0700 Subject: [PATCH 01/11] Factor out BaseClient to share between Client and AsyncClient. Also, don't compute user_agent on every request. --- geoip2/webservice.py | 149 ++++++++++++++++++++++++--------------- tests/webservice_test.py | 2 +- 2 files changed, 95 insertions(+), 56 deletions(-) diff --git a/geoip2/webservice.py b/geoip2/webservice.py index 54d49930..c3a32e96 100644 --- a/geoip2/webservice.py +++ b/geoip2/webservice.py @@ -47,58 +47,22 @@ from geoip2.types import IPAddress -class Client: - """Creates a new client object. - - It accepts the following required arguments: - - :param account_id: Your MaxMind account ID. - :param license_key: Your MaxMind license key. - - Go to https://www.maxmind.com/en/my_license_key to see your MaxMind - account ID and license key. - - The following keyword arguments are also accepted: - - :param host: The hostname to make a request against. This defaults to - "geoip.maxmind.com". In most cases, you should not need to set this - explicitly. - :param locales: This is list of locale codes. This argument will be - passed on to record classes to use when their name properties are - called. The default value is ['en']. - - The order of the locales is significant. When a record class has - multiple names (country, city, etc.), its name property will return - the name in the first locale that has one. - - Note that the only locale which is always present in the GeoIP2 - data is "en". If you do not include this locale, the name property - may end up returning None even when the record has an English name. - - Currently, the valid locale codes are: - - * de -- German - * en -- English names may still include accented characters if that is - the accepted spelling in English. In other words, English does not - mean ASCII. - * es -- Spanish - * fr -- French - * ja -- Japanese - * pt-BR -- Brazilian Portuguese - * ru -- Russian - * zh-CN -- Simplified Chinese. - :param timeout: The timeout to use when waiting on the request. This sets - both the connect timeout and the read timeout. - - """ +class BaseClient: # pylint: disable=missing-class-docstring + _account_id: str + _host: str + _license_key: str + _locales: List[str] + _timeout: Optional[float] + _user_agent: str def __init__( self, account_id: int, license_key: str, - host: str = "geoip.maxmind.com", - locales: Optional[List[str]] = None, - timeout: Optional[float] = None, + host: str, + locales: Optional[List[str]], + timeout: Optional[float], + http_user_agent: str, ) -> None: """Construct a Client.""" # pylint: disable=too-many-arguments @@ -114,6 +78,10 @@ def __init__( self._license_key = license_key self._base_uri = "https://%s/geoip/v2.1" % host self._timeout = timeout + self._user_agent = "GeoIP2-Python-Client/%s %s" % ( + geoip2.__version__, + http_user_agent, + ) def city(self, ip_address: IPAddress = "me") -> City: """Call GeoIP2 Precision City endpoint with the specified IP. @@ -167,7 +135,7 @@ def _response_for( response = requests.get( uri, auth=(self._account_id, self._license_key), - headers={"Accept": "application/json", "User-Agent": self._user_agent()}, + headers={"Accept": "application/json", "User-Agent": self._user_agent}, timeout=self._timeout, ) if response.status_code != 200: @@ -175,13 +143,6 @@ def _response_for( body = self._handle_success(response, uri) return model_class(body, locales=self._locales) - @staticmethod - def _user_agent() -> str: - return "GeoIP2 Python Client v%s (%s)" % ( - geoip2.__version__, - default_user_agent(), - ) - @staticmethod def _handle_success(response: Response, uri: str) -> Any: try: @@ -284,3 +245,81 @@ def _exception_for_non_200_status(status: int, uri: str) -> HTTPError: status, uri, ) + + +class Client(BaseClient): + """A synchronous GeoIP2 client. + + It accepts the following required arguments: + + :param account_id: Your MaxMind account ID. + :param license_key: Your MaxMind license key. + + Go to https://www.maxmind.com/en/my_license_key to see your MaxMind + account ID and license key. + + The following keyword arguments are also accepted: + + :param host: The hostname to make a request against. This defaults to + "geoip.maxmind.com". In most cases, you should not need to set this + explicitly. + :param locales: This is list of locale codes. This argument will be + passed on to record classes to use when their name properties are + called. The default value is ['en']. + + The order of the locales is significant. When a record class has + multiple names (country, city, etc.), its name property will return + the name in the first locale that has one. + + Note that the only locale which is always present in the GeoIP2 + data is "en". If you do not include this locale, the name property + may end up returning None even when the record has an English name. + + Currently, the valid locale codes are: + + * de -- German + * en -- English names may still include accented characters if that is + the accepted spelling in English. In other words, English does not + mean ASCII. + * es -- Spanish + * fr -- French + * ja -- Japanese + * pt-BR -- Brazilian Portuguese + * ru -- Russian + * zh-CN -- Simplified Chinese. + :param timeout: The timeout to use when waiting on the request. This sets + both the connect timeout and the read timeout. + + """ + + def __init__( # pylint: disable=too-many-arguments + self, + account_id: int, + license_key: str, + host: str = "geoip.maxmind.com", + locales: Optional[List[str]] = None, + timeout: Optional[float] = None, + ) -> None: + super().__init__( + account_id, license_key, host, locales, timeout, default_user_agent() + ) + + def _response_for( + self, + path: str, + model_class: Union[Type[Insights], Type[City], Type[Country]], + ip_address: IPAddress, + ) -> Union[Country, City, Insights]: + if ip_address != "me": + ip_address = ipaddress.ip_address(ip_address) + uri = "/".join([self._base_uri, path, str(ip_address)]) + response = requests.get( + uri, + auth=(self._account_id, self._license_key), + headers={"Accept": "application/json", "User-Agent": self._user_agent}, + timeout=self._timeout, + ) + if response.status_code != 200: + raise self._exception_for_error(response, uri) + body = self._handle_success(response, uri) + return model_class(body, locales=self._locales) diff --git a/tests/webservice_test.py b/tests/webservice_test.py index d2600af0..3e73de6e 100644 --- a/tests/webservice_test.py +++ b/tests/webservice_test.py @@ -288,7 +288,7 @@ def test_request(self, mock): ) self.assertRegex( request.headers["User-Agent"], - "^GeoIP2 Python Client v", + "^GeoIP2-Python-Client/", "Correct User-Agent", ) self.assertEqual( From 4aa80e452126c92a23b8a2c4d94b5cf9ef79acee Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Tue, 14 Jul 2020 15:41:51 -0700 Subject: [PATCH 02/11] Update setup.py for Python 3.6+ requirement --- setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.py b/setup.py index a3efc054..7b802c7b 100644 --- a/setup.py +++ b/setup.py @@ -24,9 +24,8 @@ package_data={"": ["LICENSE"]}, package_dir={"geoip2": "geoip2"}, include_package_data=True, - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", + python_requires=">=3.6", install_requires=requirements, - extras_require={':python_version=="2.7"': ["ipaddress"]}, tests_require=["requests_mock>=0.5"], test_suite="tests", license=geoip2.__license__, @@ -36,9 +35,7 @@ "Intended Audience :: Developers", "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", From 5bfc7c5004f4e3faabbca0ef7ec803d83528ae2c Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Tue, 14 Jul 2020 16:00:07 -0700 Subject: [PATCH 03/11] Switch _back_ to httpretty This basically reverts #24. httpretty seems well maintained these days and it makes it easy to switch out the HTTP client libraries without changing the tests. --- .travis.yml | 2 +- setup.py | 2 +- tests/webservice_test.py | 208 +++++++++++++++++++-------------------- 3 files changed, 103 insertions(+), 109 deletions(-) diff --git a/.travis.yml b/.travis.yml index ac9b5a7c..2c4db8b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: - "if [[ $RUN_SNYK && $SNYK_TOKEN ]]; then sudo apt-get install -y nodejs; npm install -g snyk; fi" install: - pip install -r requirements.txt - - pip install requests_mock coveralls + - pip install httpretty coveralls - | if [[ $RUN_LINTER ]]; then pip install --upgrade pylint black mypy diff --git a/setup.py b/setup.py index 7b802c7b..6ba33ea6 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ include_package_data=True, python_requires=">=3.6", install_requires=requirements, - tests_require=["requests_mock>=0.5"], + tests_require=["httpretty>=1.0.0"], test_suite="tests", license=geoip2.__license__, classifiers=[ diff --git a/tests/webservice_test.py b/tests/webservice_test.py index 3e73de6e..d5109bdc 100644 --- a/tests/webservice_test.py +++ b/tests/webservice_test.py @@ -3,14 +3,15 @@ import copy import ipaddress +import json import sys from typing import cast, Dict import unittest sys.path.append("..") +import httpretty # type: ignore import geoip2 -import requests_mock # type: ignore from geoip2.errors import ( AddressNotFoundError, AuthenticationError, @@ -23,6 +24,7 @@ from geoip2.webservice import Client +@httpretty.activate class TestClient(unittest.TestCase): def setUp(self): self.client = Client(42, "abcdef123456") @@ -58,13 +60,13 @@ def _content_type(self, endpoint): + "+json; charset=UTF-8; version=1.0" ) - @requests_mock.mock() - def test_country_ok(self, mock): - mock.get( + def test_country_ok(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/1.2.3.4", - json=self.country, - status_code=200, - headers={"Content-Type": self._content_type("country")}, + body=json.dumps(self.country), + status=200, + content_type=self._content_type("country"), ) country = self.client.country("1.2.3.4") self.assertEqual( @@ -103,13 +105,13 @@ def test_country_ok(self, mock): ) self.assertEqual(country.raw, self.country, "raw response is correct") - @requests_mock.mock() - def test_me(self, mock): - mock.get( + def test_me(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/me", - json=self.country, - status_code=200, - headers={"Content-Type": self._content_type("country")}, + body=json.dumps(self.country), + status=200, + content_type=self._content_type("country"), ) implicit_me = self.client.country() self.assertEqual( @@ -122,12 +124,13 @@ def test_me(self, mock): "country('me') returns Country object", ) - @requests_mock.mock() - def test_200_error(self, mock): - mock.get( + def test_200_error(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/1.1.1.1", - status_code=200, - headers={"Content-Type": self._content_type("country")}, + body="", + status=200, + content_type=self._content_type("country"), ) with self.assertRaisesRegex( GeoIP2Error, "could not decode the response as JSON" @@ -140,26 +143,26 @@ def test_bad_ip_address(self): ): self.client.country("1.2.3") - @requests_mock.mock() - def test_no_body_error(self, mock): - mock.get( + def test_no_body_error(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/" + "1.2.3.7", - text="", - status_code=400, - headers={"Content-Type": self._content_type("country")}, + body="", + status=400, + content_type=self._content_type("country"), ) with self.assertRaisesRegex( HTTPError, "Received a 400 error for .* with no body" ): self.client.country("1.2.3.7") - @requests_mock.mock() - def test_weird_body_error(self, mock): - mock.get( + def test_weird_body_error(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/" + "1.2.3.8", - text='{"wierd": 42}', - status_code=400, - headers={"Content-Type": self._content_type("country")}, + body='{"wierd": 42}', + status=400, + content_type=self._content_type("country"), ) with self.assertRaisesRegex( HTTPError, @@ -167,31 +170,32 @@ def test_weird_body_error(self, mock): ): self.client.country("1.2.3.8") - @requests_mock.mock() - def test_bad_body_error(self, mock): - mock.get( + def test_bad_body_error(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/" + "1.2.3.9", - text="bad body", - status_code=400, - headers={"Content-Type": self._content_type("country")}, + body="bad body", + status=400, + content_type=self._content_type("country"), ) with self.assertRaisesRegex( HTTPError, "it did not include the expected JSON body" ): self.client.country("1.2.3.9") - @requests_mock.mock() - def test_500_error(self, mock): - mock.get(self.base_uri + "country/" + "1.2.3.10", status_code=500) + def test_500_error(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/" + "1.2.3.10", status=500 + ) with self.assertRaisesRegex(HTTPError, r"Received a server error \(500\) for"): self.client.country("1.2.3.10") - @requests_mock.mock() - def test_300_error(self, mock): - mock.get( + def test_300_error(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/" + "1.2.3.11", - status_code=300, - headers={"Content-Type": self._content_type("country")}, + status=300, + content_type=self._content_type("country"), ) with self.assertRaisesRegex( HTTPError, r"Received a very surprising HTTP status \(300\) for" @@ -199,86 +203,76 @@ def test_300_error(self, mock): self.client.country("1.2.3.11") - @requests_mock.mock() - def test_ip_address_required(self, mock): - self._test_error(mock, 400, "IP_ADDRESS_REQUIRED", InvalidRequestError) + def test_ip_address_required(self): + self._test_error(400, "IP_ADDRESS_REQUIRED", InvalidRequestError) - @requests_mock.mock() - def test_ip_address_not_found(self, mock): - self._test_error(mock, 404, "IP_ADDRESS_NOT_FOUND", AddressNotFoundError) + def test_ip_address_not_found(self): + self._test_error(404, "IP_ADDRESS_NOT_FOUND", AddressNotFoundError) - @requests_mock.mock() - def test_ip_address_reserved(self, mock): - self._test_error(mock, 400, "IP_ADDRESS_RESERVED", AddressNotFoundError) + def test_ip_address_reserved(self): + self._test_error(400, "IP_ADDRESS_RESERVED", AddressNotFoundError) - @requests_mock.mock() - def test_permission_required(self, mock): - self._test_error(mock, 403, "PERMISSION_REQUIRED", PermissionRequiredError) + def test_permission_required(self): + self._test_error(403, "PERMISSION_REQUIRED", PermissionRequiredError) - @requests_mock.mock() - def test_auth_invalid(self, mock): - self._test_error(mock, 400, "AUTHORIZATION_INVALID", AuthenticationError) + def test_auth_invalid(self): + self._test_error(400, "AUTHORIZATION_INVALID", AuthenticationError) - @requests_mock.mock() - def test_license_key_required(self, mock): - self._test_error(mock, 401, "LICENSE_KEY_REQUIRED", AuthenticationError) + def test_license_key_required(self): + self._test_error(401, "LICENSE_KEY_REQUIRED", AuthenticationError) - @requests_mock.mock() - def test_account_id_required(self, mock): - self._test_error(mock, 401, "ACCOUNT_ID_REQUIRED", AuthenticationError) + def test_account_id_required(self): + self._test_error(401, "ACCOUNT_ID_REQUIRED", AuthenticationError) - @requests_mock.mock() - def test_user_id_required(self, mock): - self._test_error(mock, 401, "USER_ID_REQUIRED", AuthenticationError) + def test_user_id_required(self): + self._test_error(401, "USER_ID_REQUIRED", AuthenticationError) - @requests_mock.mock() - def test_account_id_unkown(self, mock): - self._test_error(mock, 401, "ACCOUNT_ID_UNKNOWN", AuthenticationError) + def test_account_id_unkown(self): + self._test_error(401, "ACCOUNT_ID_UNKNOWN", AuthenticationError) - @requests_mock.mock() - def test_user_id_unkown(self, mock): - self._test_error(mock, 401, "USER_ID_UNKNOWN", AuthenticationError) + def test_user_id_unkown(self): + self._test_error(401, "USER_ID_UNKNOWN", AuthenticationError) - @requests_mock.mock() - def test_out_of_queries_error(self, mock): - self._test_error(mock, 402, "OUT_OF_QUERIES", OutOfQueriesError) + def test_out_of_queries_error(self): + self._test_error(402, "OUT_OF_QUERIES", OutOfQueriesError) - def _test_error(self, mock, status, error_code, error_class): + def _test_error(self, status, error_code, error_class): msg = "Some error message" body = {"error": msg, "code": error_code} - mock.get( + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/1.2.3.18", - json=body, - status_code=status, - headers={"Content-Type": self._content_type("country")}, + body=json.dumps(body), + status=status, + content_type=self._content_type("country"), ) with self.assertRaisesRegex(error_class, msg): self.client.country("1.2.3.18") - @requests_mock.mock() - def test_unknown_error(self, mock): + def test_unknown_error(self): msg = "Unknown error type" ip = "1.2.3.19" body = {"error": msg, "code": "UNKNOWN_TYPE"} - mock.get( + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/" + ip, - json=body, - status_code=400, - headers={"Content-Type": self._content_type("country")}, + body=json.dumps(body), + status=400, + content_type=self._content_type("country"), ) with self.assertRaisesRegex(InvalidRequestError, msg): self.client.country(ip) - @requests_mock.mock() - def test_request(self, mock): - mock.get( + def test_request(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "country/" + "1.2.3.4", - json=self.country, - status_code=200, - headers={"Content-Type": self._content_type("country")}, + body=json.dumps(self.country), + status=200, + content_type=self._content_type("country"), ) self.client.country("1.2.3.4") - request = mock.request_history[-1] + request = httpretty.latest_requests()[-1] self.assertEqual( request.path, "/geoip/v2.1/country/1.2.3.4", "correct URI is used" @@ -297,13 +291,13 @@ def test_request(self, mock): "correct auth", ) - @requests_mock.mock() - def test_city_ok(self, mock): - mock.get( + def test_city_ok(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "city/" + "1.2.3.4", - json=self.country, - status_code=200, - headers={"Content-Type": self._content_type("city")}, + body=json.dumps(self.country), + status=200, + content_type=self._content_type("city"), ) city = self.client.city("1.2.3.4") self.assertEqual(type(city), geoip2.models.City, "return value of client.city") @@ -311,13 +305,13 @@ def test_city_ok(self, mock): city.traits.network, ipaddress.ip_network("1.2.3.0/24"), "network" ) - @requests_mock.mock() - def test_insights_ok(self, mock): - mock.get( + def test_insights_ok(self): + httpretty.register_uri( + httpretty.GET, self.base_uri + "insights/1.2.3.4", - json=self.insights, - status_code=200, - headers={"Content-Type": self._content_type("country")}, + body=json.dumps(self.insights), + status=200, + content_type=self._content_type("country"), ) insights = self.client.insights("1.2.3.4") self.assertEqual( From 238ba54a53d85610c4805679c15d2b1051ca222a Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 15 Jul 2020 09:58:03 -0700 Subject: [PATCH 04/11] Add asyncio support to for web service requests --- HISTORY.rst | 4 + geoip2/webservice.py | 300 +++++++++++++++++++++++++++++++------------ requirements.txt | 3 +- 3 files changed, 227 insertions(+), 80 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7ca4216a..19391184 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,10 @@ History * IMPORTANT: Python 2.7 and 3.5 support has been dropped. Python 3.6 or greater is required. +* Asyncio support has been added for web service requests. To make async + requests, use ``geoip.webservice.AsyncClient``. +* ``geoip.webservice.Client`` now provides a ``close()`` method and associated + context managers to be used in ``with`` statements. * Type hints have been added. * The attributes ``postal_code`` and ``postal_confidence`` have been removed from ``geoip2.record.Location``. These would previously always be ``None``. diff --git a/geoip2/webservice.py b/geoip2/webservice.py index c3a32e96..a4c40c61 100644 --- a/geoip2/webservice.py +++ b/geoip2/webservice.py @@ -26,10 +26,11 @@ """ import ipaddress +import json from typing import Any, cast, List, Optional, Type, Union +import aiohttp import requests -from requests.models import Response from requests.utils import default_user_agent import geoip2 @@ -47,7 +48,7 @@ from geoip2.types import IPAddress -class BaseClient: # pylint: disable=missing-class-docstring +class BaseClient: # pylint: disable=missing-class-docstring, too-few-public-methods _account_id: str _host: str _license_key: str @@ -83,70 +84,15 @@ def __init__( http_user_agent, ) - def city(self, ip_address: IPAddress = "me") -> City: - """Call GeoIP2 Precision City endpoint with the specified IP. - - :param ip_address: IPv4 or IPv6 address as a string. If no - address is provided, the address that the web service is - called from will be used. - - :returns: :py:class:`geoip2.models.City` object - - """ - return cast(City, self._response_for("city", geoip2.models.City, ip_address)) - - def country(self, ip_address: IPAddress = "me") -> Country: - """Call the GeoIP2 Country endpoint with the specified IP. - - :param ip_address: IPv4 or IPv6 address as a string. If no address - is provided, the address that the web service is called from will - be used. - - :returns: :py:class:`geoip2.models.Country` object - - """ - return cast( - Country, self._response_for("country", geoip2.models.Country, ip_address) - ) - - def insights(self, ip_address: IPAddress = "me") -> Insights: - """Call the GeoIP2 Precision: Insights endpoint with the specified IP. - - :param ip_address: IPv4 or IPv6 address as a string. If no address - is provided, the address that the web service is called from will - be used. - - :returns: :py:class:`geoip2.models.Insights` object - - """ - return cast( - Insights, self._response_for("insights", geoip2.models.Insights, ip_address) - ) - - def _response_for( - self, - path: str, - model_class: Union[Type[Insights], Type[City], Type[Country]], - ip_address: IPAddress, - ) -> Union[Country, City, Insights]: + def _uri(self, path: str, ip_address: IPAddress) -> str: if ip_address != "me": ip_address = ipaddress.ip_address(ip_address) - uri = "/".join([self._base_uri, path, str(ip_address)]) - response = requests.get( - uri, - auth=(self._account_id, self._license_key), - headers={"Accept": "application/json", "User-Agent": self._user_agent}, - timeout=self._timeout, - ) - if response.status_code != 200: - raise self._exception_for_error(response, uri) - body = self._handle_success(response, uri) - return model_class(body, locales=self._locales) + return "/".join([self._base_uri, path, str(ip_address)]) @staticmethod - def _handle_success(response: Response, uri: str) -> Any: + def _handle_success(body: str, uri: str) -> Any: try: - return response.json() + return json.loads(body) except ValueError as ex: raise GeoIP2Error( "Received a 200 response for %(uri)s" @@ -156,33 +102,33 @@ def _handle_success(response: Response, uri: str) -> Any: uri, ) - def _exception_for_error(self, response: Response, uri: str) -> GeoIP2Error: - status = response.status_code - + def _exception_for_error( + self, status: int, content_type: str, body: str, uri: str + ) -> GeoIP2Error: if 400 <= status < 500: - return self._exception_for_4xx_status(response, status, uri) + return self._exception_for_4xx_status(status, content_type, body, uri) if 500 <= status < 600: return self._exception_for_5xx_status(status, uri) return self._exception_for_non_200_status(status, uri) def _exception_for_4xx_status( - self, response: Response, status: int, uri: str + self, status: int, content_type: str, body: str, uri: str ) -> GeoIP2Error: - if not response.content: + if not body: return HTTPError( "Received a %(status)i error for %(uri)s " "with no body." % locals(), status, uri, ) - if response.headers["Content-Type"].find("json") == -1: + if content_type.find("json") == -1: return HTTPError( "Received a %i for %s with the following " - "body: %s" % (status, uri, str(response.content)), + "body: %s" % (status, uri, str(content_type)), status, uri, ) try: - body = response.json() + decoded_body = json.loads(body) except ValueError as ex: return HTTPError( "Received a %(status)i error for %(uri)s but it did" @@ -193,7 +139,7 @@ def _exception_for_4xx_status( else: if "code" in body and "error" in body: return self._exception_for_web_service_error( - body.get("error"), body.get("code"), status, uri + decoded_body.get("error"), decoded_body.get("code"), status, uri ) return HTTPError( "Response contains JSON but it does not specify " "code or error keys", @@ -247,6 +193,145 @@ def _exception_for_non_200_status(status: int, uri: str) -> HTTPError: ) +class AsyncClient(BaseClient): + """An async GeoIP2 client. + + It accepts the following required arguments: + + :param account_id: Your MaxMind account ID. + :param license_key: Your MaxMind license key. + + Go to https://www.maxmind.com/en/my_license_key to see your MaxMind + account ID and license key. + + The following keyword arguments are also accepted: + + :param host: The hostname to make a request against. This defaults to + "geoip.maxmind.com". In most cases, you should not need to set this + explicitly. + :param locales: This is list of locale codes. This argument will be + passed on to record classes to use when their name properties are + called. The default value is ['en']. + + The order of the locales is significant. When a record class has + multiple names (country, city, etc.), its name property will return + the name in the first locale that has one. + + Note that the only locale which is always present in the GeoIP2 + data is "en". If you do not include this locale, the name property + may end up returning None even when the record has an English name. + + Currently, the valid locale codes are: + + * de -- German + * en -- English names may still include accented characters if that is + the accepted spelling in English. In other words, English does not + mean ASCII. + * es -- Spanish + * fr -- French + * ja -- Japanese + * pt-BR -- Brazilian Portuguese + * ru -- Russian + * zh-CN -- Simplified Chinese. + :param timeout: The timeout to use when waiting on the request. This sets + both the connect timeout and the read timeout. + + """ + + _session: aiohttp.ClientSession + + def __init__( # pylint: disable=too-many-arguments + self, + account_id: int, + license_key: str, + host: str = "geoip.maxmind.com", + locales: Optional[List[str]] = None, + timeout: Optional[float] = None, + ) -> None: + super().__init__( + account_id, license_key, host, locales, timeout, default_user_agent() + ) + self._session = aiohttp.ClientSession() + + async def city(self, ip_address: IPAddress = "me") -> City: + """Call GeoIP2 Precision City endpoint with the specified IP. + + :param ip_address: IPv4 or IPv6 address as a string. If no + address is provided, the address that the web service is + called from will be used. + + :returns: :py:class:`geoip2.models.City` object + + """ + return cast( + City, await self._response_for("city", geoip2.models.City, ip_address) + ) + + async def country(self, ip_address: IPAddress = "me") -> Country: + """Call the GeoIP2 Country endpoint with the specified IP. + + :param ip_address: IPv4 or IPv6 address as a string. If no address + is provided, the address that the web service is called from will + be used. + + :returns: :py:class:`geoip2.models.Country` object + + """ + return cast( + Country, + await self._response_for("country", geoip2.models.Country, ip_address), + ) + + async def insights(self, ip_address: IPAddress = "me") -> Insights: + """Call the GeoIP2 Precision: Insights endpoint with the specified IP. + + :param ip_address: IPv4 or IPv6 address as a string. If no address + is provided, the address that the web service is called from will + be used. + + :returns: :py:class:`geoip2.models.Insights` object + + """ + return cast( + Insights, + await self._response_for("insights", geoip2.models.Insights, ip_address), + ) + + async def _response_for( + self, + path: str, + model_class: Union[Type[Insights], Type[City], Type[Country]], + ip_address: IPAddress, + ) -> Union[Country, City, Insights]: + uri = self._uri(path, ip_address) + async with await self._session.get( + uri, + auth=aiohttp.BasicAuth(self._account_id, self._license_key), + headers={"Accept": "application/json", "User-Agent": self._user_agent}, + timeout=self._timeout, + ) as response: + status = response.status + content_type = response.content_type + body = await response.text() + if status != 200: + raise self._exception_for_error(status, content_type, body, uri) + decoded_body = self._handle_success(body, uri) + return model_class(decoded_body, locales=self._locales) + + async def close(self): + """Close underlying session + + This will close the session and any associated connections. + """ + await self._session.close() + + async def __aenter__(self) -> "AsyncClient": + return self + + async def __aexit__(self, exc_type: None, exc_value: None, traceback: None) -> None: + await self.close() + + class Client(BaseClient): """A synchronous GeoIP2 client. @@ -292,6 +377,8 @@ class Client(BaseClient): """ + _session: requests.Session + def __init__( # pylint: disable=too-many-arguments self, account_id: int, @@ -303,6 +390,47 @@ def __init__( # pylint: disable=too-many-arguments super().__init__( account_id, license_key, host, locales, timeout, default_user_agent() ) + self._session = requests.Session() + + def city(self, ip_address: IPAddress = "me") -> City: + """Call GeoIP2 Precision City endpoint with the specified IP. + + :param ip_address: IPv4 or IPv6 address as a string. If no + address is provided, the address that the web service is + called from will be used. + + :returns: :py:class:`geoip2.models.City` object + + """ + return cast(City, self._response_for("city", geoip2.models.City, ip_address)) + + def country(self, ip_address: IPAddress = "me") -> Country: + """Call the GeoIP2 Country endpoint with the specified IP. + + :param ip_address: IPv4 or IPv6 address as a string. If no address + is provided, the address that the web service is called from will + be used. + + :returns: :py:class:`geoip2.models.Country` object + + """ + return cast( + Country, self._response_for("country", geoip2.models.Country, ip_address) + ) + + def insights(self, ip_address: IPAddress = "me") -> Insights: + """Call the GeoIP2 Precision: Insights endpoint with the specified IP. + + :param ip_address: IPv4 or IPv6 address as a string. If no address + is provided, the address that the web service is called from will + be used. + + :returns: :py:class:`geoip2.models.Insights` object + + """ + return cast( + Insights, self._response_for("insights", geoip2.models.Insights, ip_address) + ) def _response_for( self, @@ -310,16 +438,30 @@ def _response_for( model_class: Union[Type[Insights], Type[City], Type[Country]], ip_address: IPAddress, ) -> Union[Country, City, Insights]: - if ip_address != "me": - ip_address = ipaddress.ip_address(ip_address) - uri = "/".join([self._base_uri, path, str(ip_address)]) - response = requests.get( + uri = self._uri(path, ip_address) + response = self._session.get( uri, auth=(self._account_id, self._license_key), headers={"Accept": "application/json", "User-Agent": self._user_agent}, timeout=self._timeout, ) - if response.status_code != 200: - raise self._exception_for_error(response, uri) - body = self._handle_success(response, uri) - return model_class(body, locales=self._locales) + status = response.status_code + content_type = response.headers["Content-Type"] + body = response.text + if status != 200: + raise self._exception_for_error(status, content_type, body, uri) + decoded_body = self._handle_success(body, uri) + return model_class(decoded_body, locales=self._locales) + + def close(self): + """Close underlying session + + This will close the session and any associated connections. + """ + self._session.close() + + def __enter__(self) -> "Client": + return self + + def __exit__(self, exc_type: None, exc_value: None, traceback: None) -> None: + self.close() diff --git a/requirements.txt b/requirements.txt index 1c2b18b1..fe360ec0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +aiohttp>=3.6.2 maxminddb>=1.5.2 -requests>=2.22.0 +requests>=2.24.0 urllib3>=1.25.2 From 9fdb5e736b16da20a797a23c4e7bf9210a73f40e Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 15 Jul 2020 10:41:16 -0700 Subject: [PATCH 05/11] Run tests on Client and AsyncClient --- geoip2/webservice.py | 14 ++++-- tests/webservice_test.py | 102 +++++++++++++++++++++++++++++---------- 2 files changed, 86 insertions(+), 30 deletions(-) diff --git a/geoip2/webservice.py b/geoip2/webservice.py index a4c40c61..6c60c414 100644 --- a/geoip2/webservice.py +++ b/geoip2/webservice.py @@ -238,7 +238,7 @@ class AsyncClient(BaseClient): """ - _session: aiohttp.ClientSession + _existing_session: aiohttp.ClientSession def __init__( # pylint: disable=too-many-arguments self, @@ -251,7 +251,6 @@ def __init__( # pylint: disable=too-many-arguments super().__init__( account_id, license_key, host, locales, timeout, default_user_agent() ) - self._session = aiohttp.ClientSession() async def city(self, ip_address: IPAddress = "me") -> City: """Call GeoIP2 Precision City endpoint with the specified IP. @@ -297,6 +296,11 @@ async def insights(self, ip_address: IPAddress = "me") -> Insights: await self._response_for("insights", geoip2.models.Insights, ip_address), ) + async def _session(self) -> aiohttp.ClientSession: + if not hasattr(self, "_existing_session"): + self._existing_session = aiohttp.ClientSession() + return self._existing_session + async def _response_for( self, path: str, @@ -304,7 +308,8 @@ async def _response_for( ip_address: IPAddress, ) -> Union[Country, City, Insights]: uri = self._uri(path, ip_address) - async with await self._session.get( + session = await self._session() + async with await session.get( uri, auth=aiohttp.BasicAuth(self._account_id, self._license_key), headers={"Accept": "application/json", "User-Agent": self._user_agent}, @@ -323,7 +328,8 @@ async def close(self): This will close the session and any associated connections. """ - await self._session.close() + if hasattr(self, "_existing_session"): + await self._existing_session.close() async def __aenter__(self) -> "AsyncClient": return self diff --git a/tests/webservice_test.py b/tests/webservice_test.py index d5109bdc..8c716fb0 100644 --- a/tests/webservice_test.py +++ b/tests/webservice_test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import asyncio import copy import ipaddress import json @@ -10,7 +11,10 @@ sys.path.append("..") -import httpretty # type: ignore +# httpretty currently doesn't work, but mocket with the compat interface +# does. +from mocket import Mocket # type: ignore +from mocket.plugins.httpretty import HTTPretty as httpretty, httprettified # type: ignore import geoip2 from geoip2.errors import ( AddressNotFoundError, @@ -21,14 +25,10 @@ OutOfQueriesError, PermissionRequiredError, ) -from geoip2.webservice import Client +from geoip2.webservice import AsyncClient, Client -@httpretty.activate -class TestClient(unittest.TestCase): - def setUp(self): - self.client = Client(42, "abcdef123456") - +class TestBaseClient(unittest.TestCase): base_uri = "https://geoip.maxmind.com/geoip/v2.1/" country = { "continent": {"code": "NA", "geoname_id": 42, "names": {"en": "North America"}}, @@ -60,6 +60,7 @@ def _content_type(self, endpoint): + "+json; charset=UTF-8; version=1.0" ) + @httprettified def test_country_ok(self): httpretty.register_uri( httpretty.GET, @@ -68,7 +69,7 @@ def test_country_ok(self): status=200, content_type=self._content_type("country"), ) - country = self.client.country("1.2.3.4") + country = self.run_client(self.client.country("1.2.3.4")) self.assertEqual( type(country), geoip2.models.Country, "return value of client.country" ) @@ -105,6 +106,7 @@ def test_country_ok(self): ) self.assertEqual(country.raw, self.country, "raw response is correct") + @httprettified def test_me(self): httpretty.register_uri( httpretty.GET, @@ -113,17 +115,18 @@ def test_me(self): status=200, content_type=self._content_type("country"), ) - implicit_me = self.client.country() + implicit_me = self.run_client(self.client.country()) self.assertEqual( type(implicit_me), geoip2.models.Country, "country() returns Country object" ) - explicit_me = self.client.country() + explicit_me = self.run_client(self.client.country()) self.assertEqual( type(explicit_me), geoip2.models.Country, "country('me') returns Country object", ) + @httprettified def test_200_error(self): httpretty.register_uri( httpretty.GET, @@ -135,14 +138,16 @@ def test_200_error(self): with self.assertRaisesRegex( GeoIP2Error, "could not decode the response as JSON" ): - self.client.country("1.1.1.1") + self.run_client(self.client.country("1.1.1.1")) + @httprettified def test_bad_ip_address(self): with self.assertRaisesRegex( ValueError, "'1.2.3' does not appear to be an IPv4 " "or IPv6 address" ): - self.client.country("1.2.3") + self.run_client(self.client.country("1.2.3")) + @httprettified def test_no_body_error(self): httpretty.register_uri( httpretty.GET, @@ -154,8 +159,9 @@ def test_no_body_error(self): with self.assertRaisesRegex( HTTPError, "Received a 400 error for .* with no body" ): - self.client.country("1.2.3.7") + self.run_client(self.client.country("1.2.3.7")) + @httprettified def test_weird_body_error(self): httpretty.register_uri( httpretty.GET, @@ -168,8 +174,9 @@ def test_weird_body_error(self): HTTPError, "Response contains JSON but it does not " "specify code or error keys", ): - self.client.country("1.2.3.8") + self.run_client(self.client.country("1.2.3.8")) + @httprettified def test_bad_body_error(self): httpretty.register_uri( httpretty.GET, @@ -181,15 +188,17 @@ def test_bad_body_error(self): with self.assertRaisesRegex( HTTPError, "it did not include the expected JSON body" ): - self.client.country("1.2.3.9") + self.run_client(self.client.country("1.2.3.9")) + @httprettified def test_500_error(self): httpretty.register_uri( httpretty.GET, self.base_uri + "country/" + "1.2.3.10", status=500 ) with self.assertRaisesRegex(HTTPError, r"Received a server error \(500\) for"): - self.client.country("1.2.3.10") + self.run_client(self.client.country("1.2.3.10")) + @httprettified def test_300_error(self): httpretty.register_uri( httpretty.GET, @@ -201,38 +210,49 @@ def test_300_error(self): HTTPError, r"Received a very surprising HTTP status \(300\) for" ): - self.client.country("1.2.3.11") + self.run_client(self.client.country("1.2.3.11")) + @httprettified def test_ip_address_required(self): self._test_error(400, "IP_ADDRESS_REQUIRED", InvalidRequestError) + @httprettified def test_ip_address_not_found(self): self._test_error(404, "IP_ADDRESS_NOT_FOUND", AddressNotFoundError) + @httprettified def test_ip_address_reserved(self): self._test_error(400, "IP_ADDRESS_RESERVED", AddressNotFoundError) + @httprettified def test_permission_required(self): self._test_error(403, "PERMISSION_REQUIRED", PermissionRequiredError) + @httprettified def test_auth_invalid(self): self._test_error(400, "AUTHORIZATION_INVALID", AuthenticationError) + @httprettified def test_license_key_required(self): self._test_error(401, "LICENSE_KEY_REQUIRED", AuthenticationError) + @httprettified def test_account_id_required(self): self._test_error(401, "ACCOUNT_ID_REQUIRED", AuthenticationError) + @httprettified def test_user_id_required(self): self._test_error(401, "USER_ID_REQUIRED", AuthenticationError) + @httprettified def test_account_id_unkown(self): self._test_error(401, "ACCOUNT_ID_UNKNOWN", AuthenticationError) + @httprettified def test_user_id_unkown(self): self._test_error(401, "USER_ID_UNKNOWN", AuthenticationError) + @httprettified def test_out_of_queries_error(self): self._test_error(402, "OUT_OF_QUERIES", OutOfQueriesError) @@ -247,8 +267,9 @@ def _test_error(self, status, error_code, error_class): content_type=self._content_type("country"), ) with self.assertRaisesRegex(error_class, msg): - self.client.country("1.2.3.18") + self.run_client(self.client.country("1.2.3.18")) + @httprettified def test_unknown_error(self): msg = "Unknown error type" ip = "1.2.3.19" @@ -261,8 +282,9 @@ def test_unknown_error(self): content_type=self._content_type("country"), ) with self.assertRaisesRegex(InvalidRequestError, msg): - self.client.country(ip) + self.run_client(self.client.country(ip)) + @httprettified def test_request(self): httpretty.register_uri( httpretty.GET, @@ -271,8 +293,8 @@ def test_request(self): status=200, content_type=self._content_type("country"), ) - self.client.country("1.2.3.4") - request = httpretty.latest_requests()[-1] + self.run_client(self.client.country("1.2.3.4")) + request = httpretty.last_request self.assertEqual( request.path, "/geoip/v2.1/country/1.2.3.4", "correct URI is used" @@ -291,6 +313,7 @@ def test_request(self): "correct auth", ) + @httprettified def test_city_ok(self): httpretty.register_uri( httpretty.GET, @@ -299,12 +322,13 @@ def test_city_ok(self): status=200, content_type=self._content_type("city"), ) - city = self.client.city("1.2.3.4") + city = self.run_client(self.client.city("1.2.3.4")) self.assertEqual(type(city), geoip2.models.City, "return value of client.city") self.assertEqual( city.traits.network, ipaddress.ip_network("1.2.3.0/24"), "network" ) + @httprettified def test_insights_ok(self): httpretty.register_uri( httpretty.GET, @@ -313,7 +337,7 @@ def test_insights_ok(self): status=200, content_type=self._content_type("country"), ) - insights = self.client.insights("1.2.3.4") + insights = self.run_client(self.client.insights("1.2.3.4")) self.assertEqual( type(insights), geoip2.models.Insights, "return value of client.insights" ) @@ -326,16 +350,42 @@ def test_insights_ok(self): def test_named_constructor_args(self): id = 47 key = "1234567890ab" - client = Client(account_id=id, license_key=key) + client = self.client_class(account_id=id, license_key=key) self.assertEqual(client._account_id, str(id)) self.assertEqual(client._license_key, key) def test_missing_constructor_args(self): with self.assertRaises(TypeError): - Client(license_key="1234567890ab") + self.client_class(license_key="1234567890ab") with self.assertRaises(TypeError): - Client("47") + self.client_class("47") + + +class TestClient(TestBaseClient): + def setUp(self): + self.client_class = Client + self.client = Client(42, "abcdef123456") + + def run_client(self, v): + return v + + +class TestAsyncClient(TestBaseClient): + def setUp(self): + self._loop = asyncio.new_event_loop() + self.client_class = AsyncClient + self.client = AsyncClient(42, "abcdef123456") + + def tearDown(self): + self._loop.run_until_complete(self.client.close()) + self._loop.close() + + def run_client(self, v): + return self._loop.run_until_complete(v) + + +del TestBaseClient if __name__ == "__main__": From 0e35405ff6fd4e7f22d34ba9f88ed17acf9b0ccb Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 15 Jul 2020 14:01:17 -0700 Subject: [PATCH 06/11] Require mocket for tests --- .travis.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2c4db8b1..db77e0b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: - "if [[ $RUN_SNYK && $SNYK_TOKEN ]]; then sudo apt-get install -y nodejs; npm install -g snyk; fi" install: - pip install -r requirements.txt - - pip install httpretty coveralls + - pip install mocket coveralls - | if [[ $RUN_LINTER ]]; then pip install --upgrade pylint black mypy diff --git a/setup.py b/setup.py index 6ba33ea6..eb78ec1f 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ include_package_data=True, python_requires=">=3.6", install_requires=requirements, - tests_require=["httpretty>=1.0.0"], + tests_require=["mocket>=3.8.6"], test_suite="tests", license=geoip2.__license__, classifiers=[ From 6ece7a0cc473114376dd721d092e485d63b212b4 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 15 Jul 2020 13:54:46 -0700 Subject: [PATCH 07/11] Add AsyncClient example --- README.rst | 77 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index fda5e05b..081d3262 100644 --- a/README.rst +++ b/README.rst @@ -57,8 +57,8 @@ each of which represents part of the data returned by the web service. If the request fails, the client class throws an exception. -Web Service Example -------------------- +Sync Web Service Example +------------------------ .. code-block:: pycon @@ -67,36 +67,81 @@ Web Service Example >>> # This creates a Client object that can be reused across requests. >>> # Replace "42" with your account ID and "license_key" with your license >>> # key. - >>> client = geoip2.webservice.Client(42, 'license_key') + >>> with geoip2.webservice.Client(42, 'license_key') as client: >>> - >>> # Replace "insights" with the method corresponding to the web service - >>> # that you are using, e.g., "country", "city". - >>> response = client.insights('128.101.101.101') + >>> # Replace "insights" with the method corresponding to the web service + >>> # that you are using, e.g., "country", "city". + >>> response = client.insights('128.101.101.101') >>> - >>> response.country.iso_code + >>> response.country.iso_code 'US' - >>> response.country.name + >>> response.country.name 'United States' - >>> response.country.names['zh-CN'] + >>> response.country.names['zh-CN'] u'美国' >>> - >>> response.subdivisions.most_specific.name + >>> response.subdivisions.most_specific.name 'Minnesota' - >>> response.subdivisions.most_specific.iso_code + >>> response.subdivisions.most_specific.iso_code 'MN' >>> - >>> response.city.name + >>> response.city.name 'Minneapolis' >>> - >>> response.postal.code + >>> response.postal.code '55455' >>> - >>> response.location.latitude + >>> response.location.latitude 44.9733 - >>> response.location.longitude + >>> response.location.longitude -93.2323 >>> - >>> response.traits.network + >>> response.traits.network + IPv4Network('128.101.101.101/32') + +Async Web Service Example +------------------------ + +.. code-block:: pycon + + >>> import geoip2.webservice + >>> + >>> # This creates an AsyncClient object that can be reused across + >>> # requests on the running event loop. If you are using multiple event + >>> # loops, you must ensure the object is not used on another loop. + >>> # + >>> # Replace "42" with your account ID and "license_key" with your license + >>> # key. + >>> async with geoip2.webservice.AsyncClient(42, 'license_key') as client: + >>> + >>> # Replace "insights" with the method corresponding to the web service + >>> # that you are using, e.g., "country", "city". + >>> response = await client.insights('128.101.101.101') + >>> + >>> response.country.iso_code + 'US' + >>> response.country.name + 'United States' + >>> response.country.names['zh-CN'] + u'美国' + >>> + >>> response.subdivisions.most_specific.name + 'Minnesota' + >>> response.subdivisions.most_specific.iso_code + 'MN' + >>> + >>> response.city.name + 'Minneapolis' + >>> + >>> response.postal.code + '55455' + >>> + >>> response.location.latitude + 44.9733 + >>> response.location.longitude + -93.2323 + >>> + >>> response.traits.network IPv4Network('128.101.101.101/32') Web Service Client Exceptions From 5346e3cffca7e7bc17dd13e8306b8af9c9cef04d Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 15 Jul 2020 13:57:38 -0700 Subject: [PATCH 08/11] Use context manager consistently in examples --- README.rst | 92 ++++++++++++++++++++++++------------------------------ 1 file changed, 40 insertions(+), 52 deletions(-) diff --git a/README.rst b/README.rst index 081d3262..6cf5f39b 100644 --- a/README.rst +++ b/README.rst @@ -176,39 +176,37 @@ City Database >>> >>> # This creates a Reader object. You should use the same object >>> # across multiple requests as creation of it is expensive. - >>> reader = geoip2.database.Reader('/path/to/GeoLite2-City.mmdb') + >>> with geoip2.database.Reader('/path/to/GeoLite2-City.mmdb') as reader: >>> - >>> # Replace "city" with the method corresponding to the database - >>> # that you are using, e.g., "country". - >>> response = reader.city('128.101.101.101') + >>> # Replace "city" with the method corresponding to the database + >>> # that you are using, e.g., "country". + >>> response = reader.city('128.101.101.101') >>> - >>> response.country.iso_code + >>> response.country.iso_code 'US' - >>> response.country.name + >>> response.country.name 'United States' - >>> response.country.names['zh-CN'] + >>> response.country.names['zh-CN'] u'美国' >>> - >>> response.subdivisions.most_specific.name + >>> response.subdivisions.most_specific.name 'Minnesota' - >>> response.subdivisions.most_specific.iso_code + >>> response.subdivisions.most_specific.iso_code 'MN' >>> - >>> response.city.name + >>> response.city.name 'Minneapolis' >>> - >>> response.postal.code + >>> response.postal.code '55455' >>> - >>> response.location.latitude + >>> response.location.latitude 44.9733 - >>> response.location.longitude + >>> response.location.longitude -93.2323 >>> - >>> response.traits.network + >>> response.traits.network IPv4Network('128.101.101.0/24') - >>> - >>> reader.close() Anonymous IP Database ^^^^^^^^^^^^^^^^^^^^^ @@ -219,25 +217,24 @@ Anonymous IP Database >>> >>> # This creates a Reader object. You should use the same object >>> # across multiple requests as creation of it is expensive. - >>> reader = geoip2.database.Reader('/path/to/GeoIP2-Anonymous-IP.mmdb') + >>> with geoip2.database.Reader('/path/to/GeoIP2-Anonymous-IP.mmdb') as reader: >>> - >>> response = reader.anonymous_ip('85.25.43.84') + >>> response = reader.anonymous_ip('85.25.43.84') >>> - >>> response.is_anonymous + >>> response.is_anonymous True - >>> response.is_anonymous_vpn + >>> response.is_anonymous_vpn False - >>> response.is_hosting_provider + >>> response.is_hosting_provider False - >>> response.is_public_proxy + >>> response.is_public_proxy False - >>> response.is_tor_exit_node + >>> response.is_tor_exit_node True - >>> response.ip_address + >>> response.ip_address '85.25.43.84' - >>> response.network + >>> response.network IPv4Network('85.25.43.0/24') - >>> reader.close() ASN Database ^^^^^^^^^^^^ @@ -264,17 +261,14 @@ Connection-Type Database >>> >>> # This creates a Reader object. You should use the same object >>> # across multiple requests as creation of it is expensive. - >>> reader = geoip2.database.Reader('/path/to/GeoIP2-Connection-Type.mmdb') - >>> - >>> response = reader.connection_type('128.101.101.101') - >>> - >>> response.connection_type + >>> with geoip2.database.Reader('/path/to/GeoIP2-Connection-Type.mmdb') as reader: + >>> response = reader.connection_type('128.101.101.101') + >>> response.connection_type 'Corporate' - >>> response.ip_address + >>> response.ip_address '128.101.101.101' - >>> response.network + >>> response.network IPv4Network('128.101.101.101/24') - >>> reader.close() Domain Database @@ -286,15 +280,12 @@ Domain Database >>> >>> # This creates a Reader object. You should use the same object >>> # across multiple requests as creation of it is expensive. - >>> reader = geoip2.database.Reader('/path/to/GeoIP2-Domain.mmdb') - >>> - >>> response = reader.domain('128.101.101.101') - >>> - >>> response.domain + >>> with geoip2.database.Reader('/path/to/GeoIP2-Domain.mmdb') as reader: + >>> response = reader.domain('128.101.101.101') + >>> response.domain 'umn.edu' - >>> response.ip_address + >>> response.ip_address '128.101.101.101' - >>> reader.close() Enterprise Database ^^^^^^^^^^^^^^^^^^^ @@ -354,23 +345,20 @@ ISP Database >>> >>> # This creates a Reader object. You should use the same object >>> # across multiple requests as creation of it is expensive. - >>> reader = geoip2.database.Reader('/path/to/GeoIP2-ISP.mmdb') - >>> - >>> response = reader.isp('1.128.0.0') - >>> - >>> response.autonomous_system_number + >>> with geoip2.database.Reader('/path/to/GeoIP2-ISP.mmdb') as reader: + >>> response = reader.isp('1.128.0.0') + >>> response.autonomous_system_number 1221 - >>> response.autonomous_system_organization + >>> response.autonomous_system_organization 'Telstra Pty Ltd' - >>> response.isp + >>> response.isp 'Telstra Internet' - >>> response.organization + >>> response.organization 'Telstra Internet' - >>> response.ip_address + >>> response.ip_address '1.128.0.0' - >>> response.network + >>> response.network IPv4Network('1.128.0.0/16') - >>> reader.close() Database Reader Exceptions -------------------------- From 2b438e8cd48a6a4e4375d0fa565c61a5c333a714 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 15 Jul 2020 15:04:03 -0700 Subject: [PATCH 09/11] Use IPs from range reserved for documentation --- README.rst | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index 6cf5f39b..6093bbed 100644 --- a/README.rst +++ b/README.rst @@ -71,7 +71,7 @@ Sync Web Service Example >>> >>> # Replace "insights" with the method corresponding to the web service >>> # that you are using, e.g., "country", "city". - >>> response = client.insights('128.101.101.101') + >>> response = client.insights('203.0.113.0') >>> >>> response.country.iso_code 'US' @@ -97,7 +97,7 @@ Sync Web Service Example -93.2323 >>> >>> response.traits.network - IPv4Network('128.101.101.101/32') + IPv4Network('203.0.113.0/32') Async Web Service Example ------------------------ @@ -116,7 +116,7 @@ Async Web Service Example >>> >>> # Replace "insights" with the method corresponding to the web service >>> # that you are using, e.g., "country", "city". - >>> response = await client.insights('128.101.101.101') + >>> response = await client.insights('203.0.113.0') >>> >>> response.country.iso_code 'US' @@ -142,7 +142,7 @@ Async Web Service Example -93.2323 >>> >>> response.traits.network - IPv4Network('128.101.101.101/32') + IPv4Network('203.0.113.0/32') Web Service Client Exceptions ----------------------------- @@ -180,7 +180,7 @@ City Database >>> >>> # Replace "city" with the method corresponding to the database >>> # that you are using, e.g., "country". - >>> response = reader.city('128.101.101.101') + >>> response = reader.city('203.0.113.0') >>> >>> response.country.iso_code 'US' @@ -206,7 +206,7 @@ City Database -93.2323 >>> >>> response.traits.network - IPv4Network('128.101.101.0/24') + IPv4Network('203.0.113.0/24') Anonymous IP Database ^^^^^^^^^^^^^^^^^^^^^ @@ -219,7 +219,7 @@ Anonymous IP Database >>> # across multiple requests as creation of it is expensive. >>> with geoip2.database.Reader('/path/to/GeoIP2-Anonymous-IP.mmdb') as reader: >>> - >>> response = reader.anonymous_ip('85.25.43.84') + >>> response = reader.anonymous_ip('203.0.113.0') >>> >>> response.is_anonymous True @@ -232,9 +232,9 @@ Anonymous IP Database >>> response.is_tor_exit_node True >>> response.ip_address - '85.25.43.84' + '203.0.113.0' >>> response.network - IPv4Network('85.25.43.0/24') + IPv4Network('203.0.113.0/24') ASN Database ^^^^^^^^^^^^ @@ -246,11 +246,15 @@ ASN Database >>> # This creates a Reader object. You should use the same object >>> # across multiple requests as creation of it is expensive. >>> with geoip2.database.Reader('/path/to/GeoLite2-ASN.mmdb') as reader: - >>> response = reader.asn('1.128.0.0') + >>> response = reader.asn('203.0.113.0') >>> response.autonomous_system_number 1221 >>> response.autonomous_system_organization 'Telstra Pty Ltd' + >>> response.ip_address + '203.0.113.0' + >>> response.network + IPv4Network('203.0.113.0/24') Connection-Type Database ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -262,13 +266,13 @@ Connection-Type Database >>> # This creates a Reader object. You should use the same object >>> # across multiple requests as creation of it is expensive. >>> with geoip2.database.Reader('/path/to/GeoIP2-Connection-Type.mmdb') as reader: - >>> response = reader.connection_type('128.101.101.101') + >>> response = reader.connection_type('203.0.113.0') >>> response.connection_type 'Corporate' >>> response.ip_address - '128.101.101.101' + '203.0.113.0' >>> response.network - IPv4Network('128.101.101.101/24') + IPv4Network('203.0.113.0/24') Domain Database @@ -281,11 +285,11 @@ Domain Database >>> # This creates a Reader object. You should use the same object >>> # across multiple requests as creation of it is expensive. >>> with geoip2.database.Reader('/path/to/GeoIP2-Domain.mmdb') as reader: - >>> response = reader.domain('128.101.101.101') + >>> response = reader.domain('203.0.113.0') >>> response.domain 'umn.edu' >>> response.ip_address - '128.101.101.101' + '203.0.113.0' Enterprise Database ^^^^^^^^^^^^^^^^^^^ @@ -299,7 +303,7 @@ Enterprise Database >>> with geoip2.database.Reader('/path/to/GeoIP2-Enterprise.mmdb') as reader: >>> >>> # Use the .enterprise method to do a lookup in the Enterprise database - >>> response = reader.enterprise('128.101.101.101') + >>> response = reader.enterprise('203.0.113.0') >>> >>> response.country.confidence 99 @@ -333,7 +337,7 @@ Enterprise Database -93.2323 >>> >>> response.traits.network - IPv4Network('128.101.101.0/24') + IPv4Network('203.0.113.0/24') ISP Database @@ -346,7 +350,7 @@ ISP Database >>> # This creates a Reader object. You should use the same object >>> # across multiple requests as creation of it is expensive. >>> with geoip2.database.Reader('/path/to/GeoIP2-ISP.mmdb') as reader: - >>> response = reader.isp('1.128.0.0') + >>> response = reader.isp('203.0.113.0') >>> response.autonomous_system_number 1221 >>> response.autonomous_system_organization @@ -356,9 +360,9 @@ ISP Database >>> response.organization 'Telstra Internet' >>> response.ip_address - '1.128.0.0' + '203.0.113.0' >>> response.network - IPv4Network('1.128.0.0/16') + IPv4Network('203.0.113.0/24') Database Reader Exceptions -------------------------- From 0b07b65e450a8f4b20d2611e2b41e3d373a14857 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 15 Jul 2020 15:50:17 -0700 Subject: [PATCH 10/11] Move settings from get request to session creation --- HISTORY.rst | 2 ++ geoip2/webservice.py | 42 +++++++++++++++++++++--------------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 19391184..cb219c5a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,8 @@ History * ``user_id`` is no longer supported as a named argument for the constructor on ``geoip2.webservice.Client``. Use ``account_id`` or a positional parameter instead. +* For both ``Client`` and ``AsyncClient`` requests, the default timeout is + now 60 seconds. 3.0.0 (2019-12-20) ++++++++++++++++++ diff --git a/geoip2/webservice.py b/geoip2/webservice.py index 6c60c414..1121d9ef 100644 --- a/geoip2/webservice.py +++ b/geoip2/webservice.py @@ -53,7 +53,7 @@ class BaseClient: # pylint: disable=missing-class-docstring, too-few-public-met _host: str _license_key: str _locales: List[str] - _timeout: Optional[float] + _timeout: float _user_agent: str def __init__( @@ -62,7 +62,7 @@ def __init__( license_key: str, host: str, locales: Optional[List[str]], - timeout: Optional[float], + timeout: float, http_user_agent: str, ) -> None: """Construct a Client.""" @@ -233,8 +233,9 @@ class AsyncClient(BaseClient): * pt-BR -- Brazilian Portuguese * ru -- Russian * zh-CN -- Simplified Chinese. - :param timeout: The timeout to use when waiting on the request. This sets - both the connect timeout and the read timeout. + :param timeout: The timeout in seconts to use when waiting on the request. + This sets both the connect timeout and the read timeout. The default is + 60. """ @@ -246,7 +247,7 @@ def __init__( # pylint: disable=too-many-arguments license_key: str, host: str = "geoip.maxmind.com", locales: Optional[List[str]] = None, - timeout: Optional[float] = None, + timeout: float = 60, ) -> None: super().__init__( account_id, license_key, host, locales, timeout, default_user_agent() @@ -298,7 +299,12 @@ async def insights(self, ip_address: IPAddress = "me") -> Insights: async def _session(self) -> aiohttp.ClientSession: if not hasattr(self, "_existing_session"): - self._existing_session = aiohttp.ClientSession() + self._existing_session = aiohttp.ClientSession( + auth=aiohttp.BasicAuth(self._account_id, self._license_key), + headers={"Accept": "application/json", "User-Agent": self._user_agent}, + timeout=aiohttp.ClientTimeout(total=self._timeout), + ) + return self._existing_session async def _response_for( @@ -309,12 +315,7 @@ async def _response_for( ) -> Union[Country, City, Insights]: uri = self._uri(path, ip_address) session = await self._session() - async with await session.get( - uri, - auth=aiohttp.BasicAuth(self._account_id, self._license_key), - headers={"Accept": "application/json", "User-Agent": self._user_agent}, - timeout=self._timeout, - ) as response: + async with await session.get(uri) as response: status = response.status content_type = response.content_type body = await response.text() @@ -378,8 +379,9 @@ class Client(BaseClient): * pt-BR -- Brazilian Portuguese * ru -- Russian * zh-CN -- Simplified Chinese. - :param timeout: The timeout to use when waiting on the request. This sets - both the connect timeout and the read timeout. + :param timeout: The timeout in seconts to use when waiting on the request. + This sets both the connect timeout and the read timeout. The default is + 60. """ @@ -391,12 +393,15 @@ def __init__( # pylint: disable=too-many-arguments license_key: str, host: str = "geoip.maxmind.com", locales: Optional[List[str]] = None, - timeout: Optional[float] = None, + timeout: float = 60, ) -> None: super().__init__( account_id, license_key, host, locales, timeout, default_user_agent() ) self._session = requests.Session() + self._session.auth = (self._account_id, self._license_key) + self._session.headers["Accept"] = "application/json" + self._session.headers["User-Agent"] = self._user_agent def city(self, ip_address: IPAddress = "me") -> City: """Call GeoIP2 Precision City endpoint with the specified IP. @@ -445,12 +450,7 @@ def _response_for( ip_address: IPAddress, ) -> Union[Country, City, Insights]: uri = self._uri(path, ip_address) - response = self._session.get( - uri, - auth=(self._account_id, self._license_key), - headers={"Accept": "application/json", "User-Agent": self._user_agent}, - timeout=self._timeout, - ) + response = self._session.get(uri, timeout=self._timeout) status = response.status_code content_type = response.headers["Content-Type"] body = response.text From 8f6785b799ef483a119281bc0624db52a8ed91e0 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 17 Jul 2020 13:46:30 -0700 Subject: [PATCH 11/11] Use correct UA string for aiohttp and calc once --- geoip2/webservice.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/geoip2/webservice.py b/geoip2/webservice.py index 1121d9ef..0dbab855 100644 --- a/geoip2/webservice.py +++ b/geoip2/webservice.py @@ -30,8 +30,9 @@ from typing import Any, cast, List, Optional, Type, Union import aiohttp +import aiohttp.http import requests -from requests.utils import default_user_agent +import requests.utils import geoip2 import geoip2.models @@ -47,6 +48,15 @@ from geoip2.models import City, Country, Insights from geoip2.types import IPAddress +_AIOHTTP_UA = "GeoIP2-Python-Client/%s %s" % ( + geoip2.__version__, + aiohttp.http.SERVER_SOFTWARE, +) +_REQUEST_UA = "GeoIP2-Python-Client/%s %s" % ( + geoip2.__version__, + requests.utils.default_user_agent(), +) + class BaseClient: # pylint: disable=missing-class-docstring, too-few-public-methods _account_id: str @@ -54,7 +64,6 @@ class BaseClient: # pylint: disable=missing-class-docstring, too-few-public-met _license_key: str _locales: List[str] _timeout: float - _user_agent: str def __init__( self, @@ -63,7 +72,6 @@ def __init__( host: str, locales: Optional[List[str]], timeout: float, - http_user_agent: str, ) -> None: """Construct a Client.""" # pylint: disable=too-many-arguments @@ -79,10 +87,6 @@ def __init__( self._license_key = license_key self._base_uri = "https://%s/geoip/v2.1" % host self._timeout = timeout - self._user_agent = "GeoIP2-Python-Client/%s %s" % ( - geoip2.__version__, - http_user_agent, - ) def _uri(self, path: str, ip_address: IPAddress) -> str: if ip_address != "me": @@ -250,7 +254,7 @@ def __init__( # pylint: disable=too-many-arguments timeout: float = 60, ) -> None: super().__init__( - account_id, license_key, host, locales, timeout, default_user_agent() + account_id, license_key, host, locales, timeout, ) async def city(self, ip_address: IPAddress = "me") -> City: @@ -301,7 +305,7 @@ async def _session(self) -> aiohttp.ClientSession: if not hasattr(self, "_existing_session"): self._existing_session = aiohttp.ClientSession( auth=aiohttp.BasicAuth(self._account_id, self._license_key), - headers={"Accept": "application/json", "User-Agent": self._user_agent}, + headers={"Accept": "application/json", "User-Agent": _AIOHTTP_UA}, timeout=aiohttp.ClientTimeout(total=self._timeout), ) @@ -395,13 +399,11 @@ def __init__( # pylint: disable=too-many-arguments locales: Optional[List[str]] = None, timeout: float = 60, ) -> None: - super().__init__( - account_id, license_key, host, locales, timeout, default_user_agent() - ) + super().__init__(account_id, license_key, host, locales, timeout) self._session = requests.Session() self._session.auth = (self._account_id, self._license_key) self._session.headers["Accept"] = "application/json" - self._session.headers["User-Agent"] = self._user_agent + self._session.headers["User-Agent"] = _REQUEST_UA def city(self, ip_address: IPAddress = "me") -> City: """Call GeoIP2 Precision City endpoint with the specified IP.