getBatchDetails() - ReadTimeoutError: HTTPSConnectionPool(host='ipinfo.io', port=443): Read timed out. (read timeout=2)

[wesinator]

Got the following exception when using handler.getBatchDetails([uri]) checking large number of IPs from third-party API data source.

ReadTimeoutError: HTTPSConnectionPool(host='ipinfo.io', port=443): Read timed out. (read timeout=2)

/usr/local/lib/python3.8/site-packages/ipinfo/handler.py in getBatchDetails(self, ip_addresses)
     67         headers = self._get_headers()
     68         headers["content-type"] = "application/json"
---> 69         response = requests.post(
     70             url, json=lookup_addresses, headers=headers, **self.request_options
     71         )      

Not sure if this is fixable in this library, maybe you should add rate-limiting or something to the requests ?

Possible workaround: time.sleep() between each IP being sent to ipinfo request

Full trace:

---------------------------------------------------------------------------
timeout                                   Traceback (most recent call last)
/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    425                     # Otherwise it looks like a bug in the code.
--> 426                     six.raise_from(e, None)
    427         except (SocketTimeout, BaseSSLError, SocketError) as e:

/usr/local/lib/python3.8/site-packages/urllib3/packages/six.py in raise_from(value, from_value)

/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    420                 try:
--> 421                     httplib_response = conn.getresponse()
    422                 except BaseException as e:

/usr/local/Cellar/python@3.8/3.8.4/Frameworks/Python.framework/Versions/3.8/lib/python3.8/http/client.py in getresponse(self)
   1331             try:
-> 1332                 response.begin()
   1333             except ConnectionError:

/usr/local/Cellar/python@3.8/3.8.4/Frameworks/Python.framework/Versions/3.8/lib/python3.8/http/client.py in begin(self)
    302         while True:
--> 303             version, status, reason = self._read_status()
    304             if status != CONTINUE:

/usr/local/Cellar/python@3.8/3.8.4/Frameworks/Python.framework/Versions/3.8/lib/python3.8/http/client.py in _read_status(self)
    263     def _read_status(self):
--> 264         line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
    265         if len(line) > _MAXLINE:

/usr/local/Cellar/python@3.8/3.8.4/Frameworks/Python.framework/Versions/3.8/lib/python3.8/socket.py in readinto(self, b)
    668             try:
--> 669                 return self._sock.recv_into(b)
    670             except timeout:

/usr/local/Cellar/python@3.8/3.8.4/Frameworks/Python.framework/Versions/3.8/lib/python3.8/ssl.py in recv_into(self, buffer, nbytes, flags)
   1240                   self.__class__)
-> 1241             return self.read(nbytes, buffer)
   1242         else:

/usr/local/Cellar/python@3.8/3.8.4/Frameworks/Python.framework/Versions/3.8/lib/python3.8/ssl.py in read(self, len, buffer)
   1098             if buffer is not None:
-> 1099                 return self._sslobj.read(len, buffer)
   1100             else:

timeout: The read operation timed out

During handling of the above exception, another exception occurred:

ReadTimeoutError                          Traceback (most recent call last)
/usr/local/lib/python3.8/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    438             if not chunked:
--> 439                 resp = conn.urlopen(
    440                     method=request.method,

/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    723 
--> 724             retries = retries.increment(
    725                 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]

/usr/local/lib/python3.8/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
    402             if read is False or not self._is_method_retryable(method):
--> 403                 raise six.reraise(type(error), error, _stacktrace)
    404             elif read is not None:

/usr/local/lib/python3.8/site-packages/urllib3/packages/six.py in reraise(tp, value, tb)
    734                 raise value.with_traceback(tb)
--> 735             raise value
    736         finally:

/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    669             # Make the request on the httplib connection object.
--> 670             httplib_response = self._make_request(
    671                 conn,

/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    427         except (SocketTimeout, BaseSSLError, SocketError) as e:
--> 428             self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
    429             raise

/usr/local/lib/python3.8/site-packages/urllib3/connectionpool.py in _raise_timeout(self, err, url, timeout_value)
    334         if isinstance(err, SocketTimeout):
--> 335             raise ReadTimeoutError(
    336                 self, url, "Read timed out. (read timeout=%s)" % timeout_value

ReadTimeoutError: HTTPSConnectionPool(host='ipinfo.io', port=443): Read timed out. (read timeout=2)

During handling of the above exception, another exception occurred:

ReadTimeout                               Traceback (most recent call last)
<ipython-input-12-81b6d3ab61af> in <module>
     56 
     57         # check ipinfo data to see if the IP is multi-tenant / legitimate potential
---> 58         domains_count = ipinfo_get_domains(ipinfo_token, ip)
     59         domain_threshold = 100
     60 

<ipython-input-12-81b6d3ab61af> in ipinfo_get_domains(ipinfo_token, ip)
     15     handler = ipinfo.getHandler(ipinfo_token)
     16     uri = ip
---> 17     r = handler.getBatchDetails([uri])
     18     #print(r)
     19 

/usr/local/lib/python3.8/site-packages/ipinfo/handler.py in getBatchDetails(self, ip_addresses)
     67         headers = self._get_headers()
     68         headers["content-type"] = "application/json"
---> 69         response = requests.post(
     70             url, json=lookup_addresses, headers=headers, **self.request_options
     71         )

/usr/local/lib/python3.8/site-packages/requests/api.py in post(url, data, json, **kwargs)
    117     """
    118 
--> 119     return request('post', url, data=data, json=json, **kwargs)
    120 
    121 

/usr/local/lib/python3.8/site-packages/requests/api.py in request(method, url, **kwargs)
     59     # cases, and look like a memory leak in others.
     60     with sessions.Session() as session:
---> 61         return session.request(method=method, url=url, **kwargs)
     62 
     63 

/usr/local/lib/python3.8/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    528         }
    529         send_kwargs.update(settings)
--> 530         resp = self.send(prep, **send_kwargs)
    531 
    532         return resp

/usr/local/lib/python3.8/site-packages/requests/sessions.py in send(self, request, **kwargs)
    641 
    642         # Send the request
--> 643         r = adapter.send(request, **kwargs)
    644 
    645         # Total elapsed time of the request (approximately)

/usr/local/lib/python3.8/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    527                 raise SSLError(e, request=request)
    528             elif isinstance(e, ReadTimeoutError):
--> 529                 raise ReadTimeout(e, request=request)
    530             else:
    531                 raise

ReadTimeout: HTTPSConnectionPool(host='ipinfo.io', port=443): Read timed out. (read timeout=2)

Thanks,

[wesinator]

Code worked from adding a time.sleep(0.25) between each IP request

I don't think the issue is adjustable in this library anyway

[manish-shegokar]

Where in the code do we need to add time.sleep() function?

[UmanShahzad]

@manish-shegokar are you using the latest version of the library, and the synchronous handler? If so, you can adjust the new timeout_per_batch keyword argument.

Otherwise, you will need to update the client-level request timeout which is passed into getHandler. See https://github.com/ipinfo/python#modifying-request-options.