process_scans crashes on requests timeout or disconnect #167
Description
Occasionally, the process_scans command/service will stop because of an interruption when getting scan data or during some other request to scancode.io.
When something like the following happens, we should save that in the error log and not crash:
INFO:minecode.management.commands.process_scans:Checking or processing scan for URI: https://repo1.maven.org/maven2/com/liferay/com.liferay.portal.k8s.agent.impl/3.0.23/com.liferay.portal.k8s.agent.impl-3.0.23.jar
Traceback (most recent call last):
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/urllib3/connection.py", line 200, in _new_conn
sock = connection.create_connection(
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection
raise err
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/urllib3/util/connection.py", line 73, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 790, in urlopen
response = self._make_request(
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 496, in _make_request
conn.request(
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/urllib3/connection.py", line 388, in request
self.endheaders()
File "/home/jono/.pyenv/versions/3.10.8/lib/python3.10/http/client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/home/jono/.pyenv/versions/3.10.8/lib/python3.10/http/client.py", line 1037, in _send_output
self.send(msg)
File "/home/jono/.pyenv/versions/3.10.8/lib/python3.10/http/client.py", line 975, in send
self.connect()
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/urllib3/connection.py", line 236, in connect
self.sock = self._new_conn()
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/urllib3/connection.py", line 215, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7fbd8edb1480>: Failed to establish a new connection: [Errno 111] Connection refused
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/requests/adapters.py", line 486, in send
resp = conn.urlopen(
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 844, in urlopen
retries = retries.increment(
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/urllib3/util/retry.py", line 515, in increment
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='127.0.0.1', port=8008): Max retries exceeded with url: /api/projects/deefed09-a2b8-4a64-9dbb-15faca7fb850/ (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fbd8edb1480>: Failed to establish a new connection: [Errno 111] Connection refused'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/jono/nexb/src/purldb/manage.py", line 19, in <module>
execute_from_command_line(sys.argv)
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/django/core/management/__init__.py", line 446, in execute_from_command_line
utility.execute()
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/django/core/management/__init__.py", line 440, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/django/core/management/base.py", line 402, in run_from_argv
self.execute(*args, **cmd_options)
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/django/core/management/base.py", line 448, in execute
output = self.handle(*args, **options)
File "/home/jono/nexb/src/purldb/minecode/management/commands/process_scans.py", line 37, in handle
scanning.ScanningCommand.handle(self, *args, **options)
File "/home/jono/nexb/src/purldb/minecode/management/scanning.py", line 404, in handle
uris_counter = self.process_scans(
File "/home/jono/nexb/src/purldb/minecode/management/scanning.py", line 453, in process_scans
cls.process_scan(scannable_uri, **kwargs)
File "/home/jono/nexb/src/purldb/minecode/management/commands/process_scans.py", line 56, in process_scan
scan_info = scanning.get_scan_info(
File "/home/jono/nexb/src/purldb/minecode/management/scanning.py", line 313, in get_scan_info
results = _get_scan_info(
File "/home/jono/nexb/src/purldb/minecode/management/scanning.py", line 289, in _get_scan_info
results = _call_scan_get_api(
File "/home/jono/nexb/src/purldb/minecode/management/scanning.py", line 272, in _call_scan_get_api
response = requests.get(url=scan_url, timeout=timeout, headers=api_auth_headers)
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/home/jono/nexb/src/purldb/venv/lib/python3.10/site-packages/requests/adapters.py", line 519, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='127.0.0.1', port=8008): Max retries exceeded with url: /api/projects/deefed09-a2b8-4a64-9dbb-15faca7fb850/ (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fbd8edb1480>: Failed to establish a new connection: [Errno 111] Connection refused'))