Handle html errors

b2sdk/b2http.py

@@ -34,7 +34,6 @@
     BrokenPipe,
     ClockSkew,
     ConnectionReset,
-    InvalidJsonResponse,
     PotentialS3EndpointPassedAsRealm,
     UnknownError,
     UnknownHost,
@@ -420,9 +419,21 @@ def _translate_errors(cls, fcn, post_params=None):
         response = None
         try:
             response = fcn()
-            if response.status_code not in [200, 206]:
+            if response.status_code not in (200, 206):
                 # Decode the error object returned by the service
-                error = json.loads(response.content.decode('utf-8')) if response.content else {}
+                try:
+                    error = json.loads(response.content.decode('utf-8')) if response.content else {}
+                except (json.JSONDecodeError, UnicodeDecodeError):
+                    logger.error('failed to decode error response: %r', response.content)
+                    # When the user points to an S3 endpoint, he won't receive the JSON error
+                    # he expects. In that case, we can provide at least a hint of "what happened".
+                    # s3 url has the form of e.g. https://s3.us-west-000.backblazeb2.com
+                    if '://s3.' in response.url:
+                        raise PotentialS3EndpointPassedAsRealm(response.content)
+                    error = {
+                        'message': response.content.decode('utf-8', errors='replace'),
+                        'code': 'non_json_response',
+                    }
                 extra_error_keys = error.keys() - ('code', 'status', 'message')
                 if extra_error_keys:
                     logger.debug(
@@ -462,17 +473,6 @@ def _translate_errors(cls, fcn, post_params=None):
         except requests.Timeout as e:
             raise B2RequestTimeout(str(e))
 
-        except json.JSONDecodeError:
-            if response is None:
-                raise RuntimeError('Got JSON error without a response.')
-
-            # When the user points to an S3 endpoint, he won't receive the JSON error
-            # he expects. In that case, we can provide at least a hint of "what happened".
-            # s3 url has the form of e.g. https://s3.us-west-000.backblazeb2.com
-            if '://s3.' in response.url:
-                raise PotentialS3EndpointPassedAsRealm(response.content)
-            raise InvalidJsonResponse(response.content)
-
         except Exception as e:
             text = repr(e)
 

b2sdk/exception.py

@@ -545,8 +545,8 @@ class InvalidJsonResponse(B2SimpleError):
 
     def __init__(self, content: bytes):
         self.content = content
-        message = '%s' % self.content[:self.UP_TO_BYTES_COUNT]
-        if len(content) > self.UP_TO_BYTES_COUNT:
+        message = self.content[:self.UP_TO_BYTES_COUNT].decode('utf-8', errors='replace')
+        if len(self.content) > self.UP_TO_BYTES_COUNT:
             message += '...'
 
         super().__init__(message)

changelog.d/+handle_html_errors.fixed.md

@@ -0,0 +1 @@
+Handle non-json encoded B2 error responses, i.e. retry on 502 and 504 errors.

test/unit/b2http/test_b2http.py

@@ -15,16 +15,17 @@
 from unittest.mock import MagicMock, call, patch
 
 import apiver_deps
+import pytest
 import requests
 from apiver_deps import USER_AGENT, B2Http, B2HttpApiConfig, ClockSkewHook
 from apiver_deps_exception import (
     B2ConnectionError,
     BadDateFormat,
     BadJson,
+    BadRequest,
     BrokenPipe,
     ClockSkew,
     ConnectionReset,
-    InvalidJsonResponse,
     PotentialS3EndpointPassedAsRealm,
     ServiceError,
     TooManyRequests,
@@ -117,11 +118,10 @@ def test_invalid_json(self):
         response.content = b'{' * 500
         response.url = 'https://example.com'
 
-        with self.assertRaises(InvalidJsonResponse) as error:
+        with pytest.raises(BadRequest) as exc_info:
             B2Http._translate_errors(lambda: response)
 
-            content_length = min(len(response.content), len(error.content))
-            self.assertEqual(response.content[:content_length], error.content[:content_length])
+        assert str(exc_info.value) == f"{response.content.decode()} (non_json_response)"
 
     def test_potential_s3_endpoint_passed_as_realm(self):
         response = MagicMock()
@@ -133,6 +133,18 @@ def test_potential_s3_endpoint_passed_as_realm(self):
             B2Http._translate_errors(lambda: response)
 
 
+def test_b2_error__nginx_html():
+    """
+    While errors with HTML description should not happen, we should not crash on them.
+    """
+    response = MagicMock()
+    response.status_code = 502
+    response.content = b'<html><body><h1>502 Bad Gateway</h1></body></html>'
+    with pytest.raises(ServiceError) as exc_info:
+        B2Http._translate_errors(lambda: response)
+    assert response.content.decode('utf-8') in str(exc_info.value)
+
+
 class TestTranslateAndRetry(TestBase):
     def setUp(self):
         self.response = MagicMock()