fix(xmlupload): print reason of error if a resource cannot be created…

…, remodel BaseError (DEV-1977) (#349)

If a single resource cannot be created, but the xmlupload continues, the reason of the error could (until now) only be found in the log file. From the terminal output alone, the user doesn't know what's wrong with his XML file.

From now on, more information is printed to the terminal.

Old output:

```
>>> dsp-tools xmlupload data.xml
(...)
Created resource 47/50: 'Annotation to TestthingOhnePermissions' (ID: 'annotation_1', IRI: 'http://rdfh.ch/4123/6lqOzlhMQkSlJeYNKjPHJw')
WARNING: Unable to create resource 'First Testthing' (test_thing_1).
Created resource 49/50: 'Second Testthing' (ID: 'test_thing_2', IRI: 'http://rdfh.ch/4123/_OLrsCnaQ3STDsfYLolSoA')
Created resource 50/50: 'Link between 'Second Testthing', 'Compoundthing' and 'Partofthing-1'' (ID: 'link_obj_1', IRI: 'http://rdfh.ch/4123/wVSAp0yIRUe2H__J6D4hSA')
Upload the stashed XML texts...
  Upload XML text(s) of resource "test_thing_2"...
    WARNING: Unable to upload the xml text of "testonto:hasRichtext" of resource "test_thing_2"
```

New output:

```
>>> dsp-tools xmlupload data.xml
(...)
Created resource 47/50: 'Annotation to TestthingOhnePermissions' (ID: 'annotation_1', IRI: 'http://rdfh.ch/4123/I9r3uZ3wSfOoK7pMzVdOug')
WARNING: Unable to create resource 'First Testthing' (test_thing_1): String does not represent integer number! "hello"
Created resource 49/50: 'Second Testthing' (ID: 'test_thing_2', IRI: 'http://rdfh.ch/4123/m8mZN3D7R4y-1xIlKSjbTA')
Created resource 50/50: 'Link between 'Second Testthing', 'Compoundthing' and 'Partofthing-1'' (ID: 'link_obj_1', IRI: 'http://rdfh.ch/4123/YfV51TQ8QJqBG5ZaKTRzvg')
Upload the stashed XML texts...
  Upload XML text(s) of resource "test_thing_2"...
    WARNING: Unable to upload the xml text of "testonto:hasRichtext" of resource "test_thing_2". Original error message: Invalid standoff resource reference: IRI:test_thing_1:IRI
```

src/dsp_tools/models/connection.py

@@ -7,6 +7,25 @@
 from dsp_tools.models.exceptions import BaseError
 
 
+def check_for_api_error(response: requests.Response) -> None:
+    """
+    Check the response of an API request if it contains an error raised by DSP-API.
+
+    Args:
+        res: The requests.Response object that is returned by the API request
+    
+    Raises:
+        BaseError: If the status code of the response is not 200
+    """
+    if response.status_code != 200:
+        raise BaseError(
+            message="KNORA-ERROR: status code=" + str(response.status_code) + "\nMessage:" + response.text,
+            status_code=response.status_code,
+            json_content_of_api_response=response.text,
+            reason_from_api_response=response.reason,
+            api_route=response.url
+        )
+
 class Connection:
     """
     An Connection instance represents a connection to a Knora server.
@@ -47,13 +66,13 @@ def login(self, email: str, password: str) -> None:
         }
         jsondata = json.dumps(credentials)
 
-        req = requests.post(
+        response = requests.post(
             self._server + '/v2/authentication',
             headers={'Content-Type': 'application/json; charset=UTF-8'},
             data=jsondata
         )
-        self.on_api_error(req)
-        result = req.json()
+        check_for_api_error(response)
+        result = response.json()
         self._token = result["token"]
 
     def get_token(self) -> str:
@@ -81,29 +100,17 @@ def logout(self) -> None:
         """
 
         if self._token is not None:
-            req = requests.delete(
+            response = requests.delete(
                 self._server + '/v2/authentication',
                 headers={'Authorization': 'Bearer ' + self._token}
             )
-            self.on_api_error(req)
+            check_for_api_error(response)
             self._token = None
 
     def __del__(self):
         pass
         # self.logout()
 
-    def on_api_error(self, res) -> None:
-        """
-        Method to check for any API errors
-        :param res: The input to check, usually JSON format
-        :return: Possible Error that is being raised
-        """
-
-        if res.status_code != 200:
-            raise BaseError("KNORA-ERROR: status code=" + str(res.status_code) + "\nMessage:" + res.text)
-
-        if 'error' in res:
-            raise BaseError("KNORA-ERROR: API error: " + res.error)
 
     def post(self, path: str, jsondata: Optional[str] = None):
         """
@@ -119,22 +126,28 @@ def post(self, path: str, jsondata: Optional[str] = None):
         if jsondata is None:
             if self._token is not None:
                 headers = {'Authorization': 'Bearer ' + self._token}
-                req = requests.post(self._server + path,
-                                    headers=headers)
+                response = requests.post(
+                    self._server + path,
+                    headers=headers
+                )
             else:
-                req = requests.post(self._server + path)
+                response = requests.post(self._server + path)
         else:
             if self._token is not None:
                 headers = {'Content-Type': 'application/json; charset=UTF-8',
                            'Authorization': 'Bearer ' + self._token}
-                req = requests.post(self._server + path,
-                                    headers=headers,
-                                    data=jsondata)
+                response = requests.post(
+                    self._server + path,
+                    headers=headers,
+                    data=jsondata
+                )
             else:
                 headers = {'Content-Type': 'application/json; charset=UTF-8'}
-                req = requests.post(self._server + path,
-                                    headers=headers,
-                                    data=jsondata)
+                response = requests.post(
+                    self._server + path,
+                    headers=headers,
+                    data=jsondata
+                )
         if self._log:
             if jsondata:
                 jsonobj = json.loads(jsondata)
@@ -145,16 +158,16 @@ def post(self, path: str, jsondata: Optional[str] = None):
                 "headers": headers,
                 "route": path,
                 "body": jsonobj,
-                "return-headers": dict(req.headers),
-                "return": req.json() if req.status_code == 200 else {"status": str(req.status_code),
-                                                                     "message": req.text}
+                "return-headers": dict(response.headers),
+                "return": response.json() if response.status_code == 200 else {"status": str(response.status_code), 
+                                                                               "message": response.text}
             }
             tmp = path.split('/')
             filename = "POST" + "_".join(tmp) + ".json"
             with open(filename, 'w') as f:
                 json.dump(logobj, f, indent=4)
-        self.on_api_error(req)
-        result = req.json()
+        check_for_api_error(response)
+        result = response.json()
         return result
 
     def get(self, path: str, headers: Optional[dict[str, str]] = None) -> dict[str, Any]:
@@ -179,7 +192,7 @@ def get(self, path: str, headers: Optional[dict[str, str]] = None) -> dict[str,
                 headers['Authorization'] = 'Bearer ' + self._token
                 response = requests.get(self._server + path, headers)
 
-        self.on_api_error(response)
+        check_for_api_error(response)
         json_response = response.json()
         return json_response
 
@@ -195,15 +208,20 @@ def put(self, path: str, jsondata: Optional[str] = None, content_type: str = 'ap
         if path[0] != '/':
             path = '/' + path
         if jsondata is None:
-            req = requests.put(self._server + path,
-                               headers={'Authorization': 'Bearer ' + self._token})
+            response = requests.put(
+                self._server + path,
+                headers={'Authorization': 'Bearer ' + self._token}
+            )
         else:
-            req = requests.put(self._server + path,
-                               headers={'Content-Type': content_type + '; charset=UTF-8',
-                                        'Authorization': 'Bearer ' + self._token},
-                               data=jsondata)
-        self.on_api_error(req)
-        result = req.json()
+            response = requests.put(
+                self._server + path,
+                headers={
+                    'Content-Type': content_type + '; charset=UTF-8', 
+                    'Authorization': 'Bearer ' + self._token
+                },
+                data=jsondata)
+        check_for_api_error(response)
+        result = response.json()
         return result
 
     def delete(self, path: str, params: Optional[any] = None):
@@ -216,15 +234,19 @@ def delete(self, path: str, params: Optional[any] = None):
         if path[0] != '/':
             path = '/' + path
         if params is not None:
-            req = requests.delete(self._server + path,
-                                  headers={'Authorization': 'Bearer ' + self._token},
-                                  params=params)
+            response = requests.delete(
+                self._server + path,
+                headers={'Authorization': 'Bearer ' + self._token},
+                params=params
+            )
 
         else:
-            req = requests.delete(self._server + path,
-                                  headers={'Authorization': 'Bearer ' + self._token})
-        self.on_api_error(req)
-        result = req.json()
+            response = requests.delete(
+                self._server + path,
+                headers={'Authorization': 'Bearer ' + self._token}
+            )
+        check_for_api_error(response)
+        result = response.json()
         return result
 
     def reset_triplestore_content(self):
@@ -265,10 +287,12 @@ def reset_triplestore_content(self):
         jsondata = json.dumps(rdfdata)
         url = self._server + '/admin/store/ResetTriplestoreContent?prependdefaults=false'
 
-        req = requests.post(url,
-                            headers={'Content-Type': 'application/json; charset=UTF-8'},
-                            data=jsondata)
-        self.on_api_error(req)
-        res = req.json()
+        response = requests.post(
+            url,
+            headers={'Content-Type': 'application/json; charset=UTF-8'},
+            data=jsondata
+        )
+        check_for_api_error(response)
+        res = response.json()
         #  pprint(res)
         return res

src/dsp_tools/models/exceptions.py

@@ -1,12 +1,60 @@
+import json
+import re
+from typing import Optional
+
+
 class BaseError(Exception):
     """
-    A basic error class for DSP-TOOLS
+    A basic error class for DSP-TOOLS.
+
+    Attributes:
+        message: A message that describes the error
+        status_code: HTTP status code of the response from DSP-API (only applicable if the error comes from DSP-API)
+        json_content_of_api_response: The message that DSP-API returns (only applicable if the error comes from DSP-API)
+        original_error_msg_from_api: The original error message that DSP-API returns (only applicable if the error comes from DSP-API)
+        reason_from_api_response: The reason for the failure that DSP-API returns (only applicable if the error comes from DSP-API)
+        api_route: The route that was called (only applicable if the error comes from DSP-API)
     """
     message: str
+    status_code: Optional[int]
+    json_content_of_api_response: Optional[str]
+    original_error_msg_from_api: Optional[str]
+    reason_from_api_response: Optional[str]
+    api_route: Optional[str]
+
+    def __init__(
+        self, 
+        message: str, 
+        status_code: Optional[int] = None,
+        json_content_of_api_response: Optional[str] = None,
+        reason_from_api_response: Optional[str] = None,
+        api_route: Optional[str] = None
+    ) -> None:
+        """
+        A basic error class for DSP-TOOLS.
 
-    def __init__(self, message: str) -> None:
+        Args:
+            message: A message that describes the error
+            status_code: HTTP status code of the response from DSP-API (only applicable if the error comes from DSP-API)
+            json_content_of_api_response: The message that DSP-API returns (only applicable if the error comes from DSP-API)
+            reason_from_api_response: The reason for the failure that DSP-API returns (only applicable if the error comes from DSP-API)
+            api_route: The route that was called (only applicable if the error comes from DSP-API)
+        """
         super().__init__()
         self.message = message
+        self.status_code = status_code
+        if json_content_of_api_response:
+            self.json_content_of_api_response = json_content_of_api_response
+            try:
+                parsed_json = json.loads(json_content_of_api_response)
+                if "knora-api:error" in parsed_json:
+                    knora_api_error = parsed_json["knora-api:error"]
+                    knora_api_error = re.sub(r"^dsp\.errors\.[A-Za-z]+?: ?", "", knora_api_error)
+                    self.original_error_msg_from_api = knora_api_error
+            except json.JSONDecodeError:
+                pass
+        self.reason_from_api_response = reason_from_api_response
+        self.api_route = api_route
 
     def __str__(self) -> str:
         return self.message

src/dsp_tools/models/sipi.py

@@ -3,25 +3,7 @@
 
 import requests
 
-from dsp_tools.models.exceptions import BaseError
-
-
-def on_api_error(res):
-    """
-    Checks for any API errors
-
-    Args:
-        res: the response from the API which is checked, usually in JSON format
-
-    Returns:
-        Knora Error that is being raised
-    """
-
-    if res.status_code != 200:
-        raise BaseError("SIPI-ERROR: status code=" + str(res.status_code) + "\nMessage:" + res.text)
-
-    if 'error' in res:
-        raise BaseError("SIPI-ERROR: API error: " + res.error)
+from dsp_tools.models.connection import check_for_api_error
 
 
 class Sipi:
@@ -43,7 +25,7 @@ def upload_bitstream(self, filepath: str) -> dict[Any, Any]:
         """
         with open(filepath, 'rb') as bitstream_file:
             files = {'file': (os.path.basename(filepath), bitstream_file), }
-            req = requests.post(self.sipi_server + "/upload?token=" + self.token, files=files)
-        on_api_error(req)
-        res: dict[Any, Any] = req.json()
+            response = requests.post(self.sipi_server + "/upload?token=" + self.token, files=files)
+        check_for_api_error(response)
+        res: dict[Any, Any] = response.json()
         return res

src/dsp_tools/utils/shared.py

@@ -75,7 +75,11 @@ def try_network_action(action: Callable[..., Any]) -> Any:
             time.sleep(2 ** i)
             continue
         except BaseError as err:
-            if regex.search(r"try again later", err.message) or regex.search(r"status code=5\d\d", err.message):
+            in_500_range = False
+            if err.status_code:
+                in_500_range = 500 <= err.status_code < 600
+            try_again_later = "try again later" in err.message
+            if try_again_later or in_500_range:
                 print(f"{datetime.now().isoformat()}: Try reconnecting to DSP server, next attempt in {2 ** i} seconds...")
                 logger.error(f"Try reconnecting to DSP server, next attempt in {2 ** i} seconds...", exc_info=True)
                 time.sleep(2 ** i)