feat: save warnings and errors in second file with loguru (DEV-3406) (#…

…883)

.gitignore

@@ -63,3 +63,4 @@ testdata/preprocessed_files/
 testdata/bitstreams/nested
 processing_result_*.pkl
 processed_files.txt
+/warnings-*.log

pyproject.toml

@@ -36,6 +36,7 @@ pyyaml = "^6.0.1"
 rustworkx = "^0.14.0"
 termcolor = "^2.4.0"
 packaging = "^23.2"
+loguru = "^0.7.2"
 
 
 [tool.poetry.group.dev.dependencies]

src/dsp_tools/cli/call_action.py

@@ -1,6 +1,8 @@
 import argparse
 from pathlib import Path
 
+from loguru import logger
+
 from dsp_tools.commands.excel2json.lists import excel2lists
 from dsp_tools.commands.excel2json.lists import validate_lists_section_with_schema
 from dsp_tools.commands.excel2json.project import excel2json
@@ -20,11 +22,8 @@
 from dsp_tools.commands.template import generate_template_repo
 from dsp_tools.commands.xmlupload.upload_config import UploadConfig
 from dsp_tools.commands.xmlupload.xmlupload import xmlupload
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.xml_validation import validate_xml
 
-logger = get_logger(__name__)
-
 
 def call_requested_action(args: argparse.Namespace) -> bool:
     """

src/dsp_tools/cli/entry_point.py

@@ -9,6 +9,7 @@
 
 import regex
 import requests
+from loguru import logger
 from packaging.version import parse
 from termcolor import colored
 
@@ -17,15 +18,14 @@
 from dsp_tools.models.exceptions import BaseError
 from dsp_tools.models.exceptions import InternalError
 from dsp_tools.models.exceptions import UserError
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
+from dsp_tools.utils.logger_config import logger_config
 
 
 def main() -> None:
     """
     Main entry point of the program as referenced in pyproject.toml
     """
+    logger_config()
     run(sys.argv[1:])
 
 
@@ -71,7 +71,7 @@ def run(args: list[str]) -> None:
         print("\nThe process was terminated because of an Error:")
         print(err.message)
         sys.exit(1)
-    except Exception as err:
+    except Exception as err:  # noqa: BLE001 (blind-except)
         logger.exception(err)
         print(InternalError())
         sys.exit(1)

src/dsp_tools/commands/excel2xml/excel2xml_cli.py

@@ -515,8 +515,7 @@ def excel2xml(
             dataframe=dataframe,
             max_num_of_props=max_num_of_props,
         )
-        for resource in resources:
-            root.append(resource)
+        root.extend(resources)
         write_xml(root, output_file)
         if len(catched_warnings) > 0:
             success = False

src/dsp_tools/commands/id2iri.py

@@ -5,14 +5,12 @@
 from pathlib import Path
 
 import regex
+from loguru import logger
 from lxml import etree
 
 from dsp_tools.models.exceptions import UserError
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.xml_utils import parse_and_clean_xml_file
 
-logger = get_logger(__name__)
-
 
 def _check_input_parameters(
     xml_file: str,

src/dsp_tools/commands/ingest_xmlupload/apply_ingest_id.py

@@ -5,13 +5,11 @@
 from typing import cast
 
 import pandas as pd
+from loguru import logger
 from lxml import etree
 
 from dsp_tools.commands.ingest_xmlupload.user_information import IngestInformation
 from dsp_tools.models.exceptions import InputError
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 def get_mapping_dict_from_file(shortcode: str) -> dict[str, str]:

src/dsp_tools/commands/ingest_xmlupload/upload_xml.py

@@ -2,18 +2,16 @@
 
 from pathlib import Path
 
+from loguru import logger
 from lxml import etree
 
 from dsp_tools.commands.ingest_xmlupload.apply_ingest_id import get_mapping_dict_from_file
 from dsp_tools.commands.ingest_xmlupload.apply_ingest_id import replace_filepath_with_sipi_id
 from dsp_tools.commands.xmlupload.upload_config import UploadConfig
 from dsp_tools.commands.xmlupload.xmlupload import xmlupload
 from dsp_tools.models.exceptions import InputError
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.xml_utils import remove_comments_from_element_tree
 
-logger = get_logger(__name__)
-
 
 def ingest_xmlupload(
     xml_file: Path,

src/dsp_tools/commands/project/create/project_create.py

@@ -8,6 +8,7 @@
 from typing import cast
 
 import regex
+from loguru import logger
 
 from dsp_tools.commands.excel2json.lists import expand_lists_from_excel
 from dsp_tools.commands.project.create.project_create_lists import create_lists_on_server
@@ -27,11 +28,8 @@
 from dsp_tools.models.langstring import LangString
 from dsp_tools.utils.connection import Connection
 from dsp_tools.utils.connection_live import ConnectionLive
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.shared import parse_json_input
 
-logger = get_logger(__name__)
-
 
 def _create_project_on_server(
     project_definition: ProjectDefinition,
@@ -79,7 +77,7 @@ def _create_project_on_server(
             f"Cannot create project '{project_definition.shortname}' "
             f"({project_definition.shortcode}) on DSP server."
         )
-        logger.error(err_msg, exc_info=True)
+        logger.opt(exception=True).error(err_msg)
         raise UserError(err_msg) from None
     print(f"    Created project '{project_remote.shortname}' ({project_remote.shortcode}).")
     logger.info(f"Created project '{project_remote.shortname}' ({project_remote.shortcode}).")
@@ -119,7 +117,7 @@ def _create_groups(
             "not possible to retrieve the remote groups from the DSP server."
         )
         print(f"WARNING: {err_msg}")
-        logger.warning(err_msg, exc_info=True)
+        logger.opt(exception=True).warning(err_msg)
         remote_groups = []
         overall_success = False
 
@@ -131,7 +129,7 @@ def _create_groups(
             current_project_groups[group_name] = remotely_existing_group[0]
             err_msg = f"Group name '{group_name}' already exists on the DSP server. Skipping..."
             print(f"    WARNING: {err_msg}")
-            logger.warning(err_msg, exc_info=True)
+            logger.opt(exception=True).warning(err_msg)
             overall_success = False
             continue
 
@@ -149,7 +147,7 @@ def _create_groups(
         except BaseError:
             err_msg = "Unable to create group '{group_name}'."
             print(f"    WARNING: {err_msg}")
-            logger.warning(err_msg, exc_info=True)
+            logger.opt(exception=True).warning(err_msg)
             overall_success = False
             continue
 
@@ -198,7 +196,7 @@ def _get_group_iris_for_user(
         )
         if ":" not in full_group_name and full_group_name != "SystemAdmin":
             print(f"    WARNING: {inexisting_group_msg}")
-            logger.warning(inexisting_group_msg, exc_info=True)
+            logger.opt(exception=True).warning(inexisting_group_msg)
             success = False
             continue
 
@@ -215,7 +213,7 @@ def _get_group_iris_for_user(
             # full_group_name refers to a group inside the same project
             if group_name not in current_project_groups:
                 print(f"    WARNING: {inexisting_group_msg}")
-                logger.warning(inexisting_group_msg, exc_info=True)
+                logger.opt(exception=True).warning(inexisting_group_msg)
                 success = False
                 continue
             group = current_project_groups[group_name]
@@ -230,13 +228,13 @@ def _get_group_iris_for_user(
                     f"exists on the DSP server, but no groups could be retrieved from the DSP server."
                 )
                 print(f"    WARNING: {err_msg}")
-                logger.warning(err_msg, exc_info=True)
+                logger.opt(exception=True).warning(err_msg)
                 success = False
                 continue
             existing_group = [g for g in remote_groups if g.project == current_project.iri and g.name == group_name]
             if not existing_group:
                 print(f"    WARNING: {inexisting_group_msg}")
-                logger.warning(inexisting_group_msg, exc_info=True)
+                logger.opt(exception=True).warning(inexisting_group_msg)
                 success = False
                 continue
             group = existing_group[0]
@@ -278,7 +276,7 @@ def _get_projects_where_user_is_admin(
         if ":" not in full_project_name:
             err_msg = "Provided project '{full_project_name}' for user '{username}' is not valid. Skipping..."
             print(f"    WARNING: {err_msg}")
-            logger.warning(err_msg, exc_info=True)
+            logger.opt(exception=True).warning(err_msg)
             success = False
             continue
 
@@ -297,14 +295,14 @@ def _get_projects_where_user_is_admin(
                     f"because the projects cannot be retrieved from the DSP server."
                 )
                 print(f"    WARNING: {err_msg}")
-                logger.warning(err_msg, exc_info=True)
+                logger.opt(exception=True).warning(err_msg)
                 success = False
                 continue
             in_project_list = [p for p in remote_projects if p.shortname == project_name]
             if not in_project_list:
                 msg = f"Provided project '{full_project_name}' for user '{username}' is not valid. Skipping..."
                 print(f"    WARNING: {msg}")
-                logger.warning(msg, exc_info=True)
+                logger.opt(exception=True).warning(msg)
                 success = False
                 continue
             in_project = in_project_list[0]
@@ -349,7 +347,7 @@ def _create_users(
         if json_user_definition["email"] in [user.email for user in all_users]:
             err_msg = f"User '{username}' already exists on the DSP server. Skipping..."
             print(f"    WARNING: {err_msg}")
-            logger.warning(err_msg, exc_info=True)
+            logger.opt(exception=True).warning(err_msg)
             overall_success = False
             continue
         # add user to the group(s)
@@ -391,7 +389,7 @@ def _create_users(
             user_local.create()
         except BaseError:
             print(f"    WARNING: Unable to create user '{username}'.")
-            logger.warning(f"Unable to create user '{username}'.", exc_info=True)
+            logger.opt(exception=True).warning(f"Unable to create user '{username}'.")
             overall_success = False
             continue
         print(f"    Created user '{username}'.")
@@ -506,7 +504,7 @@ def _create_ontology(
     if onto_name in [onto.name for onto in project_ontologies]:
         err_msg = f"Ontology '{onto_name}' already exists on the DSP server. Skipping..."
         print(f"    WARNING: {err_msg}")
-        logger.warning(err_msg, exc_info=True)
+        logger.opt(exception=True).warning(err_msg)
         return None
 
     print(f"Create ontology '{onto_name}'...")
@@ -522,7 +520,7 @@ def _create_ontology(
         ontology_remote = ontology_local.create()
     except BaseError:
         # if ontology cannot be created, let the error escalate
-        logger.error(f"ERROR while trying to create ontology '{onto_name}'.", exc_info=True)
+        logger.opt(exception=True).error(f"ERROR while trying to create ontology '{onto_name}'.")
         raise UserError(f"ERROR while trying to create ontology '{onto_name}'.") from None
 
     if verbose:
@@ -583,7 +581,7 @@ def _create_ontologies(
     except BaseError:
         err_msg = "Unable to retrieve remote ontologies. Cannot check if your ontology already exists."
         print("WARNING: {err_msg}")
-        logger.warning(err_msg, exc_info=True)
+        logger.opt(exception=True).warning(err_msg)
         project_ontologies = []
 
     for ontology_definition in ontology_definitions:
@@ -698,7 +696,7 @@ def _add_resource_classes_to_remote_ontology(
         except BaseError:
             err_msg = f"Unable to create resource class '{res_class['name']}'."
             print(f"WARNING: {err_msg}")
-            logger.warning(err_msg, exc_info=True)
+            logger.opt(exception=True).warning(err_msg)
             overall_success = False
 
     return last_modification_date, new_res_classes, overall_success
@@ -794,7 +792,7 @@ def _add_property_classes_to_remote_ontology(
         except BaseError:
             err_msg = f"Unable to create property class '{prop_class['name']}'."
             print(f"WARNING: {err_msg}")
-            logger.warning(f"Unable to create property class '{prop_class['name']}'.", exc_info=True)
+            logger.opt(exception=True).warning(f"Unable to create property class '{prop_class['name']}'.")
             overall_success = False
 
     return last_modification_date, overall_success
@@ -842,7 +840,7 @@ def _add_cardinalities_to_resource_classes(
                 f"This class doesn't exist on the DSP server."
             )
             print(f"WARNINIG: {msg}")
-            logger.warning(msg, exc_info=True)
+            logger.opt(exception=True).warning(msg)
             overall_success = False
             continue
         for card_info in res_class.get("cardinalities", []):
@@ -865,7 +863,7 @@ def _add_cardinalities_to_resource_classes(
             except BaseError:
                 err_msg = f"Unable to add cardinality '{qualified_propname}' to resource class {res_class['name']}."
                 print(f"WARNING: {err_msg}")
-                logger.warning(err_msg, exc_info=True)
+                logger.opt(exception=True).warning(err_msg)
                 overall_success = False
 
             ontology_remote.lastModificationDate = last_modification_date
@@ -918,7 +916,7 @@ def _rectify_hlist_of_properties(
                 f"which is not a valid list name. "
                 f"Assuming that you meant '{deduced_list_name}' instead."
             )
-            logger.warning(msg, exc_info=True)
+            logger.opt(exception=True).warning(msg)
             print(msg)
         else:
             msg = f"Property '{prop['name']}' references an unknown list: '{prop['gui_attributes']['hlist']}'"
@@ -1062,7 +1060,7 @@ def create_project(
             f"but during the creation process, some problems occurred. Please carefully check the console output."
         )
         print(f"========================================================\nWARNING: {msg}")
-        logger.warning(msg, exc_info=True)
+        logger.opt(exception=True).warning(msg)
 
     return overall_success
 

src/dsp_tools/commands/project/create/project_create_lists.py

@@ -2,6 +2,8 @@
 from typing import Optional
 from typing import Union
 
+from loguru import logger
+
 from dsp_tools.commands.excel2json.lists import expand_lists_from_excel
 from dsp_tools.commands.project.create.project_validate import validate_project
 from dsp_tools.commands.project.models.listnode import ListNode
@@ -10,11 +12,8 @@
 from dsp_tools.models.exceptions import UserError
 from dsp_tools.utils.connection import Connection
 from dsp_tools.utils.connection_live import ConnectionLive
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.shared import parse_json_input
 
-logger = get_logger(__name__)
-
 
 def _create_list_node(
     con: Connection,
@@ -55,7 +54,7 @@ def _create_list_node(
         new_node = new_node.create()
     except BaseError:
         print(f"WARNING: Cannot create list node '{node['name']}'.")
-        logger.warning("Cannot create list node '{node['name']}'.", exc_info=True)
+        logger.opt(exception=True).warning("Cannot create list node '{node['name']}'.")
         return {}, False
 
     # if node has child nodes, call the method recursively
@@ -106,7 +105,7 @@ def create_lists_on_server(
     except BaseError:
         err_msg = "Unable to retrieve existing lists on DSP server. Cannot check if your lists are already existing."
         print(f"WARNING: {err_msg}")
-        logger.warning(err_msg, exc_info=True)
+        logger.opt(exception=True).warning(err_msg)
         existing_lists = []
         overall_success = False
 
@@ -195,7 +194,7 @@ def create_lists(
         project_remote = project_local.read()
     except BaseError:
         err_msg = f"Unable to create the lists: The project {shortcode} cannot be found on the DSP server."
-        logger.error(err_msg, exc_info=True)
+        logger.opt(exception=True).error(err_msg)
         raise UserError(err_msg) from None
 
     # create new lists