feat: save warnings and errors in second file with loguru (DEV-3406)

.gitignore

@@ -63,3 +63,4 @@ testdata/preprocessed_files/
 testdata/bitstreams/nested
 processing_result_*.pkl
 processed_files.txt
+/warnings.log

pyproject.toml

@@ -17,6 +17,7 @@ homepage = "https://www.dasch.swiss/"
 repository = "https://github.com/dasch-swiss/dsp-tools"
 include = [
     "src/dsp_tools/resources/*",
+    "src/dsp_tools/resources/logger_config.yml",
     "src/dsp_tools/py.typed"  # see https://mypy.readthedocs.io/en/stable/installed_packages.html#making-pep-561-compatible-packages
 ]
 
@@ -36,6 +37,8 @@ pyyaml = "^6.0.1"
 rustworkx = "^0.14.0"
 termcolor = "^2.4.0"
 packaging = "^23.2"
+loguru = "^0.7.2"
+loguru-config = "^0.1.0"
 
 
 [tool.poetry.group.dev.dependencies]

src/dsp_tools/cli/call_action.py

@@ -1,6 +1,8 @@
 import argparse
 from pathlib import Path
 
+from loguru import logger
+
 from dsp_tools.commands.excel2json.lists import excel2lists
 from dsp_tools.commands.excel2json.lists import validate_lists_section_with_schema
 from dsp_tools.commands.excel2json.project import excel2json
@@ -20,11 +22,8 @@
 from dsp_tools.commands.template import generate_template_repo
 from dsp_tools.commands.xmlupload.upload_config import UploadConfig
 from dsp_tools.commands.xmlupload.xmlupload import xmlupload
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.xml_validation import validate_xml
 
-logger = get_logger(__name__)
-
 
 def call_requested_action(args: argparse.Namespace) -> bool:
     """

src/dsp_tools/cli/entry_point.py

@@ -9,6 +9,7 @@
 
 import regex
 import requests
+from loguru import logger
 from packaging.version import parse
 from termcolor import colored
 
@@ -17,9 +18,11 @@
 from dsp_tools.models.exceptions import BaseError
 from dsp_tools.models.exceptions import InternalError
 from dsp_tools.models.exceptions import UserError
-from dsp_tools.utils.create_logger import get_logger
+from dsp_tools.utils.logger_config import logger_config
 
-logger = get_logger(__name__)
+# conf = LoguruConfig()
+# conf.load("src/dsp_tools/resources/logger_config.yml")
+# LOGFILES = conf.configure()
 
 
 def main() -> None:
@@ -71,7 +74,7 @@ def run(args: list[str]) -> None:
         print("\nThe process was terminated because of an Error:")
         print(err.message)
         sys.exit(1)
-    except Exception as err:
+    except Exception as err:  # noqa: BLE001 (blind-except)
         logger.exception(err)
         print(InternalError())
         sys.exit(1)
@@ -284,4 +287,5 @@ def _derive_sipi_url(
 
 
 if __name__ == "__main__":
+    logger_config()
     main()

src/dsp_tools/commands/id2iri.py

@@ -5,14 +5,12 @@
 from pathlib import Path
 
 import regex
+from loguru import logger
 from lxml import etree
 
 from dsp_tools.models.exceptions import UserError
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.xml_utils import parse_and_clean_xml_file
 
-logger = get_logger(__name__)
-
 
 def _check_input_parameters(
     xml_file: str,

src/dsp_tools/commands/ingest_xmlupload/apply_ingest_id.py

@@ -5,13 +5,11 @@
 from typing import cast
 
 import pandas as pd
+from loguru import logger
 from lxml import etree
 
 from dsp_tools.commands.ingest_xmlupload.user_information import IngestInformation
 from dsp_tools.models.exceptions import InputError
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 def get_mapping_dict_from_file(shortcode: str) -> dict[str, str]:

src/dsp_tools/commands/ingest_xmlupload/upload_xml.py

@@ -2,18 +2,16 @@
 
 from pathlib import Path
 
+from loguru import logger
 from lxml import etree
 
 from dsp_tools.commands.ingest_xmlupload.apply_ingest_id import get_mapping_dict_from_file
 from dsp_tools.commands.ingest_xmlupload.apply_ingest_id import replace_filepath_with_sipi_id
 from dsp_tools.commands.xmlupload.upload_config import UploadConfig
 from dsp_tools.commands.xmlupload.xmlupload import xmlupload
 from dsp_tools.models.exceptions import InputError
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.xml_utils import remove_comments_from_element_tree
 
-logger = get_logger(__name__)
-
 
 def ingest_xmlupload(
     xml_file: Path,

src/dsp_tools/commands/project/create/project_create.py

@@ -8,6 +8,7 @@
 from typing import cast
 
 import regex
+from loguru import logger
 
 from dsp_tools.commands.excel2json.lists import expand_lists_from_excel
 from dsp_tools.commands.project.create.project_create_lists import create_lists_on_server
@@ -27,11 +28,8 @@
 from dsp_tools.models.langstring import LangString
 from dsp_tools.utils.connection import Connection
 from dsp_tools.utils.connection_live import ConnectionLive
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.shared import parse_json_input
 
-logger = get_logger(__name__)
-
 
 def _create_project_on_server(
     project_definition: ProjectDefinition,

src/dsp_tools/commands/project/create/project_create_lists.py

@@ -2,6 +2,8 @@
 from typing import Optional
 from typing import Union
 
+from loguru import logger
+
 from dsp_tools.commands.excel2json.lists import expand_lists_from_excel
 from dsp_tools.commands.project.create.project_validate import validate_project
 from dsp_tools.commands.project.models.listnode import ListNode
@@ -10,11 +12,8 @@
 from dsp_tools.models.exceptions import UserError
 from dsp_tools.utils.connection import Connection
 from dsp_tools.utils.connection_live import ConnectionLive
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.shared import parse_json_input
 
-logger = get_logger(__name__)
-
 
 def _create_list_node(
     con: Connection,

src/dsp_tools/commands/resume_xmlupload/resume_xmlupload.py

@@ -3,6 +3,7 @@
 from copy import deepcopy
 from dataclasses import replace
 
+from loguru import logger
 from termcolor import colored
 
 from dsp_tools.commands.xmlupload.iri_resolver import IriResolver
@@ -16,9 +17,6 @@
 from dsp_tools.commands.xmlupload.xmlupload import cleanup_upload
 from dsp_tools.commands.xmlupload.xmlupload import upload_resources
 from dsp_tools.utils.connection_live import ConnectionLive
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 def resume_xmlupload(server: str, user: str, password: str, sipi: str, skip_first_resource: bool = False) -> bool:

src/dsp_tools/commands/start_stack.py

@@ -9,11 +9,9 @@
 import regex
 import requests
 import yaml
+from loguru import logger
 
 from dsp_tools.models.exceptions import UserError
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 @dataclass(frozen=True)

src/dsp_tools/commands/xmlupload/ark2iri.py

@@ -6,9 +6,6 @@
 import regex
 
 from dsp_tools.models.exceptions import BaseError
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 def convert_ark_v0_to_resource_iri(ark: str) -> str:

src/dsp_tools/commands/xmlupload/list_client.py

@@ -7,10 +7,9 @@
 from typing import Protocol
 from urllib.parse import quote_plus
 
-from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.create_logger import get_logger
+from loguru import logger
 
-logger = get_logger(__name__)
+from dsp_tools.utils.connection import Connection
 
 
 @dataclass(frozen=True)

src/dsp_tools/commands/xmlupload/ontology_client.py

@@ -6,9 +6,6 @@
 from dsp_tools.models.exceptions import BaseError
 from dsp_tools.models.exceptions import UserError
 from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 @dataclass

src/dsp_tools/commands/xmlupload/project_client.py

@@ -6,9 +6,6 @@
 from dsp_tools.models.exceptions import BaseError
 from dsp_tools.models.exceptions import UserError
 from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 @dataclass(frozen=True)

src/dsp_tools/commands/xmlupload/read_validate_xml_file.py

@@ -5,16 +5,14 @@
 from typing import Union
 
 import regex
+from loguru import logger
 from lxml import etree
 
 from dsp_tools.models.exceptions import UserError
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.iri_util import is_resource_iri
 from dsp_tools.utils.xml_utils import parse_and_clean_xml_file
 from dsp_tools.utils.xml_validation import validate_xml
 
-logger = get_logger(__name__)
-
 
 def validate_and_parse_xml_file(
     imgdir: str,

src/dsp_tools/commands/xmlupload/resource_create_client.py

@@ -4,6 +4,8 @@
 from typing import Any
 from typing import assert_never
 
+from loguru import logger
+
 from dsp_tools.commands.xmlupload.ark2iri import convert_ark_v0_to_resource_iri
 from dsp_tools.commands.xmlupload.iri_resolver import IriResolver
 from dsp_tools.commands.xmlupload.models.formatted_text_value import FormattedTextValue
@@ -15,12 +17,9 @@
 from dsp_tools.models.exceptions import BaseError
 from dsp_tools.models.exceptions import UserError
 from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.create_logger import get_logger
 from dsp_tools.utils.date_util import parse_date_string
 from dsp_tools.utils.iri_util import is_resource_iri
 
-logger = get_logger(__name__)
-
 
 @dataclass(frozen=True)
 class ResourceCreateClient:

src/dsp_tools/commands/xmlupload/resource_multimedia.py

@@ -3,15 +3,14 @@
 from datetime import datetime
 from pathlib import Path
 
+from loguru import logger
+
 from dsp_tools.commands.xmlupload.models.permission import Permissions
 from dsp_tools.commands.xmlupload.models.sipi import Sipi
 from dsp_tools.commands.xmlupload.models.xmlbitstream import XMLBitstream
 from dsp_tools.commands.xmlupload.models.xmlresource import BitstreamInfo
 from dsp_tools.commands.xmlupload.models.xmlresource import XMLResource
 from dsp_tools.models.exceptions import PermanentConnectionError
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 def handle_media_info(

src/dsp_tools/commands/xmlupload/stash/stash_circular_references.py

@@ -17,9 +17,6 @@
 from dsp_tools.commands.xmlupload.stash.stash_models import StandoffStash
 from dsp_tools.commands.xmlupload.stash.stash_models import StandoffStashItem
 from dsp_tools.commands.xmlupload.stash.stash_models import Stash
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 def _stash_standoff(

src/dsp_tools/commands/xmlupload/stash/upload_stashed_resptr_props.py

@@ -3,14 +3,13 @@
 from datetime import datetime
 from typing import Any
 
+from loguru import logger
+
 from dsp_tools.commands.xmlupload.iri_resolver import IriResolver
 from dsp_tools.commands.xmlupload.stash.stash_models import LinkValueStash
 from dsp_tools.commands.xmlupload.stash.stash_models import LinkValueStashItem
 from dsp_tools.models.exceptions import BaseError
 from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 def upload_stashed_resptr_props(

src/dsp_tools/commands/xmlupload/stash/upload_stashed_xml_texts.py

@@ -4,15 +4,14 @@
 from typing import Any
 from urllib.parse import quote_plus
 
+from loguru import logger
+
 from dsp_tools.commands.xmlupload.iri_resolver import IriResolver
 from dsp_tools.commands.xmlupload.models.formatted_text_value import FormattedTextValue
 from dsp_tools.commands.xmlupload.stash.stash_models import StandoffStash
 from dsp_tools.commands.xmlupload.stash.stash_models import StandoffStashItem
 from dsp_tools.models.exceptions import BaseError
 from dsp_tools.utils.connection import Connection
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
 
 
 def _log_unable_to_retrieve_resource(

src/dsp_tools/commands/xmlupload/upload_config.py

@@ -6,10 +6,7 @@
 from pathlib import Path
 
 import regex
-
-from dsp_tools.utils.create_logger import get_logger
-
-logger = get_logger(__name__)
+from loguru import logger
 
 
 def _transform_server_url_to_foldername(server: str) -> str:

src/dsp_tools/commands/xmlupload/write_diagnostic_info.py

@@ -3,10 +3,9 @@
 import json
 from datetime import datetime
 
-from dsp_tools.commands.xmlupload.upload_config import DiagnosticsConfig
-from dsp_tools.utils.create_logger import get_logger
+from loguru import logger
 
-logger = get_logger(__name__)
+from dsp_tools.commands.xmlupload.upload_config import DiagnosticsConfig
 
 
 def write_id2iri_mapping(