diff --git a/pyproject.toml b/pyproject.toml index 54a541aa..17500fff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "defopt>=6.4.0", "email_validator", "hdx-python-country>=3.8.6", - "hdx-python-utilities>=3.8.0", + "hdx-python-utilities>=3.8.2", "libhxl>=5.2.2", "makefun", "quantulum3", diff --git a/requirements.txt b/requirements.txt index c5d18300..e935b77b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ gspread==6.1.4 # via hdx-python-api (pyproject.toml) hdx-python-country==3.8.6 # via hdx-python-api (pyproject.toml) -hdx-python-utilities==3.8.0 +hdx-python-utilities==3.8.2 # via # hdx-python-api (pyproject.toml) # hdx-python-country diff --git a/src/hdx/api/utilities/hdx_error_handler.py b/src/hdx/api/utilities/hdx_error_handler.py index b4c6e4d1..54480048 100644 --- a/src/hdx/api/utilities/hdx_error_handler.py +++ b/src/hdx/api/utilities/hdx_error_handler.py @@ -18,15 +18,18 @@ class HDXErrorHandler(ErrorHandler): sorted. Args: - should_exit_on_error (bool): Whether to exit with a 1 code if there are errors. Default is True. + should_exit_on_error (bool): Whether to exit with a 1 code if there are errors. Default is False. + write_to_hdx (bool): Whether to write errors to HDX resources. Default is False. """ def __init__( self, - should_exit_on_error: bool = True, + should_exit_on_error: bool = False, + write_to_hdx: bool = False, ): super().__init__(should_exit_on_error) + self._write_to_hdx = write_to_hdx self.shared_errors["hdx_error"] = {} @staticmethod @@ -81,6 +84,7 @@ def add_message( fixed format: pipeline - identifier - {text} identifier is usually a dataset name. + Args: pipeline (str): Name of the pipeline originating the error identifier (str): Identifier e.g. dataset name @@ -111,6 +115,7 @@ def add_missing_value_message( to a dictionary of messages in a fixed format: pipeline - identifier - {text} identifier is usually a dataset name. + Args: pipeline (str): Name of the scaper originating the error identifier (str): Identifier e.g. dataset name @@ -148,6 +153,7 @@ def add_multi_valued_message( pipeline - identifier - n {text}. First 10 values: n1,n2,n3... If less than 10 values, ". First 10 values" is omitted. identifier is usually a dataset name. + Args: pipeline (str): Name of the scaper originating the error identifier (str): Identifier e.g. dataset name @@ -172,22 +178,46 @@ def add_multi_valued_message( ) return True - def output_errors(self, err_to_hdx: bool) -> None: + def write_errors_to_hdx(self) -> None: + """ + Write to HDX resources corresponding errors that have been flagged by + setting err_to_hdx True when adding messages + + Returns: + None + """ + logger.info("Writing errors to HDX") + for identifier, errors in self.shared_errors["hdx_error"].items(): + write_errors_to_resource(identifier, errors) + + def output_errors(self) -> None: + """ + Log errors and warnings by category and sorted. Also write to HDX + resources corresponding errors that have been flagged by setting + err_to_hdx True when adding messages. + + Returns: + None + """ self.log() - if err_to_hdx: - logger.info("Writing errors to HDX") - for identifier, errors in self.shared_errors["hdx_error"].items(): - write_error_to_resource(identifier, errors) + if self._write_to_hdx: + self.write_errors_to_hdx() + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + if self._write_to_hdx: + self.write_errors_to_hdx() + super().__exit__(exc_type, exc_value, traceback) -def write_error_to_resource( + +def write_errors_to_resource( identifier: Tuple[str, str, str], errors: set[str] ) -> bool: """ Writes error messages to a resource on HDX. If the resource already has an error message, it is only overwritten if the two messages are different. + Args: - identifier (Tuple[str, str, str]): The scraper, dataset, and resource names that the message applies to + identifier (Tuple[str, str, str]): Scraper, dataset, and resource names that the message applies to errors (set[str]): Set of errors to use e.g. "negative values removed" Returns: bool: True if a message was added, False if not diff --git a/tests/hdx/utilities/test_hdx_error_handler.py b/tests/hdx/utilities/test_hdx_error_handler.py index a0fb9c93..2cb22c4f 100644 --- a/tests/hdx/utilities/test_hdx_error_handler.py +++ b/tests/hdx/utilities/test_hdx_error_handler.py @@ -14,7 +14,7 @@ class TestHDXErrorHandler: def test_hdx_error_handler(self, caplog): with pytest.raises(SystemExit): with caplog.at_level(logging.ERROR): - with HDXErrorHandler() as errors: + with HDXErrorHandler(should_exit_on_error=True) as errors: errors.add_message( "pipeline1", "dataset1", @@ -54,5 +54,5 @@ def test_hdx_error_handler(self, caplog): ) == 3 ) - errors.output_errors(False) + errors.output_errors() assert "following values changed" in caplog.text