diff --git a/src/backups.py b/src/backups.py index 7aeb18f371..aaa68513e2 100644 --- a/src/backups.py +++ b/src/backups.py @@ -125,8 +125,13 @@ def can_use_s3_repository(self) -> Tuple[bool, Optional[str]]: timeout=30, ) except TimeoutExpired as e: - logger.error(str(e)) - return False, FAILED_TO_INITIALIZE_STANZA_ERROR_MESSAGE + # Raise an error if the connection timeouts, so the user has the possibility to + # fix network issues and call juju resolve to re-trigger the hook that calls + # this method. + logger.error( + f"error: {str(e)} - please fix the error and call juju resolve on this unit" + ) + raise TimeoutError else: if return_code != 0: @@ -195,6 +200,14 @@ def _create_bucket_if_not_exists(self) -> None: bucket.meta.client.head_bucket(Bucket=bucket_name) logger.info("Bucket %s exists.", bucket_name) exists = True + except botocore.exceptions.ConnectTimeoutError as e: + # Re-raise the error if the connection timeouts, so the user has the possibility to + # fix network issues and call juju resolve to re-trigger the hook that calls + # this method. + logger.error( + f"error: {str(e)} - please fix the error and call juju resolve on this unit" + ) + raise e except ClientError: logger.warning("Bucket %s doesn't exist or you don't have access to it.", bucket_name) exists = False @@ -351,6 +364,14 @@ def _initialise_stanza(self) -> None: "stanza-create", ] ) + if return_code == 49: + # Raise an error if the connection timeouts, so the user has the possibility to + # fix network issues and call juju resolve to re-trigger the hook that calls + # this method. + logger.error( + f"error: {stderr} - please fix the error and call juju resolve on this unit" + ) + raise TimeoutError if return_code != 0: logger.error(stderr) self.charm.unit.status = BlockedStatus(FAILED_TO_INITIALIZE_STANZA_ERROR_MESSAGE) diff --git a/tests/unit/test_backups.py b/tests/unit/test_backups.py index b5270c18fe..8c6c2a8e7a 100644 --- a/tests/unit/test_backups.py +++ b/tests/unit/test_backups.py @@ -6,6 +6,7 @@ from typing import OrderedDict from unittest.mock import ANY, MagicMock, PropertyMock, call, mock_open, patch +import botocore as botocore from boto3.exceptions import S3UploadFailedError from botocore.exceptions import ClientError from jinja2 import Template @@ -167,10 +168,8 @@ def test_can_use_s3_repository( # Test when nothing is returned from the pgBackRest info command. _execute_command.side_effect = TimeoutExpired(cmd="fake command", timeout=30) - self.assertEqual( - self.charm.backup.can_use_s3_repository(), - (False, FAILED_TO_INITIALIZE_STANZA_ERROR_MESSAGE), - ) + with self.assertRaises(TimeoutError): + self.charm.backup.can_use_s3_repository() _execute_command.side_effect = None _execute_command.return_value = (1, "", "") @@ -331,6 +330,18 @@ def test_create_bucket_if_not_exists(self, _retrieve_s3_parameters, _resource): create.assert_called_once() wait_until_exists.assert_not_called() + # Test when the bucket creation fails due to a timeout error. + head_bucket.reset_mock() + create.reset_mock() + head_bucket.side_effect = botocore.exceptions.ConnectTimeoutError( + endpoint_url="fake endpoint URL" + ) + with self.assertRaises(botocore.exceptions.ConnectTimeoutError): + self.charm.backup._create_bucket_if_not_exists() + head_bucket.assert_called_once() + create.assert_not_called() + wait_until_exists.assert_not_called() + @patch("shutil.rmtree") @patch("pathlib.Path.is_dir") @patch("pathlib.Path.exists") @@ -533,6 +544,12 @@ def test_initialise_stanza( # Assert there is no stanza name in the application relation databag. self.assertEqual(self.harness.get_relation_data(self.peer_rel_id, self.charm.app), {}) + # Test when the failure in the stanza creation is due to a timeout. + _execute_command.reset_mock() + _execute_command.return_value = (49, "", "fake stderr") + with self.assertRaises(TimeoutError): + self.charm.backup._initialise_stanza() + # Test when the stanza creation succeeds, but the archiving is not working correctly # (pgBackRest check command fails). _execute_command.reset_mock()