[DPE-1712] Enabled the user to fix network issues and rerun stanza related hooks

src/backups.py

@@ -125,8 +125,13 @@ def can_use_s3_repository(self) -> Tuple[bool, Optional[str]]:
                 timeout=30,
             )
         except TimeoutExpired as e:
-            logger.error(str(e))
-            return False, FAILED_TO_INITIALIZE_STANZA_ERROR_MESSAGE
+            # Raise an error if the connection timeouts, so the user has the possibility to
+            # fix network issues and call juju resolve to re-trigger the hook that calls
+            # this method.
+            logger.error(
+                f"error: {str(e)} - please fix the error and call juju resolve on this unit"
+            )
+            raise TimeoutError
 
         else:
             if return_code != 0:
@@ -195,6 +200,14 @@ def _create_bucket_if_not_exists(self) -> None:
             bucket.meta.client.head_bucket(Bucket=bucket_name)
             logger.info("Bucket %s exists.", bucket_name)
             exists = True
+        except botocore.exceptions.ConnectTimeoutError as e:
+            # Re-raise the error if the connection timeouts, so the user has the possibility to
+            # fix network issues and call juju resolve to re-trigger the hook that calls
+            # this method.
+            logger.error(
+                f"error: {str(e)} - please fix the error and call juju resolve on this unit"
+            )
+            raise e
         except ClientError:
             logger.warning("Bucket %s doesn't exist or you don't have access to it.", bucket_name)
             exists = False
@@ -351,6 +364,14 @@ def _initialise_stanza(self) -> None:
                 "stanza-create",
             ]
         )
+        if return_code == 49:
+            # Raise an error if the connection timeouts, so the user has the possibility to
+            # fix network issues and call juju resolve to re-trigger the hook that calls
+            # this method.
+            logger.error(
+                f"error: {stderr} - please fix the error and call juju resolve on this unit"
+            )
+            raise TimeoutError
         if return_code != 0:
             logger.error(stderr)
             self.charm.unit.status = BlockedStatus(FAILED_TO_INITIALIZE_STANZA_ERROR_MESSAGE)

tests/unit/test_backups.py

@@ -6,6 +6,7 @@
 from typing import OrderedDict
 from unittest.mock import ANY, MagicMock, PropertyMock, call, mock_open, patch
 
+import botocore as botocore
 from boto3.exceptions import S3UploadFailedError
 from botocore.exceptions import ClientError
 from jinja2 import Template
@@ -167,10 +168,8 @@ def test_can_use_s3_repository(
 
         # Test when nothing is returned from the pgBackRest info command.
         _execute_command.side_effect = TimeoutExpired(cmd="fake command", timeout=30)
-        self.assertEqual(
-            self.charm.backup.can_use_s3_repository(),
-            (False, FAILED_TO_INITIALIZE_STANZA_ERROR_MESSAGE),
-        )
+        with self.assertRaises(TimeoutError):
+            self.charm.backup.can_use_s3_repository()
 
         _execute_command.side_effect = None
         _execute_command.return_value = (1, "", "")
@@ -331,6 +330,18 @@ def test_create_bucket_if_not_exists(self, _retrieve_s3_parameters, _resource):
         create.assert_called_once()
         wait_until_exists.assert_not_called()
 
+        # Test when the bucket creation fails due to a timeout error.
+        head_bucket.reset_mock()
+        create.reset_mock()
+        head_bucket.side_effect = botocore.exceptions.ConnectTimeoutError(
+            endpoint_url="fake endpoint URL"
+        )
+        with self.assertRaises(botocore.exceptions.ConnectTimeoutError):
+            self.charm.backup._create_bucket_if_not_exists()
+        head_bucket.assert_called_once()
+        create.assert_not_called()
+        wait_until_exists.assert_not_called()
+
     @patch("shutil.rmtree")
     @patch("pathlib.Path.is_dir")
     @patch("pathlib.Path.exists")
@@ -533,6 +544,12 @@ def test_initialise_stanza(
             # Assert there is no stanza name in the application relation databag.
             self.assertEqual(self.harness.get_relation_data(self.peer_rel_id, self.charm.app), {})
 
+        # Test when the failure in the stanza creation is due to a timeout.
+        _execute_command.reset_mock()
+        _execute_command.return_value = (49, "", "fake stderr")
+        with self.assertRaises(TimeoutError):
+            self.charm.backup._initialise_stanza()
+
         # Test when the stanza creation succeeds, but the archiving is not working correctly
         # (pgBackRest check command fails).
         _execute_command.reset_mock()