diff --git a/ocp_resources/deployment.py b/ocp_resources/deployment.py index 5c75f21993..4dd790e684 100644 --- a/ocp_resources/deployment.py +++ b/ocp_resources/deployment.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ocp_resources.constants import PROTOCOL_ERROR_EXCEPTION_DICT, TIMEOUT_4MINUTES from ocp_resources.resource import NamespacedResource -from ocp_resources.utils import TimeoutSampler +from ocp_resources.utils import TimeoutSampler, TimeoutWatch class Deployment(NamespacedResource): @@ -39,9 +39,19 @@ def wait_for_replicas(self, deployed=True, timeout=TIMEOUT_4MINUTES): TimeoutExpiredError: If not availableReplicas is equal to replicas. """ self.logger.info(f"Wait for {self.kind} {self.name} to be deployed: {deployed}") - samples = TimeoutSampler( + + timeout_watcher = TimeoutWatch(timeout=timeout) + for sample in TimeoutSampler( wait_timeout=timeout, sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + samples = TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, exceptions_dict=PROTOCOL_ERROR_EXCEPTION_DICT, func=lambda: self.instance, ) diff --git a/ocp_resources/node_network_configuration_policy.py b/ocp_resources/node_network_configuration_policy.py index e066cd40a0..7b0e08fd68 100644 --- a/ocp_resources/node_network_configuration_policy.py +++ b/ocp_resources/node_network_configuration_policy.py @@ -9,7 +9,7 @@ ) from ocp_resources.node_network_state import NodeNetworkState from ocp_resources.resource import Resource, ResourceEditor -from ocp_resources.utils import TimeoutExpiredError, TimeoutSampler +from ocp_resources.utils import TimeoutExpiredError, TimeoutSampler, TimeoutWatch IPV4_STR = "ipv4" IPV6_STR = "ipv6" @@ -355,9 +355,18 @@ def status(self): return condition["reason"] def wait_for_configuration_conditions_unknown_or_progressing(self, wait_timeout=30): - samples = TimeoutSampler( + timeout_watcher = TimeoutWatch(timeout=wait_timeout) + for sample in TimeoutSampler( wait_timeout=wait_timeout, sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + samples = TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, func=lambda: self.instance.status.conditions, ) for sample in samples: diff --git a/ocp_resources/resource.py b/ocp_resources/resource.py index 0c30270fb3..49cff1fd70 100644 --- a/ocp_resources/resource.py +++ b/ocp_resources/resource.py @@ -31,6 +31,7 @@ from ocp_resources.utils import ( TimeoutExpiredError, TimeoutSampler, + TimeoutWatch, skip_existing_resource_creation_teardown, ) @@ -800,16 +801,19 @@ def update_replace(self, resource_dict): def retry_cluster_exceptions( func, exceptions_dict=DEFAULT_CLUSTER_RETRY_EXCEPTIONS, **kwargs ): - sampler = TimeoutSampler( - wait_timeout=10, - sleep=1, - func=func, - print_log=False, - exceptions_dict=exceptions_dict, - **kwargs, - ) - for sample in sampler: - return sample + try: + sampler = TimeoutSampler( + wait_timeout=10, + sleep=1, + func=func, + print_log=False, + exceptions_dict=exceptions_dict, + **kwargs, + ) + for sample in sampler: + return sample + except TimeoutExpiredError: + return None @classmethod def get( @@ -915,9 +919,18 @@ def wait_for_condition(self, condition, status, timeout=300): f" '{status}'" ) + timeout_watcher = TimeoutWatch(timeout=timeout) for sample in TimeoutSampler( wait_timeout=timeout, sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + for sample in TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, func=lambda: self.instance, ): if sample: @@ -952,8 +965,19 @@ def api_request(self, method, action, url, **params): return response.data def wait_for_conditions(self): + timeout_watcher = TimeoutWatch(timeout=30) + for sample in TimeoutSampler( + wait_timeout=30, + sleep=1, + func=lambda: self.exists, + ): + if sample: + break + samples = TimeoutSampler( - wait_timeout=30, sleep=1, func=lambda: self.instance.status.conditions + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, + func=lambda: self.instance.status.conditions, ) for sample in samples: if sample: diff --git a/ocp_resources/sriov_network_node_state.py b/ocp_resources/sriov_network_node_state.py index d7118d03e0..d345a0a5f4 100644 --- a/ocp_resources/sriov_network_node_state.py +++ b/ocp_resources/sriov_network_node_state.py @@ -1,5 +1,5 @@ from ocp_resources.resource import NamespacedResource -from ocp_resources.utils import TimeoutExpiredError, TimeoutSampler +from ocp_resources.utils import TimeoutExpiredError, TimeoutSampler, TimeoutWatch class SriovNetworkNodeState(NamespacedResource): @@ -30,8 +30,17 @@ def wait_for_status_sync(self, wanted_status, timeout=1000): f"Wait for {self.kind} {self.name} status to be {wanted_status}" ) try: + timeout_watcher = TimeoutWatch(timeout=timeout) for sample in TimeoutSampler( wait_timeout=timeout, + sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + for sample in TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), sleep=3, func=lambda: self.instance.status.syncStatus, ): diff --git a/ocp_resources/utils.py b/ocp_resources/utils.py index db80c3017a..70186f3048 100644 --- a/ocp_resources/utils.py +++ b/ocp_resources/utils.py @@ -138,9 +138,7 @@ def __iter__(self): except Exception as exp: last_exp = exp - last_exp_log = self._get_exception_log(exp=last_exp) if self._is_raisable_exception(exp=last_exp): - LOGGER.error(last_exp_log) raise TimeoutExpiredError(self._get_exception_log(exp=last_exp)) self.elapsed_time = None diff --git a/ocp_resources/virtual_machine_restore.py b/ocp_resources/virtual_machine_restore.py index 213c263a56..fe513fc249 100644 --- a/ocp_resources/virtual_machine_restore.py +++ b/ocp_resources/virtual_machine_restore.py @@ -4,7 +4,7 @@ from ocp_resources.constants import PROTOCOL_ERROR_EXCEPTION_DICT, TIMEOUT_4MINUTES from ocp_resources.resource import NamespacedResource -from ocp_resources.utils import TimeoutSampler +from ocp_resources.utils import TimeoutSampler, TimeoutWatch from ocp_resources.virtual_machine import VirtualMachine @@ -65,9 +65,18 @@ def wait_complete(self, status=True, timeout=TIMEOUT_4MINUTES): f"Wait for {self.kind} {self.name} status to be complete = {status}" ) - samples = TimeoutSampler( + timeout_watcher = TimeoutWatch(timeout=timeout) + for sample in TimeoutSampler( wait_timeout=timeout, sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + samples = TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, exceptions_dict=PROTOCOL_ERROR_EXCEPTION_DICT, func=lambda: self.instance.get("status", {}).get("complete") == status, ) diff --git a/ocp_resources/virtual_machine_snapshot.py b/ocp_resources/virtual_machine_snapshot.py index 8a10a00a91..1cb5f18442 100644 --- a/ocp_resources/virtual_machine_snapshot.py +++ b/ocp_resources/virtual_machine_snapshot.py @@ -4,7 +4,7 @@ from ocp_resources.constants import PROTOCOL_ERROR_EXCEPTION_DICT, TIMEOUT_4MINUTES from ocp_resources.resource import NamespacedResource -from ocp_resources.utils import TimeoutSampler +from ocp_resources.utils import TimeoutSampler, TimeoutWatch from ocp_resources.virtual_machine import VirtualMachine @@ -63,9 +63,18 @@ def wait_ready_to_use(self, status=True, timeout=TIMEOUT_4MINUTES): f" {'' if status else 'not '}ready to use" ) - samples = TimeoutSampler( + timeout_watcher = TimeoutWatch(timeout=timeout) + for sample in TimeoutSampler( wait_timeout=timeout, sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + samples = TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, exceptions_dict=PROTOCOL_ERROR_EXCEPTION_DICT, func=lambda: self.instance.get("status", {}).get("readyToUse") == status, )