From 1697fed31e6a5ed70e63af1a28053c6ab2cf0b58 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Sun, 3 Sep 2023 13:19:54 +0300 Subject: [PATCH 1/3] Wait for resource exists before sampling for condition --- ocp_resources/deployment.py | 14 +++++++++-- .../node_network_configuration_policy.py | 13 +++++++++-- ocp_resources/resource.py | 23 ++++++++++++++++++- ocp_resources/sriov_network_node_state.py | 11 ++++++++- ocp_resources/virtual_machine_restore.py | 13 +++++++++-- ocp_resources/virtual_machine_snapshot.py | 13 +++++++++-- 6 files changed, 77 insertions(+), 10 deletions(-) diff --git a/ocp_resources/deployment.py b/ocp_resources/deployment.py index 5c75f21993..4dd790e684 100644 --- a/ocp_resources/deployment.py +++ b/ocp_resources/deployment.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from ocp_resources.constants import PROTOCOL_ERROR_EXCEPTION_DICT, TIMEOUT_4MINUTES from ocp_resources.resource import NamespacedResource -from ocp_resources.utils import TimeoutSampler +from ocp_resources.utils import TimeoutSampler, TimeoutWatch class Deployment(NamespacedResource): @@ -39,9 +39,19 @@ def wait_for_replicas(self, deployed=True, timeout=TIMEOUT_4MINUTES): TimeoutExpiredError: If not availableReplicas is equal to replicas. """ self.logger.info(f"Wait for {self.kind} {self.name} to be deployed: {deployed}") - samples = TimeoutSampler( + + timeout_watcher = TimeoutWatch(timeout=timeout) + for sample in TimeoutSampler( wait_timeout=timeout, sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + samples = TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, exceptions_dict=PROTOCOL_ERROR_EXCEPTION_DICT, func=lambda: self.instance, ) diff --git a/ocp_resources/node_network_configuration_policy.py b/ocp_resources/node_network_configuration_policy.py index 35cdcbe954..46b28f2b37 100644 --- a/ocp_resources/node_network_configuration_policy.py +++ b/ocp_resources/node_network_configuration_policy.py @@ -9,7 +9,7 @@ ) from ocp_resources.node_network_state import NodeNetworkState from ocp_resources.resource import Resource, ResourceEditor -from ocp_resources.utils import TimeoutExpiredError, TimeoutSampler +from ocp_resources.utils import TimeoutExpiredError, TimeoutSampler, TimeoutWatch IPV4_STR = "ipv4" IPV6_STR = "ipv6" @@ -355,9 +355,18 @@ def status(self): return condition["reason"] def wait_for_configuration_conditions_unknown_or_progressing(self, wait_timeout=30): - samples = TimeoutSampler( + timeout_watcher = TimeoutWatch(timeout=wait_timeout) + for sample in TimeoutSampler( wait_timeout=wait_timeout, sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + samples = TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, func=lambda: self.instance.status.conditions, ) for sample in samples: diff --git a/ocp_resources/resource.py b/ocp_resources/resource.py index 0c30270fb3..ae9382a709 100644 --- a/ocp_resources/resource.py +++ b/ocp_resources/resource.py @@ -31,6 +31,7 @@ from ocp_resources.utils import ( TimeoutExpiredError, TimeoutSampler, + TimeoutWatch, skip_existing_resource_creation_teardown, ) @@ -915,9 +916,18 @@ def wait_for_condition(self, condition, status, timeout=300): f" '{status}'" ) + timeout_watcher = TimeoutWatch(timeout=timeout) for sample in TimeoutSampler( wait_timeout=timeout, sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + for sample in TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, func=lambda: self.instance, ): if sample: @@ -952,8 +962,19 @@ def api_request(self, method, action, url, **params): return response.data def wait_for_conditions(self): + timeout_watcher = TimeoutWatch(timeout=30) + for sample in TimeoutSampler( + wait_timeout=30, + sleep=1, + func=lambda: self.exists, + ): + if sample: + break + samples = TimeoutSampler( - wait_timeout=30, sleep=1, func=lambda: self.instance.status.conditions + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, + func=lambda: self.instance.status.conditions, ) for sample in samples: if sample: diff --git a/ocp_resources/sriov_network_node_state.py b/ocp_resources/sriov_network_node_state.py index d7118d03e0..d345a0a5f4 100644 --- a/ocp_resources/sriov_network_node_state.py +++ b/ocp_resources/sriov_network_node_state.py @@ -1,5 +1,5 @@ from ocp_resources.resource import NamespacedResource -from ocp_resources.utils import TimeoutExpiredError, TimeoutSampler +from ocp_resources.utils import TimeoutExpiredError, TimeoutSampler, TimeoutWatch class SriovNetworkNodeState(NamespacedResource): @@ -30,8 +30,17 @@ def wait_for_status_sync(self, wanted_status, timeout=1000): f"Wait for {self.kind} {self.name} status to be {wanted_status}" ) try: + timeout_watcher = TimeoutWatch(timeout=timeout) for sample in TimeoutSampler( wait_timeout=timeout, + sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + for sample in TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), sleep=3, func=lambda: self.instance.status.syncStatus, ): diff --git a/ocp_resources/virtual_machine_restore.py b/ocp_resources/virtual_machine_restore.py index b8c378170e..bde2cc86df 100644 --- a/ocp_resources/virtual_machine_restore.py +++ b/ocp_resources/virtual_machine_restore.py @@ -4,7 +4,7 @@ from ocp_resources.constants import PROTOCOL_ERROR_EXCEPTION_DICT, TIMEOUT_4MINUTES from ocp_resources.resource import NamespacedResource -from ocp_resources.utils import TimeoutSampler +from ocp_resources.utils import TimeoutSampler, TimeoutWatch from ocp_resources.virtual_machine import VirtualMachine @@ -65,9 +65,18 @@ def wait_complete(self, status=True, timeout=TIMEOUT_4MINUTES): f"Wait for {self.kind} {self.name} status to be complete = {status}" ) - samples = TimeoutSampler( + timeout_watcher = TimeoutWatch(timeout=timeout) + for sample in TimeoutSampler( wait_timeout=timeout, sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + samples = TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, exceptions_dict=PROTOCOL_ERROR_EXCEPTION_DICT, func=lambda: self.instance.get("status", {}).get("complete") == status, ) diff --git a/ocp_resources/virtual_machine_snapshot.py b/ocp_resources/virtual_machine_snapshot.py index 1235171536..11f7ba5b98 100644 --- a/ocp_resources/virtual_machine_snapshot.py +++ b/ocp_resources/virtual_machine_snapshot.py @@ -4,7 +4,7 @@ from ocp_resources.constants import PROTOCOL_ERROR_EXCEPTION_DICT, TIMEOUT_4MINUTES from ocp_resources.resource import NamespacedResource -from ocp_resources.utils import TimeoutSampler +from ocp_resources.utils import TimeoutSampler, TimeoutWatch from ocp_resources.virtual_machine import VirtualMachine @@ -63,9 +63,18 @@ def wait_ready_to_use(self, status=True, timeout=TIMEOUT_4MINUTES): f" {'' if status else 'not '}ready to use" ) - samples = TimeoutSampler( + timeout_watcher = TimeoutWatch(timeout=timeout) + for sample in TimeoutSampler( wait_timeout=timeout, sleep=1, + func=lambda: self.exists, + ): + if sample: + break + + samples = TimeoutSampler( + wait_timeout=timeout_watcher.remaining_time(), + sleep=1, exceptions_dict=PROTOCOL_ERROR_EXCEPTION_DICT, func=lambda: self.instance.get("status", {}).get("readyToUse") == status, ) From 564d825bffee3cfd6208b076bf99c16e71625804 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Sun, 3 Sep 2023 13:33:59 +0300 Subject: [PATCH 2/3] return None in self.instance --- ocp_resources/resource.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/ocp_resources/resource.py b/ocp_resources/resource.py index ae9382a709..49cff1fd70 100644 --- a/ocp_resources/resource.py +++ b/ocp_resources/resource.py @@ -801,16 +801,19 @@ def update_replace(self, resource_dict): def retry_cluster_exceptions( func, exceptions_dict=DEFAULT_CLUSTER_RETRY_EXCEPTIONS, **kwargs ): - sampler = TimeoutSampler( - wait_timeout=10, - sleep=1, - func=func, - print_log=False, - exceptions_dict=exceptions_dict, - **kwargs, - ) - for sample in sampler: - return sample + try: + sampler = TimeoutSampler( + wait_timeout=10, + sleep=1, + func=func, + print_log=False, + exceptions_dict=exceptions_dict, + **kwargs, + ) + for sample in sampler: + return sample + except TimeoutExpiredError: + return None @classmethod def get( From 80c7736365521f4a78a36bde1c1cd1bf3479b6ae Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Sun, 3 Sep 2023 13:44:13 +0300 Subject: [PATCH 3/3] TimeoutSampler: rempve dulicate exception log --- ocp_resources/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ocp_resources/utils.py b/ocp_resources/utils.py index db80c3017a..70186f3048 100644 --- a/ocp_resources/utils.py +++ b/ocp_resources/utils.py @@ -138,9 +138,7 @@ def __iter__(self): except Exception as exp: last_exp = exp - last_exp_log = self._get_exception_log(exp=last_exp) if self._is_raisable_exception(exp=last_exp): - LOGGER.error(last_exp_log) raise TimeoutExpiredError(self._get_exception_log(exp=last_exp)) self.elapsed_time = None