diff --git a/packages/jumpstarter/jumpstarter/client/lease.py b/packages/jumpstarter/jumpstarter/client/lease.py index 5bf087ade..289f5efce 100644 --- a/packages/jumpstarter/jumpstarter/client/lease.py +++ b/packages/jumpstarter/jumpstarter/client/lease.py @@ -23,6 +23,7 @@ from grpc.aio import AioRpcError, Channel from jumpstarter_protocol import jumpstarter_pb2, jumpstarter_pb2_grpc from rich.console import Console +from tenacity import retry, retry_if_exception_type, wait_exponential_jitter from .exceptions import LeaseError from jumpstarter.client import client_from_path @@ -77,6 +78,24 @@ async def get(self): svc = ClientService(channel=self.channel, namespace=self.namespace) return await svc.GetLease(name=self.name) + @retry( + wait=wait_exponential_jitter(initial=1, max=120, jitter=1), + retry=retry_if_exception_type(ConnectionError), + reraise=True, + ) + async def _get_with_retry(self): + """Get lease with exponential backoff retry on ConnectionError. + + Retries with exponential backoff and jitter indefinitely when ConnectionError occurs. + The wait time between retries is capped at 2 minutes (120 seconds). + Jitter helps prevent thundering herd problems when multiple clients retry simultaneously. + """ + try: + return await self.get() + except ConnectionError as e: + logger.error("Error while getting lease %s: %s", self.name, e) + raise + def request(self): """Request a lease, or verifies a lease which was already created. @@ -136,8 +155,7 @@ async def _acquire(self): with LeaseAcquisitionSpinner(self.name) as spinner: while True: logger.debug("Polling Lease %s", self.name) - result = await self.get() - + result = await self._get_with_retry() # lease ready if condition_true(result.conditions, "Ready"): logger.debug("Lease %s acquired", self.name) diff --git a/packages/jumpstarter/pyproject.toml b/packages/jumpstarter/pyproject.toml index cb6a0f3cd..fbd573cd5 100644 --- a/packages/jumpstarter/pyproject.toml +++ b/packages/jumpstarter/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "xdg-base-dirs>=6.0.2", "pydantic-settings>=2.9.1", "rich>=14.0.0", + "tenacity>=8.2.0", ] [dependency-groups]