Skip to content

Add retry mechanism for Azure VM update API concurrent request conflicts #4216

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
LiliDeng merged 4 commits into from
Jan 14, 2026
+47 −16
Merged

Add retry mechanism for Azure VM update API concurrent request conflicts #4216

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
649c9cb
Initial plan
Copilot Jan 14, 2026
c969d14
Add retry mechanism to Azure VM update API calls
Copilot Jan 14, 2026
ddd680c
Add unit tests for retry mechanism
Copilot Jan 14, 2026
eb31abe
Remove test_retry_mechanism.py as requested
Copilot Jan 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 47 additions & 16 deletions lisa/sut_orchestrator/azure/features.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1037,6 +1037,23 @@ def is_enabled_sriov(self) -> bool:
sriov_enabled = primary_nic.enable_accelerated_networking
return sriov_enabled

@retry(ResourceExistsError, tries=5, delay=2, backoff=1.5) # type: ignore
def _update_vm_nics(
self,
compute_client: Any,
resource_group_name: str,
vm_name: str,
network_interfaces_section: List[Dict[str, Any]],
) -> None:
"""Helper method to update VM network interfaces with retry logic."""
compute_client.virtual_machines.begin_update(
resource_group_name=resource_group_name,
vm_name=vm_name,
parameters={
"network_profile": {"network_interfaces": network_interfaces_section},
},
)

def attach_nics(
self, extra_nic_count: int, enable_accelerated_networking: bool = True
) -> None:
Expand Down Expand Up @@ -1105,12 +1122,11 @@ def attach_nics(
network_interfaces_section.append({"id": primary_nic.id, "primary": True})

self._log.debug(f"start to attach the nics into VM {self._node.name}.")
compute_client.virtual_machines.begin_update(
resource_group_name=self._resource_group_name,
vm_name=self._node.name,
parameters={
"network_profile": {"network_interfaces": network_interfaces_section},
},
self._update_vm_nics(
compute_client,
self._resource_group_name,
self._node.name,
network_interfaces_section,
)
self._log.debug(f"attach the nics into VM {self._node.name} successfully.")
startstop.start()
Expand Down Expand Up @@ -1142,12 +1158,11 @@ def remove_extra_nics(self) -> None:
network_interfaces_section.append({"id": primary_nic.id, "primary": True})
startstop = self._node.features[StartStop]
startstop.stop()
compute_client.virtual_machines.begin_update(
resource_group_name=self._resource_group_name,
vm_name=self._node.name,
parameters={
"network_profile": {"network_interfaces": network_interfaces_section},
},
self._update_vm_nics(
compute_client,
self._resource_group_name,
self._node.name,
network_interfaces_section,
)
self._log.debug(
f"Only associated nic {primary_nic.id} into VM {self._node.name}."
Expand Down Expand Up @@ -2258,6 +2273,21 @@ def create_setting(
) -> Optional[schema.FeatureSettings]:
return schema.FeatureSettings.create(cls.name())

@retry(ResourceExistsError, tries=5, delay=2, backoff=1.5) # type: ignore
def _update_vm_size(
self,
compute_client: Any,
resource_group_name: str,
vm_name: str,
vm_update: VirtualMachineUpdate,
) -> Any:
"""Helper method to update VM size with retry logic."""
return compute_client.virtual_machines.begin_update(
resource_group_name=resource_group_name,
vm_name=vm_name,
parameters=vm_update,
)

def resize(
self, resize_action: ResizeAction = ResizeAction.IncreaseCoreCount
) -> Tuple[schema.NodeSpace, str, str]:
Expand All @@ -2271,10 +2301,11 @@ def resize(
vm_update = VirtualMachineUpdate(hardware_profile=hardware_profile)

# Resizing with new Vm Size
lro_poller = compute_client.virtual_machines.begin_update(
resource_group_name=node_context.resource_group_name,
vm_name=node_context.vm_name,
parameters=vm_update,
lro_poller = self._update_vm_size(
compute_client,
node_context.resource_group_name,
node_context.vm_name,
vm_update,
)

# Waiting for the Long Running Operation to finish
Expand Down
Loading