Skip to content

Commit f2e0295

Browse files
Vemarthula-MicrosoftVemarthula-Microsoft
committed
fixing cleanup deployment failure
1 parent 91665a2 commit f2e0295

File tree

1 file changed

+48
-134
lines changed

1 file changed

+48
-134
lines changed

‎.github/workflows/deploy.yml‎

Lines changed: 48 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -198,164 +198,85 @@ jobs:
198198
id: get_azure_resources
199199
run: |
200200
set -e
201-
echo "Fetching Log Analytics workspace from resource group ${{ env.RESOURCE_GROUP_NAME }}..."
201+
echo "Collecting resource identifiers from RG ${{ env.RESOURCE_GROUP_NAME }} before deletion..."
202202
log_analytics_workspace_name=$(az monitor log-analytics workspace list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[0].name" -o tsv || true)
203203
if [ -n "$log_analytics_workspace_name" ]; then
204204
echo "LOG_ANALYTICS_WORKSPACE_NAME=${log_analytics_workspace_name}" >> $GITHUB_ENV
205-
echo "Log Analytics workspace name: ${log_analytics_workspace_name}"
206-
else
207-
echo "No Log Analytics workspace found."
208205
fi
209-
echo "Fetching OpenAI resource from resource group ${{ env.RESOURCE_GROUP_NAME }}..."
210206
openai_resource_name=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.CognitiveServices/accounts" --query "[0].name" -o tsv || true)
211207
if [ -n "$openai_resource_name" ]; then
212208
echo "OPENAI_RESOURCE_NAME=${openai_resource_name}" >> $GITHUB_ENV
213-
echo "OpenAI resource name: ${openai_resource_name}"
214-
else
215-
echo "No OpenAI resource found."
216209
fi
210+
keyvaults=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[?type=='Microsoft.KeyVault/vaults'].name" -o tsv || true)
211+
if [ -n "$keyvaults" ]; then
212+
keyvault_array="["; first=true; for kv in $keyvaults; do if [ "$first" = true ]; then keyvault_array="$keyvault_array\"$kv\""; first=false; else keyvault_array="$keyvault_array,\"$kv\""; fi; done; keyvault_array="$keyvault_array]"; echo "KEYVAULTS=$keyvault_array" >> $GITHUB_ENV; fi
213+
echo "Snapshot complete."
217214
218-
- name: List KeyVaults and Store in Array
215+
- name: Delete Resource Group (primary deletion)
219216
if: always()
220-
id: list_keyvaults
221217
run: |
222-
223218
set -e
224-
echo "Listing all KeyVaults in the resource group ${RESOURCE_GROUP_NAME}..."
225-
226-
# Get the list of KeyVaults in the specified resource group
227-
keyvaults=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[?type=='Microsoft.KeyVault/vaults'].name" -o tsv)
228-
229-
if [ -z "$keyvaults" ]; then
230-
echo "No KeyVaults found in resource group ${RESOURCE_GROUP_NAME}."
231-
echo "KEYVAULTS=[]" >> $GITHUB_ENV # If no KeyVaults found, set an empty array
232-
else
233-
echo "KeyVaults found: $keyvaults"
234-
235-
# Format the list into an array with proper formatting (no trailing comma)
236-
keyvault_array="["
237-
first=true
238-
for kv in $keyvaults; do
239-
if [ "$first" = true ]; then
240-
keyvault_array="$keyvault_array\"$kv\""
241-
first=false
242-
else
243-
keyvault_array="$keyvault_array,\"$kv\""
244-
fi
245-
done
246-
keyvault_array="$keyvault_array]"
247-
248-
# Output the formatted array and save it to the environment variable
249-
echo "KEYVAULTS=$keyvault_array" >> $GITHUB_ENV
250-
fi
219+
echo "Initiating deletion of RG ${{ env.RESOURCE_GROUP_NAME }}..."
220+
az group delete --name ${{ env.RESOURCE_GROUP_NAME }} --yes --no-wait || echo "Delete command issued."
251221
252-
- name: Delete Bicep Deployment (defer RG deletion to end)
222+
- name: Wait for RG deletion propagation
253223
if: always()
254224
run: |
255-
echo "Skipping early RG deletion to allow purges to run first."
225+
set -e
226+
echo "Polling for RG deletion..."
227+
max_retries=15
228+
sleep_interval=20
229+
for i in $(seq 1 $max_retries); do
230+
exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }} || echo true)
231+
if [ "$exists" = false ]; then
232+
echo "Resource group deleted (confirmed)."
233+
break
234+
fi
235+
echo "RG still exists (attempt $i). Waiting $sleep_interval s..."
236+
sleep $sleep_interval
237+
done
256238
257-
- name: Purge log analytics workspace
239+
- name: Purge soft-deleted OpenAI (if any)
258240
if: always()
259-
id: log_analytics_workspace
260241
run: |
261-
262242
set -e
263-
# Purge Log Analytics Workspace
264-
echo "Purging the Log Analytics Workspace..."
265-
if ! az monitor log-analytics workspace delete --force --resource-group ${{ env.RESOURCE_GROUP_NAME }} --workspace-name ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }} --yes --verbose; then
266-
echo "Failed to purge Log Analytics workspace: ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}"
243+
if [ -n "${{ env.OPENAI_RESOURCE_NAME }}" ]; then
244+
echo "Attempting purge of soft-deleted OpenAI: ${{ env.OPENAI_RESOURCE_NAME }} in region ${{ env.AI_REGION }}"
245+
az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/${{ env.AI_REGION || 'eastus' }}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose || echo "OpenAI purge not available yet or failed (non-fatal)."
267246
else
268-
echo "Purged the Log Analytics workspace: ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}"
247+
echo "No OpenAI resource recorded to purge."
269248
fi
270249
271-
echo "Log analytics workspace resource purging completed successfully"
272-
273-
274-
- name: Wait for resource deletion to complete
250+
- name: Purge soft-deleted KeyVaults (if any)
275251
if: always()
276252
run: |
277-
278-
# List of keyvaults
253+
set -e
279254
KEYVAULTS="${{ env.KEYVAULTS }}"
280-
281-
# Remove the surrounding square brackets, if they exist
282-
stripped_keyvaults=$(echo "$KEYVAULTS" | sed 's/\[\|\]//g')
283-
284-
# Convert the comma-separated string into an array
285-
IFS=',' read -r -a resources_to_check <<< "$stripped_keyvaults"
286-
287-
# Append new resources to the array
288-
resources_to_check+=("${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}" "${{ env.OPENAI_RESOURCE_NAME }}")
289-
290-
echo "List of resources to check: ${resources_to_check[@]}"
291-
292-
# Maximum number of retries
293-
max_retries=3
294-
295-
# Retry intervals in seconds (30, 60, 120)
296-
retry_intervals=(30 60 120)
297-
298-
# Retry mechanism to check resources
299-
retries=0
300-
while true; do
301-
resource_found=false
302-
303-
# Get the list of resources in YAML format again on each retry
304-
resource_list=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --output yaml)
305-
306-
# Iterate through the resources to check
307-
for resource in "${resources_to_check[@]}"; do
308-
echo "Checking resource: $resource"
309-
if echo "$resource_list" | grep -q "name: $resource"; then
310-
echo "Resource '$resource' exists in the resource group."
311-
resource_found=true
312-
else
313-
echo "Resource '$resource' does not exist in the resource group."
314-
fi
315-
done
316-
317-
# If any resource exists, retry
318-
if [ "$resource_found" = true ]; then
319-
retries=$((retries + 1))
320-
if [ "$retries" -gt "$max_retries" ]; then
321-
echo "Maximum retry attempts reached. Exiting."
322-
break
323-
else
324-
# Wait for the appropriate interval for the current retry
325-
echo "Waiting for ${retry_intervals[$retries-1]} seconds before retrying..."
326-
sleep ${retry_intervals[$retries-1]}
327-
fi
255+
stripped=$(echo "$KEYVAULTS" | sed 's/\[\|\]//g')
256+
IFS=',' read -r -a kvs <<< "$stripped"
257+
for raw in "${kvs[@]}"; do
258+
kv=$(echo "$raw" | sed 's/\"//g' | xargs)
259+
[ -z "$kv" ] && continue
260+
echo "Checking soft-delete state for KeyVault: $kv"
261+
deleted=$(az keyvault list-deleted --query "[?name=='$kv']" -o json --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} || echo '[]')
262+
if [ "$(echo "$deleted" | jq length)" -gt 0 ]; then
263+
echo "Purging KeyVault $kv ..."
264+
az keyvault purge --name "$kv" --no-wait || echo "Failed to purge $kv (non-fatal)"
328265
else
329-
echo "No resources found. Exiting."
330-
break
266+
echo "KeyVault $kv not soft-deleted; skipping."
331267
fi
332268
done
333-
334-
- name: Purging the Resources
269+
270+
- name: Purge Log Analytics Workspace (best-effort)
335271
if: always()
336272
run: |
337273
set -e
338-
echo "Azure OpenAI: ${{ env.OPENAI_RESOURCE_NAME }}"
339-
if [ -n "${{ env.OPENAI_RESOURCE_NAME }}" ]; then
340-
echo "Purging the OpenAI Resource (soft-delete) in region ${{ env.AI_REGION }}..."
341-
az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/${{ env.AI_REGION || 'eastus' }}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose || echo "OpenAI purge skipped or failed."
274+
if [ -n "${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}" ]; then
275+
echo "Attempting to purge Log Analytics workspace: ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }} (best-effort)"
276+
az monitor log-analytics workspace delete --force --workspace-name ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }} --resource-group ${{ env.RESOURCE_GROUP_NAME }} --yes --verbose || echo "Workspace purge not applicable or already gone."
277+
else
278+
echo "No Log Analytics workspace recorded."
342279
fi
343-
KEYVAULTS="${{ env.KEYVAULTS }}"
344-
stripped_keyvaults=$(echo "$KEYVAULTS" | sed 's/\[\|\]//g')
345-
IFS=',' read -r -a keyvault_array <<< "$stripped_keyvaults"
346-
for keyvault_name in "${keyvault_array[@]}"; do
347-
kv_trim=$(echo "$keyvault_name" | sed 's/\"//g' | xargs)
348-
[ -z "$kv_trim" ] && continue
349-
echo "Processing KeyVault: $kv_trim"
350-
deleted_vaults=$(az keyvault list-deleted --query "[?name=='$kv_trim']" -o json --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} || echo '[]')
351-
if [ "$(echo "$deleted_vaults" | jq length)" -gt 0 ]; then
352-
echo "KeyVault '$kv_trim' soft-deleted. Purging..."
353-
az keyvault purge --name "$kv_trim" --no-wait || echo "Failed to purge KeyVault '$kv_trim'"
354-
else
355-
echo "KeyVault '$kv_trim' not soft-deleted. Skipping."
356-
fi
357-
done
358-
echo "Resource purging completed"
359280
360281
- name: Send Notification on Failure
361282
if: failure() || needs.deploy.result == 'failure'
@@ -376,14 +297,7 @@ jobs:
376297
- name: Final Resource Group Deletion
377298
if: always()
378299
run: |
379-
set -e
380-
rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
381-
if [ "$rg_exists" = true ]; then
382-
echo "Deleting resource group ${{ env.RESOURCE_GROUP_NAME }}..."
383-
az group delete --name ${{ env.RESOURCE_GROUP_NAME }} --yes --no-wait || echo "RG delete command issued."
384-
else
385-
echo "Resource group already gone."
386-
fi
300+
echo "Final RG delete step retained for idempotency; primary deletion already initiated."
387301
388302
- name: Logout from Azure
389303
if: always()

0 commit comments

Comments
 (0)