Enable retry on functional test (#7430)

# Description

This will not configure the aggressive retry which can hide the bug. So
I set 2 retries for the first iteration.

* Enable retry on functional test to resolve the intermittent failures.
(#7174)
* Ignore the failures of `clean up cluster` step - Based on the
investigation, timeout happens by deleting resources. It is ok to skip
for failure because long-running test always clean up cluster first
before running functional test. (
* Enable retry on `Configure Radius test workspace` to avoid [the
intermittent 503 failure](#7401)

## Type of change

<!--

Please select **one** of the following options that describes your
change and delete the others. Clearly identifying the type of change you
are making will help us review your PR faster, and is used in authoring
release notes.

If you are making a bug fix or functionality change to Radius and do not
have an associated issue link please create one now.

-->

- This pull request fixes a bug in Radius and has an approved issue
(issue link required).
- This pull request adds or changes features of Radius and has an
approved issue (issue link required).
- This pull request is a minor refactor, code cleanup, test improvement,
or other maintenance task and doesn't change the functionality of Radius
(issue link optional).

<!--

Please update the following to link the associated issue. This is
required for some kinds of changes (see above).

-->

#7174

Signed-off-by: Young Bu Park <youngp@microsoft.com>

.github/workflows/functional-test.yaml

@@ -539,25 +539,30 @@ jobs:
         run: |
           make publish-test-terraform-recipes
       - name: Run functional tests
-        run: |
-          # Ensure rad cli is in path before running tests.
-          export PATH=$GITHUB_WORKSPACE/bin:$PATH
-          cd $GITHUB_WORKSPACE
+        uses: nick-fields/retry@v3
+        with:
+          timeout_seconds: 1800
+          max_attempts: 2
+          command: |
+            # Ensure rad cli is in path before running tests.
+            export PATH=$GITHUB_WORKSPACE/bin:$PATH
+            cd $GITHUB_WORKSPACE
 
-          which rad || { echo "cannot find rad"; exit 1; }
+            which rad || { echo "cannot find rad"; exit 1; }
 
-          # Populate the following test environment variables from JSON secret.
-          # AZURE_MONGODB_RESOURCE_ID
-          # AZURE_COSMOS_MONGODB_ACCOUNT_ID
-          # AZURE_TABLESTORAGE_RESOURCE_ID
-          # AZURE_SERVICEBUS_RESOURCE_ID
-          # AZURE_REDIS_RESOURCE_ID
-          # AZURE_MSSQL_RESOURCE_ID
-          # AZURE_MSSQL_USERNAME
-          # AZURE_MSSQL_PASSWORD
-          eval "export $(echo "${{ secrets.FUNCTEST_PREPROVISIONED_RESOURCE_JSON }}" | jq -r 'to_entries | map("\(.key)=\(.value)") | @sh')"
+            # Populate the following test environment variables from JSON secret.
+            # AZURE_MONGODB_RESOURCE_ID
+            # AZURE_COSMOS_MONGODB_ACCOUNT_ID
+            # AZURE_TABLESTORAGE_RESOURCE_ID
+            # AZURE_SERVICEBUS_RESOURCE_ID
+            # AZURE_REDIS_RESOURCE_ID
+            # AZURE_MSSQL_RESOURCE_ID
+            # AZURE_MSSQL_USERNAME
+            # AZURE_MSSQL_PASSWORD
+            eval "export $(echo "${{ secrets.FUNCTEST_PREPROVISIONED_RESOURCE_JSON }}" | jq -r 'to_entries | map("\(.key)=\(.value)") | @sh')"
 
-          make test-functional-${{ matrix.name }}
+            make test-functional-${{ matrix.name }}
+          on_retry_command: echo "Functional test failed. Retrying..." >> $GITHUB_STEP_SUMMARY
         env:
           DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
           TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }}

.github/workflows/long-running-azure.yaml

@@ -393,32 +393,36 @@ jobs:
             --chart ${{ env.RADIUS_CHART_LOCATION }} \
             --set rp.image=${{ env.CONTAINER_REGISTRY }}/applications-rp,rp.tag=${{ env.REL_VERSION }},controller.image=${{ env.CONTAINER_REGISTRY }}/controller,controller.tag=${{ env.REL_VERSION }},ucp.image=${{ env.CONTAINER_REGISTRY }}/ucpd,ucp.tag=${{ env.REL_VERSION }}
       - name: Configure Radius test workspace
-        run: |
-          set -x
+        uses: nick-fields/retry@v3
+        with:
+          timeout_seconds: 30
+          max_attempts: 2
+          command: |
+            set -x
 
-          export PATH=$GITHUB_WORKSPACE/bin:$PATH
-          which rad || { echo "cannot find rad"; exit 1; }
+            export PATH=$GITHUB_WORKSPACE/bin:$PATH
+            which rad || { echo "cannot find rad"; exit 1; }
 
-          echo "*** Create workspace, group and environment for test ***"
-          rad workspace create kubernetes --force
-          rad group create ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}
-          rad group switch ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}
+            echo "*** Create workspace, group and environment for test ***"
+            rad workspace create kubernetes --force
+            rad group create ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}
+            rad group switch ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}
 
-          # The functional test is designed to use default namespace. So you must create the environment for default namespace.
-          rad env create ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --namespace default
-          rad env switch ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}
+            # The functional test is designed to use default namespace. So you must create the environment for default namespace.
+            rad env create ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --namespace default
+            rad env switch ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}
 
-          echo "*** Configuring Azure provider ***"
-          rad env update ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --azure-subscription-id ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} \
-            --azure-resource-group ${{ env.AZURE_TEST_RESOURCE_GROUP }}
-          rad credential register azure --client-id ${{ secrets.AZURE_SP_TESTS_APPID }} \
-            --client-secret ${{ secrets.INTEGRATION_TEST_SP_PASSWORD }} \
-            --tenant-id ${{ secrets.AZURE_SP_TESTS_TENANTID }}
+            echo "*** Configuring Azure provider ***"
+            rad env update ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --azure-subscription-id ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} \
+              --azure-resource-group ${{ env.AZURE_TEST_RESOURCE_GROUP }}
+            rad credential register azure --client-id ${{ secrets.AZURE_SP_TESTS_APPID }} \
+              --client-secret ${{ secrets.INTEGRATION_TEST_SP_PASSWORD }} \
+              --tenant-id ${{ secrets.AZURE_SP_TESTS_TENANTID }}
 
-          echo "*** Configuring AWS provider ***"
-          rad env update ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --aws-region ${{ env.AWS_REGION }} --aws-account-id ${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}
-          rad credential register aws \
-            --access-key-id ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }} --secret-access-key ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
+            echo "*** Configuring AWS provider ***"
+            rad env update ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --aws-region ${{ env.AWS_REGION }} --aws-account-id ${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}
+            rad credential register aws \
+              --access-key-id ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }} --secret-access-key ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
       - name: Log radius installation status (failure)
         if: failure()
         run: |
@@ -430,25 +434,30 @@ jobs:
         run: |
           echo "FUNCTEST_OIDC_ISSUER=$(az aks show -n ${{ env.AKS_CLUSTER_NAME }} -g ${{ env.AKS_RESOURCE_GROUP }} --query "oidcIssuerProfile.issuerUrl" -otsv)" >> $GITHUB_ENV
       - name: Run functional tests
-        run: |
-          # Ensure rad cli is in path before running tests.
-          export PATH=$GITHUB_WORKSPACE/bin:$PATH
-          cd $GITHUB_WORKSPACE
+        uses: nick-fields/retry@v3
+        with:
+          timeout_seconds: 1800
+          max_attempts: 2
+          command: |
+            # Ensure rad cli is in path before running tests.
+            export PATH=$GITHUB_WORKSPACE/bin:$PATH
+            cd $GITHUB_WORKSPACE
 
-          which rad || { echo "cannot find rad"; exit 1; }
+            which rad || { echo "cannot find rad"; exit 1; }
 
-          # Populate the following test environment variables from JSON secret.
-          # AZURE_MONGODB_RESOURCE_ID
-          # AZURE_COSMOS_MONGODB_ACCOUNT_ID
-          # AZURE_TABLESTORAGE_RESOURCE_ID
-          # AZURE_SERVICEBUS_RESOURCE_ID
-          # AZURE_REDIS_RESOURCE_ID
-          # AZURE_MSSQL_RESOURCE_ID
-          # AZURE_MSSQL_USERNAME
-          # AZURE_MSSQL_PASSWORD
-          eval "export $(echo "${{ secrets.FUNCTEST_PREPROVISIONED_RESOURCE_JSON }}" | jq -r 'to_entries | map("\(.key)=\(.value)") | @sh')"
+            # Populate the following test environment variables from JSON secret.
+            # AZURE_MONGODB_RESOURCE_ID
+            # AZURE_COSMOS_MONGODB_ACCOUNT_ID
+            # AZURE_TABLESTORAGE_RESOURCE_ID
+            # AZURE_SERVICEBUS_RESOURCE_ID
+            # AZURE_REDIS_RESOURCE_ID
+            # AZURE_MSSQL_RESOURCE_ID
+            # AZURE_MSSQL_USERNAME
+            # AZURE_MSSQL_PASSWORD
+            eval "export $(echo "${{ secrets.FUNCTEST_PREPROVISIONED_RESOURCE_JSON }}" | jq -r 'to_entries | map("\(.key)=\(.value)") | @sh')"
 
-          make test-functional-all
+            make test-functional-all
+          on_retry_command: echo "Functional test failed. Retrying..." >> $GITHUB_STEP_SUMMARY
         env:
           TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }}
           RADIUS_CONTAINER_LOG_PATH: ${{ github.workspace }}/${{ env.RADIUS_CONTAINER_LOG_BASE }}
@@ -495,6 +504,7 @@ jobs:
             --yes --verbose
       - name: Clean up cluster
         if: always()
+        continue-on-error: true
         run: ./.github/scripts/cleanup-cluster.sh
   report-failure:
     name: Report test failure