ByoVnetCI.yml

name: InfraCI - ByoVnet cluster
# Credential prerequisites
# 1. IAM Owner on the Resource Group you're deploying into (we're making role assignments)
# 2. IAM Owner on the Vnet (we're making role assignments)

on:
# Triggering on push causes a problem with the Azure Login Federated Identity Subject Claim.
# Disabling this trigger as having Push + PR is OTT
#  push:
#    paths:
#      #- ".github/workflows/ByoVnetCI.yml"
#      - ".github/workflows_dep/AksDeploy-ByoVnet.parameters.json"
#      - "bicep/*"

  pull_request:
    branches: [main]
    paths:
      - "bicep/*"
      - ".github/workflows/ByoVnetCI.yml"
    types: #Type filtering should stop CI/CD running on draft status workflows
      - opened
      - reopened
      - synchronize
      - ready_for_review

  #Running on a schedule helps trap issues like Kubernetes versions not being available in the region, or Well Architected recommendations changing
  schedule:
    # At 11:00pm, every Wednesday week
    - cron: "0 23 * * 3"

  workflow_dispatch:
    inputs:
      environment:
        description: 'Which environment to deploy to'
        required: true
        default: "csu"
        type: environment
      ResourceGroup:
        description: 'Which Resource Group to deploy to'
        default: 'AksBicepAcc-Ci-ByoDeploy'
        type: string
        required: false
      region:
        description: 'Region (needs to be same as byo vnet location)'
        default: 'southcentralus'
        type: string
        required: false
      doWellArchitected:
        description: 'Perform the Well Architected Framework assesment'
        default: false
        type: boolean
        required: false
      doCostEstimate:
        description: 'Perform a template Cost Estimate'
        default: false
        type: boolean
        required: false
      doDebugSteps:
        description: 'Run informational steps'
        default: true
        type: boolean
        required: false
      doVerifySteps:
        description: 'Run optional verify steps'
        default: true
        type: boolean
        required: false
      doSmokeTestUninstall:
        description: 'Uninstall smoke test apps after tests'
        default: true
        type: boolean
        required: false
      defaultdeny:
        description: 'Default Deny NetworkPolicy?'
        default: false
        type: boolean
        required: false

permissions:
      id-token: write
      contents: read

concurrency: ci-${{ github.ref }}

env:
  AZCLIVERSION: 2.53.0 #2.43.0 #2.34.1 #2.29.2 #2.26.0 #latest
  ParamFilePath: ".github/workflows_dep/AksDeploy-ByoVnet.parameters.json"
  RESNAME: "Byov"
  DEPNAME: "Dep${{ github.run_number }}"
  KVACCESS: "IAM"

jobs:
  Well_Architected:
    runs-on: ubuntu-latest
    needs: [Validation]
    if:  github.event_name != 'pull_request' || github.event.inputs.doWellArchitected == 'true' ||  contains( github.event.pull_request.labels.*.name, 'test-deploy-byoconfig')
    steps:
      - uses: actions/checkout@v4.1.1

      # PSRule does this cool thing where it traverse the parameter file through to the arm template
      # PSRule performs IaC recommendations of the template.
      # https://azure.github.io/PSRule.Rules.Azure/
      - name: PSRule for Azure - Well Architected
        uses: microsoft/ps-rule@v2.7.0
        #uses: microsoft/ps-rule@main
        continue-on-error: true #Setting this whilst PSRule gets bedded in, in this project
        with:
          modules: 'PSRule.Rules.Azure'
          inputPath: "${{ env.ParamFilePath }}"
          prerelease: true

      # In addition to the Well Architected analysis, we can also leverage the CAF rules
      # Run analysis against PSRule for Cloud Adoption Framework
      - name: PSRule for CAF
        uses: microsoft/ps-rule@v2.7.0
        continue-on-error: true #Setting this whilst PSRule gets bedded in, in this project
        with:
          modules: 'PSRule.Rules.CAF'
          inputPath: "${{ env.ParamFilePath }}"
          prerelease: false

  CostEstimate:
    needs: [ReusableWF, Validation]
    if: github.event.inputs.doCostEstimate == 'true'
    uses: TheCloudTheory/arm-estimator/.github/workflows/estimateFromUrl.yml@main
    #uses: Gordonby/arm-estimator/.github/workflows/estimateFromUrl.yml@gb-workflowpolish
    with:
      rg: ${{ needs.ReusableWF.outputs.RG }}
      environment: ${{needs.ReusableWF.outputs.Environment}}
      aceVersion: 1.1-beta2
      templateFileURL: https://github.com/Azure/AKS-Construction/releases/download/${{ needs.ReusableWF.outputs.LatestAkscVersionTag }}/main.json
      templateParamFileURL: https://raw.githubusercontent.com/Azure/AKS-Construction/${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }}/.github/workflows_dep/AksDeploy-ByoVnet.parameters.json
      templateParams: ${{ needs.Validation.outputs.PARAMOVERRIDESNOSECRETS}} #${{ needs.Validation.outputs.PARAMOVERRIDES}}
    secrets:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

  #Pre deployment validation of the parameters + bicep code
  Validation:
    runs-on: ubuntu-latest
    environment: ${{ github.event.inputs.environment }}
    if: ${{ !github.event.pull_request.head.repo.fork && !github.event.pull_request.draft }}
    outputs:
      RESOURCEGROUP: ${{ steps.params.outputs.RESOURCEGROUP}}
      REGION: ${{ steps.params.outputs.REGION}}
      RESNAME: ${{ steps.params.outputs.NEWRESNAME}}
      PARAMOVERRIDES: ${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}} #This won't work if the string includes any SECRETS!!!
      PARAMOVERRIDESNOSECRETS: ${{ steps.imperitiveparams.outputs.PARAMOVERRIDESNOSECRETS}}
    steps:
      - uses: actions/checkout@v4.1.1

      - name: Job parameter check
        run: |
          echo "Param file path is: ${{ env.ParamFilePath }}"
          echo "Resource name is ${{ env.RESNAME }}"
          echo "Deployment name is ${{ env.DEPNAME }}"
          echo "Ref is ${{ github.ref }}"
          echo "Ref name is ${{GITHUB.REF_NAME}}"
          echo "EventTrigger name is ${{github.event_name}}"
          echo "PR contains bug :  ${{contains(github.event.pull_request.labels.*.name, 'bug')}}"
          echo "PR labels :  ${{github.event.pull_request.labels.*.name}}"
          echo "PR draft :  ${{github.event.pull_request.draft}}"
          echo "AZCLIVERSION is ${{ env.AZCLIVERSION }}"
          echo "doDebugSteps is ${{ github.event.inputs.doDebugSteps }}"
          echo "doVerifySteps is ${{ github.event.inputs.doVerifySteps }}"
          echo "doSmokeTestUninstall is ${{ github.event.inputs.doSmokeTestUninstall }}"

      - name: Arm Parameter file check
        if: github.event.inputs.doVerifySteps == 'true'
        shell: pwsh
        run: |
          Write-Output "Checking parameter file existence/contents"
          $paramFilePath="${{ env.ParamFilePath }}"
          Test-Path $paramFilePath

          if (Test-Path $paramFilePath) {
            $paramFileContent=Get-Content $paramFilePath
            Write-Output $paramFileContent

            Write-Output "Test Pulling a param (ingressApplicationGateway)"
            $params=$paramFileContent|ConvertFrom-Json
            Write-Output $params.parameters.ingressApplicationGateway.value
          }

      - name: Parameter Value Augmentation
        id: params
        env:
          DEFAULTRGNAME: AksBicepAcc-Ci-ByoDeploy
        run: |
            if [ -z "${{ github.event.inputs.region }}" ]
            then
              echo "Region parameter not available through GitHub event data, setting default"
              REGION="southcentralus"
            else
              echo "Region parameter found in GitHub event (${{ github.event.inputs.region }})"
              REGION="${{ github.event.inputs.region }}"
            fi
            echo $REGION
            echo "REGION=$REGION" >> $GITHUB_OUTPUT
            if [ -z "${{ github.event.inputs.ResourceGroup }}" ]
            then
              echo "ResourceGroup parameter not available through GitHub event data, setting to default"
              echo $DEFAULTRGNAME
              echo "RESOURCEGROUP=$DEFAULTRGNAME" >> $GITHUB_OUTPUT
            else
              echo "Resource Group parameter found in GitHub event (${{ github.event.inputs.ResourceGroup }})"
              echo "RESOURCEGROUP=${{ github.event.inputs.ResourceGroup }}" >> $GITHUB_OUTPUT
            fi
            NEWRESNAME="$RESNAME${REGION:0:3}"
            echo "Setting new Resource Name $NEWRESNAME"
            echo "NEWRESNAME=$NEWRESNAME" >> $GITHUB_OUTPUT

      - name: Azure Login
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          enable-AzPSSession: true
          environment: azurecloud
          allow-no-subscriptions: false

      - name: Install Pwsh modules
        shell: pwsh
        run: |
          Set-PSRepository -Name PSGallery -InstallationPolicy Trusted
          Install-Module -Name Az.KeyVault -Force

      - name: Verify any active Azure Resource Group Deployments #These can mess up our deployment
        id: activedeps
        if: github.event.inputs.doVerifySteps == 'true'
        env:
          RG: ${{ steps.params.outputs.RESOURCEGROUP }}
        uses: Azure/cli@v2
        with:
          azcliversion: ${{ env.AZCLIVERSION }}
          inlineScript: |
            echo "AZ CLI version"
            az version

            RUNCOUNT=$(az deployment group list -g $RG --query "[?properties.provisioningState=='Running'].[properties.provisioningState, name] | length(@)" -o tsv)
            echo "Active deployments : $RUNCOUNT"

            echo 'Active deployment list'
            az deployment group list -g $RG --query "[?properties.provisioningState=='Running'].[properties.provisioningState, name]"

            #echo 'Verbose deployment list'
            #az deployment group list -g $RG --query "[].[properties.provisioningState, name]"

            echo "RUNCOUNT=$RUNCOUNT" >> $GITHUB_OUTPUT

      - name: Verify AKS Preview Features are available in target Subscription
        if: github.event.inputs.doVerifySteps == 'true'
        shell: pwsh
        run: |
          write-output 'Full list of features of AKS'
          az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService')].{Name:name,State:properties.state}"

          write-output 'Features that are still registering'
          az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService') && properties.state=='Registering'].{Name:name,State:properties.state}"

          write-output 'Checking to ensure no features are still registering'
          #Not going to use JMES as i'm getting a weird error that i don't see locally in powershell
          $aksfeatures = az feature list --query "[?contains(name, 'Microsoft.ContainerService')]" | ConvertFrom-Json
          #$aksfeatures = $allfeatures | Where-Object {$_.Name -like 'Microsoft.ContainerService*'}
          $registeringfeatures = $aksfeatures |  Where-Object {$_.properties.state -eq 'Registering'}

          if ($registeringfeatures.count -gt 0) {
            Write-Error "There are still features registering"
          } else { Write-Output "-- All good, no features in the process of registering" }

          write-output 'Check specific features.'
          $paramFilePath="${{ env.ParamFilePath }}"
          $paramFileContent=Get-Content $paramFilePath
          $params=$paramFileContent|ConvertFrom-Json

          if($params.parameters.keyVaultAksCSI.value -eq $true) {
            $feature='AKS-AzureKeyVaultSecretsProvider'
            write-output "-- $feature"
            $featureCsi = $aksfeatures |  Where-Object {$_.name -like "*$feature"}
            $featureCsi.properties.state

            if ($featureCsi.properties.state -ne 'Registered') {
              Write-Output $featureCsi
              Write-Error "$feature NOT registered"
            } else { Write-Output "-- Looks like $feature is registered properly" }
          }

      - name: Create Parameter file imperative override string
        uses: Azure/cli@v2
        id: imperitiveparams
        env:
          RG: ${{ steps.params.outputs.RESOURCEGROUP }}
        with:
          azcliversion: ${{ env.AZCLIVERSION }}
          inlineScript: |
            PARAMOVERRIDES="resourceName=$RESNAME location=${{ steps.params.outputs.REGION}} byoAKSSubnetId=${{ secrets.ByoAksSubnetId }} byoAGWSubnetId=${{ secrets.ByoAgwSubnetId }} dnsZoneId=${{ secrets.BYODNSZONEID }} enableTelemetry=${{ VARS.ENABLETELEMETRY == 'true' }}"
            echo $PARAMOVERRIDES
            echo "PARAMOVERRIDES=$PARAMOVERRIDES" >> $GITHUB_OUTPUT

            PARAMOVERRIDESNOSECRETS="resourceName=$RESNAME location=${{ steps.params.outputs.REGION}} byoAKSSubnetId='' byoAGWSubnetId='' dnsZoneId=''"
            echo $PARAMOVERRIDESNOSECRETS
            echo "PARAMOVERRIDESNOSECRETS=$PARAMOVERRIDESNOSECRETS" >> $GITHUB_OUTPUT

      - name: Validate Infrastructure deployment
        uses: Azure/cli@v2
        env:
          RG: ${{ steps.params.outputs.RESOURCEGROUP }}
        with:
          azcliversion: ${{ env.AZCLIVERSION }}
          inlineScript: |
            RESNAME='${{ env.RESNAME }}'
            DEPNAME='Dep${{ github.run_number }}'
            PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}}'

            echo $PARAMS

            az deployment group validate -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS --verbose

      - name: What If (GA edge Az cli version) (has issues)
        uses: Azure/cli@v2
        id: edgeWhatIf
        env:
          RG: ${{ steps.params.outputs.RESOURCEGROUP }}
        continue-on-error: true #Setting to true due to bug in the AzureCLI https://github.com/Azure/azure-cli/issues/19850
        with:
          azcliversion: ${{ env.AZCLIVERSION }} #latest
          inlineScript: |
            RESNAME='${{ env.RESNAME }}'
            DEPNAME='${{ env.DEPNAME }}'
            WHATIFPATH='whatif.json'
            PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}}'

            #az deployment group what-if --debug -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p resourceName=$RESNAME byoAKSSubnetId=${{ secrets.ByoAksSubnetId }} byoAGWSubnetId=${{ secrets.ByoAgwSubnetId }}

            az deployment group what-if --no-pretty-print -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS > $WHATIFPATH

            if [[ -f $WHATIFPATH ]]
            then
                echo "The WhatIf json file was created"
            fi

            cat $WHATIFPATH

            echo "edgeSuccess=true" >> $GITHUB_OUTPUT

      - name: What If (No Secrets param string, used for Cost Estimate)
        uses: Azure/cli@v2
        env:
          RG: ${{ steps.params.outputs.RESOURCEGROUP }}
        continue-on-error: true
        with:
          azcliversion: ${{ env.AZCLIVERSION }} #latest
          inlineScript: |
            DEPNAME='${{ env.DEPNAME }}'
            WHATIFPATH='whatif.json'
            PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDESNOSECRETS}}'

            az deployment group what-if --no-pretty-print -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS > $WHATIFPATH

            if [[ -f $WHATIFPATH ]]
            then
                echo "The WhatIf json file was created"
            fi

            cat $WHATIFPATH

      - name: What If (workaround task)
        uses: Azure/cli@v2
        continue-on-error: true #Setting to true due to bug in the AzureCLI https://github.com/Azure/azure-cli/issues/19850
        if: steps.edgeWhatIf.outputs.edgeSuccess != 'true'
        env:
          RG: ${{ steps.params.outputs.RESOURCEGROUP }}
        with:
          azcliversion: 2.26.0
          inlineScript: |
            RESNAME='${{ env.RESNAME }}'
            DEPNAME='${{ env.DEPNAME }}'
            WHATIFPATH='whatif.json'
            PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}}'

            az deployment group what-if --no-pretty-print -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS > $WHATIFPATH

            if [[ -f $WHATIFPATH ]]
            then
                echo "The WhatIf json file was created"
            fi

            cat $WHATIFPATH

      - name: What If Analysis Output - Parse output
        if: github.event.inputs.doVerifySteps == 'true'
        shell: pwsh
        run: |
          $whatifpath='whatif.json'

          Write-Output "Checking for JSON What-If"
          $whatifexists=Test-Path -path $whatifpath
          Write-Output $whatifexists

          if ($whatifexists) {
            $jsonFileRaw=Get-Content $whatifpath
            Write-Output $jsonFileRaw

            $whatIf=$jsonFileRaw | ConvertFrom-Json

            if ($null -eq $whatIf) {
              Write-Output "What If results are null"
              #We might want to throw an error here, after the whatif bug gets fixed https://github.com/Azure/azure-cli/issues/19850
            } else {
              Write-Output $whatif.changes[0].after.type
            }
          }

      - name: What If Analysis Output - Pester Testing
        shell: pwsh
        run: |
          Write-Output "https://pester.dev/"
          Write-Output "TODO"

      - name: What If Analysis Gaps - Check for Soft deleted KeyVaults
        shell: pwsh
        run: |
          #This is a current gap in the What-If analysis.
          $whatifpath='whatif.json'
          $whatifexists=Test-Path -path $whatifpath

          if ($whatifexists) {
            $whatIf = Get-Content $whatifpath | ConvertFrom-Json

            if ($null -eq $whatIf) {
              Write-Output "What If results are null"
              #We might want to throw an error here, after the whatif bug gets fixed https://github.com/Azure/azure-cli/issues/19850
            } else {
              $paramFilePath="${{ env.ParamFilePath }}"
              $paramFileContent=Get-Content $paramFilePath
              $params=$paramFileContent|ConvertFrom-Json

              if($params.parameters.keyVaultCreate.value -eq $true) {
                Write-Output "Checking for already existing soft deleted KV"

                #lets do this properly and interrogate the whatif file for the KVNAME
                $kvafter = $whatif.changes | where-object {$_.after.type -eq 'Microsoft.KeyVault/vaults'} | where {$_.changeType -ne 'Ignore'}  | select after
                $vaultname = $kvafter.after.name

                write-output "Looking for KeyVault $vaultname"
                Get-AzKeyVault -InRemovedState | ft

                $deletedKv = Get-AzKeyVault -InRemovedState | where-object {$_.vaultname -eq $vaultname}

                if($deletedKv.length -eq 0) {
                  Write-Output "All is good, soft deleted KV not found in soft deleted list."
                }
                else {
                  Write-Output "Soft deleted KV output"
                  Write-Output $deletedKv.length
                  Write-Output $deletedKv

                  Write-Error "Existing soft deleted Key Vault ($vaultName) Found - Cannot proceed, as deployment will fail"
                }
              }
            }
          }

  Deploy:
    runs-on: ubuntu-latest
    needs: [Validation]
    environment: ${{ github.event.inputs.environment }}
    env:
      RG: ${{ needs.Validation.outputs.RESOURCEGROUP }}
      REGION: ${{ needs.Validation.outputs.REGION }}
      RESNAME: ${{ needs.Validation.outputs.RESNAME }}
    outputs:
      AKSNAME: ${{ steps.deployAks.outputs.AKSNAME}}
      LANAME : ${{ steps.deployAks.outputs.LANAME}}
      LAWGUID : ${{ steps.deployAks.outputs.LAWGUID}}
      AKVNAME : ${{ steps.deployAks.outputs.AKVNAME}}
      ACRNAME : ${{ steps.deployAks.outputs.ACRNAME}}
      AGNAME : ${{ steps.deployAks.outputs.AGNAME}}

    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || contains( github.event.pull_request.labels.*.name, 'test-deploy-byoconfig')
    steps:
      - uses: actions/checkout@v4.1.1

      - name: Job parameter check
        if: github.event.inputs.doDebugSteps == 'true'
        run: |
          echo "RG is: ${{ env.RG }}"
          echo "Param file path is: ${{ env.ParamFilePath }}"
          echo "Resource name is ${{ env.RESNAME }}
          echo "Deployment name is ${{ env.DEPNAME }}

      - name: Parameter file dependency check
        if: github.event.inputs.doVerifySteps == 'true'
        shell: pwsh
        run: |
          Write-Output "Checking parameter file existence/contents"
          $paramFilePath="${{ env.ParamFilePath }}"
          Test-Path $paramFilePath

          if (Test-Path $paramFilePath) {
            $paramFileContent=Get-Content $paramFilePath
            Write-Output $paramFileContent
          }

      - name: Azure Login
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          enable-AzPSSession: true
          environment: azurecloud
          allow-no-subscriptions: false

      - name: Deploy Infrastructure
        id: deployAks
        uses: Azure/cli@v2
        with:
          azcliversion: ${{ env.AZCLIVERSION }}
          inlineScript: |
            RESNAME='${{ env.RESNAME }}'
            DEPNAME='Dep${{ github.run_number }}'
            PARAMS="resourceName=$RESNAME location=$REGION byoAKSSubnetId=${{ secrets.ByoAksSubnetId }} byoAGWSubnetId=${{ secrets.ByoAgwSubnetId }} dnsZoneId=${{ secrets.BYODNSZONEID }}"

            az deployment group create -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS --name $DEPNAME --verbose

            DEPSTATUS=$(az deployment operation group list --resource-group $RG --name $DEPNAME) #--query "[?properties.provisioningState=='Failed']"
            echo $DEPSTATUS

            #outputs
            AKSNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.aksClusterName.value" -o tsv)
            echo "AKSNAME=$AKSNAME" >> $GITHUB_OUTPUT

            AGNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.applicationGatewayName.value" -o tsv)
            echo "AGNAME=$AGNAME" >> $GITHUB_OUTPUT

            AKVNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.keyVaultName.value" -o tsv)
            echo "AKVNAME=$AKVNAME" >> $GITHUB_OUTPUT

            LANAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.logAnalyticsName.value" -o tsv)
            echo "LANAME=$LANAME" >> $GITHUB_OUTPUT

            LAWGUID=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.logAnalyticsGuid.value" -o tsv)
            echo "LAWGUID=$LAWGUID" >> $GITHUB_OUTPUT

            ACRNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.containerRegistryName.value" -o tsv)
            echo "ACRNAME=$ACRNAME" >> $GITHUB_OUTPUT


      - name: Enable AGIC Addon
        uses: Azure/cli@v2
        if: ${{ needs.Deploy.outputs.AGNAME}} != ''
        with:
          azcliversion: ${{ env.AZCLIVERSION }}
          inlineScript: |
            AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'
            AGNAME='${{ steps.deployAks.outputs.AGNAME}}'

            echo "AKS $AKSNAME"
            echo "AG $AGNAME"

            echo "Checking if ingressApplicationGateway is enabled"
            AGICEnabled=$(az aks show -n $AKSNAME -g  $RG --query "addonProfiles.ingressApplicationGateway.enabled")
            echo "AGICEnabled = $AGICEnabled"

            if [ "$AGICEnabled" != "true" ];
            then
                az aks enable-addons -n $AKSNAME -g $RG -a ingress-appgw --appgw-id $(az network application-gateway show -g $RG -n $AGNAME --query id -o tsv)
            fi

      - name: Create AGIC role assignments (if not exist)
        if: ${{ needs.Deploy.outputs.AGNAME}} != ''
        continue-on-error: true
        shell: pwsh
        run: |
            $AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'
            $AGNAME='${{ steps.deployAks.outputs.AGNAME}}'

            write-output "AKS $AKSNAME"
            write-output "AG $AGNAME"

            write-output "Checking if ingressApplicationGateway is enabled"
            $AGICEnabled=az aks show -n $AKSNAME -g  $RG --query "addonProfiles.ingressApplicationGateway.enabled"
            write-output "AGICEnabled = $AGICEnabled"

            if ($AGICEnabled -eq $true) {
            	$agicobjid=az aks show -g $RG -n $AKSNAME --query "addonProfiles.ingressApplicationGateway.identity.objectId" -o tsv
              az role assignment list --assignee $agicobjid

              $rgScope=az group show -n $RG --query id
              az role assignment create --role Reader --assignee-object-id $agicobjid --assignee-principal-type ServicePrincipal --scope $rgScope

              $agwScope=az network application-gateway show -g $RG -n $AGNAME --query id -o tsv
              az role assignment create --role Contributor --assignee-object-id $agicobjid --assignee-principal-type ServicePrincipal --scope $agwScope

              az role assignment list --assignee $agicobjid
            }
      - name: Create Default Deny NetworkPolicy
        if: github.event.inputs.defaultdeny == 'true'
        run:
          AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'

          netpolicycmd="kubectl apply -f https://raw.githubusercontent.com/Azure/Aks-Construction/0.4.3/postdeploy/k8smanifests/networkpolicy-deny-all.yml";
          echo "Sending command $netpolicycmd to $AKSNAME in $RG";
          cmdOut=$(az aks command invoke -g $RG -n $AKSNAME -o json --command "${netpolicycmd}");
          echo $cmdOut;

      - name: Verify Default Deny NetworkPolicy
        if: github.event.inputs.defaultdeny == 'true' && github.event.inputs.doVerifySteps == 'true'
        run:
          AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}'

          netpolicycmd="kubectl get networkpolicy -A";

          echo "Sending command $netpolicycmd to $AKSNAME in $RG";
          cmdOut=$(az aks command invoke -g $RG -n $AKSNAME -o json --command "${netpolicycmd}");
          echo $cmdOut;

  ReusableWF:
    runs-on: ubuntu-latest
    needs: [Validation]
    env:
      RG: ${{ needs.Validation.outputs.RESOURCEGROUP }}
    outputs:
      RG: ${{ needs.Validation.outputs.RESOURCEGROUP }}
      ENVIRONMENT: ${{ github.event.inputs.Environment != '' && github.event.inputs.Environment || 'csu' }}
      Uninstall: ${{ github.event.inputs.doSmokeTestUninstall }}
      ExistingDnsDomainName: aksc.msftcsu.net
      ExistingDnsDomainRg: aksbicepacc-ci-deployvnet
      LatestAkscVersionTag: ${{ steps.AkscTags.outputs.LATEST}}
    steps:
      - name: Dummy step
        run: echo "Resuable workflows can't be directly passed ENV/INPUTS (yet)"
      - name: Explain more
        run: echo "see https://github.community/t/reusable-workflow-env-context-not-available-in-jobs-job-id-with/206111"
      - name: Job parameter inspection
        run: |
          echo "PARAMOVERRIDES: ${{ needs.Validation.outputs.PARAMOVERRIDES}}"
          echo "doSmokeTestUninstall: ${{ github.event.inputs.doSmokeTestUninstall }}"
          echo "RG: $RG"
      - name: Get latest AKSC version
        id: AkscTags
        run: |
          LATEST=$(curl https://api.github.com/repos/Azure/AKS-Construction/releases/latest | jq '.tag_name' -r)
          echo "LATEST=$LATEST" >> $GITHUB_OUTPUT

  Post-Deploy:
    needs: [Validation, Deploy, ReusableWF]
    uses: ./.github/workflows/PostDeploy.yml
    with:
      RG: ${{needs.ReusableWF.outputs.RG}}
      AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
      DNSDOMAIN: ${{needs.ReusableWF.outputs.ExistingDnsDomainName}}
      DNSRG: ${{needs.ReusableWF.outputs.ExistingDnsDomainRg}}
      CERTMANAGEREMAIL: "gdogg@microsoft.com"
    secrets:
      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}

  SmokeTest_SimpleApp:
    needs: [Deploy, ReusableWF]
    uses: ./.github/workflows/AppDeploy_AzureVote.yml
    with:
      ENVIRONMENT: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
      RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #$RG
      AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
      AGNAME: ${{ needs.Deploy.outputs.AGNAME}}
      APPNAME: avote-public
      INGRESSTYPE: "AGIC-Public-Ingress"
      FORCEHELMCLEANINSTALL: true
      UNINSTALLAFTERVERIFY: ${{ needs.ReusableWF.outputs.Uninstall == 'true' }}
    secrets:
      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}

  SmokeTest_JavaApp-certmgr:
    needs: [Deploy, ReusableWF, Post-Deploy]
    #uses: azure-samples/java-aks-keyvault-tls/.github/workflows/deployapp.yml@gb-workflow-bumps
    uses: azure-samples/java-aks-keyvault-tls/.github/workflows/deployapp.yml@0.9.5
    with:
      #ENVIRONMENT: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
      REPOREF: "0.9.5"
      RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows
      AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
      DNSDOMAIN: ${{needs.ReusableWF.outputs.ExistingDnsDomainName}}
      DNSRG: ${{needs.ReusableWF.outputs.ExistingDnsDomainRg}}
      DNSRECORDNAME: openjdk-demo
      AKVNAME: ${{ needs.Deploy.outputs.AKVNAME}}
      AGNAME: ${{ needs.Deploy.outputs.AGNAME}}
      APPNAME: openjdk-demo
      FRONTENDCERTTYPE: certmanager-staging
      FORCEHELMCLEANINSTALL: true
      UNINSTALLAFTERVERIFY: ${{ needs.ReusableWF.outputs.Uninstall == 'true' }}
    secrets:
      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
      # AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      # AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      # AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}


  SmokeTest_JavaApp-appgw:
    needs: [Deploy, ReusableWF, Post-Deploy, SmokeTest_JavaApp-certmgr]
    uses: azure-samples/java-aks-keyvault-tls/.github/workflows/deployapp.yml@0.9.5
    with:
      #ENVIRONMENT: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
      REPOREF: "0.9.5"
      RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows
      AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
      DNSDOMAIN: ${{needs.ReusableWF.outputs.ExistingDnsDomainName}}
      DNSRG: ${{needs.ReusableWF.outputs.ExistingDnsDomainRg}}
      DNSRECORDNAME: openjdk-keyvssl
      AKVNAME: ${{ needs.Deploy.outputs.AKVNAME}}
      AGNAME: ${{ needs.Deploy.outputs.AGNAME}}
      APPNAME: openjdk-keyvssl
      FRONTENDCERTTYPE: appgw-selfsigned
      FORCEHELMCLEANINSTALL: true
      UNINSTALLAFTERVERIFY: ${{ needs.ReusableWF.outputs.Uninstall == true }}
    secrets:
      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
      # AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      # AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      # AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}


  DeployVerify:
    runs-on: ubuntu-latest
    needs: [Validation, Deploy]
    env:
      RG: ${{ needs.Validation.outputs.RESOURCEGROUP }}
    steps:
      - uses: actions/checkout@v4.1.1

      - name: Param check
        if: github.event.inputs.doDebugSteps == 'true'
        run: |
          echo "RG is $RG"
          echo "Param file path is: ${{ env.ParamFilePath }}"
          echo "Resource name is ${{ env.RESNAME }}"
          echo "Deployment name is ${{ env.DEPNAME }}"
          echo "AKS name is ${{ needs.Deploy.outputs.AKSNAME }}"

      - name: Azure Login
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          enable-AzPSSession: true

      - name: Test Control Plane Config
        shell: pwsh
        run: |
          $RG='${{ env.RG }}'
          $AKSNAME='${{ needs.Deploy.outputs.AKSNAME }}'
          $paramFilePath="${{ env.ParamFilePath }}"
          $params = Get-Content $paramFilePath | ConvertFrom-Json

          Write-Output "Checking for User Assigned Identity"
          if($params.parameters.custom_vnet.value -eq $true) {
            $aksId=az aks show -n $AKSNAME -g $RG --query "identity.type" -o tsv
            write-output $aksId

            if($aksId -eq "UserAssigned"){
              Write-Output "UAI set correctly"
            } else {
              Write-Error "UAI not set properly"}
          }

          Write-Output "Checking for CSI Secrets driver"
          if($params.parameters.keyVaultAksCSI.value -eq $true) {
            $csiEnabled=az aks show -n $AKSNAME -g $RG --query "addonProfiles.omsagent.enabled" -o tsv
            write-output $csiEnabled

            if($csiEnabled -eq "true"){
              Write-Output "CSI Secrets driver set correctly"
            } else {
              Write-Error "CSI Secrets driver not set properly"}
          }
          #grep KeyvaultSecretsProvider

        #Sometimes cluster config works for deployment, but misconfig can
        #Prevent the cluster from scaling... So lets make sure it can.
      - name: Scale the cluster up by 1 node
        shell: pwsh
        run: |
          $RG='${{ env.RG }}'
          $AKSNAME='${{ needs.Deploy.outputs.AKSNAME }}'

          Write-Output "Scaling $AKSNAME in $RG"

          $manualScalePools = az aks show -n $AKSNAME -g $RG --query "agentPoolProfiles[?maxCount==null].{name:name, count:count}" -o json | ConvertFrom-Json
          $manualScalePools | ForEach-Object { Write-Output "scaling [m] pool $($_.name)"; az aks scale -g $RG -n $AKSNAME --node-count $($_.pool + 1) --nodepool-name $_.name }

          $autoScalePools = az aks show -n $AKSNAME -g $RG --query "agentPoolProfiles[?maxCount!=null].{name:name, minCount:minCount, maxCount:maxCount}" -o json | ConvertFrom-Json
          $autoScalePools | ForEach-Object { Write-Output "scaling [a] pool $($_.name)"; az aks nodepool update --update-cluster-autoscaler -g $RG --cluster-name $AKSNAME --name $_.name --min-count $($_.minCount + 1) --max-count $($_.maxCount + 1)  }


  Troubleshoot:
    needs: [Deploy, ReusableWF, Post-Deploy, SmokeTest_JavaApp-certmgr, SmokeTest_JavaApp-appgw]
    uses: ./.github/workflows/AksTroubleshooting.yml
    if: always()
    with:
      Environment: ''
      RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows
      AKSNAME: ${{needs.Deploy.outputs.AKSNAME}}
      AGNAME: ${{ needs.Deploy.outputs.AGNAME}}
      LANAME: ${{ needs.Deploy.outputs.LANAME}}
      LAWGUID: ${{ needs.Deploy.outputs.LAWGUID}}
      USERUNCMD: false
    secrets:
      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}

  Infra_Destroy:
    uses: ./.github/workflows/cleanupRg.yml
    if: github.event_name == 'schedule'
    needs: [ReusableWF, Validation, Deploy, Troubleshoot, DeployVerify, SmokeTest_SimpleApp, Well_Architected]
    with:
      environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }}
      rg: ${{ needs.Validation.outputs.RESOURCEGROUP }}
    secrets:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}