Merge pull request #531 from microsoft/dev

chore: Dev merge to main

Author: Prajwal-Microsoft

.github/workflows/azd-template-validation.yml

@@ -0,0 +1,40 @@
+name: AZD Template Validation
+on:
+  schedule:
+    - cron: '30 1 * * 4' # Every Thursday at 7:00 AM IST (1:30 AM UTC)
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+  pull-requests: write
+
+jobs:
+  template_validation:
+    runs-on: ubuntu-latest
+    name: azd template validation
+    environment: production
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set timestamp
+        run: echo "HHMM=$(date -u +'%H%M')" >> $GITHUB_ENV
+
+      - uses: microsoft/template-validation-action@v0.4.3
+        with:
+          validateAzd: ${{ vars.TEMPLATE_VALIDATE_AZD }}
+          validateTests: ${{ vars.TEMPLATE_VALIDATE_TESTS }}
+          useDevContainer: ${{ vars.TEMPLATE_USE_DEV_CONTAINER }}
+        id: validation
+        env:
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          AZURE_ENV_NAME: azd-${{ vars.AZURE_ENV_NAME }}-${{ env.HHMM }}
+          AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
+          AZURE_ENV_AI_SERVICE_LOCATION: ${{ vars.AZURE_LOCATION }}
+          AZURE_ENV_MODEL_CAPACITY: 1 # keep low to avoid potential quota issues
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: print result
+        run: cat ${{ steps.validation.outputs.resultFile }}

.github/workflows/azure-dev.yaml

@@ -1,38 +1,57 @@
-name: Azure Template Validation
+name: Azure Dev Deploy
+
 on:
   workflow_dispatch:
 
 permissions:
   contents: read
   id-token: write
-  pull-requests: write
 
 jobs:
-  template_validation_job:
-    environment: production
+  deploy:
     runs-on: ubuntu-latest
-    name: Template validation
-
+    environment: production
+    env:
+      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }}
+      AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
+      AZURE_ENV_MODEL_CAPACITY: 1 # keep low to avoid potential quota issues
+      AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
     steps:
-      # Step 1: Checkout the code from your repository
-      - name: Checkout code
-        uses: actions/checkout@v5
-
-      # Step 2: Validate the Azure template using microsoft/template-validation-action
-      - name: Validate Azure Template
-        uses: microsoft/template-validation-action@v0.4.3
-        id: validation
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Set timestamp and env name
+        run: |
+          HHMM=$(date -u +'%H%M')
+          echo "AZURE_ENV_NAME=azd-${{ vars.AZURE_ENV_NAME }}-${HHMM}" >> $GITHUB_ENV
+
+      - name: Install azd
+        uses: Azure/setup-azd@v2
+
+      - name: Login to Azure
+        uses: azure/login@v2
         with:
-          useDevContainer: false
-        env:
-          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
-          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
-          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-          AZURE_ENV_NAME: ${{ secrets.AZURE_ENV_NAME }}
-          AZURE_LOCATION: ${{ secrets.AZURE_LOCATION }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
-
-      # Step 3: Print the result of the validation
-      - name: Print result
-        run: cat ${{ steps.validation.outputs.resultFile }}
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Login to AZD
+        shell: bash
+        run: |
+          azd auth login \
+            --client-id "$AZURE_CLIENT_ID" \
+            --federated-credential-provider "github" \
+            --tenant-id "$AZURE_TENANT_ID"
+
+      - name: Provision and Deploy
+        shell: bash
+        run: |
+          if ! azd env select "$AZURE_ENV_NAME"; then
+            azd env new "$AZURE_ENV_NAME" --subscription "$AZURE_SUBSCRIPTION_ID" --location "$AZURE_LOCATION" --no-prompt
+          fi
+          azd config set defaults.subscription "$AZURE_SUBSCRIPTION_ID"
+          azd env set AZURE_ENV_AI_SERVICE_LOCATION="$AZURE_LOCATION"
+          azd up --no-prompt

README.md

@@ -278,6 +278,8 @@ Follow the quick deploy steps on the deployment guide to deploy this solution
 
 <br/>
 
+> **Note**: Some tenants may have additional security restrictions that run periodically and could impact the application (e.g., blocking public network access). If you experience issues or the application stops working, check if these restrictions are the cause. In such cases, consider deploying the WAF-supported version to ensure compliance. To configure, [Click here](./docs/DeploymentGuide.md#31-choose-deployment-type-optional).
+
 > ⚠️ **Important: Check Azure OpenAI Quota Availability**
  <br/>To ensure sufficient quota is available in your subscription, please follow [quota check instructions guide](./docs/quota_check.md) before you deploy the solution.
 

azure.yaml

@@ -5,6 +5,7 @@ name: content-processing
 
 requiredVersions:
   azd: '>= 1.18.0 != 1.23.9'
+  bicep: '>= 0.33.0'
 
 metadata:
   template: content-processing@1.0

docs/DeploymentGuide.md

@@ -6,6 +6,8 @@ This guide walks you through deploying the Content Processing Solution Accelerat
 
 🆘 **Need Help?** If you encounter any issues during deployment, check our [Troubleshooting Guide](./TroubleShootingSteps.md) for solutions to common problems.
 
+> **Note**: Some tenants may have additional security restrictions that run periodically and could impact the application (e.g., blocking public network access). If you experience issues or the application stops working, check if these restrictions are the cause. In such cases, consider deploying the WAF-supported version to ensure compliance. To configure, [Click here](#31-choose-deployment-type-optional).
+
 ## Step 1: Prerequisites & Setup
 
 ### 1.1 Azure Account Requirements

docs/TroubleShootingSteps.md

@@ -27,6 +27,7 @@ Use these as quick reference guides to unblock your deployments.
 | **InternalSubscriptionIsOverQuotaForSku** | Subscription quota exceeded for the requested SKU | [View Solution](#quota--capacity-limitations) |
 | **InvalidResourceGroup** | Invalid resource group configuration | [View Solution](#resource-group--deployment-management) |
 | **RequestDisallowedByPolicy** | Azure Policy blocking the requested operation | [View Solution](#subscription--access-issues) |
+| **403 Forbidden - Content Understanding** | Content Understanding returns 403 in WAF/private networking deployment | [View Solution](#network--infrastructure-configuration) |
 
 ## 📖 Table of Contents
 
@@ -127,6 +128,7 @@ Use these as quick reference guides to unblock your deployments.
 | **RouteTableCannotBeAttachedForAzureBastionSubnet** | Route table attached to Azure Bastion subnet | This error occurs because Azure Bastion subnet (`AzureBastionSubnet`) has a platform restriction that prevents route tables from being attached.<br><br>**How to reproduce:**<br><ul><li>In `virtualNetwork.bicep`, add `attachRouteTable: true` to the `AzureBastionSubnet` configuration:<br>`{ name: 'AzureBastionSubnet', addressPrefixes: ['10.0.10.0/26'], attachRouteTable: true }`</li><li>Add a Route Table module to the template</li><li>Update subnet creation to attach route table conditionally:<br>`routeTableResourceId: subnet.?attachRouteTable == true ? routeTable.outputs.resourceId : null`</li><li>Deploy the template → Azure throws `RouteTableCannotBeAttachedForAzureBastionSubnet`</li></ul><br>**Resolution:**<br><ul><li>Remove the `attachRouteTable: true` flag from `AzureBastionSubnet` configuration</li><li>Ensure no route table is associated with `AzureBastionSubnet`</li><li>Route tables can only be attached to other subnets, not `AzureBastionSubnet`</li><li>For more details, refer to [Azure Bastion subnet requirements](https://learn.microsoft.com/en-us/azure/bastion/configuration-settings#subnet)</li></ul> |
 | **VMSizeIsNotPermittedToEnableAcceleratedNetworking** | VM size does not support accelerated networking | This error occurs when you attempt to enable accelerated networking on a VM size that does not support it. This deployment's jumpbox VM **requires** accelerated networking.<br><br>**Default VM size:** `Standard_D2s_v5` — supports accelerated networking.<br><br>**How this error happens:**<br><ul><li>You override the VM size (via `AZURE_ENV_VM_SIZE`) with a size that doesn't support accelerated networking (e.g., `Standard_A2m_v2`, A-series, or B-series VMs)</li><li>Azure rejects the deployment with `VMSizeIsNotPermittedToEnableAcceleratedNetworking`</li></ul><br>**Resolution:**<br><ul><li>Use the default `Standard_D2s_v5` (recommended)</li><li>If overriding VM size, choose one that supports accelerated networking:<br>`Standard_D2s_v4`, `Standard_D2as_v5` (AMD), `Standard_D2s_v3`</li><li>Verify VM size supports accelerated networking:<br>`az vm list-skus --location <region> --size <vm-size> --query "[?capabilities[?name=='AcceleratedNetworkingEnabled' && value=='True']]"`</li><li>Avoid A-series and B-series VMs — they do not support accelerated networking</li><li>See [VM sizes with accelerated networking](https://learn.microsoft.com/en-us/azure/virtual-network/accelerated-networking-overview)</li></ul> |
 | **NetworkSecurityGroupNotCompliantForAzureBastionSubnet** / **SecurityRuleParameterContainsUnsupportedValue** | NSG rules blocking required Azure Bastion ports | This error occurs when the Network Security Group (NSG) attached to `AzureBastionSubnet` explicitly denies inbound TCP ports 443 and/or 4443, which Azure Bastion requires for management and tunneling.<br><br>**How to reproduce:**<br><ul><li>Deploy the template with `enablePrivateNetworking=true` so the virtualNetwork module creates `AzureBastionSubnet` and a Network Security Group that denies ports 443 and 4443</li><li>Attempt to deploy Azure Bastion into that subnet</li><li>During validation, Bastion detects the deny rules and fails with `NetworkSecurityGroupNotCompliantForAzureBastionSubnet`</li></ul><br>**Resolution:**<br><ul><li>**Remove or modify deny rules** for ports 443 and 4443 in the NSG attached to `AzureBastionSubnet`</li><li>**Ensure required inbound rules** per [Azure Bastion NSG requirements](https://learn.microsoft.com/en-us/azure/bastion/bastion-nsg)</li><li>**Use Bicep conditions** to skip NSG attachments for `AzureBastionSubnet` if deploying Bastion</li><li>**Validate the NSG configuration** before deploying Bastion into the subnet</li></ul> |
+| **403 Forbidden - Content Understanding** | Azure AI Content Understanding returns 403 Forbidden in WAF (private networking) deployment | This error occurs when the **Azure AI Content Understanding** service returns a `403 Forbidden` response during document processing in a **WAF-enabled (private networking)** deployment.<br><br>**Why this happens:**<br>In WAF deployments (`enablePrivateNetworking=true`), the Content Understanding AI Services account (`aicu-<suffix>`) is configured with `publicNetworkAccess: Disabled`. All traffic must flow through the **private endpoint** (`pep-aicu-<suffix>`) and resolve via private DNS zones (`privatelink.cognitiveservices.azure.com`, `privatelink.services.ai.azure.com`, `privatelink.contentunderstanding.ai.azure.com`). If any part of this chain is misconfigured, the request either reaches the public endpoint (which is blocked) or fails to route entirely, resulting in a 403.<br><br>**Common causes:**<br><ul><li>Private DNS zones not linked to the VNet — DNS resolution falls back to the public IP, which is blocked</li><li>Private endpoint connection is not in **Approved** state</li><li>Content Understanding is deployed in a different region (`contentUnderstandingLocation`, defaults to `WestUS`) than the main deployment — the private endpoint still works cross-region, but DNS misconfiguration is more likely</li><li>Container Apps are not injected into the VNet or are on a subnet that cannot reach the private endpoint</li><li>Managed Identity used by the Container App does not have the required **Cognitive Services User** role on the Content Understanding resource</li></ul><br>**Resolution:**<br><ul><li>**Verify private endpoint status:**<br>`az network private-endpoint show --name pep-aicu-<suffix> --resource-group <rg-name> --query "privateLinkServiceConnections[0].privateLinkServiceConnectionState.status"`<br>Expected: `Approved`</li><li>**Verify private DNS zone VNet links:**<br>`az network private-dns zone list --resource-group <rg-name> -o table`<br>Ensure `privatelink.cognitiveservices.azure.com`, `privatelink.services.ai.azure.com`, and `privatelink.contentunderstanding.ai.azure.com` all have VNet links</li><li>**Test DNS resolution from the jumpbox VM** (inside the VNet):<br>`nslookup aicu-<suffix>.cognitiveservices.azure.com`<br>Should resolve to a private IP (e.g., `10.x.x.x`), NOT a public IP</li><li>**Verify RBAC role assignments:** Ensure the Container App managed identity has **Cognitive Services User** role on the Content Understanding resource:<br>`az role assignment list --scope /subscriptions/<sub-id>/resourceGroups/<rg-name>/providers/Microsoft.CognitiveServices/accounts/aicu-<suffix> --query "[?roleDefinitionName=='Cognitive Services User']" -o table`</li><li>**Check Container App VNet integration:** Confirm the Container App Environment is deployed into the VNet and can reach the backend subnet where the private endpoint resides</li><li>**Redeploy if needed:**<br>`azd up`</li></ul><br>**Reference:**<br><ul><li>[Configure private endpoints for Azure AI Services](https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks)</li><li>[Azure Private DNS zones](https://learn.microsoft.com/en-us/azure/dns/private-dns-overview)</li></ul> |
 
 ---------------------------------
 

infra/main.bicep

@@ -1890,5 +1890,8 @@ output CONTAINER_REGISTRY_NAME string = avmContainerRegistry.outputs.name
 @description('The login server of the Azure Container Registry.')
 output CONTAINER_REGISTRY_LOGIN_SERVER string = avmContainerRegistry.outputs.loginServer
 
+@description('The name of the Content Understanding AI Services account.')
+output CONTENT_UNDERSTANDING_ACCOUNT_NAME string = avmAiServices_cu.outputs.name
+
 @description('The resource group the resources were deployed into.')
 output AZURE_RESOURCE_GROUP string = resourceGroup().name

infra/scripts/post_deployment.sh

@@ -98,8 +98,8 @@ else
 
   # Read schema entries from manifest
   SCHEMA_COUNT=$(cat "$SCHEMA_INFO_FILE" | grep -o '"File"' | wc -l)
-  REGISTERED_IDS=""
-  REGISTERED_NAMES=""
+  REGISTERED_IDS=()
+  REGISTERED_NAMES=()
 
   for idx in $(seq 0 $((SCHEMA_COUNT - 1))); do
     # Parse entry fields using grep/sed (no python needed)
@@ -128,8 +128,8 @@ else
 
     if [ -n "$EXISTING_ID" ]; then
       echo "  Schema '$CLASS_NAME' already exists with ID: $EXISTING_ID"
-      REGISTERED_IDS="$REGISTERED_IDS $EXISTING_ID"
-      REGISTERED_NAMES="$REGISTERED_NAMES $CLASS_NAME"
+      REGISTERED_IDS+=("$EXISTING_ID")
+      REGISTERED_NAMES+=("$CLASS_NAME")
       continue
     fi
 
@@ -148,8 +148,8 @@ else
     if [ "$HTTP_CODE" = "200" ]; then
       SCHEMA_ID=$(echo "$BODY" | sed 's/.*"Id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/')
       echo "  Successfully registered: $DESCRIPTION's Schema Id - $SCHEMA_ID"
-      REGISTERED_IDS="$REGISTERED_IDS $SCHEMA_ID"
-      REGISTERED_NAMES="$REGISTERED_NAMES $CLASS_NAME"
+      REGISTERED_IDS+=("$SCHEMA_ID")
+      REGISTERED_NAMES+=("$CLASS_NAME")
     else
       echo "  Failed to upload '$FILE_NAME'. HTTP Status: $HTTP_CODE"
       echo "  Error Response: $BODY"
@@ -205,10 +205,9 @@ else
     ALREADY_IN_SET=$(curl -s "${SCHEMASETVAULT_URL}${SCHEMASET_ID}/schemas" 2>/dev/null || echo "[]")
 
     # Iterate over registered schemas
-    IDX=0
-    for SCHEMA_ID in $REGISTERED_IDS; do
-      IDX=$((IDX + 1))
-      CLASS_NAME=$(echo "$REGISTERED_NAMES" | tr ' ' '\n' | sed -n "${IDX}p")
+    for i in "${!REGISTERED_IDS[@]}"; do
+      SCHEMA_ID="${REGISTERED_IDS[$i]}"
+      CLASS_NAME="${REGISTERED_NAMES[$i]}"
 
       if echo "$ALREADY_IN_SET" | grep -q "\"Id\"[[:space:]]*:[[:space:]]*\"$SCHEMA_ID\""; then
         echo "  Schema '$CLASS_NAME' ($SCHEMA_ID) already in schema set - skipped"
@@ -236,5 +235,29 @@ else
   echo ""
   echo "============================================================"
   echo "Schema registration process completed."
+  echo "  Schemas registered: ${#REGISTERED_IDS[@]}"
   echo "============================================================"
 fi
+
+# --- Refresh Content Understanding Cognitive Services account ---
+echo ""
+echo "============================================================"
+echo "Refreshing Content Understanding Cognitive Services account..."
+echo "============================================================"
+
+CU_ACCOUNT_NAME=$(azd env get-value CONTENT_UNDERSTANDING_ACCOUNT_NAME 2>/dev/null || echo "")
+
+if [ -z "$CU_ACCOUNT_NAME" ]; then
+  echo "  ⚠️ CONTENT_UNDERSTANDING_ACCOUNT_NAME not found in azd env. Skipping refresh."
+else
+  echo "  Refreshing account: $CU_ACCOUNT_NAME in resource group: $RESOURCE_GROUP"
+  if az cognitiveservices account update \
+    -g "$RESOURCE_GROUP" \
+    -n "$CU_ACCOUNT_NAME" \
+    --tags refresh=true \
+    --output none; then
+    echo "  ✅ Successfully refreshed Cognitive Services account '$CU_ACCOUNT_NAME'."
+  else
+    echo "  ❌ Failed to refresh Cognitive Services account '$CU_ACCOUNT_NAME'."
+  fi
+fi

infra/scripts/validate_bicep_params.py

@@ -108,7 +108,9 @@ def parse_parameters_env_vars(json_path: Path) -> dict[str, list[str]]:
         data = json.loads(sanitized)
         params = data.get("parameters", {})
     except json.JSONDecodeError:
-        pass
+        # Keep validation resilient for partially templated/malformed files:
+        # if JSON parsing fails, treat as having no parsable parameters.
+        params = {}
 
     # Walk each top-level parameter and scan its entire serialized value
     # for ${VAR} references from the original text.

src/ContentProcessor/azure_cicd.yml

@@ -2,6 +2,9 @@ trigger:
   branches:
     include:
       - main
+  paths:
+    include:
+      - src/ContentProcessor/**
 
   # When multiple commits land quickly on main, only run the latest.
   batch: true