CNI improvements and UI tweaks (#413)

* first cut of maxpods/overlay/dynamiciP Signed-off-by: Gordonby <gordon.byers@microsoft.com> * better ui Signed-off-by: Gordonby <gordon.byers@microsoft.com> * preview styling and keda preview link Signed-off-by: Gordonby <gordon.byers@microsoft.com> * CNI mainly working. Signed-off-by: Gordonby <gordon.byers@microsoft.com> * cspell typos Signed-off-by: Gordonby <gordon.byers@microsoft.com> * making cni overlay a preview feature Signed-off-by: Gordonby <gordon.byers@microsoft.com> * maxPods Signed-off-by: Gordonby <gordon.byers@microsoft.com> * removing modal dialog Signed-off-by: Gordonby <gordon.byers@microsoft.com> * pod cidr Signed-off-by: Gordonby <gordon.byers@microsoft.com> * deploy tests started Signed-off-by: Gordonby <gordon.byers@microsoft.com> * mutual feature exclusion Signed-off-by: Gordonby <gordon.byers@microsoft.com> * indentation Signed-off-by: Gordonby <gordon.byers@microsoft.com> * styling Signed-off-by: Gordonby <gordon.byers@microsoft.com> * Preview warning control Signed-off-by: Gordonby <gordon.byers@microsoft.com> * making the if statements niiiiice Signed-off-by: Gordonby <gordon.byers@microsoft.com> * scale step * overlay plus default networking compat. Signed-off-by: Gordonby <gordon.byers@microsoft.com> * Update markdownchecks.yml * psrule version lock * Update ps-rule.yaml * scale code fix * region data type * scale code tweak * autoscale code tweak * Update ByoVnetCI.yml * --update-cluster-autoscaler * casing issue? * using a tag instead of main Signed-off-by: Gordonby <gordon.byers@microsoft.com> Signed-off-by: Gordonby <gordon.byers@microsoft.com>
Azure · Oct 7, 2022 · 068c967 · 068c967
1 parent b398eae
commit 068c967
Show file tree

Hide file tree

Showing 13 changed files with 202 additions and 28 deletions.
diff --git a/.github/workflows/ByoVnetCI.yml b/.github/workflows/ByoVnetCI.yml
@@ -36,7 +36,7 @@ on:
       region:
         description: 'Region (needs to be same as byo vnet location)'
         default: 'southcentralus'
-        type: text
+        type: string
         required: false
       doWellArchitected:
         description: 'Perform the Well Architected Framework assesment'
@@ -84,7 +84,7 @@ jobs:
       # PSRule performs IaC recommendations of the template.
       # https://azure.github.io/PSRule.Rules.Azure/
       - name: PSRule for Azure - Well Architected
-        uses: Microsoft/ps-rule@main
+        uses: microsoft/ps-rule@v2.3.2
         continue-on-error: true #Setting this whilst PSRule gets bedded in, in this project
         with:
           modules: 'PSRule.Rules.Azure'
@@ -108,7 +108,7 @@ jobs:
       RESOURCEGROUP:  ${{ steps.params.outputs.RESOURCEGROUP}}
       REGION: ${{ steps.params.outputs.REGION}}
       RESNAME: ${{ steps.params.outputs.NEWRESNAME}}
-      
+
     steps:
       - uses: actions/checkout@v2
 
@@ -144,7 +144,7 @@ jobs:
             $params=$paramFileContent|ConvertFrom-Json
             Write-Output $params.parameters.ingressApplicationGateway.value
           }
-          
+
       - name: Parameter Value Augmentation
         id: params
         env:
@@ -692,6 +692,23 @@ jobs:
           }
           #grep KeyvaultSecretsProvider
 
+        #Sometimes cluster config works for deployment, but misconfig can
+        #Prevent the cluster from scaling... So lets make sure it can.
+      - name: Scale the cluster up by 1 node
+        shell: pwsh
+        run: |
+          $RG='${{ env.RG }}'
+          $AKSNAME='${{ needs.Deploy.outputs.AKSNAME }}'
+
+          Write-Output "Scaling $AKSNAME in $RG"
+
+          $manualScalePools = az aks show -n $AKSNAME -g $RG --query "agentPoolProfiles[?maxCount==null].{name:name, count:count}" -o json | ConvertFrom-Json
+          $manualScalePools | ForEach-Object { Write-Output "scaling [m] pool $($_.name)"; az aks scale -g $RG -n $AKSNAME --node-count $($_.pool + 1) --nodepool-name $_.name }
+
+          $autoScalePools = az aks show -n $AKSNAME -g $RG --query "agentPoolProfiles[?maxCount!=null].{name:name, minCount:minCount, maxCount:maxCount}" -o json | ConvertFrom-Json
+          $autoScalePools | ForEach-Object { Write-Output "scaling [a] pool $($_.name)"; az aks nodepool update --update-cluster-autoscaler -g $RG --cluster-name $AKSNAME --name $_.name --min-count $($_.minCount + 1) --max-count $($_.maxCount + 1)  }
+
+
   Troubleshoot:
     needs: [Deploy, ReusableWF, Post-Deploy, SmokeTest_JavaApp-certmgr, SmokeTest_JavaApp-appgw]
     uses: ./.github/workflows/AksTroubleshooting.yml

diff --git a/.github/workflows/markdownchecks.yml b/.github/workflows/markdownchecks.yml
@@ -12,7 +12,7 @@ jobs:
         run: cat ./.github/workflows_dep/_typos.toml
 
       - name: Check spelling of markdown files
-        uses: crate-ci/typos@master
+        uses: crate-ci/typos@v1.12.8
         with:
           files: ./*.md
           config: ./.github/workflows_dep/_typos.toml
diff --git a/.github/workflows_dep/regressionparams/cni-overlay.json b/.github/workflows_dep/regressionparams/cni-overlay.json
@@ -0,0 +1,39 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "resourceName": {
+      "value": "cniov2"
+    },
+    "custom_vnet": {
+      "value": true
+    },
+    "enable_aad": {
+      "value": true
+    },
+    "aksDisableLocalAccounts": {
+      "value": true
+    },
+    "enableAzureRBAC": {
+      "value": true
+    },
+    "registries_sku": {
+      "value": "Premium"
+    },
+    "omsagent": {
+      "value": true
+    },
+    "retentionInDays": {
+      "value": 30
+    },
+    "networkPolicy": {
+      "value": "azure"
+    },
+    "networkPluginMode": {
+      "value": "Overlay"
+    },
+    "vnetAksSubnetAddressPrefix": {
+      "value": "10.240.0.0/28"
+    }
+  }
+}
diff --git a/bicep/main.bicep b/bicep/main.bicep
@@ -870,6 +870,11 @@ param agentCount int = 3
 param agentCountMax int = 0
 var autoScale = agentCountMax > agentCount
 
+@description('Allocate pod ips dynamically')
+param cniDynamicIpAllocation bool = false
+
+@minValue(10)
+@maxValue(250)
 @description('The maximum number of pods per node.')
 param maxPods int = 30
 
@@ -880,6 +885,13 @@ param maxPods int = 30
 @description('The network plugin type')
 param networkPlugin string = 'azure'
 
+@allowed([
+  ''
+  'Overlay'
+])
+@description('The network plugin type')
+param networkPluginMode string = ''
+
 @allowed([
   ''
   'azure'
@@ -1199,7 +1211,8 @@ var aksProperties = union({
     networkPlugin: networkPlugin
     #disable-next-line BCP036 //Disabling validation of this parameter to cope with empty string to indicate no Network Policy required.
     networkPolicy: networkPolicy
-    podCidr: networkPlugin=='kubenet' ? podCidr : json('null')
+    networkPluginMode: networkPlugin=='azure' ? networkPluginMode : ''
+    podCidr: networkPlugin=='kubenet' || cniDynamicIpAllocation ? podCidr : json('null')
     serviceCidr: serviceCidr
     dnsServiceIP: dnsServiceIP
     dockerBridgeCidr: dockerBridgeCidr
@@ -1251,6 +1264,14 @@ resource aks 'Microsoft.ContainerService/managedClusters@2022-05-02-preview' = {
 }
 output aksClusterName string = aks.name
 output aksOidcIssuerUrl string = oidcIssuer ? aks.properties.oidcIssuerProfile.issuerURL : ''
+
+@description('This output can be directly leveraged when creating a ManagedId Federated Identity')
+output aksOidcFedIdentityProperties object = {
+  issuer: oidcIssuer ? aks.properties.oidcIssuerProfile.issuerURL : ''
+  audiences: ['api://AzureADTokenExchange']
+  subject: 'system:serviceaccount:ns:svcaccount'
+}
+
 output aksNodeResourceGroup string = aks.properties.nodeResourceGroup
 //output aksNodePools array = [for nodepool in agentPoolProfiles: name]
 

diff --git a/cspell.json b/cspell.json
@@ -37,6 +37,7 @@
         "Codespaces",
         "configpresets",
         "Consolas",
+        "Cred",
         "csisecret",
         "csisecrets",
         "currenttab",
@@ -84,9 +85,11 @@
         "localaccounts",
         "managedclusters",
         "MAXCOUNT",
+        "maxpods",
         "messg",
         "Microservices",
         "middleeast",
+        "Modeless",
         "monospace",
         "mounttime",
         "msrc",

diff --git a/docs/RepoCoreIacWorkflows.md b/docs/RepoCoreIacWorkflows.md
@@ -79,7 +79,7 @@ Capture the subnet id and save into the appropriate GitHub secret.
 
 You need to create the RBAC for the service principal on the resource groups.
 
-> *** Error: list: failed to list: secrets is forbidden: User \"REDACTED\" cannot list resource \"secrets\" in API group \"\" in the namespace \"default\": User does not have access to the resource in Azure. Update role assignment to allow access.\n", "provisioningState": "Succeeded", "reason": null, 
+> *** Error: list: failed to list: secrets is forbidden: User \"REDACTED\" cannot list resource \"secrets\" in API group \"\" in the namespace \"default\": User does not have access to the resource in Azure. Update role assignment to allow access.\n", "provisioningState": "Succeeded", "reason": null,
 
 ### Key Vault Certificate problem
 
@@ -100,7 +100,7 @@ az feature register -n AutoUpgradePreview --namespace Microsoft.ContainerService
 
 > SubnetNotAssociatedWithNATGateway. Subnet '***' must have a NAT gateway associated for outbound connection
 
-AKS is configured to use Nat Gatway for egress. It needs to be created and associated to the subnet.
+AKS is configured to use Nat Gateway for egress. It needs to be created and associated to the subnet.
 
 ### Nat Gateway Public IP
 

diff --git a/helper/src/components/addonsTab.js b/helper/src/components/addonsTab.js
@@ -366,10 +366,10 @@ export default function ({ tabValues, updateFn, featureFlag, invalidArray }) {
 
             <Stack.Item align="start">
                 <Label required={true}>
-                    KEDA : Enable Kubernetes Event-driven Autoscaling (KEDA) on the AKS Cluster
+                    KEDA : Enable Kubernetes Event-driven Autoscaling (KEDA) on the AKS Cluster (<a target="_new" href="https://learn.microsoft.com/en-us/azure/aks/keda-deploy-add-on-arm#prerequisites">*preview</a>)
                     (<a target="_new" href="https://docs.microsoft.com/en-us/azure/aks/keda-about">docs</a>)
                 </Label>
-                <Checkbox styles={{ root: { marginLeft: '50px' } }} checked={addons.kedaAddon} onChange={(ev, v) => updateFn("kedaAddon", v)} label="Install the KEDA AddOn" />
+                <Checkbox styles={{ root: { marginLeft: '50px' } }} checked={addons.kedaAddon} onChange={(ev, v) => updateFn("kedaAddon", v, 'https://learn.microsoft.com/azure/aks/keda-deploy-add-on-arm#prerequisites')} label="Install the KEDA AddOn" />
             </Stack.Item>
 
             <Separator className="notopmargin" />

diff --git a/helper/src/components/clusterTab.js b/helper/src/components/clusterTab.js
@@ -56,7 +56,6 @@ export default function ({ tabValues, updateFn, featureFlag, invalidArray }) {
     }
 
 
-
     return (
         <Stack tokens={{ childrenGap: 15 }} styles={adv_stackstyle}>
 
@@ -81,7 +80,6 @@ export default function ({ tabValues, updateFn, featureFlag, invalidArray }) {
                     <Stack.Item>
                         <Label >System Pool Type <Link target='_' href='https://docs.microsoft.com/azure/aks/use-system-pools#system-and-user-node-pools'>docs</Link></Label>
                         <ChoiceGroup
-
                             selectedKey={cluster.SystemPoolType}
                             options={[
                                 { "data-testid":'cluster-systempool-none', key: 'none', text: 'No separate system pool: Use a single pool for System and User workloads' },
@@ -112,7 +110,10 @@ export default function ({ tabValues, updateFn, featureFlag, invalidArray }) {
                             ]} />
                     </Stack.Item>
                     <Stack.Item>
-                        <Slider buttonProps={{ "data-testid": "cluster-agentCount-slider"}} styles={{ root: { width: 450 } }} ranged={cluster.autoscale}  lowerValue={cluster.agentCount}
+                        <Slider
+                        buttonProps={{ "data-testid": "cluster-agentCount-slider"}}
+                        styles={{ root: { width: 450 } }}
+                        ranged={cluster.autoscale}  lowerValue={cluster.agentCount}
                         label={`Node count range ${cluster.autoscale ? 'range' : ''}`} min={0}  max={100} step={1}
                         value={cluster.autoscale? cluster.maxCount : cluster.agentCount} showValue={true}
                         onChange={(val, range) => sliderUpdateFn(cluster.autoscale ? {agentCount: range[0], maxCount: range[1]} : {agentCount: val})} />

diff --git a/helper/src/components/deployTab.js b/helper/src/components/deployTab.js
@@ -68,8 +68,11 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray,
     ...(addons.azurepolicy !== "none" && addons.azurePolicyInitiative !== defaults.addons.azurePolicyInitiative && { azurePolicyInitiative: addons.azurePolicyInitiative }),
     ...(net.networkPlugin !== defaults.net.networkPlugin && {networkPlugin: net.networkPlugin}),
     ...(net.vnet_opt === "custom" && net.networkPlugin === 'kubenet' && defaults.net.podCidr !== net.podCidr && { podCidr: net.podCidr }),
+    ...((net.vnet_opt === "custom" || net.vnet_opt === "byo") && defaults.net.cniDynamicIpAllocation !== net.cniDynamicIpAllocation && { cniDynamicIpAllocation: true }),
+    ...(net.vnet_opt === "custom" && net.cniDynamicIpAllocation && defaults.net.podCidr !== net.podCidr && { podCidr: net.podCidr }),
     ...(cluster.availabilityZones === "yes" && { availabilityZones: ['1', '2', '3'] }),
     ...(cluster.apisecurity === "whitelist" && deploy.clusterIPWhitelist && apiips_array.length > 0 && { authorizedIPRanges: apiips_array }),
+    ...(defaults.net.maxPods !== net.maxPods && { maxPods: net.maxPods }),
     ...(cluster.apisecurity === "private" && { enablePrivateCluster: true }),
     ...(cluster.apisecurity === "private" && cluster.apisecurity === "private" && defaults.cluster.privateClusterDnsMethod !== cluster.privateClusterDnsMethod && { privateClusterDnsMethod: cluster.privateClusterDnsMethod }),
     ...(cluster.apisecurity === "private" && cluster.apisecurity === "private" && cluster.privateClusterDnsMethod === 'privateDnsZone' && { dnsApiPrivateZoneId: cluster.dnsApiPrivateZoneId }),
@@ -124,6 +127,7 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray,
     ...(defaults.addons.kedaAddon !== addons.kedaAddon && {kedaAddon: addons.kedaAddon }),
     ...(defaults.addons.blobCSIAddon !== addons.blobCSIAddon && {blobCSIAddon: addons.blobCSIAddon }),
     ...(defaults.addons.workloadIdentity !== addons.workloadIdentity && {workloadIdentity: addons.workloadIdentity }),
+    ...(net.networkPlugin === 'azure' && net.networkPluginMode && {networkPluginMode: 'Overlay'}),
     ...(urlParams.getAll('feature').includes('defender') && cluster.DefenderForContainers !== defaults.cluster.DefenderForContainers && { DefenderForContainers: cluster.DefenderForContainers })
   }
 
@@ -297,7 +301,7 @@ az role assignment create --role "Managed Identity Operator" --assignee-principa
 
     <Stack tokens={{ childrenGap: 15 }} styles={adv_stackstyle}>
       {!allok &&
-        <MessageBar messageBarType={MessageBarType.severeWarning}>
+        <MessageBar messageBarType={MessageBarType.error}>
           <Text >Configuration not complete, please correct the tabs with the warning symbol <b>({invalidTabs.join(' & ')})</b> before deploying</Text>
         </MessageBar>
       }
@@ -366,9 +370,13 @@ az role assignment create --role "Managed Identity Operator" --assignee-principa
       <Separator styles={{ root: { marginTop: '30px !important' } }}><div style={{ display: "flex", alignItems: 'center', }}><b style={{ marginRight: '10px' }}>Deploy Cluster</b><Image src="./bicep.png" alt="Built with bicep" /> <p style={{ marginLeft: '10px' }}>powered by Bicep</p></div> </Separator>
 
       {Object.keys(preview_params).length > 0 &&
-        <MessageBar messageBarType={MessageBarType.warning}>
-          <Text >Your deployment contains Preview features: <b>{Object.keys(preview_params).join(', ')}</b>, Ensure you have registered for these previews, and have installed the <b>'az extension add --name aks-preview'</b>  before running the script, <Link target="_pv" href="https://aka.ms/aks/previews">see here</Link>, or disable preview features here</Text>
-          <Toggle styles={{ root: { marginTop: "10px" } }} onText='preview enabled' offText="preview disabled" checked={!deploy.disablePreviews} onChange={(ev, checked) => updateFn("disablePreviews", !checked)} />
+        <MessageBar messageBarType={MessageBarType.severeWarning}>
+          <Text variant={'mediumPlus'} >Your deployment contains <b>Preview Features</b> which may require subscription registration and have Azure Region limitations. Please ensure you have registered for these previews, and have installed the <b>'az extension add --name aks-preview'</b> before running the relevant scripts.<br />Preview Features you have selected: <b>{Object.keys(preview_params).join(', ')}</b>.</Text>
+          <Checkbox
+            styles={{ root: { marginTop: "10px" } }}
+            label='Include preview features in deployment'
+            checked={!deploy.disablePreviews}
+            onChange={(ev, checked) => updateFn("disablePreviews", !checked)} />
         </MessageBar>
 
       }
@@ -452,7 +460,7 @@ on:
 
 jobs:
   reusable_workflow_job:
-    uses: Azure/AKS-Construction/.github/workflows/AKSC_Deploy.yml@main
+    uses: Azure/AKS-Construction/.github/workflows/AKSC_Deploy.yml@${deploy.selectedTemplate}
     with:` + (deploy.selectedTemplate !== 'local' ? `
       templateVersion: ${deploy.selectedTemplate}` : '') + `
       rg: ${deploy.rg}