From 068c9679df318522b0db4a6b6ca60992dab41943 Mon Sep 17 00:00:00 2001 From: Gordon Byers Date: Fri, 7 Oct 2022 15:36:07 +0100 Subject: [PATCH] CNI improvements and UI tweaks (#413) * first cut of maxpods/overlay/dynamiciP Signed-off-by: Gordonby * better ui Signed-off-by: Gordonby * preview styling and keda preview link Signed-off-by: Gordonby * CNI mainly working. Signed-off-by: Gordonby * cspell typos Signed-off-by: Gordonby * making cni overlay a preview feature Signed-off-by: Gordonby * maxPods Signed-off-by: Gordonby * removing modal dialog Signed-off-by: Gordonby * pod cidr Signed-off-by: Gordonby * deploy tests started Signed-off-by: Gordonby * mutual feature exclusion Signed-off-by: Gordonby * indentation Signed-off-by: Gordonby * styling Signed-off-by: Gordonby * Preview warning control Signed-off-by: Gordonby * making the if statements niiiiice Signed-off-by: Gordonby * scale step * overlay plus default networking compat. Signed-off-by: Gordonby * Update markdownchecks.yml * psrule version lock * Update ps-rule.yaml * scale code fix * region data type * scale code tweak * autoscale code tweak * Update ByoVnetCI.yml * --update-cluster-autoscaler * casing issue? * using a tag instead of main Signed-off-by: Gordonby Signed-off-by: Gordonby --- .github/workflows/ByoVnetCI.yml | 25 +++++- .github/workflows/markdownchecks.yml | 2 +- .../regressionparams/cni-overlay.json | 39 ++++++++ bicep/main.bicep | 23 ++++- cspell.json | 3 + docs/RepoCoreIacWorkflows.md | 4 +- helper/src/components/addonsTab.js | 4 +- helper/src/components/clusterTab.js | 7 +- helper/src/components/deployTab.js | 18 ++-- helper/src/components/networkTab.js | 88 +++++++++++++++++-- helper/src/components/portalnav.js | 13 +-- helper/src/config.json | 3 + ps-rule.yaml | 1 + 13 files changed, 202 insertions(+), 28 deletions(-) create mode 100644 .github/workflows_dep/regressionparams/cni-overlay.json diff --git a/.github/workflows/ByoVnetCI.yml b/.github/workflows/ByoVnetCI.yml index b1d11d699..23d6afdc0 100644 --- a/.github/workflows/ByoVnetCI.yml +++ b/.github/workflows/ByoVnetCI.yml @@ -36,7 +36,7 @@ on: region: description: 'Region (needs to be same as byo vnet location)' default: 'southcentralus' - type: text + type: string required: false doWellArchitected: description: 'Perform the Well Architected Framework assesment' @@ -84,7 +84,7 @@ jobs: # PSRule performs IaC recommendations of the template. # https://azure.github.io/PSRule.Rules.Azure/ - name: PSRule for Azure - Well Architected - uses: Microsoft/ps-rule@main + uses: microsoft/ps-rule@v2.3.2 continue-on-error: true #Setting this whilst PSRule gets bedded in, in this project with: modules: 'PSRule.Rules.Azure' @@ -108,7 +108,7 @@ jobs: RESOURCEGROUP: ${{ steps.params.outputs.RESOURCEGROUP}} REGION: ${{ steps.params.outputs.REGION}} RESNAME: ${{ steps.params.outputs.NEWRESNAME}} - + steps: - uses: actions/checkout@v2 @@ -144,7 +144,7 @@ jobs: $params=$paramFileContent|ConvertFrom-Json Write-Output $params.parameters.ingressApplicationGateway.value } - + - name: Parameter Value Augmentation id: params env: @@ -692,6 +692,23 @@ jobs: } #grep KeyvaultSecretsProvider + #Sometimes cluster config works for deployment, but misconfig can + #Prevent the cluster from scaling... So lets make sure it can. + - name: Scale the cluster up by 1 node + shell: pwsh + run: | + $RG='${{ env.RG }}' + $AKSNAME='${{ needs.Deploy.outputs.AKSNAME }}' + + Write-Output "Scaling $AKSNAME in $RG" + + $manualScalePools = az aks show -n $AKSNAME -g $RG --query "agentPoolProfiles[?maxCount==null].{name:name, count:count}" -o json | ConvertFrom-Json + $manualScalePools | ForEach-Object { Write-Output "scaling [m] pool $($_.name)"; az aks scale -g $RG -n $AKSNAME --node-count $($_.pool + 1) --nodepool-name $_.name } + + $autoScalePools = az aks show -n $AKSNAME -g $RG --query "agentPoolProfiles[?maxCount!=null].{name:name, minCount:minCount, maxCount:maxCount}" -o json | ConvertFrom-Json + $autoScalePools | ForEach-Object { Write-Output "scaling [a] pool $($_.name)"; az aks nodepool update --update-cluster-autoscaler -g $RG --cluster-name $AKSNAME --name $_.name --min-count $($_.minCount + 1) --max-count $($_.maxCount + 1) } + + Troubleshoot: needs: [Deploy, ReusableWF, Post-Deploy, SmokeTest_JavaApp-certmgr, SmokeTest_JavaApp-appgw] uses: ./.github/workflows/AksTroubleshooting.yml diff --git a/.github/workflows/markdownchecks.yml b/.github/workflows/markdownchecks.yml index 6a8ca25a6..9161ecdd3 100644 --- a/.github/workflows/markdownchecks.yml +++ b/.github/workflows/markdownchecks.yml @@ -12,7 +12,7 @@ jobs: run: cat ./.github/workflows_dep/_typos.toml - name: Check spelling of markdown files - uses: crate-ci/typos@master + uses: crate-ci/typos@v1.12.8 with: files: ./*.md config: ./.github/workflows_dep/_typos.toml diff --git a/.github/workflows_dep/regressionparams/cni-overlay.json b/.github/workflows_dep/regressionparams/cni-overlay.json new file mode 100644 index 000000000..8ab8f52f5 --- /dev/null +++ b/.github/workflows_dep/regressionparams/cni-overlay.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "resourceName": { + "value": "cniov2" + }, + "custom_vnet": { + "value": true + }, + "enable_aad": { + "value": true + }, + "aksDisableLocalAccounts": { + "value": true + }, + "enableAzureRBAC": { + "value": true + }, + "registries_sku": { + "value": "Premium" + }, + "omsagent": { + "value": true + }, + "retentionInDays": { + "value": 30 + }, + "networkPolicy": { + "value": "azure" + }, + "networkPluginMode": { + "value": "Overlay" + }, + "vnetAksSubnetAddressPrefix": { + "value": "10.240.0.0/28" + } + } +} diff --git a/bicep/main.bicep b/bicep/main.bicep index 52c23abef..c8cfd94d7 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -870,6 +870,11 @@ param agentCount int = 3 param agentCountMax int = 0 var autoScale = agentCountMax > agentCount +@description('Allocate pod ips dynamically') +param cniDynamicIpAllocation bool = false + +@minValue(10) +@maxValue(250) @description('The maximum number of pods per node.') param maxPods int = 30 @@ -880,6 +885,13 @@ param maxPods int = 30 @description('The network plugin type') param networkPlugin string = 'azure' +@allowed([ + '' + 'Overlay' +]) +@description('The network plugin type') +param networkPluginMode string = '' + @allowed([ '' 'azure' @@ -1199,7 +1211,8 @@ var aksProperties = union({ networkPlugin: networkPlugin #disable-next-line BCP036 //Disabling validation of this parameter to cope with empty string to indicate no Network Policy required. networkPolicy: networkPolicy - podCidr: networkPlugin=='kubenet' ? podCidr : json('null') + networkPluginMode: networkPlugin=='azure' ? networkPluginMode : '' + podCidr: networkPlugin=='kubenet' || cniDynamicIpAllocation ? podCidr : json('null') serviceCidr: serviceCidr dnsServiceIP: dnsServiceIP dockerBridgeCidr: dockerBridgeCidr @@ -1251,6 +1264,14 @@ resource aks 'Microsoft.ContainerService/managedClusters@2022-05-02-preview' = { } output aksClusterName string = aks.name output aksOidcIssuerUrl string = oidcIssuer ? aks.properties.oidcIssuerProfile.issuerURL : '' + +@description('This output can be directly leveraged when creating a ManagedId Federated Identity') +output aksOidcFedIdentityProperties object = { + issuer: oidcIssuer ? aks.properties.oidcIssuerProfile.issuerURL : '' + audiences: ['api://AzureADTokenExchange'] + subject: 'system:serviceaccount:ns:svcaccount' +} + output aksNodeResourceGroup string = aks.properties.nodeResourceGroup //output aksNodePools array = [for nodepool in agentPoolProfiles: name] diff --git a/cspell.json b/cspell.json index df7ea88a1..171f12712 100644 --- a/cspell.json +++ b/cspell.json @@ -37,6 +37,7 @@ "Codespaces", "configpresets", "Consolas", + "Cred", "csisecret", "csisecrets", "currenttab", @@ -84,9 +85,11 @@ "localaccounts", "managedclusters", "MAXCOUNT", + "maxpods", "messg", "Microservices", "middleeast", + "Modeless", "monospace", "mounttime", "msrc", diff --git a/docs/RepoCoreIacWorkflows.md b/docs/RepoCoreIacWorkflows.md index f46a1870c..ef101edf6 100644 --- a/docs/RepoCoreIacWorkflows.md +++ b/docs/RepoCoreIacWorkflows.md @@ -79,7 +79,7 @@ Capture the subnet id and save into the appropriate GitHub secret. You need to create the RBAC for the service principal on the resource groups. -> *** Error: list: failed to list: secrets is forbidden: User \"REDACTED\" cannot list resource \"secrets\" in API group \"\" in the namespace \"default\": User does not have access to the resource in Azure. Update role assignment to allow access.\n", "provisioningState": "Succeeded", "reason": null, +> *** Error: list: failed to list: secrets is forbidden: User \"REDACTED\" cannot list resource \"secrets\" in API group \"\" in the namespace \"default\": User does not have access to the resource in Azure. Update role assignment to allow access.\n", "provisioningState": "Succeeded", "reason": null, ### Key Vault Certificate problem @@ -100,7 +100,7 @@ az feature register -n AutoUpgradePreview --namespace Microsoft.ContainerService > SubnetNotAssociatedWithNATGateway. Subnet '***' must have a NAT gateway associated for outbound connection -AKS is configured to use Nat Gatway for egress. It needs to be created and associated to the subnet. +AKS is configured to use Nat Gateway for egress. It needs to be created and associated to the subnet. ### Nat Gateway Public IP diff --git a/helper/src/components/addonsTab.js b/helper/src/components/addonsTab.js index eb6cf9ff5..20b977c51 100644 --- a/helper/src/components/addonsTab.js +++ b/helper/src/components/addonsTab.js @@ -366,10 +366,10 @@ export default function ({ tabValues, updateFn, featureFlag, invalidArray }) { - updateFn("kedaAddon", v)} label="Install the KEDA AddOn" /> + updateFn("kedaAddon", v, 'https://learn.microsoft.com/azure/aks/keda-deploy-add-on-arm#prerequisites')} label="Install the KEDA AddOn" /> diff --git a/helper/src/components/clusterTab.js b/helper/src/components/clusterTab.js index a0072452e..589c0b491 100644 --- a/helper/src/components/clusterTab.js +++ b/helper/src/components/clusterTab.js @@ -56,7 +56,6 @@ export default function ({ tabValues, updateFn, featureFlag, invalidArray }) { } - return ( @@ -81,7 +80,6 @@ export default function ({ tabValues, updateFn, featureFlag, invalidArray }) { - sliderUpdateFn(cluster.autoscale ? {agentCount: range[0], maxCount: range[1]} : {agentCount: val})} /> diff --git a/helper/src/components/deployTab.js b/helper/src/components/deployTab.js index 38f25e27e..2678db8cd 100644 --- a/helper/src/components/deployTab.js +++ b/helper/src/components/deployTab.js @@ -68,8 +68,11 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray, ...(addons.azurepolicy !== "none" && addons.azurePolicyInitiative !== defaults.addons.azurePolicyInitiative && { azurePolicyInitiative: addons.azurePolicyInitiative }), ...(net.networkPlugin !== defaults.net.networkPlugin && {networkPlugin: net.networkPlugin}), ...(net.vnet_opt === "custom" && net.networkPlugin === 'kubenet' && defaults.net.podCidr !== net.podCidr && { podCidr: net.podCidr }), + ...((net.vnet_opt === "custom" || net.vnet_opt === "byo") && defaults.net.cniDynamicIpAllocation !== net.cniDynamicIpAllocation && { cniDynamicIpAllocation: true }), + ...(net.vnet_opt === "custom" && net.cniDynamicIpAllocation && defaults.net.podCidr !== net.podCidr && { podCidr: net.podCidr }), ...(cluster.availabilityZones === "yes" && { availabilityZones: ['1', '2', '3'] }), ...(cluster.apisecurity === "whitelist" && deploy.clusterIPWhitelist && apiips_array.length > 0 && { authorizedIPRanges: apiips_array }), + ...(defaults.net.maxPods !== net.maxPods && { maxPods: net.maxPods }), ...(cluster.apisecurity === "private" && { enablePrivateCluster: true }), ...(cluster.apisecurity === "private" && cluster.apisecurity === "private" && defaults.cluster.privateClusterDnsMethod !== cluster.privateClusterDnsMethod && { privateClusterDnsMethod: cluster.privateClusterDnsMethod }), ...(cluster.apisecurity === "private" && cluster.apisecurity === "private" && cluster.privateClusterDnsMethod === 'privateDnsZone' && { dnsApiPrivateZoneId: cluster.dnsApiPrivateZoneId }), @@ -124,6 +127,7 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray, ...(defaults.addons.kedaAddon !== addons.kedaAddon && {kedaAddon: addons.kedaAddon }), ...(defaults.addons.blobCSIAddon !== addons.blobCSIAddon && {blobCSIAddon: addons.blobCSIAddon }), ...(defaults.addons.workloadIdentity !== addons.workloadIdentity && {workloadIdentity: addons.workloadIdentity }), + ...(net.networkPlugin === 'azure' && net.networkPluginMode && {networkPluginMode: 'Overlay'}), ...(urlParams.getAll('feature').includes('defender') && cluster.DefenderForContainers !== defaults.cluster.DefenderForContainers && { DefenderForContainers: cluster.DefenderForContainers }) } @@ -297,7 +301,7 @@ az role assignment create --role "Managed Identity Operator" --assignee-principa {!allok && - + Configuration not complete, please correct the tabs with the warning symbol ({invalidTabs.join(' & ')}) before deploying } @@ -366,9 +370,13 @@ az role assignment create --role "Managed Identity Operator" --assignee-principa
Deploy ClusterBuilt with bicep

powered by Bicep

 {Object.keys(preview_params).length > 0 && - - Your deployment contains Preview features: {Object.keys(preview_params).join(', ')}, Ensure you have registered for these previews, and have installed the 'az extension add --name aks-preview' before running the script, see here, or disable preview features here - updateFn("disablePreviews", !checked)} /> + + Your deployment contains Preview Features which may require subscription registration and have Azure Region limitations. Please ensure you have registered for these previews, and have installed the 'az extension add --name aks-preview' before running the relevant scripts.
Preview Features you have selected: {Object.keys(preview_params).join(', ')}. + updateFn("disablePreviews", !checked)} /> } @@ -452,7 +460,7 @@ on: jobs: reusable_workflow_job: - uses: Azure/AKS-Construction/.github/workflows/AKSC_Deploy.yml@main + uses: Azure/AKS-Construction/.github/workflows/AKSC_Deploy.yml@${deploy.selectedTemplate} with:` + (deploy.selectedTemplate !== 'local' ? ` templateVersion: ${deploy.selectedTemplate}` : '') + ` rg: ${deploy.rg} diff --git a/helper/src/components/networkTab.js b/helper/src/components/networkTab.js index 3374004f0..1462b7e7b 100644 --- a/helper/src/components/networkTab.js +++ b/helper/src/components/networkTab.js @@ -1,6 +1,6 @@ import React, { useState } from 'react'; -import { Image, ImageFit, Link, Separator, TextField, DirectionalHint, Callout, Stack, Text, Label, ChoiceGroup, Checkbox, MessageBar, MessageBarType, Dropdown, Slider } from '@fluentui/react'; +import { Image, ImageFit, Link, Separator, TextField, DirectionalHint, Callout, Stack, Text, Label, ChoiceGroup, Checkbox, MessageBar, MessageBarType, Slider } from '@fluentui/react'; import { adv_stackstyle, hasError, getError } from './common' const columnProps = { @@ -9,13 +9,51 @@ const columnProps = { } -export default function NetworkTab ({ tabValues, updateFn, invalidArray, featureFlag }) { +export default function NetworkTab ({ defaults, tabValues, updateFn, invalidArray, featureFlag }) { const [callout1, setCallout1] = useState(false) const { net, addons } = tabValues var _calloutTarget1 = React.createRef() + + function UpdateDynamicIpAllocation(v) { + //update the Dynamic IP Allocation property, where this fn was called from + updateFn("cniDynamicIpAllocation", v) + + //update max pods to 250 if dynamic IP allocation is enabled + if (v) { + updateFn("maxPods", 250) + } else { + updateFn("maxPods", defaults.net.maxPods) + } + + //update pod cidr + if (v) { + updateFn("podCidr", defaults.net.podCidr.replace("/22","/24")) + } else { + updateFn("podCidr", defaults.net.podCidr) + } + } + + function UpdateCniOverlay(v) { + //update the networkPluginMode property, where this fn was called from + updateFn("networkPluginMode", v) + + //update node subnet to a nice small /24 if overlay is enabled, otherwise use the default + if (v) { + updateFn("vnetAksSubnetAddressPrefix", "10.240.0.0/24") + } else { + updateFn("vnetAksSubnetAddressPrefix", defaults.net.vnetAksSubnetAddressPrefix) + } + + if (v) { + updateFn("podCidr", '10.244.0.0/16') + } else { + updateFn("podCidr", defaults.net.podCidr) + } + } + return ( @@ -37,6 +75,46 @@ export default function NetworkTab ({ tabValues, updateFn, invalidArray, feature + + + + + Dynamic IP allocation separates node IP's and Pod IP's by subnet allowing dynamic allocation of Pod IPs docs + UpdateDynamicIpAllocation(v)} + label="Implement Dynamic Allocation of IPs" /> + + + Overlay is a preview feature that leverages a private CIDR for Pod IP's. See if it's right for you:docs + UpdateCniOverlay(v)} + label="CNI Overlay Network" /> + + + + + + + + + When using Azure CNI with Dynamic IP allocation also allows customers to set up clusters that consume fewer IPs.
This means Pods per Node can be maximised which simplifies sizing the cluster. + updateFn("maxPods", val)} + /> + + + + updateFn("vnetprivateend", v)} label="Enable Private Link" /> @@ -56,7 +134,7 @@ export default function NetworkTab ({ tabValues, updateFn, invalidArray, feature - Nat Gateway for AKS egress is currently a preview feature docs + NAT Gateway allows more traffic flows than a Load Balancer.docs {hasError(invalidArray, 'aksOutboundTrafficType') && {getError(invalidArray, 'aksOutboundTrafficType')} } @@ -144,7 +222,7 @@ export default function NetworkTab ({ tabValues, updateFn, invalidArray, feature key: 'byo', disabled: false, iconProps: { iconName: 'WebAppBuilderFragment' }, // SplitObject - text: 'BYO VNET (TBC)' + text: 'BYO VNET' } ]} /> @@ -245,7 +323,7 @@ function PodServiceNetwork({ net, updateFn }) { - updateFn("podCidr", val)} value={net.networkPlugin === 'kubenet' ? net.podCidr : "Using CNI, POD IPs from subnet"} /> + updateFn("podCidr", val)} value={net.networkPlugin === 'kubenet' || net.cniDynamicIpAllocation || net.networkPluginMode ? net.podCidr : "Using CNI, POD IPs from subnet"} /> updateFn("serviceCidr", val)} value={net.serviceCidr} /> diff --git a/helper/src/components/portalnav.js b/helper/src/components/portalnav.js index 6d8cec3f5..b1a3b9bca 100644 --- a/helper/src/components/portalnav.js +++ b/helper/src/components/portalnav.js @@ -1,6 +1,6 @@ import React, { useState, useEffect } from 'react'; -import { CommandBarButton, Image, ThemeProvider, Link, Toggle, TooltipHost, Pivot, PivotItem, Icon, Separator, Stack, Text, ChoiceGroup } from '@fluentui/react'; +import { CommandBarButton, Image, ThemeProvider, Link, Toggle, TooltipHost, Pivot, PivotItem, Icon, Separator, Stack, Text, ChoiceGroup, Modal, IconButton } from '@fluentui/react'; import { AzureThemeLight, AzureThemeDark } from '@fluentui/azure-themes'; import { mergeStyles, mergeStyleSets } from '@fluentui/merge-styles'; @@ -33,6 +33,7 @@ function useAITracking(componentName, key) { } const titleClass = mergeStyleSets({ "display": "inline-block", "marginLeft": "10px", "verticalAlign": "top" }) + function Header({ presets, setPresets, selectedPreset, featureFlag }) { @@ -286,7 +287,7 @@ export default function PortalNav({ config }) { //setTabValues(currentTabValues => updateTabValues(currentTabValues, sections, key, 'standard')) } - function mergeState(tab, field, value) { + function mergeState(tab, field, value, previewLink) { let updatevals if (typeof field === "string") { @@ -299,7 +300,9 @@ export default function PortalNav({ config }) { } } - //window.history.replaceState(null, null, "?"+urlParams.toString()) + //maintains the current config in querystring for easy bookmarking + window.history.replaceState(null, null, "?"+urlParams.toString()) + setTabValues((p) => { return { ...p, @@ -390,10 +393,10 @@ export default function PortalNav({ config }) { mergeState("cluster", field, value)} invalidArray={invalidArray['cluster']} /> _customRenderer('addons', a, b)} > - mergeState("addons", field, value)} invalidArray={invalidArray['addons']} /> + mergeState("addons", field, value, previewLink)} invalidArray={invalidArray['addons']} /> _customRenderer('net', a, b)}> - mergeState("net", field, value)} invalidArray={invalidArray['net']} /> + mergeState("net", field, value)} invalidArray={invalidArray['net']} /> _customRenderer('app', a, b)}> mergeState("app", field, value)} invalidArray={invalidArray['app']} /> diff --git a/helper/src/config.json b/helper/src/config.json index 95d9d45a3..e0d104a94 100644 --- a/helper/src/config.json +++ b/helper/src/config.json @@ -94,6 +94,9 @@ "gitops": "none" }, "net": { + "maxPods": 30, + "cniDynamicIpAllocation": false, + "networkPluginMode": false, "networkPlugin": "azure", "afw": false, "bastion": false, diff --git a/ps-rule.yaml b/ps-rule.yaml index 7f848f7e4..e32bfc234 100644 --- a/ps-rule.yaml +++ b/ps-rule.yaml @@ -38,6 +38,7 @@ configuration: # Bicep is experimental and currently disabled as testing occurs against compiled template # Enable automatic expansion of bicep source files AZURE_BICEP_FILE_EXPANSION: true + AZURE_BICEP_FILE_EXPANSION_TIMEOUT: 30 #ProjectSpecifc rules Azure_AKSNodeMinimumMaxPods: 30