diff --git a/.github/workflows/testWlsAksWithDependencyCreation.yml b/.github/workflows/testWlsAksWithDependencyCreation.yml index dc12f3b56..6e4058f2f 100644 --- a/.github/workflows/testWlsAksWithDependencyCreation.yml +++ b/.github/workflows/testWlsAksWithDependencyCreation.yml @@ -326,13 +326,14 @@ jobs: az aks get-credentials --resource-group ${{ env.resourceGroupForWlsAks }} --name $aksClusterName - name: Generate&Apply configmap run: | + wlsDomainNS=sample-domain1-ns wlsConfigmapName=sample-domain1-wdt-config-map wlsConfigmapJson=cargo-tracker-db.json modelFilePath=models mkdir ${modelFilePath} echo "create configmap" echo "export exisiting configmap" - kubectl -n sample-domain1-ns get configmap ${wlsConfigmapName} -o json >${wlsConfigmapJson} + kubectl -n ${wlsDomainNS} get configmap ${wlsConfigmapName} -o json >${wlsConfigmapJson} echo "query model keys" keyList=$(cat ${wlsConfigmapJson} | jq '.data | keys[]' | tr -d "\"") @@ -343,24 +344,25 @@ jobs: done # remove current configmap and create a new one - kubectl -n sample-domain1-ns delete configmap ${wlsConfigmapName} + kubectl -n ${wlsDomainNS} delete configmap ${wlsConfigmapName} cp cargotracker/src/test/aks/cargo-tracker-jms.yaml ${modelFilePath}/cargo-tracker-jms.yaml - kubectl -n sample-domain1-ns create configmap ${wlsConfigmapName} \ + kubectl -n ${wlsDomainNS} create configmap ${wlsConfigmapName} \ --from-file=${modelFilePath} - kubectl -n sample-domain1-ns label configmap ${wlsConfigmapName} \ + kubectl -n ${wlsDomainNS} label configmap ${wlsConfigmapName} \ weblogic.domainUID=sample-domain1 - restartVersion=$(kubectl -n sample-domain1-ns get domain sample-domain1 '-o=jsonpath={.spec.restartVersion}') + restartVersion=$(kubectl -n ${wlsDomainNS} get domain sample-domain1 '-o=jsonpath={.spec.restartVersion}') # increase restart version restartVersion=$((restartVersion + 1)) # record timestamp before apply changes timestampBeforePatchingDomain=$(date +%s) - # get the replica number - replicas=$(kubectl -n sample-domain1-ns get domain sample-domain1 -o json | jq '. | .spec.clusters[] | .replicas') + # get the replica number + clusterName=$(kubectl get cluster -n ${wlsDomainNS} -o json | jq -r '.items[0].metadata.name') + replicas=$(kubectl -n ${wlsDomainNS} get cluster ${clusterName} -o json | jq '. | .spec.replicas') echo "append configmap and update restart version" - kubectl -n sample-domain1-ns patch domain sample-domain1 \ + kubectl -n ${wlsDomainNS} patch domain sample-domain1 \ --type=json \ '-p=[{"op": "replace", "path": "/spec/restartVersion", "value": "'${restartVersion}'" }, {"op": "add", "path": "/spec/configuration/model/configMap", "value": "'${wlsConfigmapName}'" }]' echo "timestampBeforePatchingDomain=${timestampBeforePatchingDomain}" >> $GITHUB_ENV diff --git a/.github/workflows/testWlsAksWithoutDependencyCreation.yml b/.github/workflows/testWlsAksWithoutDependencyCreation.yml index e298fceff..97a59df7a 100644 --- a/.github/workflows/testWlsAksWithoutDependencyCreation.yml +++ b/.github/workflows/testWlsAksWithoutDependencyCreation.yml @@ -282,13 +282,14 @@ jobs: az aks get-credentials --resource-group ${{ needs.preflight.outputs.resourceGroupForWlsAks }} --name $aksClusterName - name: Generate&Apply configmap run: | + wlsDomainNS=sample-domain1-ns wlsConfigmapName=sample-domain1-wdt-config-map wlsConfigmapJson=cargo-tracker-db.json modelFilePath=models mkdir ${modelFilePath} echo "create configmap" echo "export exisiting configmap" - kubectl -n sample-domain1-ns get configmap ${wlsConfigmapName} -o json >${wlsConfigmapJson} + kubectl -n ${wlsDomainNS} get configmap ${wlsConfigmapName} -o json >${wlsConfigmapJson} echo "query model keys" keyList=$(cat ${wlsConfigmapJson} | jq '.data | keys[]' | tr -d "\"") @@ -299,24 +300,25 @@ jobs: done # remove current configmap and create a new one - kubectl -n sample-domain1-ns delete configmap ${wlsConfigmapName} + kubectl -n ${wlsDomainNS} delete configmap ${wlsConfigmapName} cp cargotracker/src/test/aks/cargo-tracker-jms.yaml ${modelFilePath}/cargo-tracker-jms.yaml - kubectl -n sample-domain1-ns create configmap ${wlsConfigmapName} \ + kubectl -n ${wlsDomainNS} create configmap ${wlsConfigmapName} \ --from-file=${modelFilePath} - kubectl -n sample-domain1-ns label configmap ${wlsConfigmapName} \ + kubectl -n ${wlsDomainNS} label configmap ${wlsConfigmapName} \ weblogic.domainUID=sample-domain1 - restartVersion=$(kubectl -n sample-domain1-ns get domain sample-domain1 '-o=jsonpath={.spec.restartVersion}') + restartVersion=$(kubectl -n ${wlsDomainNS} get domain sample-domain1 '-o=jsonpath={.spec.restartVersion}') # increase restart version restartVersion=$((restartVersion + 1)) # record timestamp before apply changes timestampBeforePatchingDomain=$(date +%s) - # get the replica number - replicas=$(kubectl -n sample-domain1-ns get domain sample-domain1 -o json | jq '. | .spec.clusters[] | .replicas') + # get the replica number + clusterName=$(kubectl get cluster -n ${wlsDomainNS} -o json | jq -r '.items[0].metadata.name') + replicas=$(kubectl -n ${wlsDomainNS} get cluster ${clusterName} -o json | jq '. | .spec.replicas') echo "append configmap and update restart version" - kubectl -n sample-domain1-ns patch domain sample-domain1 \ + kubectl -n ${wlsDomainNS} patch domain sample-domain1 \ --type=json \ '-p=[{"op": "replace", "path": "/spec/restartVersion", "value": "'${restartVersion}'" }, {"op": "add", "path": "/spec/configuration/model/configMap", "value": "'${wlsConfigmapName}'" }]' echo "timestampBeforePatchingDomain=${timestampBeforePatchingDomain}" >> $GITHUB_ENV diff --git a/pom.xml b/pom.xml index 7f8650a27..88a8a147e 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ - 1.0.74 + 1.0.75 1.0.27 1.0.50 diff --git a/resources/azure-common.properties b/resources/azure-common.properties index bd7ccad38..4ae0e3490 100644 --- a/resources/azure-common.properties +++ b/resources/azure-common.properties @@ -42,6 +42,8 @@ azure.apiVersionForTags=2023-07-01 azure.apiVersionForStorage=2023-01-01 # Microsoft.Storage/storageAccounts/fileServices azure.apiVersionForStorageFileService=2023-01-01 +# Microsoft.Monitor/accounts +azure.apiVersionForMonitorAccount=2023-04-03 # AzureAzCLI version azure.cli.version=2.53.0 diff --git a/weblogic-azure-aks/src/main/arm/createUiDefinition.json b/weblogic-azure-aks/src/main/arm/createUiDefinition.json index 3173f0213..d42618ec9 100644 --- a/weblogic-azure-aks/src/main/arm/createUiDefinition.json +++ b/weblogic-azure-aks/src/main/arm/createUiDefinition.json @@ -311,16 +311,30 @@ "uri": "https://aka.ms/wls-aks-well-tested-version" } } - }, + }, { "name": "aksNodeCount", "type": "Microsoft.Common.Slider", "min": 1, - "max": 1000, - "label": "Node count", + "max": 100, + "label": "Minimum node count", "defaultValue": 3, "showStepMarkers": false, - "toolTip": "The number of nodes that should be created along with the cluster. You will be able to resize the cluster later.", + "toolTip": "Set the minimum node count for the cluster.", + "constraints": { + "required": true + }, + "visible": "[bool(steps('section_aks').clusterInfo.createAKSCluster)]" + }, + { + "name": "aksNodeMaxCount", + "type": "Microsoft.Common.Slider", + "min": "[int(steps('section_aks').clusterInfo.aksNodeCount)]", + "max": 100, + "label": "Maximum node count", + "defaultValue": 5, + "showStepMarkers": false, + "toolTip": "Set the maximum node count for the cluster.", "constraints": { "required": true }, @@ -2107,12 +2121,153 @@ "visible": "[bool(steps('section_database').enableDB)]" } ] + }, + { + "name": "section_autoScaling", + "type": "Microsoft.Common.Section", + "label": "Autoscaling", + "subLabel": { + "preValidation": "Configure Horizontal Autoscaling", + "postValidation": "Done" + }, + "bladeTitle": "Horizontal Autoscaling", + "elements": [ + { + "name": "aboutAutoscaling", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Selecting 'Yes' here and providing the configuration will cause the offer configure metric to scale the WebLogic cluster." + } + }, + { + "name": "enableAutoscaling", + "type": "Microsoft.Common.OptionsGroup", + "label": "Provision resources for horizontal autoscaling?", + "defaultValue": "No", + "toolTip": "Select 'Yes' and provide required info to configure horizontal autoscaling.", + "constraints": { + "allowedValues": [ + { + "label": "Yes", + "value": "true" + }, + { + "label": "No", + "value": "false" + } + ], + "required": true + } + }, + { + "name": "autoScalingInfo", + "type": "Microsoft.Common.Section", + "label": "Horizontal autoscaling settings", + "elements": [ + { + "name": "metricSource", + "type": "Microsoft.Common.OptionsGroup", + "label": "Select autoscaling option. ", + "defaultValue": "Kubernetes Metrics Server (simple autoscaling)", + "toolTip": "If you select Kubernetes Metrics Server, this offer configures WebLogic Server to scale based on CPU or memory utilization. If you select WebLogic Monitoring Exporter, this offer configures WebLogic Monitoring Exporter to scrape WebLogic Server metrics and feed them to Azure Monitor Managed Service for Prometheus; integrates KEDA with your AKS cluster to monitor Azure Monitor workspace and feed data to AKS. You can create KEDA scaler based on Java metrics from Azure Monitor workspace", + "constraints": { + "allowedValues": [ + { + "label": "Kubernetes Metrics Server (simple autoscaling)", + "value": "kms" + }, + { + "label": "WebLogic Monitoring Exporter (advanced autoscaling)", + "value": "wme" + } + ], + "required": true + }, + "visible": true + }, + { + "name": "infoKms", + "type": "Microsoft.Common.InfoBox", + "visible": "[equals(steps('section_autoScaling').autoScalingInfo.metricSource, 'kms')]", + "options": { + "icon": "None", + "text": "This option configures and runs Kubernetes Horizontal Pod Autoscaler (HPA) to scale a WebLogic cluster, based on the CPU or memory utilization. The HPA autoscales WebLogic Server instances from a minimum of 1 cluster members up to maximum of cluster members, and the scale up or down action occur when the average CPU/memory is consistently over the utilization.
  • Default maximum of cluster member is 5. You can set it from Basics -> Optional Basic Configuration -> Maximum dynamic cluster size.
  • Default CPU request is 200m.
  • Default memory request is 1.5Gi.
    • ", + "uri": "https://aka.ms/wlsoperator-autoscaling-hpa" + } + }, + { + "name": "infoWme", + "type": "Microsoft.Common.InfoBox", + "visible": "[equals(steps('section_autoScaling').autoScalingInfo.metricSource, 'wme')]", + "options": { + "icon": "None", + "text": "This option installs all the software necessary to allow you to create Java metric aware KEDA scaling rules.
    The offer provisions the following deployments. Right-click and select Open Link in New Tab to follow links:After the provisioning is completed, you can create KEDA scaling rules. A sample rule is provided in the deployment outputs. The following steps show how to see the sample rule.", + "uri": "https://aka.ms/wls-aks-keda-scaler" + } + }, + { + "name": "kmsMetrics", + "type": "Microsoft.Common.OptionsGroup", + "label": "Select metric. ", + "defaultValue": "Average CPU Utilization", + "toolTip": "Select metric.", + "constraints": { + "allowedValues": [ + { + "label": "Average CPU Utilization", + "value": "cpu" + }, + { + "label": "Average Memory Utilization", + "value": "memory" + } + ], + "required": true + }, + "visible": "[equals(steps('section_autoScaling').autoScalingInfo.metricSource, 'kms')]" + }, + { + "name": "averageCpuUtilization", + "type": "Microsoft.Common.Slider", + "min": 10, + "max": 100, + "label": "Average CPU Utilization", + "subLabel": "Percent", + "defaultValue": 60, + "showStepMarkers": false, + "toolTip": "Pick Average CPU Utilization in Percent", + "constraints": { + "required": false + }, + "visible": "[equals(steps('section_autoScaling').autoScalingInfo.kmsMetrics, 'cpu')]" + }, + { + "name": "averageMemoryUtilization", + "type": "Microsoft.Common.Slider", + "min": 10, + "max": 100, + "label": "Average Memory Utilization", + "subLabel": "Percent", + "defaultValue": 60, + "showStepMarkers": false, + "toolTip": "Pick Average Memory Utilization in Percent", + "constraints": { + "required": false + }, + "visible": "[equals(steps('section_autoScaling').autoScalingInfo.kmsMetrics, 'memory')]" + } + ], + "visible": "[bool(steps('section_autoScaling').enableAutoscaling)]" + } + ] } ], "outputs": { "acrName": "[last(split(steps('section_aks').imageInfo.oracleAcrSelector.id, '/'))]", "acrResourceGroupName": "[last(take(split(steps('section_aks').imageInfo.oracleAcrSelector.id, '/'),5))]", "aksAgentPoolNodeCount": "[steps('section_aks').clusterInfo.aksNodeCount]", + "aksAgentPoolNodeMaxCount": "[steps('section_aks').clusterInfo.aksNodeMaxCount]", "aksClusterName": "[last(split(steps('section_aks').clusterInfo.aksClusterSelector.id, '/'))]", "aksClusterRGName": "[last(take(split(steps('section_aks').clusterInfo.aksClusterSelector.id, '/'), 5))]", "appGatewayCertificateOption": "[steps('section_appGateway').appgwIngress.certificateOption]", @@ -2124,6 +2279,8 @@ "appgwUsePrivateIP": "[steps('section_appGateway').appgwIngress.appgwUsePrivateIP]", "appPackageUrls": "[steps('section_aks').jeeAppInfo.appPackageUrl]", "appReplicas": "[int(steps('section_aks').jeeAppInfo.appReplicas)]", + "averageCpuUtilization": "[steps('section_autoScaling').autoScalingInfo.averageCpuUtilization]", + "averageMemoryUtilization": "[steps('section_autoScaling').autoScalingInfo.averageMemoryUtilization]", "createACR": "[bool(steps('section_aks').imageInfo.oracleCreateACR)]", "createAKSCluster": "[bool(steps('section_aks').clusterInfo.createAKSCluster)]", "createDNSZone": "[not(bool(steps('section_dnsConfiguration').bringDNSZone))]", @@ -2143,6 +2300,7 @@ "dnszoneRGName": "[steps('section_dnsConfiguration').dnsZoneResourceGroup]", "dsConnectionURL": "[coalesce(steps('section_database').databaseConnectionInfo.dsConnectionURL, steps('section_database').databaseConnectionInfo.dsConnectionURL2, 'null')]", "enableAppGWIngress": "[equals(steps('section_appGateway').loadBalancingOptions, 'agic')]", + "enableAutoscaling": "[bool(steps('section_autoScaling').enableAutoscaling)]", "enableAzureMonitoring": "[bool(steps('section_aks').aksAdvancedInfo.enableAzureMonitoring)]", "enableAzureFileShare": "[bool(steps('section_aks').aksAdvancedInfo.enableAzureFileShare)]", "enableCookieBasedAffinity": "[not(bool(steps('section_appGateway').appgwIngress.enableCookieBasedAffinity))]", @@ -2161,6 +2319,7 @@ "keyVaultSSLBackendRootCertDataSecretName": "[steps('section_appGateway').appgwIngress.keyVaultBackendSSLCertDataSecretName]", "keyVaultSSLCertDataSecretName": "[steps('section_appGateway').appgwIngress.keyVaultSSLCertDataSecretName]", "keyVaultSSLCertPasswordSecretName": "[steps('section_appGateway').appgwIngress.keyVaultSSLCertPasswordSecretName]", + "hpaScaleType": "[steps('section_autoScaling').autoScalingInfo.kmsMetrics]", "managedServerPrefix": "[basics('basicsOptional').managedServerPrefix]", "newOrExistingVnetForApplicationGateway": "[steps('section_appGateway').appgwIngress.vnetForApplicationGateway.newOrExisting]", "ocrSSOPSW": "[steps('section_aks').imageInfo.ocrSSOPassword]", @@ -2184,6 +2343,7 @@ "sslUploadedCustomTrustKeyStoreType": "[steps('section_sslConfiguration').uploadedCustomSSLSettings.uploadedCustomTrustKeyStoreType]", "sslUploadedPrivateKeyAlias": "[steps('section_sslConfiguration').uploadedCustomSSLSettings.uploadedPrivateKeyAlias]", "sslUploadedPrivateKeyPassPhrase": "[steps('section_sslConfiguration').uploadedCustomSSLSettings.uploadedPrivateKeyPassPhrase]", + "useHpa": "[if(equals(steps('section_autoScaling').autoScalingInfo.metricSource, 'kms'), true, false)]", "useInternalLB": "[bool(steps('section_appGateway').lbSVCInfo.enableInternalLB)]", "useOracleImage": "[if(bool(steps('section_aks').aksAdvancedInfo.useAcrImage), false, true)]", "userProvidedAcr": "[last(split(steps('section_aks').aksAdvancedInfo.userProvidedAcrSelector.id, '/'))]", diff --git a/weblogic-azure-aks/src/main/arm/scripts/common.sh b/weblogic-azure-aks/src/main/arm/scripts/common.sh index 99d0a635d..23d5ebf24 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/common.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/common.sh @@ -15,6 +15,8 @@ export checkAcrInterval=30 export checkAcrMaxAttempt=10 export checkAgicInterval=30 export checkAgicMaxAttempt=50 +export checkKedaInteval=30 +export checkKedaMaxAttempt=20 export constAdminT3AddressEnvName="T3_TUNNELING_ADMIN_ADDRESS" export constAdminServerName='admin-server' @@ -41,6 +43,7 @@ export ocrGaImagePath="middleware/weblogic" export ocrCpuImagePath="middleware/weblogic_cpu" export gitUrl4CpuImages="https://raw.githubusercontent.com/oracle/weblogic-azure/main/weblogic-azure-aks/src/main/resources/weblogic_cpu_images.json" export gitUrl4AksWellTestedVersionJsonFile="https://raw.githubusercontent.com/oracle/weblogic-azure/main/weblogic-azure-aks/src/main/resources/aks_well_tested_version.json" +export gitUrl4AksToolingWellTestedVersionJsonFile="https://raw.githubusercontent.com/oracle/weblogic-azure/main/weblogic-azure-aks/src/main/resources/aks_tooling_well_tested_versions.json" export gitUrl4WLSToolingFamilyJsonFile="https://raw.githubusercontent.com/oracle/weblogic-azure/main/weblogic-azure-aks/src/main/resources/weblogic_tooling_family.json" export gitUrl4AzureIdentityExtensionsPomFile="https://raw.githubusercontent.com/oracle/weblogic-azure/main/weblogic-azure-aks/src/main/resources/azure-identity-extensions.xml" export gitUrl4MySQLDriverPomFile="https://raw.githubusercontent.com/oracle/weblogic-azure/main/weblogic-azure-aks/src/main/resources/mysql-connector-java.xml" diff --git a/weblogic-azure-aks/src/main/arm/scripts/createAppGatewayIngress.sh b/weblogic-azure-aks/src/main/arm/scripts/createAppGatewayIngress.sh index 8e6d78d86..49916b55d 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/createAppGatewayIngress.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/createAppGatewayIngress.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Copyright (c) 2021, 2024, Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # Description: to create Azure Application Gateway ingress for the following targets. @@ -318,10 +318,9 @@ function network_peers_aks_appgw() { local aksNetworkRgName=${aksNetWorkId#*\/resourceGroups\/} local aksNetworkRgName=${aksNetworkRgName%\/providers\/*} - local appGatewaySubnetId=$(az network application-gateway show -g ${CURRENT_RG_NAME} --name ${APPGW_NAME} -o tsv --query "gatewayIpConfigurations[0].subnet.id") - local appGatewayVnetResourceGroup=$(az network application-gateway show -g ${CURRENT_RG_NAME} --name ${APPGW_NAME} -o tsv --query "gatewayIpConfigurations[0].subnet.resourceGroup") - local appGatewaySubnetName=$(az resource show --ids ${appGatewaySubnetId} --query "name" -o tsv) - local appgwNetworkId=$(echo $appGatewaySubnetId | sed s/"\/subnets\/${appGatewaySubnetName}"//) + local appGatewaySubnetId=$(az network application-gateway show -g ${CURRENT_RG_NAME} --name ${APPGW_NAME} -o tsv --query "gatewayIPConfigurations[0].subnet.id") + local appGatewayVnetResourceGroup=$(az network application-gateway show -g ${CURRENT_RG_NAME} --name ${APPGW_NAME} -o tsv --query "gatewayIPConfigurations[0].subnet.resourceGroup") + local appgwNetworkId=${appGatewaySubnetId%\/subnets\/*} local appgwVnetName=$(az resource show --ids ${appgwNetworkId} --query "name" -o tsv) local toPeer=true diff --git a/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh b/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh index a0f3782e9..b170253ca 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Copyright (c) 2021, 2024, Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # Description: to create Load Balancer Service for the following targets. @@ -359,8 +359,9 @@ ${currentDomainConfig} EOF echo ${currentDomainConfig} | kubectl -n ${wlsDomainNS} apply -f - - replicas=$(kubectl -n ${wlsDomainNS} get domain ${WLS_DOMAIN_UID} -o json | - jq '. | .spec.clusters[] | .replicas') + local clusterName=$(kubectl get cluster -n ${wlsDomainNS} -o json | jq -r '.items[0].metadata.name') + local replicas=$(kubectl -n ${wlsDomainNS} get cluster ${clusterName} -o json \ + | jq '. | .spec.replicas') # wait for the restart completed. utility_wait_for_pod_restarted \ diff --git a/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/enableHpa.sh b/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/enableHpa.sh new file mode 100644 index 000000000..36524bae5 --- /dev/null +++ b/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/enableHpa.sh @@ -0,0 +1,68 @@ +# Copyright (c) 2024, Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +function get_cluster_uid(){ + local clusterUid=$(kubectl get clusters -n ${WLS_NAMESPACE} -o=jsonpath='{.items[].metadata.name}') + utility_validate_status "Obtain cluster UID." + export WLS_CLUSTER_UID=${clusterUid} +} + +function scaling_basedon_cpu(){ + kubectl autoscale cluster ${WLS_CLUSTER_UID} \ + --cpu-percent=${UTILIZATION_PERCENTAGE} \ + --min=1 \ + --max=${WLS_CLUSTER_SIZE} \ + -n ${WLS_NAMESPACE} + utility_validate_status "Enable HPA based on CPU utilization." +} + +function scaling_basedon_memory(){ + cat <scaler-memory.yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: ${WLS_CLUSTER_UID} + namespace: ${WLS_NAMESPACE} +spec: + scaleTargetRef: + apiVersion: weblogic.oracle/v1 + kind: Cluster + name: ${WLS_CLUSTER_UID} + minReplicas: 1 + maxReplicas: ${WLS_CLUSTER_SIZE} + metrics: + - type: Resource + resource: + name: memory + target: + averageUtilization: ${UTILIZATION_PERCENTAGE} + type: Utilization +EOF + + kubectl apply -f scaler-memory.yaml + utility_validate_status "Enable HPA based on memory utilization." +} + +function check_kubernetes_metrics_server(){ + # $?=1 if there is no running kms pod. + kubectl get pod -l k8s-app=metrics-server -n kube-system | grep "Running" + # exit if $?=1 + utility_validate_status "There should be at least one pod of kubernetes metrics server running." +} + +# Main script +set -Eo pipefail + +install_kubectl + +connect_aks $AKS_CLUSTER_NAME $AKS_CLUSTER_RG_NAME + +get_cluster_uid + +check_kubernetes_metrics_server + +if [ "$HPA_SCALE_TYPE" == "cpu" ]; then + scaling_basedon_cpu +elif [ "$HPA_SCALE_TYPE" == "memory" ]; then + scaling_basedon_memory +fi diff --git a/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/enablePrometheusMetrics.sh b/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/enablePrometheusMetrics.sh new file mode 100644 index 000000000..874ff613f --- /dev/null +++ b/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/enablePrometheusMetrics.sh @@ -0,0 +1,405 @@ +# Copyright (c) 2024, Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +#!/bin/bash + +function enable_promethues_metrics(){ + # See https://learn.microsoft.com/en-us/azure/azure-monitor/containers/kubernetes-monitoring-enable?tabs=cli#enable-prometheus-and-grafana + az extension add --name k8s-extension && true + + ### Use existing Azure Monitor workspace + az aks update --enable-azure-monitor-metrics \ + --name ${AKS_CLUSTER_NAME} \ + --resource-group ${AKS_CLUSTER_RG_NAME} \ + --azure-monitor-workspace-resource-id "${AMA_WORKSPACE_ID}" \ + --only-show-errors + + utility_validate_status "Enable Promethues Metrics." + + az extension add --name aks-preview && true + az extension remove --name k8s-extension && true + + #Verify that the DaemonSet was deployed properly on the Linux node pools + #https://learn.microsoft.com/en-us/azure/azure-monitor/containers/kubernetes-monitoring-enable?tabs=cli#managed-prometheus + kubectl get ds ama-metrics-node --namespace=kube-system + #if the deployment fails, $?=1. + utility_validate_status "Validate promethues metrics is enabled." +} + +# https://learn.microsoft.com/en-us/azure/azure-monitor/containers/prometheus-metrics-scrape-configuration +function deploy_customize_scraping(){ + # https://learn.microsoft.com/en-us/azure/azure-monitor/containers/prometheus-metrics-scrape-configuration?tabs=CRDConfig%2CCRDScrapeConfig#basic-authentication + local wlsPswBase64=$(echo -n "${WLS_ADMIN_PASSWORD}" | base64) + cat <prometheus-config +global: + scrape_interval: 30s +scrape_configs: +- job_name: '${WLS_DOMAIN_UID}' + kubernetes_sd_configs: + - role: pod + namespaces: + names: [${WLS_NAMESPACE}] + basic_auth: + username: ${WLS_ADMIN_USERNAME} + password_file: /etc/prometheus/certs/password1 +EOF + + #validate the scrape config file + local podNamesinKubeSystem=$(kubectl get pods -l rsName=ama-metrics -n=kube-system -o json | jq -r '.items[].metadata.name') + mkdir promconfigvalidator + for podname in ${podNamesinKubeSystem} + do + kubectl cp -n=kube-system "${podname}":/opt/promconfigvalidator ./promconfigvalidator/promconfigvalidator + kubectl cp -n=kube-system "${podname}":/opt/microsoft/otelcollector/collector-config-template.yml ./promconfigvalidator/collector-config-template.yml + chmod 500 ./promconfigvalidator/promconfigvalidator + done + + if [ ! -f "./promconfigvalidator/promconfigvalidator" ]; then + echo_stderr "Failed to download promconfigvalidator tool that is shipped inside the Azure Monitor metrics addon pod(s)." + exit 1 + fi + + ./promconfigvalidator/promconfigvalidator --config "./prometheus-config" --otelTemplate "./promconfigvalidator/collector-config-template.yml" + utility_validate_status "Validate prometheus-config using promconfigvalidator." + + kubectl create configmap ama-metrics-prometheus-config --from-file=prometheus-config -n kube-system + utility_validate_status "Create ama-metrics-prometheus-config in kube-system namespace." +} + +function get_wls_monitoring_exporter_image_url() { + local wlsToolingFamilyJsonFile=weblogic_tooling_family.json + local imageUrl="ghcr.io/oracle/weblogic-monitoring-exporter:2.1.9" + + # download the json file that has well tested monitoring exporter image url from weblogic-azure repo. + curl -m ${curlMaxTime} --retry ${retryMaxAttempt} -fsL "${gitUrl4WLSToolingFamilyJsonFile}" -o ${wlsToolingFamilyJsonFile} + if [ $? -eq 0 ]; then + imageURL=$(cat ${wlsToolingFamilyJsonFile} | jq ".items[] | select(.key==\"WME\") | .imageURL" | tr -d "\"") + echo_stdout "well tested monitoring exporter image url: ${imageURL}" + fi + + echo_stdout "Use monitoring exporter image: ${imageURL} " + export WME_IMAGE_URL=${imageUrl} +} + +# https://github.com/oracle/weblogic-monitoring-exporter +function deploy_webLogic_monitoring_exporter(){ + local wlsVersion=$(kubectl -n ${WLS_NAMESPACE} get domain ${WLS_DOMAIN_UID} -o=jsonpath='{.spec.restartVersion}' | tr -d "\"") + wlsVersion=$((wlsVersion+1)) + + cat <patch-file.json +[ + { + "op": "replace", + "path": "/spec/restartVersion", + "value": "${wlsVersion}" + }, + { + "op": "add", + "path": "/spec/monitoringExporter", + "value": { + "configuration": { + "domainQualifier": true, + "metricsNameSnakeCase": true, + "queries": [ + { + "applicationRuntimes": { + "componentRuntimes": { + "key": "name", + "prefix": "webapp_config_", + "servlets": { + "key": "servletName", + "prefix": "weblogic_servlet_", + "values": [ + "invocationTotalCount", + "reloadTotal", + "executionTimeAverage", + "poolMaxCapacity", + "executionTimeTotal", + "reloadTotalCount", + "executionTimeHigh", + "executionTimeLow" + ] + }, + "type": "WebAppComponentRuntime", + "values": [ + "deploymentState", + "contextRoot", + "sourceInfo", + "openSessionsHighCount", + "openSessionsCurrentCount", + "sessionsOpenedTotalCount", + "sessionCookieMaxAgeSecs", + "sessionInvalidationIntervalSecs", + "sessionTimeoutSecs", + "singleThreadedServletPoolSize", + "sessionIDLength", + "servletReloadCheckSecs", + "jSPPageCheckSecs" + ] + }, + "workManagerRuntimes": { + "prefix": "workmanager_", + "key": "applicationName", + "values": [ + "pendingRequests", + "completedRequests", + "stuckThreadCount"] + }, + "key": "name", + "keyName": "app" + }, + "JVMRuntime": { + "key": "name", + "values": [ + "heapFreeCurrent", + "heapFreePercent", + "heapSizeCurrent", + "heapSizeMax", + "uptime", + "processCpuLoad" + ] + }, + "key": "name", + "keyName": "server" + } + ] + }, + "image": "${WME_IMAGE_URL}", + "port": 8080 + } + } +] +EOF + + kubectl -n ${WLS_NAMESPACE} patch domain ${WLS_DOMAIN_UID} \ + --type=json \ + --patch-file patch-file.json + utility_validate_status "Enable WebLogic Monitoring Exporter." + + local timestampBeforePatchingDomain=$(date +%s) + local clusterName=$(kubectl get cluster -n ${WLS_NAMESPACE} -o json | jq -r '.items[0].metadata.name') + local replicas=$(kubectl -n ${WLS_NAMESPACE} get cluster ${clusterName} -o json \ + | jq '. | .spec.replicas') + + # wait for the restart completed. + utility_wait_for_pod_restarted \ + ${timestampBeforePatchingDomain} \ + ${replicas} \ + ${WLS_DOMAIN_UID} \ + ${checkPodStatusMaxAttemps} \ + ${checkPodStatusInterval} +} + +function wait_for_keda_ready(){ + local ready=false + local attempt=0 + + while [[ "${ready}" == "false" && $attempt -le ${checkKedaMaxAttempt} ]]; do + echo_stdout "Check if KEDA is ready, attempt: ${attempt}." + ready=true + + local podCount=$(kubectl get pods -n ${KEDA_NAMESPACE} -o json | jq -r '.items | length') + if [ $podCount -lt 3 ];then + ready=false + fi + + local podnames=$(kubectl get pods -n ${KEDA_NAMESPACE} -o json | jq -r '.items[].metadata.name') + for podname in ${podnames} + do + kubectl get pod ${podname} -n ${KEDA_NAMESPACE} | grep "Running" + + if [ $? -eq 1 ];then + ready=false + fi + done + + attempt=$((attempt + 1)) + sleep ${checkKedaInteval} + done + + if [ ${attempt} -gt ${checkKedaMaxAttempt} ]; then + echo_stderr "Failed to enable KEDA." + exit 1 + fi + + echo_stderr "KEDA is running." +} + +function get_keda_latest_version() { + local kedaVersion + kedaVersion=$(helm search repo kedacore/keda --versions | awk '/^kedacore\/keda/ {print $2; exit}') + export KEDA_VERSION="${kedaVersion}" + echo_stderr "Use latest KEDA. KEDA version: ${KEDA_VERSION}" +} + + +function get_keda_version() { + local versionJsonFileName="aks_tooling_well_tested_version.json" + local kedaWellTestedVersion + + # Download the version JSON file + curl -L "${gitUrl4AksToolingWellTestedVersionJsonFile}" --retry "${retryMaxAttempt}" -o "${versionJsonFileName}" + + # Extract KEDA version from JSON + kedaWellTestedVersion=$(jq -r '.items[] | select(.key == "keda") | .version' "${versionJsonFileName}") + + # Check if version is available + if [ $? -ne 0 ]; then + get_keda_latest_version + return 0 + fi + + # Print KEDA well-tested version + echo_stderr "KEDA well-tested version: ${kedaWellTestedVersion}" + + # Search for KEDA version in Helm repo + if ! helm search repo kedacore/keda --versions | grep -q "${kedaWellTestedVersion}"; then + get_keda_latest_version + return 0 + fi + + # Export KEDA version + export KEDA_VERSION="${kedaWellTestedVersion}" + echo_stderr "KEDA version: ${KEDA_VERSION}" +} + +# https://learn.microsoft.com/en-us/azure/azure-monitor/containers/integrate-keda +function enable_keda_addon() { + local oidcEnabled=$(az aks show --resource-group $AKS_CLUSTER_RG_NAME --name $AKS_CLUSTER_NAME --query oidcIssuerProfile.enabled) + local workloadIdentity=$(az aks show --resource-group $AKS_CLUSTER_RG_NAME --name $AKS_CLUSTER_NAME --query securityProfile.workloadIdentity) + + if [[ "${oidcEnabled,,}" == "false" || -z "${workloadIdentity}" ]]; then + # mitigate https://github.com/Azure/azure-cli/issues/28649 + pip install --upgrade azure-core + az aks update -g $AKS_CLUSTER_RG_NAME -n $AKS_CLUSTER_NAME --enable-workload-identity --enable-oidc-issuer + utility_validate_status "Enable oidc and worload identity in AKS $AKS_CLUSTER_NAME." + fi + + export OIDC_ISSUER_URL=$(az aks show -n $AKS_CLUSTER_NAME -g $AKS_CLUSTER_RG_NAME --query "oidcIssuerProfile.issuerUrl" -otsv) + export KEDA_UAMI_CLIENT_ID=$(az identity show --resource-group $CURRENT_RG_NAME --name $KEDA_UAMI_NAME --query 'clientId' -otsv) + local tenantId=$(az identity show --resource-group $CURRENT_RG_NAME --name $KEDA_UAMI_NAME --query 'tenantId' -otsv) + + kubectl create namespace ${KEDA_NAMESPACE} + + cat <kedascalersample.yaml +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-managed-prometheus-trigger-auth + namespace: ${WLS_NAMESPACE} +spec: + podIdentity: + provider: azure-workload + identityId: ${KEDA_UAMI_CLIENT_ID} +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: azure-managed-prometheus-scaler + namespace: ${WLS_NAMESPACE} +spec: + scaleTargetRef: + apiVersion: weblogic.oracle/v1 + kind: Cluster + name: ${clusterName} + minReplicaCount: 1 + maxReplicaCount: ${WLS_CLUSTER_SIZE} + triggers: + - type: prometheus + metadata: + serverAddress: ${kedaServerAddress} + metricName: webapp_config_open_sessions_high_count + query: sum(webapp_config_open_sessions_high_count{app=""}) # Note: query must return a vector/scalar single element response + threshold: '10' + activationThreshold: '1' + authenticationRef: + name: azure-managed-prometheus-trigger-auth +EOF + + local base64ofKedaScalerSample=$(cat ./kedascalersample.yaml | base64) + local result=$(jq -n -c \ + --arg kedaScalerServerAddress "$kedaServerAddress" \ + --arg base64ofKedaScalerSample "${base64ofKedaScalerSample}" \ + '{kedaScalerServerAddress: $kedaScalerServerAddress, base64ofKedaScalerSample: $base64ofKedaScalerSample}') + echo "result is: $result" + echo $result >$AZ_SCRIPTS_OUTPUT_PATH +} + +# TBD see if we can query some of the metrics + +# Main script +set -Eo pipefail + +install_kubectl + +install_helm + +connect_aks $AKS_CLUSTER_NAME $AKS_CLUSTER_RG_NAME + +get_wls_monitoring_exporter_image_url + +deploy_webLogic_monitoring_exporter + +enable_promethues_metrics + +deploy_customize_scraping + +enable_keda_addon + +output diff --git a/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/queryDomainConfigurations.sh b/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/queryDomainConfigurations.sh index bd6e0186d..85121f908 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/queryDomainConfigurations.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/queryDomainConfigurations.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Copyright (c) 2021, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # This script runs on Azure Container Instance with Alpine Linux that Azure Deployment script creates. # @@ -15,10 +15,7 @@ echo "install kubectl" az aks install-cli echo "Connect AKS" -az aks get-credentials \ - --resource-group ${AKS_CLUSTER_RESOURCEGROUP_NAME} \ - --name ${AKS_CLUSTER_NAME} \ - --overwrite-existing +connect_aks $AKS_CLUSTER_NAME $AKS_CLUSTER_RESOURCEGROUP_NAME wlsDomainNS="${WLS_DOMAIN_UID}-ns" diff --git a/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/validateAgic.sh b/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/validateAgic.sh index df1e81f06..a7756cf14 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/validateAgic.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/inline-scripts/validateAgic.sh @@ -2,11 +2,6 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. function wait_for_azure_ingress_ready() { - az aks get-credentials \ - --resource-group ${AKS_CLUSTER_RG_NAME} \ - --name ${AKS_CLUSTER_NAME} \ - --overwrite-existing - local ready=false local attempt=0 @@ -36,4 +31,6 @@ set -Eo pipefail install_kubectl +connect_aks $AKS_CLUSTER_NAME $AKS_CLUSTER_RG_NAME + wait_for_azure_ingress_ready diff --git a/weblogic-azure-aks/src/main/arm/scripts/queryStorageAccount.sh b/weblogic-azure-aks/src/main/arm/scripts/queryStorageAccount.sh index c7467679f..fb2a38c12 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/queryStorageAccount.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/queryStorageAccount.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Copyright (c) 2021, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # Inputs: # AKS_CLUSTER_RESOURCEGROUP_NAME @@ -8,14 +8,6 @@ echo "Script ${0} starts" export currentStorageAccount="null" -# Connect to AKS cluster -function connect_aks_cluster() { - az aks get-credentials \ - --resource-group ${AKS_CLUSTER_RESOURCEGROUP_NAME} \ - --name ${AKS_CLUSTER_NAME} \ - --overwrite-existing -} - function query_storage_account() { echo "install kubectl" az aks install-cli @@ -42,7 +34,7 @@ function output_result() { echo $result >$AZ_SCRIPTS_OUTPUT_PATH } -connect_aks_cluster +connect_aks $AKS_CLUSTER_NAME $AKS_CLUSTER_RESOURCEGROUP_NAME query_storage_account diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupDBConnections.sh b/weblogic-azure-aks/src/main/arm/scripts/setupDBConnections.sh index a8dac6e0b..8d208b341 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupDBConnections.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupDBConnections.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Copyright (c) 2021, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. echo "Script ${0} starts" @@ -80,14 +80,6 @@ function validate_input() { fi } -# Connect to AKS cluster -function connect_aks_cluster() { - az aks get-credentials \ - --resource-group ${AKS_RESOURCE_GROUP_NAME} \ - --name ${AKS_NAME} \ - --overwrite-existing -} - function create_datasource_model_configmap_and_secret() { echo "get data source secret name" jndiLabel=${JDBC_DATASOURCE_NAME//\//\_} @@ -289,7 +281,7 @@ export wlsDomainNS="${WLS_DOMAIN_UID}-ns" validate_input -connect_aks_cluster +connect_aks $AKS_NAME $AKS_RESOURCE_GROUP_NAME install_kubectl diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh index 52d3fa539..084524d72 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Copyright (c) 2021, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. echo "Script ${0} starts" @@ -114,11 +114,6 @@ function validate_input() { fi } -# Connect to AKS cluster -function connect_aks_cluster() { - az aks get-credentials --resource-group ${AKS_CLUSTER_RG_NAME} --name ${AKS_CLUSTER_NAME} --overwrite-existing -} - function create_svc_lb() { # No lb svc inputs if [[ "${LB_SVC_VALUES}" != "[]" ]]; then @@ -147,7 +142,7 @@ validate_input install_utilities -connect_aks_cluster +connect_aks $AKS_CLUSTER_NAME $AKS_CLUSTER_RG_NAME create_svc_lb diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh index f10b84f38..fbdcab296 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh @@ -285,11 +285,6 @@ function install_utilities() { validate_status ${ret} } -# Connect to AKS cluster -function connect_aks_cluster() { - az aks get-credentials --resource-group ${AKS_CLUSTER_RESOURCEGROUP_NAME} --name ${AKS_CLUSTER_NAME} --overwrite-existing -} - # remove the operator if it is not running. function uninstall_operator() { echo "remove operator" @@ -772,7 +767,7 @@ query_acr_credentials build_docker_image -connect_aks_cluster +connect_aks $AKS_CLUSTER_NAME $AKS_CLUSTER_RESOURCEGROUP_NAME install_wls_operator diff --git a/weblogic-azure-aks/src/main/arm/scripts/updateApplications.sh b/weblogic-azure-aks/src/main/arm/scripts/updateApplications.sh index 8b9fa6c8c..9a6bf25a4 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/updateApplications.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/updateApplications.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Copyright (c) 2021, 2024, Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. echo "Script ${0} starts" @@ -98,14 +98,6 @@ function validate_input() { fi } -# Connect to AKS cluster -function connect_aks_cluster() { - az aks get-credentials \ - --resource-group ${AKS_CLUSTER_RESOURCEGROUP_NAME} \ - --name ${AKS_CLUSTER_NAME} \ - --overwrite-existing -} - function query_wls_cluster_info(){ WLS_CLUSTER_SIZE=$(kubectl -n ${wlsDomainNS} get domain ${WLS_DOMAIN_UID} -o json \ | jq '. | .status.clusters[] | select(.clusterName == "'${constClusterName}'") | .maximumReplicas') @@ -227,8 +219,9 @@ function apply_new_image() { function wait_for_pod_completed() { # Make sure all of the pods are running. - replicas=$(kubectl -n ${wlsDomainNS} get domain ${WLS_DOMAIN_UID} -o json \ - | jq '. | .spec.clusters[] | .replicas') + local clusterName=$(kubectl get cluster -n ${wlsDomainNS} -o json | jq -r '.items[0].metadata.name') + local replicas=$(kubectl -n ${wlsDomainNS} get cluster ${clusterName} -o json \ + | jq '. | .spec.replicas') utility_wait_for_pod_completed \ ${replicas} \ @@ -240,8 +233,9 @@ function wait_for_pod_completed() { function wait_for_image_update_completed() { # Make sure all of the pods are updated with new image. # Assumption: we have only one cluster currently. - replicas=$(kubectl -n ${wlsDomainNS} get domain ${WLS_DOMAIN_UID} -o json \ - | jq '. | .spec.clusters[] | .replicas') + local clusterName=$(kubectl get cluster -n ${wlsDomainNS} -o json | jq -r '.items[0].metadata.name') + local replicas=$(kubectl -n ${wlsDomainNS} get cluster ${clusterName} -o json \ + | jq '. | .spec.replicas') utility_wait_for_image_update_completed \ "${acrImagePath}" \ @@ -282,11 +276,14 @@ export ENABLE_CUSTOM_SSL=${constFalse} export WLS_CLUSTER_SIZE=5 export URL_3RD_DATASOURCE=$(echo "[]" | base64) +# Main script +set -Eo pipefail + validate_input install_kubectl -connect_aks_cluster +connect_aks $AKS_CLUSTER_NAME $AKS_CLUSTER_RESOURCEGROUP_NAME query_wls_cluster_info diff --git a/weblogic-azure-aks/src/main/arm/scripts/utility.sh b/weblogic-azure-aks/src/main/arm/scripts/utility.sh index d2f8c0655..878267bcf 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/utility.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/utility.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Copyright (c) 2021, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # This script runs on Azure Container Instance with Alpine Linux that Azure Deployment script creates. @@ -27,6 +27,16 @@ function utility_validate_status() { fi } +function connect_aks(){ + az aks get-credentials \ + -n $1 \ + -g $2 \ + --overwrite-existing \ + --only-show-errors + + utility_validate_status "Finished connecting to AKS cluster." +} + # JAVA_HOME=/usr/lib/jvm/java-11-openjdk function install_jdk() { local ready=false diff --git a/weblogic-azure-aks/src/main/arm/scripts/validateApplications.sh b/weblogic-azure-aks/src/main/arm/scripts/validateApplications.sh index b24db54af..a9850775a 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/validateApplications.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/validateApplications.sh @@ -1,15 +1,7 @@ -# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Copyright (c) 2021, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # This script runs on Azure Container Instance with Alpine Linux that Azure Deployment script creates. -# Connect to AKS cluster -function connect_aks_cluster() { - az aks get-credentials \ - --resource-group ${AKS_RESOURCE_GROUP_NAME} \ - --name ${AKS_NAME} \ - --overwrite-existing -} - function validate_app() { # make sure all the application are active, if not, fail the deployment. local wlsDomainNS="${WLS_DOMAIN_UID}-ns" @@ -32,6 +24,6 @@ source ${scriptDir}/utility.sh install_kubectl -connect_aks_cluster +connect_aks $AKS_NAME $AKS_RESOURCE_GROUP_NAME validate_app \ No newline at end of file diff --git a/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep b/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep index 1fc142077..e7752ed01 100644 --- a/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep +++ b/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep @@ -1,5 +1,5 @@ /* -* Copyright (c) 2021, Oracle Corporation and/or its affiliates. +* Copyright (c) 2021, 2024, Oracle Corporation and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. * * Terms: @@ -36,8 +36,12 @@ param acrResourceGroupName string = 'acr-contoso-rg' param aksAgentPoolName string = 'agentpool' @maxValue(10000) @minValue(1) -@description('The number of nodes that should be created along with the cluster. You will be able to resize the cluster later.') +@description('Set the minimum node count for the cluster.') param aksAgentPoolNodeCount int = 3 +@maxValue(1000) +@minValue(3) +@description('Set the maximum node count for the cluster.') +param aksAgentPoolNodeMaxCount int = 5 @description('The size of the virtual machines that will form the nodes in the cluster. This cannot be changed after creating the cluster') param vmSize string = 'Standard_DS2_v2' @description('Prefix for cluster name. Only The name can contain only letters, numbers, underscores and hyphens. The name must start with letter or number.') @@ -74,6 +78,10 @@ param appgwUsePrivateIP bool = false param appPackageUrls array = [] @description('The number of managed server to start.') param appReplicas int = 2 +@description('Scale up once average cpu utilization is larger then the input number ') +param averageCpuUtilization int = 60 +@description('Scale up once average memory utilization is larger then the input number ') +param averageMemoryUtilization int = 60 @description('true to create a new Azure Container Registry.') param createACR bool = false @description('true to create a new AKS cluster.') @@ -126,6 +134,8 @@ param dnszoneRGName string = 'dns-contoso-rg' param dsConnectionURL string = 'jdbc:postgresql://contoso.postgres.database.azure.com:5432/postgres' @description('true to set up Application Gateway ingress.') param enableAppGWIngress bool = false +@description('true to enable Horizontal Autoscaling.') +param enableAutoscaling bool = false @description('In addition to the CPU and memory metrics included in AKS by default, you can enable Container Insights for more comprehensive data on the overall performance and health of your cluster. Billing is based on data ingestion and retention settings.') param enableAzureMonitoring bool = false @description('true to create persistent volume using file share.') @@ -141,6 +151,11 @@ param enableAdminT3Tunneling bool = false param enableClusterT3Tunneling bool = false @description('Enable passwordless datasource connection.') param enablePswlessConnection bool = false +@allowed([ + 'cpu' + 'memory' +]) +param hpaScaleType string = 'cpu' @description('Is the specified SSO account associated with an active Oracle support contract?') param isSSOSupportEntitled bool = false @description('JNDI Name for JDBC Datasource') @@ -239,6 +254,8 @@ param t3ChannelAdminPort int = 7005 param t3ChannelClusterPort int = 8011 @description('True to use latest supported Kubernetes version.') param useLatestSupportedAksVersion bool = true +@description('True to enable HPA for auto scaling.') +param useHpa bool = true @description('True to set up internal load balancer service.') param useInternalLB bool = false @description('ture to upload Java EE applications and deploy the applications to WebLogic domain.') @@ -298,7 +315,7 @@ var _useExistingAppGatewaySSLCertificate = (appGatewayCertificateOption == const var const_appGatewaySSLCertOptionHaveCert = 'haveCert' var const_appGatewaySSLCertOptionHaveKeyVault = 'haveKeyVault' -var const_azcliVersion = '2.41.0' +var const_azcliVersion = '2.53.0' var const_azureSubjectName = format('{0}.{1}.{2}', name_domainLabelforApplicationGateway, location, 'cloudapp.azure.com') var const_hasTags = contains(resourceGroup(), 'tags') // If there is not tag 'wlsKeyVault' and key vault is created for the following usage: @@ -551,6 +568,7 @@ module wlsDomainDeployment 'modules/setupWebLogicCluster.bicep' = if (!enableCus acrResourceGroupName: preAzureResourceDeployment.outputs.acrResourceGroupName aksAgentPoolName: aksAgentPoolName aksAgentPoolNodeCount: aksAgentPoolNodeCount + aksAgentPoolNodeMaxCount: aksAgentPoolNodeMaxCount vmSize: vmSize aksClusterNamePrefix: aksClusterNamePrefix aksClusterRGName: aksClusterRGName @@ -621,6 +639,7 @@ module wlsDomainWithCustomSSLDeployment 'modules/setupWebLogicCluster.bicep' = i acrResourceGroupName: preAzureResourceDeployment.outputs.acrResourceGroupName aksAgentPoolName: aksAgentPoolName aksAgentPoolNodeCount: aksAgentPoolNodeCount + aksAgentPoolNodeMaxCount: aksAgentPoolNodeMaxCount vmSize: vmSize aksClusterNamePrefix: aksClusterNamePrefix aksClusterRGName: aksClusterRGName @@ -821,6 +840,33 @@ module validateApplciations 'modules/_deployment-scripts/_ds-validate-applicatio ] } +module horizontalAutoscaling 'modules/_enableAutoScaling.bicep' = if (enableAutoscaling) { + name: 'enable-horizontal-autoscaling' + params: { + _pidCPUUtilization: pids.outputs.cpuUtilization + _pidEnd: pids.outputs.autoScalingEnd + _pidMemoryUtilization: pids.outputs.memoryUtilization + _pidStart: pids.outputs.autoScalingStart + _pidWme: pids.outputs.enableWlsMonitoringExporter + aksClusterName: ref_wlsDomainDeployment.outputs.aksClusterName + aksClusterRGName: ref_wlsDomainDeployment.outputs.aksClusterRGName + azCliVersion: const_azcliVersion + hpaScaleType: hpaScaleType + identity: obj_uamiForDeploymentScript + location: location + useHpa: useHpa + utilizationPercentage: hpaScaleType == 'cpu' ? averageCpuUtilization : averageMemoryUtilization + wlsClusterSize: wlsClusterSize + wlsDomainUID: wlsDomainUID + wlsPassword: wlsPassword + wlsUserName: wlsUserName + + } + dependsOn: [ + validateApplciations + ] +} + /* * Query and output WebLogic domain configuration, including: * - domain deployment description @@ -839,7 +885,7 @@ module queryWLSDomainConfig 'modules/_deployment-scripts/_ds-output-domain-confi wlsDomainUID: wlsDomainUID } dependsOn: [ - validateApplciations + horizontalAutoscaling ] } @@ -857,7 +903,9 @@ output clusterExternalUrl string = const_enableNetworking ? networkingDeployment output clusterExternalSecuredUrl string = const_enableNetworking ? networkingDeployment.outputs.clusterExternalSecuredEndpoint : '' output clusterT3InternalUrl string = ref_wlsDomainDeployment.outputs.clusterT3InternalEndpoint output clusterT3ExternalEndpoint string = enableClusterT3Tunneling && const_enableNetworking ? networkingDeployment.outputs.clusterT3ChannelEndpoint : '' +output kedaScalerServerAddress string = enableAutoscaling ? horizontalAutoscaling.outputs.kedaScalerServerAddress : '' output shellCmdtoConnectAks string = format('az account set --subscription {0}; az aks get-credentials --resource-group {1} --name {2}', split(subscription().id, '/')[2], ref_wlsDomainDeployment.outputs.aksClusterRGName, ref_wlsDomainDeployment.outputs.aksClusterName) +output shellCmdtoOutputKedaScalerSample string = enableAutoscaling ? horizontalAutoscaling.outputs.base64ofKedaScalerSample : '' output shellCmdtoOutputWlsDomainYaml string = queryWLSDomainConfig.outputs.shellCmdtoOutputWlsDomainYaml output shellCmdtoOutputWlsImageModelYaml string = queryWLSDomainConfig.outputs.shellCmdtoOutputWlsImageModelYaml output shellCmdtoOutputWlsImageProperties string = queryWLSDomainConfig.outputs.shellCmdtoOutputWlsImageProperties diff --git a/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_aks.bicep b/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_aks.bicep index eea154323..e9f7bae03 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_aks.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_aks.bicep @@ -1,4 +1,4 @@ -// Copyright (c) 2021, Oracle Corporation and/or its affiliates. +// Copyright (c) 2021, 2024, Oracle Corporation and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. @description('true to use resource or workspace permissions. false to require workspace permissions.') @@ -15,6 +15,7 @@ param aksAgentPoolName string = 'agentpool' @minValue(1) @description('The number of nodes that should be created along with the cluster. You will be able to resize the cluster later.') param aksAgentPoolNodeCount int = 3 +param aksAgentPoolNodeMaxCount int = 5 @description('The size of the virtual machines that will form the nodes in the cluster. This cannot be changed after creating the cluster') param aksAgentPoolVMSize string = 'Standard_DS2_v2' @description('Prefix for cluster name. Only The name can contain only letters, numbers, underscores and hyphens. The name must start with letter or number.') @@ -26,7 +27,6 @@ param location string param utcValue string = utcNow() var const_aksAgentPoolOSDiskSizeGB = 128 -var const_aksAgentPoolMaxPods = 110 var const_aksAvailabilityZones = [ '1' '2' @@ -70,12 +70,14 @@ resource aksCluster 'Microsoft.ContainerService/managedClusters@${azure.apiVersi agentPoolProfiles: [ { name: aksAgentPoolName + enableAutoScaling: true + minCount: aksAgentPoolNodeCount + maxCount: aksAgentPoolNodeMaxCount count: aksAgentPoolNodeCount vmSize: aksAgentPoolVMSize osDiskSizeGB: const_aksAgentPoolOSDiskSizeGB osDiskType: 'Managed' kubeletDiskType: 'OS' - maxPods: const_aksAgentPoolMaxPods type: 'VirtualMachineScaleSets' availabilityZones: const_aksAvailabilityZones mode: 'System' diff --git a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-output-domain-configurations.bicep b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-output-domain-configurations.bicep index 515c5de86..5ede45003 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-output-domain-configurations.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-output-domain-configurations.bicep @@ -1,4 +1,4 @@ -// Copyright (c) 2021, Oracle Corporation and/or its affiliates. +// Copyright (c) 2021, 2024 Oracle Corporation and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. param aksClusterRGName string = '' @@ -11,7 +11,9 @@ param wlsClusterName string = 'cluster-1' param wlsDomainUID string = 'sample-domain1' // To mitigate arm-ttk error: Unreferenced variable: $fxv#0 +var base64_common = loadFileAsBase64('../../../arm/scripts/common.sh') var base64_queryDomainConfigurations = loadFileAsBase64('../../../arm/scripts/inline-scripts/queryDomainConfigurations.sh') +var base64_utility = loadFileAsBase64('../../../arm/scripts/utility.sh') resource deploymentScript 'Microsoft.Resources/deploymentScripts@${azure.apiVersionForDeploymentScript}' = { name: 'ds-query-wls-configurations' @@ -38,7 +40,7 @@ resource deploymentScript 'Microsoft.Resources/deploymentScripts@${azure.apiVers value: wlsDomainUID } ] - scriptContent: base64ToString(base64_queryDomainConfigurations) + scriptContent: format('{0}\r\n\r\n{1}\r\n\r\n{2}',base64ToString(base64_common), base64ToString(base64_utility), base64ToString(base64_queryDomainConfigurations)) cleanupPreference: 'OnSuccess' retentionInterval: 'P1D' forceUpdateTag: utcValue diff --git a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-query-storage-account.bicep b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-query-storage-account.bicep index 7dceb6622..88ca9ed37 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-query-storage-account.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-query-storage-account.bicep @@ -1,4 +1,4 @@ -// Copyright (c) 2021, Oracle Corporation and/or its affiliates. +// Copyright (c) 2021, 2024 Oracle Corporation and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. param aksClusterName string = '' @@ -10,7 +10,9 @@ param location string param utcValue string = utcNow() // To mitigate arm-ttk error: Unreferenced variable: $fxv#0 +var base64_common = loadFileAsBase64('../../../arm/scripts/common.sh') var base64_queryStorageAccount = loadFileAsBase64('../../../arm/scripts/queryStorageAccount.sh') +var base64_utility = loadFileAsBase64('../../../arm/scripts/utility.sh') var const_deploymentName = 'ds-query-storage-account' resource deploymentScript 'Microsoft.Resources/deploymentScripts@${azure.apiVersionForDeploymentScript}' = { @@ -30,7 +32,7 @@ resource deploymentScript 'Microsoft.Resources/deploymentScripts@${azure.apiVers value: aksClusterRGName } ] - scriptContent: base64ToString(base64_queryStorageAccount) + scriptContent: format('{0}\r\n\r\n{1}\r\n\r\n{2}',base64ToString(base64_common), base64ToString(base64_utility), base64ToString(base64_queryStorageAccount)) cleanupPreference: 'OnSuccess' retentionInterval: 'P1D' forceUpdateTag: utcValue diff --git a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds_enable_hpa.bicep b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds_enable_hpa.bicep new file mode 100644 index 000000000..0b2c9c233 --- /dev/null +++ b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds_enable_hpa.bicep @@ -0,0 +1,67 @@ +// Copyright (c) 2024, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +param aksClusterName string +param aksClusterRGName string +param azCliVersion string +@allowed([ + 'cpu' + 'memory' +]) +param hpaScaleType string = 'cpu' +param identity object = {} +param location string +param utcValue string = utcNow() +param utilizationPercentage int +param wlsClusterSize int +param wlsNamespace string + +// To mitigate arm-ttk error: Unreferenced variable: $fxv#0 +var base64_common = loadFileAsBase64('../../../arm/scripts/common.sh') +var base64_enableHpa = loadFileAsBase64('../../../arm/scripts/inline-scripts/enableHpa.sh') +var base64_utility = loadFileAsBase64('../../../arm/scripts/utility.sh') +var const_deploymentName='ds-enable-hpa' + +resource deploymentScript 'Microsoft.Resources/deploymentScripts@${azure.apiVersionForDeploymentScript}' = { + name: const_deploymentName + location: location + kind: 'AzureCLI' + identity: identity + properties: { + azCliVersion: azCliVersion + scriptContent: format('{0}\r\n\r\n{1}\r\n\r\n{2}',base64ToString(base64_common), base64ToString(base64_utility), base64ToString(base64_enableHpa)) + environmentVariables: [ + { + name: 'AKS_CLUSTER_RG_NAME' + value: aksClusterRGName + } + { + name: 'AKS_CLUSTER_NAME' + value: aksClusterName + } + { + name: 'CURRENT_RG_NAME' + value: resourceGroup().name + } + { + name: 'HPA_SCALE_TYPE' + value: hpaScaleType + } + { + name: 'UTILIZATION_PERCENTAGE' + value: utilizationPercentage + } + { + name: 'WLS_CLUSTER_SIZE' + value: wlsClusterSize + } + { + name: 'WLS_NAMESPACE' + value: wlsNamespace + } + ] + cleanupPreference: 'OnSuccess' + retentionInterval: 'P1D' + forceUpdateTag: utcValue + } +} diff --git a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds_enable_prometheus_metrics.bicep b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds_enable_prometheus_metrics.bicep new file mode 100644 index 000000000..5f9f93892 --- /dev/null +++ b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds_enable_prometheus_metrics.bicep @@ -0,0 +1,105 @@ +// Copyright (c) 2024, Oracle Corporation and/or its affiliates. +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +param aksClusterName string +param aksClusterRGName string +param amaName string +param azCliVersion string +param identity object = {} +param kedaUamiName string +param location string +param utcValue string = utcNow() +param wlsClusterSize int +param wlsDomainUID string +param wlsNamespace string +@secure() +param wlsPassword string +param wlsUserName string +param workspaceId string + +// To mitigate arm-ttk error: Unreferenced variable: $fxv#0 +var base64_common = loadFileAsBase64('../../../arm/scripts/common.sh') +var base64_enableHpa = loadFileAsBase64('../../../arm/scripts/inline-scripts/enablePrometheusMetrics.sh') +var base64_utility = loadFileAsBase64('../../../arm/scripts/utility.sh') +var const_deploymentName = 'ds-enable-promethues-metrics' +var const_kedaNamespace= 'keda' +var const_kedaSa= 'keda-operator' + +resource deploymentScript 'Microsoft.Resources/deploymentScripts@${azure.apiVersionForDeploymentScript}' = { + name: const_deploymentName + location: location + kind: 'AzureCLI' + identity: identity + properties: { + azCliVersion: azCliVersion + scriptContent: format('{0}\r\n\r\n{1}\r\n\r\n{2}', base64ToString(base64_common), base64ToString(base64_utility), base64ToString(base64_enableHpa)) + environmentVariables: [ + { + name: 'AKS_CLUSTER_RG_NAME' + value: aksClusterRGName + } + { + name: 'AKS_CLUSTER_NAME' + value: aksClusterName + } + { + name: 'AMA_NAME' + value: amaName + } + { + name: 'AMA_WORKSPACE_ID' + value: workspaceId + } + { + name: 'CURRENT_RG_NAME' + value: resourceGroup().name + } + { + name: 'KEDA_NAMESPACE' + value: const_kedaNamespace + } + { + name: 'KEDA_UAMI_NAME' + value: kedaUamiName + } + { + name: 'KEDA_SERVICE_ACCOUNT_NAME' + value: const_kedaSa + } + { + name: 'WLS_CLUSTER_SIZE' + value: wlsClusterSize + } + { + name: 'WLS_ADMIN_PASSWORD' + value: wlsPassword + } + { + name: 'WLS_ADMIN_USERNAME' + value: wlsUserName + } + { + name: 'WLS_DOMAIN_UID' + value: wlsDomainUID + } + { + name: 'WLS_NAMESPACE' + value: wlsNamespace + } + { + name: 'LOCATION' + value: location + } + { + name: 'SUBSCRIPTION' + value: split(subscription().id, '/')[2] + } + ] + cleanupPreference: 'OnSuccess' + retentionInterval: 'P1D' + forceUpdateTag: utcValue + } +} + +output kedaScalerServerAddress string = deploymentScript.properties.outputs.kedaScalerServerAddress +output base64ofKedaScalerSample string = deploymentScript.properties.outputs.base64ofKedaScalerSample diff --git a/weblogic-azure-aks/src/main/bicep/modules/_enableAutoScaling.bicep b/weblogic-azure-aks/src/main/bicep/modules/_enableAutoScaling.bicep new file mode 100644 index 000000000..dd178a25b --- /dev/null +++ b/weblogic-azure-aks/src/main/bicep/modules/_enableAutoScaling.bicep @@ -0,0 +1,120 @@ +/* + Copyright (c) 2024, Oracle and/or its affiliates. +Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +*/ + +param _pidCPUUtilization string = '' +param _pidEnd string = '' +param _pidMemoryUtilization string = '' +param _pidStart string = '' +param _pidWme string = '' + +param aksClusterName string +param aksClusterRGName string +param azCliVersion string + +@allowed([ + 'cpu' + 'memory' +]) +param hpaScaleType string = 'cpu' +param identity object = {} +param location string +param useHpa bool +param utilizationPercentage int +param wlsClusterSize int +param wlsDomainUID string +@secure() +param wlsPassword string +param wlsUserName string + +var const_namespace = '${wlsDomainUID}-ns' + +module pidAutoScalingStart './_pids/_pid.bicep' = { + name: 'pid-auto-scaling-start' + params: { + name: _pidStart + } +} + +module pidCpuUtilization './_pids/_pid.bicep' = if(useHpa && hpaScaleType == 'cpu') { + name: 'pid-auto-scaling-based-on-cpu-utilization' + params: { + name: _pidCPUUtilization + } + dependsOn: [ + pidAutoScalingStart + ] +} + +module pidMemoryUtilization './_pids/_pid.bicep' = if(useHpa && hpaScaleType == 'memory') { + name: 'pid-auto-scaling-based-on-memory-utilization' + params: { + name: _pidMemoryUtilization + } + dependsOn: [ + pidAutoScalingStart + ] +} + +module pidWme './_pids/_pid.bicep' = if(!useHpa) { + name: 'pid-auto-scaling-based-on-java-metrics' + params: { + name: _pidWme + } + dependsOn: [ + pidAutoScalingStart + ] +} + +module hapDeployment '_deployment-scripts/_ds_enable_hpa.bicep' = if(useHpa) { + name: 'hpa-deployment' + params: { + aksClusterName: aksClusterName + aksClusterRGName: aksClusterRGName + azCliVersion: azCliVersion + hpaScaleType: hpaScaleType + identity: identity + location: location + utilizationPercentage: utilizationPercentage + wlsClusterSize: wlsClusterSize + wlsNamespace: const_namespace + } + dependsOn: [ + pidAutoScalingStart + ] +} + +module promethuesKedaDeployment '_enablePromethuesKeda.bicep' = if (!useHpa) { + name: 'promethues-keda-weblogic-monitoring-exporter-deployment' + params: { + aksClusterName: aksClusterName + aksClusterRGName: aksClusterRGName + azCliVersion: azCliVersion + identity: identity + location: location + wlsClusterSize: wlsClusterSize + wlsDomainUID: wlsDomainUID + wlsPassword: wlsPassword + wlsUserName: wlsUserName + } + dependsOn: [ + pidAutoScalingStart + ] +} + + + +module pidAutoScalingEnd './_pids/_pid.bicep' = { + name: 'pid-auto-scaling-end' + params: { + name: _pidEnd + } + dependsOn: [ + hapDeployment + promethuesKedaDeployment + ] +} + +output kedaScalerServerAddress string = useHpa ? '' : promethuesKedaDeployment.outputs.kedaScalerServerAddress +output base64ofKedaScalerSample string = useHpa ? '' : promethuesKedaDeployment.outputs.base64ofKedaScalerSample diff --git a/weblogic-azure-aks/src/main/bicep/modules/_enablePromethuesKeda.bicep b/weblogic-azure-aks/src/main/bicep/modules/_enablePromethuesKeda.bicep new file mode 100644 index 000000000..12c6b791d --- /dev/null +++ b/weblogic-azure-aks/src/main/bicep/modules/_enablePromethuesKeda.bicep @@ -0,0 +1,80 @@ +/* + Copyright (c) 2024, Oracle and/or its affiliates. +Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +*/ +param aksClusterName string +param aksClusterRGName string +param azCliVersion string +param identity object = {} +param location string +param utcValue string = utcNow() +param wlsClusterSize int +param wlsDomainUID string +@secure() +param wlsPassword string +param wlsUserName string + +var const_namespace = '${wlsDomainUID}-ns' +// https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles +var const_roleDefinitionIdOfMonitorDataReader = 'b24988ac-6180-42a0-ab88-20f7382dd24c' +var name_azureMonitorAccountName = 'ama${uniqueString(utcValue)}' +var name_kedaUserDefinedManagedIdentity = 'kedauami${uniqueString(utcValue)}' +var name_kedaMonitorDataReaderRoleAssignmentName = guid('${resourceGroup().id}${name_kedaUserDefinedManagedIdentity}') + +resource monitorAccount 'Microsoft.Monitor/accounts@${azure.apiVersionForMonitorAccount}' = { + name: name_azureMonitorAccountName + location: location + properties: {} +} + +// UAMI for KEDA +resource uamiForKeda 'Microsoft.ManagedIdentity/userAssignedIdentities@${azure.apiVersionForIdentity}' = { + name: name_kedaUserDefinedManagedIdentity + location: location +} + +// Get role resource id +resource monitorDataReaderResourceDefinition 'Microsoft.Authorization/roleDefinitions@${azure.apiVersionForRoleDefinitions}' existing = { + name: const_roleDefinitionIdOfMonitorDataReader +} + +// Assign Monitor Data Reader role we need the permission to read data. +resource kedaUamiRoleAssignment 'Microsoft.Authorization/roleAssignments@${azure.apiVersionForRoleAssignment}' = { + name: name_kedaMonitorDataReaderRoleAssignmentName + scope: monitorAccount + properties: { + description: 'Assign Monitor Data Reader role role to KEDA Identity ' + principalId: reference(uamiForKeda.id, '${azure.apiVersionForIdentity}', 'full').properties.principalId + principalType: 'ServicePrincipal' + roleDefinitionId: monitorDataReaderResourceDefinition.id + } + dependsOn: [ + monitorAccount + uamiForKeda + ] +} + +module azureMonitorIntegrationDeployment '_deployment-scripts/_ds_enable_prometheus_metrics.bicep' = { + name: 'azure-monitor-promethues-keda-deployment' + params: { + aksClusterName: aksClusterName + aksClusterRGName: aksClusterRGName + amaName: name_azureMonitorAccountName + azCliVersion: azCliVersion + identity: identity + kedaUamiName: name_kedaUserDefinedManagedIdentity + location: location + wlsClusterSize: wlsClusterSize + wlsDomainUID: wlsDomainUID + wlsNamespace: const_namespace + wlsPassword: wlsPassword + wlsUserName: wlsUserName + workspaceId: monitorAccount.id + } + dependsOn: [ + kedaUamiRoleAssignment + ] +} + +output kedaScalerServerAddress string = azureMonitorIntegrationDeployment.outputs.kedaScalerServerAddress +output base64ofKedaScalerSample string = format('echo -e {0} | base64 -d > scaler.yaml', azureMonitorIntegrationDeployment.outputs.base64ofKedaScalerSample) diff --git a/weblogic-azure-aks/src/main/bicep/modules/_pids/_pid-dev.bicep b/weblogic-azure-aks/src/main/bicep/modules/_pids/_pid-dev.bicep index d40798ab6..98c1e07f4 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_pids/_pid-dev.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_pids/_pid-dev.bicep @@ -1,4 +1,4 @@ -// Copyright (c) 2021, Oracle Corporation and/or its affiliates. +// Copyright (c) 2021, 2024, Oracle Corporation and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. // Deployment for pids. @@ -12,15 +12,20 @@ module pidStart './_empty.bicep' = if (name != 'pid'){ output appgwEnd string = '38647ff6-ea8d-59e5-832d-b036a4d29c73' output appgwStart string = '8ba7beaa-96fd-576a-acd8-28f7a6efa83a' +output autoScalingEnd string = '074fd64c-184f-4bb9-b878-448cae164601' +output autoScalingStart string = 'f4e5974d-6531-4a34-a733-e7416258fe03' output customCertForAppgw string = 'b16ba29f-fc8e-5059-8988-f17bef4a9c5c' +output cpuUtilization string = '22a34519-24ea-4864-b304-7e47aee4071b' output dbEnd string = 'ffab0a3f-90cb-585a-a7f9-ec0a62faeec1' output dbStart string = 'e64361eb-fea0-5f15-a313-c76daadbc648' output dnsEnd string = '189306c7-39e2-5844-817d-01e883a4cf1e' output dnsStart string = '8ae63711-9fa7-56b4-a4a0-236f3ccef542' +output enableWlsMonitoringExporter string = '960632d4-918a-47c4-a32e-61ab0d4c470a' output lbEnd string = 'f76e2847-d5a1-52e7-9e52-fc8560f5d3e4' output lbStart string = 'e2a8c8b2-9b58-52c6-9636-1834ff3976dc' output networkingEnd string = '39d32fcd-1d02-50b6-9455-4b767a8e769e' output networkingStart string = 'ed47756f-2475-56dd-b13a-26027749b6e1' +output memoryUtilization string = 'b9b57564-3603-4d27-8e0c-a853ef0d60b9' output otherDb string = '551122ff-2fea-53a8-b7f4-6d6dae85af6a' output pswlessDbEnd string = '7e7aaa5b-2251-55b5-8b3d-43d514738cf2' output pswlessDbStart string = '089e9783-6707-54d0-ac8c-9b8d517914c5' diff --git a/weblogic-azure-aks/src/main/bicep/modules/_pids/_pid.bicep b/weblogic-azure-aks/src/main/bicep/modules/_pids/_pid.bicep index 4c901b125..b86bae86f 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_pids/_pid.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_pids/_pid.bicep @@ -1,4 +1,4 @@ -// Copyright (c) 2021, Oracle Corporation and/or its affiliates. +// Copyright (c) 2021, 2024, Oracle Corporation and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. // Deployment for pids. @@ -12,15 +12,20 @@ module pidStart './_empty.bicep' = if (name != 'pid'){ output appgwEnd string = '47ea43a0-95cf-52c7-aee8-7ee6106fc1bf' output appgwStart string = '01288010-2672-5831-a66b-7b8b45cace1b' +output autoScalingEnd string = 'fff261b8-f09f-4c46-b7b7-923c239f1da5' +output autoScalingStart string = '94fd65ac-bb13-475d-aba7-3352288777c2' output customCertForAppgw string = 'b80f52c3-dddd-5286-915e-e4cc64be3093' +output cpuUtilization string = 'deb6b656-aa63-4014-a686-6a01e8f87fec' output dbEnd string = 'd7a9c78e-39d9-5a47-928d-8645ed86dafd' output dbStart string = '0cc86800-37f4-5191-9368-2953394309ec' output dnsEnd string = '754e16bc-4d81-5343-b99b-7532abd6587d' output dnsStart string = '64ae895c-feb3-529e-8435-5d2e49f94e09' +output enableWlsMonitoringExporter string = '5e607302-2e52-42c4-8f02-29db35e3ddf1' output lbEnd string = 'ce664543-77bd-515a-832e-107e32f99da9' output lbStart string = '44732bbc-04c4-5df7-a0c6-b9be9ec00ee6' output networkingEnd string = '2798165c-49fa-5701-b608-b80ed3986176' output networkingStart string = '0793308f-de9d-5f0d-92f9-d9fc4b413b8b' +output memoryUtilization string = 'd1c2d027-f030-4a18-8e74-d608dd10c6f3' output otherDb string = 'fceccc86-531c-5e44-99fd-9f1250f8e409' output pswlessDbEnd string = '972084b9-2b2d-5eb9-aa37-80448a77fbe1' output pswlessDbStart string = '7190b263-7825-5ae3-bc56-7294df936d4a' diff --git a/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep b/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep index 5b2cdb284..959c7d7c2 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep @@ -1,4 +1,4 @@ -// Copyright (c) 2021, Oracle Corporation and/or its affiliates. +// Copyright (c) 2021, 2024, Oracle Corporation and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. /* @@ -35,8 +35,12 @@ param acrResourceGroupName string = '' param aksAgentPoolName string = 'agentpool' @maxValue(10000) @minValue(1) -@description('The number of nodes that should be created along with the cluster. You will be able to resize the cluster later.') +@description('Set the minimum node count for the cluster..') param aksAgentPoolNodeCount int = 3 +@maxValue(1000) +@minValue(3) +@description('Set the maximum node count for the cluster.') +param aksAgentPoolNodeMaxCount int = 5 @description('The size of the virtual machines that will form the nodes in the cluster. This cannot be changed after creating the cluster') param vmSize string = 'Standard_DS2_v2' @description('Prefix for cluster name. Only The name can contain only letters, numbers, underscores and hyphens. The name must start with letter or number.') @@ -158,6 +162,7 @@ module aksClusterDeployment './_azure-resoruces/_aks.bicep' = if (createAKSClust aciWorkspaceSku: aciWorkspaceSku aksAgentPoolName: aksAgentPoolName aksAgentPoolNodeCount: aksAgentPoolNodeCount + aksAgentPoolNodeMaxCount: aksAgentPoolNodeMaxCount aksAgentPoolVMSize: vmSize aksClusterNamePrefix: aksClusterNamePrefix aksVersion: aksVersion diff --git a/weblogic-azure-aks/src/main/resources/diagrams/wls-aks-diagram-autoscaling.vsdx b/weblogic-azure-aks/src/main/resources/diagrams/wls-aks-diagram-autoscaling.vsdx new file mode 100644 index 000000000..39ab6c68e Binary files /dev/null and b/weblogic-azure-aks/src/main/resources/diagrams/wls-aks-diagram-autoscaling.vsdx differ diff --git a/weblogic-azure-aks/src/main/resources/weblogic_tooling_family.json b/weblogic-azure-aks/src/main/resources/weblogic_tooling_family.json index e390e7bea..7d9d38323 100644 --- a/weblogic-azure-aks/src/main/resources/weblogic_tooling_family.json +++ b/weblogic-azure-aks/src/main/resources/weblogic_tooling_family.json @@ -21,6 +21,13 @@ "version": "1.12.1", "downloadURL": "https://github.com/oracle/weblogic-image-tool/releases/download/release-1.12.1/imagetool.zip", "testedDate": "2024-03-13" + }, + { + "key": "WME", + "description": "Oracle Weblogic Monitoring Exporter. IMPORTANT note to maintener: This value is hard coded in enablePrometheusMetrics.sh. Please update it there also.", + "version": "2.1.9", + "imageURL": "ghcr.io/oracle/weblogic-monitoring-exporter:2.1.9", + "testedDate": "2024-02-29" } ] }