From 8b22b532e264f289a205dc91f5358520cba1e2ef Mon Sep 17 00:00:00 2001 From: galiacheng Date: Fri, 20 Aug 2021 17:18:37 +0800 Subject: [PATCH 01/14] On branch t3tunneling: support custom t3 Upgrade wit and wdt to latest version. Changes to be committed: modified: weblogic-azure-aks/src/main/arm/createUiDefinition.json modified: weblogic-azure-aks/src/main/arm/scripts/buildWLSDockerImage.sh modified: weblogic-azure-aks/src/main/arm/scripts/createVMAndBuildImage.sh modified: weblogic-azure-aks/src/main/arm/scripts/genDomainConfig.sh modified: weblogic-azure-aks/src/main/arm/scripts/genImageModel.sh modified: weblogic-azure-aks/src/main/arm/scripts/invokeSetupNetworking.sh modified: weblogic-azure-aks/src/main/arm/scripts/invokeSetupWLSDomain.sh modified: weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh modified: weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh modified: weblogic-azure-aks/src/main/arm/scripts/updateDomainConfig.sh modified: weblogic-azure-aks/src/main/bicep/mainTemplate.bicep modified: weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep modified: weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-wls-cluster.bicep modified: weblogic-azure-aks/src/main/bicep/modules/networking.bicep modified: weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep Signed-off-by: galiacheng --- .../src/main/arm/createUiDefinition.json | 53 ++++++- .../main/arm/scripts/buildWLSDockerImage.sh | 15 +- .../main/arm/scripts/createVMAndBuildImage.sh | 9 +- .../src/main/arm/scripts/genDomainConfig.sh | 35 ++++- .../src/main/arm/scripts/genImageModel.sh | 44 +++++- .../main/arm/scripts/invokeSetupNetworking.sh | 8 +- .../main/arm/scripts/invokeSetupWLSDomain.sh | 11 +- .../src/main/arm/scripts/setupNetworking.sh | 132 +++++++++++++++++- .../src/main/arm/scripts/setupWLSDomain.sh | 33 ++++- .../main/arm/scripts/updateDomainConfig.sh | 27 +++- .../src/main/bicep/mainTemplate.bicep | 26 +++- .../_ds-create-networking.bicep | 8 +- .../_ds-create-wls-cluster.bicep | 5 +- .../src/main/bicep/modules/networking.bicep | 26 ++-- .../bicep/modules/setupWebLogicCluster.bicep | 8 +- 15 files changed, 403 insertions(+), 37 deletions(-) diff --git a/weblogic-azure-aks/src/main/arm/createUiDefinition.json b/weblogic-azure-aks/src/main/arm/createUiDefinition.json index 66c64fd0f..41849d427 100644 --- a/weblogic-azure-aks/src/main/arm/createUiDefinition.json +++ b/weblogic-azure-aks/src/main/arm/createUiDefinition.json @@ -212,6 +212,12 @@ "required": true }, "visible": "[bool(basics('basicsOptional').basicsOptionalAcceptDefaults)]" + }, + { + "name": "enableT3Tunneling", + "type": "Microsoft.Common.CheckBox", + "label": "Enable T3 tunneling", + "visible": "[bool(basics('basicsOptional').basicsOptionalAcceptDefaults)]" } ], "visible": true @@ -1399,6 +1405,27 @@ }, "visible": "[steps('section_appGateway').dnsConfiguration.enableDNSConfiguration]" }, + { + "name": "dnszoneAdminT3ChannelLabel", + "type": "Microsoft.Common.TextBox", + "label": "Label for Oracle WebLogic Admin Server T3 channel", + "defaultValue": "admin-t3", + "toolTip": "Specify a label to generate subdomain of Oracle WebLogic Admin Server T3 channel", + "constraints": { + "required": true, + "validations": [ + { + "regex": "^([0-9a-zA-Z_-]{1,63}\\.){0,33}[0-9a-zA-Z_-]{1,63}$", + "message": "Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." + }, + { + "isValid": "[less(sub(length(concat(steps('section_appGateway').dnsConfiguration.dnszoneAdminT3ChannelLabel,'.',steps('section_appGateway').dnsConfiguration.dnszoneName)),length(replace(concat(steps('section_appGateway').dnsConfiguration.dnszoneAdminT3ChannelLabel,'.',steps('section_appGateway').dnsConfiguration.dnszoneName), '.', ''))),34)]", + "message": "Subdomain must be between 2 and 34 labels. For example, \"admin-t3.contoso.com\" has 3 labels." + } + ] + }, + "visible": "[and(steps('section_appGateway').dnsConfiguration.enableDNSConfiguration,basics('basicsOptional').enableT3Tunneling,steps('section_appGateway').lbSVCInfo.enableLBSVC)]" + }, { "name": "dnszoneGatewayLabel", "type": "Microsoft.Common.TextBox", @@ -1419,6 +1446,27 @@ ] }, "visible": "[steps('section_appGateway').dnsConfiguration.enableDNSConfiguration]" + }, + { + "name": "dnszoneClusterT3ChannelLabel", + "type": "Microsoft.Common.TextBox", + "label": "Label for Oracle WebLogic cluster T3 channel", + "defaultValue": "cluster-t3", + "toolTip": "Specify a label to generate subdomain of Oracle WebLogic cluster T3 channel", + "constraints": { + "required": true, + "validations": [ + { + "regex": "^([0-9a-zA-Z_-]{1,63}\\.){0,33}[0-9a-zA-Z_-]{1,63}$", + "message": "Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." + }, + { + "isValid": "[less(sub(length(concat(steps('section_appGateway').dnsConfiguration.dnszoneClusterT3ChannelLabel,'.',steps('section_appGateway').dnsConfiguration.dnszoneName)),length(replace(concat(steps('section_appGateway').dnsConfiguration.dnszoneClusterT3ChannelLabel,'.',steps('section_appGateway').dnsConfiguration.dnszoneName), '.', ''))),34)]", + "message": "Subdomain must be between 2 and 34 labels. For example, \"application-t3.contoso.com\" has 3 labels." + } + ] + }, + "visible": "[and(steps('section_appGateway').dnsConfiguration.enableDNSConfiguration,basics('basicsOptional').enableT3Tunneling,steps('section_appGateway').lbSVCInfo.enableLBSVC)]" } ], "visible": true @@ -1576,7 +1624,9 @@ "dbUser": "[steps('section_database').databaseConnectionInfo.dbUser]", "databaseType": "[steps('section_database').databaseConnectionInfo.databaseType]", "dnszoneAdminConsoleLabel": "[steps('section_appGateway').dnsConfiguration.dnszoneAdminConsoleLabel]", - "dnszoneAppGatewayLabel": "[steps('section_appGateway').dnsConfiguration.dnszoneGatewayLabel]", + "dnszoneAdminT3ChannelLabel": "[steps('section_appGateway').dnsConfiguration.dnszoneAdminT3ChannelLabel]", + "dnszoneClusterLabel": "[steps('section_appGateway').dnsConfiguration.dnszoneGatewayLabel]", + "dnszoneClusterT3ChannelLabel": "[steps('section_appGateway').dnsConfiguration.dnszoneClusterT3ChannelLabel]", "dnszoneName": "[steps('section_appGateway').dnsConfiguration.dnszoneName]", "dnszoneRGName": "[steps('section_appGateway').dnsConfiguration.dnsZoneResourceGroup]", "dsConnectionURL": "[steps('section_database').databaseConnectionInfo.dsConnectionURL]", @@ -1587,6 +1637,7 @@ "enableCustomSSL": "[bool(steps('section_sslConfiguration').enableCustomSSL)]", "enableDB": "[bool(steps('section_database').enableDB)]", "enableDNSConfiguration": "[bool(steps('section_appGateway').dnsConfiguration.enableDNSConfiguration)]", + "enableT3Tunneling": "[basics('basicsOptional').enableT3Tunneling]", "identity": "[basics('basicsRequired').identity]", "jdbcDataSourceName": "[steps('section_database').databaseConnectionInfo.jdbcDataSourceName]", "lbSvcValues": "[steps('section_appGateway').lbSVCInfo.lbSVC]", diff --git a/weblogic-azure-aks/src/main/arm/scripts/buildWLSDockerImage.sh b/weblogic-azure-aks/src/main/arm/scripts/buildWLSDockerImage.sh index 3c19e2cb5..e6dcd4068 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/buildWLSDockerImage.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/buildWLSDockerImage.sh @@ -15,7 +15,7 @@ function read_sensitive_parameters_from_stdin() { #Function to display usage message function usage() { - echo " ./buildWLSDockerImage.sh ./buildWLSDockerImage.sh " + echo " ./buildWLSDockerImage.sh ./buildWLSDockerImage.sh " if [ $1 -eq 1 ]; then exit 1 fi @@ -80,6 +80,11 @@ function validate_inputs() { echo_stderr "enableSSL is required. " usage 1 fi + + if [ -z "$enableT3Tunneling" ]; then + echo_stderr "enableT3Tunneling is required. " + usage 1 + fi } function initialize() { @@ -179,7 +184,8 @@ EOF bash $scriptDir/genImageModel.sh \ ${modelFilePath} \ ${appPackageUrls} \ - ${enableSSL} + ${enableSSL} \ + ${enableT3Tunneling} validate_status "Generate image model file." } @@ -236,11 +242,12 @@ export appPackageUrls=$5 export ocrSSOUser=$6 export wlsClusterSize=$7 export enableSSL=$8 +export enableT3Tunneling=$9 export acrImagePath="$azureACRServer/aks-wls-images:${imageTag}" export ocrLoginServer="container-registry.oracle.com" -export wdtDownloadURL="https://github.com/oracle/weblogic-deploy-tooling/releases/download/release-1.9.14/weblogic-deploy.zip" -export witDownloadURL="https://github.com/oracle/weblogic-image-tool/releases/download/release-1.9.12/imagetool.zip" +export wdtDownloadURL="https://github.com/oracle/weblogic-deploy-tooling/releases/download/release-1.9.17/weblogic-deploy.zip" +export witDownloadURL="https://github.com/oracle/weblogic-image-tool/releases/download/release-1.9.16/imagetool.zip" export wlsPostgresqlDriverUrl="https://jdbc.postgresql.org/download/postgresql-42.2.8.jar" export wlsMSSQLDriverUrl="https://repo.maven.apache.org/maven2/com/microsoft/sqlserver/mssql-jdbc/7.4.1.jre8/mssql-jdbc-7.4.1.jre8.jar" diff --git a/weblogic-azure-aks/src/main/arm/scripts/createVMAndBuildImage.sh b/weblogic-azure-aks/src/main/arm/scripts/createVMAndBuildImage.sh index cc06553c3..15875e3b9 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/createVMAndBuildImage.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/createVMAndBuildImage.sh @@ -94,7 +94,7 @@ function build_docker_image() { --publisher Microsoft.Azure.Extensions \ --version 2.0 \ --settings "{ \"fileUris\": [\"${scriptURL}model.properties\",\"${scriptURL}genImageModel.sh\",\"${scriptURL}buildWLSDockerImage.sh\",\"${scriptURL}common.sh\"]}" \ - --protected-settings "{\"commandToExecute\":\"echo ${azureACRPassword} ${ocrSSOPSW} | bash buildWLSDockerImage.sh ${wlsImagePath} ${azureACRServer} ${azureACRUserName} ${newImageTag} \\\"${appPackageUrls}\\\" ${ocrSSOUser} ${wlsClusterSize} ${enableCustomSSL} \"}" + --protected-settings "{\"commandToExecute\":\"echo ${azureACRPassword} ${ocrSSOPSW} | bash buildWLSDockerImage.sh ${wlsImagePath} ${azureACRServer} ${azureACRUserName} ${newImageTag} \\\"${appPackageUrls}\\\" ${ocrSSOUser} ${wlsClusterSize} ${enableCustomSSL} ${enableT3Tunneling} \"}" cleanup_vm } @@ -102,6 +102,12 @@ function build_docker_image() { # Shell Global settings set -e #Exit immediately if a command exits with a non-zero status. +# Main script +export script="${BASH_SOURCE[0]}" +export scriptDir="$(cd "$(dirname "${script}")" && pwd)" + +source ${scriptDir}/common.sh + export currentResourceGroup=$1 export wlsImageTag=$2 export azureACRServer=$3 @@ -112,6 +118,7 @@ export ocrSSOUser=$7 export wlsClusterSize=$8 export enableCustomSSL=$9 export scriptURL=${10} +export enableT3Tunneling=${11} read_sensitive_parameters_from_stdin diff --git a/weblogic-azure-aks/src/main/arm/scripts/genDomainConfig.sh b/weblogic-azure-aks/src/main/arm/scripts/genDomainConfig.sh index 2f9e716b8..8483f1669 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/genDomainConfig.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/genDomainConfig.sh @@ -14,7 +14,14 @@ export wlsMemory=$7 export wlsManagedPrefix=$8 export enableSSL=${9} export enablePV=${10} -export javaOptions=${11} +export enableT3Tunneling=${11} +export t3AdminPort=${12} +export t3ClusterPort=${13} +export clusterName=${14} +export javaOptions=${15} + +export adminServiceUrl="${wlsDomainUID}-admin-server.${wlsDomainUID}-ns.svc.cluster.local" +export clusterServiceUrl="${wlsDomainUID}-cluster-${clusterName}.${wlsDomainUID}-ns.svc.cluster.local" cat <$filePath # Copyright (c) 2021, Oracle Corporation and/or its affiliates. @@ -86,7 +93,8 @@ spec: - name: MANAGED_SERVER_PREFIX value: "${wlsManagedPrefix}" EOF - if [[ "${enableSSL,,}" == "true" ]]; then + +if [[ "${enableSSL,,}" == "true" ]]; then cat <>$filePath - name: SSL_IDENTITY_PRIVATE_KEY_ALIAS valueFrom: @@ -131,15 +139,28 @@ EOF EOF fi - # Resources - cat <>$filePath +if [[ "${enableT3Tunneling,,}" == "true" ]]; then + cat <>$filePath + - name: T3_TUNNELING_ADMIN_PORT + value: "${t3AdminPort}" + - name: T3_TUNNELING_ADMIN_ADDRESS + value: "${adminServiceUrl}" + - name: T3_TUNNELING_CLUSTER_PORT + value: "${t3ClusterPort}" + - name: T3_TUNNELING_CLUSTER_ADDRESS + value: "${clusterServiceUrl}" +EOF +fi + +# Resources +cat <>$filePath resources: requests: cpu: "${wlsCPU}" memory: "${wlsMemory}" EOF - if [[ "${enablePV,,}" == "true" ]]; then +if [[ "${enablePV,,}" == "true" ]]; then cat <>$filePath # Optional volumes and mounts for the domain's pods. See also 'logHome'. volumes: @@ -150,9 +171,9 @@ EOF - mountPath: /shared name: ${wlsDomainUID}-pv-azurefile EOF - fi +fi - cat <>$filePath +cat <>$filePath # The desired behavior for starting the domain's administration server. adminServer: # The serverStartState legal values are "RUNNING" or "ADMIN" diff --git a/weblogic-azure-aks/src/main/arm/scripts/genImageModel.sh b/weblogic-azure-aks/src/main/arm/scripts/genImageModel.sh index 4d07a234e..a1afbb127 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/genImageModel.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/genImageModel.sh @@ -10,6 +10,9 @@ source ${scriptDir}/common.sh export filePath=$1 export appPackageUrls=$2 export enableCustomSSL=$3 +export enableT3Tunneling=$4 + +export enableT3s=${enableCustomSSL,,} cat <${filePath} # Copyright (c) 2020, 2021, Oracle and/or its affiliates. @@ -42,7 +45,24 @@ topology: ListenPort: 7001 EOF -if [[ "${enableCustomSSL,,}" == "true" ]];then +if [[ "${enableT3Tunneling,,}" == "true" ]];then + cat <>${filePath} + NetworkAccessPoint: + MyT3Channel: + Protocol: 't3' + ListenPort: "@@ENV:T3_TUNNELING_ADMIN_PORT@@" + PublicPort: "@@ENV:T3_TUNNELING_ADMIN_PORT@@" + HttpEnabledForThisProtocol: true + OutboundEnabled: false + Enabled: true + ClientCertificateEnforced: ${enableCustomSSL} + TunnelingEnabled: true + PublicAddress: '@@ENV:T3_TUNNELING_ADMIN_ADDRESS@@' + TwoWaySslEnabled: ${enableT3s} +EOF +fi + +if [[ "${enableCustomSSL,,}" == "true" ]]; then cat <>${filePath} SSL: HostnameVerificationIgnored: true @@ -68,6 +88,23 @@ cat <>${filePath} ListenPort: 8001 EOF +if [[ "${enableT3Tunneling,,}" == "true" ]];then + cat <>${filePath} + NetworkAccessPoint: + MyT3Channel: + Protocol: 't3' + ListenPort: "@@ENV:T3_TUNNELING_CLUSTER_PORT@@" + PublicPort: "@@ENV:T3_TUNNELING_CLUSTER_PORT@@" + HttpEnabledForThisProtocol: true + OutboundEnabled: false + Enabled: true + ClientCertificateEnforced: ${enableCustomSSL} + TunnelingEnabled: true + PublicAddress: '@@ENV:T3_TUNNELING_CLUSTER_ADDRESS@@' + TwoWaySslEnabled: ${enableT3s} +EOF +fi + if [[ "${enableCustomSSL,,}" == "true" ]];then cat <>${filePath} SSL: @@ -142,4 +179,7 @@ EOF Target: 'cluster-1' EOF index=$((index + 1)) - done \ No newline at end of file + done + +# print model +cat ${filePath} \ No newline at end of file diff --git a/weblogic-azure-aks/src/main/arm/scripts/invokeSetupNetworking.sh b/weblogic-azure-aks/src/main/arm/scripts/invokeSetupNetworking.sh index 0e87e661b..32edb91ad 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/invokeSetupNetworking.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/invokeSetupNetworking.sh @@ -32,6 +32,8 @@ Usage: + + END ) echo_stdout "${usage}" @@ -72,6 +74,8 @@ export appgwCertificateOption=${22} export enableCustomSSL=${23} export enableCookieBasedAffinity=${24} export enableRemoteConsole=${25} +export dnszoneAdminT3ChannelLabel=${26} +export dnszoneClusterT3ChannelLabel=${27} echo ${spBase64String} \ ${appgwFrontendSSLCertPsw} | \ @@ -98,7 +102,9 @@ echo ${spBase64String} \ ${appgwCertificateOption} \ ${enableCustomSSL} \ ${enableCookieBasedAffinity} \ - ${enableRemoteConsole} + ${enableRemoteConsole} \ + ${dnszoneAdminT3ChannelLabel} \ + ${dnszoneClusterT3ChannelLabel} if [ $? -ne 0 ]; then usage 1 diff --git a/weblogic-azure-aks/src/main/arm/scripts/invokeSetupWLSDomain.sh b/weblogic-azure-aks/src/main/arm/scripts/invokeSetupWLSDomain.sh index 1a5bf0cc7..2b01a1eba 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/invokeSetupWLSDomain.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/invokeSetupWLSDomain.sh @@ -37,6 +37,9 @@ Usage: + + + END ) echo_stdout "${usage}" @@ -82,6 +85,9 @@ export wlsTrustData=${27} wlsTrustPsw=${28} export wlsTrustType=${29} export enablePV=${30} +export enableT3Tunneling=${31} +export t3AdminPort=${32} +export t3ClusterPort=${33} echo ${ocrSSOPSW} \ ${wlsPassword} \ @@ -113,7 +119,10 @@ echo ${ocrSSOPSW} \ ${wlsIdentityAlias} \ ${wlsTrustData} \ ${wlsTrustType} \ - ${enablePV} + ${enablePV} \ + ${enableT3Tunneling} \ + ${t3AdminPort} \ + ${t3ClusterPort} if [ $? -ne 0 ]; then usage 1 diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh index 8c504fb25..2108199c5 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh @@ -83,6 +83,8 @@ echo | + + END ) echo_stdout ${usage} @@ -223,6 +225,16 @@ function validate_input() { echo_stderr "enableRemoteConsole is required. " usage 1 fi + + if [ -z "$dnszoneAdminT3ChannelLabel" ]; then + echo_stderr "dnszoneAdminT3ChannelLabel is required. " + usage 1 + fi + + if [ -z "$dnszoneClusterT3ChannelLabel" ]; then + echo_stderr "dnszoneClusterT3ChannelLabel is required. " + usage 1 + fi } function generate_admin_lb_definicion() { @@ -257,6 +269,38 @@ spec: EOF } +function generate_admin_t3_lb_definicion() { + cat <${adminServerT3LBDefinitionPath} +apiVersion: v1 +kind: Service +metadata: + name: ${adminServerT3LBSVCName} + namespace: ${wlsDomainNS} +EOF + + # to create internal load balancer service + if [[ "${enableInternalLB,,}" == "true" ]]; then + cat <>${adminServerT3LBDefinitionPath} + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" +EOF + fi + + cat <>${adminServerT3LBDefinitionPath} +spec: + ports: + - name: default + port: ${adminT3LBPort} + protocol: TCP + targetPort: ${adminT3Port} + selector: + weblogic.domainUID: ${wlsDomainUID} + weblogic.serverName: ${adminServerName} + sessionAffinity: None + type: LoadBalancer +EOF +} + function generate_cluster_lb_definicion() { cat <${scriptDir}/cluster-lb.yaml apiVersion: v1 @@ -289,6 +333,38 @@ spec: EOF } +function generate_cluster_t3_lb_definicion() { + cat <${clusterT3LBDefinitionPath} +apiVersion: v1 +kind: Service +metadata: + name: ${clusterT3LBSVCName} + namespace: ${wlsDomainNS} +EOF + + # to create internal load balancer service + if [[ "${enableInternalLB,,}" == "true" ]]; then + cat <>${clusterT3LBDefinitionPath} + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" +EOF + fi + + cat <>${clusterT3LBDefinitionPath} +spec: + ports: + - name: default + port: ${clusterT3LBPort} + protocol: TCP + targetPort: ${clusterT3Port} + selector: + weblogic.domainUID: ${wlsDomainUID} + weblogic.clusterName: ${clusterName} + sessionAffinity: None + type: LoadBalancer +EOF +} + function generate_appgw_cluster_config_file_expose_https() { export clusterIngressHttpsName=${wlsDomainUID}-cluster-appgw-ingress-https-svc @@ -759,7 +835,7 @@ EOF if [ "${enableCustomDNSAlias,,}" == "true" ]; then adminConsoleEndpoint="${dnsAdminLabel}.${dnsZoneName}:${adminServerEndpoint#*:}/console" fi - else + elif [[ "${target}" == "cluster1" ]]; then clusterLBSVCNamePrefix=$(cut -d',' -f1 <<<$item) clusterLBSVCName="${clusterLBSVCNamePrefix}-svc-lb-cluster" clusterLBPort=$(cut -d',' -f3 <<<$item) @@ -776,6 +852,58 @@ EOF if [ "${enableCustomDNSAlias,,}" == "true" ]; then clusterEndpoint="${dnsClusterLabel}.${dnsZoneName}:${clusterEndpoint#*:}/" fi + elif [[ "${target}" == "adminServerT3" ]]; then + echo "query admin t3 port" + adminT3Port=$( kubectl get service ${svcAdminServer} -n ${wlsDomainNS} -o json \ + | jq '.spec.ports[] | select(.name=="t3channel") | .port') + if [[ "${adminT3Port}" == "null" ]]; then + continue + fi + + adminServerT3LBSVCNamePrefix=$(cut -d',' -f1 <<<$item) + adminServerT3LBSVCName="${adminServerT3LBSVCNamePrefix}-svc-t3-lb-admin" + adminT3LBPort=$(cut -d',' -f3 <<<$item) + + export adminServerT3LBDefinitionPath=${scriptDir}/admin-server-t3-lb.yaml + generate_admin_t3_lb_definicion + + kubectl apply -f ${adminServerT3LBDefinitionPath} + waitfor_svc_completed ${adminServerT3LBSVCName} + + adminServerT3Endpoint=$(kubectl get svc ${adminServerT3LBSVCName} -n ${wlsDomainNS} \ + -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') + + create_dns_A_record "${adminServerT3Endpoint%%:*}" "${dnszoneAdminT3ChannelLabel}" + + if [ "${enableCustomDNSAlias,,}" == "true" ]; then + adminServerT3Endpoint="${dnszoneAdminT3ChannelLabel}.${dnsZoneName}:${adminServerT3Endpoint#*:}" + fi + elif [[ "${target}" == "cluster1T3" ]]; then + echo "query cluster t3 port" + clusterT3Port=$( kubectl get service ${svcCluster} -n ${wlsDomainNS} -o json \ + | jq '.spec.ports[] | select(.name=="t3channel") | .port') + if [[ "${clusterT3Port}" == "null" ]]; then + continue + fi + + clusterT3LBSVCNamePrefix=$(cut -d',' -f1 <<<$item) + clusterT3LBSVCName="${clusterT3LBSVCNamePrefix}-svc-lb-cluster" + clusterT3LBPort=$(cut -d',' -f3 <<<$item) + + export clusterT3LBDefinitionPath=${scriptDir}/cluster-t3-lb.yaml + generate_cluster_t3_lb_definicion + + kubectl apply -f ${clusterT3LBDefinitionPath} + waitfor_svc_completed ${clusterT3LBSVCName} + + clusterT3Endpoint=$(kubectl get svc ${clusterT3LBSVCName} -n ${wlsDomainNS} \ + -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') + + create_dns_A_record "${clusterT3Endpoint%%:*}" ${dnszoneClusterT3ChannelLabel} + + if [ "${enableCustomDNSAlias,,}" == "true" ]; then + clusterT3Endpoint="${dnszoneClusterT3ChannelLabel}.${dnsZoneName}:${clusterT3Endpoint#*:}" + fi fi done } @@ -975,6 +1103,8 @@ export appgwCertificateOption=${20} export enableCustomSSL=${21} export enableCookieBasedAffinity=${22} export enableRemoteConsole=${23} +export dnszoneAdminT3ChannelLabel=${24} +export dnszoneClusterT3ChannelLabel=${25} export adminServerName="admin-server" export adminConsoleEndpoint="null" diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh index 58cf72e90..af960b6f4 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh @@ -39,6 +39,9 @@ echo + + + END ) echo_stdout ${usage} @@ -174,6 +177,21 @@ function validate_input() { echo_stderr "enablePV is required. " usage 1 fi + + if [ -z "$enableT3Tunneling" ]; then + echo_stderr "enableT3Tunneling is required. " + usage 1 + fi + + if [ -z "$t3AdminPort" ]; then + echo_stderr "t3AdminPort is required. " + usage 1 + fi + + if [ -z "$t3ClusterPort" ]; then + echo_stderr "t3ClusterPort is required. " + usage 1 + fi } # Validate teminal status with $?, exit with exception if errors happen. @@ -333,7 +351,8 @@ function build_docker_image() { $ocrSSOUser \ $wlsClusterSize \ $enableCustomSSL \ - "$scriptURL" + "$scriptURL" \ + ${enableT3Tunneling} az acr repository show -n ${acrName} --image aks-wls-images:${newImageTag} if [ $? -ne 0 ]; then @@ -657,6 +676,10 @@ function setup_wls_domain() { ${managedServerPrefix} \ ${enableCustomSSL} \ ${enablePV} \ + ${enableT3Tunneling} \ + ${t3AdminPort} \ + ${t3ClusterPort} \ + ${wlsClusterName} \ "${javaOptions}" else echo "start to create domain ${wlsDomainUID}" @@ -673,6 +696,10 @@ function setup_wls_domain() { ${managedServerPrefix} \ ${enableCustomSSL} \ ${enablePV} \ + ${enableT3Tunneling} \ + ${t3AdminPort} \ + ${t3ClusterPort} \ + ${wlsClusterName} \ "${javaOptions}" fi @@ -714,6 +741,9 @@ export wlsIdentityAlias=${21} export wlsTrustData=${22} export wlsTrustType=${23} export enablePV=${24} +export enableT3Tunneling=${25} +export t3AdminPort=${26} +export t3ClusterPort=${27} export adminServerName="admin-server" export azFileShareName="weblogic" @@ -727,6 +757,7 @@ export operatorName="weblogic-operator" export storageFileShareName="weblogic" export storageResourceGroup=${currentResourceGroup} export sharedPath="/shared" +export wlsClusterName="cluster-1" export wlsDomainNS="${wlsDomainUID}-ns" export wlsOptHelmChart="https://oracle.github.io/weblogic-kubernetes-operator/charts" export wlsOptNameSpace="weblogic-operator-ns" diff --git a/weblogic-azure-aks/src/main/arm/scripts/updateDomainConfig.sh b/weblogic-azure-aks/src/main/arm/scripts/updateDomainConfig.sh index 17e3ebca7..94128d0b8 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/updateDomainConfig.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/updateDomainConfig.sh @@ -14,8 +14,14 @@ export wlsMemory=$7 export wlsManagedPrefix=$8 export enableSSL=${9} export enablePV=${10} -export javaOptions=${11} +export enableT3Tunneling=${11} +export t3AdminPort=${12} +export t3ClusterPort=${13} +export clusterName=${14} +export javaOptions=${15} +export adminServiceUrl="${wlsDomainUID}-admin-server.${wlsDomainUID}-ns.svc.cluster.local" +export clusterServiceUrl="${wlsDomainUID}-cluster-${clusterName}.${wlsDomainUID}-ns.svc.cluster.local" export wlsDomainNS="${wlsDomainUID}-ns" # output the existing domain configuration @@ -150,6 +156,19 @@ if [[ "${enableSSL,,}" == "true" ]]; then EOF fi +if [[ "${enableT3Tunneling,,}" == "true" ]]; then + cat <>$filePath + - name: T3_TUNNELING_ADMIN_PORT + value: "${t3AdminPort}" + - name: T3_TUNNELING_ADMIN_ADDRESS + value: "${adminServiceUrl}" + - name: T3_TUNNELING_CLUSTER_PORT + value: "${t3ClusterPort}" + - name: T3_TUNNELING_CLUSTER_ADDRESS + value: "${clusterServiceUrl}" +EOF +fi + index=0 while [ $index -lt ${envLength} ]; do envItemName=$(cat ${previousConfig} | jq ". | .spec.serverPod.env[$index] | .name" | tr -d "\"") @@ -168,7 +187,11 @@ while [ $index -lt ${envLength} ]; do || [[ "${envItemName}" == "SSL_IDENTITY_PRIVATE_KEYSTORE_PSW" ]] \ || [[ "${envItemName}" == "SSL_TRUST_KEYSTORE_PATH" ]] \ || [[ "${envItemName}" == "SSL_TRUST_KEYSTORE_TYPE" ]] \ - || [[ "${envItemName}" == "SSL_TRUST_KEYSTORE_PSW" ]];then + || [[ "${envItemName}" == "SSL_TRUST_KEYSTORE_PSW" ]] \ + || [[ "${envItemName}" == "T3_TUNNELING_ADMIN_PORT" ]] \ + || [[ "${envItemName}" == "T3_TUNNELING_ADMIN_ADDRESS" ]] \ + || [[ "${envItemName}" == "T3_TUNNELING_CLUSTER_PORT" ]] \ + || [[ "${envItemName}" == "T3_TUNNELING_CLUSTER_ADDRESS" ]];then continue fi diff --git a/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep b/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep index bc4b8196e..247d3e0f2 100644 --- a/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep +++ b/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep @@ -93,10 +93,14 @@ param dbPassword string = newGuid() param dbUser string = 'contosoDbUser' @description('DNS prefix for ApplicationGateway') param dnsNameforApplicationGateway string = 'wlsgw' -@description('Azure DNS Zone name.') +@description('Specify a label used to generate subdomain of Admin server. The final subdomain name will be label.dnszoneName, e.g. admin.contoso.xyz') param dnszoneAdminConsoleLabel string = 'admin' -@description('Specify a label used to generate subdomain of Application Gateway. The final subdomain name will be label.dnszoneName, e.g. applications.contoso.xyz') -param dnszoneAppGatewayLabel string = 'www' +@description('Specify a label used to generate subdomain of Admin server T3 channel. The final subdomain name will be label.dnszoneName, e.g. admin-t3.contoso.xyz') +param dnszoneAdminT3ChannelLabel string ='admin-t3' +@description('Specify a label used to generate subdomain of WebLogic cluster. The final subdomain name will be label.dnszoneName, e.g. applications.contoso.xyz') +param dnszoneClusterLabel string = 'www' +param dnszoneClusterT3ChannelLabel string = 'cluster-t3' +@description('Azure DNS Zone name.') param dnszoneName string = 'contoso.xyz' param dnszoneRGName string = 'dns-contoso-rg' @description('JDBC Connection String') @@ -112,6 +116,8 @@ param enableCookieBasedAffinity bool = false param enableCustomSSL bool = false param enableDB bool = false param enableDNSConfiguration bool = false +@description('Configure a custom channel for the T3 protocol that enables HTTP tunneling') +param enableT3Tunneling bool = false @description('An user assigned managed identity. Make sure the identity has permission to create/update/delete/list Azure resources.') param identity object @description('JNDI Name for JDBC Datasource') @@ -202,6 +208,10 @@ param sslUploadedPrivateKeyAlias string = 'contoso' @secure() @description('Password of the private key') param sslUploadedPrivateKeyPassPhrase string = newGuid() +@description('Public port of the custom T3 channel in admin server') +param t3ChannelAdminPort int = 7005 +@description('Public port of the custom T3 channel in WebLoigc cluster') +param t3ChannelClusterPort int = 8011 @description('True to set up internal load balancer service.') param useInternalLB bool = false @description('ture to upload Java EE applications and deploy the applications to WebLogic domain.') @@ -338,6 +348,7 @@ module wlsDomainDeployment 'modules/setupWebLogicCluster.bicep' = if (!enableCus createStorageAccount: const_bCreateStorageAccount enableAzureMonitoring: enableAzureMonitoring enableCustomSSL: enableCustomSSL + enableT3Tunneling: enableT3Tunneling enablePV: const_enablePV identity: identity location: location @@ -345,6 +356,8 @@ module wlsDomainDeployment 'modules/setupWebLogicCluster.bicep' = if (!enableCus ocrSSOPSW: ocrSSOPSW ocrSSOUser: ocrSSOUser storageAccountName: name_storageAccountName + t3ChannelAdminPort: t3ChannelAdminPort + t3ChannelClusterPort: t3ChannelClusterPort wdtRuntimePassword: wdtRuntimePassword wlsClusterSize: wlsClusterSize wlsCPU: wlsCPU @@ -394,6 +407,7 @@ module wlsDomainWithCustomSSLDeployment 'modules/setupWebLogicCluster.bicep' = i createStorageAccount: const_bCreateStorageAccount enableAzureMonitoring: enableAzureMonitoring enableCustomSSL: enableCustomSSL + enableT3Tunneling: enableT3Tunneling enablePV: const_enablePV identity: identity location: location @@ -401,6 +415,8 @@ module wlsDomainWithCustomSSLDeployment 'modules/setupWebLogicCluster.bicep' = i ocrSSOPSW: ocrSSOPSW ocrSSOUser: ocrSSOUser storageAccountName: name_storageAccountName + t3ChannelAdminPort: t3ChannelAdminPort + t3ChannelClusterPort: t3ChannelClusterPort wdtRuntimePassword: wdtRuntimePassword wlsClusterSize: wlsClusterSize wlsCPU: wlsCPU @@ -478,7 +494,9 @@ module networkingDeployment 'modules/networking.bicep' = if (const_enableNetwork createDNSZone: createDNSZone dnsNameforApplicationGateway: name_domainLabelforApplicationGateway dnszoneAdminConsoleLabel: dnszoneAdminConsoleLabel - dnszoneAppGatewayLabel: dnszoneAppGatewayLabel + dnszoneAdminT3ChannelLabel: dnszoneAdminT3ChannelLabel + dnszoneClusterLabel: dnszoneClusterLabel + dnszoneClusterT3ChannelLabel: dnszoneClusterT3ChannelLabel dnszoneName: dnszoneName dnszoneRGName: dnszoneRGName enableAppGWIngress: enableAppGWIngress diff --git a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep index d4a08390a..2bbcf62bb 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep @@ -23,7 +23,9 @@ param appgwFrontendSSLCertPsw string = newGuid() param aksClusterRGName string = 'aks-contoso-rg' param aksClusterName string = 'aks-contoso' param dnszoneAdminConsoleLabel string = 'admin' -param dnszoneAppGatewayLabel string = 'www' +param dnszoneAdminT3ChannelLabel string ='admin-t3' +param dnszoneClusterLabel string = 'www' +param dnszoneClusterT3ChannelLabel string = 'cluster-t3' param dnszoneName string = 'contoso.xyz' param dnszoneRGName string = 'dns-contoso-rg' param enableAppGWIngress bool = false @@ -43,11 +45,12 @@ param wlsDomainUID string = 'sample-domain1' var const_appgwHelmConfigTemplate='appgw-helm-config.yaml.template' var const_appgwSARoleBindingFile='appgw-ingress-clusterAdmin-roleBinding.yaml' -var const_arguments = '${aksClusterRGName} ${aksClusterName} ${wlsDomainName} ${wlsDomainUID} "${string(lbSvcValues)}" ${enableAppGWIngress} ${subscription().id} ${resourceGroup().name} ${appgwName} ${vnetName} ${string(servicePrincipal)} ${appgwForAdminServer} ${enableDNSConfiguration} ${dnszoneRGName} ${dnszoneName} ${dnszoneAdminConsoleLabel} ${dnszoneAppGatewayLabel} ${appgwAlias} ${useInternalLB} ${appgwFrontendSSLCertData} ${appgwFrontendSSLCertPsw} ${appgwCertificateOption} ${enableCustomSSL} ${enableCookieBasedAffinity} ${appgwForRemoteConsole}' +var const_arguments = '${aksClusterRGName} ${aksClusterName} ${wlsDomainName} ${wlsDomainUID} "${string(lbSvcValues)}" ${enableAppGWIngress} ${subscription().id} ${resourceGroup().name} ${appgwName} ${vnetName} ${string(servicePrincipal)} ${appgwForAdminServer} ${enableDNSConfiguration} ${dnszoneRGName} ${dnszoneName} ${dnszoneAdminConsoleLabel} ${dnszoneClusterLabel} ${appgwAlias} ${useInternalLB} ${appgwFrontendSSLCertData} ${appgwFrontendSSLCertPsw} ${appgwCertificateOption} ${enableCustomSSL} ${enableCookieBasedAffinity} ${appgwForRemoteConsole} ${dnszoneAdminT3ChannelLabel} ${dnszoneClusterT3ChannelLabel}' var const_commonScript = 'common.sh' var const_scriptLocation = uri(_artifactsLocation, 'scripts/') var const_setupNetworkingScript= 'setupNetworking.sh' var const_primaryScript = 'invokeSetupNetworking.sh' +var const_utilityScript= 'utility.sh' resource deploymentScript 'Microsoft.Resources/deploymentScripts@2020-10-01' = { name: 'ds-networking-deployment' @@ -63,6 +66,7 @@ resource deploymentScript 'Microsoft.Resources/deploymentScripts@2020-10-01' = { uri(const_scriptLocation, '${const_appgwHelmConfigTemplate}${_artifactsLocationSasToken}') uri(const_scriptLocation, '${const_appgwSARoleBindingFile}${_artifactsLocationSasToken}') uri(const_scriptLocation, '${const_commonScript}${_artifactsLocationSasToken}') + uri(const_scriptLocation, '${const_utilityScript}${_artifactsLocationSasToken}') ] cleanupPreference: 'OnSuccess' retentionInterval: 'P1D' diff --git a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-wls-cluster.bicep b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-wls-cluster.bicep index a2616977c..fe019c69f 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-wls-cluster.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-wls-cluster.bicep @@ -11,6 +11,7 @@ param acrName string = '' param appPackageUrls array = [] param appReplicas int = 2 param enableCustomSSL bool = false +param enableT3Tunneling bool = false param enablePV bool = false param identity object param location string = 'eastus' @@ -19,6 +20,8 @@ param managedServerPrefix string = 'managed-server' param ocrSSOPSW string param ocrSSOUser string param storageAccountName string = 'null' +param t3ChannelAdminPort int = 7005 +param t3ChannelClusterPort int = 8011 param utcValue string = utcNow() @secure() param wdtRuntimePassword string @@ -51,7 +54,7 @@ param wlsTrustKeyStorePassPhrase string = newGuid() param wlsTrustKeyStoreType string = 'PKCS12' param wlsUserName string = 'weblogic' -var const_arguments = '${ocrSSOUser} ${ocrSSOPSW} ${aksClusterRGName} ${aksClusterName} ${wlsImageTag} ${acrName} ${wlsDomainName} ${wlsDomainUID} ${wlsUserName} ${wlsPassword} ${wdtRuntimePassword} ${wlsCPU} ${wlsMemory} ${managedServerPrefix} ${appReplicas} ${string(appPackageUrls)} ${resourceGroup().name} ${const_scriptLocation} ${storageAccountName} ${wlsClusterSize} ${enableCustomSSL} ${wlsIdentityKeyStoreData} ${wlsIdentityKeyStorePassphrase} ${wlsIdentityKeyStoreType} ${wlsPrivateKeyAlias} ${wlsPrivateKeyPassPhrase} ${wlsTrustKeyStoreData} ${wlsTrustKeyStorePassPhrase} ${wlsTrustKeyStoreType} ${enablePV} ' +var const_arguments = '${ocrSSOUser} ${ocrSSOPSW} ${aksClusterRGName} ${aksClusterName} ${wlsImageTag} ${acrName} ${wlsDomainName} ${wlsDomainUID} ${wlsUserName} ${wlsPassword} ${wdtRuntimePassword} ${wlsCPU} ${wlsMemory} ${managedServerPrefix} ${appReplicas} ${string(appPackageUrls)} ${resourceGroup().name} ${const_scriptLocation} ${storageAccountName} ${wlsClusterSize} ${enableCustomSSL} ${wlsIdentityKeyStoreData} ${wlsIdentityKeyStorePassphrase} ${wlsIdentityKeyStoreType} ${wlsPrivateKeyAlias} ${wlsPrivateKeyPassPhrase} ${wlsTrustKeyStoreData} ${wlsTrustKeyStorePassPhrase} ${wlsTrustKeyStoreType} ${enablePV} ${enableT3Tunneling} ${t3ChannelAdminPort} ${t3ChannelClusterPort} ' var const_buildDockerImageScript='createVMAndBuildImage.sh' var const_commonScript = 'common.sh' var const_invokeSetUpDomainScript = 'invokeSetupWLSDomain.sh' diff --git a/weblogic-azure-aks/src/main/bicep/modules/networking.bicep b/weblogic-azure-aks/src/main/bicep/modules/networking.bicep index 72447b7ea..ab7620dd2 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/networking.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/networking.bicep @@ -32,8 +32,10 @@ param dnsNameforApplicationGateway string = 'wlsgw' @description('Azure DNS Zone name.') param dnszoneName string = 'contoso.xyz' param dnszoneAdminConsoleLabel string = 'admin' -@description('Specify a label used to generate subdomain of Application Gateway. The final subdomain name will be label.dnszoneName, e.g. applications.contoso.xyz') -param dnszoneAppGatewayLabel string = 'www' +param dnszoneAdminT3ChannelLabel string ='admin-t3' +@description('Specify a label used to generate subdomain of WebLogic cluster. The final subdomain name will be label.dnszoneName, e.g. applications.contoso.xyz') +param dnszoneClusterLabel string = 'www' +param dnszoneClusterT3ChannelLabel string = 'cluster-t3' param dnszoneRGName string = 'dns-contoso-rg' @description('true to set up Application Gateway ingress.') param enableAppGWIngress bool = false @@ -62,7 +64,7 @@ param wlsDomainName string = 'domain1' @description('UID of WebLogic domain, used in WebLogic Operator.') param wlsDomainUID string = 'sample-domain1' -var const_appgwCustomDNSAlias = format('{0}.{1}/', dnszoneAppGatewayLabel, dnszoneName) +var const_appgwCustomDNSAlias = format('{0}.{1}/', dnszoneClusterLabel, dnszoneName) var const_appgwAdminCustomDNSAlias = format('{0}.{1}/', dnszoneAdminConsoleLabel, dnszoneName) var const_appgwSSLCertOptionGenerateCert = 'generateCert' @@ -142,7 +144,9 @@ module networkingDeployment '_deployment-scripts/_ds-create-networking.bicep' = aksClusterRGName: aksClusterRGName aksClusterName: aksClusterName dnszoneAdminConsoleLabel: dnszoneAdminConsoleLabel - dnszoneAppGatewayLabel: dnszoneAppGatewayLabel + dnszoneAdminT3ChannelLabel: dnszoneAdminT3ChannelLabel + dnszoneClusterLabel: dnszoneClusterLabel + dnszoneClusterT3ChannelLabel: dnszoneClusterT3ChannelLabel dnszoneName: dnszoneName dnszoneRGName: createDNSZone ? resourceGroup().name : dnszoneRGName enableAppGWIngress: enableAppGWIngress @@ -180,7 +184,9 @@ module networkingDeployment2 '_deployment-scripts/_ds-create-networking.bicep' = aksClusterRGName: aksClusterRGName aksClusterName: aksClusterName dnszoneAdminConsoleLabel: dnszoneAdminConsoleLabel - dnszoneAppGatewayLabel: dnszoneAppGatewayLabel + dnszoneAdminT3ChannelLabel: dnszoneAdminT3ChannelLabel + dnszoneClusterLabel: dnszoneClusterLabel + dnszoneClusterT3ChannelLabel: dnszoneClusterT3ChannelLabel dnszoneName: dnszoneName dnszoneRGName: createDNSZone ? resourceGroup().name : dnszoneRGName enableAppGWIngress: enableAppGWIngress @@ -217,7 +223,9 @@ module networkingDeployment3 '_deployment-scripts/_ds-create-networking.bicep' = aksClusterRGName: aksClusterRGName aksClusterName: aksClusterName dnszoneAdminConsoleLabel: dnszoneAdminConsoleLabel - dnszoneAppGatewayLabel: dnszoneAppGatewayLabel + dnszoneAdminT3ChannelLabel: dnszoneAdminT3ChannelLabel + dnszoneClusterLabel: dnszoneClusterLabel + dnszoneClusterT3ChannelLabel: dnszoneClusterT3ChannelLabel dnszoneName: dnszoneName dnszoneRGName: createDNSZone ? resourceGroup().name : dnszoneRGName enableAppGWIngress: enableAppGWIngress @@ -245,7 +253,7 @@ module pidAppgwEnd './_pids/_pid.bicep' = if (enableAppGWIngress) { name: _pidAppgwEnd } dependsOn: [ - networkingDeployment + appgwDeployment ] } @@ -255,7 +263,9 @@ module pidNetworkingEnd './_pids/_pid.bicep' = { name: _pidNetworkingEnd } dependsOn: [ - pidAppgwEnd + networkingDeployment + networkingDeployment2 + networkingDeployment3 ] } diff --git a/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep b/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep index b081b3f46..13308bae1 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep @@ -57,6 +57,7 @@ param createStorageAccount bool = false param enableAzureMonitoring bool = false @description('true to create persistent volume using file share.') param enableCustomSSL bool = false +param enableT3Tunneling bool = false param enablePV bool = false @description('An user assigned managed identity. Make sure the identity has permission to create/update/delete/list Azure resources.') param identity object @@ -69,6 +70,8 @@ param ocrSSOPSW string @description('User name of Oracle SSO account.') param ocrSSOUser string param storageAccountName string +param t3ChannelAdminPort int = 7005 +param t3ChannelClusterPort int = 8011 @secure() @description('Password for model WebLogic Deploy Tooling runtime encrytion.') param wdtRuntimePassword string @@ -181,13 +184,16 @@ module wlsDomainDeployment './_deployment-scripts/_ds-create-wls-cluster.bicep' appPackageUrls: appPackageUrls appReplicas: appReplicas enableCustomSSL: enableCustomSSL + enableT3Tunneling: enableT3Tunneling enablePV: enablePV identity: identity location: location managedServerPrefix: managedServerPrefix - storageAccountName: storageAccountName ocrSSOUser: ocrSSOUser ocrSSOPSW: ocrSSOPSW + storageAccountName: storageAccountName + t3ChannelAdminPort: t3ChannelAdminPort + t3ChannelClusterPort: t3ChannelClusterPort wdtRuntimePassword: wdtRuntimePassword wlsClusterSize: wlsClusterSize wlsCPU: wlsCPU From 2928df4d5b31f200b2e0cd69215b985cbf54d3c8 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Wed, 25 Aug 2021 14:15:50 +0800 Subject: [PATCH 02/14] On branch t3tunneling: support t3s Changes to be committed: modified: weblogic-azure-aks/src/main/arm/createUiDefinition.json modified: weblogic-azure-aks/src/main/arm/scripts/buildWLSDockerImage.sh modified: weblogic-azure-aks/src/main/arm/scripts/common.sh modified: weblogic-azure-aks/src/main/arm/scripts/createVMAndBuildImage.sh modified: weblogic-azure-aks/src/main/arm/scripts/genDomainConfig.sh modified: weblogic-azure-aks/src/main/arm/scripts/genImageModel.sh modified: weblogic-azure-aks/src/main/arm/scripts/invokeSetupWLSDomain.sh modified: weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh modified: weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh modified: weblogic-azure-aks/src/main/arm/scripts/updateDomainConfig.sh modified: weblogic-azure-aks/src/main/bicep/mainTemplate.bicep modified: weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_keyvaultForGateway.bicep modified: weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-wls-cluster.bicep modified: weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep Signed-off-by: galiacheng --- .../src/main/arm/createUiDefinition.json | 21 +++- .../main/arm/scripts/buildWLSDockerImage.sh | 17 ++- .../src/main/arm/scripts/common.sh | 2 + .../main/arm/scripts/createVMAndBuildImage.sh | 5 +- .../src/main/arm/scripts/genDomainConfig.sh | 18 ++- .../src/main/arm/scripts/genImageModel.sh | 30 +++-- .../main/arm/scripts/invokeSetupWLSDomain.sh | 18 ++- .../src/main/arm/scripts/setupNetworking.sh | 117 +++++++++++++++++- .../src/main/arm/scripts/setupWLSDomain.sh | 39 ++++-- .../main/arm/scripts/updateDomainConfig.sh | 18 ++- .../src/main/bicep/mainTemplate.bicep | 12 +- .../_keyvaultForGateway.bicep | 4 + .../_ds-create-wls-cluster.bicep | 6 +- .../bicep/modules/setupWebLogicCluster.bicep | 8 +- 14 files changed, 254 insertions(+), 61 deletions(-) diff --git a/weblogic-azure-aks/src/main/arm/createUiDefinition.json b/weblogic-azure-aks/src/main/arm/createUiDefinition.json index 41849d427..6a89a1594 100644 --- a/weblogic-azure-aks/src/main/arm/createUiDefinition.json +++ b/weblogic-azure-aks/src/main/arm/createUiDefinition.json @@ -214,9 +214,15 @@ "visible": "[bool(basics('basicsOptional').basicsOptionalAcceptDefaults)]" }, { - "name": "enableT3Tunneling", + "name": "enableAdminT3Tunneling", "type": "Microsoft.Common.CheckBox", - "label": "Enable T3 tunneling", + "label": "Enable T3 tunneling for Admin Server", + "visible": "[bool(basics('basicsOptional').basicsOptionalAcceptDefaults)]" + }, + { + "name": "enableClusterT3Tunneling", + "type": "Microsoft.Common.CheckBox", + "label": "Enable T3 tunneling for WebLogic cluster", "visible": "[bool(basics('basicsOptional').basicsOptionalAcceptDefaults)]" } ], @@ -1014,9 +1020,17 @@ "label": "admin-server", "value": "adminServer" }, + { + "label": "admin-server-t3", + "value": "adminServerT3" + }, { "label": "cluster-1", "value": "cluster1" + }, + { + "label": "cluster-1-t3", + "value": "cluster1T3" } ], "required": true @@ -1637,7 +1651,8 @@ "enableCustomSSL": "[bool(steps('section_sslConfiguration').enableCustomSSL)]", "enableDB": "[bool(steps('section_database').enableDB)]", "enableDNSConfiguration": "[bool(steps('section_appGateway').dnsConfiguration.enableDNSConfiguration)]", - "enableT3Tunneling": "[basics('basicsOptional').enableT3Tunneling]", + "enableAdminT3Tunneling": "[basics('basicsOptional').enableAdminT3Tunneling]", + "enableClusterT3Tunneling": "[basics('basicsOptional').enableClusterT3Tunneling]", "identity": "[basics('basicsRequired').identity]", "jdbcDataSourceName": "[steps('section_database').databaseConnectionInfo.jdbcDataSourceName]", "lbSvcValues": "[steps('section_appGateway').lbSVCInfo.lbSVC]", diff --git a/weblogic-azure-aks/src/main/arm/scripts/buildWLSDockerImage.sh b/weblogic-azure-aks/src/main/arm/scripts/buildWLSDockerImage.sh index e6dcd4068..0694ebe0d 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/buildWLSDockerImage.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/buildWLSDockerImage.sh @@ -15,7 +15,7 @@ function read_sensitive_parameters_from_stdin() { #Function to display usage message function usage() { - echo " ./buildWLSDockerImage.sh ./buildWLSDockerImage.sh " + echo " ./buildWLSDockerImage.sh ./buildWLSDockerImage.sh " if [ $1 -eq 1 ]; then exit 1 fi @@ -81,8 +81,13 @@ function validate_inputs() { usage 1 fi - if [ -z "$enableT3Tunneling" ]; then - echo_stderr "enableT3Tunneling is required. " + if [ -z "$enableAdminT3Tunneling" ]; then + echo_stderr "enableAdminT3Tunneling is required. " + usage 1 + fi + + if [ -z "$enableClusterT3Tunneling" ]; then + echo_stderr "enableClusterT3Tunneling is required. " usage 1 fi } @@ -185,7 +190,8 @@ EOF ${modelFilePath} \ ${appPackageUrls} \ ${enableSSL} \ - ${enableT3Tunneling} + ${enableAdminT3Tunneling} \ + ${enableClusterT3Tunneling} validate_status "Generate image model file." } @@ -242,7 +248,8 @@ export appPackageUrls=$5 export ocrSSOUser=$6 export wlsClusterSize=$7 export enableSSL=$8 -export enableT3Tunneling=$9 +export enableAdminT3Tunneling=$9 +export enableClusterT3Tunneling=${10} export acrImagePath="$azureACRServer/aks-wls-images:${imageTag}" export ocrLoginServer="container-registry.oracle.com" diff --git a/weblogic-azure-aks/src/main/arm/scripts/common.sh b/weblogic-azure-aks/src/main/arm/scripts/common.sh index f985e809f..406aa4c76 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/common.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/common.sh @@ -3,6 +3,8 @@ export checkPodStatusMaxAttemps=30 # max attempt to check pod status. export checkPVStateInterval=5 # interval of checking pvc status. export checkPVStateMaxAttempt=10 # max attempt to check pvc status. +export constAdminT3AddressEnvName="T3_TUNNELING_ADMIN_ADDRESS" +export constClusterT3AddressEnvName="T3_TUNNELING_CLUSTER_ADDRESS" export constFalse="false" export constTrue="true" diff --git a/weblogic-azure-aks/src/main/arm/scripts/createVMAndBuildImage.sh b/weblogic-azure-aks/src/main/arm/scripts/createVMAndBuildImage.sh index 15875e3b9..51aa393db 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/createVMAndBuildImage.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/createVMAndBuildImage.sh @@ -94,7 +94,7 @@ function build_docker_image() { --publisher Microsoft.Azure.Extensions \ --version 2.0 \ --settings "{ \"fileUris\": [\"${scriptURL}model.properties\",\"${scriptURL}genImageModel.sh\",\"${scriptURL}buildWLSDockerImage.sh\",\"${scriptURL}common.sh\"]}" \ - --protected-settings "{\"commandToExecute\":\"echo ${azureACRPassword} ${ocrSSOPSW} | bash buildWLSDockerImage.sh ${wlsImagePath} ${azureACRServer} ${azureACRUserName} ${newImageTag} \\\"${appPackageUrls}\\\" ${ocrSSOUser} ${wlsClusterSize} ${enableCustomSSL} ${enableT3Tunneling} \"}" + --protected-settings "{\"commandToExecute\":\"echo ${azureACRPassword} ${ocrSSOPSW} | bash buildWLSDockerImage.sh ${wlsImagePath} ${azureACRServer} ${azureACRUserName} ${newImageTag} \\\"${appPackageUrls}\\\" ${ocrSSOUser} ${wlsClusterSize} ${enableCustomSSL} ${enableAdminT3Tunneling} ${enableClusterT3Tunneling} \"}" cleanup_vm } @@ -118,7 +118,8 @@ export ocrSSOUser=$7 export wlsClusterSize=$8 export enableCustomSSL=$9 export scriptURL=${10} -export enableT3Tunneling=${11} +export enableAdminT3Tunneling=${11} +export enableClusterT3Tunneling=${12} read_sensitive_parameters_from_stdin diff --git a/weblogic-azure-aks/src/main/arm/scripts/genDomainConfig.sh b/weblogic-azure-aks/src/main/arm/scripts/genDomainConfig.sh index 8483f1669..d4072b1d4 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/genDomainConfig.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/genDomainConfig.sh @@ -14,11 +14,12 @@ export wlsMemory=$7 export wlsManagedPrefix=$8 export enableSSL=${9} export enablePV=${10} -export enableT3Tunneling=${11} -export t3AdminPort=${12} -export t3ClusterPort=${13} -export clusterName=${14} -export javaOptions=${15} +export enableAdminT3Tunneling=${11} +export enableClusterT3Tunneling=${12} +export t3AdminPort=${13} +export t3ClusterPort=${14} +export clusterName=${15} +export javaOptions=${16} export adminServiceUrl="${wlsDomainUID}-admin-server.${wlsDomainUID}-ns.svc.cluster.local" export clusterServiceUrl="${wlsDomainUID}-cluster-${clusterName}.${wlsDomainUID}-ns.svc.cluster.local" @@ -139,12 +140,17 @@ if [[ "${enableSSL,,}" == "true" ]]; then EOF fi -if [[ "${enableT3Tunneling,,}" == "true" ]]; then +if [[ "${enableAdminT3Tunneling,,}" == "true" ]]; then cat <>$filePath - name: T3_TUNNELING_ADMIN_PORT value: "${t3AdminPort}" - name: T3_TUNNELING_ADMIN_ADDRESS value: "${adminServiceUrl}" +EOF +fi + +if [[ "${enableClusterT3Tunneling,,}" == "true" ]]; then + cat <>$filePath - name: T3_TUNNELING_CLUSTER_PORT value: "${t3ClusterPort}" - name: T3_TUNNELING_CLUSTER_ADDRESS diff --git a/weblogic-azure-aks/src/main/arm/scripts/genImageModel.sh b/weblogic-azure-aks/src/main/arm/scripts/genImageModel.sh index a1afbb127..8b5c3d818 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/genImageModel.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/genImageModel.sh @@ -10,9 +10,17 @@ source ${scriptDir}/common.sh export filePath=$1 export appPackageUrls=$2 export enableCustomSSL=$3 -export enableT3Tunneling=$4 +export enableAdminT3Tunneling=$4 +export enableClusterT3Tunneling=$5 export enableT3s=${enableCustomSSL,,} +export t3Protocol="t3" +export t3ChannelName="T3Channel" + +if [ "${enableCustomSSL,,}" == "true" ]; then + t3Protocol="t3s" + t3ChannelName="T3sChannel" +fi cat <${filePath} # Copyright (c) 2020, 2021, Oracle and/or its affiliates. @@ -45,20 +53,20 @@ topology: ListenPort: 7001 EOF -if [[ "${enableT3Tunneling,,}" == "true" ]];then +if [[ "${enableAdminT3Tunneling,,}" == "true" ]];then cat <>${filePath} NetworkAccessPoint: - MyT3Channel: - Protocol: 't3' + ${t3ChannelName}: + Protocol: '${t3Protocol}' ListenPort: "@@ENV:T3_TUNNELING_ADMIN_PORT@@" PublicPort: "@@ENV:T3_TUNNELING_ADMIN_PORT@@" HttpEnabledForThisProtocol: true OutboundEnabled: false Enabled: true - ClientCertificateEnforced: ${enableCustomSSL} + TwoWaySSLEnabled: ${enableT3s} + ClientCertificateEnforced: false TunnelingEnabled: true PublicAddress: '@@ENV:T3_TUNNELING_ADMIN_ADDRESS@@' - TwoWaySslEnabled: ${enableT3s} EOF fi @@ -88,20 +96,20 @@ cat <>${filePath} ListenPort: 8001 EOF -if [[ "${enableT3Tunneling,,}" == "true" ]];then +if [[ "${enableClusterT3Tunneling,,}" == "true" ]];then cat <>${filePath} NetworkAccessPoint: - MyT3Channel: - Protocol: 't3' + ${t3ChannelName}: + Protocol: '${t3Protocol}' ListenPort: "@@ENV:T3_TUNNELING_CLUSTER_PORT@@" PublicPort: "@@ENV:T3_TUNNELING_CLUSTER_PORT@@" HttpEnabledForThisProtocol: true OutboundEnabled: false Enabled: true - ClientCertificateEnforced: ${enableCustomSSL} + TwoWaySSLEnabled: ${enableT3s} + ClientCertificateEnforced: false TunnelingEnabled: true PublicAddress: '@@ENV:T3_TUNNELING_CLUSTER_ADDRESS@@' - TwoWaySslEnabled: ${enableT3s} EOF fi diff --git a/weblogic-azure-aks/src/main/arm/scripts/invokeSetupWLSDomain.sh b/weblogic-azure-aks/src/main/arm/scripts/invokeSetupWLSDomain.sh index 2b01a1eba..22d559a25 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/invokeSetupWLSDomain.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/invokeSetupWLSDomain.sh @@ -37,9 +37,11 @@ Usage: - + + + END ) echo_stdout "${usage}" @@ -85,9 +87,11 @@ export wlsTrustData=${27} wlsTrustPsw=${28} export wlsTrustType=${29} export enablePV=${30} -export enableT3Tunneling=${31} -export t3AdminPort=${32} -export t3ClusterPort=${33} +export enableAdminT3Tunneling=${31} +export enableClusterT3Tunneling=${32} +export t3AdminPort=${33} +export t3ClusterPort=${34} +export wlsJavaOption=${35} echo ${ocrSSOPSW} \ ${wlsPassword} \ @@ -120,9 +124,11 @@ echo ${ocrSSOPSW} \ ${wlsTrustData} \ ${wlsTrustType} \ ${enablePV} \ - ${enableT3Tunneling} \ + ${enableAdminT3Tunneling} \ + ${enableClusterT3Tunneling} \ ${t3AdminPort} \ - ${t3ClusterPort} + ${t3ClusterPort} \ + ${wlsJavaOption} if [ $? -ne 0 ]; then usage 1 diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh index 2108199c5..05206a213 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh @@ -791,6 +791,97 @@ function create_dns_CNAME_record() { fi } +function patch_admin_t3_public_address() { + # patch admin t3 public address + if [ "${enableCustomDNSAlias,,}" == "true" ]; then + adminT3Address="${dnszoneAdminT3ChannelLabel}.${dnsZoneName}" + else + adminT3Address=$(kubectl -n ${wlsDomainNS} get svc ${adminServerT3LBSVCName} -o json \ + | jq '. | .status.loadBalancer.ingress[0].ip' \ + | tr -d "\"") + fi + + if [ $? == 1 ]; then + echo_stderr "Failed to query public IP of admin t3 channel." + fi + + currentDomainConfig=$(echo ${currentDomainConfig} \ + | jq \ + --arg match "${constAdminT3AddressEnvName}" \ + --arg replace "${adminT3Address}" \ + '.spec.serverPod.env |= map(if .name==$match then (.value=$replace) else . end)') +} + +function patch_cluster_t3_public_address() { + #patch cluster t3 pubilc address + if [ "${enableCustomDNSAlias,,}" == "true" ]; then + clusterT3Adress="${dnszoneClusterT3ChannelLabel}.${dnsZoneName}" + else + clusterT3Adress=$(kubectl -n ${wlsDomainNS} get svc ${clusterT3LBSVCName} -o json \ + | jq '. | .status.loadBalancer.ingress[0].ip' \ + | tr -d "\"") + fi + + if [ $? == 1 ]; then + echo_stderr "Failed to query public IP of cluster t3 channel." + fi + + currentDomainConfig=$(echo ${currentDomainConfig} \ + | jq \ + --arg match "${constClusterT3AddressEnvName}" \ + --arg replace "${clusterT3Adress}" \ + '.spec.serverPod.env |= map(if .name==$match then (.value=$replace) else . end)') +} + +function rolling_update_with_t3_public_address() { + timestampBeforePatchingDomain=$(date +%s) + currentDomainConfig=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json) + + # update public address of t3 channel + if [[ "${enableAdminT3Channel,,}" == "true" ]]; then + patch_admin_t3_public_address + fi + + if [[ "${enableClusterT3Channel,,}" == "true" ]]; then + patch_cluster_t3_public_address + fi + + if [[ "${enableClusterT3Channel,,}" == "true" ]] || [[ "${enableAdminT3Channel,,}" == "true" ]]; then + # restart cluster + restartVersion=$( kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json \ + | jq '. | .spec.restartVersion' \ + | tr -d "\"") + restartVersion=$((restartVersion+1)) + + currentDomainConfig=$(echo ${currentDomainConfig} \ + | jq \ + --arg version "${restartVersion}" \ + '.spec.restartVersion |= $version') + + echo "rolling restart the cluster with t3 public address." + # echo the configuration for debugging + echo -e ${currentDomainConfig} >${scriptDir}/domainNewConfiguration.yaml + echo ${currentDomainConfig} | kubectl -n ${wlsDomainNS} apply -f - + + replicas=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json \ + | jq '. | .spec.clusters[] | .replicas') + + # wait for the restart completed. + utility_wait_for_pod_restarted \ + ${timestampBeforePatchingDomain} \ + ${replicas} \ + ${wlsDomainUID} \ + ${checkPodStatusMaxAttemps} \ + ${checkPodStatusInterval} + + utility_wait_for_pod_completed \ + ${replicas} \ + ${wlsDomainNS} \ + ${checkPodStatusMaxAttemps} \ + ${checkPodStatusInterval} + fi +} + function create_svc_lb() { # No lb svc inputs if [[ "${lbSvcValues}" == "[]" ]]; then @@ -856,15 +947,21 @@ EOF echo "query admin t3 port" adminT3Port=$( kubectl get service ${svcAdminServer} -n ${wlsDomainNS} -o json \ | jq '.spec.ports[] | select(.name=="t3channel") | .port') - if [[ "${adminT3Port}" == "null" ]]; then + adminT3sPort=$( kubectl get service ${svcAdminServer} -n ${wlsDomainNS} -o json \ + | jq '.spec.ports[] | select(.name=="t3schannel") | .port') + if [[ "${adminT3Port}" == "null" ]] && [[ "${adminT3sPort}" == "null" ]]; then continue fi + if [[ "${adminT3sPort}" != "null" ]]; then + adminT3Port=${adminT3sPort} + fi + adminServerT3LBSVCNamePrefix=$(cut -d',' -f1 <<<$item) adminServerT3LBSVCName="${adminServerT3LBSVCNamePrefix}-svc-t3-lb-admin" adminT3LBPort=$(cut -d',' -f3 <<<$item) - export adminServerT3LBDefinitionPath=${scriptDir}/admin-server-t3-lb.yaml + adminServerT3LBDefinitionPath=${scriptDir}/admin-server-t3-lb.yaml generate_admin_t3_lb_definicion kubectl apply -f ${adminServerT3LBDefinitionPath} @@ -878,19 +975,27 @@ EOF if [ "${enableCustomDNSAlias,,}" == "true" ]; then adminServerT3Endpoint="${dnszoneAdminT3ChannelLabel}.${dnsZoneName}:${adminServerT3Endpoint#*:}" fi + + enableAdminT3Channel=true elif [[ "${target}" == "cluster1T3" ]]; then echo "query cluster t3 port" clusterT3Port=$( kubectl get service ${svcCluster} -n ${wlsDomainNS} -o json \ | jq '.spec.ports[] | select(.name=="t3channel") | .port') - if [[ "${clusterT3Port}" == "null" ]]; then + clusterT3sPort=$( kubectl get service ${svcCluster} -n ${wlsDomainNS} -o json \ + | jq '.spec.ports[] | select(.name=="t3schannel") | .port') + if [[ "${clusterT3Port}" == "null" ]] && [[ "${clusterT3sPort}" == "null" ]]; then continue fi + if [[ "${clusterT3sPort}" != "null" ]]; then + clusterT3Port=${clusterT3sPort} + fi + clusterT3LBSVCNamePrefix=$(cut -d',' -f1 <<<$item) clusterT3LBSVCName="${clusterT3LBSVCNamePrefix}-svc-lb-cluster" clusterT3LBPort=$(cut -d',' -f3 <<<$item) - export clusterT3LBDefinitionPath=${scriptDir}/cluster-t3-lb.yaml + clusterT3LBDefinitionPath=${scriptDir}/cluster-t3-lb.yaml generate_cluster_t3_lb_definicion kubectl apply -f ${clusterT3LBDefinitionPath} @@ -904,8 +1009,12 @@ EOF if [ "${enableCustomDNSAlias,,}" == "true" ]; then clusterT3Endpoint="${dnszoneClusterT3ChannelLabel}.${dnsZoneName}:${clusterT3Endpoint#*:}" fi + + enableClusterT3Channel=true fi done + + rolling_update_with_t3_public_address } # Create network peers for aks and appgw diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh index af960b6f4..7c7ec95b0 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh @@ -39,7 +39,8 @@ echo - + + END @@ -178,8 +179,13 @@ function validate_input() { usage 1 fi - if [ -z "$enableT3Tunneling" ]; then - echo_stderr "enableT3Tunneling is required. " + if [ -z "$enableAdminT3Tunneling" ]; then + echo_stderr "enableAdminT3Tunneling is required. " + usage 1 + fi + + if [ -z "$enableClusterT3Tunneling" ]; then + echo_stderr "enableClusterT3Tunneling is required. " usage 1 fi @@ -352,7 +358,8 @@ function build_docker_image() { $wlsClusterSize \ $enableCustomSSL \ "$scriptURL" \ - ${enableT3Tunneling} + ${enableAdminT3Tunneling} \ + ${enableClusterT3Tunneling} az acr repository show -n ${acrName} --image aks-wls-images:${newImageTag} if [ $? -ne 0 ]; then @@ -595,7 +602,6 @@ function create_domain_namespace() { } function parsing_ssl_certs_and_create_ssl_secret() { - export javaOptions="" if [[ "${enableCustomSSL,,}" == "${constTrue}" ]]; then # use default Java, if no, install open jdk 11. # why not Microsoft open jdk? No apk installation package! @@ -628,7 +634,7 @@ function parsing_ssl_certs_and_create_ssl_secret() { --from-literal=ssltruststorepassword=${wlsTrustPsw} kubectl -n ${wlsDomainNS} label secret ${kubectlWLSSSLCredentialsName} weblogic.domainUID=${wlsDomainUID} - javaOptions="-Dweblogic.security.SSL.ignoreHostnameVerification=true -Dweblogic.security.SSL.trustedCAKeyStore=${sharedPath}/${wlsTrustKeyStoreJKSFileName}" + javaOptions=" -Dweblogic.security.SSL.ignoreHostnameVerification=true -Dweblogic.security.SSL.trustedCAKeyStore=${sharedPath}/${wlsTrustKeyStoreJKSFileName} ${javaOptions}" fi } @@ -661,6 +667,13 @@ function setup_wls_domain() { echo "print pvc info" kubectl -n ${wlsDomainNS} get pvc -o wide + export javaOptions=${wlsJavaOption} + if [[ "${enableClusterT3Channel,,}" == "true" ]] || [[ "${enableAdminT3Channel,,}" == "true" ]]; then + # for remote t3/t3s access. + # refer to https://oracle.github.io/weblogic-kubernetes-operator/faq/external-clients/#enabling-unknown-host-access + javaOptions="-Dweblogic.rjvm.allowUnknownHost=true ${javaOptions}" + fi + customDomainYaml=${scriptDir}/custom-domain.yaml if [[ "${updateNamepace}" == "${constTrue}" ]]; then echo "start to update domain ${wlsDomainUID}" @@ -676,7 +689,8 @@ function setup_wls_domain() { ${managedServerPrefix} \ ${enableCustomSSL} \ ${enablePV} \ - ${enableT3Tunneling} \ + ${enableAdminT3Tunneling} \ + ${enableClusterT3Tunneling} \ ${t3AdminPort} \ ${t3ClusterPort} \ ${wlsClusterName} \ @@ -696,7 +710,8 @@ function setup_wls_domain() { ${managedServerPrefix} \ ${enableCustomSSL} \ ${enablePV} \ - ${enableT3Tunneling} \ + ${enableAdminT3Tunneling} \ + ${enableClusterT3Tunneling} \ ${t3AdminPort} \ ${t3ClusterPort} \ ${wlsClusterName} \ @@ -741,9 +756,11 @@ export wlsIdentityAlias=${21} export wlsTrustData=${22} export wlsTrustType=${23} export enablePV=${24} -export enableT3Tunneling=${25} -export t3AdminPort=${26} -export t3ClusterPort=${27} +export enableAdminT3Tunneling=${25} +export enableClusterT3Tunneling=${26} +export t3AdminPort=${27} +export t3ClusterPort=${28} +export wlsJavaOption=${29} export adminServerName="admin-server" export azFileShareName="weblogic" diff --git a/weblogic-azure-aks/src/main/arm/scripts/updateDomainConfig.sh b/weblogic-azure-aks/src/main/arm/scripts/updateDomainConfig.sh index 94128d0b8..86db0c86c 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/updateDomainConfig.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/updateDomainConfig.sh @@ -14,11 +14,12 @@ export wlsMemory=$7 export wlsManagedPrefix=$8 export enableSSL=${9} export enablePV=${10} -export enableT3Tunneling=${11} -export t3AdminPort=${12} -export t3ClusterPort=${13} -export clusterName=${14} -export javaOptions=${15} +export enableAdminT3Tunneling=${11} +export enableClusterT3Tunneling=${12} +export t3AdminPort=${13} +export t3ClusterPort=${14} +export clusterName=${15} +export javaOptions=${16} export adminServiceUrl="${wlsDomainUID}-admin-server.${wlsDomainUID}-ns.svc.cluster.local" export clusterServiceUrl="${wlsDomainUID}-cluster-${clusterName}.${wlsDomainUID}-ns.svc.cluster.local" @@ -156,12 +157,17 @@ if [[ "${enableSSL,,}" == "true" ]]; then EOF fi -if [[ "${enableT3Tunneling,,}" == "true" ]]; then +if [[ "${enableAdminT3Tunneling,,}" == "true" ]]; then cat <>$filePath - name: T3_TUNNELING_ADMIN_PORT value: "${t3AdminPort}" - name: T3_TUNNELING_ADMIN_ADDRESS value: "${adminServiceUrl}" +EOF +fi + +if [[ "${enableClusterT3Tunneling,,}" == "true" ]]; then + cat <>$filePath - name: T3_TUNNELING_CLUSTER_PORT value: "${t3ClusterPort}" - name: T3_TUNNELING_CLUSTER_ADDRESS diff --git a/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep b/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep index 247d3e0f2..7b341fda9 100644 --- a/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep +++ b/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep @@ -116,8 +116,10 @@ param enableCookieBasedAffinity bool = false param enableCustomSSL bool = false param enableDB bool = false param enableDNSConfiguration bool = false -@description('Configure a custom channel for the T3 protocol that enables HTTP tunneling') -param enableT3Tunneling bool = false +@description('Configure a custom channel in Admin Server for the T3 protocol that enables HTTP tunneling') +param enableAdminT3Tunneling bool = false +@description('Configure a custom channel in WebLogic cluster for the T3 protocol that enables HTTP tunneling') +param enableClusterT3Tunneling bool = false @description('An user assigned managed identity. Make sure the identity has permission to create/update/delete/list Azure resources.') param identity object @description('JNDI Name for JDBC Datasource') @@ -229,6 +231,7 @@ param wlsDomainName string = 'domain1' param wlsDomainUID string = 'sample-domain1' @description('Docker tag that comes after "container-registry.oracle.com/middleware/weblogic:"') param wlsImageTag string = '12.2.1.4' +param wlsJavaOption string = 'null' @description('Memory requests for admin server and managed server.') param wlsMemory string = '1.5Gi' @secure() @@ -348,7 +351,8 @@ module wlsDomainDeployment 'modules/setupWebLogicCluster.bicep' = if (!enableCus createStorageAccount: const_bCreateStorageAccount enableAzureMonitoring: enableAzureMonitoring enableCustomSSL: enableCustomSSL - enableT3Tunneling: enableT3Tunneling + enableAdminT3Tunneling: enableAdminT3Tunneling + enableClusterT3Tunneling: enableClusterT3Tunneling enablePV: const_enablePV identity: identity location: location @@ -367,6 +371,7 @@ module wlsDomainDeployment 'modules/setupWebLogicCluster.bicep' = if (!enableCus wlsIdentityKeyStorePassphrase: sslUploadedCustomIdentityKeyStorePassphrase wlsIdentityKeyStoreType: const_defaultKeystoreType wlsImageTag: wlsImageTag + wlsJavaOption: wlsJavaOption wlsMemory: wlsMemory wlsPassword: wlsPassword wlsPrivateKeyAlias: sslUploadedPrivateKeyAlias @@ -426,6 +431,7 @@ module wlsDomainWithCustomSSLDeployment 'modules/setupWebLogicCluster.bicep' = i wlsIdentityKeyStorePassphrase: sslKeyvault.getSecret(name_identityKeyStorePswSecret) wlsIdentityKeyStoreType: const_identityKeyStoreType wlsImageTag: wlsImageTag + wlsJavaOption: wlsJavaOption wlsMemory: wlsMemory wlsPassword: wlsPassword wlsPrivateKeyAlias: sslKeyvault.getSecret(name_privateKeyAliasSecret) diff --git a/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_keyvaultForGateway.bicep b/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_keyvaultForGateway.bicep index 70fd23fb7..5e682f89a 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_keyvaultForGateway.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_keyvaultForGateway.bicep @@ -76,6 +76,10 @@ module keyvaultBackendRootCert '_keyvault/_keyvaultForGatewayBackendCert.bicep' keyVaultName: keyVaultName sku: sku } + dependsOn:[ + keyVaultwithSelfSignedAppGatewaySSLCert + keyVaultwithExistingAppGatewaySSLCert + ] } output keyVaultName string = (useExistingAppGatewaySSLCertificate ? keyVaultwithExistingAppGatewaySSLCert.outputs.keyVaultName : keyVaultwithSelfSignedAppGatewaySSLCert.outputs.keyVaultName) diff --git a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-wls-cluster.bicep b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-wls-cluster.bicep index fe019c69f..6ae022f00 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-wls-cluster.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-wls-cluster.bicep @@ -11,7 +11,8 @@ param acrName string = '' param appPackageUrls array = [] param appReplicas int = 2 param enableCustomSSL bool = false -param enableT3Tunneling bool = false +param enableAdminT3Tunneling bool = false +param enableClusterT3Tunneling bool = false param enablePV bool = false param identity object param location string = 'eastus' @@ -38,6 +39,7 @@ param wlsIdentityKeyStorePassphrase string = newGuid() ]) param wlsIdentityKeyStoreType string = 'PKCS12' param wlsImageTag string = '12.2.1.4' +param wlsJavaOption string = 'null' param wlsMemory string = '1.5Gi' @secure() param wlsPassword string @@ -54,7 +56,7 @@ param wlsTrustKeyStorePassPhrase string = newGuid() param wlsTrustKeyStoreType string = 'PKCS12' param wlsUserName string = 'weblogic' -var const_arguments = '${ocrSSOUser} ${ocrSSOPSW} ${aksClusterRGName} ${aksClusterName} ${wlsImageTag} ${acrName} ${wlsDomainName} ${wlsDomainUID} ${wlsUserName} ${wlsPassword} ${wdtRuntimePassword} ${wlsCPU} ${wlsMemory} ${managedServerPrefix} ${appReplicas} ${string(appPackageUrls)} ${resourceGroup().name} ${const_scriptLocation} ${storageAccountName} ${wlsClusterSize} ${enableCustomSSL} ${wlsIdentityKeyStoreData} ${wlsIdentityKeyStorePassphrase} ${wlsIdentityKeyStoreType} ${wlsPrivateKeyAlias} ${wlsPrivateKeyPassPhrase} ${wlsTrustKeyStoreData} ${wlsTrustKeyStorePassPhrase} ${wlsTrustKeyStoreType} ${enablePV} ${enableT3Tunneling} ${t3ChannelAdminPort} ${t3ChannelClusterPort} ' +var const_arguments = '${ocrSSOUser} ${ocrSSOPSW} ${aksClusterRGName} ${aksClusterName} ${wlsImageTag} ${acrName} ${wlsDomainName} ${wlsDomainUID} ${wlsUserName} ${wlsPassword} ${wdtRuntimePassword} ${wlsCPU} ${wlsMemory} ${managedServerPrefix} ${appReplicas} ${string(appPackageUrls)} ${resourceGroup().name} ${const_scriptLocation} ${storageAccountName} ${wlsClusterSize} ${enableCustomSSL} ${wlsIdentityKeyStoreData} ${wlsIdentityKeyStorePassphrase} ${wlsIdentityKeyStoreType} ${wlsPrivateKeyAlias} ${wlsPrivateKeyPassPhrase} ${wlsTrustKeyStoreData} ${wlsTrustKeyStorePassPhrase} ${wlsTrustKeyStoreType} ${enablePV} ${enableAdminT3Tunneling} ${enableClusterT3Tunneling} ${t3ChannelAdminPort} ${t3ChannelClusterPort} "${wlsJavaOption}"' var const_buildDockerImageScript='createVMAndBuildImage.sh' var const_commonScript = 'common.sh' var const_invokeSetUpDomainScript = 'invokeSetupWLSDomain.sh' diff --git a/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep b/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep index 13308bae1..87adf1097 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep @@ -57,7 +57,8 @@ param createStorageAccount bool = false param enableAzureMonitoring bool = false @description('true to create persistent volume using file share.') param enableCustomSSL bool = false -param enableT3Tunneling bool = false +param enableAdminT3Tunneling bool = false +param enableClusterT3Tunneling bool = false param enablePV bool = false @description('An user assigned managed identity. Make sure the identity has permission to create/update/delete/list Azure resources.') param identity object @@ -94,6 +95,7 @@ param wlsIdentityKeyStorePassphrase string = newGuid() param wlsIdentityKeyStoreType string = 'PKCS12' @description('Docker tag that comes after "container-registry.oracle.com/middleware/weblogic:"') param wlsImageTag string = '12.2.1.4' +param wlsJavaOption string = 'null' @description('Memory requests for admin server and managed server.') param wlsMemory string = '1.5Gi' @secure() @@ -184,7 +186,8 @@ module wlsDomainDeployment './_deployment-scripts/_ds-create-wls-cluster.bicep' appPackageUrls: appPackageUrls appReplicas: appReplicas enableCustomSSL: enableCustomSSL - enableT3Tunneling: enableT3Tunneling + enableAdminT3Tunneling: enableAdminT3Tunneling + enableClusterT3Tunneling: enableClusterT3Tunneling enablePV: enablePV identity: identity location: location @@ -203,6 +206,7 @@ module wlsDomainDeployment './_deployment-scripts/_ds-create-wls-cluster.bicep' wlsIdentityKeyStorePassphrase: wlsIdentityKeyStorePassphrase wlsIdentityKeyStoreType: wlsIdentityKeyStoreType wlsImageTag: wlsImageTag + wlsJavaOption: wlsJavaOption wlsMemory: wlsMemory wlsPassword: wlsPassword wlsPrivateKeyAlias: wlsPrivateKeyAlias From 0d755c65cdf3cff0c08bdc05771dc8b17bad3583 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Wed, 1 Sep 2021 14:21:15 +0800 Subject: [PATCH 03/14] On branch t3tunneling: enhance UI Checkbox for "admin server T3 tunneling" Checkbox for "cluster T3 tunneling" InputText for "Java options" Changes to be committed: modified: weblogic-azure-aks/src/main/arm/createUiDefinition.json modified: weblogic-azure-aks/src/main/bicep/mainTemplate.bicep Signed-off-by: galiacheng --- .../src/main/arm/createUiDefinition.json | 348 ++++++++++-------- .../src/main/bicep/mainTemplate.bicep | 3 +- 2 files changed, 188 insertions(+), 163 deletions(-) diff --git a/weblogic-azure-aks/src/main/arm/createUiDefinition.json b/weblogic-azure-aks/src/main/arm/createUiDefinition.json index 6a89a1594..f0155ca8d 100644 --- a/weblogic-azure-aks/src/main/arm/createUiDefinition.json +++ b/weblogic-azure-aks/src/main/arm/createUiDefinition.json @@ -213,6 +213,18 @@ }, "visible": "[bool(basics('basicsOptional').basicsOptionalAcceptDefaults)]" }, + { + "name": "wlsJavaOption", + "type": "Microsoft.Common.TextBox", + "label": "Custom Java Options to start WebLogic Server", + "placeholder": "-Xms1200m -XX:MaxNewSize=300m -Dweblogic.webservice.i18n.charset=utf-8", + "toolTip": "Specify Java options to start WebLogic Server.", + "constraints": { + "regex": "^-[a-z0-9A-Z:./+_=-]+(\\s-[a-z0-9A-Z:./+_=-]+){0,99}$", + "validationMessage": "Allow letters, numbers, space, colon(:), plus(+), minus(-), full stop(.), slash(/), equal sign(=), underscore(_), and the value must be less then 100 Java options." + }, + "visible": "[bool(basics('basicsOptional').basicsOptionalAcceptDefaults)]" + }, { "name": "enableAdminT3Tunneling", "type": "Microsoft.Common.CheckBox", @@ -535,7 +547,7 @@ "type": "Microsoft.Common.TextBlock", "visible": true, "options": { - "text": "Selecting 'Yes' here will cause the template to provision WebLogic Administration Console on HTTPS (Secure) port, with your own TLS/SSL certificate.", + "text": "Selecting 'Yes' here will cause the template to provision WebLogic Administration Console, Remote Console, cluster and custom T3 channel on HTTPS (Secure) port, with your own TLS/SSL certificate.", "link": { "label": "Learn more", "uri": "https://aka.ms/arm-oraclelinux-wls-ssl-config" @@ -983,7 +995,7 @@ "width": "Full", "rows": { "count": { - "min": 1, + "min": 0, "max": 10 } }, @@ -1184,11 +1196,11 @@ { "name": "keyVaultBackendSSLCertData", "type": "Microsoft.Common.FileUpload", - "label": "Trusted root certificate(.cer, cert)", + "label": "Trusted root certificate(.cer, .cert)", "toolTip": "Trusted root certificate (CA certificate) used to set up end to end TLS/SSL", "constraints": { "required": true, - "accept": ".cer, cert" + "accept": ".cer, .cert" }, "options": { "multiple": false, @@ -1326,164 +1338,175 @@ } ], "visible": true + } + ] + }, + { + "name": "section_dnsConfiguration", + "type": "Microsoft.Common.Section", + "label": "DNS Configuration", + "elements": [ + { + "name": "dnsConfigurationText", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Selecting 'Yes' here will cause the template to provision Oracle WebLogic Server Administration Console, cluster, Remote Console and custom T3 channel using custom DNS Name (example: admin.contoso.com)", + "link": { + "label": "Learn more", + "uri": "https://aka.ms/arm-oraclelinux-wls-dns" + } + } }, { - "name": "dnsConfiguration", - "type": "Microsoft.Common.Section", - "label": "Custom DNS Configuration", - "elements": [ - { - "name": "enableDNSConfiguration", - "type": "Microsoft.Common.OptionsGroup", - "label": "Configure Custom DNS Alias", - "defaultValue": "No", - "toolTip": "Select 'Yes' to configure Custom DNS Alias.", - "constraints": { - "allowedValues": [ - { - "label": "Yes", - "value": true - }, - { - "label": "No", - "value": false - } - ], - "required": false - } - }, - { - "name": "bringDNSZone", - "type": "Microsoft.Common.OptionsGroup", - "label": "Use an existing Azure DNS Zone", - "defaultValue": "No", - "toolTip": "Select 'Yes' to configure Custom DNS Alias based on an existing Azure DNS Zone. Select 'No' to create an Azure DNS Zone and Custom DNS Alias.", - "constraints": { - "allowedValues": [ - { - "label": "Yes", - "value": true - }, - { - "label": "No", - "value": false - } - ] - }, - "visible": "[steps('section_appGateway').dnsConfiguration.enableDNSConfiguration]" - }, - { - "name": "dnszoneName", - "type": "Microsoft.Common.TextBox", - "label": "DNS Zone Name", - "defaultValue": "", - "toolTip": "Use only letters and numbers and periods to separate Domains", - "constraints": { - "required": true, - "regex": "^([0-9a-zA-Z_-]{1,63}\\.){1,33}[0-9a-zA-Z_-]{1,63}$", - "validationMessage": "There must be between 2 and 34 labels. For example, \"contoso.com\" has 2 labels. Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." + "name": "enableDNSConfiguration", + "type": "Microsoft.Common.OptionsGroup", + "label": "Configure Custom DNS Alias", + "defaultValue": "No", + "toolTip": "Select 'Yes' to configure Custom DNS Alias.", + "constraints": { + "allowedValues": [ + { + "label": "Yes", + "value": true }, - "visible": "[steps('section_appGateway').dnsConfiguration.enableDNSConfiguration]" - }, - { - "name": "dnsZoneResourceGroup", - "type": "Microsoft.Common.TextBox", - "label": "Name of the resource group contains the DNS Zone in current subscription", - "defaultValue": "", - "toolTip": "Name of the resource group which contains the DNS Zone in current subscription", - "constraints": { - "required": true, - "regex": "^[a-z0-9A-Z.\\-_()]{0,89}([a-z0-9A-Z\\-_()]{1})$", - "validationMessage": "[if(greater(length(steps('section_appGateway').dnsConfiguration.dnsZoneResourceGroup), 90),'Resource group names only allow up to 90 characters.', 'Resource group names only allow alphanumeric characters, periods, underscores, hyphens and parenthesis and cannot end in a period.')]" + { + "label": "No", + "value": false + } + ], + "required": false + } + }, + { + "name": "bringDNSZone", + "type": "Microsoft.Common.OptionsGroup", + "label": "Use an existing Azure DNS Zone", + "defaultValue": "No", + "toolTip": "Select 'Yes' to configure Custom DNS Alias based on an existing Azure DNS Zone. Select 'No' to create an Azure DNS Zone and Custom DNS Alias.", + "constraints": { + "allowedValues": [ + { + "label": "Yes", + "value": true }, - "visible": "[and(steps('section_appGateway').dnsConfiguration.enableDNSConfiguration,steps('section_appGateway').dnsConfiguration.bringDNSZone)]" - }, - { - "name": "dnszoneAdminConsoleLabel", - "type": "Microsoft.Common.TextBox", - "label": "Label for Oracle WebLogic Administration Console", - "defaultValue": "admin", - "toolTip": "Specify a label to generate subdomain of Oracle WebLogic Administration Console", - "constraints": { - "required": true, - "validations": [ - { - "regex": "^([0-9a-zA-Z_-]{1,63}\\.){0,33}[0-9a-zA-Z_-]{1,63}$", - "message": "Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." - }, - { - "isValid": "[less(sub(length(concat(steps('section_appGateway').dnsConfiguration.dnszoneAdminConsoleLabel,'.',steps('section_appGateway').dnsConfiguration.dnszoneName)),length(replace(concat(steps('section_appGateway').dnsConfiguration.dnszoneAdminConsoleLabel,'.',steps('section_appGateway').dnsConfiguration.dnszoneName), '.', ''))),34)]", - "message": "Subdomain must be between 2 and 34 labels. For example, \"admin.contoso.com\" has 3 labels." - } - ] + { + "label": "No", + "value": false + } + ] + }, + "visible": "[steps('section_dnsConfiguration').enableDNSConfiguration]" + }, + { + "name": "dnszoneName", + "type": "Microsoft.Common.TextBox", + "label": "DNS Zone Name", + "defaultValue": "", + "toolTip": "Use only letters and numbers and periods to separate Domains", + "constraints": { + "required": true, + "regex": "^([0-9a-zA-Z_-]{1,63}\\.){1,33}[0-9a-zA-Z_-]{1,63}$", + "validationMessage": "There must be between 2 and 34 labels. For example, \"contoso.com\" has 2 labels. Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." + }, + "visible": "[steps('section_dnsConfiguration').enableDNSConfiguration]" + }, + { + "name": "dnsZoneResourceGroup", + "type": "Microsoft.Common.TextBox", + "label": "Name of the resource group contains the DNS Zone in current subscription", + "defaultValue": "", + "toolTip": "Name of the resource group which contains the DNS Zone in current subscription", + "constraints": { + "required": true, + "regex": "^[a-z0-9A-Z.\\-_()]{0,89}([a-z0-9A-Z\\-_()]{1})$", + "validationMessage": "[if(greater(length(steps('section_dnsConfiguration').dnsZoneResourceGroup), 90),'Resource group names only allow up to 90 characters.', 'Resource group names only allow alphanumeric characters, periods, underscores, hyphens and parenthesis and cannot end in a period.')]" + }, + "visible": "[and(steps('section_dnsConfiguration').enableDNSConfiguration,steps('section_dnsConfiguration').bringDNSZone)]" + }, + { + "name": "dnszoneAdminConsoleLabel", + "type": "Microsoft.Common.TextBox", + "label": "Label for Oracle WebLogic Administration Console", + "defaultValue": "admin", + "toolTip": "Specify a label to generate subdomain of Oracle WebLogic Administration Console", + "constraints": { + "required": true, + "validations": [ + { + "regex": "^([0-9a-zA-Z_-]{1,63}\\.){0,33}[0-9a-zA-Z_-]{1,63}$", + "message": "Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." }, - "visible": "[steps('section_appGateway').dnsConfiguration.enableDNSConfiguration]" - }, - { - "name": "dnszoneAdminT3ChannelLabel", - "type": "Microsoft.Common.TextBox", - "label": "Label for Oracle WebLogic Admin Server T3 channel", - "defaultValue": "admin-t3", - "toolTip": "Specify a label to generate subdomain of Oracle WebLogic Admin Server T3 channel", - "constraints": { - "required": true, - "validations": [ - { - "regex": "^([0-9a-zA-Z_-]{1,63}\\.){0,33}[0-9a-zA-Z_-]{1,63}$", - "message": "Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." - }, - { - "isValid": "[less(sub(length(concat(steps('section_appGateway').dnsConfiguration.dnszoneAdminT3ChannelLabel,'.',steps('section_appGateway').dnsConfiguration.dnszoneName)),length(replace(concat(steps('section_appGateway').dnsConfiguration.dnszoneAdminT3ChannelLabel,'.',steps('section_appGateway').dnsConfiguration.dnszoneName), '.', ''))),34)]", - "message": "Subdomain must be between 2 and 34 labels. For example, \"admin-t3.contoso.com\" has 3 labels." - } - ] + { + "isValid": "[less(sub(length(concat(steps('section_dnsConfiguration').dnszoneAdminConsoleLabel,'.',steps('section_dnsConfiguration').dnszoneName)),length(replace(concat(steps('section_dnsConfiguration').dnszoneAdminConsoleLabel,'.',steps('section_dnsConfiguration').dnszoneName), '.', ''))),34)]", + "message": "Subdomain must be between 2 and 34 labels. For example, \"admin.contoso.com\" has 3 labels." + } + ] + }, + "visible": "[steps('section_dnsConfiguration').enableDNSConfiguration]" + }, + { + "name": "dnszoneAdminT3ChannelLabel", + "type": "Microsoft.Common.TextBox", + "label": "Label for Oracle WebLogic Admin Server T3 channel", + "defaultValue": "admin-t3", + "toolTip": "Specify a label to generate subdomain of Oracle WebLogic Admin Server T3 channel", + "constraints": { + "required": true, + "validations": [ + { + "regex": "^([0-9a-zA-Z_-]{1,63}\\.){0,33}[0-9a-zA-Z_-]{1,63}$", + "message": "Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." }, - "visible": "[and(steps('section_appGateway').dnsConfiguration.enableDNSConfiguration,basics('basicsOptional').enableT3Tunneling,steps('section_appGateway').lbSVCInfo.enableLBSVC)]" - }, - { - "name": "dnszoneGatewayLabel", - "type": "Microsoft.Common.TextBox", - "label": "Label for WebLogic Cluster", - "defaultValue": "www", - "toolTip": "Specify a label to generate subdomain of WebLogic Cluster", - "constraints": { - "required": true, - "validations": [ - { - "regex": "^([0-9a-zA-Z_-]{1,63}\\.){0,33}[0-9a-zA-Z_-]{1,63}$", - "message": "Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." - }, - { - "isValid": "[less(sub(length(concat(if(empty(steps('section_appGateway').dnsConfiguration.dnszoneGatewayLabel), '', steps('section_appGateway').dnsConfiguration.dnszoneGatewayLabel),'.',steps('section_appGateway').dnsConfiguration.dnszoneName)),length(replace(concat(if(empty(steps('section_appGateway').dnsConfiguration.dnszoneGatewayLabel), '', steps('section_appGateway').dnsConfiguration.dnszoneGatewayLabel),'.',steps('section_appGateway').dnsConfiguration.dnszoneName), '.', ''))),34)]", - "message": "Subdomain must be between 2 and 34 labels. For example, \"applications.contoso.com\" has 3 labels." - } - ] + { + "isValid": "[less(sub(length(concat(steps('section_dnsConfiguration').dnszoneAdminT3ChannelLabel,'.',steps('section_dnsConfiguration').dnszoneName)),length(replace(concat(steps('section_dnsConfiguration').dnszoneAdminT3ChannelLabel,'.',steps('section_dnsConfiguration').dnszoneName), '.', ''))),34)]", + "message": "Subdomain must be between 2 and 34 labels. For example, \"admin-t3.contoso.com\" has 3 labels." + } + ] + }, + "visible": "[and(steps('section_dnsConfiguration').enableDNSConfiguration,basics('basicsOptional').enableAdminT3Tunneling, steps('section_appGateway').lbSVCInfo.enableLBSVC)]" + }, + { + "name": "dnszoneGatewayLabel", + "type": "Microsoft.Common.TextBox", + "label": "Label for WebLogic Cluster", + "defaultValue": "www", + "toolTip": "Specify a label to generate subdomain of WebLogic Cluster", + "constraints": { + "required": true, + "validations": [ + { + "regex": "^([0-9a-zA-Z_-]{1,63}\\.){0,33}[0-9a-zA-Z_-]{1,63}$", + "message": "Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." }, - "visible": "[steps('section_appGateway').dnsConfiguration.enableDNSConfiguration]" - }, - { - "name": "dnszoneClusterT3ChannelLabel", - "type": "Microsoft.Common.TextBox", - "label": "Label for Oracle WebLogic cluster T3 channel", - "defaultValue": "cluster-t3", - "toolTip": "Specify a label to generate subdomain of Oracle WebLogic cluster T3 channel", - "constraints": { - "required": true, - "validations": [ - { - "regex": "^([0-9a-zA-Z_-]{1,63}\\.){0,33}[0-9a-zA-Z_-]{1,63}$", - "message": "Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." - }, - { - "isValid": "[less(sub(length(concat(steps('section_appGateway').dnsConfiguration.dnszoneClusterT3ChannelLabel,'.',steps('section_appGateway').dnsConfiguration.dnszoneName)),length(replace(concat(steps('section_appGateway').dnsConfiguration.dnszoneClusterT3ChannelLabel,'.',steps('section_appGateway').dnsConfiguration.dnszoneName), '.', ''))),34)]", - "message": "Subdomain must be between 2 and 34 labels. For example, \"application-t3.contoso.com\" has 3 labels." - } - ] + { + "isValid": "[less(sub(length(concat(if(empty(steps('section_dnsConfiguration').dnszoneGatewayLabel), '', steps('section_dnsConfiguration').dnszoneGatewayLabel),'.',steps('section_dnsConfiguration').dnszoneName)),length(replace(concat(if(empty(steps('section_dnsConfiguration').dnszoneGatewayLabel), '', steps('section_dnsConfiguration').dnszoneGatewayLabel),'.',steps('section_dnsConfiguration').dnszoneName), '.', ''))),34)]", + "message": "Subdomain must be between 2 and 34 labels. For example, \"applications.contoso.com\" has 3 labels." + } + ] + }, + "visible": "[steps('section_dnsConfiguration').enableDNSConfiguration]" + }, + { + "name": "dnszoneClusterT3ChannelLabel", + "type": "Microsoft.Common.TextBox", + "label": "Label for Oracle WebLogic cluster T3 channel", + "defaultValue": "cluster-t3", + "toolTip": "Specify a label to generate subdomain of Oracle WebLogic cluster T3 channel", + "constraints": { + "required": true, + "validations": [ + { + "regex": "^([0-9a-zA-Z_-]{1,63}\\.){0,33}[0-9a-zA-Z_-]{1,63}$", + "message": "Each label must contain between 1 and 63 characters. Each label must only contain letters, numbers, underscores, and dashes." }, - "visible": "[and(steps('section_appGateway').dnsConfiguration.enableDNSConfiguration,basics('basicsOptional').enableT3Tunneling,steps('section_appGateway').lbSVCInfo.enableLBSVC)]" - } - ], - "visible": true + { + "isValid": "[less(sub(length(concat(steps('section_dnsConfiguration').dnszoneClusterT3ChannelLabel,'.',steps('section_dnsConfiguration').dnszoneName)),length(replace(concat(steps('section_dnsConfiguration').dnszoneClusterT3ChannelLabel,'.',steps('section_dnsConfiguration').dnszoneName), '.', ''))),34)]", + "message": "Subdomain must be between 2 and 34 labels. For example, \"application-t3.contoso.com\" has 3 labels." + } + ] + }, + "visible": "[and(steps('section_dnsConfiguration').enableDNSConfiguration,basics('basicsOptional').enableClusterT3Tunneling, steps('section_appGateway').lbSVCInfo.enableLBSVC)]" } ] }, @@ -1633,16 +1656,16 @@ "appReplicas": "[int(steps('section_aks').jeeAppInfo.appReplicas)]", "createACR": "[bool(steps('section_aks').acrInfo.createACR)]", "createAKSCluster": "[bool(steps('section_aks').clusterInfo.createAKSCluster)]", - "createDNSZone": "[not(bool(steps('section_appGateway').dnsConfiguration.bringDNSZone))]", + "createDNSZone": "[not(bool(steps('section_dnsConfiguration').bringDNSZone))]", "dbPassword": "[steps('section_database').databaseConnectionInfo.dbPassword]", "dbUser": "[steps('section_database').databaseConnectionInfo.dbUser]", "databaseType": "[steps('section_database').databaseConnectionInfo.databaseType]", - "dnszoneAdminConsoleLabel": "[steps('section_appGateway').dnsConfiguration.dnszoneAdminConsoleLabel]", - "dnszoneAdminT3ChannelLabel": "[steps('section_appGateway').dnsConfiguration.dnszoneAdminT3ChannelLabel]", - "dnszoneClusterLabel": "[steps('section_appGateway').dnsConfiguration.dnszoneGatewayLabel]", - "dnszoneClusterT3ChannelLabel": "[steps('section_appGateway').dnsConfiguration.dnszoneClusterT3ChannelLabel]", - "dnszoneName": "[steps('section_appGateway').dnsConfiguration.dnszoneName]", - "dnszoneRGName": "[steps('section_appGateway').dnsConfiguration.dnsZoneResourceGroup]", + "dnszoneAdminConsoleLabel": "[steps('section_dnsConfiguration').dnszoneAdminConsoleLabel]", + "dnszoneAdminT3ChannelLabel": "[steps('section_dnsConfiguration').dnszoneAdminT3ChannelLabel]", + "dnszoneClusterLabel": "[steps('section_dnsConfiguration').dnszoneGatewayLabel]", + "dnszoneClusterT3ChannelLabel": "[steps('section_dnsConfiguration').dnszoneClusterT3ChannelLabel]", + "dnszoneName": "[steps('section_dnsConfiguration').dnszoneName]", + "dnszoneRGName": "[steps('section_dnsConfiguration').dnsZoneResourceGroup]", "dsConnectionURL": "[steps('section_database').databaseConnectionInfo.dsConnectionURL]", "enableAppGWIngress": "[steps('section_appGateway').appgwIngress.enableAppGateway]", "enableAzureMonitoring": "[bool(steps('section_aks').clusterInfo.enableAzureMonitoring)]", @@ -1650,7 +1673,7 @@ "enableCookieBasedAffinity": "[bool(steps('section_appGateway').appgwIngress.enableCookieBasedAffinity)]", "enableCustomSSL": "[bool(steps('section_sslConfiguration').enableCustomSSL)]", "enableDB": "[bool(steps('section_database').enableDB)]", - "enableDNSConfiguration": "[bool(steps('section_appGateway').dnsConfiguration.enableDNSConfiguration)]", + "enableDNSConfiguration": "[bool(steps('section_dnsConfiguration').enableDNSConfiguration)]", "enableAdminT3Tunneling": "[basics('basicsOptional').enableAdminT3Tunneling]", "enableClusterT3Tunneling": "[basics('basicsOptional').enableClusterT3Tunneling]", "identity": "[basics('basicsRequired').identity]", @@ -1691,6 +1714,7 @@ "wlsDomainName": "[basics('basicsOptional').wlsDomainName]", "wlsDomainUID": "[basics('basicsOptional').wlsDomainUID]", "wlsImageTag": "[steps('section_aks').imageInfo.fromImage]", + "wlsJavaOption": "[basics('basicsOptional').wlsJavaOption]", "wlsPassword": "[basics('basicsRequired').wlsPassword]", "wlsUserName": "[basics('basicsRequired').wlsUserName]" } diff --git a/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep b/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep index 7b341fda9..85302355f 100644 --- a/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep +++ b/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep @@ -412,7 +412,8 @@ module wlsDomainWithCustomSSLDeployment 'modules/setupWebLogicCluster.bicep' = i createStorageAccount: const_bCreateStorageAccount enableAzureMonitoring: enableAzureMonitoring enableCustomSSL: enableCustomSSL - enableT3Tunneling: enableT3Tunneling + enableAdminT3Tunneling: enableAdminT3Tunneling + enableClusterT3Tunneling: enableClusterT3Tunneling enablePV: const_enablePV identity: identity location: location From 199968f4d533dcb28161ee231fd625ef182e91d5 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Fri, 3 Sep 2021 13:48:29 +0800 Subject: [PATCH 04/14] On branch t3tunneling: fix mount permission issue Use `az storage` commands to upload files instead of mounting the file share. Changes to be committed: modified: weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh modified: weblogic-azure-aks/src/main/arm/scripts/utility.sh Signed-off-by: galiacheng --- .../src/main/arm/scripts/setupWLSDomain.sh | 98 +++++++++++-------- .../src/main/arm/scripts/utility.sh | 33 +++++++ 2 files changed, 89 insertions(+), 42 deletions(-) diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh index 7c7ec95b0..393b0fd11 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh @@ -368,39 +368,21 @@ function build_docker_image() { fi } -function mount_fileshare() { - fileShareName="${azFileShareName}" - - # Disable https-only - az storage account update --name ${storageAccountName} --resource-group ${storageResourceGroup} --https-only false - +function create_source_folder_for_certificates() { mntRoot="/wls" - mntPath="$mntRoot/$storageAccountName/$fileShareName" + mntPath="$mntRoot/$storageAccountName/$azFileShareName" mkdir -p $mntPath - httpEndpoint=$( - az storage account show \ - --resource-group $storageResourceGroup \ - --name $storageAccountName \ - --query "primaryEndpoints.file" | tr -d '"' - ) - smbPath=$(echo $httpEndpoint | cut -c7-$(expr length $httpEndpoint))$fileShareName - - export storageAccountKey=$(az storage account keys list \ - --resource-group $storageResourceGroup \ - --account-name $storageAccountName \ - --query "[0].value" -o tsv) - - mount -t cifs $smbPath $mntPath -o username=$storageAccountName,password=$storageAccountKey,serverino,vers=3.0,file_mode=0777,dir_mode=0777 - validate_status "Mounting path." -} - -function unmount_fileshare() { - echo "unmount fileshare." - umount ${mntPath} - # Disable https-only - az storage account update --name ${storageAccountName} --resource-group ${storageResourceGroup} --https-only true + # Create a folder for certificates + securityDir=${mntPath}/security + if [ ! -d "${securityDir}" ]; then + mkdir ${mntPath}/security + else + rm -f ${mntPath}/$wlsIdentityKeyStoreFileName + rm -f ${mntPath}/$wlsTrustKeyStoreFileName + rm -f ${mntPath}/${wlsTrustKeyStoreJKSFileName} + fi } function validate_ssl_keystores() { @@ -444,18 +426,50 @@ function validate_ssl_keystores() { echo "Validate SSL key stores successfull !!" } +function upload_certificates_to_fileshare() { + expiryData=$(( `date +%s`+${sasTokenValidTime})) + sasTokenEnd=`date -d@"$expiryData" -u '+%Y-%m-%dT%H:%MZ'` + sasToken=$(az storage share generate-sas \ + --name ${fileShareName} \ + --account-name ${storageAccountName} \ + --https-only \ + --permissions dlrw \ + --expiry $sasTokenEnd -o tsv) + + echo "create directory security" + fsSecurityDirName="security" + utility_create_directory_to_fileshare \ + ${fsSecurityDirName} \ + ${fileShareName} \ + ${storageAccountName} \ + $sasToken + + echo "upload $wlsIdentityKeyStoreFileName" + utility_upload_file_to_fileshare \ + ${fileShareName} \ + ${storageAccountName} \ + "${fsSecurityDirName}/$wlsIdentityKeyStoreFileName" \ + ${mntPath}/$wlsIdentityKeyStoreFileName \ + $sasToken + + echo "upload $wlsTrustKeyStoreFileName" + utility_upload_file_to_fileshare \ + ${fileShareName} ${storageAccountName} \ + "${fsSecurityDirName}/$wlsTrustKeyStoreFileName" \ + ${mntPath}/$wlsTrustKeyStoreFileName \ + $sasToken + + echo "upload $wlsTrustKeyStoreJKSFileName" + utility_upload_file_to_fileshare \ + ${fileShareName} \ + ${storageAccountName} \ + "${fsSecurityDirName}/$wlsTrustKeyStoreJKSFileName" \ + ${mntPath}/${wlsTrustKeyStoreJKSFileName} \ + $sasToken +} + function output_ssl_keystore() { echo "Custom SSL is enabled. Storing CertInfo as files..." - # Create a folder for certificates - securityDir=${mntPath}/security - if [ ! -d "${securityDir}" ]; then - mkdir ${mntPath}/security - else - rm -f ${mntPath}/$wlsIdentityKeyStoreFileName - rm -f ${mntPath}/$wlsTrustKeyStoreFileName - rm -f ${mntPath}/${wlsTrustKeyStoreJKSFileName} - fi - #decode cert data once again as it would got base64 encoded echo "$wlsIdentityData" | base64 -d >${mntPath}/$wlsIdentityKeyStoreFileName echo "$wlsTrustData" | base64 -d >${mntPath}/$wlsTrustKeyStoreFileName @@ -604,17 +618,17 @@ function create_domain_namespace() { function parsing_ssl_certs_and_create_ssl_secret() { if [[ "${enableCustomSSL,,}" == "${constTrue}" ]]; then # use default Java, if no, install open jdk 11. - # why not Microsoft open jdk? No apk installation package! + # why not use Microsoft open jdk? No apk installation package! export JAVA_HOME=/usr/lib/jvm/default-jvm/ if [ ! -d "${JAVA_HOME}" ]; then install_jdk JAVA_HOME=/usr/lib/jvm/java-11-openjdk fi - mount_fileshare + create_source_folder_for_certificates output_ssl_keystore validate_ssl_keystores - unmount_fileshare + upload_certificates_to_fileshare echo "check if ${kubectlWLSSSLCredentialsName} exists." ret=$(kubectl get secret -n ${wlsDomainNS} | grep "${kubectlWLSSSLCredentialsName}") diff --git a/weblogic-azure-aks/src/main/arm/scripts/utility.sh b/weblogic-azure-aks/src/main/arm/scripts/utility.sh index b23d2006d..5ce783757 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/utility.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/utility.sh @@ -62,6 +62,39 @@ function utility_check_pv_state { fi } +# +# Create directory in specified file share +# $1 - name of directory +# $2 - name of file share +# $3 - name of storage account +# $4 - sas token +function utility_create_directory_to_fileshare() { + ret=$(az storage directory exists --name $1 --share-name $2 --account-name $3 --sas-token ${4} | jq '.exists') + if [[ "${ret,,}" == "false" ]]; then + az storage directory create --name $1 --share-name $2 --account-name $3 --sas-token ${4} --timeout 30 + fi + + if [ $? != 0 ]; then + echo_stderr "Failed to create directory ${1} in file share ${3}/${2}" + exit 1 + fi +} + +# +# Upload file to file share +# $1 - name of file share +# $2 - name of storage account +# $3 - path of file +# $4 - source path of file +# $5 - sas token +function utility_upload_file_to_fileshare(){ + az storage file upload --share-name ${1} --account-name ${2} --path ${3} --source ${4} --sas-token ${5} --timeout 60 + if [ $? != 0 ]; then + echo_stderr "Failed to upload ${3} to file share ${2}/${1}" + exit 1 + fi +} + # Call this function to make sure pods of a domain are running. # * Make sure the admin server pod is running # * Make sure all the managed server pods are running From 18457423ec1c5869179767467ad79a214d14d678 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Sun, 5 Sep 2021 10:48:14 +0800 Subject: [PATCH 05/14] On branch t3tunneling: check ingress-azure status Changes to be committed: modified: weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh Signed-off-by: galiacheng --- weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh index 05206a213..b05fa6075 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh @@ -1126,8 +1126,8 @@ function create_appgw_ingress() { echo Waiting for Pod running...${attempts} sleep ${perfRetryInterval} - ret=$(kubectl get pod | grep "ingress-azure") - if [ -z "${ret}" ]; then + ret=$(kubectl get pod -o json | jq '.items[] | .status.containerStatuses[] | select(.name=="ingress-azure") | .ready') + if [[ "${ret}"=="false" ]]; then podState="running" if [ $attempts -ge ${perfPodAttemps} ]; then From 5426fab9ba1675d18daa04a59571db1db1a6075a Mon Sep 17 00:00:00 2001 From: galiacheng Date: Sun, 5 Sep 2021 10:59:34 +0800 Subject: [PATCH 06/14] On branch t3tunneling: support t3s Changes to be committed: modified: weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh modified: weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh Signed-off-by: galiacheng --- .../src/main/arm/scripts/setupNetworking.sh | 7 +- .../src/main/arm/scripts/setupWLSDomain.sh | 183 ++++++++++-------- 2 files changed, 104 insertions(+), 86 deletions(-) diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh index b05fa6075..715fea547 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh @@ -836,6 +836,9 @@ function patch_cluster_t3_public_address() { function rolling_update_with_t3_public_address() { timestampBeforePatchingDomain=$(date +%s) currentDomainConfig=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json) + cat<${scriptDir}/domainPreviousConfiguration.yaml +${currentDomainConfig} +EOF # update public address of t3 channel if [[ "${enableAdminT3Channel,,}" == "true" ]]; then @@ -860,7 +863,9 @@ function rolling_update_with_t3_public_address() { echo "rolling restart the cluster with t3 public address." # echo the configuration for debugging - echo -e ${currentDomainConfig} >${scriptDir}/domainNewConfiguration.yaml + cat<${scriptDir}/domainNewConfiguration.yaml +${currentDomainConfig} +EOF echo ${currentDomainConfig} | kubectl -n ${wlsDomainNS} apply -f - replicas=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json \ diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh index 393b0fd11..d2b73079b 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh @@ -11,7 +11,8 @@ function read_sensitive_parameters_from_stdin() { #Function to display usage message function usage() { - usage=$(cat <<-END + usage=$( + cat <<-END Usage: echo | ./setupWLSDomain.sh @@ -44,7 +45,7 @@ echo END -) + ) echo_stdout ${usage} if [ $1 -eq 1 ]; then echo_stderr ${usage} @@ -198,6 +199,15 @@ function validate_input() { echo_stderr "t3ClusterPort is required. " usage 1 fi + + if [ -z "$wlsJavaOption" ]; then + echo_stderr "wlsJavaOption is required. " + usage 1 + fi + + if [[ "${wlsJavaOption}" == "null" ]];then + wlsJavaOption="" + fi } # Validate teminal status with $?, exit with exception if errors happen. @@ -311,13 +321,13 @@ function install_wls_operator() { echo "install the operator" helm install ${wlsOptRelease} weblogic-operator/weblogic-operator \ - --namespace ${wlsOptNameSpace} \ - --set serviceAccount=${wlsOptSA} \ - --set "enableClusterRoleBinding=true" \ - --set "domainNamespaceSelectionStrategy=LabelSelector" \ - --set "domainNamespaceLabelSelector=weblogic-operator\=enabled" \ - --version ${wlsOptVersion} \ - --wait + --namespace ${wlsOptNameSpace} \ + --set serviceAccount=${wlsOptSA} \ + --set "enableClusterRoleBinding=true" \ + --set "domainNamespaceSelectionStrategy=LabelSelector" \ + --set "domainNamespaceLabelSelector=weblogic-operator\=enabled" \ + --version ${wlsOptVersion} \ + --wait validate_status "Installing WLS operator." @@ -346,21 +356,21 @@ function query_acr_credentials() { function build_docker_image() { echo "build a new image including the new applications" chmod ugo+x $scriptDir/createVMAndBuildImage.sh - echo $azureACRPassword $ocrSSOPSW | \ + echo $azureACRPassword $ocrSSOPSW | bash $scriptDir/createVMAndBuildImage.sh \ - $currentResourceGroup \ - $wlsImageTag \ - $azureACRServer \ - $azureACRUserName \ - $newImageTag \ - "$appPackageUrls" \ - $ocrSSOUser \ - $wlsClusterSize \ - $enableCustomSSL \ - "$scriptURL" \ - ${enableAdminT3Tunneling} \ - ${enableClusterT3Tunneling} - + $currentResourceGroup \ + $wlsImageTag \ + $azureACRServer \ + $azureACRUserName \ + $newImageTag \ + "$appPackageUrls" \ + $ocrSSOUser \ + $wlsClusterSize \ + $enableCustomSSL \ + "$scriptURL" \ + ${enableAdminT3Tunneling} \ + ${enableClusterT3Tunneling} + az acr repository show -n ${acrName} --image aks-wls-images:${newImageTag} if [ $? -ne 0 ]; then echo "Failed to create image ${azureACRServer}/aks-wls-images:${newImageTag}" @@ -427,10 +437,10 @@ function validate_ssl_keystores() { } function upload_certificates_to_fileshare() { - expiryData=$(( `date +%s`+${sasTokenValidTime})) - sasTokenEnd=`date -d@"$expiryData" -u '+%Y-%m-%dT%H:%MZ'` + expiryData=$(($(date +%s) + ${sasTokenValidTime})) + sasTokenEnd=$(date -d@"$expiryData" -u '+%Y-%m-%dT%H:%MZ') sasToken=$(az storage share generate-sas \ - --name ${fileShareName} \ + --name ${azFileShareName} \ --account-name ${storageAccountName} \ --https-only \ --permissions dlrw \ @@ -440,30 +450,31 @@ function upload_certificates_to_fileshare() { fsSecurityDirName="security" utility_create_directory_to_fileshare \ ${fsSecurityDirName} \ - ${fileShareName} \ + ${azFileShareName} \ ${storageAccountName} \ $sasToken echo "upload $wlsIdentityKeyStoreFileName" utility_upload_file_to_fileshare \ - ${fileShareName} \ + ${azFileShareName} \ ${storageAccountName} \ - "${fsSecurityDirName}/$wlsIdentityKeyStoreFileName" \ + "$wlsIdentityKeyStoreFileName" \ ${mntPath}/$wlsIdentityKeyStoreFileName \ $sasToken echo "upload $wlsTrustKeyStoreFileName" utility_upload_file_to_fileshare \ - ${fileShareName} ${storageAccountName} \ - "${fsSecurityDirName}/$wlsTrustKeyStoreFileName" \ + ${azFileShareName} \ + ${storageAccountName} \ + "$wlsTrustKeyStoreFileName" \ ${mntPath}/$wlsTrustKeyStoreFileName \ $sasToken echo "upload $wlsTrustKeyStoreJKSFileName" utility_upload_file_to_fileshare \ - ${fileShareName} \ + ${azFileShareName} \ ${storageAccountName} \ - "${fsSecurityDirName}/$wlsTrustKeyStoreJKSFileName" \ + "$wlsTrustKeyStoreJKSFileName" \ ${mntPath}/${wlsTrustKeyStoreJKSFileName} \ $sasToken } @@ -524,8 +535,8 @@ function create_pv() { export storageAccountKey=$(az storage account keys list --resource-group $storageResourceGroup --account-name $storageAccountName --query "[0].value" -o tsv) export azureSecretName="azure-secret" kubectl -n ${wlsDomainNS} create secret generic ${azureSecretName} \ - --from-literal=azurestorageaccountname=${storageAccountName} \ - --from-literal=azurestorageaccountkey=${storageAccountKey} + --from-literal=azurestorageaccountname=${storageAccountName} \ + --from-literal=azurestorageaccountkey=${storageAccountKey} # generate pv configurations customPVYaml=${scriptDir}/pv.yaml @@ -555,7 +566,7 @@ function create_pv() { } function wait_for_pod_completed() { - echo "Waiting for $((appReplicas+1)) pods are running." + echo "Waiting for $((appReplicas + 1)) pods are running." utility_wait_for_pod_completed \ ${appReplicas} \ @@ -568,8 +579,8 @@ function wait_for_image_update_completed() { # Make sure all of the pods are updated with new image. # Assumption: we have only one cluster currently. acrImagePath=${azureACRServer}/aks-wls-images:${newImageTag} - echo "Waiting for $((appReplicas+1)) new pods created with image ${acrImagePath}" - + echo "Waiting for $((appReplicas + 1)) new pods created with image ${acrImagePath}" + utility_wait_for_image_update_completed \ "${acrImagePath}" \ ${appReplicas} \ @@ -596,21 +607,21 @@ function create_domain_namespace() { fi kubectl -n ${wlsDomainNS} create secret generic \ - ${kubectlWLSCredentialName} \ - --from-literal=username=${wlsUserName} \ - --from-literal=password=${wlsPassword} + ${kubectlWLSCredentialName} \ + --from-literal=username=${wlsUserName} \ + --from-literal=password=${wlsPassword} kubectl -n ${wlsDomainNS} label secret ${kubectlWLSCredentialName} weblogic.domainUID=${wlsDomainUID} kubectl -n ${wlsDomainNS} create secret generic ${kubectlWDTEncryptionSecret} \ - --from-literal=password=${wdtRuntimePassword} + --from-literal=password=${wdtRuntimePassword} kubectl -n ${wlsDomainNS} label secret ${kubectlWDTEncryptionSecret} weblogic.domainUID=${wlsDomainUID} kubectl create secret docker-registry ${kubectlSecretForACR} \ - --docker-server=${azureACRServer} \ - --docker-username=${azureACRUserName} \ - --docker-password=${azureACRPassword} \ - -n ${wlsDomainNS} + --docker-server=${azureACRServer} \ + --docker-username=${azureACRUserName} \ + --docker-password=${azureACRPassword} \ + -n ${wlsDomainNS} kubectl -n ${wlsDomainNS} label secret ${kubectlSecretForACR} weblogic.domainUID=${wlsDomainUID} } @@ -660,6 +671,13 @@ function parsing_ssl_certs_and_create_ssl_secret() { # * Deploy WebLogic domain using image in ACR # * Wait for the domain completed function setup_wls_domain() { + export javaOptions=${wlsJavaOption} + if [[ "${enableClusterT3Channel,,}" == "true" ]] || [[ "${enableAdminT3Channel,,}" == "true" ]]; then + # for remote t3/t3s access. + # refer to https://oracle.github.io/weblogic-kubernetes-operator/faq/external-clients/#enabling-unknown-host-access + javaOptions="-Dweblogic.rjvm.allowUnknownHost=true ${javaOptions}" + fi + # create namespace create_domain_namespace @@ -681,55 +699,48 @@ function setup_wls_domain() { echo "print pvc info" kubectl -n ${wlsDomainNS} get pvc -o wide - export javaOptions=${wlsJavaOption} - if [[ "${enableClusterT3Channel,,}" == "true" ]] || [[ "${enableAdminT3Channel,,}" == "true" ]]; then - # for remote t3/t3s access. - # refer to https://oracle.github.io/weblogic-kubernetes-operator/faq/external-clients/#enabling-unknown-host-access - javaOptions="-Dweblogic.rjvm.allowUnknownHost=true ${javaOptions}" - fi - customDomainYaml=${scriptDir}/custom-domain.yaml if [[ "${updateNamepace}" == "${constTrue}" ]]; then echo "start to update domain ${wlsDomainUID}" chmod ugo+x $scriptDir/updateDomainConfig.sh bash $scriptDir/updateDomainConfig.sh \ - ${customDomainYaml} \ - ${appReplicas} \ - ${wlsCPU} \ - ${wlsDomainUID} \ - ${wlsDomainName} \ - "${azureACRServer}/aks-wls-images:${newImageTag}" \ - ${wlsMemory} \ - ${managedServerPrefix} \ - ${enableCustomSSL} \ - ${enablePV} \ - ${enableAdminT3Tunneling} \ - ${enableClusterT3Tunneling} \ - ${t3AdminPort} \ - ${t3ClusterPort} \ - ${wlsClusterName} \ - "${javaOptions}" + ${customDomainYaml} \ + ${appReplicas} \ + ${wlsCPU} \ + ${wlsDomainUID} \ + ${wlsDomainName} \ + "${azureACRServer}/aks-wls-images:${newImageTag}" \ + ${wlsMemory} \ + ${managedServerPrefix} \ + ${enableCustomSSL} \ + ${enablePV} \ + ${enableAdminT3Tunneling} \ + ${enableClusterT3Tunneling} \ + ${t3AdminPort} \ + ${t3ClusterPort} \ + ${wlsClusterName} \ + "${javaOptions}" else echo "start to create domain ${wlsDomainUID}" # generate domain yaml chmod ugo+x $scriptDir/genDomainConfig.sh bash $scriptDir/genDomainConfig.sh \ - ${customDomainYaml} \ - ${appReplicas} \ - ${wlsCPU} \ - ${wlsDomainUID} \ - ${wlsDomainName} \ - "${azureACRServer}/aks-wls-images:${newImageTag}" \ - ${wlsMemory} \ - ${managedServerPrefix} \ - ${enableCustomSSL} \ - ${enablePV} \ - ${enableAdminT3Tunneling} \ - ${enableClusterT3Tunneling} \ - ${t3AdminPort} \ - ${t3ClusterPort} \ - ${wlsClusterName} \ - "${javaOptions}" + ${customDomainYaml} \ + ${appReplicas} \ + ${wlsCPU} \ + ${wlsDomainUID} \ + ${wlsDomainName} \ + "${azureACRServer}/aks-wls-images:${newImageTag}" \ + ${wlsMemory} \ + ${managedServerPrefix} \ + ${enableCustomSSL} \ + ${enablePV} \ + ${enableAdminT3Tunneling} \ + ${enableClusterT3Tunneling} \ + ${t3AdminPort} \ + ${t3ClusterPort} \ + ${wlsClusterName} \ + "${javaOptions}" fi kubectl apply -f ${customDomainYaml} @@ -785,6 +796,8 @@ export kubectlWLSCredentialName="${wlsDomainUID}-weblogic-credentials" export kubectlWLSSSLCredentialsName="${wlsDomainUID}-weblogic-ssl-credentials" export newImageTag=$(date +%s) export operatorName="weblogic-operator" +# seconds +export sasTokenValidTime=3600 export storageFileShareName="weblogic" export storageResourceGroup=${currentResourceGroup} export sharedPath="/shared" From 4cd67ac0029a339be8c3325815f88c7cc47d2476 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Mon, 6 Sep 2021 09:34:25 +0800 Subject: [PATCH 07/14] On branch t3tunneling: make sure azure-ingress is running. Changes to be committed: modified: weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh Signed-off-by: galiacheng --- .../src/main/arm/scripts/setupNetworking.sh | 253 +++++++++--------- 1 file changed, 122 insertions(+), 131 deletions(-) diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh index 715fea547..13e3c8cec 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh @@ -5,7 +5,7 @@ echo "Script ${0} starts" # read from stdin function read_sensitive_parameters_from_stdin() { - read spBase64String appgwFrontendSSLCertPsw + read spBase64String appgwFrontendSSLCertPsw } function install_helm() { @@ -56,7 +56,8 @@ function output_result() { #Function to display usage message function usage() { - usage=$(cat <<-END + usage=$( + cat <<-END Usage: echo | ./setupNetworking.sh @@ -86,12 +87,12 @@ echo | END -) - echo_stdout ${usage} - if [ $1 -eq 1 ]; then - echo_stderr ${usage} - exit 1 - fi + ) + echo_stdout ${usage} + if [ $1 -eq 1 ]; then + echo_stderr ${usage} + exit 1 + fi } #Validate teminal status with $?, exit with exception if errors happen. @@ -365,8 +366,7 @@ spec: EOF } -function generate_appgw_cluster_config_file_expose_https() -{ +function generate_appgw_cluster_config_file_expose_https() { export clusterIngressHttpsName=${wlsDomainUID}-cluster-appgw-ingress-https-svc export clusterAppgwIngressHttpsYamlPath=${scriptDir}/appgw-cluster-ingress-https-svc.yaml cat <${clusterAppgwIngressHttpsYamlPath} @@ -379,13 +379,13 @@ metadata: kubernetes.io/ingress.class: azure/application-gateway EOF -if [[ "${enableCookieBasedAffinity,,}" == "true" ]];then - cat <>${clusterAppgwIngressHttpsYamlPath} + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${clusterAppgwIngressHttpsYamlPath} appgw.ingress.kubernetes.io/cookie-based-affinity: "true" EOF -fi + fi -cat <>${clusterAppgwIngressHttpsYamlPath} + cat <>${clusterAppgwIngressHttpsYamlPath} spec: tls: - secretName: ${appgwFrontendSecretName} @@ -402,8 +402,7 @@ spec: EOF } -function generate_appgw_cluster_config_file_nossl() -{ +function generate_appgw_cluster_config_file_nossl() { export clusterIngressName=${wlsDomainUID}-cluster-appgw-ingress-svc export clusterAppgwIngressYamlPath=${scriptDir}/appgw-cluster-ingress-svc.yaml cat <${clusterAppgwIngressYamlPath} @@ -416,13 +415,13 @@ metadata: kubernetes.io/ingress.class: azure/application-gateway EOF -if [[ "${enableCookieBasedAffinity,,}" == "true" ]];then - cat <>${clusterAppgwIngressYamlPath} + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${clusterAppgwIngressYamlPath} appgw.ingress.kubernetes.io/cookie-based-affinity: "true" EOF -fi + fi -cat <>${clusterAppgwIngressYamlPath} + cat <>${clusterAppgwIngressYamlPath} spec: rules: - http: @@ -437,8 +436,7 @@ spec: EOF } -function generate_appgw_cluster_config_file_ssl() -{ +function generate_appgw_cluster_config_file_ssl() { export clusterIngressName=${wlsDomainUID}-cluster-appgw-ingress-svc export clusterAppgwIngressYamlPath=${scriptDir}/appgw-cluster-ingress-svc.yaml cat <${clusterAppgwIngressYamlPath} @@ -452,7 +450,7 @@ metadata: appgw.ingress.kubernetes.io/ssl-redirect: "true" appgw.ingress.kubernetes.io/backend-protocol: "https" EOF - if [[ "${enableCustomDNSAlias,,}" == "true" ]];then + if [[ "${enableCustomDNSAlias,,}" == "true" ]]; then cat <>${clusterAppgwIngressYamlPath} appgw.ingress.kubernetes.io/backend-hostname: "${dnsClusterLabel}.${dnsZoneName}" EOF @@ -462,13 +460,13 @@ EOF EOF fi - if [[ "${enableCookieBasedAffinity,,}" == "true" ]];then - cat <>${clusterAppgwIngressYamlPath} + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${clusterAppgwIngressYamlPath} appgw.ingress.kubernetes.io/cookie-based-affinity: "true" EOF -fi + fi -cat <>${clusterAppgwIngressYamlPath} + cat <>${clusterAppgwIngressYamlPath} appgw.ingress.kubernetes.io/appgw-trusted-root-certificate: "${appgwBackendSecretName}" spec: @@ -487,8 +485,7 @@ spec: EOF } -function generate_appgw_admin_config_file_nossl() -{ +function generate_appgw_admin_config_file_nossl() { export adminIngressName=${wlsDomainUID}-admin-appgw-ingress-svc export adminAppgwIngressYamlPath=${scriptDir}/appgw-admin-ingress-svc.yaml cat <${adminAppgwIngressYamlPath} @@ -501,13 +498,13 @@ metadata: kubernetes.io/ingress.class: azure/application-gateway EOF - if [[ "${enableCookieBasedAffinity,,}" == "true" ]];then - cat <>${adminAppgwIngressYamlPath} + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${adminAppgwIngressYamlPath} appgw.ingress.kubernetes.io/cookie-based-affinity: "true" EOF -fi + fi -cat <>${adminAppgwIngressYamlPath} + cat <>${adminAppgwIngressYamlPath} spec: rules: - http: @@ -522,8 +519,7 @@ spec: EOF } -function generate_appgw_admin_remote_config_file_nossl() -{ +function generate_appgw_admin_remote_config_file_nossl() { cat <${adminRemoteAppgwIngressYamlPath} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -535,13 +531,13 @@ metadata: appgw.ingress.kubernetes.io/backend-path-prefix: "/" EOF - if [[ "${enableCookieBasedAffinity,,}" == "true" ]];then - cat <>${adminRemoteAppgwIngressYamlPath} + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${adminRemoteAppgwIngressYamlPath} appgw.ingress.kubernetes.io/cookie-based-affinity: "true" EOF -fi + fi -cat <>${adminRemoteAppgwIngressYamlPath} + cat <>${adminRemoteAppgwIngressYamlPath} spec: rules: - http: @@ -556,8 +552,7 @@ spec: EOF } -function generate_appgw_admin_config_file_ssl() -{ +function generate_appgw_admin_config_file_ssl() { export adminIngressName=${wlsDomainUID}-admin-appgw-ingress-svc export adminAppgwIngressYamlPath=${scriptDir}/appgw-admin-ingress-svc.yaml cat <${adminAppgwIngressYamlPath} @@ -572,7 +567,7 @@ metadata: appgw.ingress.kubernetes.io/backend-protocol: "https" EOF - if [[ "${enableCustomDNSAlias,,}" == "true" ]];then + if [[ "${enableCustomDNSAlias,,}" == "true" ]]; then cat <>${adminAppgwIngressYamlPath} appgw.ingress.kubernetes.io/backend-hostname: "${dnsAdminLabel}.${dnsZoneName}" EOF @@ -582,11 +577,11 @@ EOF EOF fi - if [[ "${enableCookieBasedAffinity,,}" == "true" ]];then - cat <>${adminAppgwIngressYamlPath} + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${adminAppgwIngressYamlPath} appgw.ingress.kubernetes.io/cookie-based-affinity: "true" EOF -fi + fi cat <>${adminAppgwIngressYamlPath} appgw.ingress.kubernetes.io/appgw-trusted-root-certificate: "${appgwBackendSecretName}" @@ -607,8 +602,7 @@ spec: EOF } -function generate_appgw_admin_remote_config_file_ssl() -{ +function generate_appgw_admin_remote_config_file_ssl() { cat <${adminRemoteAppgwIngressYamlPath} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -623,7 +617,7 @@ metadata: EOF - if [[ "${enableCustomDNSAlias,,}" == "true" ]];then + if [[ "${enableCustomDNSAlias,,}" == "true" ]]; then cat <>${adminRemoteAppgwIngressYamlPath} appgw.ingress.kubernetes.io/backend-hostname: "${dnsAdminLabel}.${dnsZoneName}" EOF @@ -633,11 +627,11 @@ EOF EOF fi - if [[ "${enableCookieBasedAffinity,,}" == "true" ]];then - cat <>${adminRemoteAppgwIngressYamlPath} + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${adminRemoteAppgwIngressYamlPath} appgw.ingress.kubernetes.io/cookie-based-affinity: "true" EOF -fi + fi cat <>${adminRemoteAppgwIngressYamlPath} appgw.ingress.kubernetes.io/appgw-trusted-root-certificate: "${appgwBackendSecretName}" @@ -659,7 +653,7 @@ EOF } function generate_appgw_cluster_config_file() { - if [[ "${enableCustomSSL,,}" == "true" ]];then + if [[ "${enableCustomSSL,,}" == "true" ]]; then generate_appgw_cluster_config_file_ssl else generate_appgw_cluster_config_file_nossl @@ -668,7 +662,7 @@ function generate_appgw_cluster_config_file() { } function generate_appgw_admin_config_file() { - if [[ "${enableCustomSSL,,}" == "true" ]];then + if [[ "${enableCustomSSL,,}" == "true" ]]; then generate_appgw_admin_config_file_ssl else generate_appgw_admin_config_file_nossl @@ -676,28 +670,28 @@ function generate_appgw_admin_config_file() { } function generate_appgw_admin_remote_config_file() { - if [[ "${enableCustomSSL,,}" == "true" ]];then + if [[ "${enableCustomSSL,,}" == "true" ]]; then generate_appgw_admin_remote_config_file_ssl else generate_appgw_admin_remote_config_file_nossl fi } -function output_create_gateway_ssl_k8s_secret(){ +function output_create_gateway_ssl_k8s_secret() { echo "export gateway frontend certificates" echo "$appgwFrontendSSLCertData" | base64 -d >${scriptDir}/$appgwFrontCertFileName appgwFrontendSSLCertPassin=${appgwFrontendSSLCertPsw} - if [[ "$appgwCertificateOption" == "${appgwSelfsignedCert}" ]];then + if [[ "$appgwCertificateOption" == "${appgwSelfsignedCert}" ]]; then appgwFrontendSSLCertPassin="" # empty password fi openssl pkcs12 \ - -in ${scriptDir}/$appgwFrontCertFileName \ - -nocerts \ - -out ${scriptDir}/$appgwFrontCertKeyFileName \ - -passin pass:${appgwFrontendSSLCertPassin} \ - -passout pass:${appgwFrontendSSLCertPsw} + -in ${scriptDir}/$appgwFrontCertFileName \ + -nocerts \ + -out ${scriptDir}/$appgwFrontCertKeyFileName \ + -passin pass:${appgwFrontendSSLCertPassin} \ + -passout pass:${appgwFrontendSSLCertPsw} validate_status "Export key from frontend certificate." @@ -708,11 +702,11 @@ function output_create_gateway_ssl_k8s_secret(){ validate_status "Decryte private key." openssl pkcs12 \ - -in ${scriptDir}/$appgwFrontCertFileName \ - -clcerts \ - -nokeys \ - -out ${scriptDir}/$appgwFrontPublicCertFileName \ - -passin pass:${appgwFrontendSSLCertPassin} \ + -in ${scriptDir}/$appgwFrontCertFileName \ + -clcerts \ + -nokeys \ + -out ${scriptDir}/$appgwFrontPublicCertFileName \ + -passin pass:${appgwFrontendSSLCertPassin} validate_status "Export cert from frontend certificate." @@ -723,7 +717,7 @@ function output_create_gateway_ssl_k8s_secret(){ } function query_admin_target_port() { - if [[ "${enableCustomSSL,,}" == "true" ]];then + if [[ "${enableCustomSSL,,}" == "true" ]]; then adminTargetPort=$(kubectl describe service ${svcAdminServer} -n ${wlsDomainNS} | grep 'default-secure' | tr -d -c 0-9) else adminTargetPort=$(kubectl describe service ${svcAdminServer} -n ${wlsDomainNS} | grep 'default' | tr -d -c 0-9) @@ -734,7 +728,7 @@ function query_admin_target_port() { } function query_cluster_target_port() { - if [[ "${enableCustomSSL,,}" == "true" ]];then + if [[ "${enableCustomSSL,,}" == "true" ]]; then clusterTargetPort=$(kubectl describe service ${svcCluster} -n ${wlsDomainNS} | grep 'default-secure' | tr -d -c 0-9) else clusterTargetPort=$(kubectl describe service ${svcCluster} -n ${wlsDomainNS} | grep 'default' | tr -d -c 0-9) @@ -796,20 +790,20 @@ function patch_admin_t3_public_address() { if [ "${enableCustomDNSAlias,,}" == "true" ]; then adminT3Address="${dnszoneAdminT3ChannelLabel}.${dnsZoneName}" else - adminT3Address=$(kubectl -n ${wlsDomainNS} get svc ${adminServerT3LBSVCName} -o json \ - | jq '. | .status.loadBalancer.ingress[0].ip' \ - | tr -d "\"") + adminT3Address=$(kubectl -n ${wlsDomainNS} get svc ${adminServerT3LBSVCName} -o json | + jq '. | .status.loadBalancer.ingress[0].ip' | + tr -d "\"") fi if [ $? == 1 ]; then echo_stderr "Failed to query public IP of admin t3 channel." fi - currentDomainConfig=$(echo ${currentDomainConfig} \ - | jq \ - --arg match "${constAdminT3AddressEnvName}" \ - --arg replace "${adminT3Address}" \ - '.spec.serverPod.env |= map(if .name==$match then (.value=$replace) else . end)') + currentDomainConfig=$(echo ${currentDomainConfig} | + jq \ + --arg match "${constAdminT3AddressEnvName}" \ + --arg replace "${adminT3Address}" \ + '.spec.serverPod.env |= map(if .name==$match then (.value=$replace) else . end)') } function patch_cluster_t3_public_address() { @@ -817,26 +811,26 @@ function patch_cluster_t3_public_address() { if [ "${enableCustomDNSAlias,,}" == "true" ]; then clusterT3Adress="${dnszoneClusterT3ChannelLabel}.${dnsZoneName}" else - clusterT3Adress=$(kubectl -n ${wlsDomainNS} get svc ${clusterT3LBSVCName} -o json \ - | jq '. | .status.loadBalancer.ingress[0].ip' \ - | tr -d "\"") + clusterT3Adress=$(kubectl -n ${wlsDomainNS} get svc ${clusterT3LBSVCName} -o json | + jq '. | .status.loadBalancer.ingress[0].ip' | + tr -d "\"") fi if [ $? == 1 ]; then echo_stderr "Failed to query public IP of cluster t3 channel." fi - currentDomainConfig=$(echo ${currentDomainConfig} \ - | jq \ - --arg match "${constClusterT3AddressEnvName}" \ - --arg replace "${clusterT3Adress}" \ - '.spec.serverPod.env |= map(if .name==$match then (.value=$replace) else . end)') + currentDomainConfig=$(echo ${currentDomainConfig} | + jq \ + --arg match "${constClusterT3AddressEnvName}" \ + --arg replace "${clusterT3Adress}" \ + '.spec.serverPod.env |= map(if .name==$match then (.value=$replace) else . end)') } function rolling_update_with_t3_public_address() { timestampBeforePatchingDomain=$(date +%s) currentDomainConfig=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json) - cat<${scriptDir}/domainPreviousConfiguration.yaml + cat <${scriptDir}/domainPreviousConfiguration.yaml ${currentDomainConfig} EOF @@ -851,39 +845,39 @@ EOF if [[ "${enableClusterT3Channel,,}" == "true" ]] || [[ "${enableAdminT3Channel,,}" == "true" ]]; then # restart cluster - restartVersion=$( kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json \ - | jq '. | .spec.restartVersion' \ - | tr -d "\"") - restartVersion=$((restartVersion+1)) + restartVersion=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json | + jq '. | .spec.restartVersion' | + tr -d "\"") + restartVersion=$((restartVersion + 1)) - currentDomainConfig=$(echo ${currentDomainConfig} \ - | jq \ - --arg version "${restartVersion}" \ - '.spec.restartVersion |= $version') + currentDomainConfig=$(echo ${currentDomainConfig} | + jq \ + --arg version "${restartVersion}" \ + '.spec.restartVersion |= $version') echo "rolling restart the cluster with t3 public address." # echo the configuration for debugging - cat<${scriptDir}/domainNewConfiguration.yaml + cat <${scriptDir}/domainNewConfiguration.yaml ${currentDomainConfig} EOF echo ${currentDomainConfig} | kubectl -n ${wlsDomainNS} apply -f - - replicas=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json \ - | jq '. | .spec.clusters[] | .replicas') + replicas=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json | + jq '. | .spec.clusters[] | .replicas') # wait for the restart completed. utility_wait_for_pod_restarted \ - ${timestampBeforePatchingDomain} \ - ${replicas} \ - ${wlsDomainUID} \ - ${checkPodStatusMaxAttemps} \ - ${checkPodStatusInterval} + ${timestampBeforePatchingDomain} \ + ${replicas} \ + ${wlsDomainUID} \ + ${checkPodStatusMaxAttemps} \ + ${checkPodStatusInterval} utility_wait_for_pod_completed \ - ${replicas} \ - ${wlsDomainNS} \ - ${checkPodStatusMaxAttemps} \ - ${checkPodStatusInterval} + ${replicas} \ + ${wlsDomainNS} \ + ${checkPodStatusMaxAttemps} \ + ${checkPodStatusInterval} fi } @@ -950,15 +944,15 @@ EOF fi elif [[ "${target}" == "adminServerT3" ]]; then echo "query admin t3 port" - adminT3Port=$( kubectl get service ${svcAdminServer} -n ${wlsDomainNS} -o json \ - | jq '.spec.ports[] | select(.name=="t3channel") | .port') - adminT3sPort=$( kubectl get service ${svcAdminServer} -n ${wlsDomainNS} -o json \ - | jq '.spec.ports[] | select(.name=="t3schannel") | .port') + adminT3Port=$(kubectl get service ${svcAdminServer} -n ${wlsDomainNS} -o json | + jq '.spec.ports[] | select(.name=="t3channel") | .port') + adminT3sPort=$(kubectl get service ${svcAdminServer} -n ${wlsDomainNS} -o json | + jq '.spec.ports[] | select(.name=="t3schannel") | .port') if [[ "${adminT3Port}" == "null" ]] && [[ "${adminT3sPort}" == "null" ]]; then continue fi - if [[ "${adminT3sPort}" != "null" ]]; then + if [[ "${adminT3sPort}" != "null" ]]; then adminT3Port=${adminT3sPort} fi @@ -984,10 +978,10 @@ EOF enableAdminT3Channel=true elif [[ "${target}" == "cluster1T3" ]]; then echo "query cluster t3 port" - clusterT3Port=$( kubectl get service ${svcCluster} -n ${wlsDomainNS} -o json \ - | jq '.spec.ports[] | select(.name=="t3channel") | .port') - clusterT3sPort=$( kubectl get service ${svcCluster} -n ${wlsDomainNS} -o json \ - | jq '.spec.ports[] | select(.name=="t3schannel") | .port') + clusterT3Port=$(kubectl get service ${svcCluster} -n ${wlsDomainNS} -o json | + jq '.spec.ports[] | select(.name=="t3channel") | .port') + clusterT3sPort=$(kubectl get service ${svcCluster} -n ${wlsDomainNS} -o json | + jq '.spec.ports[] | select(.name=="t3schannel") | .port') if [[ "${clusterT3Port}" == "null" ]] && [[ "${clusterT3sPort}" == "null" ]]; then continue fi @@ -1008,7 +1002,7 @@ EOF clusterT3Endpoint=$(kubectl get svc ${clusterT3LBSVCName} -n ${wlsDomainNS} \ -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') - + create_dns_A_record "${clusterT3Endpoint%%:*}" ${dnszoneClusterT3ChannelLabel} if [ "${enableCustomDNSAlias,,}" == "true" ]; then @@ -1068,8 +1062,6 @@ function network_peers_aks_appgw() { validate_status "Associate the route table ${routeTableId} to Application Gateway's subnet ${appGatewaySubnetId}" } - - function create_appgw_ingress() { if [[ "${enableAppGWIngress,,}" != "true" ]]; then return @@ -1125,43 +1117,42 @@ function create_appgw_ingress() { attempts=0 podState="running" - while [ ! "$podState" == "completed" ] && [ $attempts -lt ${perfPodAttemps} ]; do + while [ "$podState" == "running" ] && [ $attempts -lt ${perfPodAttemps} ]; do podState="completed" attempts=$((attempts + 1)) echo Waiting for Pod running...${attempts} sleep ${perfRetryInterval} ret=$(kubectl get pod -o json | jq '.items[] | .status.containerStatuses[] | select(.name=="ingress-azure") | .ready') - if [[ "${ret}"=="false" ]]; then + if [[ "${ret}" == "false" ]]; then podState="running" - - if [ $attempts -ge ${perfPodAttemps} ]; then - echo_stderr "Failed to install app gateway ingress controller." - exit 1 - fi fi done + if [ "$podState" == "running" ] && [ $attempts -ge ${perfPodAttemps} ]; then + echo_stderr "Failed to install app gateway ingress controller." + exit 1 + fi + # create tsl secret output_create_gateway_ssl_k8s_secret - - if [[ "${enableCustomSSL,,}" == "true" ]];then + if [[ "${enableCustomSSL,,}" == "true" ]]; then az network application-gateway root-cert list \ --gateway-name $appgwName \ - --resource-group $curRGName \ - | jq '.[] | .name' | grep "${appgwBackendSecretName}" + --resource-group $curRGName | + jq '.[] | .name' | grep "${appgwBackendSecretName}" validate_status "check if backend cert exists." fi - # generate ingress svc config for cluster + # generate ingress svc config for cluster generate_appgw_cluster_config_file kubectl apply -f ${clusterAppgwIngressYamlPath} validate_status "Create appgw ingress svc." waitfor_svc_completed ${clusterIngressName} # expose https if e2e ssl is not set up. - if [[ "${enableCustomSSL,,}" != "true" ]];then + if [[ "${enableCustomSSL,,}" != "true" ]]; then kubectl apply -f ${clusterAppgwIngressHttpsYamlPath} validate_status "Create appgw ingress https svc." waitfor_svc_completed ${clusterIngressHttpsName} @@ -1236,7 +1227,7 @@ export clusterEndpoint="null" export httpsListenerName="myHttpsListenerName$(date +%s)" export httpsRuleName="myHttpsRule$(date +%s)" export perfRetryInterval=30 # seconds -export perfPodAttemps=5 +export perfPodAttemps=10 export perfSVCAttemps=10 export sharedPath="/shared" export svcAdminServer="${wlsDomainUID}-${adminServerName}" From 349f4a29f026ae6cbdc7f59f4bf976b8a14049fd Mon Sep 17 00:00:00 2001 From: galiacheng Date: Mon, 6 Sep 2021 14:15:42 +0800 Subject: [PATCH 08/14] On branch t3tunneling: expose t3 internal and external endpoints Changes to be committed: modified: weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh modified: weblogic-azure-aks/src/main/bicep/mainTemplate.bicep modified: weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep modified: weblogic-azure-aks/src/main/bicep/modules/networking.bicep modified: weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep Signed-off-by: galiacheng --- .../src/main/arm/scripts/setupNetworking.sh | 8 +++++++- .../src/main/bicep/mainTemplate.bicep | 16 +++++++++++----- .../_ds-create-networking.bicep | 2 ++ .../src/main/bicep/modules/networking.bicep | 3 +++ .../bicep/modules/setupWebLogicCluster.bicep | 2 ++ 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh index 13e3c8cec..0e3aeb8cf 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh @@ -45,11 +45,15 @@ function install_utilities() { function output_result() { echo ${adminConsoleEndpoint} echo ${clusterEndpoint} + echo ${adminServerT3Endpoint} + echo ${clusterT3Endpoint} result=$(jq -n -c \ --arg adminEndpoint $adminConsoleEndpoint \ --arg clusterEndpoint $clusterEndpoint \ - '{adminConsoleEndpoint: $adminEndpoint, clusterEndpoint: $clusterEndpoint}') + --arg adminT3Endpoint $adminServerT3Endpoint \ + --arg clusterT3Endpoint $clusterT3Endpoint \ + '{adminConsoleEndpoint: $adminEndpoint, clusterEndpoint: $clusterEndpoint, adminServerT3Endpoint: $adminT3Endpoint, clusterT3Endpoint: $clusterT3Endpoint}') echo "result is: $result" echo $result >$AZ_SCRIPTS_OUTPUT_PATH } @@ -1213,6 +1217,7 @@ export dnszoneClusterT3ChannelLabel=${25} export adminServerName="admin-server" export adminConsoleEndpoint="null" +export adminServerT3Endpoint="null" export appgwIngressHelmRepo="https://appgwingress.blob.core.windows.net/ingress-azure-helm-package/" export appgwFrontCertFileName="appgw-frontend-cert.pfx" export appgwFrontCertKeyDecrytedFileName="appgw-frontend-cert.key" @@ -1224,6 +1229,7 @@ export appgwSelfsignedCert="generateCert" export azureAppgwIngressVersion="1.4.0" export clusterName="cluster-1" export clusterEndpoint="null" +export clusterT3Endpoint="null" export httpsListenerName="myHttpsListenerName$(date +%s)" export httpsRuleName="myHttpsRule$(date +%s)" export perfRetryInterval=30 # seconds diff --git a/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep b/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep index 85302355f..497302265 100644 --- a/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep +++ b/weblogic-azure-aks/src/main/bicep/mainTemplate.bicep @@ -96,7 +96,7 @@ param dnsNameforApplicationGateway string = 'wlsgw' @description('Specify a label used to generate subdomain of Admin server. The final subdomain name will be label.dnszoneName, e.g. admin.contoso.xyz') param dnszoneAdminConsoleLabel string = 'admin' @description('Specify a label used to generate subdomain of Admin server T3 channel. The final subdomain name will be label.dnszoneName, e.g. admin-t3.contoso.xyz') -param dnszoneAdminT3ChannelLabel string ='admin-t3' +param dnszoneAdminT3ChannelLabel string = 'admin-t3' @description('Specify a label used to generate subdomain of WebLogic cluster. The final subdomain name will be label.dnszoneName, e.g. applications.contoso.xyz') param dnszoneClusterLabel string = 'www' param dnszoneClusterT3ChannelLabel string = 'cluster-t3' @@ -256,6 +256,7 @@ var const_hasStorageAccount = !createAKSCluster && reference('query-existing-sto var const_identityKeyStoreType = (sslConfigurationAccessOption == const_wlsSSLCertOptionKeyVault) ? sslKeyVaultCustomIdentityKeyStoreType : sslUploadedCustomIdentityKeyStoreType var const_keyvaultNameFromTag = const_hasTags && contains(resourceGroup().tags, name_tagNameForKeyVault) ? resourceGroup().tags.wlsKeyVault : '' var const_trustKeyStoreType = (sslConfigurationAccessOption == const_wlsSSLCertOptionKeyVault) ? sslKeyVaultCustomTrustKeyStoreType : sslUploadedCustomTrustKeyStoreType +var const_wlsJavaOptions = wlsJavaOption == '' ? 'null' : wlsJavaOption var const_wlsSSLCertOptionKeyVault = 'keyVaultStoredConfig' var name_defaultPidDeployment = 'pid' var name_dnsNameforApplicationGateway = '${concat(dnsNameforApplicationGateway, take(utcValue, 6))}' @@ -371,7 +372,7 @@ module wlsDomainDeployment 'modules/setupWebLogicCluster.bicep' = if (!enableCus wlsIdentityKeyStorePassphrase: sslUploadedCustomIdentityKeyStorePassphrase wlsIdentityKeyStoreType: const_defaultKeystoreType wlsImageTag: wlsImageTag - wlsJavaOption: wlsJavaOption + wlsJavaOption: const_wlsJavaOptions wlsMemory: wlsMemory wlsPassword: wlsPassword wlsPrivateKeyAlias: sslUploadedPrivateKeyAlias @@ -432,7 +433,7 @@ module wlsDomainWithCustomSSLDeployment 'modules/setupWebLogicCluster.bicep' = i wlsIdentityKeyStorePassphrase: sslKeyvault.getSecret(name_identityKeyStorePswSecret) wlsIdentityKeyStoreType: const_identityKeyStoreType wlsImageTag: wlsImageTag - wlsJavaOption: wlsJavaOption + wlsJavaOption: const_wlsJavaOptions wlsMemory: wlsMemory wlsPassword: wlsPassword wlsPrivateKeyAlias: sslKeyvault.getSecret(name_privateKeyAliasSecret) @@ -558,8 +559,13 @@ output adminConsoleInternalUrl string = ref_wlsDomainDeployment.outputs.adminSer output adminConsoleExternalUrl string = const_enableNetworking ? networkingDeployment.outputs.adminConsoleExternalUrl : '' output adminConsoleExternalSecuredUrl string = const_enableNetworking ? networkingDeployment.outputs.adminConsoleExternalSecuredUrl : '' // If TLS/SSL enabled, only secured url is working, will not output HTTP url. -output adminRemoteConsoleUrl string = const_enableNetworking && !enableCustomSSL ? networkingDeployment.outputs.adminRemoteConsoleUrl: '' -output adminRemoteConsoleSecuredUrl string = const_enableNetworking ? networkingDeployment.outputs.adminRemoteConsoleSecuredUrl: '' +output adminRemoteConsoleUrl string = const_enableNetworking && !enableCustomSSL ? networkingDeployment.outputs.adminRemoteConsoleUrl : '' +output adminRemoteConsoleSecuredUrl string = const_enableNetworking ? networkingDeployment.outputs.adminRemoteConsoleSecuredUrl : '' +output adminServerT3InternalUrl string = ref_wlsDomainDeployment.outputs.adminServerT3InternalUrl.value +output adminServerT3ExternalUrl string = enableAdminT3Tunneling && const_enableNetworking ? format('{0}://{1}', enableCustomSSL ? 't3s' : 't3', networkingDeployment.outputs.adminServerT3ChannelUrl) : '' output clusterInternalUrl string = ref_wlsDomainDeployment.outputs.clusterSVCUrl.value output clusterExternalUrl string = const_enableNetworking ? networkingDeployment.outputs.clusterExternalUrl : '' output clusterExternalSecuredUrl string = const_enableNetworking ? networkingDeployment.outputs.clusterExternalSecuredUrl : '' +output clusterT3InternalUrl string = ref_wlsDomainDeployment.outputs.clusterT3InternalUrl.value +output clusterT3ExternalUrl string = enableAdminT3Tunneling && const_enableNetworking ? format('{0}://{1}', enableCustomSSL ? 't3s' : 't3', networkingDeployment.outputs.clusterT3ChannelUrl) : '' + diff --git a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep index 2bbcf62bb..76612e55a 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep @@ -75,5 +75,7 @@ resource deploymentScript 'Microsoft.Resources/deploymentScripts@2020-10-01' = { } output adminConsoleLBUrl string = length(lbSvcValues) > 0 && (reference('ds-networking-deployment').outputs.adminConsoleEndpoint != 'null') ? format('http://{0}/',reference('ds-networking-deployment').outputs.adminConsoleEndpoint): '' +output adminServerT3LBUrl string = length(lbSvcValues) > 0 && (reference('ds-networking-deployment').outputs.adminServerT3Endpoint != 'null') ? reference('ds-networking-deployment').outputs.adminServerT3Endpoint: '' output clusterLBUrl string = length(lbSvcValues) > 0 && (reference('ds-networking-deployment').outputs.clusterEndpoint != 'null') ? format('http://{0}/',reference('ds-networking-deployment').outputs.clusterEndpoint): '' +output clusterT3LBUrl string = length(lbSvcValues) > 0 && (reference('ds-networking-deployment').outputs.clusterT3Endpoint != 'null') ? reference('ds-networking-deployment').outputs.clusterT3Endpoint: '' diff --git a/weblogic-azure-aks/src/main/bicep/modules/networking.bicep b/weblogic-azure-aks/src/main/bicep/modules/networking.bicep index ab7620dd2..e9804ae82 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/networking.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/networking.bicep @@ -67,6 +67,7 @@ param wlsDomainUID string = 'sample-domain1' var const_appgwCustomDNSAlias = format('{0}.{1}/', dnszoneClusterLabel, dnszoneName) var const_appgwAdminCustomDNSAlias = format('{0}.{1}/', dnszoneAdminConsoleLabel, dnszoneName) var const_appgwSSLCertOptionGenerateCert = 'generateCert' +var ref_networkDeployment = enableAppGWIngress ? (appGatewayCertificateOption == const_appgwSSLCertOptionGenerateCert ? reference('ds-networking-deployment-1'): reference('ds-networking-deployment')) : reference('ds-networking-deployment-2') module pidNetworkingStart './_pids/_pid.bicep' = { name: 'pid-networking-start-deployment' @@ -273,5 +274,7 @@ output adminConsoleExternalUrl string = enableAppGWIngress ? (enableDNSConfigura output adminConsoleExternalSecuredUrl string = enableAppGWIngress && enableCustomSSL && enableDNSConfiguration ? format('https://{0}console', const_appgwAdminCustomDNSAlias) : '' output adminRemoteConsoleUrl string = enableAppGWIngress ? (enableDNSConfiguration ? format('http://{0}remoteconsole', const_appgwAdminCustomDNSAlias) : format('http://{0}/remoteconsole', appgwDeployment.outputs.appGatewayAlias)) : networkingDeployment3.outputs.adminConsoleLBUrl output adminRemoteConsoleSecuredUrl string = enableAppGWIngress && enableCustomSSL && enableDNSConfiguration ? format('https://{0}remoteconsole', const_appgwAdminCustomDNSAlias) : '' +output adminServerT3ChannelUrl string = ref_networkDeployment.outputs.adminServerT3LBUrl.value output clusterExternalUrl string = enableAppGWIngress ? (enableDNSConfiguration ? format('http://{0}', const_appgwCustomDNSAlias) : appgwDeployment.outputs.appGatewayURL) : networkingDeployment3.outputs.clusterLBUrl output clusterExternalSecuredUrl string = enableAppGWIngress ? (enableDNSConfiguration ? format('https://{0}', const_appgwCustomDNSAlias) : appgwDeployment.outputs.appGatewaySecuredURL) : '' +output clusterT3ChannelUrl string = ref_networkDeployment.outputs.clusterT3LBUrl.value diff --git a/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep b/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep index 87adf1097..f2effe569 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep @@ -240,4 +240,6 @@ module pidEnd './_pids/_pid.bicep' = { output aksClusterName string = createAKSCluster ? aksClusterDeployment.outputs.aksClusterName : aksClusterName output aksClusterRGName string = createAKSCluster ? resourceGroup().name : aksClusterRGName output adminServerUrl string = format('http://{0}-admin-server.{0}-ns.svc.cluster.local:7001/console', wlsDomainUID) +output adminServerT3InternalUrl string = enableAdminT3Tunneling ? format('{0}://{1}-admin-server.{1}-ns.svc.cluster.local:{2}', enableCustomSSL ? 't3s' : 't3', wlsDomainUID, t3ChannelAdminPort): '' output clusterSVCUrl string = format('http://{0}-cluster-cluster-1.{0}-ns.svc.cluster.local:8001/', wlsDomainUID) +output clusterT3InternalUrl string = enableClusterT3Tunneling ? format('{0}://{1}-cluster-cluster-1.{1}-ns.svc.cluster.local:{2}', enableCustomSSL ? 't3s' : 't3', wlsDomainUID, t3ChannelAdminPort): '' From d25908a3d434063de2cfd3f2369bc7c4d1505061 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Wed, 8 Sep 2021 15:17:29 +0800 Subject: [PATCH 09/14] On branch t3tunneling: refactoring Changes to be committed: modified: weblogic-azure-aks/src/main/arm/scripts/common.sh new file: weblogic-azure-aks/src/main/arm/scripts/createAppGatewayIngress.sh new file: weblogic-azure-aks/src/main/arm/scripts/createDnsRecord.sh new file: weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh modified: weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh modified: weblogic-azure-aks/src/main/arm/scripts/utility.sh modified: weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep modified: weblogic-azure-aks/src/main/bicep/modules/networking.bicep modified: weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep Signed-off-by: galiacheng --- .../src/main/arm/scripts/common.sh | 4 + .../arm/scripts/createAppGatewayIngress.sh | 639 ++++++++++ .../src/main/arm/scripts/createDnsRecord.sh | 65 + .../src/main/arm/scripts/createLbSvc.sh | 471 ++++++++ .../src/main/arm/scripts/setupNetworking.sh | 1059 +---------------- .../src/main/arm/scripts/setupWLSDomain.sh | 5 +- .../src/main/arm/scripts/utility.sh | 208 ++-- .../_ds-create-networking.bicep | 6 + .../src/main/bicep/modules/networking.bicep | 12 +- .../bicep/modules/setupWebLogicCluster.bicep | 2 +- 10 files changed, 1373 insertions(+), 1098 deletions(-) create mode 100644 weblogic-azure-aks/src/main/arm/scripts/createAppGatewayIngress.sh create mode 100644 weblogic-azure-aks/src/main/arm/scripts/createDnsRecord.sh create mode 100644 weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh diff --git a/weblogic-azure-aks/src/main/arm/scripts/common.sh b/weblogic-azure-aks/src/main/arm/scripts/common.sh index 406aa4c76..dcf214f3d 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/common.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/common.sh @@ -2,8 +2,12 @@ export checkPodStatusInterval=20 # interval of checking pod status. export checkPodStatusMaxAttemps=30 # max attempt to check pod status. export checkPVStateInterval=5 # interval of checking pvc status. export checkPVStateMaxAttempt=10 # max attempt to check pvc status. +export checkSVCStateMaxAttempt=10 +export checkSVCInterval=30 #seconds export constAdminT3AddressEnvName="T3_TUNNELING_ADMIN_ADDRESS" +export constAdminServerName='admin-server' +export constClusterName='cluster-1' export constClusterT3AddressEnvName="T3_TUNNELING_CLUSTER_ADDRESS" export constFalse="false" export constTrue="true" diff --git a/weblogic-azure-aks/src/main/arm/scripts/createAppGatewayIngress.sh b/weblogic-azure-aks/src/main/arm/scripts/createAppGatewayIngress.sh new file mode 100644 index 000000000..ba1ec299e --- /dev/null +++ b/weblogic-azure-aks/src/main/arm/scripts/createAppGatewayIngress.sh @@ -0,0 +1,639 @@ +# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +# Description: to create Azure Application Gateway ingress for the following targets. +# * [Optional] Admin console, with path host/console +# * [Optional] Admin remote console, with path host/remoteconsole +# * Cluster, with path host/* + +echo "Script ${0} starts" + +# read from stdin +function read_sensitive_parameters_from_stdin() { + read spBase64String appgwFrontendSSLCertPsw +} + +function generate_appgw_cluster_config_file_expose_https() { + clusterIngressHttpsName=${wlsDomainUID}-cluster-appgw-ingress-https-svc + clusterAppgwIngressHttpsYamlPath=${scriptDir}/appgw-cluster-ingress-https-svc.yaml + cat <${clusterAppgwIngressHttpsYamlPath} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ${clusterIngressHttpsName} + namespace: ${wlsDomainNS} + annotations: + kubernetes.io/ingress.class: azure/application-gateway +EOF + + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${clusterAppgwIngressHttpsYamlPath} + appgw.ingress.kubernetes.io/cookie-based-affinity: "true" +EOF + fi + + cat <>${clusterAppgwIngressHttpsYamlPath} +spec: + tls: + - secretName: ${appgwFrontendSecretName} + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ${svcCluster} + port: + number: ${clusterTargetPort} +EOF +} + +function generate_appgw_cluster_config_file_nossl() { + clusterIngressName=${wlsDomainUID}-cluster-appgw-ingress-svc + clusterAppgwIngressYamlPath=${scriptDir}/appgw-cluster-ingress-svc.yaml + cat <${clusterAppgwIngressYamlPath} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ${clusterIngressName} + namespace: ${wlsDomainNS} + annotations: + kubernetes.io/ingress.class: azure/application-gateway +EOF + + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${clusterAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/cookie-based-affinity: "true" +EOF + fi + + cat <>${clusterAppgwIngressYamlPath} +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ${svcCluster} + port: + number: ${clusterTargetPort} +EOF +} + +function generate_appgw_cluster_config_file_ssl() { + clusterIngressName=${wlsDomainUID}-cluster-appgw-ingress-svc + clusterAppgwIngressYamlPath=${scriptDir}/appgw-cluster-ingress-svc.yaml + cat <${clusterAppgwIngressYamlPath} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ${clusterIngressName} + namespace: ${wlsDomainNS} + annotations: + kubernetes.io/ingress.class: azure/application-gateway + appgw.ingress.kubernetes.io/ssl-redirect: "true" + appgw.ingress.kubernetes.io/backend-protocol: "https" +EOF + if [[ "${enableCustomDNSAlias,,}" == "true" ]]; then + cat <>${clusterAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/backend-hostname: "${dnsClusterLabel}.${dnsZoneName}" +EOF + else + cat <>${clusterAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/backend-hostname: "${appgwAlias}" +EOF + fi + + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${clusterAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/cookie-based-affinity: "true" +EOF + fi + + cat <>${clusterAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/appgw-trusted-root-certificate: "${appgwBackendSecretName}" + +spec: + tls: + - secretName: ${appgwFrontendSecretName} + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ${svcCluster} + port: + number: ${clusterTargetPort} +EOF +} + +function generate_appgw_admin_config_file_nossl() { + adminIngressName=${wlsDomainUID}-admin-appgw-ingress-svc + adminAppgwIngressYamlPath=${scriptDir}/appgw-admin-ingress-svc.yaml + cat <${adminAppgwIngressYamlPath} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ${adminIngressName} + namespace: ${wlsDomainNS} + annotations: + kubernetes.io/ingress.class: azure/application-gateway +EOF + + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${adminAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/cookie-based-affinity: "true" +EOF + fi + + cat <>${adminAppgwIngressYamlPath} +spec: + rules: + - http: + paths: + - path: /console* + pathType: Prefix + backend: + service: + name: ${svcAdminServer} + port: + number: ${adminTargetPort} +EOF +} + +function generate_appgw_admin_remote_config_file_nossl() { + cat <${adminRemoteAppgwIngressYamlPath} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ${adminRemoteIngressName} + namespace: ${wlsDomainNS} + annotations: + kubernetes.io/ingress.class: azure/application-gateway + appgw.ingress.kubernetes.io/backend-path-prefix: "/" +EOF + + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${adminRemoteAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/cookie-based-affinity: "true" +EOF + fi + + cat <>${adminRemoteAppgwIngressYamlPath} +spec: + rules: + - http: + paths: + - path: /remoteconsole* + pathType: Prefix + backend: + service: + name: ${svcAdminServer} + port: + number: ${adminTargetPort} +EOF +} + +function generate_appgw_admin_config_file_ssl() { + adminIngressName=${wlsDomainUID}-admin-appgw-ingress-svc + adminAppgwIngressYamlPath=${scriptDir}/appgw-admin-ingress-svc.yaml + cat <${adminAppgwIngressYamlPath} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ${adminIngressName} + namespace: ${wlsDomainNS} + annotations: + kubernetes.io/ingress.class: azure/application-gateway + appgw.ingress.kubernetes.io/ssl-redirect: "true" + appgw.ingress.kubernetes.io/backend-protocol: "https" +EOF + + if [[ "${enableCustomDNSAlias,,}" == "true" ]]; then + cat <>${adminAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/backend-hostname: "${dnsAdminLabel}.${dnsZoneName}" +EOF + else + cat <>${adminAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/backend-hostname: "${appgwAlias}" +EOF + fi + + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${adminAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/cookie-based-affinity: "true" +EOF + fi + + cat <>${adminAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/appgw-trusted-root-certificate: "${appgwBackendSecretName}" + +spec: + tls: + - secretName: ${appgwFrontendSecretName} + rules: + - http: + paths: + - path: /console* + pathType: Prefix + backend: + service: + name: ${svcAdminServer} + port: + number: ${adminTargetPort} +EOF +} + +function generate_appgw_admin_remote_config_file_ssl() { + cat <${adminRemoteAppgwIngressYamlPath} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ${adminRemoteIngressName} + namespace: ${wlsDomainNS} + annotations: + kubernetes.io/ingress.class: azure/application-gateway + appgw.ingress.kubernetes.io/backend-path-prefix: "/" + appgw.ingress.kubernetes.io/ssl-redirect: "true" + appgw.ingress.kubernetes.io/backend-protocol: "https" + +EOF + + if [[ "${enableCustomDNSAlias,,}" == "true" ]]; then + cat <>${adminRemoteAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/backend-hostname: "${dnsAdminLabel}.${dnsZoneName}" +EOF + else + cat <>${adminRemoteAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/backend-hostname: "${appgwAlias}" +EOF + fi + + if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then + cat <>${adminRemoteAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/cookie-based-affinity: "true" +EOF + fi + + cat <>${adminRemoteAppgwIngressYamlPath} + appgw.ingress.kubernetes.io/appgw-trusted-root-certificate: "${appgwBackendSecretName}" + +spec: + tls: + - secretName: ${appgwFrontendSecretName} + rules: + - http: + paths: + - path: /remoteconsole* + pathType: Prefix + backend: + service: + name: ${svcAdminServer} + port: + number: ${adminTargetPort} +EOF +} + +function query_admin_target_port() { + if [[ "${enableCustomSSL,,}" == "true" ]]; then + adminTargetPort=$(utility_query_service_port ${svcAdminServer} ${wlsDomainNS} 'default-secure') + else + adminTargetPort=$(utility_query_service_port ${svcAdminServer} ${wlsDomainNS} 'default') + fi + + echo "Admin port of ${adminServerName}: ${adminTargetPort}" +} + +# Create network peers for aks and appgw +function network_peers_aks_appgw() { + # To successfully peer two virtual networks command 'az network vnet peering create' must be called twice with the values + # for --vnet-name and --remote-vnet reversed. + aksMCRGName=$(az aks show -n $aksClusterName -g $aksClusterRGName -o tsv --query "nodeResourceGroup") + ret=$(az group exists -n ${aksMCRGName}) + if [ "${ret,,}" == "false" ]; then + echo_stderr "AKS namaged resource group ${aksMCRGName} does not exist." + exit 1 + fi + + aksNetWorkId=$(az resource list -g ${aksMCRGName} --resource-type Microsoft.Network/virtualNetworks -o tsv --query '[*].id') + aksNetworkName=$(az resource list -g ${aksMCRGName} --resource-type Microsoft.Network/virtualNetworks -o tsv --query '[*].name') + az network vnet peering create \ + --name aks-appgw-peer \ + --remote-vnet ${aksNetWorkId} \ + --resource-group ${curRGName} \ + --vnet-name ${vnetName} \ + --allow-vnet-access + utility_validate_status "Create network peers for $aksNetWorkId and ${vnetName}." + + appgwNetworkId=$(az resource list -g ${curRGName} --name ${vnetName} -o tsv --query '[*].id') + az network vnet peering create \ + --name aks-appgw-peer \ + --remote-vnet ${appgwNetworkId} \ + --resource-group ${aksMCRGName} \ + --vnet-name ${aksNetworkName} \ + --allow-vnet-access + + utility_validate_status "Create network peers for $aksNetWorkId and ${vnetName}." + + # For Kbectl network plugin: https://azure.github.io/application-gateway-kubernetes-ingress/how-tos/networking/#with-kubenet + # find route table used by aks cluster + routeTableId=$(az network route-table list -g $aksMCRGName --query "[].id | [0]" -o tsv) + + # get the application gateway's subnet + appGatewaySubnetId=$(az network application-gateway show -n $appgwName -g $curRGName -o tsv --query "gatewayIpConfigurations[0].subnet.id") + + # associate the route table to Application Gateway's subnet + az network vnet subnet update \ + --ids $appGatewaySubnetId \ + --route-table $routeTableId + + utility_validate_status "Associate the route table ${routeTableId} to Application Gateway's subnet ${appGatewaySubnetId}" +} + +function query_cluster_target_port() { + if [[ "${enableCustomSSL,,}" == "true" ]]; then + clusterTargetPort=$(utility_query_service_port ${svcCluster} ${wlsDomainNS} 'default-secure') + else + clusterTargetPort=$(utility_query_service_port ${svcCluster} ${wlsDomainNS} 'default') + fi + + echo "Cluster port of ${clusterName}: ${clusterTargetPort}" +} + +function install_azure_ingress() { + # create sa and bind cluster-admin role + # grant azure ingress permission to access WebLogic service + kubectl apply -f ${scriptDir}/appgw-ingress-clusterAdmin-roleBinding.yaml + + install_helm + helm repo add application-gateway-kubernetes-ingress ${appgwIngressHelmRepo} + helm repo update + + # generate Helm config for azure ingress + customAppgwHelmConfig=${scriptDir}/appgw-helm-config.yaml + cp ${scriptDir}/appgw-helm-config.yaml.template ${customAppgwHelmConfig} + subID=${subID#*\/subscriptions\/} + sed -i -e "s:@SUB_ID@:${subID}:g" ${customAppgwHelmConfig} + sed -i -e "s:@APPGW_RG_NAME@:${curRGName}:g" ${customAppgwHelmConfig} + sed -i -e "s:@APPGW_NAME@:${appgwName}:g" ${customAppgwHelmConfig} + sed -i -e "s:@WATCH_NAMESPACE@:${wlsDomainNS}:g" ${customAppgwHelmConfig} + sed -i -e "s:@SP_ENCODING_CREDENTIALS@:${spBase64String}:g" ${customAppgwHelmConfig} + + helm install ingress-azure \ + -f ${customAppgwHelmConfig} \ + application-gateway-kubernetes-ingress/ingress-azure \ + --version ${azureAppgwIngressVersion} + + utility_validate_status "Install app gateway ingress controller." + + attempts=0 + podState="running" + while [ "$podState" == "running" ] && [ $attempts -lt ${checkPodStatusMaxAttemps} ]; do + podState="completed" + attempts=$((attempts + 1)) + echo Waiting for Pod running...${attempts} + sleep ${checkPodStatusInterval} + + ret=$(kubectl get pod -o json | + jq '.items[] | .status.containerStatuses[] | select(.name=="ingress-azure") | .ready') + if [[ "${ret}" == "false" ]]; then + podState="running" + fi + done + + if [ "$podState" == "running" ] && [ $attempts -ge ${checkPodStatusMaxAttemps} ]; then + echo_stderr "Failed to install app gateway ingress controller." + exit 1 + fi +} + +function output_create_gateway_ssl_k8s_secret() { + echo "export gateway frontend certificates" + echo "$appgwFrontendSSLCertData" | base64 -d >${scriptDir}/$appgwFrontCertFileName + + appgwFrontendSSLCertPassin=${appgwFrontendSSLCertPsw} + if [[ "$appgwCertificateOption" == "${appgwSelfsignedCert}" ]]; then + appgwFrontendSSLCertPassin="" # empty password + fi + + openssl pkcs12 \ + -in ${scriptDir}/$appgwFrontCertFileName \ + -nocerts \ + -out ${scriptDir}/$appgwFrontCertKeyFileName \ + -passin pass:${appgwFrontendSSLCertPassin} \ + -passout pass:${appgwFrontendSSLCertPsw} + + utility_validate_status "Export key from frontend certificate." + + openssl rsa -in ${scriptDir}/$appgwFrontCertKeyFileName \ + -out ${scriptDir}/$appgwFrontCertKeyDecrytedFileName \ + -passin pass:${appgwFrontendSSLCertPsw} + + utility_validate_status "Decryte private key." + + openssl pkcs12 \ + -in ${scriptDir}/$appgwFrontCertFileName \ + -clcerts \ + -nokeys \ + -out ${scriptDir}/$appgwFrontPublicCertFileName \ + -passin pass:${appgwFrontendSSLCertPassin} + + utility_validate_status "Export cert from frontend certificate." + + echo "create k8s tsl secret for app gateway frontend ssl termination" + kubectl -n ${wlsDomainNS} create secret tls ${appgwFrontendSecretName} \ + --key="${scriptDir}/$appgwFrontCertKeyDecrytedFileName" \ + --cert="${scriptDir}/$appgwFrontPublicCertFileName" + + utility_validate_status "create k8s tsl secret for app gateway frontend ssl termination." +} + +function validate_backend_ca_cert() { + az network application-gateway root-cert list \ + --gateway-name $appgwName \ + --resource-group $curRGName | + jq '.[] | .name' | grep "${appgwBackendSecretName}" + + utility_validate_status "check if backend cert exists." +} + +function generate_appgw_cluster_config_file() { + if [[ "${enableCustomSSL,,}" == "true" ]]; then + generate_appgw_cluster_config_file_ssl + else + generate_appgw_cluster_config_file_nossl + generate_appgw_cluster_config_file_expose_https + fi +} + +function generate_appgw_admin_config_file() { + if [[ "${enableCustomSSL,,}" == "true" ]]; then + generate_appgw_admin_config_file_ssl + else + generate_appgw_admin_config_file_nossl + fi +} + +function generate_appgw_admin_remote_config_file() { + if [[ "${enableCustomSSL,,}" == "true" ]]; then + generate_appgw_admin_remote_config_file_ssl + else + generate_appgw_admin_remote_config_file_nossl + fi +} + +function appgw_ingress_svc_for_cluster() { + # generate ingress svc config for cluster + generate_appgw_cluster_config_file + kubectl apply -f ${clusterAppgwIngressYamlPath} + utility_validate_status "Create appgw ingress svc." + utility_waitfor_lb_svc_completed \ + ${clusterIngressName} \ + ${wlsDomainNS} \ + ${checkSVCStateMaxAttempt} \ + ${checkSVCInterval} + + # expose https for cluster if e2e ssl is not set up. + if [[ "${enableCustomSSL,,}" != "true" ]]; then + kubectl apply -f ${clusterAppgwIngressHttpsYamlPath} + utility_validate_status "Create appgw ingress https svc." + utility_waitfor_lb_svc_completed \ + ${clusterIngressHttpsName} \ + ${wlsDomainNS} \ + ${checkSVCStateMaxAttempt} \ + ${checkSVCInterval} + fi +} + +function appgw_ingress_svc_for_admin_server() { + generate_appgw_admin_config_file + kubectl apply -f ${adminAppgwIngressYamlPath} + utility_validate_status "Create appgw ingress svc." + utility_waitfor_lb_svc_completed \ + ${adminIngressName} \ + ${wlsDomainNS} \ + ${checkSVCStateMaxAttempt} \ + ${checkSVCInterval} +} + +function appgw_ingress_svc_for_remote_console() { + adminRemoteIngressName=${wlsDomainUID}-admin-remote-appgw-ingress-svc + adminRemoteAppgwIngressYamlPath=${scriptDir}/appgw-admin-remote-ingress-svc.yaml + generate_appgw_admin_remote_config_file + + kubectl apply -f ${adminRemoteAppgwIngressYamlPath} + utility_validate_status "Create appgw ingress svc." + utility_waitfor_lb_svc_completed \ + ${adminRemoteIngressName} \ + ${wlsDomainNS} \ + ${checkSVCStateMaxAttempt} \ + ${checkSVCInterval} +} + +function create_dns_record() { + if [[ "${enableCustomDNSAlias,,}" == "true" ]]; then + create_dns_CNAME_record \ + ${appgwAlias} \ + ${dnsClusterLabel} \ + ${dnsRGName} \ + ${dnsZoneName} + fi + + if [[ "${enableCustomDNSAlias,,}" == "true" ]] && + [[ "${appgwForAdminServer,,}" == "true" ]]; then + create_dns_CNAME_record \ + ${appgwAlias} \ + ${dnsAdminLabel} \ + ${dnsRGName} \ + ${dnsZoneName} + fi +} + +function create_gateway_ingress() { + # query admin server port used for non-ssl or ssl + query_admin_target_port + # query cluster port used for non-ssl or ssl + query_cluster_target_port + # create network peers between gateway vnet and aks vnet + network_peers_aks_appgw + # install azure ingress controllor + install_azure_ingress + # create tsl/ssl frontend secrets + output_create_gateway_ssl_k8s_secret + + # validate backend CA certificate + # the certificate has been upload to Application Gateway in + # weblogic-azure-aks\src\main\bicep\modules\networking.bicep + if [[ "${enableCustomSSL,,}" == "true" ]]; then + validate_backend_ca_cert + fi + + # create ingress svc for cluster + appgw_ingress_svc_for_cluster + + # create ingress svc for admin console + if [[ "${appgwForAdminServer,,}" == "true" ]]; then + appgw_ingress_svc_for_admin_server + fi + + # create ingress svc for admin remote console + if [[ "${enableRemoteConsole,,}" == "true" ]]; then + appgw_ingress_svc_for_remote_console + fi + + create_dns_record +} + +# Initialize +script="${BASH_SOURCE[0]}" +scriptDir="$(cd "$(dirname "${script}")" && pwd)" + +source ${scriptDir}/common.sh +source ${scriptDir}/utility.sh +source ${scriptDir}/createDnsRecord.sh + +aksClusterRGName=$1 +aksClusterName=$2 +wlsDomainUID=$3 +subID=$4 +curRGName=$5 +appgwName=$6 +vnetName=$7 +appgwForAdminServer=$8 +enableCustomDNSAlias=$9 +dnsRGName=${10} +dnsZoneName=${11} +dnsAdminLabel=${12} +dnsClusterLabel=${13} +appgwAlias=${14} +appgwFrontendSSLCertData=${15} +appgwCertificateOption=${16} +enableCustomSSL=${17} +enableCookieBasedAffinity=${18} +enableRemoteConsole=${19} + +adminServerName=${constAdminServerName} # define in common.sh +appgwIngressHelmRepo="https://appgwingress.blob.core.windows.net/ingress-azure-helm-package/" +appgwFrontCertFileName="appgw-frontend-cert.pfx" +appgwFrontCertKeyDecrytedFileName="appgw-frontend-cert.key" +appgwFrontCertKeyFileName="appgw-frontend-cert-decryted.key" +appgwFrontPublicCertFileName="appgw-frontend-cert.crt" +appgwFrontendSecretName="frontend-tls" +appgwBackendSecretName="backend-tls" +appgwSelfsignedCert="generateCert" +azureAppgwIngressVersion="1.4.0" +clusterName=${constClusterName} +httpsListenerName="myHttpsListenerName$(date +%s)" +httpsRuleName="myHttpsRule$(date +%s)" +svcAdminServer="${wlsDomainUID}-${adminServerName}" +svcCluster="${wlsDomainUID}-cluster-${clusterName}" +wlsDomainNS="${wlsDomainUID}-ns" + +read_sensitive_parameters_from_stdin + +create_gateway_ingress diff --git a/weblogic-azure-aks/src/main/arm/scripts/createDnsRecord.sh b/weblogic-azure-aks/src/main/arm/scripts/createDnsRecord.sh new file mode 100644 index 000000000..abe94ae47 --- /dev/null +++ b/weblogic-azure-aks/src/main/arm/scripts/createDnsRecord.sh @@ -0,0 +1,65 @@ +# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +# Description: to create DNS record in an existing DNS Zone. + +echo "Script ${0} starts" + +# Initialize +export script="${BASH_SOURCE[0]}" +export scriptDir="$(cd "$(dirname "${script}")" && pwd)" + +source ${scriptDir}/utility.sh + +# create dns alias for lb service +# $1: ipv4 address +# $2: label of subdomain +# $3: resource group name that has the DNS Zone. +# $4: DNS Zone name +function create_dns_A_record() { + ipv4Addr=$1 + label=$2 + dnsRGName=$3 + dnsZoneName=$4 + + az network dns record-set a add-record --ipv4-address ${ipv4Addr} \ + --if-none-match \ + --record-set-name ${label} \ + --resource-group ${dnsRGName} \ + --zone-name ${dnsZoneName} + + if [ $? != 0 ]; then + echo_stderr "Failed to create DNS record: ${label}.${dnsZoneName}, ipv4: ${ipv4Addr}" + exit 1 + fi +} + +# create dns alias for app gateway +# $1: ipv4 address +# $2: label of subdomain +# $3: resource group name that has the DNS Zone. +# $4: DNS Zone name +function create_dns_CNAME_record() { + cname=$1 + label=$2 + dnsRGName=$3 + dnsZoneName=$4 + + az network dns record-set cname create \ + --if-none-match \ + -g ${dnsRGName} \ + -z ${dnsZoneName} \ + -n ${label} + + az network dns record-set cname set-record \ + --if-none-match \ + -g ${dnsRGName} \ + -z ${dnsZoneName} \ + --cname ${cname} \ + --record-set-name ${label} + + if [ $? != 0 ]; then + echo_stderr "Failed to create DNS record: ${label}.${dnsZoneName}, cname: ${cname}" + exit 1 + fi +} diff --git a/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh b/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh new file mode 100644 index 000000000..9ea912f66 --- /dev/null +++ b/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh @@ -0,0 +1,471 @@ +# Copyright (c) 2021, Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. + +# Description: to create Load Balancer Service for the following targets. +# * [Optional] admin server default channel +# * [Optional] admin server T3 channel +# * [Optional] cluster default channel +# * [Optional] cluster T3 channel +# +# Special parameter example: +# * lbSvcValues: [{"colName":"admin-t3","colTarget":"adminServerT3","colPort":"7005"},{"colName":"cluster","colTarget":"cluster1T3","colPort":"8011"}] + +echo "Script ${0} starts" + +function generate_admin_lb_definicion() { + cat <${scriptDir}/admin-server-lb.yaml +apiVersion: v1 +kind: Service +metadata: + name: ${adminServerLBSVCName} + namespace: ${wlsDomainNS} +EOF + + # to create internal load balancer service + if [[ "${enableInternalLB,,}" == "true" ]]; then + cat <>${scriptDir}/admin-server-lb.yaml + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" +EOF + fi + + cat <>${scriptDir}/admin-server-lb.yaml +spec: + ports: + - name: default + port: ${adminLBPort} + protocol: TCP + targetPort: ${adminTargetPort} + selector: + weblogic.domainUID: ${wlsDomainUID} + weblogic.serverName: ${adminServerName} + sessionAffinity: None + type: LoadBalancer +EOF +} + +function generate_admin_t3_lb_definicion() { + cat <${adminServerT3LBDefinitionPath} +apiVersion: v1 +kind: Service +metadata: + name: ${adminServerT3LBSVCName} + namespace: ${wlsDomainNS} +EOF + + # to create internal load balancer service + if [[ "${enableInternalLB,,}" == "true" ]]; then + cat <>${adminServerT3LBDefinitionPath} + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" +EOF + fi + + cat <>${adminServerT3LBDefinitionPath} +spec: + ports: + - name: default + port: ${adminT3LBPort} + protocol: TCP + targetPort: ${adminT3Port} + selector: + weblogic.domainUID: ${wlsDomainUID} + weblogic.serverName: ${adminServerName} + sessionAffinity: None + type: LoadBalancer +EOF +} + +function generate_cluster_lb_definicion() { + cat <${scriptDir}/cluster-lb.yaml +apiVersion: v1 +kind: Service +metadata: + name: ${clusterLBSVCName} + namespace: ${wlsDomainNS} +EOF + + # to create internal load balancer service + if [[ "${enableInternalLB,,}" == "true" ]]; then + cat <>${scriptDir}/cluster-lb.yaml + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" +EOF + fi + + cat <>${scriptDir}/cluster-lb.yaml +spec: + ports: + - name: default + port: ${clusterLBPort} + protocol: TCP + targetPort: ${clusterTargetPort} + selector: + weblogic.domainUID: ${wlsDomainUID} + weblogic.clusterName: ${clusterName} + sessionAffinity: None + type: LoadBalancer +EOF +} + +function generate_cluster_t3_lb_definicion() { + cat <${clusterT3LBDefinitionPath} +apiVersion: v1 +kind: Service +metadata: + name: ${clusterT3LBSVCName} + namespace: ${wlsDomainNS} +EOF + + # to create internal load balancer service + if [[ "${enableInternalLB,,}" == "true" ]]; then + cat <>${clusterT3LBDefinitionPath} + annotations: + service.beta.kubernetes.io/azure-load-balancer-internal: "true" +EOF + fi + + cat <>${clusterT3LBDefinitionPath} +spec: + ports: + - name: default + port: ${clusterT3LBPort} + protocol: TCP + targetPort: ${clusterT3Port} + selector: + weblogic.domainUID: ${wlsDomainUID} + weblogic.clusterName: ${clusterName} + sessionAffinity: None + type: LoadBalancer +EOF +} + +function query_admin_target_port() { + if [[ "${enableCustomSSL,,}" == "true" ]]; then + adminTargetPort=$(utility_query_service_port ${svcAdminServer} ${wlsDomainNS} 'default-secure') + else + adminTargetPort=$(utility_query_service_port ${svcAdminServer} ${wlsDomainNS} 'default') + fi + + echo "Admin port of ${adminServerName}: ${adminTargetPort}" +} + +function query_cluster_target_port() { + if [[ "${enableCustomSSL,,}" == "true" ]]; then + clusterTargetPort=$(utility_query_service_port ${svcCluster} ${wlsDomainNS} 'default-secure') + else + clusterTargetPort=$(utility_query_service_port ${svcCluster} ${wlsDomainNS} 'default') + fi + + echo "Cluster port of ${clusterName}: ${clusterTargetPort}" +} + +function create_lb_svc_for_admin_server_default_channel() { + item=$1 # input values + + echo ${item} + + adminServerLBSVCNamePrefix=$(cut -d',' -f1 <<<$item) + adminServerLBSVCName="${adminServerLBSVCNamePrefix}-svc-lb-admin" + adminLBPort=$(cut -d',' -f3 <<<$item) + + generate_admin_lb_definicion + + kubectl apply -f ${scriptDir}/admin-server-lb.yaml + utility_validate_status "create lb service for admin server" + utility_waitfor_lb_svc_completed ${adminServerLBSVCName} \ + ${wlsDomainNS} \ + ${checkSVCStateMaxAttempt} \ + ${checkSVCInterval} + + adminServerEndpoint=$(kubectl get svc ${adminServerLBSVCName} -n ${wlsDomainNS} \ + -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') + adminConsoleEndpoint="${adminServerEndpoint}/console" + + if [ "${enableCustomDNSAlias,,}" == "true" ]; then + create_dns_A_record "${adminServerEndpoint%%:*}" ${dnsAdminLabel} ${dnsRGName} ${dnsZoneName} + adminConsoleEndpoint="${dnsAdminLabel}.${dnsZoneName}:${adminServerEndpoint#*:}/console" + fi +} + +function create_lb_svc_for_admin_t3_channel() { + item=$1 # input values + + adminServerT3LBSVCNamePrefix=$(cut -d',' -f1 <<<$item) + adminServerT3LBSVCName="${adminServerT3LBSVCNamePrefix}-svc-t3-lb-admin" + adminT3LBPort=$(cut -d',' -f3 <<<$item) + + adminServerT3LBDefinitionPath=${scriptDir}/admin-server-t3-lb.yaml + generate_admin_t3_lb_definicion + + kubectl apply -f ${adminServerT3LBDefinitionPath} + utility_validate_status "create lb service for admin server t3 channel" + utility_waitfor_lb_svc_completed ${adminServerT3LBSVCName} \ + ${wlsDomainNS} \ + ${checkSVCStateMaxAttempt} \ + ${checkSVCInterval} + + adminServerT3Endpoint=$(kubectl get svc ${adminServerT3LBSVCName} -n ${wlsDomainNS} \ + -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') + + if [ "${enableCustomDNSAlias,,}" == "true" ]; then + create_dns_A_record "${adminServerT3Endpoint%%:*}" "${dnszoneAdminT3ChannelLabel}" ${dnsRGName} ${dnsZoneName} + adminServerT3Endpoint="${dnszoneAdminT3ChannelLabel}.${dnsZoneName}:${adminServerT3Endpoint#*:}" + fi +} + +function create_lb_svc_for_cluster_default_channel() { + item=$1 # input values + + clusterLBSVCNamePrefix=$(cut -d',' -f1 <<<$item) + clusterLBSVCName="${clusterLBSVCNamePrefix}-svc-lb-cluster" + clusterLBPort=$(cut -d',' -f3 <<<$item) + + generate_cluster_lb_definicion + + kubectl apply -f ${scriptDir}/cluster-lb.yaml + utility_validate_status "create lb service for cluster" + utility_waitfor_lb_svc_completed ${clusterLBSVCName} \ + ${wlsDomainNS} \ + ${checkSVCStateMaxAttempt} \ + ${checkSVCInterval} + + clusterEndpoint=$(kubectl get svc ${clusterLBSVCName} -n ${wlsDomainNS} -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') + + if [ "${enableCustomDNSAlias,,}" == "true" ]; then + create_dns_A_record "${clusterEndpoint%%:*}" ${dnsClusterLabel} ${dnsRGName} ${dnsZoneName} + clusterEndpoint="${dnsClusterLabel}.${dnsZoneName}:${clusterEndpoint#*:}/" + fi +} + +function create_lb_svc_for_cluster_t3_channel() { + item=$1 # input values + + clusterT3LBSVCNamePrefix=$(cut -d',' -f1 <<<$item) + clusterT3LBSVCName="${clusterT3LBSVCNamePrefix}-svc-lb-cluster" + clusterT3LBPort=$(cut -d',' -f3 <<<$item) + + clusterT3LBDefinitionPath=${scriptDir}/cluster-t3-lb.yaml + generate_cluster_t3_lb_definicion + + kubectl apply -f ${clusterT3LBDefinitionPath} + utility_validate_status "create lb service for cluster t3 channel" + utility_waitfor_lb_svc_completed ${clusterT3LBSVCName} \ + ${wlsDomainNS} \ + ${checkSVCStateMaxAttempt} \ + ${checkSVCInterval} + + clusterT3Endpoint=$(kubectl get svc ${clusterT3LBSVCName} -n ${wlsDomainNS} \ + -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') + + if [ "${enableCustomDNSAlias,,}" == "true" ]; then + create_dns_A_record "${clusterT3Endpoint%%:*}" ${dnszoneClusterT3ChannelLabel} ${dnsRGName} ${dnsZoneName} + clusterT3Endpoint="${dnszoneClusterT3ChannelLabel}.${dnsZoneName}:${clusterT3Endpoint#*:}" + fi +} + +function patch_admin_t3_public_address() { + # patch admin t3 public address + if [ "${enableCustomDNSAlias,,}" == "true" ]; then + adminT3Address="${dnszoneAdminT3ChannelLabel}.${dnsZoneName}" + else + adminT3Address=$(kubectl -n ${wlsDomainNS} get svc ${adminServerT3LBSVCName} -o json | + jq '. | .status.loadBalancer.ingress[0].ip' | + tr -d "\"") + fi + + if [ $? == 1 ]; then + echo_stderr "Failed to query public IP of admin t3 channel." + fi + + currentDomainConfig=$(echo ${currentDomainConfig} | + jq \ + --arg match "${constAdminT3AddressEnvName}" \ + --arg replace "${adminT3Address}" \ + '.spec.serverPod.env |= map(if .name==$match then (.value=$replace) else . end)') +} + +function patch_cluster_t3_public_address() { + #patch cluster t3 pubilc address + if [ "${enableCustomDNSAlias,,}" == "true" ]; then + clusterT3Adress="${dnszoneClusterT3ChannelLabel}.${dnsZoneName}" + else + clusterT3Adress=$(kubectl -n ${wlsDomainNS} get svc ${clusterT3LBSVCName} -o json | + jq '. | .status.loadBalancer.ingress[0].ip' | + tr -d "\"") + fi + + if [ $? == 1 ]; then + echo_stderr "Failed to query public IP of cluster t3 channel." + fi + + currentDomainConfig=$(echo ${currentDomainConfig} | + jq \ + --arg match "${constClusterT3AddressEnvName}" \ + --arg replace "${clusterT3Adress}" \ + '.spec.serverPod.env |= map(if .name==$match then (.value=$replace) else . end)') +} + +function rolling_update_with_t3_public_address() { + timestampBeforePatchingDomain=$(date +%s) + currentDomainConfig=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json) + cat <${scriptDir}/domainPreviousConfiguration.yaml +${currentDomainConfig} +EOF + + # update public address of t3 channel + if [[ "${enableAdminT3Channel,,}" == "true" ]]; then + patch_admin_t3_public_address + fi + + if [[ "${enableClusterT3Channel,,}" == "true" ]]; then + patch_cluster_t3_public_address + fi + + if [[ "${enableClusterT3Channel,,}" == "true" ]] || [[ "${enableAdminT3Channel,,}" == "true" ]]; then + # restart cluster + restartVersion=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json | + jq '. | .spec.restartVersion' | + tr -d "\"") + restartVersion=$((restartVersion + 1)) + + currentDomainConfig=$(echo ${currentDomainConfig} | + jq \ + --arg version "${restartVersion}" \ + '.spec.restartVersion |= $version') + + echo "rolling restart the cluster with t3 public address." + # echo the configuration for debugging + cat <${scriptDir}/domainNewConfiguration.yaml +${currentDomainConfig} +EOF + echo ${currentDomainConfig} | kubectl -n ${wlsDomainNS} apply -f - + + replicas=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json | + jq '. | .spec.clusters[] | .replicas') + + # wait for the restart completed. + utility_wait_for_pod_restarted \ + ${timestampBeforePatchingDomain} \ + ${replicas} \ + ${wlsDomainUID} \ + ${checkPodStatusMaxAttemps} \ + ${checkPodStatusInterval} + + utility_wait_for_pod_completed \ + ${replicas} \ + ${wlsDomainNS} \ + ${checkPodStatusMaxAttemps} \ + ${checkPodStatusInterval} + fi +} + +#Output value to deployment scripts +function output_result() { + echo ${adminConsoleEndpoint} + echo ${clusterEndpoint} + echo ${adminServerT3Endpoint} + echo ${clusterT3Endpoint} + + result=$(jq -n -c \ + --arg adminEndpoint $adminConsoleEndpoint \ + --arg clusterEndpoint $clusterEndpoint \ + --arg adminT3Endpoint $adminServerT3Endpoint \ + --arg clusterT3Endpoint $clusterT3Endpoint \ + '{adminConsoleEndpoint: $adminEndpoint, clusterEndpoint: $clusterEndpoint, adminServerT3Endpoint: $adminT3Endpoint, clusterT3Endpoint: $clusterT3Endpoint}') + echo "result is: $result" + echo $result >$AZ_SCRIPTS_OUTPUT_PATH +} + +function create_svc_lb() { + query_admin_target_port + query_cluster_target_port + + # Parse lb svc input values + # Generate valid json + ret=$(echo $lbSvcValues | sed "s/\:/\\\"\:\\\"/g" | + sed "s/{/{\"/g" | + sed "s/}/\"}/g" | + sed "s/,/\",\"/g" | + sed "s/}\",\"{/},{/g" | + tr -d \(\)) + + cat <${scriptDir}/lbConfiguration.json +${ret} +EOF + + array=$(jq -r '.[] | "\(.colName),\(.colTarget),\(.colPort)"' ${scriptDir}/lbConfiguration.json) + for item in $array; do + # LB config for admin-server + target=$(cut -d',' -f2 <<<$item) + if [[ "${target}" == "adminServer" ]]; then + create_lb_svc_for_admin_server_default_channel ${item} + elif [[ "${target}" == "cluster1" ]]; then + create_lb_svc_for_cluster_default_channel ${item} + elif [[ "${target}" == "adminServerT3" ]]; then + echo "query admin t3 port" + adminT3Port=$(utility_query_service_port ${svcAdminServer} ${wlsDomainNS} 't3channel') + adminT3sPort=$(utility_query_service_port ${svcAdminServer} ${wlsDomainNS} 't3schannel') + + if [[ "${adminT3Port}" == "null" ]] && [[ "${adminT3sPort}" == "null" ]]; then + continue + fi + + if [[ "${adminT3sPort}" != "null" ]]; then + adminT3Port=${adminT3sPort} + fi + + create_lb_svc_for_admin_t3_channel $item + enableAdminT3Channel=true + elif [[ "${target}" == "cluster1T3" ]]; then + echo "query cluster t3 port" + clusterT3Port=$(utility_query_service_port ${svcCluster} ${wlsDomainNS} 't3channel') + clusterT3sPort=$(utility_query_service_port ${svcCluster} ${wlsDomainNS} 't3schannel') + + if [[ "${clusterT3Port}" == "null" ]] && [[ "${clusterT3sPort}" == "null" ]]; then + continue + fi + + if [[ "${clusterT3sPort}" != "null" ]]; then + clusterT3Port=${clusterT3sPort} + fi + + create_lb_svc_for_cluster_t3_channel ${item} + enableClusterT3Channel=true + fi + done + + rolling_update_with_t3_public_address +} + +# Initialize +script="${BASH_SOURCE[0]}" +scriptDir="$(cd "$(dirname "${script}")" && pwd)" + +source ${scriptDir}/common.sh +source ${scriptDir}/utility.sh +source ${scriptDir}/createDnsRecord.sh + +enableInternalLB=$1 +enableCustomSSL=$2 +enableCustomDNSAlias=$3 +dnsRGName=$4 +dnsZoneName=$5 +dnsAdminLabel=$6 +dnszoneAdminT3ChannelLabel=$7 +dnsClusterLabel=$8 +dnszoneClusterT3ChannelLabel=$9 +lbSvcValues=${10} +wlsDomainUID=${11} + +adminServerName=${constAdminServerName} # define in common.sh +clusterName=${constClusterName} +svcAdminServer="${wlsDomainUID}-${adminServerName}" +svcCluster="${wlsDomainUID}-cluster-${clusterName}" +wlsDomainNS="${wlsDomainUID}-ns" + +echo ${lbSvcValues} + +create_svc_lb + +output_result diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh index 0e3aeb8cf..0fb9500e7 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupNetworking.sh @@ -8,23 +8,6 @@ function read_sensitive_parameters_from_stdin() { read spBase64String appgwFrontendSSLCertPsw } -function install_helm() { - # Install Helm - browserURL=$(curl -m ${curlMaxTime} -s https://api.github.com/repos/helm/helm/releases/latest | - grep "browser_download_url.*linux-amd64.tar.gz.asc" | - cut -d : -f 2,3 | - tr -d \") - helmLatestVersion=${browserURL#*download\/} - helmLatestVersion=${helmLatestVersion%%\/helm*} - helmPackageName=helm-${helmLatestVersion}-linux-amd64.tar.gz - curl -m ${curlMaxTime} -fL https://get.helm.sh/${helmPackageName} -o /tmp/${helmPackageName} - tar -zxvf /tmp/${helmPackageName} -C /tmp - mv /tmp/linux-amd64/helm /usr/local/bin/helm - echo "Helm version" - helm version - validate_status "Finished installing Helm." -} - # Install latest kubectl and Helm function install_utilities() { if [ -d "apps" ]; then @@ -35,27 +18,7 @@ function install_utilities() { cd apps # Install kubectl - az aks install-cli - echo "kubectl version" - ret=$(kubectl --help) - validate_status ${ret} -} - -#Output value to deployment scripts -function output_result() { - echo ${adminConsoleEndpoint} - echo ${clusterEndpoint} - echo ${adminServerT3Endpoint} - echo ${clusterT3Endpoint} - - result=$(jq -n -c \ - --arg adminEndpoint $adminConsoleEndpoint \ - --arg clusterEndpoint $clusterEndpoint \ - --arg adminT3Endpoint $adminServerT3Endpoint \ - --arg clusterT3Endpoint $clusterT3Endpoint \ - '{adminConsoleEndpoint: $adminEndpoint, clusterEndpoint: $clusterEndpoint, adminServerT3Endpoint: $adminT3Endpoint, clusterT3Endpoint: $clusterT3Endpoint}') - echo "result is: $result" - echo $result >$AZ_SCRIPTS_OUTPUT_PATH + install_kubectl } #Function to display usage message @@ -99,36 +62,6 @@ END fi } -#Validate teminal status with $?, exit with exception if errors happen. -function validate_status() { - if [ $? == 1 ]; then - echo_stderr "$@" - echo_stderr "Errors happen, exit 1." - exit 1 - else - echo_stdout "$@" - fi -} - -function waitfor_svc_completed() { - svcName=$1 - - attempts=0 - svcState="running" - while [ ! "$svcState" == "completed" ] && [ $attempts -lt ${perfSVCAttemps} ]; do - svcState="completed" - attempts=$((attempts + 1)) - echo Waiting for job completed...${attempts} - sleep ${perfRetryInterval} - - ret=$(kubectl get svc ${svcName} -n ${wlsDomainNS} | - grep -c "Running") - if [ -z "${ret}" ]; then - svcState="running" - fi - done -} - #Function to validate input function validate_input() { if [[ -z "$aksClusterRGName" || -z "${aksClusterName}" ]]; then @@ -242,944 +175,55 @@ function validate_input() { fi } -function generate_admin_lb_definicion() { - cat <${scriptDir}/admin-server-lb.yaml -apiVersion: v1 -kind: Service -metadata: - name: ${adminServerLBSVCName} - namespace: ${wlsDomainNS} -EOF - - # to create internal load balancer service - if [[ "${enableInternalLB,,}" == "true" ]]; then - cat <>${scriptDir}/admin-server-lb.yaml - annotations: - service.beta.kubernetes.io/azure-load-balancer-internal: "true" -EOF - fi - - cat <>${scriptDir}/admin-server-lb.yaml -spec: - ports: - - name: default - port: ${adminLBPort} - protocol: TCP - targetPort: ${adminTargetPort} - selector: - weblogic.domainUID: ${wlsDomainUID} - weblogic.serverName: ${adminServerName} - sessionAffinity: None - type: LoadBalancer -EOF -} - -function generate_admin_t3_lb_definicion() { - cat <${adminServerT3LBDefinitionPath} -apiVersion: v1 -kind: Service -metadata: - name: ${adminServerT3LBSVCName} - namespace: ${wlsDomainNS} -EOF - - # to create internal load balancer service - if [[ "${enableInternalLB,,}" == "true" ]]; then - cat <>${adminServerT3LBDefinitionPath} - annotations: - service.beta.kubernetes.io/azure-load-balancer-internal: "true" -EOF - fi - - cat <>${adminServerT3LBDefinitionPath} -spec: - ports: - - name: default - port: ${adminT3LBPort} - protocol: TCP - targetPort: ${adminT3Port} - selector: - weblogic.domainUID: ${wlsDomainUID} - weblogic.serverName: ${adminServerName} - sessionAffinity: None - type: LoadBalancer -EOF -} - -function generate_cluster_lb_definicion() { - cat <${scriptDir}/cluster-lb.yaml -apiVersion: v1 -kind: Service -metadata: - name: ${clusterLBSVCName} - namespace: ${wlsDomainNS} -EOF - - # to create internal load balancer service - if [[ "${enableInternalLB,,}" == "true" ]]; then - cat <>${scriptDir}/cluster-lb.yaml - annotations: - service.beta.kubernetes.io/azure-load-balancer-internal: "true" -EOF - fi - - cat <>${scriptDir}/cluster-lb.yaml -spec: - ports: - - name: default - port: ${clusterLBPort} - protocol: TCP - targetPort: ${clusterTargetPort} - selector: - weblogic.domainUID: ${wlsDomainUID} - weblogic.clusterName: ${clusterName} - sessionAffinity: None - type: LoadBalancer -EOF -} - -function generate_cluster_t3_lb_definicion() { - cat <${clusterT3LBDefinitionPath} -apiVersion: v1 -kind: Service -metadata: - name: ${clusterT3LBSVCName} - namespace: ${wlsDomainNS} -EOF - - # to create internal load balancer service - if [[ "${enableInternalLB,,}" == "true" ]]; then - cat <>${clusterT3LBDefinitionPath} - annotations: - service.beta.kubernetes.io/azure-load-balancer-internal: "true" -EOF - fi - - cat <>${clusterT3LBDefinitionPath} -spec: - ports: - - name: default - port: ${clusterT3LBPort} - protocol: TCP - targetPort: ${clusterT3Port} - selector: - weblogic.domainUID: ${wlsDomainUID} - weblogic.clusterName: ${clusterName} - sessionAffinity: None - type: LoadBalancer -EOF -} - -function generate_appgw_cluster_config_file_expose_https() { - export clusterIngressHttpsName=${wlsDomainUID}-cluster-appgw-ingress-https-svc - export clusterAppgwIngressHttpsYamlPath=${scriptDir}/appgw-cluster-ingress-https-svc.yaml - cat <${clusterAppgwIngressHttpsYamlPath} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ${clusterIngressHttpsName} - namespace: ${wlsDomainNS} - annotations: - kubernetes.io/ingress.class: azure/application-gateway -EOF - - if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then - cat <>${clusterAppgwIngressHttpsYamlPath} - appgw.ingress.kubernetes.io/cookie-based-affinity: "true" -EOF - fi - - cat <>${clusterAppgwIngressHttpsYamlPath} -spec: - tls: - - secretName: ${appgwFrontendSecretName} - rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: ${svcCluster} - port: - number: ${clusterTargetPort} -EOF -} - -function generate_appgw_cluster_config_file_nossl() { - export clusterIngressName=${wlsDomainUID}-cluster-appgw-ingress-svc - export clusterAppgwIngressYamlPath=${scriptDir}/appgw-cluster-ingress-svc.yaml - cat <${clusterAppgwIngressYamlPath} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ${clusterIngressName} - namespace: ${wlsDomainNS} - annotations: - kubernetes.io/ingress.class: azure/application-gateway -EOF - - if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then - cat <>${clusterAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/cookie-based-affinity: "true" -EOF - fi - - cat <>${clusterAppgwIngressYamlPath} -spec: - rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: ${svcCluster} - port: - number: ${clusterTargetPort} -EOF -} - -function generate_appgw_cluster_config_file_ssl() { - export clusterIngressName=${wlsDomainUID}-cluster-appgw-ingress-svc - export clusterAppgwIngressYamlPath=${scriptDir}/appgw-cluster-ingress-svc.yaml - cat <${clusterAppgwIngressYamlPath} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ${clusterIngressName} - namespace: ${wlsDomainNS} - annotations: - kubernetes.io/ingress.class: azure/application-gateway - appgw.ingress.kubernetes.io/ssl-redirect: "true" - appgw.ingress.kubernetes.io/backend-protocol: "https" -EOF - if [[ "${enableCustomDNSAlias,,}" == "true" ]]; then - cat <>${clusterAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/backend-hostname: "${dnsClusterLabel}.${dnsZoneName}" -EOF - else - cat <>${clusterAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/backend-hostname: "${appgwAlias}" -EOF - fi - - if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then - cat <>${clusterAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/cookie-based-affinity: "true" -EOF - fi - - cat <>${clusterAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/appgw-trusted-root-certificate: "${appgwBackendSecretName}" - -spec: - tls: - - secretName: ${appgwFrontendSecretName} - rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: ${svcCluster} - port: - number: ${clusterTargetPort} -EOF -} - -function generate_appgw_admin_config_file_nossl() { - export adminIngressName=${wlsDomainUID}-admin-appgw-ingress-svc - export adminAppgwIngressYamlPath=${scriptDir}/appgw-admin-ingress-svc.yaml - cat <${adminAppgwIngressYamlPath} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ${adminIngressName} - namespace: ${wlsDomainNS} - annotations: - kubernetes.io/ingress.class: azure/application-gateway -EOF - - if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then - cat <>${adminAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/cookie-based-affinity: "true" -EOF - fi - - cat <>${adminAppgwIngressYamlPath} -spec: - rules: - - http: - paths: - - path: /console* - pathType: Prefix - backend: - service: - name: ${svcAdminServer} - port: - number: ${adminTargetPort} -EOF -} - -function generate_appgw_admin_remote_config_file_nossl() { - cat <${adminRemoteAppgwIngressYamlPath} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ${adminRemoteIngressName} - namespace: ${wlsDomainNS} - annotations: - kubernetes.io/ingress.class: azure/application-gateway - appgw.ingress.kubernetes.io/backend-path-prefix: "/" -EOF - - if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then - cat <>${adminRemoteAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/cookie-based-affinity: "true" -EOF - fi - - cat <>${adminRemoteAppgwIngressYamlPath} -spec: - rules: - - http: - paths: - - path: /remoteconsole* - pathType: Prefix - backend: - service: - name: ${svcAdminServer} - port: - number: ${adminTargetPort} -EOF -} - -function generate_appgw_admin_config_file_ssl() { - export adminIngressName=${wlsDomainUID}-admin-appgw-ingress-svc - export adminAppgwIngressYamlPath=${scriptDir}/appgw-admin-ingress-svc.yaml - cat <${adminAppgwIngressYamlPath} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ${adminIngressName} - namespace: ${wlsDomainNS} - annotations: - kubernetes.io/ingress.class: azure/application-gateway - appgw.ingress.kubernetes.io/ssl-redirect: "true" - appgw.ingress.kubernetes.io/backend-protocol: "https" -EOF - - if [[ "${enableCustomDNSAlias,,}" == "true" ]]; then - cat <>${adminAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/backend-hostname: "${dnsAdminLabel}.${dnsZoneName}" -EOF - else - cat <>${adminAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/backend-hostname: "${appgwAlias}" -EOF - fi - - if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then - cat <>${adminAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/cookie-based-affinity: "true" -EOF - fi - - cat <>${adminAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/appgw-trusted-root-certificate: "${appgwBackendSecretName}" - -spec: - tls: - - secretName: ${appgwFrontendSecretName} - rules: - - http: - paths: - - path: /console* - pathType: Prefix - backend: - service: - name: ${svcAdminServer} - port: - number: ${adminTargetPort} -EOF -} - -function generate_appgw_admin_remote_config_file_ssl() { - cat <${adminRemoteAppgwIngressYamlPath} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ${adminRemoteIngressName} - namespace: ${wlsDomainNS} - annotations: - kubernetes.io/ingress.class: azure/application-gateway - appgw.ingress.kubernetes.io/backend-path-prefix: "/" - appgw.ingress.kubernetes.io/ssl-redirect: "true" - appgw.ingress.kubernetes.io/backend-protocol: "https" - -EOF - - if [[ "${enableCustomDNSAlias,,}" == "true" ]]; then - cat <>${adminRemoteAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/backend-hostname: "${dnsAdminLabel}.${dnsZoneName}" -EOF - else - cat <>${adminRemoteAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/backend-hostname: "${appgwAlias}" -EOF - fi - - if [[ "${enableCookieBasedAffinity,,}" == "true" ]]; then - cat <>${adminRemoteAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/cookie-based-affinity: "true" -EOF - fi - - cat <>${adminRemoteAppgwIngressYamlPath} - appgw.ingress.kubernetes.io/appgw-trusted-root-certificate: "${appgwBackendSecretName}" - -spec: - tls: - - secretName: ${appgwFrontendSecretName} - rules: - - http: - paths: - - path: /remoteconsole* - pathType: Prefix - backend: - service: - name: ${svcAdminServer} - port: - number: ${adminTargetPort} -EOF -} - -function generate_appgw_cluster_config_file() { - if [[ "${enableCustomSSL,,}" == "true" ]]; then - generate_appgw_cluster_config_file_ssl - else - generate_appgw_cluster_config_file_nossl - generate_appgw_cluster_config_file_expose_https - fi -} - -function generate_appgw_admin_config_file() { - if [[ "${enableCustomSSL,,}" == "true" ]]; then - generate_appgw_admin_config_file_ssl - else - generate_appgw_admin_config_file_nossl - fi -} - -function generate_appgw_admin_remote_config_file() { - if [[ "${enableCustomSSL,,}" == "true" ]]; then - generate_appgw_admin_remote_config_file_ssl - else - generate_appgw_admin_remote_config_file_nossl - fi -} - -function output_create_gateway_ssl_k8s_secret() { - echo "export gateway frontend certificates" - echo "$appgwFrontendSSLCertData" | base64 -d >${scriptDir}/$appgwFrontCertFileName - - appgwFrontendSSLCertPassin=${appgwFrontendSSLCertPsw} - if [[ "$appgwCertificateOption" == "${appgwSelfsignedCert}" ]]; then - appgwFrontendSSLCertPassin="" # empty password - fi - - openssl pkcs12 \ - -in ${scriptDir}/$appgwFrontCertFileName \ - -nocerts \ - -out ${scriptDir}/$appgwFrontCertKeyFileName \ - -passin pass:${appgwFrontendSSLCertPassin} \ - -passout pass:${appgwFrontendSSLCertPsw} - - validate_status "Export key from frontend certificate." - - openssl rsa -in ${scriptDir}/$appgwFrontCertKeyFileName \ - -out ${scriptDir}/$appgwFrontCertKeyDecrytedFileName \ - -passin pass:${appgwFrontendSSLCertPsw} - - validate_status "Decryte private key." - - openssl pkcs12 \ - -in ${scriptDir}/$appgwFrontCertFileName \ - -clcerts \ - -nokeys \ - -out ${scriptDir}/$appgwFrontPublicCertFileName \ - -passin pass:${appgwFrontendSSLCertPassin} - - validate_status "Export cert from frontend certificate." - - echo "create k8s tsl secret for app gateway frontend ssl certificate" - kubectl -n ${wlsDomainNS} create secret tls ${appgwFrontendSecretName} \ - --key="${scriptDir}/$appgwFrontCertKeyDecrytedFileName" \ - --cert="${scriptDir}/$appgwFrontPublicCertFileName" -} - -function query_admin_target_port() { - if [[ "${enableCustomSSL,,}" == "true" ]]; then - adminTargetPort=$(kubectl describe service ${svcAdminServer} -n ${wlsDomainNS} | grep 'default-secure' | tr -d -c 0-9) - else - adminTargetPort=$(kubectl describe service ${svcAdminServer} -n ${wlsDomainNS} | grep 'default' | tr -d -c 0-9) - fi - - validate_status "Query admin target port." - echo "Target port of ${adminServerName}: ${adminTargetPort}" -} - -function query_cluster_target_port() { - if [[ "${enableCustomSSL,,}" == "true" ]]; then - clusterTargetPort=$(kubectl describe service ${svcCluster} -n ${wlsDomainNS} | grep 'default-secure' | tr -d -c 0-9) - else - clusterTargetPort=$(kubectl describe service ${svcCluster} -n ${wlsDomainNS} | grep 'default' | tr -d -c 0-9) - fi - - validate_status "Query cluster 1 target port." - echo "Target port of ${clusterName}: ${clusterTargetPort}" -} - # Connect to AKS cluster function connect_aks_cluster() { az aks get-credentials --resource-group ${aksClusterRGName} --name ${aksClusterName} --overwrite-existing } -# create dns alias for lb service -function create_dns_A_record() { - if [ "${enableCustomDNSAlias,,}" == "true" ]; then - ipv4Addr=$1 - label=$2 - az network dns record-set a add-record --ipv4-address ${ipv4Addr} \ - --record-set-name ${label} \ - --resource-group ${dnsRGName} \ - --zone-name ${dnsZoneName} - fi -} - -# create dns alias for app gateway -function create_dns_CNAME_record() { - if [ "${enableCustomDNSAlias,,}" == "true" ]; then - - az network dns record-set cname create \ - -g ${dnsRGName} \ - -z ${dnsZoneName} \ - -n ${dnsClusterLabel} - - az network dns record-set cname set-record \ - -g ${dnsRGName} \ - -z ${dnsZoneName} \ - --cname ${appgwAlias} \ - --record-set-name ${dnsClusterLabel} - - if [[ ${appgwForAdminServer,,} == "true" ]]; then - az network dns record-set cname create \ - -g ${dnsRGName} \ - -z ${dnsZoneName} \ - -n ${dnsAdminLabel} - - az network dns record-set cname set-record \ - -g ${dnsRGName} \ - -z ${dnsZoneName} \ - --cname ${appgwAlias} \ - --record-set-name ${dnsAdminLabel} - fi - fi -} - -function patch_admin_t3_public_address() { - # patch admin t3 public address - if [ "${enableCustomDNSAlias,,}" == "true" ]; then - adminT3Address="${dnszoneAdminT3ChannelLabel}.${dnsZoneName}" - else - adminT3Address=$(kubectl -n ${wlsDomainNS} get svc ${adminServerT3LBSVCName} -o json | - jq '. | .status.loadBalancer.ingress[0].ip' | - tr -d "\"") - fi - - if [ $? == 1 ]; then - echo_stderr "Failed to query public IP of admin t3 channel." - fi - - currentDomainConfig=$(echo ${currentDomainConfig} | - jq \ - --arg match "${constAdminT3AddressEnvName}" \ - --arg replace "${adminT3Address}" \ - '.spec.serverPod.env |= map(if .name==$match then (.value=$replace) else . end)') -} - -function patch_cluster_t3_public_address() { - #patch cluster t3 pubilc address - if [ "${enableCustomDNSAlias,,}" == "true" ]; then - clusterT3Adress="${dnszoneClusterT3ChannelLabel}.${dnsZoneName}" - else - clusterT3Adress=$(kubectl -n ${wlsDomainNS} get svc ${clusterT3LBSVCName} -o json | - jq '. | .status.loadBalancer.ingress[0].ip' | - tr -d "\"") - fi - - if [ $? == 1 ]; then - echo_stderr "Failed to query public IP of cluster t3 channel." - fi - - currentDomainConfig=$(echo ${currentDomainConfig} | - jq \ - --arg match "${constClusterT3AddressEnvName}" \ - --arg replace "${clusterT3Adress}" \ - '.spec.serverPod.env |= map(if .name==$match then (.value=$replace) else . end)') -} - -function rolling_update_with_t3_public_address() { - timestampBeforePatchingDomain=$(date +%s) - currentDomainConfig=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json) - cat <${scriptDir}/domainPreviousConfiguration.yaml -${currentDomainConfig} -EOF - - # update public address of t3 channel - if [[ "${enableAdminT3Channel,,}" == "true" ]]; then - patch_admin_t3_public_address - fi - - if [[ "${enableClusterT3Channel,,}" == "true" ]]; then - patch_cluster_t3_public_address - fi - - if [[ "${enableClusterT3Channel,,}" == "true" ]] || [[ "${enableAdminT3Channel,,}" == "true" ]]; then - # restart cluster - restartVersion=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json | - jq '. | .spec.restartVersion' | - tr -d "\"") - restartVersion=$((restartVersion + 1)) - - currentDomainConfig=$(echo ${currentDomainConfig} | - jq \ - --arg version "${restartVersion}" \ - '.spec.restartVersion |= $version') - - echo "rolling restart the cluster with t3 public address." - # echo the configuration for debugging - cat <${scriptDir}/domainNewConfiguration.yaml -${currentDomainConfig} -EOF - echo ${currentDomainConfig} | kubectl -n ${wlsDomainNS} apply -f - - - replicas=$(kubectl -n ${wlsDomainNS} get domain ${wlsDomainUID} -o json | - jq '. | .spec.clusters[] | .replicas') - - # wait for the restart completed. - utility_wait_for_pod_restarted \ - ${timestampBeforePatchingDomain} \ - ${replicas} \ - ${wlsDomainUID} \ - ${checkPodStatusMaxAttemps} \ - ${checkPodStatusInterval} - - utility_wait_for_pod_completed \ - ${replicas} \ - ${wlsDomainNS} \ - ${checkPodStatusMaxAttemps} \ - ${checkPodStatusInterval} - fi -} - function create_svc_lb() { # No lb svc inputs - if [[ "${lbSvcValues}" == "[]" ]]; then - return + if [[ "${lbSvcValues}" != "[]" ]]; then + chmod ugo+x $scriptDir/createLbSvc.sh + bash $scriptDir/createLbSvc.sh \ + ${enableInternalLB} \ + ${enableCustomSSL} \ + ${enableCustomDNSAlias} \ + ${dnsRGName} \ + ${dnsZoneName} \ + ${dnsAdminLabel} \ + ${dnszoneAdminT3ChannelLabel} \ + ${dnsClusterLabel} \ + ${dnszoneClusterT3ChannelLabel} \ + "${lbSvcValues}" \ + ${wlsDomainUID} fi - - query_admin_target_port - query_cluster_target_port - - # Parse lb svc input values - # Generate valid json - ret=$(echo $lbSvcValues | sed "s/\:/\\\"\:\\\"/g" | - sed "s/{/{\"/g" | - sed "s/}/\"}/g" | - sed "s/,/\",\"/g" | - sed "s/}\",\"{/},{/g" | - tr -d \(\)) - - cat <${scriptDir}/lbConfiguration.json -${ret} -EOF - - array=$(jq -r '.[] | "\(.colName),\(.colTarget),\(.colPort)"' ${scriptDir}/lbConfiguration.json) - for item in $array; do - # LB config for admin-server - target=$(cut -d',' -f2 <<<$item) - if [[ "${target}" == "adminServer" ]]; then - adminServerLBSVCNamePrefix=$(cut -d',' -f1 <<<$item) - adminServerLBSVCName="${adminServerLBSVCNamePrefix}-svc-lb-admin" - adminLBPort=$(cut -d',' -f3 <<<$item) - - generate_admin_lb_definicion - - kubectl apply -f ${scriptDir}/admin-server-lb.yaml - waitfor_svc_completed ${adminServerLBSVCName} - - adminServerEndpoint=$(kubectl get svc ${adminServerLBSVCName} -n ${wlsDomainNS} -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') - adminConsoleEndpoint="${adminServerEndpoint}/console" - - create_dns_A_record "${adminServerEndpoint%%:*}" ${dnsAdminLabel} - - if [ "${enableCustomDNSAlias,,}" == "true" ]; then - adminConsoleEndpoint="${dnsAdminLabel}.${dnsZoneName}:${adminServerEndpoint#*:}/console" - fi - elif [[ "${target}" == "cluster1" ]]; then - clusterLBSVCNamePrefix=$(cut -d',' -f1 <<<$item) - clusterLBSVCName="${clusterLBSVCNamePrefix}-svc-lb-cluster" - clusterLBPort=$(cut -d',' -f3 <<<$item) - - generate_cluster_lb_definicion - - kubectl apply -f ${scriptDir}/cluster-lb.yaml - waitfor_svc_completed ${clusterLBSVCName} - - clusterEndpoint=$(kubectl get svc ${clusterLBSVCName} -n ${wlsDomainNS} -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') - - create_dns_A_record "${clusterEndpoint%%:*}" ${dnsClusterLabel} - - if [ "${enableCustomDNSAlias,,}" == "true" ]; then - clusterEndpoint="${dnsClusterLabel}.${dnsZoneName}:${clusterEndpoint#*:}/" - fi - elif [[ "${target}" == "adminServerT3" ]]; then - echo "query admin t3 port" - adminT3Port=$(kubectl get service ${svcAdminServer} -n ${wlsDomainNS} -o json | - jq '.spec.ports[] | select(.name=="t3channel") | .port') - adminT3sPort=$(kubectl get service ${svcAdminServer} -n ${wlsDomainNS} -o json | - jq '.spec.ports[] | select(.name=="t3schannel") | .port') - if [[ "${adminT3Port}" == "null" ]] && [[ "${adminT3sPort}" == "null" ]]; then - continue - fi - - if [[ "${adminT3sPort}" != "null" ]]; then - adminT3Port=${adminT3sPort} - fi - - adminServerT3LBSVCNamePrefix=$(cut -d',' -f1 <<<$item) - adminServerT3LBSVCName="${adminServerT3LBSVCNamePrefix}-svc-t3-lb-admin" - adminT3LBPort=$(cut -d',' -f3 <<<$item) - - adminServerT3LBDefinitionPath=${scriptDir}/admin-server-t3-lb.yaml - generate_admin_t3_lb_definicion - - kubectl apply -f ${adminServerT3LBDefinitionPath} - waitfor_svc_completed ${adminServerT3LBSVCName} - - adminServerT3Endpoint=$(kubectl get svc ${adminServerT3LBSVCName} -n ${wlsDomainNS} \ - -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') - - create_dns_A_record "${adminServerT3Endpoint%%:*}" "${dnszoneAdminT3ChannelLabel}" - - if [ "${enableCustomDNSAlias,,}" == "true" ]; then - adminServerT3Endpoint="${dnszoneAdminT3ChannelLabel}.${dnsZoneName}:${adminServerT3Endpoint#*:}" - fi - - enableAdminT3Channel=true - elif [[ "${target}" == "cluster1T3" ]]; then - echo "query cluster t3 port" - clusterT3Port=$(kubectl get service ${svcCluster} -n ${wlsDomainNS} -o json | - jq '.spec.ports[] | select(.name=="t3channel") | .port') - clusterT3sPort=$(kubectl get service ${svcCluster} -n ${wlsDomainNS} -o json | - jq '.spec.ports[] | select(.name=="t3schannel") | .port') - if [[ "${clusterT3Port}" == "null" ]] && [[ "${clusterT3sPort}" == "null" ]]; then - continue - fi - - if [[ "${clusterT3sPort}" != "null" ]]; then - clusterT3Port=${clusterT3sPort} - fi - - clusterT3LBSVCNamePrefix=$(cut -d',' -f1 <<<$item) - clusterT3LBSVCName="${clusterT3LBSVCNamePrefix}-svc-lb-cluster" - clusterT3LBPort=$(cut -d',' -f3 <<<$item) - - clusterT3LBDefinitionPath=${scriptDir}/cluster-t3-lb.yaml - generate_cluster_t3_lb_definicion - - kubectl apply -f ${clusterT3LBDefinitionPath} - waitfor_svc_completed ${clusterT3LBSVCName} - - clusterT3Endpoint=$(kubectl get svc ${clusterT3LBSVCName} -n ${wlsDomainNS} \ - -o=jsonpath='{.status.loadBalancer.ingress[0].ip}:{.spec.ports[0].port}') - - create_dns_A_record "${clusterT3Endpoint%%:*}" ${dnszoneClusterT3ChannelLabel} - - if [ "${enableCustomDNSAlias,,}" == "true" ]; then - clusterT3Endpoint="${dnszoneClusterT3ChannelLabel}.${dnsZoneName}:${clusterT3Endpoint#*:}" - fi - - enableClusterT3Channel=true - fi - done - - rolling_update_with_t3_public_address -} - -# Create network peers for aks and appgw -function network_peers_aks_appgw() { - # To successfully peer two virtual networks command 'az network vnet peering create' must be called twice with the values - # for --vnet-name and --remote-vnet reversed. - aksMCRGName=$(az aks show -n $aksClusterName -g $aksClusterRGName -o tsv --query "nodeResourceGroup") - ret=$(az group exists ${aksMCRGName}) - if [ "${ret,,}" == "false" ]; then - echo_stderr "AKS namaged resource group ${aksMCRGName} does not exist." - exit 1 - fi - - aksNetWorkId=$(az resource list -g ${aksMCRGName} --resource-type Microsoft.Network/virtualNetworks -o tsv --query '[*].id') - aksNetworkName=$(az resource list -g ${aksMCRGName} --resource-type Microsoft.Network/virtualNetworks -o tsv --query '[*].name') - az network vnet peering create \ - --name aks-appgw-peer \ - --remote-vnet ${aksNetWorkId} \ - --resource-group ${curRGName} \ - --vnet-name ${vnetName} \ - --allow-vnet-access - validate_status "Create network peers for $aksNetWorkId and ${vnetName}." - - appgwNetworkId=$(az resource list -g ${curRGName} --name ${vnetName} -o tsv --query '[*].id') - az network vnet peering create \ - --name aks-appgw-peer \ - --remote-vnet ${appgwNetworkId} \ - --resource-group ${aksMCRGName} \ - --vnet-name ${aksNetworkName} \ - --allow-vnet-access - - validate_status "Create network peers for $aksNetWorkId and ${vnetName}." - - # For Kbectl network plugin: https://azure.github.io/application-gateway-kubernetes-ingress/how-tos/networking/#with-kubenet - # find route table used by aks cluster - routeTableId=$(az network route-table list -g $aksMCRGName --query "[].id | [0]" -o tsv) - - # get the application gateway's subnet - appGatewaySubnetId=$(az network application-gateway show -n $appgwName -g $curRGName -o tsv --query "gatewayIpConfigurations[0].subnet.id") - - # associate the route table to Application Gateway's subnet - az network vnet subnet update \ - --ids $appGatewaySubnetId \ - --route-table $routeTableId - - validate_status "Associate the route table ${routeTableId} to Application Gateway's subnet ${appGatewaySubnetId}" } function create_appgw_ingress() { - if [[ "${enableAppGWIngress,,}" != "true" ]]; then - return - fi - - query_admin_target_port - query_cluster_target_port - network_peers_aks_appgw - - # create sa and bind cluster-admin role - kubectl apply -f ${scriptDir}/appgw-ingress-clusterAdmin-roleBinding.yaml - - # Keep the aad pod identity controller installation, may be used for CNI network usage - # Install aad pod identity controller - # https://github.com/Azure/aad-pod-identity - # latestAADPodIdentity=$(curl -s https://api.github.com/repos/Azure/aad-pod-identity/releases/latest \ - # | grep "browser_download_url.*deployment-rbac.yaml" \ - # | cut -d : -f 2,3 \ - # | tr -d \") - - # kubectl apply -f https://raw.githubusercontent.com/Azure/aad-pod-identity/v1.8.0/deploy/infra/deployment-rbac.yaml - - install_helm - - helm repo add application-gateway-kubernetes-ingress ${appgwIngressHelmRepo} - helm repo update - - # Keep the identity parsing, may be used for CNI network usage - # {type:UserAssigned,userAssignedIdentities:{/subscriptions/05887623-95c5-4e50-a71c-6e1c738794e2/resourceGroups/haiche-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/wls-aks-mvp:{}}} - # identityId=${identity#*userAssignedIdentities:\{} - # identityId=${identityId%%:\{\}*} - # query identity client id - # identityClientId=$(az identity show --ids ${identityId} -o tsv --query "clientId") - - # generate Helm config - customAppgwHelmConfig=${scriptDir}/appgw-helm-config.yaml - cp ${scriptDir}/appgw-helm-config.yaml.template ${customAppgwHelmConfig} - subID=${subID#*\/subscriptions\/} - sed -i -e "s:@SUB_ID@:${subID}:g" ${customAppgwHelmConfig} - sed -i -e "s:@APPGW_RG_NAME@:${curRGName}:g" ${customAppgwHelmConfig} - sed -i -e "s:@APPGW_NAME@:${appgwName}:g" ${customAppgwHelmConfig} - sed -i -e "s:@WATCH_NAMESPACE@:${wlsDomainNS}:g" ${customAppgwHelmConfig} - # sed -i -e "s:@INDENTITY_ID@:${identityId}:g" ${customAppgwHelmConfig} - # sed -i -e "s:@IDENTITY_CLIENT_ID@:${identityClientId}:g" ${customAppgwHelmConfig} - sed -i -e "s:@SP_ENCODING_CREDENTIALS@:${spBase64String}:g" ${customAppgwHelmConfig} - - helm install ingress-azure \ - -f ${customAppgwHelmConfig} \ - application-gateway-kubernetes-ingress/ingress-azure \ - --version ${azureAppgwIngressVersion} - - validate_status "Install app gateway ingress controller." - - attempts=0 - podState="running" - while [ "$podState" == "running" ] && [ $attempts -lt ${perfPodAttemps} ]; do - podState="completed" - attempts=$((attempts + 1)) - echo Waiting for Pod running...${attempts} - sleep ${perfRetryInterval} - - ret=$(kubectl get pod -o json | jq '.items[] | .status.containerStatuses[] | select(.name=="ingress-azure") | .ready') - if [[ "${ret}" == "false" ]]; then - podState="running" - fi - done - - if [ "$podState" == "running" ] && [ $attempts -ge ${perfPodAttemps} ]; then - echo_stderr "Failed to install app gateway ingress controller." - exit 1 - fi - - # create tsl secret - output_create_gateway_ssl_k8s_secret - - if [[ "${enableCustomSSL,,}" == "true" ]]; then - az network application-gateway root-cert list \ - --gateway-name $appgwName \ - --resource-group $curRGName | - jq '.[] | .name' | grep "${appgwBackendSecretName}" - - validate_status "check if backend cert exists." - fi - - # generate ingress svc config for cluster - generate_appgw_cluster_config_file - kubectl apply -f ${clusterAppgwIngressYamlPath} - validate_status "Create appgw ingress svc." - waitfor_svc_completed ${clusterIngressName} - # expose https if e2e ssl is not set up. - if [[ "${enableCustomSSL,,}" != "true" ]]; then - kubectl apply -f ${clusterAppgwIngressHttpsYamlPath} - validate_status "Create appgw ingress https svc." - waitfor_svc_completed ${clusterIngressHttpsName} - fi - - if [[ "${appgwForAdminServer,,}" == "true" ]]; then - generate_appgw_admin_config_file - kubectl apply -f ${adminAppgwIngressYamlPath} - validate_status "Create appgw ingress svc." - waitfor_svc_completed ${adminIngressName} + if [[ "${enableAppGWIngress,,}" == "true" ]]; then + chmod ugo+x $scriptDir/createAppGatewayIngress.sh + echo "$spBase64String" "$appgwFrontendSSLCertPsw" | + bash $scriptDir/createAppGatewayIngress.sh \ + ${aksClusterRGName} \ + ${aksClusterName} \ + ${wlsDomainUID} \ + ${subID} \ + ${curRGName} \ + ${appgwName} \ + ${vnetName} \ + ${appgwForAdminServer} \ + ${enableCustomDNSAlias} \ + ${dnsRGName} \ + ${dnsZoneName} \ + ${dnsAdminLabel} \ + ${dnsClusterLabel} \ + ${appgwAlias} \ + ${appgwFrontendSSLCertData} \ + ${appgwCertificateOption} \ + ${enableCustomSSL} \ + ${enableCookieBasedAffinity} \ + ${enableRemoteConsole} fi - - if [[ "${enableRemoteConsole,,}" == "true" ]]; then - export adminRemoteIngressName=${wlsDomainUID}-admin-remote-appgw-ingress-svc - export adminRemoteAppgwIngressYamlPath=${scriptDir}/appgw-admin-remote-ingress-svc.yaml - generate_appgw_admin_remote_config_file - - kubectl apply -f ${adminRemoteAppgwIngressYamlPath} - validate_status "Create appgw ingress svc." - waitfor_svc_completed ${adminRemoteIngressName} - fi - - create_dns_CNAME_record } # Main script @@ -1215,31 +259,6 @@ export enableRemoteConsole=${23} export dnszoneAdminT3ChannelLabel=${24} export dnszoneClusterT3ChannelLabel=${25} -export adminServerName="admin-server" -export adminConsoleEndpoint="null" -export adminServerT3Endpoint="null" -export appgwIngressHelmRepo="https://appgwingress.blob.core.windows.net/ingress-azure-helm-package/" -export appgwFrontCertFileName="appgw-frontend-cert.pfx" -export appgwFrontCertKeyDecrytedFileName="appgw-frontend-cert.key" -export appgwFrontCertKeyFileName="appgw-frontend-cert-decryted.key" -export appgwFrontPublicCertFileName="appgw-frontend-cert.crt" -export appgwFrontendSecretName="frontend-tls" -export appgwBackendSecretName="backend-tls" -export appgwSelfsignedCert="generateCert" -export azureAppgwIngressVersion="1.4.0" -export clusterName="cluster-1" -export clusterEndpoint="null" -export clusterT3Endpoint="null" -export httpsListenerName="myHttpsListenerName$(date +%s)" -export httpsRuleName="myHttpsRule$(date +%s)" -export perfRetryInterval=30 # seconds -export perfPodAttemps=10 -export perfSVCAttemps=10 -export sharedPath="/shared" -export svcAdminServer="${wlsDomainUID}-${adminServerName}" -export svcCluster="${wlsDomainUID}-cluster-${clusterName}" -export wlsDomainNS="${wlsDomainUID}-ns" - read_sensitive_parameters_from_stdin validate_input @@ -1251,5 +270,3 @@ connect_aks_cluster create_svc_lb create_appgw_ingress - -output_result diff --git a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh index d2b73079b..b95814e1f 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/setupWLSDomain.sh @@ -718,7 +718,7 @@ function setup_wls_domain() { ${enableClusterT3Tunneling} \ ${t3AdminPort} \ ${t3ClusterPort} \ - ${wlsClusterName} \ + ${constClusterName} \ "${javaOptions}" else echo "start to create domain ${wlsDomainUID}" @@ -739,7 +739,7 @@ function setup_wls_domain() { ${enableClusterT3Tunneling} \ ${t3AdminPort} \ ${t3ClusterPort} \ - ${wlsClusterName} \ + ${constClusterName} \ "${javaOptions}" fi @@ -801,7 +801,6 @@ export sasTokenValidTime=3600 export storageFileShareName="weblogic" export storageResourceGroup=${currentResourceGroup} export sharedPath="/shared" -export wlsClusterName="cluster-1" export wlsDomainNS="${wlsDomainUID}-ns" export wlsOptHelmChart="https://oracle.github.io/weblogic-kubernetes-operator/charts" export wlsOptNameSpace="weblogic-operator-ns" diff --git a/weblogic-azure-aks/src/main/arm/scripts/utility.sh b/weblogic-azure-aks/src/main/arm/scripts/utility.sh index 5ce783757..f2eb83bef 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/utility.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/utility.sh @@ -2,6 +2,31 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. # This script runs on Azure Container Instance with Alpine Linux that Azure Deployment script creates. +function echo_stderr() { + echo >&2 "$@" + # The function is used for scripts running within Azure Deployment Script + # The value of AZ_SCRIPTS_OUTPUT_PATH is /mnt/azscripts/azscriptoutput + echo -e "$@" >>${AZ_SCRIPTS_PATH_OUTPUT_DIRECTORY}/errors.log +} + +function echo_stdout() { + echo "$@" + # The function is used for scripts running within Azure Deployment Script + # The value of AZ_SCRIPTS_OUTPUT_PATH is /mnt/azscripts/azscriptoutput + echo -e "$@" >>${AZ_SCRIPTS_PATH_OUTPUT_DIRECTORY}/debug.log +} + +#Validate teminal status with $?, exit with exception if errors happen. +function utility_validate_status() { + if [ $? == 1 ]; then + echo_stderr "$@" + echo_stderr "Errors happen, exit 1." + exit 1 + else + echo_stdout "$@" + fi +} + function install_jdk() { # Install Microsoft OpenJDK apk --no-cache add openjdk11 --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community @@ -9,6 +34,7 @@ function install_jdk() { echo "java version" java -version if [ $? -eq 1 ]; then + echo_stderr "Failed to install open jdk 11." exit 1 fi # JAVA_HOME=/usr/lib/jvm/java-11-openjdk @@ -20,49 +46,69 @@ function install_kubectl() { echo "validate kubectl" kubectl --help if [ $? -eq 1 ]; then - echo "Failed to install kubectl." + echo_stderr "Failed to install kubectl." exit 1 fi } -function echo_stderr() { - >&2 echo "$@" - # The function is used for scripts running within Azure Deployment Script - # The value of AZ_SCRIPTS_OUTPUT_PATH is /mnt/azscripts/azscriptoutput - echo -e "$@" >>${AZ_SCRIPTS_PATH_OUTPUT_DIRECTORY}/errors.log +function install_helm() { + # Install Helm + browserURL=$(curl -m ${curlMaxTime} -s https://api.github.com/repos/helm/helm/releases/latest | + grep "browser_download_url.*linux-amd64.tar.gz.asc" | + cut -d : -f 2,3 | + tr -d \") + helmLatestVersion=${browserURL#*download\/} + helmLatestVersion=${helmLatestVersion%%\/helm*} + helmPackageName=helm-${helmLatestVersion}-linux-amd64.tar.gz + curl -m ${curlMaxTime} -fL https://get.helm.sh/${helmPackageName} -o /tmp/${helmPackageName} + tar -zxvf /tmp/${helmPackageName} -C /tmp + mv /tmp/linux-amd64/helm /usr/local/bin/helm + echo "Helm version" + helm version + utility_validate_status "Finished installing Helm." } -function echo_stdout() { - echo "$@" - # The function is used for scripts running within Azure Deployment Script - # The value of AZ_SCRIPTS_OUTPUT_PATH is /mnt/azscripts/azscriptoutput - echo -e "$@" >>${AZ_SCRIPTS_PATH_OUTPUT_DIRECTORY}/debug.log +# Query service port +# $1: service name +# $2: namespace +# $3: channel name +# return: port +# Notes: channel name will be different if istio is enabled. +function utility_query_service_port() { + local port=$(kubectl get service ${1} -n ${2} -o json | + jq ".spec.ports[] | select(.name==\"${3}\") | .port") + if [ $? != 0 ] || [[ "$port" == "null" ]]; then + echo_stderr "Failed to query port of ${1}/${3} in namespace ${2}" + exit 1 + fi + + echo $port } # # Check the state of a persistent volume. -# Leverage source code from function "checkPvState" in weblogic-operator, kubernetes\samples\scripts\common\utility.sh +# Leverage source code from function "checkPvState" in weblogic-operator, kubernetes\samples\scripts\common\utility.sh # $1 - name of volume # $2 - expected state of volume # $3 - max attempt # $4 - interval function utility_check_pv_state { - echo_stdout "Checking if the persistent volume ${1:?} is ${2:?}" - local pv_state=`kubectl get pv $1 -o jsonpath='{.status.phase}'` - attempts=0 - while [ ! "$pv_state" = "$2" ] && [ ! $attempts -eq $3 ]; do - attempts=$((attempts + 1)) - sleep $4 - pv_state=`kubectl get pv $1 -o jsonpath='{.status.phase}'` - done - if [ "$pv_state" != "$2" ]; then - echo_stderr "The persistent volume state should be $2 but is $pv_state" - exit 1 - fi + echo_stdout "Checking if the persistent volume ${1:?} is ${2:?}" + local pv_state=$(kubectl get pv $1 -o jsonpath='{.status.phase}') + attempts=0 + while [ ! "$pv_state" = "$2" ] && [ ! $attempts -eq $3 ]; do + attempts=$((attempts + 1)) + sleep $4 + pv_state=$(kubectl get pv $1 -o jsonpath='{.status.phase}') + done + if [ "$pv_state" != "$2" ]; then + echo_stderr "The persistent volume state should be $2 but is $pv_state" + exit 1 + fi } -# +# # Create directory in specified file share # $1 - name of directory # $2 - name of file share @@ -73,29 +119,29 @@ function utility_create_directory_to_fileshare() { if [[ "${ret,,}" == "false" ]]; then az storage directory create --name $1 --share-name $2 --account-name $3 --sas-token ${4} --timeout 30 fi - + if [ $? != 0 ]; then echo_stderr "Failed to create directory ${1} in file share ${3}/${2}" exit 1 fi } -# +# # Upload file to file share # $1 - name of file share # $2 - name of storage account # $3 - path of file # $4 - source path of file # $5 - sas token -function utility_upload_file_to_fileshare(){ - az storage file upload --share-name ${1} --account-name ${2} --path ${3} --source ${4} --sas-token ${5} --timeout 60 +function utility_upload_file_to_fileshare() { + az storage file upload --share-name ${1} --account-name ${2} --path ${3} --source ${4} --sas-token ${5} --timeout 60 if [ $? != 0 ]; then echo_stderr "Failed to upload ${3} to file share ${2}/${1}" exit 1 fi } -# Call this function to make sure pods of a domain are running. +# Call this function to make sure pods of a domain are running. # * Make sure the admin server pod is running # * Make sure all the managed server pods are running # Assuming there is only one cluster in the domain @@ -110,34 +156,33 @@ function utility_wait_for_pod_completed() { checkPodStatusMaxAttemps=$3 checkPodStatusInterval=$4 - echo "Waiting for $((appReplicas+1)) pods are running." + echo "Waiting for $((appReplicas + 1)) pods are running." readyPodNum=0 attempt=0 - while [[ ${readyPodNum} -le ${appReplicas} && $attempt -le ${checkPodStatusMaxAttemps} ]];do - ret=$(kubectl get pods -n ${wlsDomainNS} -o json \ - | jq '.items[] | .status.phase' \ - | grep "Running") - if [ -z "${ret}" ];then + while [[ ${readyPodNum} -le ${appReplicas} && $attempt -le ${checkPodStatusMaxAttemps} ]]; do + ret=$(kubectl get pods -n ${wlsDomainNS} -o json | + jq '.items[] | .status.phase' | + grep "Running") + if [ -z "${ret}" ]; then readyPodNum=0 else - readyPodNum=$(kubectl get pods -n ${wlsDomainNS} -o json \ - | jq '.items[] | .status.phase' \ - | grep -c "Running") + readyPodNum=$(kubectl get pods -n ${wlsDomainNS} -o json | + jq '.items[] | .status.phase' | + grep -c "Running") fi echo "Number of new running pod: ${readyPodNum}" - attempt=$((attempt+1)) + attempt=$((attempt + 1)) sleep ${checkPodStatusInterval} done - if [ ${attempt} -gt ${checkPodStatusMaxAttemps} ];then - echo "It takes too long to wait for all the pods are running, please refer to http://oracle.github.io/weblogic-kubernetes-operator/samples/simple/azure-kubernetes-service/#troubleshooting" + if [ ${attempt} -gt ${checkPodStatusMaxAttemps} ]; then + echo_stderr "It takes too long to wait for all the pods are running, please refer to http://oracle.github.io/weblogic-kubernetes-operator/samples/simple/azure-kubernetes-service/#troubleshooting" exit 1 fi } - -# Call this function to make sure pods of a domain are updated with expected image. +# Call this function to make sure pods of a domain are updated with expected image. # * Make sure the admin server pod is updated with expected image # * Make sure all the managed server pods are updated with expected image # Assuming there is only one cluster in the domain @@ -156,31 +201,31 @@ function utility_wait_for_image_update_completed() { checkPodStatusMaxAttemps=$4 checkPodStatusInterval=$5 - echo "Waiting for $((appReplicas+1)) new pods created with image ${acrImagePath}" - + echo "Waiting for $((appReplicas + 1)) new pods created with image ${acrImagePath}" + updatedPodNum=0 attempt=0 - while [ ${updatedPodNum} -le ${appReplicas} ] && [ $attempt -le ${checkPodStatusMaxAttemps} ];do + while [ ${updatedPodNum} -le ${appReplicas} ] && [ $attempt -le ${checkPodStatusMaxAttemps} ]; do echo "attempts ${attempt}" - ret=$(kubectl get pods -n ${wlsDomainNS} -o json \ - | jq '.items[] | .spec | .containers[] | select(.name == "weblogic-server") | .image' \ - | grep "${acrImagePath}") - - if [ -z "${ret}" ];then + ret=$(kubectl get pods -n ${wlsDomainNS} -o json | + jq '.items[] | .spec | .containers[] | select(.name == "weblogic-server") | .image' | + grep "${acrImagePath}") + + if [ -z "${ret}" ]; then updatedPodNum=0 else - updatedPodNum=$(kubectl get pods -n ${wlsDomainNS} -o json \ - | jq '.items[] | .spec | .containers[] | select(.name == "weblogic-server") | .image' \ - | grep -c "${acrImagePath}") + updatedPodNum=$(kubectl get pods -n ${wlsDomainNS} -o json | + jq '.items[] | .spec | .containers[] | select(.name == "weblogic-server") | .image' | + grep -c "${acrImagePath}") fi echo "Number of new pod: ${updatedPodNum}" - attempt=$((attempt+1)) + attempt=$((attempt + 1)) sleep ${checkPodStatusInterval} done - if [ ${attempt} -gt ${checkPodStatusMaxAttemps} ];then - echo "Failed to update image ${acrImagePath} to all weblogic server pods. " + if [ ${attempt} -gt ${checkPodStatusMaxAttemps} ]; then + echo_stderr "Failed to update image ${acrImagePath} to all weblogic server pods. " exit 1 fi } @@ -204,11 +249,11 @@ function utility_wait_for_pod_restarted() { updatedPodNum=0 attempt=0 - while [ ${updatedPodNum} -le ${appReplicas} ] && [ $attempt -le ${checkPodStatusMaxAttemps} ];do + while [ ${updatedPodNum} -le ${appReplicas} ] && [ $attempt -le ${checkPodStatusMaxAttemps} ]; do echo "attempts ${attempt}" - ret=$(kubectl get pods -n ${wlsDomainNS} -l weblogic.domainUID=${wlsDomainUID} -o json \ - | jq '.items[] | .metadata.creationTimestamp' | tr -d "\"") - + ret=$(kubectl get pods -n ${wlsDomainNS} -l weblogic.domainUID=${wlsDomainUID} -o json | + jq '.items[] | .metadata.creationTimestamp' | tr -d "\"") + counter=0 for item in $ret; do # conver the time format from YYYY-MM-DDThh:mm:ssZ to YYYY.MM.DD-hh:mm:ss @@ -216,19 +261,48 @@ function utility_wait_for_pod_restarted() { podCreateTimeStamp=$(date -u -d "${alpineItem}" +"%s") echo "pod create time: $podCreateTimeStamp, base time: ${baseTime}" if [ ${podCreateTimeStamp} -gt ${baseTime} ]; then - counter=$((counter+1)) + counter=$((counter + 1)) fi done updatedPodNum=$counter echo "Number of new pod: ${updatedPodNum}" - attempt=$((attempt+1)) + attempt=$((attempt + 1)) sleep ${checkPodStatusInterval} done - if [ ${attempt} -gt ${checkPodStatusMaxAttemps} ];then - echo "Failed to restart all weblogic server pods. " + if [ ${attempt} -gt ${checkPodStatusMaxAttemps} ]; then + echo_stderr "Failed to restart all weblogic server pods. " + exit 1 + fi +} + +# Call this function to make sure the lb service is avaliable. +function utility_waitfor_lb_svc_completed() { + svcName=$1 + wlsDomainNS=$2 + perfSVCAttemps=$3 + perfRetryInterval=$4 + + attempts=0 + svcState="running" + while [ "$svcState" == "running" ] && [ $attempts -lt ${perfSVCAttemps} ]; do + svcState="completed" + attempts=$((attempts + 1)) + echo Waiting for job completed...${attempts} + sleep ${perfRetryInterval} + + ip=$(kubectl get svc ${svcName} -n ${wlsDomainNS} -o json | + jq '.status.loadBalancer.ingress[0].ip') + echo "ip: ${ip}" + if [[ "${ip}" == "null" ]]; then + svcState="running" + fi + done + + if [ "$svcState" == "running" ] && [ $attempts -ge ${perfSVCAttemps} ]; then + echo_stderr "Failed to create service: ${svcName}" exit 1 fi -} \ No newline at end of file +} diff --git a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep index 76612e55a..ffed6373a 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_deployment-scripts/_ds-create-networking.bicep @@ -47,6 +47,9 @@ var const_appgwHelmConfigTemplate='appgw-helm-config.yaml.template' var const_appgwSARoleBindingFile='appgw-ingress-clusterAdmin-roleBinding.yaml' var const_arguments = '${aksClusterRGName} ${aksClusterName} ${wlsDomainName} ${wlsDomainUID} "${string(lbSvcValues)}" ${enableAppGWIngress} ${subscription().id} ${resourceGroup().name} ${appgwName} ${vnetName} ${string(servicePrincipal)} ${appgwForAdminServer} ${enableDNSConfiguration} ${dnszoneRGName} ${dnszoneName} ${dnszoneAdminConsoleLabel} ${dnszoneClusterLabel} ${appgwAlias} ${useInternalLB} ${appgwFrontendSSLCertData} ${appgwFrontendSSLCertPsw} ${appgwCertificateOption} ${enableCustomSSL} ${enableCookieBasedAffinity} ${appgwForRemoteConsole} ${dnszoneAdminT3ChannelLabel} ${dnszoneClusterT3ChannelLabel}' var const_commonScript = 'common.sh' +var const_createDnsRecordScript = 'createDnsRecord.sh' +var const_createLbSvcScript = 'createLbSvc.sh' +var const_createGatewayIngressSvcScript = 'createAppGatewayIngress.sh' var const_scriptLocation = uri(_artifactsLocation, 'scripts/') var const_setupNetworkingScript= 'setupNetworking.sh' var const_primaryScript = 'invokeSetupNetworking.sh' @@ -67,6 +70,9 @@ resource deploymentScript 'Microsoft.Resources/deploymentScripts@2020-10-01' = { uri(const_scriptLocation, '${const_appgwSARoleBindingFile}${_artifactsLocationSasToken}') uri(const_scriptLocation, '${const_commonScript}${_artifactsLocationSasToken}') uri(const_scriptLocation, '${const_utilityScript}${_artifactsLocationSasToken}') + uri(const_scriptLocation, '${const_createDnsRecordScript}${_artifactsLocationSasToken}') + uri(const_scriptLocation, '${const_createLbSvcScript}${_artifactsLocationSasToken}') + uri(const_scriptLocation, '${const_createGatewayIngressSvcScript}${_artifactsLocationSasToken}') ] cleanupPreference: 'OnSuccess' retentionInterval: 'P1D' diff --git a/weblogic-azure-aks/src/main/bicep/modules/networking.bicep b/weblogic-azure-aks/src/main/bicep/modules/networking.bicep index e9804ae82..91346ea42 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/networking.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/networking.bicep @@ -67,7 +67,8 @@ param wlsDomainUID string = 'sample-domain1' var const_appgwCustomDNSAlias = format('{0}.{1}/', dnszoneClusterLabel, dnszoneName) var const_appgwAdminCustomDNSAlias = format('{0}.{1}/', dnszoneAdminConsoleLabel, dnszoneName) var const_appgwSSLCertOptionGenerateCert = 'generateCert' -var ref_networkDeployment = enableAppGWIngress ? (appGatewayCertificateOption == const_appgwSSLCertOptionGenerateCert ? reference('ds-networking-deployment-1'): reference('ds-networking-deployment')) : reference('ds-networking-deployment-2') +var name_networkDeployment = enableAppGWIngress ? (appGatewayCertificateOption == const_appgwSSLCertOptionGenerateCert ? 'ds-networking-deployment-1': 'ds-networking-deployment') : 'ds-networking-deployment-2' +var ref_networkDeployment = reference(name_networkDeployment) module pidNetworkingStart './_pids/_pid.bicep' = { name: 'pid-networking-start-deployment' @@ -108,7 +109,7 @@ module appgwDeployment '_azure-resoruces/_appgateway.bicep' = if (enableAppGWIng module appgwBackendCertDeployment '_deployment-scripts/_ds-appgw-upload-trusted-root-certificate.bicep' = if (enableAppGWIngress && enableCustomSSL) { name: 'app-gateway-backend-cert-deployment' params: { - appgwName: appgwDeployment.outputs.appGatewayName + appgwName: enableAppGWIngress ? appgwDeployment.outputs.appGatewayName : 'null' sslBackendRootCertData: existingKeyvault.getSecret(keyvaultBackendCertDataSecretName) identity: identity } @@ -214,8 +215,8 @@ module networkingDeployment3 '_deployment-scripts/_ds-create-networking.bicep' = params: { _artifactsLocation: _artifactsLocation _artifactsLocationSasToken: _artifactsLocationSasToken - appgwName: enableAppGWIngress ? appgwDeployment.outputs.appGatewayName : 'null' - appgwAlias: enableAppGWIngress ? appgwDeployment.outputs.appGatewayAlias : 'null' + appgwName: 'null' + appgwAlias: 'null' appgwCertificateOption: appGatewayCertificateOption appgwForAdminServer: appgwForAdminServer appgwForRemoteConsole: appgwForRemoteConsole @@ -238,12 +239,11 @@ module networkingDeployment3 '_deployment-scripts/_ds-create-networking.bicep' = location: location servicePrincipal: servicePrincipal useInternalLB: useInternalLB - vnetName: enableAppGWIngress ? appgwDeployment.outputs.vnetName : 'null' + vnetName: 'null' wlsDomainName: wlsDomainName wlsDomainUID: wlsDomainUID } dependsOn: [ - appgwBackendCertDeployment dnsZoneDeployment ] } diff --git a/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep b/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep index f2effe569..8281508fc 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/setupWebLogicCluster.bicep @@ -242,4 +242,4 @@ output aksClusterRGName string = createAKSCluster ? resourceGroup().name : aksCl output adminServerUrl string = format('http://{0}-admin-server.{0}-ns.svc.cluster.local:7001/console', wlsDomainUID) output adminServerT3InternalUrl string = enableAdminT3Tunneling ? format('{0}://{1}-admin-server.{1}-ns.svc.cluster.local:{2}', enableCustomSSL ? 't3s' : 't3', wlsDomainUID, t3ChannelAdminPort): '' output clusterSVCUrl string = format('http://{0}-cluster-cluster-1.{0}-ns.svc.cluster.local:8001/', wlsDomainUID) -output clusterT3InternalUrl string = enableClusterT3Tunneling ? format('{0}://{1}-cluster-cluster-1.{1}-ns.svc.cluster.local:{2}', enableCustomSSL ? 't3s' : 't3', wlsDomainUID, t3ChannelAdminPort): '' +output clusterT3InternalUrl string = enableClusterT3Tunneling ? format('{0}://{1}-cluster-cluster-1.{1}-ns.svc.cluster.local:{2}', enableCustomSSL ? 't3s' : 't3', wlsDomainUID, t3ChannelClusterPort): '' From 6d51421ab79f138ee8f8e95e1bb451b894ec9058 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Wed, 8 Sep 2021 23:40:15 +0800 Subject: [PATCH 10/14] On branch t3tunneling: fix dnszone location Changes to be committed: modified: weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_dnsZones.bicep modified: weblogic-azure-aks/src/main/bicep/modules/networking.bicep Signed-off-by: galiacheng --- .../src/main/bicep/modules/_azure-resoruces/_dnsZones.bicep | 5 +---- weblogic-azure-aks/src/main/bicep/modules/networking.bicep | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_dnsZones.bicep b/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_dnsZones.bicep index ae220178f..c5dba80b3 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_dnsZones.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/_azure-resoruces/_dnsZones.bicep @@ -1,12 +1,9 @@ @description('Azure DNS Zone name.') param dnszoneName string -@description('Location for all resources.') -param location string - resource dnszoneName_resource 'Microsoft.Network/dnszones@2018-05-01' = { name: dnszoneName - location: location + location: 'global' properties: { zoneType: 'Public' } diff --git a/weblogic-azure-aks/src/main/bicep/modules/networking.bicep b/weblogic-azure-aks/src/main/bicep/modules/networking.bicep index 91346ea42..a0d7438b4 100644 --- a/weblogic-azure-aks/src/main/bicep/modules/networking.bicep +++ b/weblogic-azure-aks/src/main/bicep/modules/networking.bicep @@ -124,7 +124,6 @@ module dnsZoneDeployment '_azure-resoruces/_dnsZones.bicep' = if (enableDNSConfi name: 'dnszone-deployment' params: { dnszoneName: dnszoneName - location: location } dependsOn: [ pidNetworkingStart From f0468dd14050968df603b8f7daa15653e58db139 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Thu, 2 Sep 2021 09:37:05 +0800 Subject: [PATCH 11/14] On branch t3tunneling: increase pom version Changes to be committed: modified: weblogic-azure-aks/pom.xml Signed-off-by: galiacheng On branch t3tunneling: pom version Signed-off-by: galiacheng --- weblogic-azure-aks/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weblogic-azure-aks/pom.xml b/weblogic-azure-aks/pom.xml index affeb4e91..e594b6be1 100644 --- a/weblogic-azure-aks/pom.xml +++ b/weblogic-azure-aks/pom.xml @@ -11,7 +11,7 @@ com.oracle.weblogic.azure wls-on-aks-azure-marketplace - 1.0.16 + 1.0.17 com.microsoft.azure.iaas From 84a2b0242075e25464bc22620ac09fd1fdbea614 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Fri, 10 Sep 2021 15:08:01 +0800 Subject: [PATCH 12/14] On branch t3tunneling: add ejb application to test t3 tunneling Changes to be committed: new file: weblogic-azure-aks/src/resources/ejb-client-stateless-1.0.0.war new file: weblogic-azure-aks/src/resources/ejb-server-stateless-1.0.0.jar Signed-off-by: galiacheng --- .../resources/ejb-client-stateless-1.0.0.war | Bin 0 -> 93912 bytes .../resources/ejb-server-stateless-1.0.0.jar | Bin 0 -> 3852 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 weblogic-azure-aks/src/resources/ejb-client-stateless-1.0.0.war create mode 100644 weblogic-azure-aks/src/resources/ejb-server-stateless-1.0.0.jar diff --git a/weblogic-azure-aks/src/resources/ejb-client-stateless-1.0.0.war b/weblogic-azure-aks/src/resources/ejb-client-stateless-1.0.0.war new file mode 100644 index 0000000000000000000000000000000000000000..2028d66b42ff72e82108f400eaeae247ff26b6a3 GIT binary patch literal 93912 zcmaI61CS^|vo1KcZQHhO+qP}no-?*>8)t0Wwmp0Px9{$~yDwtfQT|j zq?o9(3Z1OjqwLhAj5ICXJghV=)y&jvlM=%c^WL!&jnvE-jWn$g1o%<0S_&FeHg*1gWd1+V|B?B> zp7~#BQ!AtYpB^y(qlb;Q5h(cu7oE@_03gL>n~d+l%hwc z2Ect7CUqwCv9#jQKLVs9dE2)ELepqv~~^1-WdUZ z8pjC$q@b?Qtn30*Y4L=^_d$m=_Rb?V{t$u(61EetrSZP~nn6*&abEx^UiM!-Ez%Nr zndl?(=n{OpX|aA6!^OMBP%t;a-zKiwM9%-3VJnZ-csC671*sKVbNJXx1H`vC)|kui zCRdP@Ku$M+aaqlZ0eOk&!y#}0@E(A8vcDZLw2h$j|zq*35+MCdo zs`E?1%@afdy@lsUtM-%sK~V{%yuFg>rm`2e6Om%#yvf$7+r$2^$p!&H{O@A`{GmZ# z0s;Uy2m8O3=i+2&V)`HD<)|wDFXi3VwPUJUjbJekkfg@0$N5l(n2^`Pf_Qy}jw0<FN3+OrQJ`vclmX z>F!rK0qHdS>%VAf6t+mBsBZ1CRAf*ekYFp1+Cx2S5^n!rz}Yd|JPpvO?c36^~3 z<>2%6KDmc{9`L?SaLAJ;nK=u*(5RIJf4W+9KAXk3hT-7s?#77;h^g>#=pb_b{)9tU zx=X)5#{CAOJmQq2MJc1e)k`FR7BP~raO`wW3(s3|;^L)U;w%!HlIaaeHaxdhQ*Ydx zO=_5$n&)U@@D2Xup1}uZa^wPAT`zXrhq5xjAUZH~-^LuFp$2iI+_rdMQFQgjoKcOl z(cw}hA2NXhO*18WOmm1Y==?mU>WfH@dNKk5h)|oNGJI_>ppUIsvnS6AY@thAeEBYE zYiFB5L0iv$>OpBtW`q0uLI0-4cTTo;P#a?!>ziY|30R{18{`-E$FMtKr}hGLhu!O< ze2Y4USZwS7Tp86-_^q37&!QyJJeOuZP;`gb+xzF-f+dbfV0{b&X-(gH16+xGS|?Mu z{zVoS3?(UQVW}R|H!-Zn89Q?mmW{V_pKZ#Tss4>DnCnPpelAvLJqS&ldaTFd+f7RibUs+&vQc^DkV#0Bs zkcIpqEjUjNZ><>&NKHvVid5nat2;VIP|u9HcJB$7W+wHUd~v-i!)#N=7F>6x5k+z> z?n2r7=$db?@j83QNf!#oZ9}){9J_OBs_3FaRX;;+lyxKBBg)KLjp#Omg=wMt?-QQa z3^l_?Xn>!sLrq}7%TXc1pibx~utSuQe* zOWt^0>^q#@1Tt&dvt{f!s@~n_X&?ED~gcrav#6yI(f9Ec*q0qM88^ka7CHl z*$`{pXJ_~54}rqMi{JC`5uo}h422)P4#mDf@KqidQ}N-Akl&3{@P*2=?=insXZy-k z`UnnJ86hYlwINHpz+xibrp^D;4`9}qX6$STvn}c=5KT}4)%KS+- zqcYz(tT5bdxw@AFWff&F%kQ0Rr2Pd*zt<&(vhvQ3QK&cSZyYgTtT+)=YumDsqhkgREsVSRpZRt)^f|I~H%{gr8 z>~mf#5+lT8p?j{gCO>ajc2+uKD7~S}xQio6Ikg0fYP&lTjX2qKCfsJr-HREZEqGbT zm2o`1RF08l-`FX+{@9#NM5jz6(q3ZBe>RZjSv1OazRX5U|5csL_$NDi>}-s%?w~P_ z+Cy>#-laPR6IVC?q9t0GN2S>bbHA~Kx`mGfjry9wMQ4N+x2e7MaGIm&AS@35K%B+_ zk}FJIrY!U{hxSCJyYP_gfy*x@Z){zDT=d&JCO4bZX7`NY1I%0XOQ!O(BTj!$m5lAU z%S?=fygMjP|3IC}ujY{TOKf0NyCpuT3q4nf%CG9sy}kC}mFp*fzw#|Pj{lIJiPnGF z5v}@p1r(N59;X^UateW(=4+HZ-$Z#hFL2s2xrVJ>-?<(*f9 zs1}jHLRxI6ujEB=xAafK^aD(r#pVKcC_4???gxYJAi@GQfaXN=aq&mIx61H)pH!2FDREVzhUQa5WTjWcnfmXMsmJoE*emnxYmFR@4zt=Illd0uO5q3C2)G z^*7N~Jh~H4h}DtLPg8)Fdq2-0b&N~bD8Q?%!yeV|1>Sx4Pn}p))2!QSBoENzw>1z# zj4h<9zmu++{s0tfT{b6n-*#(G*l1yXD~@F{?{JK3{uN(y$t8B{29B^t#kRF;r4&px1qLuCoT!eDON+>%X&V z#P_WIZ903_5xoOg#XX~`YkPc!v3H(PU3eqg^GK*d>EjtxSan04I1YFgjC%%j1n%U# zAO*t_ivQva)O$*lhQdE$CsJ)4A!07iK#vDIQmb4?EZ0yrf>s?{H{=mf73ZisZp%AV z>~1Jl%h;1ToEZ_Rc0+ST)Div;6p_ivx~-BOZZ^0_!rHJS@VcPFGX8+M=?nskV{Xsd zVaP|tbycrtT44#+klrpeqtaFl_I)m9U7a;(*|I*-BWf+}_{p%=;Er}>l^E7CYv2ol zYY~-go;4DN)f>JTSS!V}=?$Y*H=yV;uYYz*NHuo^#Ln=+d?KtqTAT5?7K2V|4TXXG6~Eb#HQF4g~k0sU5`W`qfLJ1z$!nKgxSMjZR;N(FmG8} z3_s);kcv?{94cN`76%SvmZ&Zmqd+#^% z*b+HXIwL*lU=xnpZ5YZ8tBx{TFif2}s5=tcOr5rX2H~8!CLqLD*@`!O6KEVGYThx9 zd2mE5XW?QMG&C0$u*4}FaN&q@A-L)XTsX@v|x{ z1!|*VVB8!8EJQXp9BVjt%5<&aJ+7Q*ub#Zx~r?E7u@t@F&S_l##RnuZ>qApi^*aJoZx+9j#9 zDLfE>LyB<{&QO*X=Fr^dG|TZ>5zO9?lw=3ZcZ;^-(%MSC*)A~xQad#>WM7z!kC?j{cWEBfAUfZECsUw zK*As}fRg{Rg$z*vuZ4*@(=ztWk=WCV8-Kv74O>#T;-_+1wInxiNZ{6^Ix_{ji4>*e zJz`-oP2AH8!L<;Pn2$s}+|;hg4|KEW6|jTaf@6qaJI1oN8BbwfYrVf=L7zTyx&6~j zZ$A0zLpKQLO9&e@a-SZAdxY`x z9N8~DZ<}kOL2#U`YpJQVU!dhYiS>`G-01=m&Gqg_#vS8(Pbu)?1?ZML`)}rD86=v_ zYyTPw;fsmOS(yOhHu)#l(p&yXtLwsS3O`?`!tSXgwR)A7S1~l((Q#H!#~spl&Fsce zUg;0sEKp=cS*y-rBXHu$6-&JLnS`a41%ZL`@w^-MDX6&>8T)ZB5^zy`D%%^wU}4=- zinBJZIyN+ljK59I(^6Vbrvl0Gr(eWMl>@k9-7mXTzL~jD))HXgUKTn)<4kACF11Fvo*iT#Q4aP&xYtVr;#u3X&0Yad!Q1KIi_NUSwWrVR?x{LBe6 zFnCeAxmWGb){rk(*FYXihCO@z-PQ0(;x)npBBWFFDr_Z6IxN7C#h-8-qh(Y*kW6xg zX+$g4@wZaOad!6Z+b?C5$eST(1;|u4Th-9r3R=6YD|qmbDsyYk_WV^Dr2CMjI$Pg1 zy)02cgq?Xf%whoOQ(;ov7fd+=1pu^Qb`UVb^AwnA%Cd=ix(o*fer`DbP! zPGTP5w+`5Q)%76+_9Y7bANgI;A!@#$>g9HkZWcSyJ)2 zcZXj;Uv?xjgw5P)N10(y(b0COTf-adv{NYr7~Kyek@}DWML0?Gc15D1*HJad%&gZ6 zY6qd)LxxmxUmV&%=7p->tCb!Gbl+ELH(w~ZZ5vVf4cAR5xUkgRTa+t`fR8x1=FRW3IVjT7BvF5r+Dv&uvo9Y=-jzTk!J;`2quSJ@7| z@(gW^m0iykjw9O(7=i}+k*j_>bjQiFAE0wTV#rZMvw8Be`&yd#kX{(P0XB37QjGDT zQFo(6=gKsv#B$BJ-%oESb@7GHlA&(Q&2f}!Z!8_|@`7oU$7GkFLPg+0=~lX2Q#gBK z5CH;se?P&V4&G>;ZDQfA#UrqbS!hI`ZKtAQRU*1r%}H^Hpps*p6F!bjKcd?ml5+oc zFwYvpNX`n2TeXx?n&sM{kQUFYb-zlMuyw4(>ZUeb3LG?7o1pi_lpzB1YSf+aq%$b( z&m-qNR71I(SelQV1*lt;g}--XvCksnUaBZw zz5ax7FA8EN(bNl4>IzX)*CKilGp1-x&dkzGK54Tol~r}&{~~5U$Uh=at_(49RSG;X zhfHMh%S?n-_BUM=3d39JpZ2tETG1ltnL8Z;Nib4Nj!3H*ilJ6{E#Lf|Ghj|?8HSda zFQ#YvwOzjFN_RB8*h8%8>X_bFN132EPf>fgHLFWUi_VdOcS@P3lICDdI`SSCY+}{o z_Zp}ak%@GSx{taB`yO z79!XLFCo+?&Gx@Jer8U;m04Ik3-rF$#$5qzxs)oMQYs9Wz204bV>>p@{W0ZsFL44J2rMDzZW2g-ljhuyjUK>&i&yv&CSwZr_UC;%)N^FXasy zQ{d1&8)RJcUy0RIk*8-$J&}58eflQxh09;4?k)6T$W+a&hty%~t<4RcRtPuLz&d%W zlzQ~i!3V3P@l9n^oHLox=jUk&d9x`$dhw?%qWK-V)IF zR#6WidjFY3o`ijFIqn4#MR+T#`kDOb!`V*l+6nlT54H(BM%M@nap04lk*9s*0QCp@ zZwdtGdaPtR9KfF<2?)UF1_(g@|DixuhHi!)bpIhi|AIKQh7Ojr%yj=@K>zqpN*8cK ziS!Dqj;`*qZmaF)|FbQdZnn;k^xg0?-*Ugs_%QFf6EPheMe;c2>UbaTSl{M|i+yn; zVD#hHmFKJQ1^9af`oT&Hg6c^rQW9Ea^w9$YMto_la9{pdzhVOjApZ-`5AxF|AEhFp zBBiM3C!tKJBs9yH1)j5@e;jQmmjz!z8#Lgohif+B2VYcC2jGiiuA&*E$XNl86LBA>)dEE-OfVK|N7t%jD^ zL7$Z*IIs*6&eP~M{c#&=gFy7~oTCVl45VadMG!^?X4`;_%%F zp|+clF+{)iT2q^v+7etFT+xF1{QJ)M^K1JjF*HxTdH_wga-7#R&^ zCj}M~H?D)=Aq$X%EQhnX@Nt5jms>tNqa`D>%MUo_Vl^RUFW3IqEw;fs_)fc!hU^1m zDt7^ksbpzPt%jTu`YtqmQB3ps2Uk;2@_Gv3yb*`5K^bu;u&ob*aPwP_^{+PlaYq2e z16k&`z{N}@9{mdIfgzGpVT*jWT%c33KBxAM9l?rvH9RXN-5g42jmXqi%#JC~hon7? z#lxpSywNyv`pO;w6s^SUDB#clXJ54uYhXv3S=FQyd0qYlEdBPI;75OoRjS6v+Fbi@ z485q*wZuGnEB!7Y0L3qHuYT)(TWh0x{UU3N(;N0+=tI)OT{lDSmxEM&qyUrm7*{2O zN_y}4>~4VRF{H0LVT%UA$U7ybvM4=S=uH|&ZeQOxnIO9m#qro0eVz`Lc3TgZo@Y1z z57?{W5kSxA`8jxm7{OyR2yo4KVuOp*YmnhcovVcp!*tPlxpHOERNrDfv`UwTD;TJ^%-9TGoyYZDO1-%MoWCl zWjzO&u$fYHE{qwhOCfh0?NdM-j?{?&tYD%$FG5qiuR@Vj&aBvQ1C*r;7+TqgO@pG% zOl@!iIJW2acao~08hj|U_3d@dMBznk-hgT$v)>ISm#H-TQs7Y-qK;RH}TrI zxnK5RX)LZk8rWaa>xX}j_B;`T_=3SqKphNEb$iK=p6@YlC0y_BW~hrFFir}nWv_tm zxLd~;oo8H%1C3unR(r-LLFbp)zIbdZsUfG0MwwY6BBg z()?J@^KogK(d3=2E6KM*2-v*uv)o}=oAbMsywLl7@)f&UXN`qrmpGl0>`(-$qrYJX zC~g=ZG{wHF4;h>FUVpj>wi3^8W|_Yy^L5PY7R!87iwK?KuEv9f{9Bwftsg*UnI=U) zm2N#AS!jdsem2EXOw++Hh9TT?Tut6-tisA%#K_TvS5hf9CVjv|iXWc#Jc!AT)Z)f~ z#d}cVgD1*n^3C5%P{qlEn>(1# z*GKXP*KZhl1yzA~8_5d*IJJhV`Kb{HRYGEG3m4bV>(xEHy_@3!P>=3d`riwHBLDMY=rEJvFyIpRybr%q@&F~EiU>RN zA-jN024h!x>RIx4co!aT83G&ZMJSVIsXH zHLzmCmLy`~FujxI`g2MF&SI(OlsMtOm&{K?0|M+A{PuE2ytQ z%^C#dvzciOqH@Hdp5f%6d`Tk1g;FX|=svtWi5>Bx?HHvaer@ivN#*HBpaZg^nq^go z=vA5Gltr!tdb%#&?y5w1*>*zP2xG2V1| z;Ceu${fHA94zrX=9W+k>pULGX2ioWM@PXG;LV!Qh5LMoV8?K%Jd|ZRkV)A8?2`CDM z`=&>DK90BlwXWnhs(JkE?7UrdEcr@`3i%yZVe9-j=^PDHA%jap%azJpu_$D~6Q6*K z2S@wED(miMh!;)GTYfOff#gsr8If?Ea8-1AXf_JquKw`!jE5b>zTc^1W_o;lU4y?! z4r>LtjZdj5{ABwFng3|MSaw~9pB9v zt&kDLNMENEyban00^*fYnwR}^tQb9a=^npj878T$-7mg$N2+2YlKq9x9c#m2 z)8cJ3*E1UDKpRWrnmfil)w|&RmZnDJrFYyR+i9MP^5O7;iba1~a0M1I3Lix(YsHhjyKNOCsaA55QAEfD-3F_KI&Sq`JnLlnWVQKiVcCKkSCmR;n^ zhc!o)@rx%Xl(pQk`EBA@6T|??^R2x~sNsF(Qoo*B({G zTCtL}G95k-iiuyt{fPH3TYP`Yre|6Bp-{Ibp)zZmaRfi}lnBUTyhuTsfC5>$yYJHW zbZ><0&w_SO72=$ex5??76<6}+zSO;ug#PT*SIqh^ICK>w5?!qcg*C2#LNhYvbGOq| z!^pa?NL%lMt{C~x3f;a!?=wKOFZSv|?>=%EB@f95Xg->=<|pJx^o8xk1}Qh|i*u0^ zyL)kECOVzsnKg=8?D0Dq8Q1FgK>S!3y2*z!7&`v_v)(u4B1|@A;^AK*OXsD2-1@+= znt7f!X;1e=zeDfTt>AZ>=5S+Kp6Si!*`W10^F8sPgVD&xMB6W?5VqW7qGxK)j(@Xc zJ=;%Yo-lTm&hZHuru=0GodVAw13Hu+M}hTRZchrH55BIjQWF09#{SMMRo8duf>~?J zROrKIJGM+>&c9{``xVAcq7AuhP%)Aggfhfc{ilLZ-Q&9h^C&x1{&a4_dB!zHZyMTJ z;L98zEKD_y-~rW2ulEaM*8joq!T(cYM`3qijeWGsN7%=KZviQr_X&+drfgIY zhaxI7P{tg^=NY*03-!o1bXqRKfn_O%OG{JCs^{sK$L$vO- z8}|}d65PIiVu<0?y0QJ7u-u>Tzoqyj!5XsKnrxYTR-;-Ia^e@a;y295ji(&09+6KW zklhqK@&>S8{WtU>lGA+9l_5;;m?VPhwCxMlcrqyJ$pxGj_@p_1{(2z@^_Y!pOJN)T zNKbSR>msYmx?Ej#jGLw(5%+rI3l6+|?o^bcK#j|(p%~81&RQtr){{%b_6F0T3=rax zEaiVDOQH})m%K5EQr%dJdmEo0+czuU9_Dj}8ew>^HVgVH> zYWPO)-9_!kh|FQVxhhB@NUd9<;!Eva|qTZRM zQSbq4c3O)V?|Xwn{6S!mix%-hNUTG{Yf720qFny9aIA4)}{8ww&n&m6yL6Eg9{7$m0sk; z1fi#r0)GG&c45k{#d&|(dPq)fQZJo@0|HGN#sf#;b@1_)a_zh>|!>AM- zL>A_1oDoeV`;Ay75c@fe-}wp>{(4rU^3yoxn=pl!WnPmlIGFe+OI#gQ^DI7h%9WO( z8~AOBLGlF`oHm2z)l8tpy&O&9nF z=~KZ2iWe^H+=|zUIFwR!(^aKjvOucXMPy_)>o26q^I^9W zkF3l;rin3Kp;z4)a+gU9GA!^4FsLk0f&eir`vd%yt@55EE^3~B zQ-Z_jU6(~VJfEX1F|VO+CHe+zLql9Uo>>bd62gw12V`FMA$EX?v>2V7zk4cFCd6)_ zH2LYtxmc62h-e~3_J9V;wM|iyNV^~JSV+URMU)2-(irEETcs)27%EmgyN82L86qki2HFu5i z%gP5=&aw!Bq$zRT0KYP7Y{1NE3-q9QZ{K|sZ+!{R_ER!3tIc7bQU|RPZR$fwW(6aN z`p%A+B#=@$`2Kg3QI&`*8Z#RVS|Y0$#e1!*&_3mgeZHRr>)IKz6opDfY)rW_ycZl; z`jXQ(wGTiJkgZ=8a3v!LPD#Q@hhII-{Af?z8b@wua7zHXvU*~n6G(p-nY50Je-9E? znE6`a?vQ)E7d@g}N~%~7+57B&YgXq`x0Cy#L@P}7oNck> z=?-G8>xWG1(;s9)!&S0;&204sjz@{kKpaJD%;eQNT_~=Y!AqE%LyUMautG8F`>v5RPa)*tnu_Soq3uu|;#f3jOqWfl6W zeK36wCeKKI$Im9hyo5^8;KJPe<=2kLs`o4v8U9eNLO_r@2M*js20jAycG<@VB&87r zL`9B`7yn(|{76>k#^-^!rmxolb|ARg@Q!ST-dZj1aJK2dxVCop?gP{%rW8_kcG=Po zKW-$5xWLfBK+EUK;1{6xjPZu!i_xjr=f^RAk?&MLh&Mz#RO?7l^wF@48H-?AI9n2cg(-WuZUxuTtV!;{*%U= z+8a26F9j2S&hQrrHE;~SSn)hHb5})c_%97vB(Z8lne5wSH0^^cMas{X=FCDuv9$;Q zm0+l362wMSzOb5irQ+=ovERN#QUd*}k?-y`mHkkFsPzGmh>jp^>&~?~AsJ<*h~ni^ zinggsU?$b6O~h_D70mB^&1>om5ksH#c}@i^X?|8MgQp==J&~1Bm%@vtb##gat$mmG z5XL_1*aJsCH4qX!o19!!L1@8km6cro2-XIms4Ne|$_w1+LkIs^hu%|jy|j4T2_410 z)IFfq3f+Y_%XknkcIuNn1vX*=bUwwp-rQxM$upHNJSL6)Q$N5L{qX|5SA#68HQIUIQVj{lSf2H+t*}0H z_~20W{p-`>yt|BReN!J__iljQ(07PXu-7mLxO_wa!gR*|#mv{T*KR+6y=|erL@|uV z9G5)TiiGr-wqY>|#nR%qh>E!QjGH`(J!A}VFr`TpBQ9rG?+~lHQMLnPDN;!G##&SG zP&tZ+KEPj&&3|DuloFNHt7~_blfiV6DXxa4+)?qia$dY7>{m{2&@e}p-yXU_yRki( zE`UvxM4*v<@u%cf)s)hw8m=sx7{ni!2}$0NOJtoj#SKnPW$cvO$*T%F;R5)FhxVcu zQ^Wv}x?xQ?p<@-` zAFw-f;4gq%^!KlSlpy9p}z`kB|bmFZtp{E_L#W@&-*Vj zUo05ALfHuf;m5U7$)L{Jp?T!QEa~tp1t_SB_44%vjuUC%-E~v8kW-E|pr;YMNsgpACqx$vKEDr&i(;rvq*{M+iX+p% ztAo%swxw`k$v7rA`5PmC(4M7OxVqEBe)=O#S$5q!Twyt(I_lGdDh81Sq@X_3SyVgms}w@43bw7wTH z8!bT$mxbB@nlb$$Go-<8S(?QG<(muQ_u-TJ(4rEZS9O(vSIVei=Vs#G)xY5qn zof=Gm;j=>3BsDM)m!>EoTud$#&MJGnH1_9&Vd`&53&ttgP~5qUU}TU~DBu<&X-c%K ztE~&)5T%!lhr*Go%a4!GrGdKsH^61~6-|WG6l7`NXt|}dJUnj1)$`SH|9bWoOKYcD zg5gcKs3^%4URO?OK+e=Qs|`tvyEJ79dY=5gv(TEyi}``mFP1X`Obm)a}}UofNyvenaEr+ z5Jkur)*1tb3I*zrES|`F+B)=o7FXC6mCmm<_ELCZ~c1NARKSWg=Hh8;*%+WD99YXT_ zGWI|3j_%sn``f|jtwY-fW?I&o)~@xQ&Vyr#JZ=_N3s*DK3`x8 zI;C-2&fSbEJHaMnt?rv8ggMzQR&1`XO}t+NyPX=&m6q;qm4Zi!a)}d+uaJVugSOjP zfTK@q0Kdhi`cB0fqCHixJ5FCrHLtqr{$Pc`d8IzF%2an&>L!m+Ej_qDH{+RxIeZ7{ z>ILyGBcL@m4~iqpxnW-wH*D6?QhZX&yfA&6Y0M?baNHxzxx4WhBMf@l{mcp^JO;Z1 zTDyN0m%K8;7QzJ)uYq=J4+*@*W=jfyg%+YRgXYM`tX%07Ibu{wXQyw30t0#46*$6i zYnrR!{0Yd~&El37BkThnW|fJ)Es(wpDlMoX4mp&8Y5nkIgvA+0KkOCSun?8MR06jR zs04Vh{K5)mxHW+zXxF&S0OvqdgV^KfNZ9K?-!2(x^Dn4w&aMP9G`{bs^_k*^ekE?e z6bOOROi1|FKR5GmlNqpxl?KsCyZiwF48Fakptvx9iZT|;8O&ZwhByg7w~+ z2K)mYWW~uS7tjhP;R7Q{$O;ZEu=WHo^XGJ|zQU@#z^h}BxzB@pXUgjRk|2QIYd1D7 zG`})gZ7f}1xmZhJ9Xc*S$8e$}qG@3E!wkFX@&YIXkko=F#V^X>y6jK}`}J55Z4p_1 z#x5y`WMAe6EsS<|{^gj5ma-XNl?RSW^Dd>_PSc}q@++N7c{Pi*{w|0@ofgLyXqE8H z64ZRISc?)`YGFB4-DPXa%u0;@}tPEj*N9jHvj7(sS30dhmi@zQrLr4eBh}2NNuQjLr!| zK6QM(ilAisQ09-V)~pe^Eq0IOy1z@`FB~gb8QuW0M+NSapK=qi(QeQkOXLp1X|#!9 z&|O2|O+e@QG_|HjY^~2-$#tXnyFNlij@e-6KdsUmM0?YB`k0p*W9m}3=B^i!Kl?7` z`v^*EQ61=31LJRK$>WN_AQu{Eb&s*YI=>_{z38-hX^%(Je8uU3-VV}dAKc;J*WnAa z783q4`m*v8bdlt9N0k%0KcuXR9eb?E#EL7Iyv)m6th`I=>DKH`x$c(OX=KJ%cOj7< z;<1@XjyNF7;|vf8Hp~aDbSD|p!a9sV>cBzPXo}FV#*HO&jk11doxRxocM!&QiW=05 zeTNj8ERV}*wsjuun30Z#vp0sGA%Wd?QyqOxI zo|XlzBMr-FjU}(F&J;V; zDnY9R*N@qm#9(Vh&i#5`3`7{KBqBzPX^c?VkI$YOoahI6I43wDT8Z?82a{e}6vJ^O zX!=`kb;AnfDgf8(P&TV92=Q<_SbqThd?qaUNK|}i>cJw2{W4M~YFW6A9uWC$JlEn@ z+>X>SUp_AaW`{r;-stElMRY%6{M6sl*EA!0nG{p6BuS zjgR@oFPZdWV9~^oqbpR{)m@?R;QQB?TOo9O!Pf)o2i+@i)WuotNDNsC`%OYSJmaQI zl#tFV+*QXb`UNyi47zP3JI8UVM)(3&3fIwUQCO+z8WJL-WDdHK{4fZ>f4Omq&nswU z+U8i9BHF>1&;-~UEFkr`sHFTInm~pvo%}N>r_coO)>`94S8m-Nw-`16pWxja zHRNkT35mo~*;hth7Mb@_<-mL)%f+RpsPB%r>5}gl5VgRq!bV(j8qs~(;$hG`^Cl#y zHFa)4U(0&`&Ac!2rk@u6j!~dPM%5l4EeNS)7H@x~Qp2InG#HY?aQiB_VQOM#>QZq$ zXo^m^MPJn9IbTdS4h(4^QyY`&t&_oQR96mF_)bPYSJQ_#?k->0w7I8nZ>uYn-UK z+|kR{AdaytSykz{-T`byf`njTuEAsX$Q46SQJf}Zq|5_FKqDW8J-+N^M=^iOUG~`~ zwbF!zPS~9FcyRUh#)QNPiSLL{$&>dLk>mchSbV~Q;S_3F%Qt7Y$1nK;Cw@X5*^@?` zI8`wRAj$RRQpS@Rqnfbg)QZB9jEVMpaFl>Ju}b+sokd!ExDIC(h0ruW8p@`V;o=`J zrVv?YB4ySma4~czj98-1m{({VYzy>w1JyKi@hIvMf$+RBi+2kTlBnAPAv(Yw21kpR zackP@Jhy9H%rIgz#9tuk0LlPQB=|TDm#a%ijmw#y$?yu&T45Pe<9-0m0*&k4Rbzya z@=&RoTI1JCx+!j>}usOhaZj}}M;m)MATlWIPbem!CoR=#x49&nrpvNxhNaO4(r((%emFpFKmf-LX{atldryCfK5)VjK>@+6lH zbkc~Use3{`Z}xVqYRN(()# z4Ubd-W6n0TxpLTvf$ zRO{_kPmg+~z2#~mlj(;4*b^VEJnVj3`omKm&87Y9c-wefo}_*g!45vzJ7x{pIlm&qA#BYT&HV9NB2Q5YJtz+bp}hTS{c;DVdMx?v}Wl38&evjlU{L z0iK;ups1ySo}zi55JlI&W{ z_C=c(s0RJK}Ky5W55p-HxUjgFr1 z5uMkQJhJ(N4^Yk*$b4~kYl&*1PQrLoP8dn^!R+P<5{ntl6Ymg^?lSgP%n5K7H2maA zXY_(z!IzO5E@Qg?JN<|eZz_S|)0*vXktWh|48r<9W<7v+|^KM^kN@A^az-(cX8e4u-0z_csY^;g9uZqfRNUt zFOhopStI;ksExuMQ<~QWr-fN=m2YgcFTYfL~Y;4%L0DaHTjyT;6n4Z*nB@1nn*V76=bEW$(xV@p~(U(KHE=YG%kU!MWn;KI}Z z=JIHuJ|Qw8GN3^GF$h*M{}%v5K)k<9Pgm@)MNe=LNMB1Z5YTdec&eOK$V-Jyf{zt43j9zgzo~X&O2bdxR#H z%C;4xJO`90Zb3CgXzlKpg!rEP_z2;R^yv2Wd-GofRJg2zq$N5Fiyrb&E-x(FzC<>fps#QL4eKo zxfq-HWlTP=zV9O0+wA_I__S$Oi%6}4WzsuM%GtrvZ;=ByQ=UTG5OmDXRE*;?^OIvH zzU_F@#X_-!N!!K9GKYu7(y$y|d78bRk5NDKh#(PB!;Yl5rlp=uJpO*;S5+vMnZ7%!*KLZne{%tN$T)p;p8y`eup@T+y=mR<44tz;P!QRdsT@u zNN}bNFnpPeZAEmuBZ+m)Hg<%%1H}ILOK0mxkX^&k*DUZ$E28}iSO2-K;V9j1)yD&q z0X#g656veRpTwXM8%HNrAQBtHBgSty4j5d)UIQ)QMJdST6CGC91Pkm(ZpB#7USI#( zovjT(Z9=|IwhjxZA4$yN$eldHw0aUTXW6IKojzdya(X^K{yL={FtkQ>n8-Y71V+hc zdHM^r?^%I^>?~#pebVDoj#)hvHME>{)I ztrdLdCIdA$agHtYTmza&JnCS7az$a7927cmh~=G@r4H9uhWDe)cYqh?SxiQYUl+mb zCs9JHTQB1@f<#MFw_m#VpW(q#Vx(yOn&E!IgZ+Pihu=ik)avgTMYV(V$-oF0d+xso z)zsB44ymiGg5=?V`1yH8gbVTIjWX`9QM>_gi>;%8fVEB783ko}a#%Y9;5Q(cfjROS zVvlQ1*@&dvClj+b!DHH@xLe#4t(kXeu+r^)lliuQ{O4UyAFW~Be$D=9dH!pq@9(?*rz?t%F}#1|2!h^&wEB57 z5XXGVRRmNG$W-`N0=zJYhd`Dj=}_@l$MXRh6M;XkW%MQlW>S{Z$&R1Sma3nkmzV*N zN94{6kKx^-4i=I7ZhECsYPeZCvDZ==10g5(;XG~l`HEB~ZIx;?l6hUI0`{mKtct?1 z>m^Dms_WXf4@QMDtB3;Pdr#jQlYV+PKKRM^?K}k~KKHE$nDxzsQTv;*wL15TAu^7x zc&{InttFbZyQ&Vtp3y2~P+H}QYIU$zyY|i4EpG)7y{G%Z-}`5n#ZouNl?1f7c%V8(wmT#-@Kw^zH`;r@L0{lNs;vyHW37Lfs$%oUq8(t{uL@9nt^A&i@ha#HGJ>oHZy0 zEPL}Uj~E(j8pZuur4N{3<~d9e+D&b&fJ(J#!byDpca}VTVh&1wqmjqH26U)9?=3L>cLio2?1LTXmL;7F z&&od|*Ri`DlfL;wtN^})jJbVBPa^pxo8a3qsDlc|eFL^r-hJI@no+^RQOTd=kp&ZO zL7V>imwq)ZEIss5kZGl`1o%IJy+A5w5BG!nUx({|?IiS8!5 z?{ZXXFf^7vVlabBdodV&Jv98p@M50#lAg!WGT0vGjeg6G_oD4dA3)wgFNb!mQ=>3^ zR^eRx8S(a$n_JD@olmJ-pRXsz0Mt8KVN~mKGoj9i6pYguAbP5U?kci7!>^GoA&clEz9Z9 zncNKA%30`Av7(1h(FE+Kzf@r{I+DXnwnXU~4xvm+tSK!nP&x}T+h^dK97fkcWd43- zU82bs=P*CYK*;{3Y8oG}>G`W^F%qI*(AAte?O?KMbi)LXTY602)^@!H!a1&&o1vo* zCZ%Z(Xbyv!qAO-iYj3iy%gC;GQtIR|>=I?SyBXDn8A8vl6i7cxeVtYg3&EGfG|Lh5 zm8*XR!dV~4Mcnz`^3pC?SEq#IdPhtRh=NY{+EBhgyl+8X{q7~ofG>=$A+b#>Cv@2n z0uq^&QU9Q+HNQQ`GIlH|AhyfZVU#rxjhRsgd9@7TG3LZQ?Q94=So+l(=q*2{LYm`g?3y+S{iHn zq_GWQar6!gBvfRQCnGh3T?sg!N*ZM*3e)j)b_U4m6swlik0Y{oVH#sFC#ZAIPA!Q@ znjUFO19OQ5&~1;g(U)qHDZ%W1Q3}GP_gFvOPVi?0vlFdea5VB-;ge^+2Wh<^{ho$W zRDiGKh8N_uhtHIyoMjRC%wv#`&9T}CIFDsOEvnVBveE_0^VtUupcSJZm7#$e`y^o= zM?|gj(-p`A?SBPkV*_Jadif$Hh?~?sdWWb=x33>|%eBK}4fDZJEj0xtMk!k|7ipW55djih|6zS3e z2&p&I^Gn4;aI>2TX$vOYnlSv%l^ZLZO)R9xh;X@DELAJt)9vRkxNDTtemG2QWeI?X zMq`gM0JO31&xsxHLf<$F0X|%U08oZ#cXYKMLSFgRNH4g=%CE7l}!A)z)y9LAUQC-uIs3Ku_7M+l6PSTkLVu!)Z9HkvDp?Jf9*HyKl{r8 zD#IEF5C8xZ*ng+({{KJ7@mpJ)oBlmVi&QdofL}uV5Q<~eckK}x+79$l?F1p|S97L_KIqktq@$-JGEHlcL{BRqm0W9xX)+Z-pA90QxS56FTLmzUzw~wZ zJ@auM{iy9OHFc7-`D8?K7IMSNnzXgHPWhk`BNX#2q9PIh6?a0D{gn z6+|6yO%fIUjF-+qv~vTK)sxHzvO5Z6+eNVh6#pC+P)y*V*tbOBq1iVF*M+;24kuMN zC=Ztfds7M16?sL&I90dPZsAgeP<+D?{~IgaD;tu!W(Q)gbEE{D<}?^$MI3K3%!M@o z(S>O8l&PM$RVT#)ynmxiz~~FU>+r-w3)2FcMVV zN1zZ=eHWF9WkA38p_)9y5ELaL^|=Nj0uhe#W&eTtd4)-Xj_(b|1GaoTBK1uxPEN`~ z+(LBd4G~7oASNaqh*y;vMOd>~%rBy*uEj~R?RNd1Wh<5UCuDcaBC<6IV5k$CMly^O znM!E@Ff(R!%H+^UB%2G)0+K7^!tOHpgKS+)X!#@2$7Pu41SPdCx)7!mF!gawacHZh z2CYt6rDOZCg{wEEF$%6MeLCSQ8L5mEeh6a7AybwXw_*yUr54Wm>So-RY@8f# z0Aoy)qvgNL330Ls7Tswas!4OrDD?`U;0mTKSVEMhqWTFG1t7c+pJeyEI1;JG%|=G3 zVHA5T0;#FV*1#CgY6fMF8`cMzmcL`>lQNpa2Rt|+04Kl}ot(xq6*x41j3|cjl8l0T zhYyJrkBhn|m?qbLhaNK#4vqjqZ_-CU&>Bj|84)k~M&O_+NzSII7DdQnttAA`6uYMo zID@dXHU_0=jeUUzXDc|Q{!Ck6ryT@L1~Eeki^{z-=QJTkYQ%luUWPOPmE@8brc}^S zZD-aeYHdU2zv}>)H0qLWT1FL=quSXNpA@zW65c|#hgLPV905tEwlWKJXDfH_6N{bWTPe}n#}#4}DaH`7q1vgl(3DHL7iDQl0@c=)(*<_50O?6xrO!wY(4 ze6%$_6Y8Zs1i6lI;~TNH^F;UIgL{fnJ5&TonXX*b{W=}zAIxm&vEBP3N0yp+b_P5@ zzD#YR9lInYo;KE+1}lDhS+#LhLmnm#R2Un@lXg(=R&;otrL61~xKw*QbnxBv^9Xs8}i4Q*dp`GPINMF>t%-KWM~*mQYxF~$oDF~8@?>eXnu#n-(jFw z;wMg!*NvRS5W8ZpqwyT3P{$;iuGmNuQew0;W5WJ4Va`ZKw=Z(F49YDBV}F06GpcgY z%nUSJr#yeTvKQH`tWC)xTe#kVS`Sq%8Prq(_Y=J$!(N=ph@zQkTo&0P^`hsih_!X2 z*M6$4>gl$;WWMw&RlU7Ja+aF#^@gYNX3tiRy~I2&uqDR95CR|^z_iZtnLbDNuU_H) z!Ho_Wdj=vnBx?tjm<+S-{Y_5=P51Eh!{#rP$+zc?j z`DJNqnc{F{GWNc8SX^S_O#h2~8==ry39oUl&v}wUHPN%HZvG zzgFH6{xxpuZTK;)E%m$4t+)P`9Nc>FyHp8iUi60#PJj0{hcu`^*6#Y#=rbggeinWr zGwQ%ME4dzC3_HicXABZkFG%po14UrgyD5#xbC4D@AdJcHro>k%pslQS-@P$Cp6L(i{)KxSdo0!<2y z(^W6XgU0E5?@nCOg(3p}ZH3`G%tGXdkFmgMIR5MjC_yA|bqEF354#W_jI3`VLc$rS zTz~pMu+vJTn zbOyURXjdgf!U={w)x~q{%?BEp8b9K#I^c}JdROt6RdZ}x!2tRImxcN8Kj`b`>4t;L zRW3Xe%wFT_b1zdm0%g$Lku^a61%`ZVVIVhHAHU1;Mn^O{fKos@ zF>I>{^TMy|GEo6ySoGo3coOg}FIe#@9&}~t^Da0z z3Q$E9-^!vN9?N2=8=)uM+SFw>(t3{+Qo6ksZOVlvz&Nw`f*WzFb*J~v%IG~Rqcdkm zo6$qLzq|C=nc2;DJQDea6Op?0hE6MXpxo|v3&%|w`oVKoP)mY`M1LgOqj`(y;F;C) zO+XO#$d&iV4kRS^{B<^(SHaFO+UESVHkw!7PO;scv0Q1k`Ey{9w%7}0=|?rgTl3q; zs{d#2=dT@RGOzL-XSW(ow*2xGYdg|1|-iew8VH(&$uJUuB$)-GBC4 z%A>Ojg9tLw(y+|ZPjh<&Ios&C>_c2_CrTVM(bCO#`BrBU4?03!9G{$`AER8JB%2F9 zs#(8YgH+$9Ztle9W_?7mx4F5wYKIpkO28o%FiNM!n#{$-Mij&454|W*U@$HNF0G1C z=rzG&G&-t-{0eeOxgfcW9EkXsf0n|hgX zbpNfZfF*U+^u|z-f`hnQxJWE1S%|fKsx6k$sPXYaUppvohQfaEe145RR%W)dr8qy0~TFM7ln5k1V@w!68wVnwxIi)5o-a2lCh+Cvfxcj#wF zax3+^W59N&&^PSx&tghPp&wm{r0TFD6P}>4v_#R9M?kpeCW(WS$3h~&LhE9~Ic}`f=T&~XYrS%_xx`qq)f!QS+VdMReUxf2 zELDnK2>In|ThXP*%crYk%M+Hk;XQV4GzMUt*rppTkHwlYvzlw>Em-xYw8pTXEeqvR zyPdZv)wxbRy+QF$A_~pb8J4LBsk2?cg=D)gRq` zkrcnA#u%lEkldH$1?GH4pYpnG(Hx0s_$haUKsJ9XYT4=a&2~I&d&~RPzWGqtq^;!n za@OBGx(I-W1ksnjnh^1N-gbiG!dmphP=d${&+Fx8PoE3KGM2Kl*W|) zQ$`UpV#ys>+DwvS)yjI(1X}yR`kl}%jEZ_CDts4okYt7OX1MH*_$OwMLY(C=6Us$( z`e|v7&YPg}eK>_jF*LV6wUUbgHTVUrdr;P9(SieS zkr1DFY1}ex%^`bHX)mAFoI2A>TedCpaZdQ6&eNE8BuTIPYQoHGCKl34X)auWU{MbVn|f%tAnTM2i|#9>n2w8J3_TlxNrU(FWT! zR$ACFH`#}QyIMG^C9lmmlx+}Ewn(-HS0|h3zti~O=Gwxn&c2@m_t0S84HmHby~z>P ziUYnV%WzT_Q7*$io_<}yn`z-r+2X=-0)B%Lp6ZxyIvsKqD}C)03GoyzQl9+l-)}*> zSR-<=oiWQRfpT>6+l8oLTL5v}0i%x74;?2=o%_<)g{bUVD?rbyB|ya%uw)Bhu>~~S z0v>Gvu(#Rh!oRU3b%dU24OUNHfZA)}OYh_hpOCt6$`nV#cq7pVi#-a7*AJekBz$L! z*LZ^-{PGW=xHJ5a(D}1aJN zr@cz$O-yd&JukTXz_d5UgMl9AKgizwR(oSPabessv(|I=rI>ow`h6occ8E_&{aY7s zT|Pfs%n7WvJ%H*nTXjFz+`_}GMdUZf4Ew7n@8g3?RepHw)>0jBMqZ+MYN3WZi>`+2pD=a0FdA8rr4GYx0(=+@%ikqIPbpGoD> zIaUww?xEw35XI~q5vNAkRwZUvYCS70Es6&z%2$sPvPK_XrW3De`KB%ZhN&3_rdPmc zTMuH`i-vp6;mm9~&eYrBZ6*ZM4)1SRO!Fr`WN=DQsn5`MjIfaRv|>YeoDOx?4(0JG zD@sz|tM@E3B4#_F&qFWB0(rB^>wG8n!Z8@yz3XLOVURetQWw*3YLDtyyEmJ4YWfqP zqB%66r0WCn0nMpUs0+}TTxR(*GYI&}j4t$3Q!n-v!82PMEi-TDcu8}R$O}5K_*!p5 z1c%vc@!;s3S~z;@G;6SQ#F~9Vi)Ejw#$L%chG+Uj<#>Yap+q5f&`daQo7?Nd>!*M9 z+(hNd>YiQZKC7U}3f$_01w?wY;b7JDgWfeBU8x8 z7?;%xDYd=0@K%uTf%+9@{)pXyN{7gnIrx*v)jEPIQ=P$#y@gHlnF~?$JmeK4MF;v9 zb-0c@V*BAOZHWHp)~hpRj^$C=war&n2UaC-2crMn1Hav?>3Q#`mjXl$%mFJy<*$H2 z;n(^KL!Vw(dBHeSVresJh$FJ@kKivwaB^|R2-~XD3!QfD$^^ZIK#}=N(3tJeWJ@^D zO^XJ7!`A8{wL-iO)Y4~o^hs8BQ%+aEskFI<=Jnq{gHY5pcR+OX=^akV?BS(N6FDtv zUFC-0q&X+*Y;#6t3q}&Iq+iJ(f>CgrX!w#XVWlbm`j90W?BiVDkf(c${Pches9$JM zxaT4iHHaSK{{2$(>P}eoi`*cT`bI4B4d*1fewg7#B9}8L^#FJ_kE+cFMQLZ~rn<)q9`(3O@dVyQXV~xX#CC|# zxGVES??a>pm|uhE1fPvEyi@W~Kg0bSrvqAWR)DI&5jcfRU0V~OWLh&ojEH~dvQ(k>dEG^7*ro`=> zLp^Z*SRsYfbN!|iVY642=#G<}MVXB^u26*12Wxny{WREY_NyZ( z5KuYPzuVyaXFsiM=HzDP^1s{aShY>|ENZ z(#1--OclGDia&5H?8+{8fp%qTnV1pJ;U#k&5Rm9GoQZf45_*7ZL>axWPQW5dZs&!$ zypU<6W4WQlNS!vz3yHej(Qnq2*%`M(unhWZjlolI^k83uAv8CX^H%({3h-M?Xr+FS zz)Y?CisM1iLT!b+8Tv^^%Efhxtr6Qyz!(o#ynZCHp7Wu?<+}TcC%qh#y;v-Lchj!p zE8P0nE;%g3DX2KiJjWv7B0OWKq2sh$LJzJ+P!E%)FwNlx z^TSxbm-G}W1~TR}&BI+B|5rkgK!Tq)Qz6nU^z^6#ijYb+fI&)_kmdeog13xXPagXS zpLxg3`JOqB=PJ%Ac@~VZDcxN6^vz3_%|4bpq7N;&o#21)~m< z;`UV{vup~w33x4NE@_C+x$Q{4Narvi{`RI7Jf@^NVJis7yZ&smD zG5h84{_Tg&YamJ~JUB=4DJJmFn3LP^&WN%Sj%$UOK#hnW;0OZ+1H@PuZ%E?v#;Tez zg}ex;ZeA$I5{%@Y@um4zkqTt9k~yhwst0(}u(|;q10l--$sJI0;Pg^n*l3XqV0|ec z8gF|pK6Ll7Bkd7jJ4QLW-iU;X+G2(Hhx)-}7yLtA%VJ2K=AiuB3${ZT54cyRCse$f zVXnz@a93at&Eu+o`t5%(DeCD_F4IB)0Ue?KyFHNq%fr=hHga_QpJBGtG%ZIoF|4uv z9Xwq2Vfg(;D4}r*V0B=05h!8FabqYhcA;RT@a@@~0dw=r%+L`~G{l}kogToUSnVQp zb%kc>aWZ1x+eLMK--or9pTV4|P4hOsvE7RQ&#wO8&ua#+Yuo+bpNDfm4X}MtgX*3+ zN0F4g*uiv;jRaMDC=)7V3dy<@USUUCNvD!|KRSgUKqjKe7!rGAA7Cb`$?6kz$ved# zxF>MQ9Fls(9@r;%$ZnH*B%E`Pe3K9*`N`fRjt(h(P(l^3m)$knjy;$yV_$Qb=CW0w z{R8G!$S8L8(7R~*Z zUFkj$XsW++j+xdQ8-U}w%?k&%d*O=g6^9GF0{fUOvF4l&?2YD89JClq2(Tfe{OH;A z1Yb2m`B1%r$GB{qa&%03jzeGwu?n7zTlh^sdeXD8Zq?s&v4y&)Se-}j~TpM`K zA^d7WwUUvJdpbm0$aHi1p5Jt{!|4dTE4iM--8MxUEZ&;q$}~)blod#C`GE&9HC&%! zv!PFbNsa2cb-d1>k0K`oFih-T0%q1+A?81S^BF02f}biDULBeBpF;6WJt9EI=G zz{y116hnb-F332QQGw(Lk$q_LcofShEN-`PjKu;0XR*d=P&V_TnKfI;&|`~oNvn{z zS&`Y^P)jXuWTUF})G=$CV1~t)8yFGPw*nZ)rJ&$#_obnuSaiQUoTuSz8v_KbcJ5;a zgTD?7dMK271-iFlhRzi^1`EmUOlV`hm;+{&uE=?#E~7Y=9W9s9W#_v0IqjR#KpTC^O(RQCYcJgj6xS&|);zNnuL;^qQ;V zl^5AgJm5rey}dlEs5~=ZEG1TM#=8kgY?Y?d1T|?-kj83TO!Aal6d0sndup)BQ-~ zRi7k-Yix4J94wneQ{#0s{?$}IEPgIneFmI;H_?vHxOvC59`a`fNy2WhUOr+Ev7oK> znxf&h5iL-CIimtU6o(VRPSYBy#t_#Q5~&nFK~OwomBaLU#6RqhTP+9i1FWjAM$t;t z2a^ypNOr=ArA^4UNBFnNdcQY>HI^#MqFIP$WWcAiTtzCh7dbG8ndCa*XmK8t9Sc(I zKs$Pbv6kS43~*&QpJP830Y^K8I%eWmrzb2H1$0}bVYKObs+Upjkhee@KApvP&>C<7H$N}pVu%0v3sU{exS*RT+dwj=EUB}GR3xL7qc}L=yQ=$?KTjYGQIkg-43T+?p@g{*^DQ|0NDAa0 zM!O1(W=4rRYVmBxUh^o%vyZPjd2j95IL6PlVYp zsl#ASm_u8aik{Z}|BgLIF>J7}U_d}7us}ex|39$D#nHju<$p#Jl=X?vJ0M!<@Vx>v4D zSIpHb*-sW*?<$yZ0<~DFjJqj&gi%8Lt~UE{rR*jx`C5L1LklqCBq58~1GW;A2;C5- z+Pvn`x6NIVNakKqs_`(t6-U%*sd`)>E_vaSm* zr>ovK`W4#$kfN2|VyKSe*IFO$p>e&;XT}Mr*!xL6X0fq$tWQi&ANkR#KMTb2$R|-z z<0aFf{Rv*|D=x34#hW_Pm*1hTsaV)=+A_vS6)t?~TTirNz_d|ahMx+`i>!|I=oFGV zq7wM<7_s6w0b5f78pCuagYS+p?Utb>r_dKTBkYP5SoIAl+W&-^&YKFDE)*080T@ z04D+p8qx%{8bj|(>B4RdM0IP{wzXdU;iJFdbgz1^R_>hJ;>(P9)^VNrFI#z=xfl(_ zUDxVNwaXcKrrOm-_LBlW=U8hO3QV(WZ@Xud0Xhw9cDjBO6 z*RI?uv%P}X5PgX`t{73)X{+iuGQorL8#VXt-Jz4>qoVmOXGRfo3gyjO_`7kSPZ-5+ zCc;r1GQ0XMYB<5a9DmHgPm8jA)=zh# zzG1mDJEf>KcOu-dM8CG0916>Lr-{INRK(Z)wsz6#4Xm|fWu%HhX5heDvB-kj97W0f z+>Nu)z{wM7po2B@T*0N1#>UrSyKhuetqviG(ssE%XEksrm|q}i<5|omD9hEN74;VG zGBW%0f&0)r{9(%!>wlcJk@GF%tI=_#O8=}b{|1}g0W9#V1>%p5RnE&Iw|4v|#Men} zA@H4eFyx>Jgae2JyC+r;V6S%14#ENQmeKQvXZm3GASuKxa6C>AroRVP528=`;O8J@ z+PxkGA=s|n9q}C>#4c>#+}`jWDMWnW7(72r-|n8t-XTOj&?y{0G{4y$=-v{90F>YO zUie@<))(a)Q!pW9KGPfR8+7noU_R>`RfE|b*gE?eXffu>PEV%#p7Q3 zc`CRcdi&b)_47YD8x(QAR7(F!pi&6`jurd=mO%eez0Z04E06Xw$oZ3lLPI6+9!Mi<1C)gk!P5l^IA-xK6)s8TWG-KD{RaLGY;RTxi`np3XFFER<5nr7ccJ_T zm}}?L97V*uEVlgA=k$1{H{av5@AKv|KAyl2c8``{jwsYaa|eIqCp2&DYx3E!T^gdM zYZ_Y`vrMR1lh3%anU)2U=1qfbLk4X&Erv{N>Ffh?^vnU=5Wc90q2X!+9LJ9YDa@NQ zoe%fb#FsRx?W?ZK)#hk2atC8A?&qxI@ytlIz&G7m67UyvI2jDwY@5Zty8293mjXqa z%aba5+^^02nC{Ck(u8ZLP~0~CdYPZ%hT|{gVBQD{hEMKlUBvZZ6icYRCSlLgW_9YD zjN|u`XAe6%sR@C9)jb=7iiQrI=AB(eT+zz)k~(_{kh57`YBOoNn+;VgsO^+yX?>N8 z*q$%0p*NG#^BM4A_*QBd zw%y{5@g+sNRts|F((|F$O*P#R$8JbEPj=_LI#w~(gQUzt<{}hoXgsi2 zP9;1Fjjc%>C-@>rg;_dxp(-HdUWsOP^@HHDKk@P=~9pvTO+TQ zb-3{H%tFAs=yqH)MyY0?P42rM4XGg4lgvjd*BYfCn+59t7S8+}!+P0b_D$e7-$JbW z+ly=VxpI~rVXWY>Wv#{!yx^B;EwuDay`UMIb>|a?#o*KLy28NIraVPN^30$etSK9G zjk;;BoX{PtIz|J`Qdnya!UoR&tQ>>M^V$=dxhs;-RhT_$Y@SN(d3u<@T&_-7TcGk{ zSO#tLbBcQ?!KlW$D#R|jz|AEP6dG`ioFQUmO!EeGC=_GGB~dJ2!F*6b9E#zaW0jjXCIl|R5dSW+yZkqbk26U^B+)~3UV(AcZMDKP;a?f%m#WcNc zpo?x79{s>R*izHwS=q?@@uFBU9gZnjc1CtQ1LUBGha~`bD5yJtpFi&$Kd_D*Tjv5@j;OQ2I3RZi+|%;5koCW zXDshVN#MxkcWJ^NdKMP)Hk~G%hc0a7ORFPYlSUkbccSF=BH)}k-qJQaxZx_U;d0VD zRQzc!5b^qBv|Kd6i|PoMZ*lXl<~r8?8}=W!0Kg)X0l+{&HK6}tcE`o#|5=mqRi3h6 z5=84&NurQbS)4CZBq&w1yax9iLw;KjvJRUO(E@yaV<@JMcVU zg_fEal6ZBR&2@3``}6)D9igA7Q9O);4{Z&w-L*$noL zH#_00^`R3n^Zm1V9an+wgnCR3JaiJbyIIquRGp4A&QxFaAom|O(>2mTh*ql>*X{@x z+8Th$mQ09OfSj)b#_m1FR_w&Ov;5d&KSQ_1()2HGZDX4SwZ_j6#_Tt1s~be;yu@?b zO$5oy!`Y-YMXI_|@5Q)3dX^Q`E|UVfu5n>EL`%s2s?a#>7WPj~ikL00e`JClZ9W& zCJKoR|Db84Z4~b3Y^|XE(JCrXTv8&q3n~dFvfQ0kdXlPX6UM|0B_!uKA?qYdA9!&J zGs{3HE$=op4eXqf@FbK9q{W{JAg}bQ6!RvE#=?fX+YGOaPZRLKy*&JwDz{w+-6A1bbBEL^- z8v6OaB)-3l!uFqy;_B*bW$fl^_II23CJ5OK2_l8=TC>{Omge!knls!C7MG*Z5`Yy7 z-`X&bz+Y03CGR=>7oB42O%z7q{P1)z2TOcBmp`Ebkhv!Uawt}aCkhtziL?W58K)c- zdP1^Vowi#7n_4Y#8qbL`z22sx-sMoAO}2G5opuwi|0jy)d+b|@@4`pILarK4swcw+ z+FyM~I6K*$HzwAuyGECJld2qF0O@=F;an!^J?0{1_QW%Lc?kF@X0?&sQRrLS7ADeJ z$3<7VD`+4P-+FL4p7QrT$eoG5$vr30v8lTG?CtAJkX5 z|D8(sP=7e#Zmq%}e@hbz8pG(T)&yxP?o>r74*dLqO^$Gb^(z_be!bxoG=RYn5=D(1 z!v~0Pu$<^7W?NMbM2MD`<#os9rW5hc+w(O~0Q|BvrLh?XcukdwTqdTRF0)hSpu8vE zTjAItS;WW^WIOh{NLc5LB-tY7;^Qzs!=1&m0^#nTEyHnG|8SO+cJJQ<9wG*=TVXMv z3zaNr5HLQRZxz1|>7)h^Vg7tAS@lT#EE<$J&si&^R|1h<9!AkhMT;}xL^xZ6&M7gy zC=GGu{;s6;&A#H@ei8CMy`YZ)jctnSCyUi6xzCdJDR#?A;aF}A=XNhE)~_ND?qU6u zT`1KlgqbCH$PZ^0yY{J8(V-%ek+T`?WO~pz5}&xGNSZqw0&8ND{>_-EyHIhw`}d&2 z7_+cz|4Tv#{tuP-{~1(&e|516jR?R(XgOm}c%kx@rG~Jj^1^5+q4R{V&M!^Bk~g=b z9aP?c2+3dMRdE16lV#0!rhoR`AOg8OiMWc05_^YK#H1dY-M9)P9ok`}-^*dqO9Rw( zYkH$%4SuqVpLkQbk+^kHYi$CKB+u4fg86+BL_fFCE%apoAq(|RG_htgfSM>`*+0^s zSx~{!45~FMjXY*PZ6QW-Tr{Z_I=%oqkiz})a|3&@5Fok%lWyU;aP;_D+`rpl81sQ( z=P$|h-yXjDryc&rap)U2C=bGf6!r&r3taSY<_(C#bDrQRY(EG~R+94yb=j;btwwJ# zmz)tMK7MItx|%zA0uZr|Z-JLV*^UprCLbLaYUIFq zVa{5F!q~Vq3n6cG;urr`VGhpu$94maQ@KC?AiNDJer9$5%S25e|BeOppJw_$l22@+ z+`ra+!*<0A0*E3GAgPN+!5AAODfAh2?>5Cz1v)Ud1f}fcy2(fV_W{NILJDLAF&6$Q zPF<`N-be_!+g^{eEIdnd_3a)&o0JqtgE@pJ=tt#2^KgdQKQ$&HM;6ncdwlpAz6^Q^yK)1j5$vlS` zRVi~hxO+{lL0+oMZCh7^D$#S^%%Rs9absPkL(@Pl2|!{!LSlw^>U zihD)OTACFjzEJa_qM<7Dk%xt`Zkt2Un=(R=>tLvi6#O0suc6BXZImV~M2zYEnlDoB zPbfJWKuy3(^j*8+_`CDS8tn?2|B}G}?TY$8ohN1b4{(j|{nv_mhy@#tKMv2}kBC@} z^-N{7&;YRkehL|9VqG#1i2Ru$$h*Q37LvxMbtA1lBXh#JP5c2A6HHJ{5lqq{(dN}A z)lbu0G3w)xhKC^Ds^aQ--4$@$L~Iuf9-95|?d6)7UBRLT3gj&5s^xq_=;DSRRVD|s z!H{3^R{3DcxazueJK{mF0wu?G6YLj`lU(c}RxVvwS`lTlm^Y|U_&j=_)LdYYy{}wH_^hw(UlQdQd@%o2S%}JgiCjH_6~*dkYJUWW8WOKd;IpEj z{K5~`pBV!6q;%rIX=uv8_VjVsnSONh2f`}EF2E=tC=jy5W~Y!&jyzHHR5*K0FK2|U zpKO>l#&AR9st&cgMC6*{OuRP~vGng%VJ}+Bc3E3H7rXT&0LY&G?np~Sme)kwb{4+% z7Z=)ugRJ6|MX+w=yO)Q`)$Ni*tNuy@d4UF-Q=jlVPlYy91>*7Ls3qXUU-^goVM-q) zOaGGAfc`&IoSLJBvyrLUe;@e!ABw@aMwlT+G%<&kHf>Cfe!+@pY&?AMb0W?N?(*y6 zvXGLJ#~F3sfEB_v^9H~I2*c4A_k^Qk1=Fsku?K>x6@ zOb5l&6x#Sv;^gD9Fv203BfTq%2JDiK=-;~q#ohnI+kdup0wn$t5zzkKCj9@at*e#o z|2|-ts{!kyvV#9x-Z?qD1d}op?L$~;pp7cFh$JLj8EnE<#wrj&=$yRQ4#Wz@-d+<~ ztb7`hS*gsWgDLZ;t9<*u32<%w4A;U87f0?=<ofU z^B;*rww+z8Q$BiZ`wEJr00dlLMTqT2_369@4}0!}eKGZtqy}pDY=qSFr-N&ySNO_P z9I9LA!;xpn=C?^l4wDP*K5y(Vl^r?&z9i3jvYj8+zsXod)cPlG^={ zh?5f{1iZAR1owMNQh(}f`%VW>~+Z$7d!NKOCS%u zi+I+}GaE%;@dA}#lOtP7f7GOU^O`_;9(0zr2ztc{m;eq2)k$-wGmwjklh0ysQ(Wd^ z$&FWifGyl+q=C(hnvxeya7>_!xt5w|8PhK>55icpV$SYUyRefVd6gQAm~b*oanIn( z;9XR<8iJIjq`}i0w1kqaaH!+bcrIJOv1UF`0Q}D?f&GO;VG0NqNN-U$?A(~7l>id$D`-9Vn%HoHQnN2H3+KY3}_$)6)>Mhk=EJ=cKX z_ELfkl9yp>$BGg&WU8UDK~y)yd6cQijHTaI!C_odqsc5pMfh!voeMu9|KYenT!tR$ zN|7p^rK_D1LB`X0Sg1VP!MYdG$aVh4gF!i|^UD@H250O70-Xw7p@FOc2)L3S)-5rX1*YS1!=RQ5B@G24@rkmx)8f@aZ6Og=8F4YUV~h{yg4GlKjHXOCP7*El zd7&`L(C(1#xk-R4u6lJNlZY6NHZC}ZDxwDZL%?w+tZf(Z(u&z>;u0%6Ob)ZK*wI|Z z^h@M0jcYlnlE_ficz}^upQmGtGVQC?8xKu{F*uDB!DTiy%!4IXxWV2|6D6)zrlyJ^GK!aHWfs6eY7W+ZeV#OP~+o7Faz_-U!+_y-$rD53Qm za^C8l32*WK4^Q=B3LoKNg?kqGxqUBqfic*cA&KmTJJ_D__0>BRzsh~v98$(r#+eVz zCVqT!K^yFuTx$K*I~e`cRX;Y*5Yl9o;re^lg0dSq$ua#44PdmtRK$}?ar~lp zt={;47woh3)E&UX`;X*<$#cpdJ>XzF4M-dAq}jUz4JDeyAvB^|aLdFim<9(io*J_R3~INldv*K8h1{a|uk%W;oJSD2)M&SJLWjP=qJ& zxsAbBUPnn=ptFe|>nr3;&Es{4WT*IjF+`BZPlAN7`Kmb~^kqx?h_1WC!CVF344^-j z*b-nh;1Fhc&)#q7+UmV5qyDVaQo_pZ(G)8bA<$0!#E%L7IP95-^UgG3?&mtW104@^ zo(AwjO}AS#?ZnQ&P3c(2nR(T?jkc&yxncWj#Tam<0$43M;G#{ow>5&W#LnLjFO zI8^n6Lr;83v8?e05&L$m*+4uc{Z?G>&idFluV;T9Zmwz@=_c$;P%bl7YQgO`s}I#R zt2^-0ipe^E>Bj$wXhZ43^?gGtt{@P=MBF-k>_LNXR!18JX_eQ-%og3)%``@k!W-p8 z?2!3tRC-7pxsJjHA0i&ZK)8jy@?4Q0CTaEr@$ z@p8`^6+7S!0;L~;@PraOgB2^I7$w65t)mn@#R!^i!p<*6R8K^bFAQ5ZD%Jr(Q3t@V z9aHN-;AsSHKUn&r+zjO5+ziKcWUwCrz8VtgfQ~;j#~YzzpQIw`ghjn$%2CkJ6T^+Q zB-v=D+LV>(vOJfjJ+8`j@3L|rP}Q~>W{%@sgta`D^u8milzY;qx*2x%UKrHd4NbKS zPxgL2IMf8F=qJ3|#o4fRt|l}=WsOF&1XiFG-m#APWF>}C%^`Fzq*2ji!6VD zF;F|DWvJYUmzs$wM16s$S4f-Mk|AfZ*?;{El_Z)_a4*M)oWUA z4BMJRSvgeDA4Xvr3ZhRdZ znr2Qfzm?oY%jL9W^8l(Z@HZMnwEPS}$_qa$xHU~`oU{jIb8E*NP}xn=lO2Ktp?OG( zP<`qhuB-VYn-4shQJ+qel%u_miov?nw5hwicWE(=LR60cQ$8R;NH8 zCTE2pp)`Zq^n41KJ2((R2{Jw-j;V;Clq5px08SR!f`rK2xQsK}aCQ_2S-71O5_-W9 z)LqqkN)SA^s9A9Kuv1#gwXO-&2SunWixmK2L6QTiI_TId4rqN(0*S$v zq1!~}BpJiH zk%=VXg4Gc#c9TXvM{OnP(pWG2+2PpI^X05uD(cgeaVD_X>B-!WF2-EOUo_+_e0eV6 zDB6z@@AViq@LtB=@xgj#sHbS!ynPyN;^@4HKd)N5c@du}#$>!%_c?qr#y;cZDtuh4 z@ArSdxcb7y;Z8M`oh+fdl!R2U=3mQH5A;pm&=OymOg5RjNnAp)Z!(pxfwpZhJ4H*| zK)nh$%J5wK zkt>vPt#Vi+`PV@hXixCz)+2-Q&56Y%mPd-g!Vgv zg!-tUHwehc5g0_qzq8eC5~*%aHyE<11lavm;;XDUTOP(GWpe)DEGW#CpU#+Rngv}` zYD@%cH`ti0!pTDa%u#Ro)c70Mza_-zL8C%4#M653@c1k zZ3^9>%-i8<(TXX&1vdx4=L1n$MizDnJ$f?UKpMogRTEbDbs386mTrGMIhaVIQr9&i zpypr#yAA}mmf_6o@uF0ovuLYwNLoPYz}ckIh`GJLR)}y>FpQE|DOyq*1EK9o+h`^B z$dfI-F{vv_`OQ^{$1J-{3pztBM|y<^(rzO)3>VtSPt;)g?fPS*^Mv*p1?jA$wvy;F?pq3-s4k@0PiyB}Q^{-IDj{F1c5iWRoz6H~@K=F+iRMLVa7+Zd zN@0r37g{Q^2M`E1dC%x}p24ediYp>RJHc63+Rscc+LBK_OARiMRD}bm`L?>P;>F1< zYejrd>lqEB(;^bYcqFH$7G+k$!X}}Iz~epNxj?1}& z>vM||+TM{OnE7{^72gBY7hv+_5^cfC>$Sg{2op;c4fj|C1i1@l%f$PAJ!KFz8_a$z zZ;=ru#2mlTb~-8{oFxLpM(y=A=Xo=19Q`pGDApQ9g3r*~Xj3hnIcZ=!0L{#%xgl+u z1zs3iqGFg^r(0uxP3_MBbYt2TFU{g^Z9yNPU(W{Ef)PhE$(`R9S67>y9-C)DZya@@ zud5{fjR?hJY7ggmFVxYOS^0EkGc4TvIUxcdLQVJn5X40Msz?So!JIVy%xGIepH76%UN2PeWjsxtYFKcdz5Mvz2QqM| zKb@6xf${Ia)OS%S9=C@elwJ{#k?@g*m+4k=%Q;COjA%V`4e$RJ~os` zC$()JNPh)^O+UH&xQ@)Sq1q6#DwK9BwB|#64xALBVp6qz8Q}WoeLvfXRH$x3k^x-u zfu@nRhXbFh?uf0C^mfoZBZr{%kBw06TJ0u{H@R9nvBurwnDpMcHzhYyYPnwn4jWpm z?4nO53B$#(X8n^M?lFBIBMcKy-?v!AF?svrYs<-B&22Fh-?#HmHEU6nx7GAPt$r$o zqSpWJs%N;DnI5SezUsxAQoz}_=#12*1<@`MblQR>| zh!CJ-q^JEIvv4{RgsAwc;D_)Dz4#J#ovY9oU&5PB9;jk!N?m2<<>lgiPfw5fmc%R5 zr<`Y#IbPuR)D}GYyp4D~Rk!sb7KefDF6RWNXS&zpyHq~hy{FpaVB_q>mbs-`Su(YA zM9A3a&C|snFj?!5aB0r@o52v!tl9ntm>R$f`-KJ%Y29Lf+r@Fthxo?vcfhgWXgGZW z2Lh6U|0^f`XTUKq`#a`r+W&<)zOR2_4p6I-7$!#qE~b|vIRK+dGK}m8Bb-QqJ5}WU zzc7c&dcpq|`=*Xw%8;Yt+mK;`i~r@b?3UZ+hiwO_jS~D+s>`4cw+(~_<22ShI80qMlC1Es ztFfeqlqtVj6q-RVR$EGVa56Dfexo(UrDH-_*|a4cN5@>=uM%$%K~gtA4R|b=iNMd)S7(k=sf)Eo19h>eMZ~_i?&4#q}NB~3C_s;K{ z^7c)HIz5@BvuV^_Ouz|3?DNF60RC$3EiOk7w9;+?Fp|x zDZJxj5i2jB1eAeHoQJgx2Fuu@3&)qLz|LIET9pn3Nv1*R=pOW`JD5Rg0>{uYy_;Ik zXaktbezlE+iwNxyHEA@OZ14A*;HSuclJlY>%g9)Qw+PZQJ0YJSQk=9LcN%JVyP!~+ zVnu2NF-CJDG`K)**>b|OppBkT26!!3|6G_&l{FqiAl@21=AYghE*2Hr6)4zQ5*2;_ zST#+X7Dvrl(VuDeiCjcll}@>@-NMSQHrutnX;)r{>{>G>|MhO!4$5dnNG)cw*clYV z=VlQa+<{`c&D{S}FXW+OiAbPXd5~*vo=*6`P^Uw4va~p+f!ze5a{AsxReT*7YG~p{ z?FP}4&#|E6CuiTYhOw z1EeakE@cF21`#@reHdXZb;|hRArGw<+FFaw0QZCA6FU@9+7t_{Ff*b;I1?2U5&vIU zpHMV!8P$&+;;*ehe}I~9`*{QQCXO{+K6@5C@lWhw^^p8WOYe4Yj~JGX8Yrdt`RDbG z78nkjVYO<7%EyxLIEWnCf6oztjpza2U*U~t{oc2VFzWbaYfX|+i``v;3GHT9><>OP zc%=Sg@jm-#P8NM(+e1{g9sW|4w#GEqad2^zI!|#$uNTD30QbI~G!8)q8&Xp|i@KKS zc@+Jlm+dCDniuqE^5y<@2!E@1@v7nCRKCIWe!)`qQy8dEZ~5w8-0w4Z?v%LA~UERLEg1Mr|a+6246B}s;5q@rG`lb<;p^OWLkSXxJVdBBXEjyE|s4vta z=cSvsBT;Rg=ol|m=gCLos;%X0iOO9!)E*3VmZ@Rf=Ybz*x1$0L!NDMkkpp|w1QjG0 zz*5+}#TX>KuO<3#$K8Dku*c0A5bs?BXCf?xsZSK_LPSlhZZdGqm4JDlH3^Avpxx0Z zPA;RuRl_FMd$eb!<1Z<{;K__gzwyD0Jn?K-{eUN3t57lB^u0yqT#^G6yj};=aRWzqD?9^#hP(Y= zmJd^h|6D#;;a%j#T>SH`6%Pa&jkSIfVa8|n)WG_!;Yw7f9V8c4{1o9^@ttMb(9wx! zmX+@wk#c~Wp2$qwt?x8jI8$h5D?A#k*(}k`dA)S)e6r>|)zN+8GCLMpz07O?uVdrB zhB@ijv}$~d>NKP0tsNXYJ*A!zc+J%9^7*)rP#dO;DLqzBXQD{J)ez%!Q%ne(YJ|GL zUYKwUSIx%pbb-%Ivg^lG`Yxj%K*N{U02AP^JFn`O}tk=6hjyHbe*vYi~*zuo8VfO zDQkK0SK^taz;`eyh6b2f;H_a=lP=|tZ%CRNP-l<_ZFO=&=?FN0Bwi)$1y{YcvD9mFoDLEkfdPT!70BorY-nh_-6 zxJ+ery8+WmnqhkM_%6IjQYpxRpm8v!lJYFKm{9?U++!C-K?ZGx=QM;FrN+XG6MhF4 z7Zu)l0{3-!mg~Yfv6Ch#qB&-sD`HqWSw`PopH^5o5!5o57E6iWuN>tVs9~ZJn8m;2 zLScbXK%dM7NL}F5f_N$HnC)&4vD?)3U;u-6g^B9z7~(?QfPz9s8--@*h*k@Ad5NpuAOvWxN*pFyMRibQ_;P1j~~)|uvs@m_tTsePu{ z!G=;+5X9V)()DLUT`^J_?sUZ}Ja%|EH%9NilWCDTLeLo4D0njPn;;Prvl6easZ^N% zc{;2wNqk&UOUvnW-{TFmVzREG0_e$m(HZZ4pf~?0aVeVo_w~m7p%IL=g;t&X0HK67 zCr#rH&y+;|#kC-GA=?}FBfok&J1uY$aig_xp$fFeMjR)N;NXi@=x1x6sk@=%?@+UFNnE>x-|G%*KqYs^`=(&*e1&6JK!gv@w~4IKyLzD&cvnK z@G##{>$))dz;rI!R;kFRA+Bgd*+D(cr3xClv)2-VCC+^B74bVNM10g269k~WjsQ0Y zxV`G9F_s|)ZOV9XxH0yUOiWqeYuju$!0)!j#Nkid;+R){V#7&Tysqd}RvaxtrdZS9 zq)EE4+otPRR~mJ3k0`cY6bVd7BqZU;pCusJ%>tg)n>W!jb<^R>QV2ik)4hu8GIqQy zQN4Z!!_`UKC0n}MS-j1)xn!WD|6Xg?r>CopH1u5N;b7z2zmPopxv-I(LM*pH2%osF zeE3S|M&@(%=%g^`q4jfsuEwEY{&VXa*RptJ=Q4Nv74Mu6$^}nGGdejnmzY?QLxxR4 zcg$C-OeoKS@eSs*K`+l(#2tg6MTFY4^58ONhDB;8E@jjcR@$(GibuZtK#ad$YyOK- zs>J0MX}m_4%y$gFejF^02`+hwejrdJGj3pOTREqt)cTcZ{Oh>DoI}V_2d^loA>+rV zAP2v#Bpz92()-uD&r`^;+Y-Id{zryxa{0GihAr`zPoMj`fQq(|%>JuICAizyF+tkx z7AIaD-8l`<=j|B^@Q3^I&p|kIgAb*jDV@Df{j}M_ov$`CFcxs{AU5URv)`J!0n;Bp zLI1rRhzi$vae@H=Y{CEEm4ojF|56SnRkt12L=k+R^lF8^ok`>N@ICK3-SNZLTHxLQ z$PO8KrV45H4hoiRWl5Wte0jQF6N7|ZI<2KV3!u3+Z+o}DT~_H{cKTl-fb~N<(UNr{ z!idqId1Lro^se5VnJ(~kr{36nRj?g(j*9rgpm)kPvz1+S67Dl4RW)YPQD1;p@7_E# z{5tySqaKonCJ>1qUsKF1Zpkkk7gS;VV0ul#+jzvXs{Y0I0bfX@}xyz~>pS-;&kGm;K+3aGK1Xlp;^(%F|2Z_ zY3#%mK%jx{_wpj7e%FJIV((Gg;ZQw{jcwXvXY`EmDwixUX@#8U^RmD>iT!1%H?91h7k46esXXe?#OY9gTc;Xw(XqcnN84l(je3{apHoG z13;)p2A{ctvKiB$?Q=2wg~a{*6NXUr5JKSCaXpLS^YjIM>X#rhVjsCE*2D~Ouw25s zszYz|M0np~DPJG6&v>Flc9xeZ=W|sdW9!3`X4C27AR#R&!k<;E5+A=)XiZ*X1$Bk2 z3sc8s!LxFK1iyC0Hy003iJyAkbGpV>5!adSUi(Z4wa#0R^VJLXW|y3!Rf)*2%4gZt zqDXOJ8dwM8C_X$Bs06PQAV86|tF>O7M|qXqq%fHo)6 zwb`O94sE@2>aT$yrVu(+wpi9vmE?;>eK}cd)haNdiS-3heAN5Ke=xdUee|f%>9coa z6VitNfsNkhSxxR+1(zkFd$*hX=nyeZ$eR(Xa$eKT1?fuwGBxyt83dHr zZz0}q*QluyM5LwSp*xQJJzVrF7=kO|oLIDfo7mTwF~(*U(W0$Vy~MC8&bbd1nu^9< z5ZCG~Nq%e%q8rz~MrX|Fhe;xT5mWGI3sASq(8n#{bkFC3p&O5%T-NMQrqqXzD26_rXB}Cq0$zto1L$@Au|kTN_7o5|P4aTA?cV zhA(99!+*+x>YQmtX^S32&fr6Hc_+_88y5L5eEYFYZ_3GqGH~gwO_O0~#9JnImumvP zf*o4R%mtyA@^_W5w<@5Hl|m!36`O67${HVs6qed8m-CZ}^&kU442~Yth@2=TpQRWa z#L|-t3)+olTvEq&x8YM4a<`Fb4Rs`_OcO_!lty3$x+^a025y^x95N@%`xaTMsMS*P z<@IQpaRRpY7xR=5jTwy0?^w+!VNEw%%_s6)>7aw{-MM!83Um;SeF;RaQC#zA)4EN= zzZ7-yb$;4OnsAz668J(HJzsG&`w`|0dp~?ezDm^VXkR$)dRvR^Jy^zM$cU{Wet*+> z(tTQH9+M8bA_mh!Z{||_V6>n7$qd;)H&gN!M$hF@GA!rLD|X(&rXeYl;@zDpUF9V? z?U{gnkC*h0t*hq7)AmlseZP~*J?@_2P&RR&SslEOm*DgWbbO8%?;xJlvsYiy-Z{Qi z_*8)KchjM>O+z^`2mnAi`2T2JlC>4IHFlSCcCvGJ`dy1m46Og_j23GpY!wtAo2%d; zk`5(uam^ar-9cf?a#;KnahO-*t;gI??o%AD_tW=$(EuG$BEZ%yOFvX2&oC11-FUXx>~Ps& z?7ZY-AaNyt6Iv|vq2@VlFhj-PtWOi?Zu6pJFbGdC-nG9!E*;O8KA;GBdh~BFAUB;J zAZu*bXTD}dc+KCUOSh9_oL$Z-_~kVp=snPIF_1yo%#Mc{&uTOVMUJ;d)8X*Z zWOj5<^(ewP?3Xj)3@HgL zs~~|G8WN@N!_83cYxC^85^=avs2ZacR-u`4D_%t{LkOCZ@#L=7mH1wDt9=BU8ru`r zQ>p8K)<)mi$eO$(>V6W*jHnvOrj~n#PnZqF8(vHDT9orE2My2Qapl#Xc}dq1s z^+(7?Z{BDTqD9U&yE;t`(?`REbPmbmVsFk=sg)ubt02+&jwES)K00vr%^vcM z11WyoYLv1N_RI($TrcUxjewUzgQ!M>((oO`^_HSP=xdP!26)j&z@yBY6NO=1rjQX; zSho92fSy^NE%}6AJF@eA4$P9LOQ{iug2p<@;#Ner=|>k4o_X+EUBoNrM?841GgSed z%f~<@Im$#ws!JzLf=9CrjkJ`A;{z%HTMIRV9Xf4^V~>x6$Bi3FHH5mHUhVVT9CO7j%J~`b!&JCLCiB7 z{c|%^aruGf(M;%lVI)Nlv=b>{j!;3l-{fxh>@AAdEkncO&Gk>NADvn%L2 z-*`QKVBb|_Pl-0Rb{@_xFo^&-)b!Kg(XmBVt@D#>GF+B*Omc zPp=k8m!NoaO>PfAGm5vz{q@cM6S(RIk6{XLm-Gvg>D`R_Mz3Z9&nSRTp|1-0kX=Nt z5a@f0Kcco^xGYEQ{Y=+6_Ko?_R!TZl6F^U^ z)aRsV-F120C9HL6Jzny~+~(AeVu!0vCDd|DV5~Stse}(FshBvNOeQZX9!PQ56Pi+z zU8b~<5Sl@n(4Y%Tv0`>))0|ry3F%DmeOCUb=0q9+IUc2diE~V0t7dvaIJdK{%nXr^=#g259QDRv*!q18MRS9=r=*C+Pq);T5*uK+QmD7Do<$?ZqO7UQ zrL9&_%{3V+2(6niR;@ra$dVIr5qrxsgsNbGG8z2=ET(2zJjQH@zvaviE&WC*X-AnvS?-lC zKHB!CvghWjDZi(8l4ZBU^5CADS3ao?=Ric?b5jA zPj@Z{gAeJ;H3hA08Z!wmagG=uGZA+aW#RgR472>iBDVUFWy$Ig5E^}n{VZ55kB@N$qSQVI9 z!qQ8P7P!kyvwRjj@M$%50!6ucmY&Ta2|ux5p-IfEXPy(FB<5B`X=jkk=i*Oh;C2yK zJ;nvxz^{S6J7b7JAT5Lsdznk{T46?ig%{(|^DEu!h2Ayb8G-RHXroFAfoTBmH;TTC-#F&HirP_k;+v9;nYEY@CKEvPJs8A1Yg`Q0GH%W zt|>xu2Af<1ZTM~x#HsuUFYlZ=1WrlDIW-fWz@q`)O@s0UwW~++ZkaJua_mI($q#v% z_v{$RC-=<8=EqljQ*yrJ8o0Mj`py;T>zeQk`)+V^Tm6{y1IueF?rQoePmUGs1&eDeFBkF24jpZSA@|N(@WRLug zVdtU zKu7QC1-n26Z1Mf$5<2k^w|ZuR_Pp$MR(hPURFgnOu13f+Jn#|5Xe|Pt3Uba3Q!Q~S zj7_V-)(roFGT*5->B&Id9hfF7pxfXC-O2GTDBtRc#+ogv{%orH>u*EmPR!*LQh#Ha z0_J}l)bp?7u8UN!ZLv5IKBn{-n)PUBL0Ki_I2%*Sg@1rKQQ+fO=98CATmjgidrq@j zJzv`nZQ;$RUP=t3>v#J2+#gLw+WJ^DYtiJ&`2y{a0<}Fw^umxbrNVB>@xtV?rSVRQ zy7q%sM|%WtRMP|(aKM0Xe(EnlwtI5&VYF_svrA3wp8Y!KTU$<1OQB~V97PC}G_vOB zgf?u9PeXyT25-{<^GN{>K%4wFk7QdbAl~5c(_E&y$GQ-7OkCct)$+Z*{Wbu+O(;mzn0}LnDXphdT3#a%-n?P~s!E{z0>Er`^MRKW9GCU!!9^;< zd&8*so+j=_9R>8E$XS%uHQF`B+)SZ*`}ooJVP)M9(LPe|mDYt|inN8y@#MiqzO z0-VcmPB@qj@(8Xlf25hiq7W(2U+|8fM3&!Am^`L1MRo>EKQ3uSbuACzr&b+W5~{XPFp z;l>Yz0*nWZ6+Q8%pB#b6?GRn^mE7!Dqv1HXO)LF?X$quLwo$i>ld_QLHwY?iA2v1* zIwffX_n)%4Egh#}()(89vW0pjRpy4S*iiDsR;+o%yY3EQvQ)j>=ZTIOb4Y|(mHY<> zlfu+Y@!Ex@!@6J93z>vLxMaORgiUt)-_gW}f$HA*;xd=b!CL_^i}Wlt=6aCQ=SnBfy-F zDTvC6ZDa=+Zs<|xco{yGLHQ$}2zYFiq=$P@x-+20sLi_@jV_d&vZ8~rAJhGu+jt1PEgyKawLmPYB z7s|S8hBut~`B#Hm22dUbkMw+ryS@UqZCCcwa!N|~oBd{yu7uwGSlBf)$-J8RisECy z!${O&@|$FdX4@1=W|r(U9}k$>(HD#y{GGc0yOqmpN;#n9eLZ=2^i3q!TE#wNdlMFv zwIS{!_fHraJ&VuPQN`n8i#cZMl~zORSe^bLn4mcB8QL?>M|PRP!fiMac3KAyKkFj= zvnd1Sn4Kmryh*yg6gb+}#iz(C1TSJs8yUx)TKFvr{qt_?6wtcZ@Pv$Ec<5@*P>DY|7pSo4uR!BbPbf!1ea610TjWDoWNlg3c|& zr#t79qJGyI7@ZI%-ANx`c#;AHclq|q_?A2BUBIZO4*nTfCz1QHrg!Ng_Z)f6zDvTv zBxUs_Di+7yOV6+>e&g#TGO&*1RSG%{qEWlrREK7V^s8i*b3l)239RowALq;m*%sQ0 z8rjK=K@BgwP!Y7YxrNyW*5g`lKo0xALlI2P2-z5-ozqwz55E5`qr)|0`>78I0I&uG z0D$|Sp)POW@Lvf+#lI4SSt7ZLs0#8oZ>Hf4k2{F{LFNk>45hbSJ;k&X*7eee9~R%< z0YAz2sRzIRAiXjL4li8OMLhCq`PUI~rfi!8q{jL&lhgL=*R+ag!ql8;&53NM}TzjqR0a zqZB+{t?aoMR_o481Ql!J9Tryrp7e#LI|`c=e7VQo4zuW}wnvO8!ki&w==UiMRvOT#xCFaYMn$kYNlmL!v? z&^ntCj2bytt7Iv24OKmBg@t^Id?}9^dG*I6WHStExRnAa@OogmywnYxdpol2_22Ta zS1L%&YrkJ<0RE?;u7CYBY9@w4wl+22EMj$U0G$`s0$MluZxy0N==mGKt5+!IaE@e7lsx?e?=!EZYDs2%UeXs#{YiFNNY?f#w=kSf z0j;BSqol!L@Hp|p?KUl|S zDF*u|SN8kL{F!%aVdErXWn%rmVz7;>+kf_>K5bZQh^gi01VN=Tn(#+S5l|~N$@u~s zgbDBkP`owgEZdywGgh_NOug;~)wZ*0nLf>Ny}xrY+Mk8mdf(VOEI6{QGsC{|n|QVI zO=aI_Id(Jvcb)B;Ohi4_85T5BAZm3SA2}x&@-cZlaH%TpG=^W@9qvSUQ$=W?A`KCL>c%fK~uRTv2r*s@?kztrDAVJ)W(HAl0 zEV%bBEUB<3wa?0*6Y zqfBkXRD%5Wx|Q8kW0!|rJlr^ixM90I(^MFGfSr7zzPAFqtn(-z!f?NY{Ob7g!$ingETs^9R71;fYrbjSSs@3UuuWRUs4-8kuO6Ist5<5f+k$q1(E# z-sWSJ05jPpwm>^R1g*sRvHQx4ae`p=>8g?nxO{de9)Wu$Q>0aF?;VQ62P4V>kM87_HTULp#u&m zxe3POKBOYfS8}48orSxRf#x3ohZwcI2KJJviI1^wO84-eFAZ2lj2}50(QztCY+$^j zdYzNYE%pthY`o8nIv!y4oJ0H(21*b015i##ses5DeOlG zr6n$2TGeL}dO;!MeW0Y16Q`eK!26K@nrHwt!d5yon9k&=Klb%;bTRGzdVT=yAx0y# zqetnb7qyCPGNATHm}=g~pgl=yVTH7gUn{hVTyGgcpnz&Tw5__$*FVv9?`?Kp+pmkA zD(guKvIxG3q|#v-pFywIldWKCpnlCr+^F4?tW~A33B?~~<0d*jZf%|dR-3l$MQc1v z##9uwB-26bZ6Z^C%~mW@38*_6uY+o#T8t+P7o;={HAS87);=U5GF&0UD8>R{iLFFm zScrQUNHwS^WuC7w7x=lj?XiR{$9$$qe6Hr-(XxNWGyGZC4w-e43v98L5FphuLGMN0 za^are)A^DsKWj^Oc_1Ru+GTv=ddT{;F}c#IedSZ}ifI&|+Mr$%KpxH2;(rcjqd(Tm zZ2Hh=9V|SbWrksqPY09b%ld-=WmyK!QxI(-IgM7S`lqYZVEmf*i6ya~iQVFrc@~a$Ja1VxbPILLJO5^- zokLP?e_~#OCZ1X9TZ;cLAm!Qs%I~3f9mg-#1I1lJVP;X$Z0=F{;Z0g8nBwf&GQi;-Fz-oTfs|&|yFe#ph;OTSg#4F$^&%8=NCPyW( zzWTWYFv$1233(7;DUIIptUNT#&UeYN%MkJ8B5}a%Ar9SONN_WV*OpxXDwYKob}BtI zne2ElZ{Q26uw$2$dzqP=sdO<%ah9^Y0PeaTs24k|faGLI%Ki${B#B_HwuV_YzaX`Z zPj_Kq`jNrBH3swkH@_NsmBBC_C;$LGGynj}fBS#@kI@>PKhYYmr)qU__(lbhx&Sm8 z?Pq}`<$wCr=wSldv?Qcu50VZf6e2#KrtYpD;y@()UV9&LM|UqSF0MM+-32-kVa1fr zHe}z7u%awBe7QW9ORi3?47gjm(#Fnm*KkUd6+o(*_WKUGdthnd7Ty(XB zsz=hTq)MXB+t-IMSMSW1t5nr5Zqu1Es`>v!Ydrpn*2LDME?E?;pO5a&513+#TV$@W z63G{=&9hq~^86L8Au$3*)vt{S<0Hugp95p4b_g{>Du(qBqeqnt69{AMsjbhg(b?ti zq5)mACMF3?AvpM~+tf84dux3FKS03099JDXUjlDbrRmSh5))Yy{1g-Z^1Ah!)eKf4 zU5R!`@=}I&xr{`owA?1V?3d_95;tO+50FBK-e>UR^5n0Ow6x>NT(t$!J7|;psiKL6xeWjJio@5utDNyUgRZ~^O4g$BXD8$(F-vaO8_mnsgaP7sjRN;;!dg5a3Qa33$h z&f_%6u_rAp_B!l;Bx}Y<)_T;bRL<91>Z_c_yB4`^G+N0#nEy)FXaQohm;6cAoZ_F$ zerTZLSAl4BQ(x<y^U$Du~B zXKYbqo-6454~LpI5mSTmABS3m|9Ld5uh8g-qTt4pnZf#dcv4Y{x-uZA?`azyy8*5| zqeK}xd1Jq(0+TU}_1TckOz*CQ4j2t9ZJs4r{PML-!#Ab!bR4^%O1ixACF281^6x zDc5hyCtYX=c>IqaYG4S2t?yPg-JE=16jPgUZTO$k-}gUk;=0vCZiQ9uPW_(BkeeHM zxkfdQojkZ9eL?+=_rG8S`MH%5wzC3OsQCT_YuGYmq(tgy&e1L@^SW#v=mglmwX$J) zzL37u&m^_I{R!3}=9C2)-G-r1TW2OiE^aJR$v-y6`Ecx=)FjplSQ-DEO`Fl5%iZQo z=q;pRvZL?B+>|@@Hsij6DG9i{FXzH~1e@APH|T(9w&JgG1j`<4%m(<>mP@U$=W# zOP?@$5Lg1@XhsUUIV=<~sGe`i!_y1jZK#)!DNgq7CHr#3aj1T5+Ff?JMC(RWd2L1* zxLKqF?}|&Eszz_+4pdjX-r~KsNp|(#b;7?Yl0U?CNWSbMQZ) zyZi^bwf}&w;(pOOC^yF3=$d7r7G`!6{iE+LQqN;H0D{Yfr~vH?6sHAa`kA}{=qKI(QoXE{=u&2AM8$#rdx@bO_r}`9C7@{?sV0%`o(?l zZ|n;E!S3sC>>mGxU5)zX>+#+Ce`2?hP`-X`p4%3Aa}|i*vXSTycI`v|7j_;0V3+1! zuzUXx>{5ZQ+5X0^+HdTtl+;foK0#lqT_&9!iLCxLp_Cf!9)1iKioI*L|sbL z=NePFQ=nOOX4EaKa13aODGqRXM(Hse^K#hCpPM|+JPDEoh@+^d2H=-!H!y3^P>d$F zpsRZ$H!WV~{E3hQouFz_ZWKlOTw^G7s_XP)!1997YS$m_L9&jpGO%J*yB?_~4%Uqz zzD`mVfNC-z0=^_#Ff?LU%U(=WKHP)Qyec+RKpLCSEvRVhgu=6vdG9J-wF)N`CCy=x zIXN3G)nL?LcwT zgsnBD1(hQ~9t?V+!wS8^Jq%+xb5sp9#)b_EtRu?Mqi~Pnuch7w=t+$;68sTsZ_Iui z1}|gfhY#R0o@r&puI<&nxJ@j<8q$+NS=w2!A=R~7fC;`qD5d3?V6wSPATtC3+M+7=U1q&!nl5>)|! zI7qrpz7grd6aE!w%zkcb@=SB@5RYk}$#b}(`eO|Cq=f%t3{F=D&(^$W#v!M?(+>%hye|8rh$%)&IO&;Iqx2dA@!!a zW!?4*;2~s{(Gsz(a@mcA*9#I$y_wZh z;&;&hG6t&&c)7wER=WQ*{<+kp>*w$@>R}vFY`7>A){sbO@+vWSp7dia3$BDF&98s;*Yjp1 zuM=6ql{qUlU$@7;f-gIEw3wTR(B>@e2P@7mEoZ;mk_|}TmpvrReYHi`<451rHO94W zw&p{rc=WOnAQM)Fv`^$&iOdonoy-P3G|F4ZijDmYiGtp##@!gP%gYFKzD}O-|(9r9_%%eQPSSW$mR>vMp( z1=Vk~B0>MZqV?a5wneJz4(qIcau*lB^=t_g375Us>0fBAByuMv!Hn6zuWsFdogbMbM;< zL;;@|3czE|oyx5BxD$2B;J2K;J?dW}7bzsZ)MNveFYqX_M*-YO%hXRcgk5tR2^(Z3 z^$KUW$7dPrm=MRoB|<@w{!uC~2@@JbioWxH&Xz@@Y5A7$x zYN#y!E=IzuzxE7qr(qIVQY=e`B?=5HbN08ILh?OOzoetQfaZ1=L5PS?F?(>)LKG*B z8EJWkR|0F4UJ@D=QMsAXR#VrM21bK0@v6W|ID5VJ=2}221em4NcT>|zNz&9FE1i8M z{@yKN<8#B|t^&O`-k$-F(BaYpRy27ed7y^15|nA%>4EdANB%;Xh4_qh2GpkE#%K~q z`bvx~6zs4}l#zyVzrlL6)KDs%#10pFz*-e)Aj;q+T6XdVUiKs%*5cL0+6t;yMj6oAAGYoapMwl06ngL%HEB^9 zieeV9I$J}X?HovGfK9S)@kL1e*0dAQqqZpaBfzBFyG@9Ljz(*<1j^8DYjAbcLQx{@ zK;pq9Ny9d~?cBNx?Pf)OtVf4O)+iKcV@xppn&J+@wQ03^-6%so#X2ZPr&AwTn@d20 zV)*Ny$9=hBEilkT5X1Oc8C|x@kj~CTzcL|0Xs#OshqsCK95hPX zFGHC(p&m<9wTyG@dIb$gq44nVUIQw!MRAK>FQ%!m^+cur!wz6O&QZ$+`-^T zSBqU2-{diUa}AysczIdGh4ldgr-sMbKo;5A8pxkpDI$LK2a;cWiu zUpSimtZ$vvi%$*Lc9$nbll&B!wac`2X2ua9{-y?CJJrl_XS)62g3rG4`Y=EZ|^|9mh)>bd=6 zx>xci4w0>+*JR2WFY4ItxgF2*-}do|N0bvcfB*pC!T!_f95T*UP8N0s4o*V0Hl`M4 z|F*Ff^=6IG&2m%->tAX#%m&4JCqnM_jLS@Sd1*!8jQC# zl7S3W@de_{912Lwqp_{@Zl};CQ8j3iDorFa5kn(0nogx5oQk4dW$B;-Nyo>Pw>iz$ z?j)?OB~;NONu(Gd1vDZ=2M;>YVK_+YDeWYFt&ORen9Rq^+H{$69JMLfla4=zYh>0P zq&MIa+KzIG?p}7(R-G++Uqd`R*Gx=J(aW+f&aBm3))fp`Wai%(fgc0Cj#^4Ar=|yHMj2)(Zv?7Jlu-q0(k_hcWIZS5a&QYyo|Gq-S3}(O^V=D znnx7i;@UgaN#slQvxkJZ72L1V=fFb4WFN4AtijPfJ6WipMF($iQR&aII4hDc@awty z`#=zYq^5NM7?n59Ye8nK!us2|hFig3^5Tn={iyN{8F?4gzJ|CUd`48@5h~9XHb+Nv zoqgD$&F4zQ9k8fFMiQyX)<^a3A9w1#v3KhIp@i2?qy4n*Mzb4?BA?caj86NX5 zSnAOY=%%up3UtG5XQUi|n0%)^HBtL%5@(~*2QeX3W_o_0!PzNZa4Iwq8xpJJiC^I> zbYt1+EDXw~oY9i8)0pQ=Nj&LepR0@WvLWAbsg-$0vxF6~{eo6<+Rq-1`>`PZzSYcL zP$&BNH&If}@roSuTa2vJ;}6~s3h83#lW6n+ zOldQE?2a5C41QaWFLvxrSo99gGuWe!CXDDKdK{;>01dK>tJ8P=1CRY3YL1ZdjPOW-*Tgz}_`LOXM=nMuZ*c4~|U^T*U=w+L9l{3Ia}OAk;RBAOXd ze?|zG=J&`j;1UbK-n`YkdY?-V}xG(Mn$T2u$m^| z>=9R2!?PSGHLWGhTACO8-8!9*#i{eUW7^@V|Edza>a%dZXXrKNg_Z&jRGgJ1A-#uamVX&3Sfg+ zkHQQ5d{sg4dmL)r=nC6+CautsXg@7%x%NWyoK-m4+n_>rgy@cg%BkYjab~s20QCZH z74HgVNoVhG9NA+?(`UqU+K3>cbsR__QY-Rj9Tu)K-V{1cr!2zODS&o0Xdjcs8)rq)FAOMrkf1&|L>V@8nDT#iA_sk z?c*NSg5{G@#(uYhz8=Lkv1Wvj)9@Z<{OC~RriTY-J;UN6vg_fZXD7P*rYh>1FNjso98U!6-9xiO#!o)-Tk+{UcMW+qo8I)>`S|$uN6VFBbz!yp;ecy; z{3#W)6~&u?$H9=pc*|UAa+M>bS;_ppTug7;3t#_y0CzGzUq;sF;rAbN@6_S>MBMw) ztmiEgRh7YgptW$G-;{%stC_zpj7uHonJuu;F0^Y|N2+!OgZsq4nP0X=^T;YOoHh#~ zMn`1o>T6MiwmD)<617}TiQT(kWE2eAz1IAQ*<7vQ(unLstzyf15Q~^~68zc{aJ_@@ zjUoMTQE_>W$By=+v!*7~o4~j*yZ_{yc(o&W9_{FEaa;es`97EPh!*n%j6u5Sp{AR4c0G66Jl*@MQw>7lTt-I^o`b(6DZ*(9p}{H@;1rw3-giL z4#CD0QK~L4*3Iz4VrpcfV^~=$FFgbj2(C&Ivs*jk({mz0oIIY5fr@^ufu@T1g$;3+ux-L2?N1nDmqGHj-wpj->&Iq%f0WzyiK7oUUaol+sET6%Rx6iHfa4d zUe9?9mvmoQ&5!1-#F&7)ur>kpZOtzB3pKEp!lh}=4imx^wh?x1B;OQUc5mIZAj>P` zLuZQ=L9@2**(Dj-L0fnsnmk5H_~hIllZaLja-S1bI0RO4mRx~lQ z_;W|~kGX|d*xkm!+QR6cCYKlnct8P!plv&wO@cg?aQ`7xA3uaJNx}S?N|v~Udb0NO zz0dSM_?vs zABON_i=k|-TY|I_1oly{^LjnYF36_8u!$)F{x;?RpX3zt=Ml&IyeJ6j*^cG))ivC#-^+5l5 zP5kfg{ZFYIFC*1Yk291rvAV|Ty!hgQglJ9oxYHX@7eNT))xYHuL6R6ri*3RBQ0{&M z{9E1nT6*VY)$qIB$@$3L5g&Z)|AOtIaIv;pU1_TA*VpSL=S^e~DlbW#Nuii#jGGlC zoWR9{3XaCxfO$>~p-_{+z@$-ekmUvh{SawnGVqX02d^q+6=cGBbJp`FJxSmwO^Y#2 zH9NW~16O+fnB!((gtdKnUe{QNqc?UWOe41;O|N^RdLHC%ZC{izYiV=uI@=`AIR;Pq z0b{n<%ArKxGHaf1WKW~i7V$)ml!iGOg;Sc4X|^(c$T1O^UECMw3+eH;&2yatM#u^S z{&gR6=v#|{fi;q`Xk4QYTqsqvnop?qfdFx>7&mn zuvo#?3Pb_qNXS*54AW*?<1qAG)&mV>k7z+vAwO0Q)scwgRHhQ4OxzJq$R6y{<0M3t z=z_Y^hX`F{RU2P$7Yfxegn|g+>Km{rB7F9#D*5{I8#@}DH2 zyt$on#zO*WlsAa*rP!+lZP}|Lkkd3S(> z=j+ibw%;6J&TQuE-L|TTCZ5WUthx=rmKdJmFYiE9`*gspKul@3V_)|JwE+DZWfkjE z5-%zfAKFGMPFF#eLOiLLA>F9-)j!pp@aP)@CS0O7;!Hd?sF?Z!=iQDcuDRd+@;r~3 zUCehexf*1Jk9(LI-J@;^Knx81E zhHwlpM!R#CL42C{7f(|+f8eBq@S1tFO6K2BS+S|e(=Ct#0wKOHPWAw4`#~0#iz9{- z4il@!R!eY)dFkw7k0>E+3uh;{=crqtG^%E z|83R&b2#5W*Q$K4Y#%>@cR*2jXfDvVMWfc005MupQDh`!qzt!eXY*v{_7z$X-^l>H z9Wl(}M!(b_o~ff9Y`=BC&zF}EV9Q`oU#noujzt0PE~i`i;J6zqS|%Nni@`(2b!FO_ z6SEgV!#7pP>8z$=`4ePWB^Tc|2({iDJxqT}$SnF@o9Q7lA3eKW(YF;tS}=7S3K)p2 z%COorO~LM_|A)DE2(Uz3wnfvnZ5x%9&Pv<1S!vt0ZQHhO+qPAySNq@d?mM;rzUOs* z<2P0#R%^|Om@6Vi#EgMpTr1W;a=51GzAebyj33~i!$DRT&b2F&8Jg%+X`=v#1jfla z-m9@WdLDu7OXu8V#S=DDZrlJn91nYj3;~-^eHu#WBF|b#RZkz%mx~fW-0*WK*$aQz zuhCZC7S{n3T>%N3O#}E(-4Sq%a9{kEDt_yZ@PE?Xf8h%MJF4`G1M|TIeJwPamoF(l z5NLCWai8FDQE3bEA)r1%4j~0ouB)XOOJq;_!|w=$u38$+uC|JI7G%M)7@v$-91#wf|Johk_bpmtin8%+pu2?wj9aQypp*~;oQ zftx%1d};Kkfcf3CcW)KEPseiL;(+{YXo=g_R_Mckd{i9}M~+nIUU5v8(N~TuH^=qJ zw#^EP{g7+Vsyi>_!^!q=aRQubiSiI*%OalcC|43opxnSSQb;}E(C-0K4d-e4&K{S%$RzbWazX=yT5JT;I-5VBDvczL(v)C;?Ez-0A3eZ}R~ zNos|Wdr68an;IqRAiGe` zE!9WR!$GWj=xGR>sAHKYU_j6kr*1cPxk#RkD^pYrHJ4h#o9kLYVda^~qk(%2VdzTID z>NH(b7)R)=WZ(=Zy@_~+gyGV6rH(Od7R$HLhGH_JFw)JRvnqs*7?I~&%4B?+R&hte zG!0Pn8=J8XMxPYn2DQ3@5q%0l%pQnu;wL0ZbdN}cbY0t^ctU+$y~$5h8FQYq&C`V0 z62|bb%~0sPpbAp{I7_NLDlG?J$}7jtf{8EmYn)<8>JTZ;Njvc;Q#OZ(3G&KTuU>)SZK8QQ2{ z+Wyasp`7mEY*Q`f@mp+4BPE+bi}?_sq)qx8`YiRMT($M&!)0`&vB0H5<3>i&J&apr zqXzy<4M=1%*kJWdjz8D)l9oR%c^T^MD#}(j^d|Ghe7Mp389o-caBr-h^^FvY1Gr9r znQD9H*d|`g+Hz=@vif*(yy~X&(WQoBRrZ)GqP2Z5#r1;-y_%S z>_71ae#sdCGdKWJPBTPZn-REcl&QPt(L8T>OpfkhxyEK?+lG*K0&|YX^wU@Ah%8K- z*r{Yu`^XzFBZ!`)U=nSa1VKQZeEt~FfCP^xM9+j|qr9N9<>3%ha#Aq>M6`XhBtbHiR7dliX!gW*O&VB9Rud~m8&7Rx%WXYr-vQ5b zF-Ks3iQNATUgMY?z^{sH8T5c1@{*`$6F z6al;lZ50FHYyV>F33sFiFqQulx&S5@?eakA;(+I^vI_(-hKP}92SX>$UUd)<7cVZ| z+mOJ_5H4zubc%GAjF{O|ToMkGoqGtSnV1EA)0(iH&;iHFP;}^uj0RZnfwm#szJPLt z+aT<@MA|^t9~Pk&)E+X+uNveoUsZ->P}ck={smMU0)7a{mGC5TqdL|Y_2~E_zm{EM zfel3ur$hZ38K+^jp|H?QxVn^EAJ>;?a#I=M0%yj$2YNuz%Ou(shsp5|F#)_$Z?5z-mD5Cyb~_Kva$VHlppc8y&sk z=(Jlk=Yp*^JJ~(-6rwre5VycDav`+yjxOHhL%L6NUo3O3*Z?aVjhQ4&u|&gLg% z<`cQg3~AShYiB2U!z!12>G!HP;3GX0i=)^cg`TsR^S!_z1&nf@%82;73kQTXL=_y6 z8uvJ6O~98YV$2~Ljf9zNWv^H>2!UKSU%7<2Bq?;5^|OJrtj%PN#?vefP_8Yw{4EVY zrZF?VtRwH>fxdd*_Y24H#2=Dre(9QZ@T0NIW;;merJyMpTTo3AoK(u@ct%*cA571v z*`Rs*vnNUvWE4siOvC1jy3Y=DD?H(k7zDSpV~|JBAL;?c`_vXONU1dzqACo{rK#V7 zYCoZ9aj=Zp(Zz7M$BbeXTg$I&@zS&8Cuu(u7JrekzQviqoK157=?KUS&Ar3FC7yre z(EA@8;eTTD)c!9jPbV-uVX&@&xVXHDg1I=w;e><*6k#w=*zAgdx&f$aK52a>(Fkg`?gk3>MOtR^ z#8^Ji;L`s?=*h~!bb@q9L~RK9 zZutxMBkE)-oe+T^-Tt~g*nXP(d1`rjozuq?#UO-WKhP0Y4VE3&m46Hsc)n&$e90A4gZwJlPPl0`uRE5@o5D--bq7=GAl!AxoRzCnDuH zM*YRp5vtgi{bDN8n@nM>IrV?n>ome>=hDf+SX^o605Id&^Q0D13*4BH8hADE%b0L& z8%ik_{xMgO>~Kh7hhq&*suHklEZpX)-jWw?K(-SG#8E)sAz#J9JZehK5tX^3coK>V zsS8$5!>fYdhL>pjuTFVN;JA)2(Qx$HSIi;;kow+-0r}D7OfyVcq4c7D=-=B$K?;{? z2D6DYJ8SdNht>Bm7R`W}e~Ud>E}Q0uGUM$;s~y1N!l(9K$}00+y;KR{VYq)UMpJr8 zln;B$;0;w>HK_am`qOf~{iMu)u|h%v{!?G{Ps{zCD6JI=hz~)qymN^f0U1~UQ7WTn zW(F9fmzP0jS=boUSg%=YL5BGT_?^Obh?s%V6e>(bYx8;d5llD2*hk&R%?B(OmsI(M zM50_MSXjqJ<^*2ZHYJ!m(KZbmt=)FKIq@Tl(aK`n9fVz+f!kDQBrYg_zJmdr7WAuG*l>CXkn+g|bZqnv@^XsK;!u z{2ur^{};%g$@UyOp;OcEqg(wWQS-l!FXUoiWb=PeGSApa={;GL;V*P5)RH{Y54=HK zG-j=qg%(B*Y$j;F=fqY@mX)+73W-N<9DF{hZzOa_fJeCz$C_#SD3emsGSeT~+pjHO zIy#?#D}8Q&97!uQ^=hNU!T1-IXR7mZh$F##2-obJqEIQ13deo5oKe?nlTH|#9OSsI zv@z?q9`swNBg(8%pbecn@TC%t)5+cKCge;Ob+S2{@aodXE0EluGJfhCPei_9`p*}l zLFl-c>yf?A-OF*uwdgO3Lls@++aUte&dse^US#U(} zY??zJj{D(`C|G=~7Nfw?!^}V6?&9NNP-G36CykeS*zPV;fy%e9`E-p{St9i_@uZ8% zQAZfUI~eauA=Cu88*>EY%HeNCrNNL5p&vlgW^4k)?jn34NF^yTSW*C-#75ULl+KCo zV|V1GH4u-$$^v?{F`3KQCR1zRVhaTpE7pwh)y(fP*J9X8uc|2S)7HA>7RGq9e z@S@Y@JeYUC#NBA^stuGaK|YhPJ|!rfuhZy?Uh@3x=Iv8ta-75BcHiJO~q z6vM6mw0k!HF*4}yKPxEUe|q%zFT0Cb{cXoopxn6CyetB@fevS>*mlaDsi-=E_;|l) zVw;&_E)kU)6akA8=?U0ys}gJ}?y9s>Qcg(nfFHb1-FXrAcY@j-V1%!pXvH2)a{dyC z-9c{V z3Pb1G^^+5V+;^Q%LFnLA_%TGAa7cX?AGyuvm+YYt&g)E$Se_33(yW#RwGU275UxW> zKP5(#SXDHgr$}Yvb1bWB+!`7ODHYr_fvc685mN84zo1Z*Z#r!EG z?OJ+VV=C%G4K;n>0MYFs<-Y&;3J_D;1Btzdf4Z_(=umDfn+T&eeldY8nbtKFtKpo&5thGr zC!Ya_`*!g9jrk5R=!ND1T6Y(O*p_EP>Fk!(>L=4un6=(X@UOg-FNn*nyxhxt2iLaZ zSj!HzYser*b9_(@lzL+2zyPO_>qp7@xKi%j+bp|fJd3^uHe%NcgFRZzf*iyOt9E5{I1rY|8a@XzZ~iRy{#-Taa{r1 z5oNghhux~78dTt%9c`hhgaBz%V`E_K) z=S`tC$C>2VNl8Ba)Jv7O|0m?n#;kfs-x@mE&yOQ_fZ&^~h)owF zA4bf5w&A@F@Rnc#bw_#m_%Nr3u}jD2TEw0j!5 zItEo5kQl>2lyD)m={g2ART>MlQ-GwEB~r+7Bsyq{+Su7$-}++meBfFy-?NpDwt^s7 z7ItjBS}AUJcg-B4IC1r+w1#0t;z~%6hznwg9K!PKCP!kf3boRP)ET$=Q8R)3=Xdgl z0;_D4n}%y_NtvG}^Uw-u>}ArJSlCjeiUyJ_j$6lYolhCkU~4$Q<9ZUz z$uN^}(T2yl4V@`Xqp(t11QK0Bcf)8Umh?~AQg1Bt5gPSCNVIN}PaQ{nV2in>uC#i{ zq9mq*rYGOPOk;|MXW|k95H+MUBg(_szYabR1DZ_F6R;qsW4kYrEk^5mj0aF6At9&l zT?Zh$JP9~q??_;<{5v8!)-kcSRFOO zPm?N1Srl|t-ijnqxt!||*kT#%Sc#(9R8SY$6F8C!oR}o}IGqMpezb4oX8ZO3akpF@VGrJ?tB8wp|l4a?QHFxNIMF4z_gQW6nd#$lyU~;W)*fv zGJeWO(wY?sy-tuPP5vm^VVMaWk1?%%W#c zDJJa}TtabP;$lF@@o)xryt0&ZX`@+BVc$kWh%TCWoX*09S1v2lH&PGG-1(*~w( zMeBx?ryC{Mmk(4d9!C8jPRh1`a)=t`5~|e|72Fnfxb*AI z2G^wYOIXYVh&2;oYDVKt>>j{qKqO7-8=PDW>`9*hnUi1!anlFK(RCkAx9-aXZ(Q-rocT<(+&zAVgDnF%oF7shM?M_ z76{3M#a$Kau@?L!3{()7id^PBvHNS?{f<4zO`w6hyE3CS=f1(3)_4-4H6KIFRs zB!f<`v5gMih3y?COk;(9I`oSQ(3djMMzks7i$mfFM7tG!)v9nx#?_lcgvaY8 zSun@*4%&-u(*T)ogcND4EmX%^iS|APws}G=*e(8GPI@w!h&!$F)OPL8ymBh+oqd9j@X?R-Aa!_&`Y?-^7ujRr zk|7O@G4)lTo=H_yR8lmx{mAleA!+d<7a5OZi-Yk2|7V7wn#zSB_&c)<0QwJD_y62_ z&Xn&J2WCX@EHz(VDr;1)+(LoB9OQPnsUZ?oqELp?y1`p+DJWQtn?!pv$9o6(A)c`* zzNn5!5|cX7(eWH~@Ob$O+5`EdY0Rv+#;_}(eS9jJ|R)52vW-*ariltFy^$4$KOifl|o9 z#tGg?_;F9f(E%(gql)NybCOP*rs|&j@z5gyOs1Clin&?!l%_o~A|$tD8R*V8UkKX% z-s}!KESTVJy>Q28Z*Wvh45)^rotbD`oFF3@(1g>DFmY|2emCv)k!PwV ziX{qz0XgcV5EJt7>%T3y=2-;6`)fGjKh>!J&=Z&YFGH9u4{51Xsu#K-yP5KB_@n0% zi3%a0()7gH*JF@FeNha%S`f7lv+Tq&C+Wy&{6{8(= z8DXxH0KyI?;k7I@S33J3UKlV|{crMQMy8F|Rl)i+M6{AvgH!U2>#0dl#SSPE%SJZ_ ziCDW()R@wrgzO)qY{cn}ifyZl3!?Sh)n`WP;{&A=2)8z)vq;-99&dr{({HIoaM;;r zu6W#BQ&8z65=wSywG9SROsQoXYI1G(i9|vEOx(h+2?!tjh9%x_+gtFZ>_Hzwt3=)R z#U|$8tFeFu0`tTbD&3}cR7nqz%Lr~xkP*Fh9s96XGA_>J9idsq02#T$64So^HfO;e zD)5Z?JEOh>{|~K;fAzrsl?N%Qsr=5Uvk2rVj3~pQh;r&0s1{&_!cBdN4i$H05Baj_ zP74F#$rxLAdvxyqYJA_ZIg7ck7xj%ZY=0PMi+#O+JYoZA_j&<^L9Vb6DCJl5q*!ML zMPSt=Q0l};#MOr?j4L5H{d}ze!_Cl^jX#7HJ$KR(%LD_HG(ur9Ov-(u0rto4AL0pDt*4vL-_f_Ni-?%Jh*C_v~T1EP)cTDcolv4qkJV)-g~W0d$8a`Bi`MGrAddN65~qgLwClVeJ$ zneO(5K;JS?s{nU)TEJ}C7R&0VqGE7{v_K-aRY_cia((?`tR~_2?Mplq|7R-Q0h7R; zHAdK!%boDxkW2us*?7qg2_^Hcb@QFw$N4CrYIp8nb`t)=Z4?sB`u%0302$#NI3*ZElH+-UmJOM8B zV|s$Zx{4hIdXgiygBdL32BmZdJ-&n7<-)Ei#_~|TLRz-a^q|$jte~-9zW=Sq+~DX1{#S4Q=05?@|GHTJ1V9(c zZ%QxlqI?=RZ-UbJK~v@Aox|%%#oZBTQ*g-?eL!PKsj+bXvRlPDZw64wPoo z3!HKTzy*A*%f(ztQhq)(5T43zGWDE!oxyznF+9uYBQz1&GgPUkYI=6*gzJ3PGR}|{ zMsg5KANMWda2;I+b44O(c2Tt`OycmK0{LmR`2FF0bQm5c>44KOx3+20{z^#Gsk2F( ze^pDr#vb+xw(HCyoL8=<31>{Uk^$tNtm8V?4iECMF+3h6aYaSqBfol3n{1A0E&r`p zb-b8EW&c@mY(<@7(5-16aTI$iRMOU#_rvZi?fq^}UPEuJhC%i%l&Dv_XrNwGbW@%b z7Q-0axnEWW>X%M#w>q(@N{J)$_UNl}HzYh_4}H)^I2-=GDCblhFBiO{#v~ss*MbGO zLCf(D5Ds{L%gD~>8wkZ0fla9EyUz6H-E6`2&w2bReK3{3>d^8Xb)jhmpZnx?M#|2KI0uj_M^^5xb4TPIP#|1tFcKYICh zcv@acW}O$obKipv1i1^^l9EIg0VJT1NSc66mI_tK()asWE|T3GDh8&;%6Z6kZ|RTx z#IP8*5;ohNA$Ni>lZRS<5yF^6TH3MNc+S?|8Chv$9;zpsr@*E5A3a&xft^ z;d~$DRFPtS_qlc@)y)lnghp$l=@=r>E6AkDy*qG308@&_l~}H4xvQq|h0F|2`p1@!j|9vZV6Qpxb+4 zw(8z$^%-$2dA>e-r>vOmf&>0XlvHNJujA{DU39A#ddKAvmOt+YWe*TX)Bg?Z{+A#8 z9qjIe>f=QSFv%zTMAbLYqt1&|QC%w|009AhVx+UP6YS9R`(=vu*WAh0tHO#Z4RAke zd;Onacj0S_y>XG#!|sFzzc<3wJ`qo7gk$bX$Vp{6D?2qCOZ;xy%%>rG={F|jyS=%` zlbG{>V|m8hH*8oVV>TmhyEn*AZf$J2esyk= zq+1n9**_FCt@BDm&4K0y7oe>*_%6%@6*MI9JJfNC6>4{Tm(0??`xal#{0bIt&Gd>EuOOMFcV|hq zt!I}uib=L@bjM9L&Ey(2`Ypa*vTNEfX!Krkit!aN9-qn8cQjm5TmOzJUM1x{WHegg zMJ&zX%M9vx{sq%3tFPS5VgUKkC9=C)_lW-dmOAS_t9=BdkfCuFGof4kt#1GxUNbGwu93T}_jx)oc_n5DT}{7J}&oE3JXGnf`#y z(>XEd-nZv<_D&mKXUVMK2`uXk2hgR-Q1_YHaG`!eDKt1&x2vi2hrI4g58fl_f?=vW zpwy?R*9)I4RwhT1rZDm;lrB5?c{RDT6nCz8<3 zpbTzE*ctAH$yQs8GDEL!Dx3A$I7~Ot#_2E^Mya)_*g@Qcv9(&>iLIMiQ)ezRS8=*c zF=uQlPfg83)vj|YrZu*SpOR9QQ>KXZkHvO7$s9%yo$nOg1oi0i%r$ZK-~EC$pTH*X zeTwRmO+Ml8o;+!25z(Cu9Q+s#6Ye>w4~td})?&@D6z-1O@fEGX!y5hZg{lm9y+|??7g^be;w~ z(_;~*;lVpnzDEbG=u!>z1=D8URjkVBlT>Z110ijp4pjPkuFyz#kAe zS;d3Hm2E<2a6t#9YT&Y??e#Mh@EfZDK;}p6zPysl!MOVKE4AU{5%RUB;52j(C#wfC z^RU;k+Uu<$+AG(~y{AVOf4xZC$(q6bKLXWMaa)rxy~@2Yunyjp{BMq>!hvth?3;BK^vwE(74o?~2m)AJzUC|&Av~}jEn@uuR!a;3F+k40BhcWs`gH$;1Ps@T+(#!@C5Mw~Kzf&MJpwQfmmGF-`b8={R&Nobua$zo=D zq8RNMHGapnu7x81#uXHw@VZ!;NH=Sj+&ySF{wUjZLdDXeu;k8~ zfu-|_i>eM2Clw1+baqYf?8!YtESB+?CCB0^5!b5ifpNf~|~zBo_eYYf^@grA&kqWgkPvdD4L7`>TdS z71}5O6O@0Ixc?#F2`^qK{D8*ID2pb~oUc+fzhcF_RxN>8n}-e`eO%OfA~o89ZxCaJ z1pDQP3oL4Iuu4t&JbP> z2%0%iL}d2`*AI`I(8-ZggR+siHkL!qDT|~4JYj7^!qC2%Vlp$Lyh@3w@3D5#5&iZA zvUusx(vIqp(lPg0N7Az@=}ZA;sO9J**Zc$*6H6JkNf-<}3;DwTQ}U)<(Na&iX+}}q zMMl%DDzWpD5;5xcNk$=@sS&Rt>-XeWp&lzG>1|p1jOyT`DG^UFtEbu=nf%W7s97VU zUsT}(RNc#AgE<3){Mj2lIF7B)P`vIRI{?&^qx{EE*P zK8fK|QbPM(V70MufoGP-ti)N9dQ00Q7=7pTyKO|rhYuROo zrGof&`C!(S1|bGoZ>HH_eMF~hgOzT2Jjyo73ZAz7l2$Do_N_avp$jSJ{!ZNQbPo7;YES$>CpG_0;9)5@F5M@Kz^zhVLXorg_8~qtM5oy{Z|X>}N^*6`@_bpj5-S}J5?RB%tTO2`RRJrF9g6Y` zcF4g^pM?37eHqN=wAK^X1vw_DcpNy;qSkp&8doUa#NKlzyQl zD-iKfn=P!suAh*qPD&RQU5XNyMocB9~2yVObG-e7b!UD}bNNMANV*?zxJiJ*V-;gqL%?%pJ2bh}i2P zb1V50X5Ug@SrZ&q%s7@nfi%9fO$qe3yEZF+@Wc<#)55{YSHjb?QW7wLeI(LogQNAZ z#qN^WWn}MF46;p5z}yi}&>TfH`Q-H?9l_EWzUvPq*BaVjQWN7Ur_QFbRA6t6>`7T* zZVbVdgF3zb2)1G(Lia zU{8S9U{p{!3Y4%yAwLw9@_C0eI7#Eai2^QE{1@eg%$E5;Xd^-qnqoV+GvO)QU1eu0 z$5IJyk&my@Z&J+2W22*A+ppK&-ESY!?$ZBbL$yYQPeVw_3CO=S7^j(ic~Z7Y?B6CA*O~Qp7}x04O|fEg={E;3T5+L zQXi=;JXpe(U-p@p-7+_~Lw2NmOG;IR1e#HN6aC2#DPJvyVlYMEE-crG|B&`noz z&&oO#a`No~x2y7ja(N6gbdmkbs$deK48mLm@#aV3aK72qIVpZu`gDpUbeC@gS=z*P z8Au2h8YH2UQq!2qFhaoIY#FojtlnhM9>#%zjr$iW5KVJ9F1%N|~%2>Fn1LEa6mZ7%vtJ4P<_PTkVve zSlel0FFvbG83eI~WJ*$is8A>cx@N+v5II4(Hkz5E6wanB(2Tl?r})spEhcn=vs2@j zmIoaI$cn$n;}0f+aK_c+wdccopD`S3FH{^8^>b=@C4I!!*-W3GvHr~UsUBxBA=?_x z$!WIh7P8V%D(NyR(kZ-LIa`jRq^UB3v&jUcZ=oyWvnAi;1KaH4A%#h$Ew1usYMre+ zyD+f|rN*NnyJ!74bz|B!QEAe8*hi2n{jqP0@Dv;M5N1Q!r)QdlT9m+sBgL_K?XXCW z<>x5jkLQPipP9}M*9W3?KedxD<4SFaUtVvzYAfWJ?8UpsM91G{HiWgQ^>BkI*4Q%u zr4;V!n#L!Rp%hQW@#mcC0Zub2&Vw@X6z1H-<}nP)16Wk4XF3|1(2 zZMg_@;oV6d?k~KuaCO(?F%oPL1dxJkMf()EI3|d~iIIeDBy}e{H*k@n{Qb1ORJI|u zl5XInWpNtG9i7D%hg!!AJkJvHitRJT8UmbevjU!LBa;f)wb?BbhfX>uj{-6j=7vCC zHFr&UdV(i>YtSe4;G;7Y?G{F8>rEVWTc}b))5d1y7uSr`G_}gSyUk`SXM1)}7qqe8 zt*E0%BmLV*_&<0^&`VzA)+uD>Vy3NKtCL)xm0Hnj!?I!)5gtAv+*V359(Ee%TyXIS zR>gFR5eqPN#h#@?vN%oK0())^Z^-z!9iKh?ZyeL0^TG|W`$$9(`WOW%S;V-)oiccv znU8gIz|R3q2HfRhzDe`Gi%j`Kd>4)(^zZRsIk3tmq!kTrw*z7TYVe=2+wR}h}&x=;dD`pne=7Er-??c58 zxO{+*H3I)QrFb)7ajWIg1_-wT=~kpo=bK0`lM(Y)CRb{&Stk@|mH=fC{ef;&)40utkAvrpJV?M`v25@B`-rB9rnBH;PeyK6x+a_lIB>yY+A(JBku0<^W zd^ua5e@7j*Fx!)IcJuOSpy?GNJRo?M-gKZE`T>+1_%i~{9rNR8JY)+2JvT^hZv4bq z{~?%uY}bu~Ry*Mb$m}<)(*=ZWEki69RDes38dVj{|PNZi8eOi0QNRO*=6w;?+*pN+9`(5W=UU7^F+n zDt9AE=o=}05Km<4gq&vh#0FECkzap{`PfUZxi??{0C7nFAwVb?893URIk^6@%cW8! z(ivqD^{aDJ%6Jtf$Olre3kXEi2^~qUlnjAKKW2f%4?EvSJ$RE5GiAcc#Dvgi4W5Fw zk+z|+avSRBX)^Z%4(SliZTa!aeD_Gc$nCQ+S^DM$nDb!0$(FnQai{zB_13oM_2pN$ z*B6!#=gl=l(oT+_03wKu4dBxLECgQvzTE@y>m}r>X&pOFx?upmyyMKRA+9hZ8MeT8 zk%Eg7pnACd8k7Q~?AS=cbYjGBJHw7Ov5CKM*~b=lF`~we`MtPvcMALz5G#gy8M5L& z=IdO^{FGio{j#AJBG0E(IamrJ&((a3e2Lc>D~XRWj76;1zC9Kg6jWOIfNIz8Ly`4@MyS{mVMFw+DpyIJSx<`I{#4@+*IuAH-__Oge+ zNqTBF!1LABcnjezVa?35e*_LHgg+dDKYyxmS}8ER<>e?%B~G&(an`}G z8-j7G(~Fu6SSEikocf-D%zOm0wl?OXa+d5s5;qYJx}?$|nbTV7aZuP-qz^#O#7uxj zhC}P>AUoc2Em`cB4S7LXUDkFZ-p-wi9Gp?0c+=nsu}^;!&psR=Giw_lZ3oszmh~$rX1MP1-kLm5E?p^qD zI`uQ4v=gYbkyKTBm2>ut%xI%jT2f10P^(K_aNiU)2S~xR2eDlg1~p#F!^Y*0*H%>@ znSJupTF3Dw#iTru#kltqm*-jmH#3K?sgF|PvU7u&r0D#{x7A{|RP_Ygt;kxc+E73LW%f_jfonV*+8BKT<)W#wVinhw8_fn zT(PT@LHh2DvJ*WF*Miw*ed&=0dd(QXesnwN#7ycBVFyAJDa|ifjL60iy3-CJv@I~% zUh)y$UgCeLAF%RNv+3)0A&Bgh8;`W85w%_$*s16lT+K1s3ix!MS6eRo67WisB6zSq z38RZRJ#x4b+_-X`W_G9eYJ6}ay&)tpx1;P84@Qp^p0RkQALj>gR1J`S1F7k_EQ>F)nZb^;&K1QpUado)YgG@_#Cli3IqgYBR>b)@52#t%sczE zlfUA-h29V03k`xD%+l{FyWk=LI(KkBJ$U96Tq=qrg%>bI8eOsn z6ggg3nfQdGvlBtaw**5z8rm#4*{v1~T2|sU(9{B|tOo_nZu~L0c>?YnH0-qpJ{$yW zf7#7XLm)CYF?j~t8piUpEVib2)+VRb7%CjJsCEE%YwU;91y*O(s1j>vp@J$}PrnKO z-rXboQ4q~ZkOeAAlSyi`eZVKpFIOS=v=1}wf~kFWRuArhndz8Kv9@IfMc{&WEB(v7 z2J~n7d8Z3Ld=aL1^gF#S-cwH=`lK;gE0IX-icY9sp#C51=#cLS&u=g*dq|iN5;EHY z_lX|t3vG82powTtlP-GrITke)S!+ygRTJeVA@KBtU0Kx-qD9sNG7|*zcIiDcMU*t? zE`XeDs)RR)SZ(~1x#g;>O32B#CNnw%XJ!=~43UePD|IfDN)?t>diJR2WV63^56_qP zRPH(+<8BB&UsF0h0$|<~-rwmZKABIy;@M&H#KC`r z=oIJ?t;LwMsEiSNwYqJuc~-F!yr7Mq)tyVby)Wkc%)`?Wy zKYs%E{-+5JuORg5DEUEcGAl$S@x zUZ?oZTS}P^bm&0fWMJGa++eT+_w*r3`LYOC>u~O0T~?SkiSte7qi9Ojsxk4#YXL2XLC2a`%5B$h2ASJVVb&gC2*e+M5|Dm4j@Q0(2OUyRanU$ITz1!%beCAc z;`HOU51`Q82o`KLo9vw7G@V8SKY|Wi&7OwwV6}VRhYlKTwdDOwzb{eUV~&x#MQDd9 z|2kBT9VA9=wly)16D<|B&nFy=X{kseTxb@76B|JZv(IdBCpX?EPhEU%H(LohUi^IG z?N&2?Lroh2iL7tPK7<0eJ560~kvViR=>U?`>}SHLQfLs)qBnwi0`!Fj*;Wul7yM3R z$ElbtWTmVl#CtK6M3Bfsw?A8z*G>>{q3hG7yuZz|Gq8dp#QX&N6G59Xl9|OZ(;Pe7dVOXQZmSL zDBRY@C%hU31c&y5;U$-{Cw-(tuz7@h2J%L|Ruu6Z>P3zm2%kQ;#AjPCjfVaF$f02S z!)uw{JwJBUxvw47sS+?)ueY3!H|?jmox48P-e7jPjQ2x!E#TX1r18{T4N8xLH_LojsP1scZAx#Fe}&r`Gd$FZ_<(cZGaCo*+P! z=oE3fH7SC8kCTt>#({=B94ApvH10f}*M@$TVQ<(dQ&+&yC`m{gtwM6SMu!Qzj1%uJ z0EVUWRBH=$sjX18HqMm)Qmm+69!EMssO5c+(}3QHLvAw%VqP8mn%mHT$(+k9mzFA( z$2*dmfGDWX#17(EKtBu)Q0eZvF3XynU3OW84WD$3Ft!(=Bbf}gj7LvDCxAgmqE}}1 zi*sL%?}~J?GgSs6pM|D|MBWs8AjlE~67wDPexqiXl~Y~!cNhi-D)l!e=+9;x+ExrE zD`c22z(S=AnFaA2X54yn(a>AKRB-y_4&d&Dp8W2V+pUC4nFY+?vJ~Q;%m&zgc4Bg< zm><0z4NMj9!EfX*TJJ8pNV;w_@RT8!XLNx%nnb?-B0*wdi9Ynb?b23QBl^Tq=m($b z4$mom%qL;s5271`hE+zR3FjEO};Ycb)G9fdn_&kRQ2K<1hbb_EzIX(Re5)8zr zLv^mg4eE=8PDpq4v5)m+gP))1C{JO1+SG&sk;k-zkU~@_C!r7z+6OI#yp>P4mHqqC zrIua<<9N?6u{7BucY$HuO_ses8PyT5=jN;2a=f%p(AmLd)G zJ#^u~?ttBF;n>}_dAOr!$$_fiz&pCNA2r22G1$RiQ|*RTBFB` zTO!a}LnRb;_5(drKWUhMt&KFNeAf_iUS(1v4R2vOb~Tc27w2S!kF`7!I}`2|+2?XC zh+dlgrjbZ*YsE49UFPI*3}v>8VcfJUdT$Jce3U9|{=+Qz(|PoNb#@j|Rc_thr&GGS zyOHkhF6r*>lx~qu3F+=`q`Q%jk`Sd!LXh~j_uP9u?xURdemEF|b1Ya;_ceezM&!#Agtqx@lM-{i`L}dqXKNrty?aU2Wt;D<- zNr!>k8gw<@GjSQ;rJ%`9pPDz9QsEvG*Tudysae5=n6>;S(|KkF(onm?w$$R;IKb<|lC_nJ!*joF@ga z%6X>e){1M(XDNtWKprv5)U$f`)0HygH>sq*RE6o|SUi_q^xbk-RAi_huorD_ioG8Z zz$ENZy?brahc6ahjQZk;C3ddPglUcPf$YN}NBm8GL67C8c>FX7!E)i|Hi`E+YGQQl z3)&fz&G?!%V@{@V{}amq_O+Y={}WVFsc%5a`RhqVayt`N{gxg@eP;s|P%!iN`&+d_v7wO@nXboCEgq?qPFJE z!&oC#TyvX|m(D!(*g{-x#ti6B2c(5n1Sj>CHSiS z>9wmNHJuG^h*Tr!SeF8Kx}xb)QQr7zXC`YUsx}qlE8F;fLr6Niknd)T7c9|YAx4`T zhnS?Bt18mlz9a5yyCH;ov+vH2OY`02y@zI&`DO*xUu6i&bxFa+sY5~$l}saLXrC~Q zh;ABl*Q)a0?53N;3@SdG=@^rpiyq!P_@vP>$3MKlmDT%DYaThV&pAV(_38kvj+_uT z(cXg_R&Cp1Xcm-4JFrVtu+Q!)sFE?V2a{e;nD#Ei?244Kpm%B{oc^^G*MQZXrK3n% z)RaimHY7I-+7$L!$+0*y%(W#5{h2lUduXy8|ov7`Bv&)rP*aFJXiF$Cww2m@q!f2DN%a7f9a7y#?Uj zd)jYaUIdgyf@|*9T?wueWVO7BOi-hd>S1l`XDnsqog5Qy|){+%0Z(Wh+phF>hB~@xP1Ykf#gJ!xS z>AM7_kwYTd9BOF0-3tU>IuTufz!|Zlyc=DEv`63?8nMS+oF3iPSq5W95Q9=K4Gn~3 z`cB5MpH3vy=DSCTj6AV5s_u;@5_iv_n6Io6M3u=^+l;x=>;4(yjguFBwT`(QKD%F( z!lc3tJ|ADVnJKXa#9*hfS=Ly4iW|qX1I^{2D9cl^eiyuFe9Wc^4a=5CNl6(!9&Io= zs|j)VM$m-|UTaW@kXw%~PQ#eSE$RN%7eN zpTxAnDPymD5_)b@>F z(Uiq(#~Al|Ruwnq9j~L;*0F?{`G`Ui7l|^3Q_#AheNz`2rRvu!PHMTrz>7k)fvOKF zBKwLvD$G{32=SvjWAyc5{L}=c&u4Iw0PBjNrDHFbEDqh$0~cZRWwfd^nGPOHJ1XL^ynGG&womg1crkmb<%8Q4oJiF8!v&!+eU-Ly#WQb;a zmP2=|q72skBaMlpF4AJH#PJY&J3b0zC&I3Ty7pXgusXpZ{wvePR0e1i=muRKIKQEJ^xF~c5vd6>quXW!(YT{V zyxBPH$NZD>MD_4$6L!syEgr8LX%UfO!@667{Os`U4269S#=9n-qbrPM1dme1proos z(nEhR8vlUlra|N0s%&nt+fch))Pq>Q=AG;qH>zJACc?u*N^S_5gu(5Wb46BB7Vl*x zRkz0&eVicv7UO$RmDx;gAyb+;fybzp4uA zrIb^C6{L-{g2xjH-vHq#41LRY%o#A$Z<3hwl~LV1*pLtx1X7MUTyjFXID;yMzP?3T zzLvZqstLLl7*Nvxz|T)J=qin|#bFEGCD<#nTmJ}wXPaNKGw*qidIG6Jxi`rLy;C0c zqTKaXqM!?^86RzTZk@bTY!NGLx5L(gA!6c6=Qy|mw8+6DXs(uz;wyv&49>`qAm_Jo zV{HHDOu_|W@-o~tiyK9d7Z~pM*`f=zv2)&xG<6_*4^&*A8C-kGXE)9wy7mgDoC_DWcMHsdLfgrUUVZwN8j9}O53#pjR7%)XDiiD$Lu08H=f(o3_0)o4_Mp_V`nG#(vP(1G1V3q@`AW>`QJyr zUH4%4eq(OS^Du9|cr(F35SzD7y?<47VaR#=J_E5RYxk?6JFj0_Yc7kt!v)3dfuZYs zww87x#M_GVjy(&(9O*d^iSaZ1(}+|k5qNdJYh+>^Nu;?hJyEr~N6ND)APbGUxW4v4DP+Qno^-hiDD~Of4!4Aj6Jj;9wP?1k(Mltpb>iy*H0NrVY=OjE zI4O@~xsn!YIR($L+o@iqNU?Y=2y3Xd*=b2HIf-t8+add#VZ{N|WF9~q)RpsXz?MR7 zts8%D0X29lbpctP41&uP!!ZU%;Efj3OuE1BlHgG%zI2a1V)0p-)mD88>LWBP;qYl) zWQ(0}2Ue#4_I*K#ot%-c?0I+YwGNF_S*yu`Z<;<7$NX2=dJ^sfn#xb-3&-7oJrRb7 zVH)2VG?#M4XiZm$0^+t1O6*SGS5gxv$dALLnN(Cdt@As4*@_{TyQ_F1YeY*o5SR1d zji$uVgo><3XGDvo-`aCmEAQ;#lZMC|o)yHAN-XM{S|zV6q_E3?l${(70W#A(#^Nd6M((*#(65yAJ|?1{;WOEcQC|y}8W}Oe^`>=~8Gd3Kp7w}KheFChr-|5Qu%<(s zcz=2tsf)}JbO|~=6|zY5bc@CIa!#>2C^n*0h|ofpDA}}jpYX{X2#8P#Ib3AY@{XbN zjU#6kD7+jFeoJ-OJ)TNo4_7&cn`Iu0?IgWg zx{fP-S5}}O1H!NP$9W-INYKSKV>P7w&m1|o&*s1M#8KY!y}e^kpB%itzJRlQ>G2pq zQ{@x9*%xB8q37Uv&)k+&>qf^VtF)$ai8x$WXa3e3u5Oz9b)K+L1(UaFGrx#y_U$)! zh+!CH`ha-kT~}iIo#6^H4+eK@Ga&+;h#JY_OGv8CcM!r=xBOslFOazLwa*PNQo~_Q zM#DFs>3Zdd8+7A&~pj?9#HXaDr3o@pdmVOEm1O1gY8o$#opnvCi5z5$}k?d-ggf4 z(EVFS%&ga&6nx(=7qfSR5>Z}6B*L?@ z|7N`itd|b;wRZi{YML9-V|qXeuKMHBj&!YDzv{CR3VkvK24(uibeQRcqoXW=o`hf_ zu8HNyg46fn>u@KZg{w+}>EyB;?ewk)t1Tv3AnZ}RK7M+x>nY%t|D)k^pV@t zAAK%L$JQkarVCox=_fj{MX_Nt+Lo^2-|Wu1NtIeWV;OWCj6_S&gg%7F$#deut)!01 z;V7Te&Kwv^v6zqgf*KOx|9mT$AH~+}^-FoBw}H7yMg8-(aP)hD6`15rmPOqZ=SC`C z#2DZd5(~l66VG7uK7(P^4tP#neV0#RfY*gdN926jx!A`c%guwR~V8~M=PhDnqEKuR=>k&>RfGZRZ$ zFy1^(bB~>&l|1t*qes1l{29V)yPG+Cv`gjv#A!*p5e(&i!kqj_2B>-Db;N~S8U-V& zsN$Oy0hbd(hcQ+Wv8|n|M1DLFy!xW_{BIWcf{9SoWfSVj7<1W4{mj!X{mWioairi z2=Mf&-kvJHbN^Ppi#vXNaltO%p{Cd4be!!MZ4UK~s?A_Xs_+f!_w?QEAtcj<1l*or zkIvn`n-3p8e6Nos@*~;K@o;@FSq1??meq$T5LDMN-oCbo2DSTvd!n@`Ra%K_A57#` zD=!K^p%ZP{pv=5DX8!RS@wvMqoSkT@Y{bqTF5RSFFxGHuXuytqJYP{=Rl>kW*$&AC zToKVhdK7}uf(|q*@F{L1{y>!j%y^&HuQ<^pZa7%7QI>0BqfG|cHH(T(s&kYU|d<#E1s`^|#?DM0F$wr<4yxtDjgGq`V<(JjoSL=#ucZQ~g76)Eou1VHDUYd!vs9fpF`VJ2fBr&|z zh()F9GWG_Dk1!D<-_4{!bE>V{A|m-Ir#KEq&Ay4GLy&aCWiPlm@Kep<`E+R#44Hvi zjA#(d2`4h$)52ByENOBWF3R-i)?#VId%h)v*VCIDB!X)*U^OMQg)a?u-aBIVUh!rh zdBB8%Ws6J0%TeqxkTHSOum$T>t9P@OW*0PVok}?WI+N~m5EK!tYIsU_ey(T4X?!HP z9K+lu4`w8yBVnSAS8k2t?^Kw^86g%>^hOr+YBrZtNS9KT+QXOo^IL(B*mA}TuacT0 z@Ox1mskzX$cc>n@m7c@WkPM(JogRRrA8BV26w3m4)rI zo}e+eGNUi|U++n+r_+O+Z!92+gr`lCLgo0xhcj#*w#4Fbwb{8M$AE~KK>p2EGrjJ; z=On7IzwEH6yfBH|PB3MYssb0$N%u8)AS4-vz`TgSkqiZib*b1+>mhRqJSANM3UVUM$CiXC?HIXfpMeyky`%(btJeK2)>mzxgj}xiQ@JYIgBqK-ygc*@Eh1CQ zv8)jtuC41Ktv+RXVQQqxqLGEQ-;ZQLgfkB1mj|{Ceptr;=)Z|AIUoUu=I!#H%Z&uz zWl2e*{k>;|$J9|Q2w7ihXLm=N2%UvxyA8sHJMg_m7pBp~J6_1WdR?xqyprpB<(^4A znX>JpweBQIyTA*^zq(%;HEjnv9@4}53 zYs}lVbf188rDM8+EUccVubUA9X_vE4w&JV@3R!i&BJ8izt939*M*7}Li)-5R(b;fe zMKtX^rP9KE+jaG;cIzh3H53wLe8ofw6Gs3;mo=HY4c0y24PV8l)`%|L`T@J{8=W}wgF`h)(9!o< zB-YKFdpSZlK3_$*aR!6d+sYSShF%CNdBHdD=3o2l&$%jfSypOiQd%Eb84$necV;&* zwQ(GPt2K_oG*RF-8cwxSZ!w%gJb*BS)4o8m(a<@hBY1Q83ccj0FmTwNiGWvUY2ejp zx)fcud&M3(_tEK54MAtR?kH=g21|ChaZ8?vE!^Xq2_h?J3MzSr*EGQ~MhWQU=~_Io z{l$bM-}QA6AEZCZ9rq=FkyBqL4bM#axN+?aYxEK?ZJN~jlO3IB%09xj_FPVh=ej1? zSMsJ~iOROBONmnT(WAAxMij}K_uqLb-tM46Pjc^ckX#%(ZDpx|c@jjAc~$gfF?)q! z<2B#G+x0h3HSvM8ca*|UZ@o9Rm$WROR5d1*PmE375;Iam;zZ>%*@B?II!HuEpy$fP zPm6z7;4+fSR#~F=VL9KBSi-#wasW1oNjm;BGK_6sPc4aA_cCGFfJVAGFDfD4h-1UU zax)BDcnRVPwdDz@XugS2vI(e}Q)Z~Z%Y8lA!?Ue&{B8%;p0Q(7+Ge54ye7AvubDvu zVV0QuRG)VO(#SSQcG%fQ9h6m)Q46>xyv^49Q&t^~+GGShrbSFxR?6QWMF{a!*kq+F zlDc>pHwLy9KYJggt*nl`2gh|bfML%vRqtg;*1J!Ci#4f>f!3_6>&ECHw32Mblpbn`}AG$zpc)A@t$EH(Ju9>Kkosp|qt z-KCp=@tD!~YrAt*d>4Ey`;^#?v%yjYVM`d&XKm(&C|Y#&0~V}djqmF&Y8>}Upci*HSwTu^OzRRo%unCua_fKU^&m~E8~m?pt*cuzaDU_Js7=nXG+7zmLtOB6cxpk><1<*qen zx2uf|~q*lk$U%68@9%v{ic|S zLW~~}%=0=UU1Fj=Wr-~h-YY+=B&MDSS}1*$O_6B|dhhMvGwK8uJ5Bp(0?|g3!Y#PP z=kz-c{${oCZ?}~%b{T}`Fb9) zj!kD`P<)O7+w(zMD2w)#A>)VLucT@-M-u)%BuWyRx_JV0Va%sO)xOq9HhUXipmb9) zXsRW{aJ>0-*p{tqsX8$6If9`av%+UPO>;HhMTr(|+ECTx4LB8(p+$Z+X4ptl;9qgr z^0$OjOJSYli$^dXFo;LYi#Ba#d3{nTCn6#zUD_0@TlT{3dyOUB3!%J;EjQYzc9YJS zNcRo+2uV1f`RTi!-2hq86RIu?yn1dOo+wDgN*sh)d4u5g@n<^HsfGdj>tLsp@pSC^ z$)LF77V*!)WeMe++pu+;g!^BSi=l%kVMX?;ue|5Q=hW$oM$=AF`nnF$A7>9|)JN_# zx`dh6Wn@_lCRkR}&H4E&jlY6TbHR~wRjggqmdk?f`%3Qbbcu5<5g%Y4o$!ma2S35T z6+vEZqV<4gZ~_hG-r5o?uMF!+}H$OBZ<9`)&CY<-?szJBYzT z%W~zSFkD^|$BOsYwjLPs)f9H#JgE%=S7}&JNmVRj)3tTg34`t79QPhG)lO6b^)Jht zs)4aZ^!uC8SZ=YEW*BU8$R6D1y|(nacAcN?J+UX>a_FMaj?HkGK5)$A?^5mK@VxFE zxDMi4?`3%{M^tXp=-gpOIZ?w?6)~k%=J4#z$gBHv8}bS*8r!)0ihI$P!ie4Pi_^E9 zS2|%gpp6;%x&{zgX_@#gxmq_8DvY=+U#&YE2(`UufJ3o18fYx)_V9c0_SD}{Nht*3!jma2=!@W|) zrKu!ssM9o$1Kk-}0ZufTvMUDGk|8#CcGxan`#e-_Srf{%WK7MJ5o9}4%})cgoU~vF zyI)-*gnOV^<2ji!Vr0szk89`%i;&Rm7nq4L0XC>#c_?}A7=23ELL>N<0$=c}R_GDm zgld*cakR7RDUtQy!V*K=A0vK(f^M}BLrkcZI8Cp_smk6I*cb)BRsi2q=6^?3)qLqY z;%)82b?>4B0RLMP#S*=Y)@Z!>k` z2~6%8K?992&0ZZ>bt@r#;aZtnims9GILwA5VYRcvl?U+A6{1dea<&-fvX!j9rp}BU zCY63;YW*tA6MiD3XM^kTb+_^ph9KUYOa@GQ1DMx&9#5;?qne>d51d_`d@>5hecD09 zjw(ZBqjx{pIqy1WdcY-#q*$W^1yir{yL%Ei)Ju;JqeK#x55+Bfq}gDCpgZ29+VydE zk`Rc_F(9PkRkfrX0WP8At`zeSn;`-iJLmbFO^(&dNZa~tN{?Uc4}vN)<1YGGWkVli zRrq8TCQR3`F%-vHyZzI3wc`z|eCw}{-14NbXaX@(%W_1Nl5?bt>_dkiKI|95wyNP# znej#0%wFel)FvfCi2Q@}M+glO{IMWR2*Hl zZsYFm4#719cMb0D?oNOJjY}Yb;K3o#Nbun9?ry=|2?VG6k$Lx@%-or|>#p~zR<+bR z-?z`+eQNKotE=lk9*C5SLPCm`FM}th5!s(-%3sMY)WHBLdF-F0q>YVGLBtx08b&ySKR6btMYq9`2~|rMCO5$3srP1U78Q? zPxG4fkstj7^dD2ZYO9&OBxa``?-2vKcR+4Gq+NCFPCxj9^I$VrOK~j8T~y~AklLe{ z<-*plXw1`eE*+L9V$zep%)E~Jq{3d;h0;T57{G$amshs2UflJJe(Sg=b>*eb+O6>M z6E+?Oyd!lzEL26MJJk(tiQkI%5RpI^x7Nx0CIe$(Z9I`k)Q^Fwz3wjL{Z>Lgt}*m_ zG*ad!$ynA%TIR*(IqC{1lOYbGk33cv?&%}f#lq+$_E=J=N8Wn*_>$3GkFQJt8pL6v zFcGR~`up;Q=RFq^Im>CJVbQG2JTcgKO9;i-z`87@0Cvw!0i;E(5lN@WFRP9X{(X;} z666jHpux{Bd*4sh*dLh=@vE^27mF%8uMZ4BJZWkjJ7U2t`a0NT69e6RD;iMD-yJ7%NhCfgY$tM1ji=EkTF_?Q92FW@ zlC^2pdaQ4%HhJdad4)9=wS9d-<$%p!EYlUyn401JJ!Vk7@z-rvih(l9B%`79I7aM> ze+y*m%tk95(fR zr&7mY-P=8+QZ-w!J-++qkoC+UeSbwO7Jt)rR~YZ-7cmjJGp!@}71QTzsTH(1VsOjQ zs#ZEfPOcS=i|6ug=)ok(&*uln{_@F%@`o|;20i$J?H;3#X1<&a0`H{)(;OTbt@^Tk zjOb#sf_gf`PpnSL15Bm0V+);uY0VkfI7!v+YP+(uw{B*1u|9GiCbb`uDKS(GYV@h{ z#xqz`$Y3h$;NYCc(Y$;=5~oKa;1S1!C$-{P%a=?r0QUaX4!vv*j<{SRCCrWvc>Y%CWwf2D?4VoKDK3w^n(*V~yt!_06DqB}y(ac0qTbv= z*}U3zD0P$A_bSZ9L19DM-qaQsYm{s+Q;-4XUHtdPCpSi3JnJrv()d$yd%O+nNAvM<5WFjgaE+=qyfNeV zSU3g5Cn-|%1Dkx}V%*v{da#-bXwLlIq(E+P6kI0j}grf4lFu^`%a_S+5Ivb$7mO3GEoNDG%+2; zM8X(davheT#$AcQRiv3HI96Nu>io!~fx7YaT9>9g4=ehf z7V5#2_B$_RQ61)FrXx=E;v3;y#Lvp{PyNf)7!KvQ+Xlq+=c8N4@=W_<_Cw?{$qakT!pAts?w*CIdK z8M4S^>46!cSld4=^=`)Ft@3k16Nb&wHJF<|aj zENER=Dg;GI=Rhi|K&=tc=aa~QO0gm8PMX0Pkai5Rx(UcAjNXSy;h?tt(UBya<(dCF zs3FPvOZZKsrpjdn3Eu*GHP#8~*DLcc6~*pf;M>2NCX@GjC2*mVDR5%2AI`i#dD8yj z|GA$F;)wrYi=&RdA+=n(rf$ljZzbYQhFsU|P{mf1?&!9ll}t%#c-<3nl@PR0W38?B z71`ZtNjz#jy`R&74R+kblOy~XJ}2J?i>a`VQF9%Tl%rUr2TN*cdxlDT^~|b7QZg0l zX}?S4Es8|PXeqr(`gthl!v_WE`T$F|i&5iq^*fLa-G05SG&L;GQ?@p!tNti~(+6|N0-`<;?Qe7^K6{tIKA^amdG}gW(9T*ow5&v-p%DkENe~eS zAY^+7rUM6dHt)l1wri%{=g%-oIHoAaGgeph%*+s32Dx2x$8ZQLDhNaK))_j*V0WI%Q8}>mWIGhoRH34ANE%7(wgVl zJMW)lB5VN-rCmNvK3k@;KhgG`&J5ceLm>)hyi_PA#|kv#xHyE8{hjNp z3dB;8=esVH#0gyPZza@cyZI@^1&fN0j0Vjc z3`HhcAjt)4ouqfFgX^yR=L z%+ycjt2z@EJ#FEDO)p;1m{=7NF5zBcqJhNpbW~U$E?l&w`p9Ae7Y+%sJUbh+Sw0D; z;h_v|)~e7XhBec)j@79zPrH6<)(=-FH4pWFflcb94#F^Jw{ub>9#s^2ID5i2lKESO0-3(I8`l7PV z*E{$10++25a}y3iyW1|a|3s~Fo~(C*rq4gexRXI*|I9g;M;f~XyTW;OQB zul!pGo5-(WxZpzCj%jld?W;_Udo5aI&yvaL?E*#v${ud1iQLvCF{yM_Zmd){)T#W} zqbeVZFQe3Ir$72y$Ci|Z@P7K_;#l>Vhpv{2#U~2q8$Ggg^`!?^NjC8UZX6-TC|nD> z64s_zv)+hw+I`EkWxH=Kj}KYpSi4(lE3lM8%FJuiSd|*l8qjb=l#c||=_*X*r_>V87`mcN0Si=usioKD^X zb#9gH{i_|yXj13|Ix^l1Y00cHaqI>4Z^nU>u5cDT+z~ze5abRRQ!d6J?4~h^#y&^bMJj*`wEhZA@Ozh)} z(slG=3QCVQxh?wgHbUQg(&F}fdxbeXaY9#P%d^+;%(#pTqjr%Ohh|>lyl%_gpp?N7 zR&T&;IIm<(6+%irnPL9}NntWU6Vp=(q!_~CHEVCT?8Q>!z8#le>Y?cqjas{evf^nh zuC#DOyBu7CUJGlrgD+lca%Q+dWFMsfoJHs zx&=Ecc2acI)@O!kEV4ZXnr2RCZ$#yt0;zq(i>*ExJ?6BTpmVJ>^myYrg%aiS05Gih zYt}K{dKiZd>R6((L42WQ$aKnQ)$c&PR|mY8UUUmfs)0!Zs$kudn*|~>Zc-uAS3dkW zY1TMUqAm(d%2 zj97qOEOrgI38YKeS#hKPfM}N$O}soFtS0ClA01TGF#-sG^Kyr-QZBalu&1hmtHEui ztF>aRr#c}|@wMI|ij6r?uNP{k+ZCMCtxAj}aGDs{iLJJJt0}`@hZZC%xMDwS^-eUG zI3YSm$~d{EGq3dLDKzLB4+NKP^C2!!KcmUIB+`V~IrgKox(W8)Zh=112?}?=M(5>? z>x{!(?~|2&|7c7=_Im2^OTub^AOv>^Q>J#Fw>rJf1-pL2u;FDgt~#n$`1CcOW#`Jo z(bTT&+Nxhigk<5rtBvWasm7297w$HKlgu(rtVg;TZ{2%u{u8S{!N3neliW~#-e$L( zia^6VJ?_<5JV7qn)Q@=WF#^wL^ibp8YUrKlgfVYfUZ0Y{QdcJTcLP5nx9eP?zPXJu za3B=(=O4A;yAL!&QGAfdr$>wl-J;i=GI;US@uVOAzj^0;W`hfNFBDr5@sWtYdHDsXLrz^rw+aW3P@3>gk3T_hnpYGo!JK#H&el02HB6mY5BK>^t>2zwj zj&w={(ltgfmMc4TEyXxDIBj*ZFf@oxeRXbU*(5;J7){)i<2SQ%L1fp>7c`V?7xveUN882RpaM>7JL{e_%&=8&yBoGo)6=5_oEN z_OR>y%kV;XOK`GeO{jo?g26NjrCGDp(5cjfv_mIVn28CMsmwbVr^OqrKIdM=@RRz) zbr#MUhMa56-$IKo5_V1627j2Hl$x%41a#Zk)8F>BwG-Y!yCV9L8aAaM4zhCDZWaUl z*5Rjf(*Of=8yV@5?{Oo&G;$sK20E!N#bvwP8+-cg)sHmAuWLUuFbYY~#R`0pfrYYB(T-*&8!CBiPu&2*RbKUnTY)u!g$=X3KGS8-1g1g9M& zUn3g$65@UFbN@cjY8%QflUwL4%bIHU@s2$5zRPK)H?SD1DBR>o5(u+a6S)qM&9Tcj z?MX`r7H=ILA7(xqnz!ybp$jBXC_2H2p&{s=CgAqI&0&x8!Y*$czWtmL8s5yCN+R7n zr2^6lTbvdlxKp~*U2F}j@cByO;}Di$q|Z|vkzVaId0xrqia_->!@pB4-x^$u&&nw4 z@Aeda{D3O8+9pQZ>!{7w5GA#=(I_-Yxst-Q*2n)Y$c9OY?M)h}qpZhYcx<{Kf}Mi% zfP#|Q@R$c(K7aR>BI`k5yaukfj?HTpfu4FR^$2Y&3w~WBm1B1yAsnUN%Ihksx~>JY z)HIHQ*|Uuw9;g;^s`=4wp@LzUFDjLuGoC~|uH~yuqZcLdp<^tsdzY(}7k4zZFU}OO z?>(!~f{-!NwzCtLRhui3oF0z%BCX#k?nsA;0ZUvmu!CQ{=7*8LrUr}Aq2biD=a=v| zl(lP4Q_Q(}_93%op0~x7DsFzRwO3eIoxYK5AI_@PPqLB1#>D(u``??->CwVg=-;KN zOwh4uCkE&5DCZv+?B6o$pM2_FV9M&LQ_A zF|G5Uy4ys4IW%sD%YdlD$5nQQsL&ul%S=V7Wvwob z%EzE(L0f$n===Tt3&HIvyr1mHnJY%Ek@(H|Pi{uEvl*0zNBB72^2eFoWqJ2Rq`Ts2 zI=XUQ$#G)XP(oeCGWbCWC>t3Nh)d}1!O4d0-if`{Uj9)9AJqt@Bs*&*ix$Y6Nrhdo zx;%tWqWnxwrM!EzbbM5MV%T_KxDpAh_pTfk! zU@6gW&keJOWfKQ+vkXd@z<|~}U^_B&3D{W4SXW$M!cgm@n(@|ogb55$#6uqk`y#~O z=keStKlq3etUdrhy@Od-nd`eRbe_bnH?(Ib@kYj3Dtay)aLeuG9J`X!nBi6jt-0au74 zO{CDf5fRV^XG$0d)e+)@jOwaB+XaqXOPqsPM#0EvEHyX0ftg&z1j@ghN`Dl4M$umz zoty5uUPl!x9RR)t5OToQQf;%thnQt^ylD4h`(4|elZRVHVW17(GR5)`8h;L)*l)vDJjlyb$n{><&RPG zigTmmgKNKlniE0gUyMQsnE_?uk|9Be5OY?vU5KgD*wWxy-&EI<*7O+S#n*3Nmd?j6 z)AT)(p&zJ;NE_LEryw349q0S`NT3x8Y!RFd;I6>^&|a5+$aD9oZuju~FW^s6fX3_h z?A%QNO@EG!`04JzbS5X4-BMUvw3hYIJj4B;a!?y1kd)y&B;$Tx;wOi0WYdcI@=%B% zK!f6m9qU7mA!ln*6XeHm$2q^<6sCcf7)dU)G47U06&Gx$t{DOQ`$R}eEx0W)q46Y5 z90+9q*d}m9 zbtIhzb{ic8Z<=efeUho5Pq7{s1(mH|<#uP1v;*Ge1%djX!)1G6M;|qwARCwtRWl9H zZ)pMOQ|zQrd+Q!jtWX|P=Vmh|+7}a0g%pw-mEyhqKM@vj5>-NRX?5{(dl8X^($Z_d z**7+?6-{pd{G|oBLC459ss6Vr_28?Q@gj1-pP>Jw4E4hS9b>ic(Xmr;hyu#q>gNE} zAwp%&R9QbBN4P;g;2j#$ClS*+dR3w7*$>1n2-M7fOVJe zlle_?WRYFYa-pg-Q7=>5H&t*#=ONY-y~As>!)sd@8W$J>d-35$+=C{1^?|24LM@PZ zaP4D&?ta*DP{N@oFR$ed#Cl^WTuu&5mmvtyM}TkDPjI_?bAf{49%YZ0MWhgI2twBH2v`AEK`Ki5S* z!Q8od35#K?N=YG*-?L`}TJ)E|)`AZf-pwdY@0DQ)p`tN^7sf>bJ9dZo(+HPJp{e`U zx_4a$m^%$o#?#peg5WB6pf7lDId)8%;H_2i+HzCw9)V?9kYG^E!@Jt1D;!&->gH6T zpm|+cXql^Ch~V<_0mcCZ*}nCt@ z8M(QLa&7j^f)UtuxlH#e2RJ%*juE`Q#)LBi&@XxqRXZJ?Zc5k9g5ylVh>%Tg?c^y; zoLpeqX7DX7b6w{Jgu^^*eFM=P5YOP+nGtxuR}`-zJKJE&$SeTB!y`}ApT4C171wng z`c|YYr54%NfcDjf8cGpC8nWlZwdHqhkM;a;=+y3#wGdAebx%U7%bpHD4pRy&hU|QBb_zC- z>F}pyIXUuOIBMn-!)b^1xae(mhpfF=D+3$?-`zGX@;L$R{wPHl54h@bhiFd}Hi-%B zR<_CYh%lMM8zTyx(TD79g{0~Z2Zw&r-~sG!f;L|sy&sTIN&Z}(e49hQ9Ly|cjCTRQ zl?Od>9&nx$1aZ^y)B+OVm)#M=2Pq&a14mvLEvS7} zh_-MB_@EJ9?Ce$F)CM!Oo*jPm0b6{07M*Su&10d+i4Y+x`o2ngrUU>zi`FZSk^ncL z>0;pNtz-PfLt%vkNck>2W!ruso`dhNsdpm3rn}t5 z_W|O_q2Wn!DsLdgA8v9!X<|cPVT#rQq9B*p8)RUFi*?B+8tF%vlwca|K8SThRwfz{EoU&qW8Yt|QPWNKavb($5Ijw>8k6avQJ^ z(a&18or9u9j|d=W8zqQ*c|R-&Odl!WY5`qD28llIg?@Iz{#nFEOZj}RGDsO0cd?ZV zY4w5Z55sn7?mgI8ZN%IGWjLQh-0$Dap0n*i96{?wV~{1qACPx$TP81y8e8Swoi50(!+*_#m_malRo^_)4LCHW$ z1sWo;_MssL{GB2F@&d$CUdwI}Af|!G8ysOu0p-0#nYK6`W>BoTU>;G;Nyj=!pShGpxlM1v60!7WV+ zz_s*jCqEWOD(Gonv^HP}r^$03Lo*eP!NVIwzIJKTZY0jerDS-F7j;GSMP_IE9zVI3 zCqIaPu9p68;M(@1Bvgwu6*|KX!C$Tc0HZIiy?FEY<@d-?lGYUe+t6?_^R{wk|Idx7 z|FyA&ixZofpO?E8+snB6TVVbN{G0z3-pbyb)xy!v%Gs0E!_&;u%F)Wh?1Jiy4R!52n;rmc-(^uxMkUD;$>0U zG!MRMSVmqSOd?{tvgtKY)YBFBZ1nB$@1SMKGNQ$+@nPr21cpxg1em)-*-?JiHlQ2X z@=7UiS6KuWLc536#T#Vz*G}n4f>771W;ck)7u1;ecIdlfQ4uEvK-s}wVnGy(Df=rs zgPMvhU+2hQuCYfr*{~R1BF9FdC&sWsE3+$+;_6aRWZoSiK#xs;qZFH*pV&)0yMvPJ z8f9zpFa4%6uPE6+C;Jw$ToImdAjkqemHx_bhj+yuH(rfmC|%HoHj01TFvKPH3&yg%qA6GNQrj5 zu=Bp=!(pphfCipe(51hQ7SyBphya>x12rx7Pt>2Y%+kz)3)q$H6idNbaslrb6TFC4 zDo_+@v@gU}^yA<+cQr7s2CC4QpI|H}IzRNK4d`iIB_32fYHSXwe4c>S6TU#@viy-P zisAIe6yB4=*khxpt-r?Y?plvzG19h8^Bb?RG{!s8AEHl}XO-1u<&ZX#%tPaW2j3K& z0-q2NbGeU^`xGGVjgZY&g+76Fl5C_Z z`C*TfY#-x|iS93#1-M`Jfz~vJB4cd#UC6cL+``?l;yWFPpw~4nn$bHEHJwi_uk@dy z4cygaqw`fOtxD88WxFu=M$QB$NkH|w1m7w4?~8oE6IDEuZRp7KDtXS7?`kb95smPkDbta^kuABw}UlA@y*vq zHBcs!t)1PbhcpG}T+hc)1;`w9R7#8b%lVHUZdaacPq(YK2s z2TZ`J`0wYx=kGr-M8L>PqOWdeTy=hF5cabDu3>*I1;Gq2FaHix5?7X&k=D>;Qgh;_@fZ001QT-OFA(XG<#|HhT}( ze}vJz1R4s=LNNQHvku|^JB;Ur$kW}-((1p%et$l3@FKP0FJbk@0sv_L-p}9G*k6eL zou|@oB{Zzuy&bJQ+5Q%Qf9%KyMHmgFIu5*+7e?xr!pQVbj5p|i7=L^IyUKdJP+7VE zsj{q`Y#cAEy_x$z@}hg~s$2C2i6+TI0Yto^0Kdo1UkfDop5UK(?ajQ+eAs?7zjTt- z%+-#So9#F4eDXj-TyDW`j0*OclzuP7|Y_Hz<*b3e;EFCZ2y))|95#A zXONQ@hQA*je~F_1JV<}bqCa3F%YOm?Taomi`F_i!KYV)sE583An*K8K?`6|pW+1^8 z|L;crOGN#N?!QmRe{6Nz{%->GZyow=>rZl4Re*arOaK7vmz5q40Lc8U)dBwp0w%$? literal 0 HcmV?d00001 diff --git a/weblogic-azure-aks/src/resources/ejb-server-stateless-1.0.0.jar b/weblogic-azure-aks/src/resources/ejb-server-stateless-1.0.0.jar new file mode 100644 index 0000000000000000000000000000000000000000..1c2d04522ed73c8933e74a079eab0e3b6bd5d987 GIT binary patch literal 3852 zcmbtW2{@bS7LI*aRZ3A?TYGS$)I=9cso1L42GcerR2xep$g~*yQj=0UMVGO)mZ3#M zV>*^v#!fS~CMXesi2z46SWc&5ah#+gi;`!K&SSreLLxw$7(k z>aR3LC}^ejHpOn=$K_VWCT-@wcTy>6RTFa+m^oAIcl!^^x#cLW$fe{?D2l#KpE_WG z$(g*7>k?*G+NBYF403Fo@`l0;vNYx3WQ@RaGd2(>Xfrk-oNeO*-nL`=`ZiWz8}jO9 zwXg5k!wBSo@I!ijW3%j+o1MKpR1r6P(MVN)4rr6s5+w%K0Y4j%)NSbg)-aE zki-mz%9%`fasuQ|G;rXBWrSI+;)N~0r?f%g5x5y&mtW4&?%{u9I?TTY3KF_JQ*IR< zPYImEQQHaB^>r#$4b<7~Qx>Ts-rrOUeOHZ-qVYrd`41i!IDhj2u2PL{nnX6fzW~ir z2x-?9*HlR`M!G?6#gvC)RL1VKVc-`-`DidI^H4~J;l~yih`OD*ZHg2RKK+TIYn|*J z=Jt1&y2Lp-Cm#N^mkZ(G;=F8^s_$spzMj5jMkc!>iY#6kQ!`xU8*}U&vc$wH!xDVn zrB=$E6D}k`%*O*t!Df_$Cu_g;}qv7NQD+b#DP)Q>o11==Nsy+F;yGC z>t~i?mYm)5KGLq_HKxca=U6A_*sUAzsi6x6KfW$6wRfgLmruD2_IJ6O$VuF!o+=I2 zP%G>&{Y6{_=1|u=M&^vV^4=9TuGuGAC37~Fs9{vwe|(i`p<*=vm*2|yT9?IfKJ5n{ z!IFI}i{@{5LS)jH6BhU??ACR}XdF^6iYCk5u}^!ZRPd=#WF^incNP%^$8~xVNv<3P zGTyOsvl2IdWTIC9Be`1__Y>_1y%pkPfbMQ!iGNAE71F~CgWMt?`BM#dovuK*n{&`n zEm?&X_CYKz%T`QAnv<)a>c%CKpku81<|%vybTSFs-2?V>P4W7$9CVyXLVb@qkAPs^ z?e?xmm@68g&v0Ykqv9S(R)?N_f~bQkwVppvf`7Zuvtl$bi=!(QA4zganV# zgdRItZhhI}HK=BqmS!icnKV4RuvJ@4EA|s^`9Q z=`}e2u)JsTq|P{SaoMCwZ!=z;d^Jiy-c&t#k%;eR?uAPI(`tsk3~}A0{nC`jfkk-+rk5{>R{~YK);t zl2NTUY+=#VOrj{!A6iAdlXz&zH+!;=wVX^W5@Ne1uI~a_ALl6if!ztr(+-K!6nAz~ z3QkIx#0d^|jtKZ^t__JEIXdiGVpkp&j1f7>>#clC@Iu_XM`tvYJr5dD$_I+CD8EkY zxF-B8MZ~l#_C~XEX7=Tm$F4=cA{#z6k|f08{#=qUZ(f9H3!Kho&XA&~%9s#CsunEP zw}r=TUIRTD56q{NjHzx-mR*L$uFc*X7%ISZuCN=vHl!{P)NyIUz;4y1H0fxd!pp^8 zQCl+zC5Qs(mk6Gd$N4MDgy6eVOp_27V(ZQYXdRU0KdY zSGjerXdk&S)7Gg)BedNy9H+-$IbYYn#Fee~Bu;8#5tGj?5O+x>vN;oy%aU%8Y(!jj zyhFKqrvKg`XrSJqAc3lQVp<|T{nn{6LY)SxFNUo*7*_h2oAvzN9?%GHhH6_+lxJN* z+J=`M8eaLU)eR`L*hQz=2?9Hw#G*Y}JyF+GA7)$8GRkwT{|qwQad^+;0=67X(XyQS}V!T8286S>3NT3dApo-<|j`jSr=CK z`1;C*wx-@*X%Vw-$O_609|@cgCxb{lj=}q<(;>xd{|@kQ&rEJCkuS?f?K@|LVPU^@ zc&1Oj!x%B=5`H&tRiQ>cRgJVlkPSJ2s*+f=?@#yFH`fN|Wnh+Z%L~>0lz64`@p~lRN zg5Rnl%wmHTcl3TT#p5j(`sQ5b8!6_2lH>}H5nD_5X?t^mvM z|EnW!w3jy$jd4f%@U{JmKTKVKt(%-*db*uE{4`bep~)SCqV}fitfI4GmgrTyWR1HV zlStbiKhBPg%tr<;M3DH8_FPhp&Lcd}hEAI$1xR)dL=g6FSTK?1uiaJ@O-b0#pKBKo zP#ACG2wtwyC$4QyaK`X0O_`X5ScJZ+uz~D0EhmKj*zS!CdV33j8}v0DyyY(dn$^o> zMEI5)T^Dbe0BA$69(dhS$@EZnKZvfAw~T%e1$f;K^^Iq%XK zzZE`RNAKWS{Uf|x8|~-hFiJ1_sQP3ZA^E@|^l$Wq%KjXJQOeLY+9%rx$!7Tif>E(C z0_~=Jc7W~x^lg8@-E`3BP>ciX?`hev?*c`iJ%5APWwPx=bBcoj;(wZRjA!ju{X1tx g0he<-^Z)*=onByR$_8W(0`UUB6o8jYJoJD60a{bQwg3PC literal 0 HcmV?d00001 From 843e2295f2ba8ade7f288ce56899b84e90d80c14 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Fri, 10 Sep 2021 17:15:15 +0800 Subject: [PATCH 13/14] On branch t3tunneling: initialize the output vaules of lb endpoints Changes to be committed: modified: weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh Signed-off-by: galiacheng --- weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh b/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh index 9ea912f66..b47ebd266 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/createLbSvc.sh @@ -458,8 +458,12 @@ dnszoneClusterT3ChannelLabel=$9 lbSvcValues=${10} wlsDomainUID=${11} +adminConsoleEndpoint="null" adminServerName=${constAdminServerName} # define in common.sh +adminServerT3Endpoint="null" +clusterEndpoint="null" clusterName=${constClusterName} +clusterT3Endpoint="null" svcAdminServer="${wlsDomainUID}-${adminServerName}" svcCluster="${wlsDomainUID}-cluster-${clusterName}" wlsDomainNS="${wlsDomainUID}-ns" From 983a16bf234b5ec6a1cd61a9e0786ad52a8db459 Mon Sep 17 00:00:00 2001 From: galiacheng Date: Wed, 15 Sep 2021 14:59:03 +0800 Subject: [PATCH 14/14] On branch t3tunneling: override the existing dns record. Signed-off-by: galiacheng Changes to be committed: modified: createDnsRecord.sh --- weblogic-azure-aks/src/main/arm/scripts/createDnsRecord.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/weblogic-azure-aks/src/main/arm/scripts/createDnsRecord.sh b/weblogic-azure-aks/src/main/arm/scripts/createDnsRecord.sh index abe94ae47..61492c9d7 100644 --- a/weblogic-azure-aks/src/main/arm/scripts/createDnsRecord.sh +++ b/weblogic-azure-aks/src/main/arm/scripts/createDnsRecord.sh @@ -23,7 +23,6 @@ function create_dns_A_record() { dnsZoneName=$4 az network dns record-set a add-record --ipv4-address ${ipv4Addr} \ - --if-none-match \ --record-set-name ${label} \ --resource-group ${dnsRGName} \ --zone-name ${dnsZoneName} @@ -46,13 +45,11 @@ function create_dns_CNAME_record() { dnsZoneName=$4 az network dns record-set cname create \ - --if-none-match \ -g ${dnsRGName} \ -z ${dnsZoneName} \ -n ${label} az network dns record-set cname set-record \ - --if-none-match \ -g ${dnsRGName} \ -z ${dnsZoneName} \ --cname ${cname} \ @@ -60,6 +57,6 @@ function create_dns_CNAME_record() { if [ $? != 0 ]; then echo_stderr "Failed to create DNS record: ${label}.${dnsZoneName}, cname: ${cname}" - exit 1 + exit 1 fi }