From cf2a8c4ee3a257ef672aa70691cdc86f1bdb9f65 Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Fri, 12 Apr 2019 12:27:10 -0700 Subject: [PATCH 01/13] feat: pre-pull Azure Stack's custom Hyperkube --- packer/install-dependencies.sh | 10 +++++++ parts/k8s/cloud-init/artifacts/cse_helpers.sh | 2 ++ parts/k8s/cloud-init/artifacts/cse_install.sh | 27 ++++++++++++++----- parts/k8s/cloud-init/artifacts/cse_main.sh | 5 ++-- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index f5607eaaec..b8ff32b252 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -363,6 +363,16 @@ for KUBERNETES_VERSION in ${K8S_VERSIONS}; do echo " - ${CONTAINER_IMAGE}" >> ${RELEASE_NOTES_FILEPATH} done +# TODO: fetch supported k8s versions from an aks-engine command instead of hardcoding them here +AZURE_STACK_K8S_VERSIONS=" +1.11.7 +" +for AZS_KUBERNETES_VERSION in ${AZURE_STACK_K8S_VERSIONS}; do + HYPERKUBE_URL="${AZURE_STACK_HYPERKUBE_REPOSITORY}:v${AZS_KUBERNETES_VERSION}" + extractHyperkube "docker" ${AZURE_STACK_SUFFIX} + echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH} +done + df -h echo "Install completed successfully on " `date` >> ${RELEASE_NOTES_FILEPATH} diff --git a/parts/k8s/cloud-init/artifacts/cse_helpers.sh b/parts/k8s/cloud-init/artifacts/cse_helpers.sh index 373c89d5cc..230114aa71 100755 --- a/parts/k8s/cloud-init/artifacts/cse_helpers.sh +++ b/parts/k8s/cloud-init/artifacts/cse_helpers.sh @@ -53,6 +53,8 @@ OS=$(cat /etc/*-release | grep ^ID= | tr -d 'ID="' | awk '{print toupper($0)}') UBUNTU_OS_NAME="UBUNTU" RHEL_OS_NAME="RHEL" COREOS_OS_NAME="COREOS" +AZURE_STACK_SUFFIX="-azs" +AZURE_STACK_HYPERKUBE_REPOSITORY="mcr.microsoft.com/azurestack/k8s/core/hyperkube-amd64" KUBECTL=/usr/local/bin/kubectl DOCKER=/usr/bin/docker GPU_DV=418.40.04 diff --git a/parts/k8s/cloud-init/artifacts/cse_install.sh b/parts/k8s/cloud-init/artifacts/cse_install.sh index 21a165de8e..4bc7ecb365 100755 --- a/parts/k8s/cloud-init/artifacts/cse_install.sh +++ b/parts/k8s/cloud-init/artifacts/cse_install.sh @@ -253,7 +253,10 @@ installImg() { extractHyperkube() { CLI_TOOL=$1 - path="/home/hyperkube-downloads/${KUBERNETES_VERSION}" + # CUSTOM_CLOUD_SUFFIX only set when extracting custom Hyperkube builds (e.x.: Azure Stack) + CUSTOM_CLOUD_SUFFIX=$2 + kubernetes_version="${KUBERNETES_VERSION}${CUSTOM_CLOUD_SUFFIX}" + path="/home/hyperkube-downloads/${kubernetes_version}" pullContainerImage $CLI_TOOL ${HYPERKUBE_URL} if [[ "$CLI_TOOL" == "docker" ]]; then mkdir -p "$path" @@ -267,22 +270,25 @@ extractHyperkube() { mv "$path/hyperkube" "/opt/kubectl" chmod a+x /opt/kubelet /opt/kubectl else - cp "$path/hyperkube" "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" - mv "$path/hyperkube" "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" + cp "$path/hyperkube" "/usr/local/bin/kubelet-${kubernetes_version}" + mv "$path/hyperkube" "/usr/local/bin/kubectl-${kubernetes_version}" fi } installKubeletAndKubectl() { - if [[ ! -f "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" ]]; then + # CUSTOM_CLOUD_SUFFIX only set when extracting custom Hyperkube builds (e.x.: Azure Stack) + CUSTOM_CLOUD_SUFFIX=$1 + kubernetes_version=${KUBERNETES_VERSION}${CUSTOM_CLOUD_SUFFIX} + if [[ ! -f "/usr/local/bin/kubectl-${kubernetes_version}" ]]; then if [[ "$CONTAINER_RUNTIME" == "docker" ]]; then - extractHyperkube "docker" + extractHyperkube "docker" ${CUSTOM_CLOUD_SUFFIX} else installImg extractHyperkube "img" fi fi - mv "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" "/usr/local/bin/kubelet" - mv "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" "/usr/local/bin/kubectl" + mv "/usr/local/bin/kubelet-${kubernetes_version}" "/usr/local/bin/kubelet" + mv "/usr/local/bin/kubectl-${kubernetes_version}" "/usr/local/bin/kubectl" chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-* /home/hyperkube-downloads & } @@ -298,8 +304,15 @@ pullContainerImage() { cleanUpContainerImages() { # TODO remove all unused container images at runtime + # CUSTOM_CLOUD_SUFFIX only set when extracting custom Hyperkube builds (e.x.: Azure Stack) + CUSTOM_CLOUD_SUFFIX=$1 docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'hyperkube') & docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'cloud-controller-manager') & + if [[ "${CUSTOM_CLOUD_SUFFIX}" == "${AZURE_STACK_SUFFIX}" ]]; then + docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${AZURE_STACK_HYPERKUBE_REPOSITORY} | grep 'hyperkube') & + else + docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep ${AZURE_STACK_HYPERKUBE_REPOSITORY} | grep 'hyperkube') & + fi if [ "$IS_HOSTED_MASTER" = "false" ]; then echo "Cleaning up AKS container images, not an AKS cluster" docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep 'hcp-tunnel-front') & diff --git a/parts/k8s/cloud-init/artifacts/cse_main.sh b/parts/k8s/cloud-init/artifacts/cse_main.sh index 253c7ce54a..e4dedb4826 100755 --- a/parts/k8s/cloud-init/artifacts/cse_main.sh +++ b/parts/k8s/cloud-init/artifacts/cse_main.sh @@ -33,6 +33,7 @@ if [[ "${TARGET_ENVIRONMENT,,}" == "${AZURE_STACK_ENV}" ]]; then config_script_custom_cloud=/opt/azure/containers/provision_configs_custom_cloud.sh wait_for_file 3600 1 $config_script_custom_cloud || exit $ERR_FILE_WATCH_TIMEOUT source $config_script_custom_cloud + CUSTOM_CLOUD_SUFFIX=${AZURE_STACK_SUFFIX} fi CUSTOM_SEARCH_DOMAIN_SCRIPT=/opt/azure/containers/setup-custom-search-domains.sh @@ -88,7 +89,7 @@ if [[ "${GPU_NODE}" = true ]]; then else cleanUpGPUDrivers fi -installKubeletAndKubectl +installKubeletAndKubectl ${CUSTOM_CLOUD_SUFFIX} if [[ $OS != $COREOS_OS_NAME ]]; then ensureRPC fi @@ -180,7 +181,7 @@ ps auxfww > /opt/azure/provision-ps.log & if $FULL_INSTALL_REQUIRED; then applyCIS || exit $ERR_CIS_HARDENING_ERROR else - cleanUpContainerImages + cleanUpContainerImages ${CUSTOM_CLOUD_SUFFIX} fi if $REBOOTREQUIRED; then From 2d169db48e094db9f74c281390ba0e546763e247 Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Fri, 12 Apr 2019 14:50:57 -0700 Subject: [PATCH 02/13] Changed approach from optional params to global variables --- packer/install-dependencies.sh | 4 +++- parts/k8s/cloud-init/artifacts/cse_helpers.sh | 1 + parts/k8s/cloud-init/artifacts/cse_install.sh | 14 ++++---------- parts/k8s/cloud-init/artifacts/cse_main.sh | 5 +++-- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index b8ff32b252..40d47747b4 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -340,6 +340,7 @@ pullContainerImage "docker" "busybox" echo " - busybox" >> ${RELEASE_NOTES_FILEPATH} # TODO: fetch supported k8s versions from an aks-engine command instead of hardcoding them here +KUBERNETES_VERSION_SUFFIX=${AZURE_SUFFIX} K8S_VERSIONS=" 1.14.1 1.14.0 @@ -364,12 +365,13 @@ for KUBERNETES_VERSION in ${K8S_VERSIONS}; do done # TODO: fetch supported k8s versions from an aks-engine command instead of hardcoding them here +KUBERNETES_VERSION_SUFFIX=${AZURE_STACK_SUFFIX} AZURE_STACK_K8S_VERSIONS=" 1.11.7 " for AZS_KUBERNETES_VERSION in ${AZURE_STACK_K8S_VERSIONS}; do HYPERKUBE_URL="${AZURE_STACK_HYPERKUBE_REPOSITORY}:v${AZS_KUBERNETES_VERSION}" - extractHyperkube "docker" ${AZURE_STACK_SUFFIX} + extractHyperkube "docker" echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH} done diff --git a/parts/k8s/cloud-init/artifacts/cse_helpers.sh b/parts/k8s/cloud-init/artifacts/cse_helpers.sh index 230114aa71..0a18becb90 100755 --- a/parts/k8s/cloud-init/artifacts/cse_helpers.sh +++ b/parts/k8s/cloud-init/artifacts/cse_helpers.sh @@ -53,6 +53,7 @@ OS=$(cat /etc/*-release | grep ^ID= | tr -d 'ID="' | awk '{print toupper($0)}') UBUNTU_OS_NAME="UBUNTU" RHEL_OS_NAME="RHEL" COREOS_OS_NAME="COREOS" +AZURE_SUFFIX="" AZURE_STACK_SUFFIX="-azs" AZURE_STACK_HYPERKUBE_REPOSITORY="mcr.microsoft.com/azurestack/k8s/core/hyperkube-amd64" KUBECTL=/usr/local/bin/kubectl diff --git a/parts/k8s/cloud-init/artifacts/cse_install.sh b/parts/k8s/cloud-init/artifacts/cse_install.sh index 4bc7ecb365..f140710b7d 100755 --- a/parts/k8s/cloud-init/artifacts/cse_install.sh +++ b/parts/k8s/cloud-init/artifacts/cse_install.sh @@ -253,9 +253,7 @@ installImg() { extractHyperkube() { CLI_TOOL=$1 - # CUSTOM_CLOUD_SUFFIX only set when extracting custom Hyperkube builds (e.x.: Azure Stack) - CUSTOM_CLOUD_SUFFIX=$2 - kubernetes_version="${KUBERNETES_VERSION}${CUSTOM_CLOUD_SUFFIX}" + kubernetes_version="${KUBERNETES_VERSION}${KUBERNETES_VERSION_SUFFIX}" path="/home/hyperkube-downloads/${kubernetes_version}" pullContainerImage $CLI_TOOL ${HYPERKUBE_URL} if [[ "$CLI_TOOL" == "docker" ]]; then @@ -276,12 +274,10 @@ extractHyperkube() { } installKubeletAndKubectl() { - # CUSTOM_CLOUD_SUFFIX only set when extracting custom Hyperkube builds (e.x.: Azure Stack) - CUSTOM_CLOUD_SUFFIX=$1 - kubernetes_version=${KUBERNETES_VERSION}${CUSTOM_CLOUD_SUFFIX} + kubernetes_version=${KUBERNETES_VERSION}${KUBERNETES_VERSION_SUFFIX} if [[ ! -f "/usr/local/bin/kubectl-${kubernetes_version}" ]]; then if [[ "$CONTAINER_RUNTIME" == "docker" ]]; then - extractHyperkube "docker" ${CUSTOM_CLOUD_SUFFIX} + extractHyperkube "docker" else installImg extractHyperkube "img" @@ -304,11 +300,9 @@ pullContainerImage() { cleanUpContainerImages() { # TODO remove all unused container images at runtime - # CUSTOM_CLOUD_SUFFIX only set when extracting custom Hyperkube builds (e.x.: Azure Stack) - CUSTOM_CLOUD_SUFFIX=$1 docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'hyperkube') & docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'cloud-controller-manager') & - if [[ "${CUSTOM_CLOUD_SUFFIX}" == "${AZURE_STACK_SUFFIX}" ]]; then + if [[ "${KUBERNETES_VERSION_SUFFIX}" == "${AZURE_STACK_SUFFIX}" ]]; then docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${AZURE_STACK_HYPERKUBE_REPOSITORY} | grep 'hyperkube') & else docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep ${AZURE_STACK_HYPERKUBE_REPOSITORY} | grep 'hyperkube') & diff --git a/parts/k8s/cloud-init/artifacts/cse_main.sh b/parts/k8s/cloud-init/artifacts/cse_main.sh index e4dedb4826..6d1b440087 100755 --- a/parts/k8s/cloud-init/artifacts/cse_main.sh +++ b/parts/k8s/cloud-init/artifacts/cse_main.sh @@ -36,6 +36,7 @@ if [[ "${TARGET_ENVIRONMENT,,}" == "${AZURE_STACK_ENV}" ]]; then CUSTOM_CLOUD_SUFFIX=${AZURE_STACK_SUFFIX} fi +KUBERNETES_VERSION_SUFFIX="${CUSTOM_CLOUD_SUFFIX:-$AZURE_SUFFIX}" CUSTOM_SEARCH_DOMAIN_SCRIPT=/opt/azure/containers/setup-custom-search-domains.sh set +x @@ -89,7 +90,7 @@ if [[ "${GPU_NODE}" = true ]]; then else cleanUpGPUDrivers fi -installKubeletAndKubectl ${CUSTOM_CLOUD_SUFFIX} +installKubeletAndKubectl if [[ $OS != $COREOS_OS_NAME ]]; then ensureRPC fi @@ -181,7 +182,7 @@ ps auxfww > /opt/azure/provision-ps.log & if $FULL_INSTALL_REQUIRED; then applyCIS || exit $ERR_CIS_HARDENING_ERROR else - cleanUpContainerImages ${CUSTOM_CLOUD_SUFFIX} + cleanUpContainerImages fi if $REBOOTREQUIRED; then From fad7228b2ca2710e2378764cb218dd14748aa9a8 Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Fri, 12 Apr 2019 15:21:37 -0700 Subject: [PATCH 03/13] Update caller code instead of provisioning functions --- packer/install-dependencies.sh | 3 +-- parts/k8s/cloud-init/artifacts/cse_install.sh | 16 +++++++--------- parts/k8s/cloud-init/artifacts/cse_main.sh | 5 +++++ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index 40d47747b4..9266ae7c91 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -340,7 +340,6 @@ pullContainerImage "docker" "busybox" echo " - busybox" >> ${RELEASE_NOTES_FILEPATH} # TODO: fetch supported k8s versions from an aks-engine command instead of hardcoding them here -KUBERNETES_VERSION_SUFFIX=${AZURE_SUFFIX} K8S_VERSIONS=" 1.14.1 1.14.0 @@ -365,11 +364,11 @@ for KUBERNETES_VERSION in ${K8S_VERSIONS}; do done # TODO: fetch supported k8s versions from an aks-engine command instead of hardcoding them here -KUBERNETES_VERSION_SUFFIX=${AZURE_STACK_SUFFIX} AZURE_STACK_K8S_VERSIONS=" 1.11.7 " for AZS_KUBERNETES_VERSION in ${AZURE_STACK_K8S_VERSIONS}; do + KUBERNETES_VERSION=${AZS_KUBERNETES_VERSION}${AZURE_STACK_SUFFIX} HYPERKUBE_URL="${AZURE_STACK_HYPERKUBE_REPOSITORY}:v${AZS_KUBERNETES_VERSION}" extractHyperkube "docker" echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH} diff --git a/parts/k8s/cloud-init/artifacts/cse_install.sh b/parts/k8s/cloud-init/artifacts/cse_install.sh index f140710b7d..52ff73cb71 100755 --- a/parts/k8s/cloud-init/artifacts/cse_install.sh +++ b/parts/k8s/cloud-init/artifacts/cse_install.sh @@ -253,8 +253,7 @@ installImg() { extractHyperkube() { CLI_TOOL=$1 - kubernetes_version="${KUBERNETES_VERSION}${KUBERNETES_VERSION_SUFFIX}" - path="/home/hyperkube-downloads/${kubernetes_version}" + path="/home/hyperkube-downloads/${KUBERNETES_VERSION}" pullContainerImage $CLI_TOOL ${HYPERKUBE_URL} if [[ "$CLI_TOOL" == "docker" ]]; then mkdir -p "$path" @@ -268,14 +267,13 @@ extractHyperkube() { mv "$path/hyperkube" "/opt/kubectl" chmod a+x /opt/kubelet /opt/kubectl else - cp "$path/hyperkube" "/usr/local/bin/kubelet-${kubernetes_version}" - mv "$path/hyperkube" "/usr/local/bin/kubectl-${kubernetes_version}" + cp "$path/hyperkube" "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" + mv "$path/hyperkube" "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" fi } installKubeletAndKubectl() { - kubernetes_version=${KUBERNETES_VERSION}${KUBERNETES_VERSION_SUFFIX} - if [[ ! -f "/usr/local/bin/kubectl-${kubernetes_version}" ]]; then + if [[ ! -f "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" ]]; then if [[ "$CONTAINER_RUNTIME" == "docker" ]]; then extractHyperkube "docker" else @@ -283,8 +281,8 @@ installKubeletAndKubectl() { extractHyperkube "img" fi fi - mv "/usr/local/bin/kubelet-${kubernetes_version}" "/usr/local/bin/kubelet" - mv "/usr/local/bin/kubectl-${kubernetes_version}" "/usr/local/bin/kubectl" + mv "/usr/local/bin/kubelet-${KUBERNETES_VERSION}" "/usr/local/bin/kubelet" + mv "/usr/local/bin/kubectl-${KUBERNETES_VERSION}" "/usr/local/bin/kubectl" chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl rm -rf /usr/local/bin/kubelet-* /usr/local/bin/kubectl-* /home/hyperkube-downloads & } @@ -302,7 +300,7 @@ cleanUpContainerImages() { # TODO remove all unused container images at runtime docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'hyperkube') & docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'cloud-controller-manager') & - if [[ "${KUBERNETES_VERSION_SUFFIX}" == "${AZURE_STACK_SUFFIX}" ]]; then + if [[ "${TARGET_ENVIRONMENT,,}" == "${AZURE_STACK_ENV}" ]]; then docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${AZURE_STACK_HYPERKUBE_REPOSITORY} | grep 'hyperkube') & else docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep ${AZURE_STACK_HYPERKUBE_REPOSITORY} | grep 'hyperkube') & diff --git a/parts/k8s/cloud-init/artifacts/cse_main.sh b/parts/k8s/cloud-init/artifacts/cse_main.sh index 6d1b440087..5077a5f4ea 100755 --- a/parts/k8s/cloud-init/artifacts/cse_main.sh +++ b/parts/k8s/cloud-init/artifacts/cse_main.sh @@ -90,7 +90,12 @@ if [[ "${GPU_NODE}" = true ]]; then else cleanUpGPUDrivers fi + +KUBERNETES_VERSION_TEMP=${KUBERNETES_VERSION} +KUBERNETES_VERSION=${KUBERNETES_VERSION}${KUBERNETES_VERSION_SUFFIX} installKubeletAndKubectl +KUBERNETES_VERSION=${KUBERNETES_VERSION_TEMP} + if [[ $OS != $COREOS_OS_NAME ]]; then ensureRPC fi From 9589577cb74b6fc4673596d6f243532ee7c10133 Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Fri, 12 Apr 2019 16:15:18 -0700 Subject: [PATCH 04/13] Updated AZS hyperkube repository --- packer/install-dependencies.sh | 6 +++--- parts/k8s/cloud-init/artifacts/cse_helpers.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index 9266ae7c91..5068987733 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -367,9 +367,9 @@ done AZURE_STACK_K8S_VERSIONS=" 1.11.7 " -for AZS_KUBERNETES_VERSION in ${AZURE_STACK_K8S_VERSIONS}; do - KUBERNETES_VERSION=${AZS_KUBERNETES_VERSION}${AZURE_STACK_SUFFIX} - HYPERKUBE_URL="${AZURE_STACK_HYPERKUBE_REPOSITORY}:v${AZS_KUBERNETES_VERSION}" +for AZURE_STACK_KUBERNETES_VERSION in ${AZURE_STACK_K8S_VERSIONS}; do + KUBERNETES_VERSION=${AZURE_STACK_KUBERNETES_VERSION}${AZURE_STACK_SUFFIX} + HYPERKUBE_URL="${AZURE_STACK_HYPERKUBE_REPOSITORY}:v${AZURE_STACK_KUBERNETES_VERSION}" extractHyperkube "docker" echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH} done diff --git a/parts/k8s/cloud-init/artifacts/cse_helpers.sh b/parts/k8s/cloud-init/artifacts/cse_helpers.sh index 0a18becb90..7354e8680f 100755 --- a/parts/k8s/cloud-init/artifacts/cse_helpers.sh +++ b/parts/k8s/cloud-init/artifacts/cse_helpers.sh @@ -55,7 +55,7 @@ RHEL_OS_NAME="RHEL" COREOS_OS_NAME="COREOS" AZURE_SUFFIX="" AZURE_STACK_SUFFIX="-azs" -AZURE_STACK_HYPERKUBE_REPOSITORY="mcr.microsoft.com/azurestack/k8s/core/hyperkube-amd64" +AZURE_STACK_HYPERKUBE_REPOSITORY="msazurestackdocker/hyperkube-amd64" KUBECTL=/usr/local/bin/kubectl DOCKER=/usr/bin/docker GPU_DV=418.40.04 From 4867cd406a235638298d454958bea00387cf7134 Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Fri, 12 Apr 2019 21:39:22 -0700 Subject: [PATCH 05/13] Trust the pipeline --- parts/k8s/cloud-init/artifacts/cse_helpers.sh | 1 - parts/k8s/cloud-init/artifacts/cse_main.sh | 10 +++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/parts/k8s/cloud-init/artifacts/cse_helpers.sh b/parts/k8s/cloud-init/artifacts/cse_helpers.sh index 7354e8680f..314517e43d 100755 --- a/parts/k8s/cloud-init/artifacts/cse_helpers.sh +++ b/parts/k8s/cloud-init/artifacts/cse_helpers.sh @@ -53,7 +53,6 @@ OS=$(cat /etc/*-release | grep ^ID= | tr -d 'ID="' | awk '{print toupper($0)}') UBUNTU_OS_NAME="UBUNTU" RHEL_OS_NAME="RHEL" COREOS_OS_NAME="COREOS" -AZURE_SUFFIX="" AZURE_STACK_SUFFIX="-azs" AZURE_STACK_HYPERKUBE_REPOSITORY="msazurestackdocker/hyperkube-amd64" KUBECTL=/usr/local/bin/kubectl diff --git a/parts/k8s/cloud-init/artifacts/cse_main.sh b/parts/k8s/cloud-init/artifacts/cse_main.sh index 5077a5f4ea..ce8427f83f 100755 --- a/parts/k8s/cloud-init/artifacts/cse_main.sh +++ b/parts/k8s/cloud-init/artifacts/cse_main.sh @@ -33,10 +33,10 @@ if [[ "${TARGET_ENVIRONMENT,,}" == "${AZURE_STACK_ENV}" ]]; then config_script_custom_cloud=/opt/azure/containers/provision_configs_custom_cloud.sh wait_for_file 3600 1 $config_script_custom_cloud || exit $ERR_FILE_WATCH_TIMEOUT source $config_script_custom_cloud - CUSTOM_CLOUD_SUFFIX=${AZURE_STACK_SUFFIX} + KUBERNETES_VERSION_NO_SUFFIX="${KUBERNETES_VERSION}" + KUBERNETES_VERSION=${KUBERNETES_VERSION}${AZURE_STACK_SUFFIX} fi -KUBERNETES_VERSION_SUFFIX="${CUSTOM_CLOUD_SUFFIX:-$AZURE_SUFFIX}" CUSTOM_SEARCH_DOMAIN_SCRIPT=/opt/azure/containers/setup-custom-search-domains.sh set +x @@ -90,12 +90,7 @@ if [[ "${GPU_NODE}" = true ]]; then else cleanUpGPUDrivers fi - -KUBERNETES_VERSION_TEMP=${KUBERNETES_VERSION} -KUBERNETES_VERSION=${KUBERNETES_VERSION}${KUBERNETES_VERSION_SUFFIX} installKubeletAndKubectl -KUBERNETES_VERSION=${KUBERNETES_VERSION_TEMP} - if [[ $OS != $COREOS_OS_NAME ]]; then ensureRPC fi @@ -187,6 +182,7 @@ ps auxfww > /opt/azure/provision-ps.log & if $FULL_INSTALL_REQUIRED; then applyCIS || exit $ERR_CIS_HARDENING_ERROR else + KUBERNETES_VERSION="${KUBERNETES_VERSION_NO_SUFFIX}" cleanUpContainerImages fi From e6dd2e9b81f210d5aefcc5cda02fa9c78ebc1c87 Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Mon, 15 Apr 2019 10:07:39 -0700 Subject: [PATCH 06/13] Updated AZURE_STACK_K8S_VERSIONS --- packer/install-dependencies.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index c9488cc35c..e24e4129c2 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -366,6 +366,8 @@ done # TODO: fetch supported k8s versions from an aks-engine command instead of hardcoding them here AZURE_STACK_K8S_VERSIONS=" 1.11.7 +1.11.8 +1.11.9 " for AZURE_STACK_KUBERNETES_VERSION in ${AZURE_STACK_K8S_VERSIONS}; do KUBERNETES_VERSION=${AZURE_STACK_KUBERNETES_VERSION}${AZURE_STACK_SUFFIX} From 9ecf58b244a970aebfddbd01aadc68ce13ebf8ab Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Mon, 15 Apr 2019 11:37:05 -0700 Subject: [PATCH 07/13] Updated AZURE_STACK_K8S_VERSIONS --- packer/install-dependencies.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index e24e4129c2..69c28577d1 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -365,7 +365,6 @@ done # TODO: fetch supported k8s versions from an aks-engine command instead of hardcoding them here AZURE_STACK_K8S_VERSIONS=" -1.11.7 1.11.8 1.11.9 " From 9358809e464c59711b36145e1bcad67e819c2a0b Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Wed, 17 Apr 2019 21:59:24 -0700 Subject: [PATCH 08/13] Add AzS prefix to KUBERNETES_VERSION's ARM param --- packer/install-dependencies.sh | 32 ++++++++----------- parts/k8s/cloud-init/artifacts/cse_helpers.sh | 2 -- parts/k8s/cloud-init/artifacts/cse_install.sh | 5 --- parts/k8s/cloud-init/artifacts/cse_main.sh | 3 -- pkg/engine/armvariables.go | 7 +++- 5 files changed, 20 insertions(+), 29 deletions(-) diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index 0bed594eb4..54f85b0725 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -361,31 +361,27 @@ K8S_VERSIONS=" 1.12.7 1.12.6 1.11.9 +1.11.9-azs 1.11.8 +1.11.8-azs 1.10.13 1.10.12 1.9.11 1.9.10 " for KUBERNETES_VERSION in ${K8S_VERSIONS}; do - HYPERKUBE_URL="k8s.gcr.io/hyperkube-amd64:v${KUBERNETES_VERSION}" - extractHyperkube "docker" - CONTAINER_IMAGE="k8s.gcr.io/cloud-controller-manager-amd64:v${KUBERNETES_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} - echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH} - echo " - ${CONTAINER_IMAGE}" >> ${RELEASE_NOTES_FILEPATH} -done - -# TODO: fetch supported k8s versions from an aks-engine command instead of hardcoding them here -AZURE_STACK_K8S_VERSIONS=" -1.11.8 -1.11.9 -" -for AZURE_STACK_KUBERNETES_VERSION in ${AZURE_STACK_K8S_VERSIONS}; do - KUBERNETES_VERSION=${AZURE_STACK_KUBERNETES_VERSION}${AZURE_STACK_SUFFIX} - HYPERKUBE_URL="${AZURE_STACK_HYPERKUBE_REPOSITORY}:v${AZURE_STACK_KUBERNETES_VERSION}" - extractHyperkube "docker" - echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH} + if [[ $KUBERNETES_VERSION == *"azs"* ]]; then + HYPERKUBE_URL="msazurestackdocker/hyperkube-amd64:v${KUBERNETES_VERSION}" + extractHyperkube "docker" + echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH} +  else + HYPERKUBE_URL="k8s.gcr.io/hyperkube-amd64:v${KUBERNETES_VERSION}" + extractHyperkube "docker" + CONTAINER_IMAGE="k8s.gcr.io/cloud-controller-manager-amd64:v${KUBERNETES_VERSION}" + pullContainerImage "docker" ${CONTAINER_IMAGE} + echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH} + echo " - ${CONTAINER_IMAGE}" >> ${RELEASE_NOTES_FILEPATH} + fi done df -h diff --git a/parts/k8s/cloud-init/artifacts/cse_helpers.sh b/parts/k8s/cloud-init/artifacts/cse_helpers.sh index 5cc77542fb..4ddd02ac96 100755 --- a/parts/k8s/cloud-init/artifacts/cse_helpers.sh +++ b/parts/k8s/cloud-init/artifacts/cse_helpers.sh @@ -55,8 +55,6 @@ OS=$(cat /etc/*-release | grep ^ID= | tr -d 'ID="' | awk '{print toupper($0)}') UBUNTU_OS_NAME="UBUNTU" RHEL_OS_NAME="RHEL" COREOS_OS_NAME="COREOS" -AZURE_STACK_SUFFIX="-azs" -AZURE_STACK_HYPERKUBE_REPOSITORY="msazurestackdocker/hyperkube-amd64" KUBECTL=/usr/local/bin/kubectl DOCKER=/usr/bin/docker GPU_DV=418.40.04 diff --git a/parts/k8s/cloud-init/artifacts/cse_install.sh b/parts/k8s/cloud-init/artifacts/cse_install.sh index 52ff73cb71..21a165de8e 100755 --- a/parts/k8s/cloud-init/artifacts/cse_install.sh +++ b/parts/k8s/cloud-init/artifacts/cse_install.sh @@ -300,11 +300,6 @@ cleanUpContainerImages() { # TODO remove all unused container images at runtime docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'hyperkube') & docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'cloud-controller-manager') & - if [[ "${TARGET_ENVIRONMENT,,}" == "${AZURE_STACK_ENV}" ]]; then - docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${AZURE_STACK_HYPERKUBE_REPOSITORY} | grep 'hyperkube') & - else - docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep ${AZURE_STACK_HYPERKUBE_REPOSITORY} | grep 'hyperkube') & - fi if [ "$IS_HOSTED_MASTER" = "false" ]; then echo "Cleaning up AKS container images, not an AKS cluster" docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep 'hcp-tunnel-front') & diff --git a/parts/k8s/cloud-init/artifacts/cse_main.sh b/parts/k8s/cloud-init/artifacts/cse_main.sh index a37908fd14..732efac64f 100755 --- a/parts/k8s/cloud-init/artifacts/cse_main.sh +++ b/parts/k8s/cloud-init/artifacts/cse_main.sh @@ -33,8 +33,6 @@ if [[ "${TARGET_ENVIRONMENT,,}" == "${AZURE_STACK_ENV}" ]]; then config_script_custom_cloud=/opt/azure/containers/provision_configs_custom_cloud.sh wait_for_file 3600 1 $config_script_custom_cloud || exit $ERR_FILE_WATCH_TIMEOUT source $config_script_custom_cloud - KUBERNETES_VERSION_NO_SUFFIX="${KUBERNETES_VERSION}" - KUBERNETES_VERSION=${KUBERNETES_VERSION}${AZURE_STACK_SUFFIX} fi CUSTOM_SEARCH_DOMAIN_SCRIPT=/opt/azure/containers/setup-custom-search-domains.sh @@ -187,7 +185,6 @@ if $FULL_INSTALL_REQUIRED; then applyCIS fi else - KUBERNETES_VERSION="${KUBERNETES_VERSION_NO_SUFFIX}" cleanUpContainerImages fi diff --git a/pkg/engine/armvariables.go b/pkg/engine/armvariables.go index 135c92499b..e8214c2f0b 100644 --- a/pkg/engine/armvariables.go +++ b/pkg/engine/armvariables.go @@ -74,6 +74,11 @@ func getK8sMasterVars(cs *api.ContainerService) (map[string]interface{}, error) isCustomVnet := cs.Properties.AreAgentProfilesCustomVNET() hasAgentPool := len(profiles) > 0 hasCosmosEtcd := masterProfile != nil && to.Bool(masterProfile.CosmosEtcd) + + kubernetesVersion := orchProfile.OrchestratorVersion + if cs.Properties.IsAzureStackCloud() { + kubernetesVersion = orchProfile.OrchestratorVersion + "-azs" + } masterVars := map[string]interface{}{ "maxVMsPerPool": 100, @@ -131,7 +136,7 @@ func getK8sMasterVars(cs *api.ContainerService) (map[string]interface{}, error) "modprobeConfCIS": getBase64EncodedGzippedCustomScript(modprobeConfCIS), }, "provisionScriptParametersCommon": fmt.Sprintf("[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' CONTAINERD_VERSION=',parameters('containerdVersion'),' MOBY_VERSION=',parameters('mobyVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION=%s HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' MAXIMUM_LOADBALANCER_RULE_COUNT=',variables('maximumLoadBalancerRuleCount'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER=%t',' PRIVATE_AZURE_REGISTRY_SERVER=',parameters('privateAzureRegistryServer'),' AUTHENTICATION_METHOD=',variables('customCloudAuthenticationMethod'),' IDENTITY_SYSTEM=',variables('customCloudIdentifySystem'))]", - orchProfile.OrchestratorVersion, isHostedMaster), + kubernetesVersion, isHostedMaster), "orchestratorNameVersionTag": fmt.Sprintf("%s:%s", orchProfile.OrchestratorType, orchProfile.OrchestratorVersion), "subnetNameResourceSegmentIndex": 10, "vnetNameResourceSegmentIndex": 8, From b27553f18e76ac71f5d2a86943d811d047d56c6b Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Thu, 18 Apr 2019 22:56:36 -0700 Subject: [PATCH 09/13] cleanUpContainerImages --- packer/install-dependencies.sh | 6 ++---- parts/k8s/cloud-init/artifacts/cse_install.sh | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index 54f85b0725..66798db0ff 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -372,16 +372,14 @@ K8S_VERSIONS=" for KUBERNETES_VERSION in ${K8S_VERSIONS}; do if [[ $KUBERNETES_VERSION == *"azs"* ]]; then HYPERKUBE_URL="msazurestackdocker/hyperkube-amd64:v${KUBERNETES_VERSION}" - extractHyperkube "docker" - echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH}  else HYPERKUBE_URL="k8s.gcr.io/hyperkube-amd64:v${KUBERNETES_VERSION}" - extractHyperkube "docker" CONTAINER_IMAGE="k8s.gcr.io/cloud-controller-manager-amd64:v${KUBERNETES_VERSION}" pullContainerImage "docker" ${CONTAINER_IMAGE} - echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH} echo " - ${CONTAINER_IMAGE}" >> ${RELEASE_NOTES_FILEPATH} fi + extractHyperkube "docker" + echo " - ${HYPERKUBE_URL}" >> ${RELEASE_NOTES_FILEPATH} done df -h diff --git a/parts/k8s/cloud-init/artifacts/cse_install.sh b/parts/k8s/cloud-init/artifacts/cse_install.sh index 21a165de8e..3ecdc2c06d 100755 --- a/parts/k8s/cloud-init/artifacts/cse_install.sh +++ b/parts/k8s/cloud-init/artifacts/cse_install.sh @@ -298,8 +298,8 @@ pullContainerImage() { cleanUpContainerImages() { # TODO remove all unused container images at runtime - docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'hyperkube') & - docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v ${KUBERNETES_VERSION} | grep 'cloud-controller-manager') & + docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v "${KUBERNETES_VERSION}$" | grep 'hyperkube') & + docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -v "${KUBERNETES_VERSION}$" | grep 'cloud-controller-manager') & if [ "$IS_HOSTED_MASTER" = "false" ]; then echo "Cleaning up AKS container images, not an AKS cluster" docker rmi $(docker images --format '{{.Repository}}:{{.Tag}}' | grep 'hcp-tunnel-front') & From 7ce621db8af907a4e90c7ee13d7513ebc2465d9e Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Fri, 19 Apr 2019 12:50:56 -0700 Subject: [PATCH 10/13] fixed armvariables_test --- pkg/engine/armvariables.go | 8 ++++---- pkg/engine/armvariables_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/engine/armvariables.go b/pkg/engine/armvariables.go index e8214c2f0b..9de5810e5e 100644 --- a/pkg/engine/armvariables.go +++ b/pkg/engine/armvariables.go @@ -74,10 +74,10 @@ func getK8sMasterVars(cs *api.ContainerService) (map[string]interface{}, error) isCustomVnet := cs.Properties.AreAgentProfilesCustomVNET() hasAgentPool := len(profiles) > 0 hasCosmosEtcd := masterProfile != nil && to.Bool(masterProfile.CosmosEtcd) - - kubernetesVersion := orchProfile.OrchestratorVersion - if cs.Properties.IsAzureStackCloud() { - kubernetesVersion = orchProfile.OrchestratorVersion + "-azs" + + kubernetesVersion := orchProfile.OrchestratorVersion + if cs.Properties.IsAzureStackCloud() { + kubernetesVersion = orchProfile.OrchestratorVersion + "-azs" } masterVars := map[string]interface{}{ diff --git a/pkg/engine/armvariables_test.go b/pkg/engine/armvariables_test.go index 2070b7bf2b..cfeed0222e 100644 --- a/pkg/engine/armvariables_test.go +++ b/pkg/engine/armvariables_test.go @@ -527,7 +527,7 @@ func TestK8sVars(t *testing.T) { }, "masterPublicLbFQDN": "blueorange.local.cloudapp.azurestack.external", "provisionConfigsCustomCloud": "H4sIAAAAAAAA/7RX7XLaSBb9z1PcyGzZrlgI72ZdKe+SlCKUCRMbXJKcVCak5Lb6CjoW3Up3yx9j8+5TLRBgwIakZvhhG3E559zT96O988K5ZNy5JGpYqyFXhUQPpWYpS4hGtbcP9zUAAPeP88API9f7GPvdT52g1z31u1H8e9jrxmdu9KFlOagT56q4RMlRo3LIn4VEpUlylWSioI3vSnBrGSvww9554Pnxqdt1f/OD2O+2z3qdbtS6SIiG+iZaeIDvP6AhUYlCJnhKOBmg9DnNBeMaHkBLsClYfcu6eJrbj9y2G7lzcqu+jUZnhJpQoomDU0L1luTMvkapmOCtfzcP/2s3D+3m4STvpJAZrEdeVlDGe+fBSRz40XnQ9Xptv1V/Wz7+eP7Oj71eNwp6Jyd+MJP1vnPit5ZPYUQ4S1FpVT60E8G1FFmG0h5NvGrckVFW4rIUvkJ9iRRetKAJ3/4Heoi8DDOvHQgwz0iCUP4cioyihFRIUCqDS8Yp44NZdAlsp1B/TvkKh3kppGAzsNTD/69FVoxQqezNgw2cjPDYcPX5JHAolD4jenhcPQDIzVvo940j/b5jgvtOglKrh4G1QcvzIk5FwfUjJZZSmTWnBhiZkLMnFSyGSiS0x7O7Y9CywI3aZtJSVls4D0/kd4wPjIXgms4LTeeBFEJDMm9o0KIMIXkuRS6ZeaS0kOUHlwhFTolG2pghL1ZrrxfFnh9Enfcdz438eFq80/a/JtLJ2KVzQ8gAuXYWx0gjx5G1FWbc9sOogiyUdDKRkMxRQyLRSYi9kIsZMWqBpZFIPSdJ8set9oz45wPnimbYE5fsJTnlx5gp/BsKf2dykMEpVKMNpv0K1ayp2uzR8TJlqkhppAcgcSSusQTapksXS91ZaDeHbluR6xBmvbI9TMpqC79qEztWVxPcsCwzNStRS4bUuG1yxVumIRG0NIMLDc1fGr4lTP1tbVyrJYKnbFBI/PhaeYXSYuSZjTbbjXibC6nNQa+qrE1aXMu7ZERjlsYpYVkhEY6acNiE/zTBrF6wk6e+q1DDy9upC5+HyME9jz743chUZ6fXNfI/9Nom2SRjyHW8UA8HpSNLBVL2OwXGtYArvINrkhX6oDLa5RRCP/jUMc0RdLpe58w9ib2Tjtm7oe8FfjRz3oCnIsvEjalDs+AhJ3eZIBRumB6a1JAevQLk5jwmBPdTHgCwzO60jsGq5+mtq94RhUev/DKWhloyPrAOlqKjuxwtOLby9PbRZzlR6kZIOkGr3kwCxrPt9hWs+v1a+w4Oxha0WmCtemgBfFtu0Q0GxW3f7M526wKToYD6/Yb4MTyUXh29AtumaAy42JrLzKqKaEtd01uTMXR7njM3DD/3gvavcVVnMudbWF5aslFZTEoTqU0tEU7NoDN/Wj/nxEazy7AdcyMcwz8B/a8p9M8buxm/Cv1Z+b9AMUljnoUZ3UHXj/ywnNZxuxO06nuUSXMDKvvKjNb5zXxs7c+/+zqsWIxD0/Vev1+DOXauXquYFHq42IMN0+8zuO3qzzCt9hW8gfo6OY/By38+1uUED9B/tO++/wDbJnKwNkeo3697PIbdBryEe0KoV44bM/TLy+LeWm37492nebc9z81TaBb6hL5JAx/vbXS+wil1r8jepZjtNWbYISYS9f7uhTmZFcOrm8DjEd5pmwEefYnDL2Hkn86GN6GpWjeud6Y3tnLGaOTINTBa3oTc9vsQkF8zKfgIuW5sXwgml4ZGTrjuUKjod+H5TKqdbt/WxrW/AgAA///LSOTXfg8AAA==", - "provisionScriptParametersCommon": fmt.Sprintf("[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' CONTAINERD_VERSION=',parameters('containerdVersion'),' MOBY_VERSION=',parameters('mobyVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION=%s HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' MAXIMUM_LOADBALANCER_RULE_COUNT=',variables('maximumLoadBalancerRuleCount'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER=false',' PRIVATE_AZURE_REGISTRY_SERVER=',parameters('privateAzureRegistryServer'),' AUTHENTICATION_METHOD=',variables('customCloudAuthenticationMethod'),' IDENTITY_SYSTEM=',variables('customCloudIdentifySystem'))]", testK8sVersion), + "provisionScriptParametersCommon": fmt.Sprintf("[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' CONTAINERD_VERSION=',parameters('containerdVersion'),' MOBY_VERSION=',parameters('mobyVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION=%s-azs HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' MAXIMUM_LOADBALANCER_RULE_COUNT=',variables('maximumLoadBalancerRuleCount'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER=false',' PRIVATE_AZURE_REGISTRY_SERVER=',parameters('privateAzureRegistryServer'),' AUTHENTICATION_METHOD=',variables('customCloudAuthenticationMethod'),' IDENTITY_SYSTEM=',variables('customCloudIdentifySystem'))]", testK8sVersion), "provisionScriptParametersMaster": "[concat('COSMOS_URI= MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true NO_OUTBOUND=false CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", "readerRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]", "resourceGroup": "[resourceGroup().name]", From 8397555d648328b131159da7cbdd38e8e4a3168f Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Fri, 19 Apr 2019 13:47:58 -0700 Subject: [PATCH 11/13] test --- pkg/engine/armvariables_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/engine/armvariables_test.go b/pkg/engine/armvariables_test.go index cfeed0222e..747471f152 100644 --- a/pkg/engine/armvariables_test.go +++ b/pkg/engine/armvariables_test.go @@ -22,7 +22,7 @@ func TestSizeMap(t *testing.T) { sizeMap := getSizeMap() _, err := json.MarshalIndent(sizeMap["vmSizesMap"], "", " ") if err != nil { - t.Errorf("unxpected error while attempting to marshal the size map: %s", err.Error()) + t.Errorf("unexpected error while attempting to marshal the size map: %s", err.Error()) } } @@ -371,6 +371,12 @@ func TestK8sVars(t *testing.T) { containerRegistryDNSSuffix = "azurecr.io" tokenAudience = "https://management.azurestack.external/" ) + + customCloudK8sVersion = testK8sVersion + if name == "azurestackcloud" { + customCloudK8sVersion = testK8sVersion + "-azs" + } + cs = &api.ContainerService{ Location: "local", Properties: &api.Properties{ @@ -527,7 +533,7 @@ func TestK8sVars(t *testing.T) { }, "masterPublicLbFQDN": "blueorange.local.cloudapp.azurestack.external", "provisionConfigsCustomCloud": "H4sIAAAAAAAA/7RX7XLaSBb9z1PcyGzZrlgI72ZdKe+SlCKUCRMbXJKcVCak5Lb6CjoW3Up3yx9j8+5TLRBgwIakZvhhG3E559zT96O988K5ZNy5JGpYqyFXhUQPpWYpS4hGtbcP9zUAAPeP88API9f7GPvdT52g1z31u1H8e9jrxmdu9KFlOagT56q4RMlRo3LIn4VEpUlylWSioI3vSnBrGSvww9554Pnxqdt1f/OD2O+2z3qdbtS6SIiG+iZaeIDvP6AhUYlCJnhKOBmg9DnNBeMaHkBLsClYfcu6eJrbj9y2G7lzcqu+jUZnhJpQoomDU0L1luTMvkapmOCtfzcP/2s3D+3m4STvpJAZrEdeVlDGe+fBSRz40XnQ9Xptv1V/Wz7+eP7Oj71eNwp6Jyd+MJP1vnPit5ZPYUQ4S1FpVT60E8G1FFmG0h5NvGrckVFW4rIUvkJ9iRRetKAJ3/4Heoi8DDOvHQgwz0iCUP4cioyihFRIUCqDS8Yp44NZdAlsp1B/TvkKh3kppGAzsNTD/69FVoxQqezNgw2cjPDYcPX5JHAolD4jenhcPQDIzVvo940j/b5jgvtOglKrh4G1QcvzIk5FwfUjJZZSmTWnBhiZkLMnFSyGSiS0x7O7Y9CywI3aZtJSVls4D0/kd4wPjIXgms4LTeeBFEJDMm9o0KIMIXkuRS6ZeaS0kOUHlwhFTolG2pghL1ZrrxfFnh9Enfcdz438eFq80/a/JtLJ2KVzQ8gAuXYWx0gjx5G1FWbc9sOogiyUdDKRkMxRQyLRSYi9kIsZMWqBpZFIPSdJ8set9oz45wPnimbYE5fsJTnlx5gp/BsKf2dykMEpVKMNpv0K1ayp2uzR8TJlqkhppAcgcSSusQTapksXS91ZaDeHbluR6xBmvbI9TMpqC79qEztWVxPcsCwzNStRS4bUuG1yxVumIRG0NIMLDc1fGr4lTP1tbVyrJYKnbFBI/PhaeYXSYuSZjTbbjXibC6nNQa+qrE1aXMu7ZERjlsYpYVkhEY6acNiE/zTBrF6wk6e+q1DDy9upC5+HyME9jz743chUZ6fXNfI/9Nom2SRjyHW8UA8HpSNLBVL2OwXGtYArvINrkhX6oDLa5RRCP/jUMc0RdLpe58w9ib2Tjtm7oe8FfjRz3oCnIsvEjalDs+AhJ3eZIBRumB6a1JAevQLk5jwmBPdTHgCwzO60jsGq5+mtq94RhUev/DKWhloyPrAOlqKjuxwtOLby9PbRZzlR6kZIOkGr3kwCxrPt9hWs+v1a+w4Oxha0WmCtemgBfFtu0Q0GxW3f7M526wKToYD6/Yb4MTyUXh29AtumaAy42JrLzKqKaEtd01uTMXR7njM3DD/3gvavcVVnMudbWF5aslFZTEoTqU0tEU7NoDN/Wj/nxEazy7AdcyMcwz8B/a8p9M8buxm/Cv1Z+b9AMUljnoUZ3UHXj/ywnNZxuxO06nuUSXMDKvvKjNb5zXxs7c+/+zqsWIxD0/Vev1+DOXauXquYFHq42IMN0+8zuO3qzzCt9hW8gfo6OY/By38+1uUED9B/tO++/wDbJnKwNkeo3697PIbdBryEe0KoV44bM/TLy+LeWm37492nebc9z81TaBb6hL5JAx/vbXS+wil1r8jepZjtNWbYISYS9f7uhTmZFcOrm8DjEd5pmwEefYnDL2Hkn86GN6GpWjeud6Y3tnLGaOTINTBa3oTc9vsQkF8zKfgIuW5sXwgml4ZGTrjuUKjod+H5TKqdbt/WxrW/AgAA///LSOTXfg8AAA==", - "provisionScriptParametersCommon": fmt.Sprintf("[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' CONTAINERD_VERSION=',parameters('containerdVersion'),' MOBY_VERSION=',parameters('mobyVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION=%s-azs HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' MAXIMUM_LOADBALANCER_RULE_COUNT=',variables('maximumLoadBalancerRuleCount'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER=false',' PRIVATE_AZURE_REGISTRY_SERVER=',parameters('privateAzureRegistryServer'),' AUTHENTICATION_METHOD=',variables('customCloudAuthenticationMethod'),' IDENTITY_SYSTEM=',variables('customCloudIdentifySystem'))]", testK8sVersion), + "provisionScriptParametersCommon": fmt.Sprintf("[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' CONTAINERD_VERSION=',parameters('containerdVersion'),' MOBY_VERSION=',parameters('mobyVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION=%s HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' MAXIMUM_LOADBALANCER_RULE_COUNT=',variables('maximumLoadBalancerRuleCount'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER=false',' PRIVATE_AZURE_REGISTRY_SERVER=',parameters('privateAzureRegistryServer'),' AUTHENTICATION_METHOD=',variables('customCloudAuthenticationMethod'),' IDENTITY_SYSTEM=',variables('customCloudIdentifySystem'))]", customCloudK8sVersion), "provisionScriptParametersMaster": "[concat('COSMOS_URI= MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true NO_OUTBOUND=false CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", "readerRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]", "resourceGroup": "[resourceGroup().name]", From 329babe342947d95a2f632923e6bedeecfac6dee Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Fri, 19 Apr 2019 14:08:37 -0700 Subject: [PATCH 12/13] fixed build --- pkg/engine/armvariables_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/engine/armvariables_test.go b/pkg/engine/armvariables_test.go index 747471f152..28b22c8fa8 100644 --- a/pkg/engine/armvariables_test.go +++ b/pkg/engine/armvariables_test.go @@ -372,7 +372,7 @@ func TestK8sVars(t *testing.T) { tokenAudience = "https://management.azurestack.external/" ) - customCloudK8sVersion = testK8sVersion + customCloudK8sVersion := testK8sVersion if name == "azurestackcloud" { customCloudK8sVersion = testK8sVersion + "-azs" } From 822d74fb9b377a3233e967fb455f733a4b8fd4a7 Mon Sep 17 00:00:00 2001 From: Javier Darsie Date: Fri, 19 Apr 2019 14:55:49 -0700 Subject: [PATCH 13/13] Removed invalid chars, hopefully will get a Mac soon --- packer/install-dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packer/install-dependencies.sh b/packer/install-dependencies.sh index 66798db0ff..20f8cd5f63 100644 --- a/packer/install-dependencies.sh +++ b/packer/install-dependencies.sh @@ -370,9 +370,9 @@ K8S_VERSIONS=" 1.9.10 " for KUBERNETES_VERSION in ${K8S_VERSIONS}; do - if [[ $KUBERNETES_VERSION == *"azs"* ]]; then + if [[ $KUBERNETES_VERSION == *"azs"* ]]; then HYPERKUBE_URL="msazurestackdocker/hyperkube-amd64:v${KUBERNETES_VERSION}" -  else + else HYPERKUBE_URL="k8s.gcr.io/hyperkube-amd64:v${KUBERNETES_VERSION}" CONTAINER_IMAGE="k8s.gcr.io/cloud-controller-manager-amd64:v${KUBERNETES_VERSION}" pullContainerImage "docker" ${CONTAINER_IMAGE}