From 859690d72a4a1a63e8e088f33de8a6679e973477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Banaszewski?= Date: Fri, 14 Apr 2023 15:08:42 +0000 Subject: [PATCH] Added support for image credential provider for windows and arm64 on gce --- cluster/gce/config-common.sh | 14 +++++++ cluster/gce/gci/configure.sh | 41 ++++++++++--------- cluster/gce/util.sh | 20 +++++++++- cluster/gce/windows/configure.ps1 | 2 + cluster/gce/windows/k8s-node-setup.psm1 | 52 +++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 21 deletions(-) diff --git a/cluster/gce/config-common.sh b/cluster/gce/config-common.sh index 17a744ddf66d..a154466acdbf 100644 --- a/cluster/gce/config-common.sh +++ b/cluster/gce/config-common.sh @@ -167,3 +167,17 @@ export CSI_PROXY_STORAGE_PATH="https://storage.googleapis.com/gke-release/csi-pr export CSI_PROXY_VERSION="${CSI_PROXY_VERSION:-v1.1.1-gke.0}" # csi-proxy additional flags, there are additional flags that cannot be unset in k8s-node-setup.psm1 export CSI_PROXY_FLAGS="${CSI_PROXY_FLAGS:-}" +# Storage path for auth-provider-gcp binaries +export AUTH_PROVIDER_GCP_STORAGE_PATH="${AUTH_PROVIDER_GCP_STORAGE_PATH:-https://storage.googleapis.com/gke-release/auth-provider-gcp}" +# auth-provider-gcp version +export AUTH_PROVIDER_GCP_VERSION="${AUTH_PROVIDER_GCP_VERSION:-v0.0.2-gke.4}" +# Hash of auth-provider-gcp.exe binary +export AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64="${AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64:-348af2c189d938e1a4fa5ac5c640d21e003da1f000abcd6fd7eef2acd0678638286e40703618758d4fdfe2cc4b90e920f0422128ec777c74054af9dd4405de12}" +# Directory of kubelet image credential provider binary files on windows +export AUTH_PROVIDER_GCP_LINUX_BIN_DIR="${AUTH_PROVIDER_GCP_LINUX_BIN_DIR:-/home/kubernetes/bin}" +# Location of kubelet image credential provider config file on windows +export AUTH_PROVIDER_GCP_LINUX_CONF_FILE="${AUTH_PROVIDER_GCP_LINUX_CONF_FILE:-/home/kubernetes/cri-auth-config.yaml}" +# Directory of kubelet image credential provider binary files on windows +export AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR=${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR:-${WINDOWS_NODE_DIR}} +# Location of kubelet image credential provider config file on windows +export AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE="${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE:-${WINDOWS_K8S_DIR}\cri-auth-config.yaml}" diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index 8138ad7ad79b..f97c8fcc2d0f 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -28,14 +28,14 @@ DEFAULT_CNI_VERSION='v1.2.0' DEFAULT_CNI_HASH='29ea9be8e81e0b4c44469c4307cd8be83647e30ade8b737d94df81477b494662308b2566fce80cfa993c761afb6e5bad9382455260b857c7f941fa18bb7919b4' DEFAULT_NPD_VERSION='v0.8.9' DEFAULT_NPD_HASH_AMD64='4919c47447c5f3871c1dc3171bbb817a38c8c8d07a6ce55a77d43cadc098e9ad608ceeab121eec00c13c0b6a2cc3488544d61ce84cdade1823f3fd5163a952de' -DEFAULT_AUTH_PROVIDER_GCP_HASH_AMD64='88d9fa581002973170ca58427763f00355b24fbabd66f7fee725a0845ad88bee644e60eed2d95a5721e6ae0056a81a5990bf02148ea49817c174bcb2cc9c0626' -DEFAULT_AUTH_PROVIDER_GCP_VERSION='v0.24.0' # TODO (SergeyKanzhelev): fill up for npd 0.8.9+ DEFAULT_NPD_HASH_ARM64='8ccb42a862efdfc1f25ca9a22f3fd36f9fdff1ac618dd7d39e3b5991505dd610d432364420896ad71f42197a116f28a85dde58b129baa075ebb7312caa57f852' DEFAULT_CRICTL_VERSION='v1.26.1' DEFAULT_CRICTL_AMD64_SHA512='e3a20c4d18bbbd9f5dc303e90b649181f9b8242758de58d04ea8acd0d3da32919b8859e90b6602354755dc82b79833caf30510e5a27b0ebc6a2b1545d744d4cc' DEFAULT_CRICTL_ARM64_SHA512='605d8fe73e7e0b93dfac9ea90548e7334b55dda1f0abeb4c3382ae15d6d250a71fccfde20e8fd68bde59b4c1d54ec69ef295a2aa2119e8579d60d0dbcf380d2e' DEFAULT_MOUNTER_TAR_SHA='7956fd42523de6b3107ddc3ce0e75233d2fcb78436ff07a1389b6eaac91fb2b1b72a08f7a219eaf96ba1ca4da8d45271002e0d60e0644e796c665f99bb356516' +AUTH_PROVIDER_GCP_HASH_LINUX_AMD64="${AUTH_PROVIDER_GCP_HASH_LINUX_AMD64:-156058e5b3994cba91c23831774033e0d505d6d8b80f43541ef6af91b320fd9dfaabe42ec8a8887b51d87104c2b57e1eb895649d681575ffc80dd9aee8e563db}" +AUTH_PROVIDER_GCP_HASH_LINUX_ARM64="${AUTH_PROVIDER_GCP_HASH_LINUX_ARM64:-1aa3b0bea10a9755231989ffc150cbfa770f1d96932db7535473f7bfeb1108bafdae80202ae738d59495982512e716ff7366d5f414d0e76dd50519f98611f9ab}" ### # Standard curl flags. @@ -549,30 +549,33 @@ function install-containerd-ubuntu { } function install-auth-provider-gcp { - local -r auth_provider_tar="auth-provider-gcp-${DEFAULT_AUTH_PROVIDER_GCP_VERSION}-${HOST_PLATFORM}_${HOST_ARCH}.tar.gz" - echo "Downloading auth-provider-gcp ${auth_provider_tar}" . + local -r filename="auth-provider-gcp" + local -r auth_provider_storage_full_path="${AUTH_PROVIDER_GCP_STORAGE_PATH}/${AUTH_PROVIDER_GCP_VERSION}/${HOST_PLATFORM}_${HOST_ARCH}/${filename}" + echo "Downloading auth-provider-gcp ${auth_provider_storage_full_path}" . - local -r auth_provider_release_path="https://storage.googleapis.com/cloud-provider-gcp" - download-or-bust "${DEFAULT_AUTH_PROVIDER_GCP_HASH_AMD64}" "${auth_provider_release_path}/${auth_provider_tar}" + case "${HOST_ARCH}" in + amd64) + local -r auth_provider_gcp_hash="${AUTH_PROVIDER_GCP_HASH_LINUX_AMD64}" + ;; + arm64) + local -r auth_provider_gcp_hash="${AUTH_PROVIDER_GCP_HASH_LINUX_ARM64}" + ;; + *) + echo "Unrecognized version and platform/arch combination: ${HOST_PLATFORM}/${HOST_ARCH}" + exit 1 + esac - # Keep in sync with --image-credential-provider-bin-dir in ../util.sh - local auth_provider_dir="${KUBE_HOME}/auth-provider-gcp" - mkdir -p "${auth_provider_dir}" - tar xzf "${KUBE_HOME}/${auth_provider_tar}" -C "${auth_provider_dir}" --overwrite - mv "${auth_provider_dir}/auth-provider-gcp" "${KUBE_BIN}" - chmod a+x "${KUBE_BIN}/auth-provider-gcp" + download-or-bust "${auth_provider_gcp_hash}" "${auth_provider_storage_full_path}" - rm -f "${KUBE_HOME}/${auth_provider_tar}" - rmdir "${auth_provider_dir}" + mv "${KUBE_HOME}/${filename}" "${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}" + chmod a+x "${KUBE_BIN}/${filename}" - # Keep in sync with --image-credential-provider-config in ../util.sh - local auth_config_file="${KUBE_HOME}/cri_auth_config.yaml" - cat >> "${auth_config_file}" << EOF + cat >> "${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}" << EOF kind: CredentialProviderConfig -apiVersion: kubelet.config.k8s.io/v1beta1 +apiVersion: kubelet.config.k8s.io/v1 providers: - name: auth-provider-gcp - apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 + apiVersion: credentialprovider.kubelet.k8s.io/v1 matchImages: - "container.cloud.google.com" - "gcr.io" diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 76008e4ca0d9..74c946241317 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -758,8 +758,8 @@ function construct-linux-kubelet-flags { # Keep the values of --image-credential-provider-config and --image-credential-provider-bin-dir # in sync with value of auth_config_file and auth_provider_dir set in install-auth-provider-gcp function # in gci/configure.sh. - flags+=" --image-credential-provider-config=/home/kubernetes/cri_auth_config.yaml" - flags+=" --image-credential-provider-bin-dir=/home/kubernetes/bin" + flags+=" --image-credential-provider-config=${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}" + flags+=" --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}" fi if [[ "${node_type}" == "master" ]]; then @@ -864,6 +864,13 @@ function construct-windows-kubelet-flags { WINDOWS_CONTAINER_RUNTIME_ENDPOINT=${KUBE_WINDOWS_CONTAINER_RUNTIME_ENDPOINT:-npipe:////./pipe/containerd-containerd} flags+=" --container-runtime-endpoint=${WINDOWS_CONTAINER_RUNTIME_ENDPOINT}" + # If ENABLE_AUTH_PROVIDER_GCP is set to true, kubelet is enabled to use out-of-tree auth + # credential provider. https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider + if [[ "${ENABLE_AUTH_PROVIDER_GCP:-false}" == "true" ]]; then + flags+=" --image-credential-provider-config=${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}" + flags+=" --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}" + fi + KUBELET_ARGS="${flags}" } @@ -1199,6 +1206,10 @@ ${CUSTOM_CALICO_NODE_DAEMONSET_YAML//\'/\'\'} CUSTOM_TYPHA_DEPLOYMENT_YAML: | ${CUSTOM_TYPHA_DEPLOYMENT_YAML//\'/\'\'} CONCURRENT_SERVICE_SYNCS: $(yaml-quote "${CONCURRENT_SERVICE_SYNCS:-}") +AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}") +AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}") +AUTH_PROVIDER_GCP_LINUX_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}") +AUTH_PROVIDER_GCP_LINUX_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}") EOF if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || \ [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]] || \ @@ -1581,6 +1592,11 @@ NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_RELEAS NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote "${WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS}") NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TOKEN:-}") WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE}") +AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}") +AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}") +AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64: $(yaml-quote "${AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64}") +AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}") +AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}") EOF } diff --git a/cluster/gce/windows/configure.ps1 b/cluster/gce/windows/configure.ps1 index 7063605d94ff..adb5749635f0 100644 --- a/cluster/gce/windows/configure.ps1 +++ b/cluster/gce/windows/configure.ps1 @@ -165,11 +165,13 @@ try { DownloadAndInstall-KubernetesBinaries DownloadAndInstall-NodeProblemDetector DownloadAndInstall-CSIProxyBinaries + DownloadAndInstall-AuthProviderGcpBinary Start-CSIProxy Create-NodePki Create-KubeletKubeconfig Create-KubeproxyKubeconfig Create-NodeProblemDetectorKubeConfig + Create-AuthProviderGcpConfig Set-PodCidr Configure-HostNetworkingService Prepare-CniNetworking diff --git a/cluster/gce/windows/k8s-node-setup.psm1 b/cluster/gce/windows/k8s-node-setup.psm1 index 4377c4e48332..e2964a968fbd 100644 --- a/cluster/gce/windows/k8s-node-setup.psm1 +++ b/cluster/gce/windows/k8s-node-setup.psm1 @@ -297,6 +297,12 @@ function Set-EnvironmentVars { "WINDOWS_ENABLE_HYPERV" = ${kube_env}['WINDOWS_ENABLE_HYPERV'] "ENABLE_NODE_PROBLEM_DETECTOR" = ${kube_env}['ENABLE_NODE_PROBLEM_DETECTOR'] "NODEPROBLEMDETECTOR_KUBECONFIG_FILE" = ${kube_env}['WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE'] + "ENABLE_AUTH_PROVIDER_GCP" = ${kube_env}['ENABLE_AUTH_PROVIDER_GCP'] + "AUTH_PROVIDER_GCP_STORAGE_PATH" = ${kube_env}['AUTH_PROVIDER_GCP_STORAGE_PATH'] + "AUTH_PROVIDER_GCP_VERSION" = ${kube_env}['AUTH_PROVIDER_GCP_VERSION'] + "AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64" = ${kube_env}['AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64'] + "AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR" = ${kube_env}['AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR'] + "AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE" = ${kube_env}['AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE'] "Path" = ${env:Path} + ";" + ${kube_env}['NODE_DIR'] "KUBE_NETWORK" = "l2bridge".ToLower() @@ -2286,5 +2292,51 @@ $FLUENTD_CONFIG = @' '@ +# Downloads the out-of-tree kubelet image credential provider binaries. +function DownloadAndInstall-AuthProviderGcpBinary { + if ("${env:ENABLE_AUTH_PROVIDER_GCP}" -eq "true") { + $filename = 'auth-provider-gcp.exe' + if (ShouldWrite-File ${env:AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}\$filename) { + Log-Output "Installing auth provider gcp binaries" + $tmp_dir = 'C:\k8s_tmp' + New-Item -Force -ItemType 'directory' $tmp_dir | Out-Null + $url = "${env:AUTH_PROVIDER_GCP_STORAGE_PATH}/${env:AUTH_PROVIDER_GCP_VERSION}/windows_amd64/$filename" + MustDownload-File -Hash $AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64 -Algorithm SHA512 -OutFile $tmp_dir\$filename -URLs $url + Move-Item -Force $tmp_dir\$filename ${env:AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR} + Remove-Item -Force -Recurse $tmp_dir + } else { + Log-Output "Skipping auth provider gcp binaries installation, auth-provider-gcp.exe file already exists." + } + } +} + +# Creates config file for the out-of-tree kubelet image credential provider. +function Create-AuthProviderGcpConfig { + if ("${env:ENABLE_AUTH_PROVIDER_GCP}" -eq "true") { + if (ShouldWrite-File ${env:AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}) { + Log-Output "Creating auth provider gcp config file" + Set-Content ${env:AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE} @' +kind: CredentialProviderConfig +apiVersion: kubelet.config.k8s.io/v1 +providers: + - name: auth-provider-gcp.exe + apiVersion: credentialprovider.kubelet.k8s.io/v1 + matchImages: + - "container.cloud.google.com" + - "gcr.io" + - "*.gcr.io" + - "*.pkg.dev" + args: + - get-credentials + - --v=3 + defaultCacheDuration: 1m +'@ + } else { + Log-Output "Skipping auth provider gcp config file creation, it already exists" + } + } +} + + # Export all public functions: Export-ModuleMember -Function *-*