From 0cb70ec0eb47940fe803bb439e284504abe7ffc9 Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Fri, 29 Sep 2023 12:15:21 -0700 Subject: [PATCH 01/16] upgrade azuredisk-csi-driver to v1.28.3 - Upstream AKS uses azuredisk-csi-driver v1.26.5 for k8s v1.26 and azuredisk-csi-driver v1.28.3 for k8s v1.27. - Upgrade azuredisk csi driver to v1.28.3 - Upgrade all components according to new v1.28.3 csi driver values: https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.28.3/azuredisk-csi-driver/values.yaml --- docs/topics/azure-stack.md | 6 +- pkg/api/k8s_versions.go | 502 +++++++++++++++++---------- vhd/packer/configure-windows-vhd.ps1 | 4 + vhd/packer/install-dependencies.sh | 8 + 4 files changed, 329 insertions(+), 191 deletions(-) diff --git a/docs/topics/azure-stack.md b/docs/topics/azure-stack.md index d49aa58e6..d07a2c915 100644 --- a/docs/topics/azure-stack.md +++ b/docs/topics/azure-stack.md @@ -203,7 +203,8 @@ If the data persisted in the underlying Azure disks should be preserved, then th The following script uses `Helm` to install the Azure Disk CSI Driver: ```bash -DRIVER_VERSION=v1.26.5 +DRIVER_VERSION=v1.26.5 # if using k8s v1.26 +DRIVER_VERSION=v1.28.3 # if using k8s v1.27 helm repo add azuredisk-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver \ --namespace kube-system \ @@ -278,7 +279,8 @@ In this section, please follow the example commands to deploy a StatefulSet appl ```bash # Install CSI Driver -DRIVER_VERSION=v1.26.5 +DRIVER_VERSION=v1.26.5 # if using k8s v1.26 +DRIVER_VERSION=v1.28.3 # if using k8s v1.27 helm repo add azuredisk-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver --namespace kube-system --set cloud=AzureStackCloud --set controller.runOnMaster=true --set node.supportZone=false --set windows.getNodeInfoFromLabels=true --set linux.getNodeInfoFromLabels=true --version ${DRIVER_VERSION} diff --git a/pkg/api/k8s_versions.go b/pkg/api/k8s_versions.go index 31bd1609f..52f3a42a9 100644 --- a/pkg/api/k8s_versions.go +++ b/pkg/api/k8s_versions.go @@ -33,16 +33,6 @@ const ( azurePolicyImageReference string = "mcr.microsoft.com/azure-policy/policy-kubernetes-addon-prod:prod_20201023.1" gatekeeperImageReference string = "mcr.microsoft.com/oss/open-policy-agent/gatekeeper:v3.2.3" nodeProblemDetectorImageReference string = "registry.k8s.io/node-problem-detector/node-problem-detector:v0.8.4" - csiProvisionerImageReference string = "oss/kubernetes-csi/csi-provisioner:v3.3.0" - csiAttacherImageReference string = "oss/kubernetes-csi/csi-attacher:v4.0.0" - csiLivenessProbeImageReference string = "oss/kubernetes-csi/livenessprobe:v2.8.0" - csiLivenessProbeWindowsImageReference string = "oss/kubernetes-csi/livenessprobe:v2.8.0" - csiNodeDriverRegistrarImageReference string = "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2" - csiNodeDriverRegistrarWindowsImageReference string = "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2" - csiResizerImageReference string = "oss/kubernetes-csi/csi-resizer:v1.6.0" - csiSnapshotterImageReference string = "oss/kubernetes-csi/csi-snapshotter:v5.0.1" - csiSnapshotControllerImageReference string = "oss/kubernetes-csi/snapshot-controller:v5.0.1" - csiAzureDiskImageReference string = "oss/kubernetes-csi/azuredisk-csi:v1.26.5" csiAzureFileImageReference string = "oss/kubernetes-csi/azurefile-csi:v1.9.0" azureCloudControllerManagerImageReference string = "oss/kubernetes/azure-cloud-controller-manager:v1.1.1" azureCloudNodeManagerImageReference string = "oss/kubernetes/azure-cloud-node-manager:v1.1.1" @@ -261,92 +251,226 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string }, }, common.KubernetesImageBaseTypeMCR: { + "1.27": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.5.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.10.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.10.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v6.2.2", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v6.2.2", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.8.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.28.3", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.6", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + }, "1.26": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.6", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.3.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.0.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v5.0.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v5.0.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.6.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.26.5", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.6", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.25": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.6", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.3.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.0.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v5.0.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v5.0.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.6.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.26.5", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.6", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.24": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.23": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.22": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", - }, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", "1.21": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.20": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.19": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.18": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.0", }, "1.17": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", }, "1.16": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.16.7", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.16.7", }, "1.15": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.15.7", - common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.15.7", + common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", }, "1.14": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.0.2", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.14.8", - common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.0.2", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.14.8", + common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", }, "1.13": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v8.9.1", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.13.9", - common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v8.9.1", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.13.9", + common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", }, "1.12": { common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", @@ -573,16 +697,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -656,16 +780,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -739,16 +863,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -822,16 +946,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -905,16 +1029,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -988,16 +1112,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1071,16 +1195,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1154,16 +1278,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1237,16 +1361,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1320,14 +1444,14 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1399,12 +1523,12 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1475,11 +1599,11 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1548,11 +1672,11 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1619,11 +1743,11 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, diff --git a/vhd/packer/configure-windows-vhd.ps1 b/vhd/packer/configure-windows-vhd.ps1 index 98e276b36..11764099b 100644 --- a/vhd/packer/configure-windows-vhd.ps1 +++ b/vhd/packer/configure-windows-vhd.ps1 @@ -56,10 +56,14 @@ function Get-ContainerImages { "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.10.0-windows-hp", "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.26.5", "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.26.5-windows-hp", + "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.28.3", + "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.28.3-windows-hp", "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.5.0", "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.8.0", + "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.10.0", "mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0") } '2004' { diff --git a/vhd/packer/install-dependencies.sh b/vhd/packer/install-dependencies.sh index a73d5eff6..d5b8f2cee 100644 --- a/vhd/packer/install-dependencies.sh +++ b/vhd/packer/install-dependencies.sh @@ -239,6 +239,7 @@ done AZUREDISK_CSI_VERSIONS=" 1.10.0 1.26.5 +1.28.3 " for AZUREDISK_CSI_VERSION in ${AZUREDISK_CSI_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v${AZUREDISK_CSI_VERSION}" @@ -256,6 +257,7 @@ for AZUREFILE_CSI_VERSION in ${AZUREFILE_CSI_VERSIONS}; do done CSI_ATTACHER_VERSIONS=" +4.3.0 4.0.0 3.3.0 " @@ -266,6 +268,7 @@ for CSI_ATTACHER_VERSION in ${CSI_ATTACHER_VERSIONS}; do done CSI_NODE_DRIVER_REGISTRAR_VERSIONS=" +2.8.0 2.6.2 2.4.0 " @@ -276,6 +279,7 @@ for CSI_NODE_DRIVER_REGISTRAR_VERSION in ${CSI_NODE_DRIVER_REGISTRAR_VERSIONS}; done CSI_PROVISIONER_VERSIONS=" +3.5.0 3.3.0 3.0.0 " @@ -286,6 +290,7 @@ for CSI_PROVISIONER_VERSION in ${CSI_PROVISIONER_VERSIONS}; do done LIVENESSPROBE_VERSIONS=" +2.10.0 2.8.0 2.5.0 " @@ -296,6 +301,7 @@ for LIVENESSPROBE_VERSION in ${LIVENESSPROBE_VERSIONS}; do done CSI_RESIZER_VERSIONS=" +1.8.0 1.6.0 1.3.0 " @@ -306,6 +312,7 @@ for CSI_RESIZER_VERSION in ${CSI_RESIZER_VERSIONS}; do done CSI_SNAPSHOTTER_VERSIONS=" +6.2.2 5.0.1 4.2.1 " @@ -316,6 +323,7 @@ for CSI_SNAPSHOTTER_VERSION in ${CSI_SNAPSHOTTER_VERSIONS}; do done SNAPSHOT_CONTROLLER_VERSIONS=" +6.2.2 5.0.1 4.2.1 " From 10d6615298a9af0479e65411d90fffb7ddfed1c1 Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Fri, 29 Sep 2023 12:15:34 -0700 Subject: [PATCH 02/16] update v1.28.3 csi driver template: csi-azuredisk-node-windows.yaml - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.26.6/azuredisk-csi-driver/templates/csi-azuredisk-node-windows.yaml - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.28.3/azuredisk-csi-driver/templates/csi-azuredisk-node-windows.yaml --- parts/k8s/addons/azuredisk-csi-driver-deployment.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml index 68024e7a4..fdce012e2 100644 --- a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml +++ b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml @@ -265,6 +265,9 @@ spec: nodeSelector: kubernetes.io/os: windows priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - key: "node.kubernetes.io/os" operator: "Exists" @@ -333,10 +336,10 @@ spec: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" - - "--metrics-address=0.0.0.0:29605" - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" + - "--get-nodeid-from-imds=false" ports: - containerPort: 29603 name: healthz From fe06af53464871c56be22f981d3e9a8959eb927c Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Fri, 29 Sep 2023 15:54:18 -0700 Subject: [PATCH 03/16] update v1.28.3 csi driver template: csi-azuredisk-node.yaml - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.26.6/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.28.3/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml --- parts/k8s/addons/azuredisk-csi-driver-deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml index fdce012e2..706038946 100644 --- a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml +++ b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml @@ -461,6 +461,9 @@ spec: nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - operator: "Exists" containers: @@ -523,6 +526,7 @@ spec: - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" + - "--get-nodeid-from-imds=false" ports: - containerPort: 29603 name: healthz From 780c9324072549a95fb5497b9d59339f6d4115fd Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Fri, 29 Sep 2023 17:06:20 -0700 Subject: [PATCH 04/16] update v1.28.3 csi driver template: csi-azuredisk-controller.yaml - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.26.6/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.28.3/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml --- parts/k8s/addons/azuredisk-csi-driver-deployment.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml index 706038946..20f8b00cd 100644 --- a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml +++ b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml @@ -642,6 +642,9 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - key: "node-role.kubernetes.io/master" operator: "Exists" @@ -662,8 +665,8 @@ spec: - "--worker-threads=40" - "--extra-create-metadata=true" - "--strict-topology=true" - - "--kube-api-qps=50" - - "--kube-api-burst=100" + - "--kube-api-qps=200" + - "--kube-api-burst=400" env: - name: ADDRESS value: /csi/csi.sock @@ -771,6 +774,8 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=aks-engine" - "--vmss-cache-ttl-seconds=-1" + - "--enable-traffic-manager=false" + - "--traffic-manager-port=7788" ports: - containerPort: 29602 name: healthz From 413790a978bdf8d11663a62a115c1487873de93a Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Fri, 29 Sep 2023 17:09:50 -0700 Subject: [PATCH 05/16] update v1.28.3 csi driver template: csi-snapshot-controller.yaml - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.26.6/azuredisk-csi-driver/templates/csi-snapshot-controller.yaml - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.28.3/azuredisk-csi-driver/templates/csi-snapshot-controller.yaml --- .../k8s/addons/azuredisk-csi-driver-deployment.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml index 20f8b00cd..1c8d75a11 100644 --- a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml +++ b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml @@ -845,6 +845,15 @@ spec: selector: matchLabels: app: csi-snapshot-controller + # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable + # in #504 the snapshot-controller will exit after around 7.5 seconds if it + # can't find the v1 CRDs so this value should be greater than that + minReadySeconds: 15 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: @@ -855,6 +864,9 @@ spec: kubernetes.io/os: linux kubernetes.io/role: master priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - key: "node-role.kubernetes.io/master" operator: "Equal" From cee0ed50f33bf4109795a73782351bfa320e9b8a Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Fri, 29 Sep 2023 17:31:20 -0700 Subject: [PATCH 06/16] update v1.28.3 csi driver template: crd-csi-snapshot.yaml - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.26.6/azuredisk-csi-driver/templates/crd-csi-snapshot.yaml - https://github.com/kubernetes-sigs/azuredisk-csi-driver/blob/master/charts/v1.28.3/azuredisk-csi-driver/templates/crd-csi-snapshot.yaml --- .../azuredisk-csi-driver-deployment.yaml | 298 ++++++++++++++---- 1 file changed, 239 insertions(+), 59 deletions(-) diff --git a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml index 1c8d75a11..0bfe3cbbb 100644 --- a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml +++ b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml @@ -978,8 +978,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.0 - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -1000,15 +1000,18 @@ spec: jsonPath: .status.readyToUse name: ReadyToUse type: boolean - - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + - description: If a new snapshot needs to be created, this contains the name of + the source PVC from which this snapshot was (or will be) created. jsonPath: .spec.source.persistentVolumeClaimName name: SourcePVC type: string - - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + - description: If a snapshot already exists, this contains the name of the existing + VolumeSnapshotContent object representing the existing snapshot. jsonPath: .spec.source.volumeSnapshotContentName name: SourceSnapshotContent type: string - - description: Represents the minimum size of volume required to rehydrate from this snapshot. + - description: Represents the minimum size of volume required to rehydrate from + this snapshot. jsonPath: .status.restoreSize name: RestoreSize type: string @@ -1016,11 +1019,16 @@ spec: jsonPath: .spec.volumeSnapshotClassName name: SnapshotClass type: string - - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage of + this object. jsonPath: .status.boundVolumeSnapshotContentName name: SnapshotContent type: string - - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + - description: Timestamp when the point-in-time snapshot was taken by the underlying + storage system. jsonPath: .status.creationTime name: CreationTime type: date @@ -1030,51 +1038,102 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: - description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. properties: source: - description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. properties: persistentVolumeClaimName: - description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object representing the volume from which + a snapshot should be created. This PVC is assumed to be in the + same namespace as the VolumeSnapshot object. This field should + be set if the snapshot does not exists, and needs to be created. + This field is immutable. type: string volumeSnapshotContentName: - description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + description: volumeSnapshotContentName specifies the name of a + pre-existing VolumeSnapshotContent object representing an existing + volume snapshot. This field should be set if the snapshot already + exists and only needs a representation in Kubernetes. This field + is immutable. type: string type: object oneOf: - required: ["persistentVolumeClaimName"] - required: ["volumeSnapshotContentName"] volumeSnapshotClassName: - description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be used. + A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one VolumeSnapshotClass + exist for a given CSI Driver and more than one have been marked + as default, CreateSnapshot will fail and generate an event. Empty + string is not allowed for this field.' type: string required: - source type: object status: - description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object. properties: boundVolumeSnapshotContentName: - description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent + object to which this VolumeSnapshot object intends to bind to. If + not specified, it indicates that the VolumeSnapshot object has not + been successfully bound to a VolumeSnapshotContent object yet. NOTE: + To avoid possible security issues, consumers must verify binding + between VolumeSnapshot and VolumeSnapshotContent objects is successful + (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object.' type: string creationTime: - description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the snapshot controller + with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may indicate + that the creation time of the snapshot is unknown. format: date-time type: string error: - description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. The snapshot controller will keep retrying when an error + occurs during the snapshot creation. Upon success, this error field + will be cleared. properties: message: - description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' type: string time: description: time is the timestamp when the error was encountered. @@ -1082,11 +1141,27 @@ spec: type: string type: object readyToUse: - description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + description: readyToUse indicates if the snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. type: boolean restoreSize: type: string - description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller with + the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the + "size_bytes" value returned from the CSI "ListSnapshots" gRPC call + if the driver supports it. When restoring a volume from this snapshot, + the size of the volume MUST NOT be smaller than the restoreSize + if it is specified, otherwise the restoration will fail. If not + specified, it indicates that the size is unknown. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -1175,7 +1250,7 @@ spec: format: date-time type: string error: - description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. properties: message: description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' @@ -1197,7 +1272,7 @@ spec: required: - spec type: object - served: true + served: false storage: false subresources: status: {} @@ -1213,8 +1288,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.0 - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -1235,7 +1310,8 @@ spec: - jsonPath: .driver name: Driver type: string - - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. jsonPath: .deletionPolicy name: DeletionPolicy type: string @@ -1245,27 +1321,42 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string deletionPolicy: - description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its bound + VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. + Required. enum: - Delete - Retain type: string driver: - description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string parameters: additionalProperties: type: string - description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. type: object required: - deletionPolicy @@ -1319,7 +1410,7 @@ spec: - deletionPolicy - driver type: object - served: true + served: false storage: false subresources: {} status: @@ -1334,8 +1425,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.0 - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -1358,11 +1449,14 @@ spec: jsonPath: .status.restoreSize name: RestoreSize type: integer - - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. jsonPath: .spec.deletionPolicy name: DeletionPolicy type: string - - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. jsonPath: .spec.driver name: Driver type: string @@ -1370,7 +1464,8 @@ spec: jsonPath: .spec.volumeSnapshotClassName name: VolumeSnapshotClass type: string - - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. jsonPath: .spec.volumeSnapshotRef.name name: VolumeSnapshot type: string @@ -1384,50 +1479,102 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: - description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. properties: deletionPolicy: - description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are deleted. For dynamically provisioned + snapshots, this field will automatically be filled in by the CSI + snapshotter sidecar with the "DeletionPolicy" field defined in the + corresponding VolumeSnapshotClass. For pre-existing snapshots, users + MUST specify this field when creating the VolumeSnapshotContent + object. Required. enum: - Delete - Retain type: string driver: - description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be + the same as the name returned by the CSI GetPluginName() call for + that driver. Required. type: string source: - description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + description: source specifies whether the snapshot is (or should be) + dynamically provisioned or already exists, and just requires a Kubernetes + object representation. This field is immutable after creation. Required. properties: snapshotHandle: - description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + description: snapshotHandle specifies the CSI "snapshot_id" of + a pre-existing snapshot on the underlying storage system for + which a Kubernetes object representation was (or should be) + created. This field is immutable. type: string volumeHandle: - description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. type: string type: object oneOf: - required: ["snapshotHandle"] - required: ["volumeHandle"] + sourceVolumeMode: + description: SourceVolumeMode is the mode of the volume whose snapshot + is taken. Can be either “Filesystem” or “Block”. If not specified, + it indicates the source volume's mode is unknown. This field is + immutable. This field is an alpha field. + type: string volumeSnapshotClassName: - description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and as + such, should not be referenced post-snapshot creation. type: string volumeSnapshotRef: - description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after creation. + Required. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' type: string kind: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' @@ -1439,7 +1586,8 @@ spec: description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' type: string resourceVersion: - description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' type: string uid: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' @@ -1455,14 +1603,27 @@ spec: description: status represents the current information of a snapshot. properties: creationTime: - description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the CSI snapshotter + sidecar with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it indicates + the creation time is unknown. The format of this field is a Unix + nanoseconds time encoded as an int64. On Unix, the command `date + +%s%N` returns the current time in nanoseconds since 1970-01-01 + 00:00:00 UTC. format: int64 type: integer error: - description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. properties: message: - description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' type: string time: description: time is the timestamp when the error was encountered. @@ -1470,15 +1631,34 @@ spec: type: string type: object readyToUse: - description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. type: boolean restoreSize: - description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" value + returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. When restoring a volume from this snapshot, the size of the + volume MUST NOT be smaller than the restoreSize if it is specified, + otherwise the restoration will fail. If not specified, it indicates + that the size is unknown. format: int64 minimum: 0 type: integer snapshotHandle: - description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. type: string type: object required: @@ -1625,7 +1805,7 @@ spec: required: - spec type: object - served: true + served: false storage: false subresources: status: {} From 56f6fc8eef2bcc389ac6c603844b7b5513dc2c67 Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Fri, 29 Sep 2023 18:01:38 -0700 Subject: [PATCH 07/16] Update generated template --- pkg/engine/templates_generated.go | 328 ++++++++++++++++++++++++------ 1 file changed, 266 insertions(+), 62 deletions(-) diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index ea36d6b87..d9c9b1e22 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -7628,6 +7628,9 @@ spec: nodeSelector: kubernetes.io/os: windows priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - key: "node.kubernetes.io/os" operator: "Exists" @@ -7696,10 +7699,10 @@ spec: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" - - "--metrics-address=0.0.0.0:29605" - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" + - "--get-nodeid-from-imds=false" ports: - containerPort: 29603 name: healthz @@ -7821,6 +7824,9 @@ spec: nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - operator: "Exists" containers: @@ -7883,6 +7889,7 @@ spec: - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" + - "--get-nodeid-from-imds=false" ports: - containerPort: 29603 name: healthz @@ -7998,6 +8005,9 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - key: "node-role.kubernetes.io/master" operator: "Exists" @@ -8018,8 +8028,8 @@ spec: - "--worker-threads=40" - "--extra-create-metadata=true" - "--strict-topology=true" - - "--kube-api-qps=50" - - "--kube-api-burst=100" + - "--kube-api-qps=200" + - "--kube-api-burst=400" env: - name: ADDRESS value: /csi/csi.sock @@ -8127,6 +8137,8 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=aks-engine" - "--vmss-cache-ttl-seconds=-1" + - "--enable-traffic-manager=false" + - "--traffic-manager-port=7788" ports: - containerPort: 29602 name: healthz @@ -8196,6 +8208,15 @@ spec: selector: matchLabels: app: csi-snapshot-controller + # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable + # in #504 the snapshot-controller will exit after around 7.5 seconds if it + # can't find the v1 CRDs so this value should be greater than that + minReadySeconds: 15 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: @@ -8206,6 +8227,9 @@ spec: kubernetes.io/os: linux kubernetes.io/role: master priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault tolerations: - key: "node-role.kubernetes.io/master" operator: "Equal" @@ -8317,8 +8341,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.0 - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -8339,15 +8363,18 @@ spec: jsonPath: .status.readyToUse name: ReadyToUse type: boolean - - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + - description: If a new snapshot needs to be created, this contains the name of + the source PVC from which this snapshot was (or will be) created. jsonPath: .spec.source.persistentVolumeClaimName name: SourcePVC type: string - - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + - description: If a snapshot already exists, this contains the name of the existing + VolumeSnapshotContent object representing the existing snapshot. jsonPath: .spec.source.volumeSnapshotContentName name: SourceSnapshotContent type: string - - description: Represents the minimum size of volume required to rehydrate from this snapshot. + - description: Represents the minimum size of volume required to rehydrate from + this snapshot. jsonPath: .status.restoreSize name: RestoreSize type: string @@ -8355,11 +8382,16 @@ spec: jsonPath: .spec.volumeSnapshotClassName name: SnapshotClass type: string - - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage of + this object. jsonPath: .status.boundVolumeSnapshotContentName name: SnapshotContent type: string - - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + - description: Timestamp when the point-in-time snapshot was taken by the underlying + storage system. jsonPath: .status.creationTime name: CreationTime type: date @@ -8369,51 +8401,102 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: - description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. properties: source: - description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. properties: persistentVolumeClaimName: - description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object representing the volume from which + a snapshot should be created. This PVC is assumed to be in the + same namespace as the VolumeSnapshot object. This field should + be set if the snapshot does not exists, and needs to be created. + This field is immutable. type: string volumeSnapshotContentName: - description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + description: volumeSnapshotContentName specifies the name of a + pre-existing VolumeSnapshotContent object representing an existing + volume snapshot. This field should be set if the snapshot already + exists and only needs a representation in Kubernetes. This field + is immutable. type: string type: object oneOf: - required: ["persistentVolumeClaimName"] - required: ["volumeSnapshotContentName"] volumeSnapshotClassName: - description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be used. + A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one VolumeSnapshotClass + exist for a given CSI Driver and more than one have been marked + as default, CreateSnapshot will fail and generate an event. Empty + string is not allowed for this field.' type: string required: - source type: object status: - description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object. properties: boundVolumeSnapshotContentName: - description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent + object to which this VolumeSnapshot object intends to bind to. If + not specified, it indicates that the VolumeSnapshot object has not + been successfully bound to a VolumeSnapshotContent object yet. NOTE: + To avoid possible security issues, consumers must verify binding + between VolumeSnapshot and VolumeSnapshotContent objects is successful + (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object.' type: string creationTime: - description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the snapshot controller + with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may indicate + that the creation time of the snapshot is unknown. format: date-time type: string error: - description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. The snapshot controller will keep retrying when an error + occurs during the snapshot creation. Upon success, this error field + will be cleared. properties: message: - description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' type: string time: description: time is the timestamp when the error was encountered. @@ -8421,11 +8504,27 @@ spec: type: string type: object readyToUse: - description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + description: readyToUse indicates if the snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. type: boolean restoreSize: type: string - description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller with + the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the + "size_bytes" value returned from the CSI "ListSnapshots" gRPC call + if the driver supports it. When restoring a volume from this snapshot, + the size of the volume MUST NOT be smaller than the restoreSize + if it is specified, otherwise the restoration will fail. If not + specified, it indicates that the size is unknown. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -8514,7 +8613,7 @@ spec: format: date-time type: string error: - description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. properties: message: description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' @@ -8536,7 +8635,7 @@ spec: required: - spec type: object - served: true + served: false storage: false subresources: status: {} @@ -8552,8 +8651,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.0 - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -8574,7 +8673,8 @@ spec: - jsonPath: .driver name: Driver type: string - - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. jsonPath: .deletionPolicy name: DeletionPolicy type: string @@ -8584,27 +8684,42 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string deletionPolicy: - description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its bound + VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. + Required. enum: - Delete - Retain type: string driver: - description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string parameters: additionalProperties: type: string - description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. type: object required: - deletionPolicy @@ -8658,7 +8773,7 @@ spec: - deletionPolicy - driver type: object - served: true + served: false storage: false subresources: {} status: @@ -8673,8 +8788,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.0 - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -8697,11 +8812,14 @@ spec: jsonPath: .status.restoreSize name: RestoreSize type: integer - - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. jsonPath: .spec.deletionPolicy name: DeletionPolicy type: string - - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. jsonPath: .spec.driver name: Driver type: string @@ -8709,7 +8827,8 @@ spec: jsonPath: .spec.volumeSnapshotClassName name: VolumeSnapshotClass type: string - - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. jsonPath: .spec.volumeSnapshotRef.name name: VolumeSnapshot type: string @@ -8723,50 +8842,102 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: - description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. properties: deletionPolicy: - description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are deleted. For dynamically provisioned + snapshots, this field will automatically be filled in by the CSI + snapshotter sidecar with the "DeletionPolicy" field defined in the + corresponding VolumeSnapshotClass. For pre-existing snapshots, users + MUST specify this field when creating the VolumeSnapshotContent + object. Required. enum: - Delete - Retain type: string driver: - description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be + the same as the name returned by the CSI GetPluginName() call for + that driver. Required. type: string source: - description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + description: source specifies whether the snapshot is (or should be) + dynamically provisioned or already exists, and just requires a Kubernetes + object representation. This field is immutable after creation. Required. properties: snapshotHandle: - description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + description: snapshotHandle specifies the CSI "snapshot_id" of + a pre-existing snapshot on the underlying storage system for + which a Kubernetes object representation was (or should be) + created. This field is immutable. type: string volumeHandle: - description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. type: string type: object oneOf: - required: ["snapshotHandle"] - required: ["volumeHandle"] + sourceVolumeMode: + description: SourceVolumeMode is the mode of the volume whose snapshot + is taken. Can be either “Filesystem” or “Block”. If not specified, + it indicates the source volume's mode is unknown. This field is + immutable. This field is an alpha field. + type: string volumeSnapshotClassName: - description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and as + such, should not be referenced post-snapshot creation. type: string volumeSnapshotRef: - description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after creation. + Required. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' type: string kind: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' @@ -8778,7 +8949,8 @@ spec: description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' type: string resourceVersion: - description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' type: string uid: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' @@ -8794,14 +8966,27 @@ spec: description: status represents the current information of a snapshot. properties: creationTime: - description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command ` + "`" + `date +%s%N` + "`" + ` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the CSI snapshotter + sidecar with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it indicates + the creation time is unknown. The format of this field is a Unix + nanoseconds time encoded as an int64. On Unix, the command ` + "`" + `date + +%s%N` + "`" + ` returns the current time in nanoseconds since 1970-01-01 + 00:00:00 UTC. format: int64 type: integer error: - description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. properties: message: - description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' type: string time: description: time is the timestamp when the error was encountered. @@ -8809,15 +8994,34 @@ spec: type: string type: object readyToUse: - description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. type: boolean restoreSize: - description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" value + returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. When restoring a volume from this snapshot, the size of the + volume MUST NOT be smaller than the restoreSize if it is specified, + otherwise the restoration will fail. If not specified, it indicates + that the size is unknown. format: int64 minimum: 0 type: integer snapshotHandle: - description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. type: string type: object required: @@ -8964,7 +9168,7 @@ spec: required: - spec type: object - served: true + served: false storage: false subresources: status: {} From 9b25baebb0ff031e355d8c7edd38562fa8d8015c Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Fri, 29 Sep 2023 18:04:04 -0700 Subject: [PATCH 08/16] Fix syntax --- pkg/api/k8s_versions.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/api/k8s_versions.go b/pkg/api/k8s_versions.go index 52f3a42a9..700fcad15 100644 --- a/pkg/api/k8s_versions.go +++ b/pkg/api/k8s_versions.go @@ -346,6 +346,7 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + }, "1.21": { common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", From 7315a924ceae997250ac9561be2e3a31187ce26d Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Tue, 3 Oct 2023 16:31:23 -0700 Subject: [PATCH 09/16] Fix k8s_versions_test --- pkg/api/k8s_versions_test.go | 78 ++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/pkg/api/k8s_versions_test.go b/pkg/api/k8s_versions_test.go index a8ddcc24c..3e738b596 100644 --- a/pkg/api/k8s_versions_test.go +++ b/pkg/api/k8s_versions_test.go @@ -68,16 +68,16 @@ func TestGetK8sVersionComponents(t *testing.T) { common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -157,14 +157,14 @@ func TestGetK8sVersionComponents(t *testing.T) { common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -224,12 +224,12 @@ func TestGetK8sVersionComponents(t *testing.T) { common.SMBFlexVolumeAddonName: smbFlexVolumeImageReference, common.IPMASQAgentAddonName: getDefaultImage(common.IPMASQAgentAddonName, kubernetesImageBaseType), common.AzureNetworkPolicyAddonName: azureNPMContainerImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.CalicoTyphaComponentName: calicoTyphaImageReference, common.CalicoCNIComponentName: calicoCNIImageReference, @@ -307,11 +307,11 @@ func TestGetK8sVersionComponents(t *testing.T) { common.SMBFlexVolumeAddonName: smbFlexVolumeImageReference, common.IPMASQAgentAddonName: getDefaultImage(common.IPMASQAgentAddonName, kubernetesImageBaseType), common.AzureNetworkPolicyAddonName: azureNPMContainerImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.CalicoTyphaComponentName: calicoTyphaImageReference, common.CalicoCNIComponentName: calicoCNIImageReference, @@ -385,11 +385,11 @@ func TestGetK8sVersionComponents(t *testing.T) { common.SMBFlexVolumeAddonName: smbFlexVolumeImageReference, common.IPMASQAgentAddonName: getDefaultImage(common.IPMASQAgentAddonName, kubernetesImageBaseType), common.AzureNetworkPolicyAddonName: azureNPMContainerImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.CalicoTyphaComponentName: calicoTyphaImageReference, common.CalicoCNIComponentName: calicoCNIImageReference, @@ -462,11 +462,11 @@ func TestGetK8sVersionComponents(t *testing.T) { common.SMBFlexVolumeAddonName: smbFlexVolumeImageReference, common.IPMASQAgentAddonName: getDefaultImage(common.IPMASQAgentAddonName, kubernetesImageBaseType), common.AzureNetworkPolicyAddonName: azureNPMContainerImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.CalicoTyphaComponentName: calicoTyphaImageReference, common.CalicoCNIComponentName: calicoCNIImageReference, From 4c5f2e575a0f59883db469799e9db425ef6d5656 Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Tue, 3 Oct 2023 16:59:04 -0700 Subject: [PATCH 10/16] Change addon template based on version --- .../azuredisk-csi-driver-deployment.yaml | 65 ++++++++++++++++++- pkg/engine/templates_generated.go | 65 ++++++++++++++++++- 2 files changed, 126 insertions(+), 4 deletions(-) diff --git a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml index 0bfe3cbbb..54e212685 100644 --- a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml +++ b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml @@ -265,9 +265,11 @@ spec: nodeSelector: kubernetes.io/os: windows priorityClassName: system-node-critical +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault +{{- end}} tolerations: - key: "node.kubernetes.io/os" operator: "Exists" @@ -336,10 +338,15 @@ spec: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" +{{- if not (IsKubernetesVersionGe "1.27.0")}} + - "--metrics-address=0.0.0.0:29605" +{{- end}} - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" +{{- if IsKubernetesVersionGe "1.27.0"}} - "--get-nodeid-from-imds=false" +{{- end}} ports: - containerPort: 29603 name: healthz @@ -461,9 +468,11 @@ spec: nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault +{{- end}} tolerations: - operator: "Exists" containers: @@ -526,7 +535,9 @@ spec: - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" +{{- if IsKubernetesVersionGe "1.27.0"}} - "--get-nodeid-from-imds=false" +{{- end}} ports: - containerPort: 29603 name: healthz @@ -642,9 +653,11 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault +{{- end}} tolerations: - key: "node-role.kubernetes.io/master" operator: "Exists" @@ -665,8 +678,8 @@ spec: - "--worker-threads=40" - "--extra-create-metadata=true" - "--strict-topology=true" - - "--kube-api-qps=200" - - "--kube-api-burst=400" + - "--kube-api-qps=50" + - "--kube-api-burst=100" env: - name: ADDRESS value: /csi/csi.sock @@ -688,9 +701,16 @@ spec: - "-timeout=1200s" - "-leader-election" - "--leader-election-namespace=kube-system" +{{- if IsKubernetesVersionGe "1.27.0"}} + - "-worker-threads=1000" + - "-kube-api-qps=200" + - "-kube-api-burst=400" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} - "-worker-threads=500" - "-kube-api-qps=50" - "-kube-api-burst=100" +{{- end}} env: - name: ADDRESS value: /csi/csi.sock @@ -774,8 +794,10 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=aks-engine" - "--vmss-cache-ttl-seconds=-1" +{{- if IsKubernetesVersionGe "1.27.0"}} - "--enable-traffic-manager=false" - "--traffic-manager-port=7788" +{{- end}} ports: - containerPort: 29602 name: healthz @@ -845,6 +867,7 @@ spec: selector: matchLabels: app: csi-snapshot-controller +{{- if IsKubernetesVersionGe "1.27.0"}} # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable # in #504 the snapshot-controller will exit after around 7.5 seconds if it # can't find the v1 CRDs so this value should be greater than that @@ -854,6 +877,7 @@ spec: maxSurge: 0 maxUnavailable: 1 type: RollingUpdate +{{- end}} template: metadata: labels: @@ -864,9 +888,11 @@ spec: kubernetes.io/os: linux kubernetes.io/role: master priorityClassName: system-cluster-critical +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault +{{- end}} tolerations: - key: "node-role.kubernetes.io/master" operator: "Equal" @@ -978,8 +1004,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -1272,7 +1304,12 @@ spec: required: - spec type: object +{{- if IsKubernetesVersionGe "1.27.0"}} served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + served: true +{{- end}} storage: false subresources: status: {} @@ -1288,8 +1325,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -1410,7 +1453,12 @@ spec: - deletionPolicy - driver type: object +{{- if IsKubernetesVersionGe "1.27.0"}} served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + served: true +{{- end}} storage: false subresources: {} status: @@ -1425,8 +1473,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -1539,12 +1593,14 @@ spec: oneOf: - required: ["snapshotHandle"] - required: ["volumeHandle"] +{{- if IsKubernetesVersionGe "1.27.0"}} sourceVolumeMode: description: SourceVolumeMode is the mode of the volume whose snapshot is taken. Can be either “Filesystem” or “Block”. If not specified, it indicates the source volume's mode is unknown. This field is immutable. This field is an alpha field. type: string +{{- end}} volumeSnapshotClassName: description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass @@ -1805,7 +1861,12 @@ spec: required: - spec type: object +{{- if IsKubernetesVersionGe "1.27.0"}} served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + served: true +{{- end}} storage: false subresources: status: {} diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index d9c9b1e22..a45813d1e 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -7628,9 +7628,11 @@ spec: nodeSelector: kubernetes.io/os: windows priorityClassName: system-node-critical +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault +{{- end}} tolerations: - key: "node.kubernetes.io/os" operator: "Exists" @@ -7699,10 +7701,15 @@ spec: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" +{{- if not (IsKubernetesVersionGe "1.27.0")}} + - "--metrics-address=0.0.0.0:29605" +{{- end}} - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" +{{- if IsKubernetesVersionGe "1.27.0"}} - "--get-nodeid-from-imds=false" +{{- end}} ports: - containerPort: 29603 name: healthz @@ -7824,9 +7831,11 @@ spec: nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault +{{- end}} tolerations: - operator: "Exists" containers: @@ -7889,7 +7898,9 @@ spec: - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" +{{- if IsKubernetesVersionGe "1.27.0"}} - "--get-nodeid-from-imds=false" +{{- end}} ports: - containerPort: 29603 name: healthz @@ -8005,9 +8016,11 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault +{{- end}} tolerations: - key: "node-role.kubernetes.io/master" operator: "Exists" @@ -8028,8 +8041,8 @@ spec: - "--worker-threads=40" - "--extra-create-metadata=true" - "--strict-topology=true" - - "--kube-api-qps=200" - - "--kube-api-burst=400" + - "--kube-api-qps=50" + - "--kube-api-burst=100" env: - name: ADDRESS value: /csi/csi.sock @@ -8051,9 +8064,16 @@ spec: - "-timeout=1200s" - "-leader-election" - "--leader-election-namespace=kube-system" +{{- if IsKubernetesVersionGe "1.27.0"}} + - "-worker-threads=1000" + - "-kube-api-qps=200" + - "-kube-api-burst=400" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} - "-worker-threads=500" - "-kube-api-qps=50" - "-kube-api-burst=100" +{{- end}} env: - name: ADDRESS value: /csi/csi.sock @@ -8137,8 +8157,10 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=aks-engine" - "--vmss-cache-ttl-seconds=-1" +{{- if IsKubernetesVersionGe "1.27.0"}} - "--enable-traffic-manager=false" - "--traffic-manager-port=7788" +{{- end}} ports: - containerPort: 29602 name: healthz @@ -8208,6 +8230,7 @@ spec: selector: matchLabels: app: csi-snapshot-controller +{{- if IsKubernetesVersionGe "1.27.0"}} # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable # in #504 the snapshot-controller will exit after around 7.5 seconds if it # can't find the v1 CRDs so this value should be greater than that @@ -8217,6 +8240,7 @@ spec: maxSurge: 0 maxUnavailable: 1 type: RollingUpdate +{{- end}} template: metadata: labels: @@ -8227,9 +8251,11 @@ spec: kubernetes.io/os: linux kubernetes.io/role: master priorityClassName: system-cluster-critical +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault +{{- end}} tolerations: - key: "node-role.kubernetes.io/master" operator: "Equal" @@ -8341,8 +8367,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -8635,7 +8667,12 @@ spec: required: - spec type: object +{{- if IsKubernetesVersionGe "1.27.0"}} served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + served: true +{{- end}} storage: false subresources: status: {} @@ -8651,8 +8688,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -8773,7 +8816,12 @@ spec: - deletionPolicy - driver type: object +{{- if IsKubernetesVersionGe "1.27.0"}} served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + served: true +{{- end}} storage: false subresources: {} status: @@ -8788,8 +8836,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -8902,12 +8956,14 @@ spec: oneOf: - required: ["snapshotHandle"] - required: ["volumeHandle"] +{{- if IsKubernetesVersionGe "1.27.0"}} sourceVolumeMode: description: SourceVolumeMode is the mode of the volume whose snapshot is taken. Can be either “Filesystem” or “Block”. If not specified, it indicates the source volume's mode is unknown. This field is immutable. This field is an alpha field. type: string +{{- end}} volumeSnapshotClassName: description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass @@ -9168,7 +9224,12 @@ spec: required: - spec type: object +{{- if IsKubernetesVersionGe "1.27.0"}} served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} + served: true +{{- end}} storage: false subresources: status: {} From 459206abae779101d2d098258a65b119c5a0ee6f Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Tue, 3 Oct 2023 17:15:25 -0700 Subject: [PATCH 11/16] Fix lint --- pkg/api/k8s_versions.go | 104 ++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/pkg/api/k8s_versions.go b/pkg/api/k8s_versions.go index 700fcad15..aabf4341a 100644 --- a/pkg/api/k8s_versions.go +++ b/pkg/api/k8s_versions.go @@ -412,66 +412,66 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.0", }, "1.17": { - common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", - common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", - common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", - common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", - common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", - common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", - common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", - common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", }, "1.16": { - common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", - common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", - common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", - common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", - common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", - common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.16.7", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.16.7", }, "1.15": { - common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", - common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", - common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", - common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", - common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.15.7", - common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.15.7", + common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", }, "1.14": { - common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", - common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", - common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", - common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", - common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.0.2", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.14.8", - common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.0.2", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.14.8", + common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", }, "1.13": { - common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", - common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", - common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", - common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", - common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v8.9.1", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.13.9", - common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v8.9.1", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.13.9", + common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", }, "1.12": { common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", From 45da9ebc84cab93766c2429d3f542ee08f6c85fb Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Tue, 3 Oct 2023 21:43:19 -0700 Subject: [PATCH 12/16] Fix GCR for unit tests --- pkg/api/k8s_versions.go | 223 ++++++++++++++++++++++++++++++++-------- 1 file changed, 179 insertions(+), 44 deletions(-) diff --git a/pkg/api/k8s_versions.go b/pkg/api/k8s_versions.go index aabf4341a..7c745ee3a 100644 --- a/pkg/api/k8s_versions.go +++ b/pkg/api/k8s_versions.go @@ -103,73 +103,198 @@ func getDefaultImage(image, kubernetesImageBaseType string) string { // The map supports GCR or MCR image string flavors var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string{ common.KubernetesImageBaseTypeGCR: { + "1.27": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.5.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.10.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.10.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v6.2.2", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v6.2.2", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.8.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.28.3", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.6", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + }, "1.26": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.6", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.3.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.0.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v5.0.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v5.0.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.6.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.26.5", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.6", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.25": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.6", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.3.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.0.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v5.0.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v5.0.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.6.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.26.5", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.6", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.24": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.23": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.22": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.21": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.20": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.19": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.18": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.17": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.17.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.17.1", }, "1.16": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.16.4", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.16.4", }, "1.15": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.AddonResizerComponentName: "addon-resizer:1.8.7", common.MetricsServerAddonName: "metrics-server-amd64:v0.2.1", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", @@ -177,6 +302,11 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string common.CloudControllerManagerComponentName: "cloud-controller-manager-amd64", }, "1.14": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.AddonResizerComponentName: "addon-resizer:1.8.4", common.MetricsServerAddonName: "metrics-server-amd64:v0.2.1", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.0.2", @@ -184,6 +314,11 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string common.CloudControllerManagerComponentName: "cloud-controller-manager-amd64", }, "1.13": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.AddonResizerComponentName: "addon-resizer:1.8.4", common.MetricsServerAddonName: "metrics-server-amd64:v0.2.1", common.AddonManagerComponentName: "kube-addon-manager-amd64:v8.9.1", From 11b8696657429c9392233581cf92097a45e7038e Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Wed, 4 Oct 2023 15:04:26 -0700 Subject: [PATCH 13/16] Update documentation for Windows HostProcess containers --- docs/topics/azure-stack.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/topics/azure-stack.md b/docs/topics/azure-stack.md index d07a2c915..cd60c29c0 100644 --- a/docs/topics/azure-stack.md +++ b/docs/topics/azure-stack.md @@ -185,6 +185,7 @@ The [in-tree volume provisioner](https://kubernetes.io/blog/2019/12/09/kubernete AKS Engine will **not** enable any CSI driver by default on Azure Stack Hub. For workloads that require a CSI driver, it is possible to either: * For AKS Engine versions 0.75.3 and above: explicitly enable the `azuredisk-csi-driver` [addon](../topics/clusterdefinitions.md#addons) (Linux and/or Windows cluster) + * If you are using Windows HostProcess containers in your workloads, do not use the addon. Instead use `Helm` to [install the `azuredisk-csi-driver` chart](#1-install-azure-disk-csi-driver-manually) and set the following value: `--set windows.useHostProcessContainers=true`. * For AKS Engine versions 0.70.0 and above: explicitly enable the `azuredisk-csi-driver` [addon](../topics/clusterdefinitions.md#addons) (Linux cluster only) * For AKS Engine versions 0.70.0 and above: use `Helm` to [install the `azuredisk-csi-driver` chart](#1-install-azure-disk-csi-driver-manually) (Linux and/or Windows clusters). @@ -213,6 +214,7 @@ helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver \ --set node.supportZone=false \ --set windows.getNodeInfoFromLabels=true \ --set linux.getNodeInfoFromLabels=true \ + # --set windows.useHostProcessContainers=true \ # if using Windows HostProcess containers --version ${DRIVER_VERSION} ``` @@ -282,7 +284,15 @@ In this section, please follow the example commands to deploy a StatefulSet appl DRIVER_VERSION=v1.26.5 # if using k8s v1.26 DRIVER_VERSION=v1.28.3 # if using k8s v1.27 helm repo add azuredisk-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts -helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver --namespace kube-system --set cloud=AzureStackCloud --set controller.runOnMaster=true --set node.supportZone=false --set windows.getNodeInfoFromLabels=true --set linux.getNodeInfoFromLabels=true --version ${DRIVER_VERSION} +helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver \ + --namespace kube-system \ + --set cloud=AzureStackCloud \ + --set controller.runOnMaster=true \ + --set node.supportZone=false \ + --set windows.getNodeInfoFromLabels=true \ + --set linux.getNodeInfoFromLabels=true \ + # --set windows.useHostProcessContainers=true \ # if using Windows HostProcess containers + --version ${DRIVER_VERSION} # Deploy Storage Class kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/deploy/example/storageclass-azuredisk-csi-azurestack.yaml From db74df8d5ca0f58bf60774367d9544de98e760a9 Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Wed, 4 Oct 2023 15:09:18 -0700 Subject: [PATCH 14/16] Test PVC by default --- .pipelines/e2e-job-template.yaml | 2 ++ .pipelines/pr-e2e.yaml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml index 6ed62273a..f3208fef2 100644 --- a/.pipelines/e2e-job-template.yaml +++ b/.pipelines/e2e-job-template.yaml @@ -15,6 +15,7 @@ parameters: stabilityIterations: 0 windowsVHDBuild: "" linuxVHDBuild: "" + testPVC: true jobs: - job: ${{ parameters.name }} @@ -55,6 +56,7 @@ jobs: LINUX_NODE_IMAGE_GALLERY: AKSeGallery LINUX_NODE_IMAGE_NAME: ubuntu-20.04 LINUX_NODE_IMAGE_VERSION: ${{ parameters.linuxVHDBuild }} + TEST_PVC: ${{ parameters.testPVC }} steps: - template: e2e-step-template.yaml diff --git a/.pipelines/pr-e2e.yaml b/.pipelines/pr-e2e.yaml index 7dd493386..009559ba1 100644 --- a/.pipelines/pr-e2e.yaml +++ b/.pipelines/pr-e2e.yaml @@ -74,6 +74,7 @@ jobs: msiId: $(MSI_USER_ASSIGNED_ID_AKS_ENGINE_E2E) windowsVHDBuild: $(WINDOWS_VHD_BUILD) linuxVHDBuild: $(LINUX_VHD_BUILD) + testPVC: true - template: e2e-job-template.yaml parameters: @@ -93,6 +94,7 @@ jobs: msiId: $(MSI_USER_ASSIGNED_ID_AKS_ENGINE_E2E) windowsVHDBuild: $(WINDOWS_VHD_BUILD) linuxVHDBuild: $(LINUX_VHD_BUILD) + testPVC: true - template: e2e-job-template.yaml parameters: @@ -112,3 +114,4 @@ jobs: msiId: $(MSI_USER_ASSIGNED_ID_AKS_ENGINE_E2E) windowsVHDBuild: $(WINDOWS_VHD_BUILD) linuxVHDBuild: $(LINUX_VHD_BUILD) + testPVC: true From 3c440320415e8dc4e2c5d1a4753e7e5b8251bd0d Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Wed, 4 Oct 2023 15:59:56 -0700 Subject: [PATCH 15/16] Add IsAzureDiskCSIDriverVersionGe --- .../azuredisk-csi-driver-deployment.yaml | 48 ++++++++-------- pkg/engine/template_generator.go | 24 ++++++++ pkg/engine/template_generator_test.go | 56 +++++++++++++++++++ pkg/engine/templates_generated.go | 48 ++++++++-------- 4 files changed, 128 insertions(+), 48 deletions(-) diff --git a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml index 54e212685..bd14e8dd5 100644 --- a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml +++ b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml @@ -265,7 +265,7 @@ spec: nodeSelector: kubernetes.io/os: windows priorityClassName: system-node-critical -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} securityContext: seccompProfile: type: RuntimeDefault @@ -338,13 +338,13 @@ spec: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} - "--metrics-address=0.0.0.0:29605" {{- end}} - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} - "--get-nodeid-from-imds=false" {{- end}} ports: @@ -468,7 +468,7 @@ spec: nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} securityContext: seccompProfile: type: RuntimeDefault @@ -535,7 +535,7 @@ spec: - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} - "--get-nodeid-from-imds=false" {{- end}} ports: @@ -653,7 +653,7 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} securityContext: seccompProfile: type: RuntimeDefault @@ -701,12 +701,12 @@ spec: - "-timeout=1200s" - "-leader-election" - "--leader-election-namespace=kube-system" -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} - "-worker-threads=1000" - "-kube-api-qps=200" - "-kube-api-burst=400" {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} - "-worker-threads=500" - "-kube-api-qps=50" - "-kube-api-burst=100" @@ -794,7 +794,7 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=aks-engine" - "--vmss-cache-ttl-seconds=-1" -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} - "--enable-traffic-manager=false" - "--traffic-manager-port=7788" {{- end}} @@ -867,7 +867,7 @@ spec: selector: matchLabels: app: csi-snapshot-controller -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable # in #504 the snapshot-controller will exit after around 7.5 seconds if it # can't find the v1 CRDs so this value should be greater than that @@ -888,7 +888,7 @@ spec: kubernetes.io/os: linux kubernetes.io/role: master priorityClassName: system-cluster-critical -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} securityContext: seccompProfile: type: RuntimeDefault @@ -1004,11 +1004,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -1304,10 +1304,10 @@ spec: required: - spec type: object -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} served: false {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} served: true {{- end}} storage: false @@ -1325,11 +1325,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -1453,10 +1453,10 @@ spec: - deletionPolicy - driver type: object -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} served: false {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} served: true {{- end}} storage: false @@ -1473,11 +1473,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -1593,7 +1593,7 @@ spec: oneOf: - required: ["snapshotHandle"] - required: ["volumeHandle"] -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} sourceVolumeMode: description: SourceVolumeMode is the mode of the volume whose snapshot is taken. Can be either “Filesystem” or “Block”. If not specified, @@ -1861,10 +1861,10 @@ spec: required: - spec type: object -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} served: false {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} served: true {{- end}} storage: false diff --git a/pkg/engine/template_generator.go b/pkg/engine/template_generator.go index de41ce145..ad1e6bdd0 100644 --- a/pkg/engine/template_generator.go +++ b/pkg/engine/template_generator.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "regexp" "runtime/debug" "sort" "strings" @@ -247,6 +248,12 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap { "IsKubernetesVersionLt": func(version string) bool { return !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, version) }, + "IsAzureDiskCSIDriverVersionGe": func(version string) bool { + return common.IsKubernetesVersionGe(getAzureDiskCSIDriverVersion(cs), version) + }, + "IsAzureDiskCSIDriverVersionLt": func(version string) bool { + return !common.IsKubernetesVersionGe(getAzureDiskCSIDriverVersion(cs), version) + }, "GetMasterKubernetesLabels": func(rg string) string { return common.GetMasterKubernetesLabels(rg, false) }, @@ -854,6 +861,23 @@ version = 2 } } +func getAzureDiskCSIDriverVersion(cs *api.ContainerService) string { + for _, addon := range cs.Properties.OrchestratorProfile.KubernetesConfig.Addons { + if addon.Name == "azuredisk-csi-driver" && addon.Enabled != nil && to.Bool(addon.Enabled) { + for _, container := range addon.Containers { + if container.Name == "azuredisk-csi" { + re := regexp.MustCompile(`^mcr\.microsoft\.com/oss/kubernetes-csi/azuredisk-csi:v(\d+\.\d+\.\d+)$`) + matches := re.FindStringSubmatch(container.Image) + if len(matches) == 2 { + return matches[1] + } + } + } + } + } + return "" +} + func (t *TemplateGenerator) GenerateTemplateV2(containerService *api.ContainerService, generatorCode string, acsengineVersion string) (templateRaw string, parametersRaw string, err error) { armParams, _ := t.getParameterDescMap(containerService) diff --git a/pkg/engine/template_generator_test.go b/pkg/engine/template_generator_test.go index 5dea3e5b3..35427c484 100644 --- a/pkg/engine/template_generator_test.go +++ b/pkg/engine/template_generator_test.go @@ -54,6 +54,8 @@ func TestGetTemplateFuncMap(t *testing.T) { "IsIPMasqAgentEnabled", "IsKubernetesVersionGe", "IsKubernetesVersionLt", + "IsAzureDiskCSIDriverVersionGe", + "IsAzureDiskCSIDriverVersionLt", "GetMasterKubernetesLabels", "GetAgentKubernetesLabels", "GetKubeletConfigKeyVals", @@ -1940,3 +1942,57 @@ func TestGenerateUserAssignedIdentityClientIDParameterForWindows(t *testing.T) { }) } } + +func TestGetAzureDiskCSIDriverVersion(t *testing.T) { + // Define a test case with a mock AKS Engine API model + testCases := []struct { + name string + containerImage string + expectedResult string + }{ + { + name: "valid version", + containerImage: "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.10.0", + expectedResult: "1.10.0", + }, + { + name: "invalid version", + containerImage: "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:latest", + expectedResult: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create a mock AKS Engine API model with the specified container image + cs := &api.ContainerService{ + Properties: &api.Properties{ + OrchestratorProfile: &api.OrchestratorProfile{ + KubernetesConfig: &api.KubernetesConfig{ + Addons: []api.KubernetesAddon{ + { + Name: "azuredisk-csi-driver", + Enabled: to.BoolPtr(true), + Containers: []api.KubernetesContainerSpec{ + { + Name: "azuredisk-csi", + Image: tc.containerImage, + }, + }, + }, + }, + }, + }, + }, + } + + // Call the getAzureDiskCSIDriverVersion function with the mock AKS Engine API model + actualResult := getAzureDiskCSIDriverVersion(cs) + + // Check that the actual result matches the expected result + if actualResult != tc.expectedResult { + t.Errorf("Expected result %s, but got %s", tc.expectedResult, actualResult) + } + }) + } +} diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index a45813d1e..d8d28e91c 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -7628,7 +7628,7 @@ spec: nodeSelector: kubernetes.io/os: windows priorityClassName: system-node-critical -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} securityContext: seccompProfile: type: RuntimeDefault @@ -7701,13 +7701,13 @@ spec: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} - "--metrics-address=0.0.0.0:29605" {{- end}} - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} - "--get-nodeid-from-imds=false" {{- end}} ports: @@ -7831,7 +7831,7 @@ spec: nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} securityContext: seccompProfile: type: RuntimeDefault @@ -7898,7 +7898,7 @@ spec: - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} - "--get-nodeid-from-imds=false" {{- end}} ports: @@ -8016,7 +8016,7 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} securityContext: seccompProfile: type: RuntimeDefault @@ -8064,12 +8064,12 @@ spec: - "-timeout=1200s" - "-leader-election" - "--leader-election-namespace=kube-system" -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} - "-worker-threads=1000" - "-kube-api-qps=200" - "-kube-api-burst=400" {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} - "-worker-threads=500" - "-kube-api-qps=50" - "-kube-api-burst=100" @@ -8157,7 +8157,7 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=aks-engine" - "--vmss-cache-ttl-seconds=-1" -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} - "--enable-traffic-manager=false" - "--traffic-manager-port=7788" {{- end}} @@ -8230,7 +8230,7 @@ spec: selector: matchLabels: app: csi-snapshot-controller -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable # in #504 the snapshot-controller will exit after around 7.5 seconds if it # can't find the v1 CRDs so this value should be greater than that @@ -8251,7 +8251,7 @@ spec: kubernetes.io/os: linux kubernetes.io/role: master priorityClassName: system-cluster-critical -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} securityContext: seccompProfile: type: RuntimeDefault @@ -8367,11 +8367,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -8667,10 +8667,10 @@ spec: required: - spec type: object -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} served: false {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} served: true {{- end}} storage: false @@ -8688,11 +8688,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -8816,10 +8816,10 @@ spec: - deletionPolicy - driver type: object -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} served: false {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} served: true {{- end}} storage: false @@ -8836,11 +8836,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -8956,7 +8956,7 @@ spec: oneOf: - required: ["snapshotHandle"] - required: ["volumeHandle"] -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} sourceVolumeMode: description: SourceVolumeMode is the mode of the volume whose snapshot is taken. Can be either “Filesystem” or “Block”. If not specified, @@ -9224,10 +9224,10 @@ spec: required: - spec type: object -{{- if IsKubernetesVersionGe "1.27.0"}} +{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} served: false {{- end}} -{{- if not (IsKubernetesVersionGe "1.27.0")}} +{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} served: true {{- end}} storage: false From 35ae08836a04b609ed295c00860c490de82208b5 Mon Sep 17 00:00:00 2001 From: Christina Lau Date: Wed, 4 Oct 2023 17:54:01 -0700 Subject: [PATCH 16/16] Revert "Add IsAzureDiskCSIDriverVersionGe" This reverts commit 3c440320415e8dc4e2c5d1a4753e7e5b8251bd0d. --- .../azuredisk-csi-driver-deployment.yaml | 48 ++++++++-------- pkg/engine/template_generator.go | 24 -------- pkg/engine/template_generator_test.go | 56 ------------------- pkg/engine/templates_generated.go | 48 ++++++++-------- 4 files changed, 48 insertions(+), 128 deletions(-) diff --git a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml index bd14e8dd5..54e212685 100644 --- a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml +++ b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml @@ -265,7 +265,7 @@ spec: nodeSelector: kubernetes.io/os: windows priorityClassName: system-node-critical -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault @@ -338,13 +338,13 @@ spec: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} - "--metrics-address=0.0.0.0:29605" {{- end}} - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} - "--get-nodeid-from-imds=false" {{- end}} ports: @@ -468,7 +468,7 @@ spec: nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault @@ -535,7 +535,7 @@ spec: - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} - "--get-nodeid-from-imds=false" {{- end}} ports: @@ -653,7 +653,7 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault @@ -701,12 +701,12 @@ spec: - "-timeout=1200s" - "-leader-election" - "--leader-election-namespace=kube-system" -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} - "-worker-threads=1000" - "-kube-api-qps=200" - "-kube-api-burst=400" {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} - "-worker-threads=500" - "-kube-api-qps=50" - "-kube-api-burst=100" @@ -794,7 +794,7 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=aks-engine" - "--vmss-cache-ttl-seconds=-1" -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} - "--enable-traffic-manager=false" - "--traffic-manager-port=7788" {{- end}} @@ -867,7 +867,7 @@ spec: selector: matchLabels: app: csi-snapshot-controller -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable # in #504 the snapshot-controller will exit after around 7.5 seconds if it # can't find the v1 CRDs so this value should be greater than that @@ -888,7 +888,7 @@ spec: kubernetes.io/os: linux kubernetes.io/role: master priorityClassName: system-cluster-critical -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault @@ -1004,11 +1004,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -1304,10 +1304,10 @@ spec: required: - spec type: object -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} served: false {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true {{- end}} storage: false @@ -1325,11 +1325,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -1453,10 +1453,10 @@ spec: - deletionPolicy - driver type: object -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} served: false {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true {{- end}} storage: false @@ -1473,11 +1473,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -1593,7 +1593,7 @@ spec: oneOf: - required: ["snapshotHandle"] - required: ["volumeHandle"] -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} sourceVolumeMode: description: SourceVolumeMode is the mode of the volume whose snapshot is taken. Can be either “Filesystem” or “Block”. If not specified, @@ -1861,10 +1861,10 @@ spec: required: - spec type: object -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} served: false {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true {{- end}} storage: false diff --git a/pkg/engine/template_generator.go b/pkg/engine/template_generator.go index ad1e6bdd0..de41ce145 100644 --- a/pkg/engine/template_generator.go +++ b/pkg/engine/template_generator.go @@ -9,7 +9,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "regexp" "runtime/debug" "sort" "strings" @@ -248,12 +247,6 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap { "IsKubernetesVersionLt": func(version string) bool { return !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, version) }, - "IsAzureDiskCSIDriverVersionGe": func(version string) bool { - return common.IsKubernetesVersionGe(getAzureDiskCSIDriverVersion(cs), version) - }, - "IsAzureDiskCSIDriverVersionLt": func(version string) bool { - return !common.IsKubernetesVersionGe(getAzureDiskCSIDriverVersion(cs), version) - }, "GetMasterKubernetesLabels": func(rg string) string { return common.GetMasterKubernetesLabels(rg, false) }, @@ -861,23 +854,6 @@ version = 2 } } -func getAzureDiskCSIDriverVersion(cs *api.ContainerService) string { - for _, addon := range cs.Properties.OrchestratorProfile.KubernetesConfig.Addons { - if addon.Name == "azuredisk-csi-driver" && addon.Enabled != nil && to.Bool(addon.Enabled) { - for _, container := range addon.Containers { - if container.Name == "azuredisk-csi" { - re := regexp.MustCompile(`^mcr\.microsoft\.com/oss/kubernetes-csi/azuredisk-csi:v(\d+\.\d+\.\d+)$`) - matches := re.FindStringSubmatch(container.Image) - if len(matches) == 2 { - return matches[1] - } - } - } - } - } - return "" -} - func (t *TemplateGenerator) GenerateTemplateV2(containerService *api.ContainerService, generatorCode string, acsengineVersion string) (templateRaw string, parametersRaw string, err error) { armParams, _ := t.getParameterDescMap(containerService) diff --git a/pkg/engine/template_generator_test.go b/pkg/engine/template_generator_test.go index 35427c484..5dea3e5b3 100644 --- a/pkg/engine/template_generator_test.go +++ b/pkg/engine/template_generator_test.go @@ -54,8 +54,6 @@ func TestGetTemplateFuncMap(t *testing.T) { "IsIPMasqAgentEnabled", "IsKubernetesVersionGe", "IsKubernetesVersionLt", - "IsAzureDiskCSIDriverVersionGe", - "IsAzureDiskCSIDriverVersionLt", "GetMasterKubernetesLabels", "GetAgentKubernetesLabels", "GetKubeletConfigKeyVals", @@ -1942,57 +1940,3 @@ func TestGenerateUserAssignedIdentityClientIDParameterForWindows(t *testing.T) { }) } } - -func TestGetAzureDiskCSIDriverVersion(t *testing.T) { - // Define a test case with a mock AKS Engine API model - testCases := []struct { - name string - containerImage string - expectedResult string - }{ - { - name: "valid version", - containerImage: "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.10.0", - expectedResult: "1.10.0", - }, - { - name: "invalid version", - containerImage: "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:latest", - expectedResult: "", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Create a mock AKS Engine API model with the specified container image - cs := &api.ContainerService{ - Properties: &api.Properties{ - OrchestratorProfile: &api.OrchestratorProfile{ - KubernetesConfig: &api.KubernetesConfig{ - Addons: []api.KubernetesAddon{ - { - Name: "azuredisk-csi-driver", - Enabled: to.BoolPtr(true), - Containers: []api.KubernetesContainerSpec{ - { - Name: "azuredisk-csi", - Image: tc.containerImage, - }, - }, - }, - }, - }, - }, - }, - } - - // Call the getAzureDiskCSIDriverVersion function with the mock AKS Engine API model - actualResult := getAzureDiskCSIDriverVersion(cs) - - // Check that the actual result matches the expected result - if actualResult != tc.expectedResult { - t.Errorf("Expected result %s, but got %s", tc.expectedResult, actualResult) - } - }) - } -} diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index d8d28e91c..a45813d1e 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -7628,7 +7628,7 @@ spec: nodeSelector: kubernetes.io/os: windows priorityClassName: system-node-critical -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault @@ -7701,13 +7701,13 @@ spec: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} - "--metrics-address=0.0.0.0:29605" {{- end}} - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} - "--get-nodeid-from-imds=false" {{- end}} ports: @@ -7831,7 +7831,7 @@ spec: nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault @@ -7898,7 +7898,7 @@ spec: - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} - "--get-nodeid-from-imds=false" {{- end}} ports: @@ -8016,7 +8016,7 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault @@ -8064,12 +8064,12 @@ spec: - "-timeout=1200s" - "-leader-election" - "--leader-election-namespace=kube-system" -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} - "-worker-threads=1000" - "-kube-api-qps=200" - "-kube-api-burst=400" {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} - "-worker-threads=500" - "-kube-api-qps=50" - "-kube-api-burst=100" @@ -8157,7 +8157,7 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=aks-engine" - "--vmss-cache-ttl-seconds=-1" -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} - "--enable-traffic-manager=false" - "--traffic-manager-port=7788" {{- end}} @@ -8230,7 +8230,7 @@ spec: selector: matchLabels: app: csi-snapshot-controller -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable # in #504 the snapshot-controller will exit after around 7.5 seconds if it # can't find the v1 CRDs so this value should be greater than that @@ -8251,7 +8251,7 @@ spec: kubernetes.io/os: linux kubernetes.io/role: master priorityClassName: system-cluster-critical -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} securityContext: seccompProfile: type: RuntimeDefault @@ -8367,11 +8367,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -8667,10 +8667,10 @@ spec: required: - spec type: object -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} served: false {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true {{- end}} storage: false @@ -8688,11 +8688,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -8816,10 +8816,10 @@ spec: - deletionPolicy - driver type: object -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} served: false {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true {{- end}} storage: false @@ -8836,11 +8836,11 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} controller-gen.kubebuilder.io/version: v0.8.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" {{- end}} @@ -8956,7 +8956,7 @@ spec: oneOf: - required: ["snapshotHandle"] - required: ["volumeHandle"] -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} sourceVolumeMode: description: SourceVolumeMode is the mode of the volume whose snapshot is taken. Can be either “Filesystem” or “Block”. If not specified, @@ -9224,10 +9224,10 @@ spec: required: - spec type: object -{{- if IsAzureDiskCSIDriverVersionGe "1.28.3"}} +{{- if IsKubernetesVersionGe "1.27.0"}} served: false {{- end}} -{{- if not (IsAzureDiskCSIDriverVersionGe "1.28.3")}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true {{- end}} storage: false