diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml index 6ed62273a..f3208fef2 100644 --- a/.pipelines/e2e-job-template.yaml +++ b/.pipelines/e2e-job-template.yaml @@ -15,6 +15,7 @@ parameters: stabilityIterations: 0 windowsVHDBuild: "" linuxVHDBuild: "" + testPVC: true jobs: - job: ${{ parameters.name }} @@ -55,6 +56,7 @@ jobs: LINUX_NODE_IMAGE_GALLERY: AKSeGallery LINUX_NODE_IMAGE_NAME: ubuntu-20.04 LINUX_NODE_IMAGE_VERSION: ${{ parameters.linuxVHDBuild }} + TEST_PVC: ${{ parameters.testPVC }} steps: - template: e2e-step-template.yaml diff --git a/.pipelines/pr-e2e.yaml b/.pipelines/pr-e2e.yaml index 7dd493386..009559ba1 100644 --- a/.pipelines/pr-e2e.yaml +++ b/.pipelines/pr-e2e.yaml @@ -74,6 +74,7 @@ jobs: msiId: $(MSI_USER_ASSIGNED_ID_AKS_ENGINE_E2E) windowsVHDBuild: $(WINDOWS_VHD_BUILD) linuxVHDBuild: $(LINUX_VHD_BUILD) + testPVC: true - template: e2e-job-template.yaml parameters: @@ -93,6 +94,7 @@ jobs: msiId: $(MSI_USER_ASSIGNED_ID_AKS_ENGINE_E2E) windowsVHDBuild: $(WINDOWS_VHD_BUILD) linuxVHDBuild: $(LINUX_VHD_BUILD) + testPVC: true - template: e2e-job-template.yaml parameters: @@ -112,3 +114,4 @@ jobs: msiId: $(MSI_USER_ASSIGNED_ID_AKS_ENGINE_E2E) windowsVHDBuild: $(WINDOWS_VHD_BUILD) linuxVHDBuild: $(LINUX_VHD_BUILD) + testPVC: true diff --git a/docs/topics/azure-stack.md b/docs/topics/azure-stack.md index d49aa58e6..cd60c29c0 100644 --- a/docs/topics/azure-stack.md +++ b/docs/topics/azure-stack.md @@ -185,6 +185,7 @@ The [in-tree volume provisioner](https://kubernetes.io/blog/2019/12/09/kubernete AKS Engine will **not** enable any CSI driver by default on Azure Stack Hub. For workloads that require a CSI driver, it is possible to either: * For AKS Engine versions 0.75.3 and above: explicitly enable the `azuredisk-csi-driver` [addon](../topics/clusterdefinitions.md#addons) (Linux and/or Windows cluster) + * If you are using Windows HostProcess containers in your workloads, do not use the addon. Instead use `Helm` to [install the `azuredisk-csi-driver` chart](#1-install-azure-disk-csi-driver-manually) and set the following value: `--set windows.useHostProcessContainers=true`. * For AKS Engine versions 0.70.0 and above: explicitly enable the `azuredisk-csi-driver` [addon](../topics/clusterdefinitions.md#addons) (Linux cluster only) * For AKS Engine versions 0.70.0 and above: use `Helm` to [install the `azuredisk-csi-driver` chart](#1-install-azure-disk-csi-driver-manually) (Linux and/or Windows clusters). @@ -203,7 +204,8 @@ If the data persisted in the underlying Azure disks should be preserved, then th The following script uses `Helm` to install the Azure Disk CSI Driver: ```bash -DRIVER_VERSION=v1.26.5 +DRIVER_VERSION=v1.26.5 # if using k8s v1.26 +DRIVER_VERSION=v1.28.3 # if using k8s v1.27 helm repo add azuredisk-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver \ --namespace kube-system \ @@ -212,6 +214,7 @@ helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver \ --set node.supportZone=false \ --set windows.getNodeInfoFromLabels=true \ --set linux.getNodeInfoFromLabels=true \ + # --set windows.useHostProcessContainers=true \ # if using Windows HostProcess containers --version ${DRIVER_VERSION} ``` @@ -278,9 +281,18 @@ In this section, please follow the example commands to deploy a StatefulSet appl ```bash # Install CSI Driver -DRIVER_VERSION=v1.26.5 +DRIVER_VERSION=v1.26.5 # if using k8s v1.26 +DRIVER_VERSION=v1.28.3 # if using k8s v1.27 helm repo add azuredisk-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts -helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver --namespace kube-system --set cloud=AzureStackCloud --set controller.runOnMaster=true --set node.supportZone=false --set windows.getNodeInfoFromLabels=true --set linux.getNodeInfoFromLabels=true --version ${DRIVER_VERSION} +helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver \ + --namespace kube-system \ + --set cloud=AzureStackCloud \ + --set controller.runOnMaster=true \ + --set node.supportZone=false \ + --set windows.getNodeInfoFromLabels=true \ + --set linux.getNodeInfoFromLabels=true \ + # --set windows.useHostProcessContainers=true \ # if using Windows HostProcess containers + --version ${DRIVER_VERSION} # Deploy Storage Class kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/deploy/example/storageclass-azuredisk-csi-azurestack.yaml diff --git a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml index 68024e7a4..54e212685 100644 --- a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml +++ b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml @@ -265,6 +265,11 @@ spec: nodeSelector: kubernetes.io/os: windows priorityClassName: system-node-critical +{{- if IsKubernetesVersionGe "1.27.0"}} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- end}} tolerations: - key: "node.kubernetes.io/os" operator: "Exists" @@ -333,10 +338,15 @@ spec: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" +{{- if not (IsKubernetesVersionGe "1.27.0")}} - "--metrics-address=0.0.0.0:29605" +{{- end}} - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" +{{- if IsKubernetesVersionGe "1.27.0"}} + - "--get-nodeid-from-imds=false" +{{- end}} ports: - containerPort: 29603 name: healthz @@ -458,6 +468,11 @@ spec: nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical +{{- if IsKubernetesVersionGe "1.27.0"}} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- end}} tolerations: - operator: "Exists" containers: @@ -520,6 +535,9 @@ spec: - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" +{{- if IsKubernetesVersionGe "1.27.0"}} + - "--get-nodeid-from-imds=false" +{{- end}} ports: - containerPort: 29603 name: healthz @@ -635,6 +653,11 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical +{{- if IsKubernetesVersionGe "1.27.0"}} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- end}} tolerations: - key: "node-role.kubernetes.io/master" operator: "Exists" @@ -678,9 +701,16 @@ spec: - "-timeout=1200s" - "-leader-election" - "--leader-election-namespace=kube-system" +{{- if IsKubernetesVersionGe "1.27.0"}} + - "-worker-threads=1000" + - "-kube-api-qps=200" + - "-kube-api-burst=400" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} - "-worker-threads=500" - "-kube-api-qps=50" - "-kube-api-burst=100" +{{- end}} env: - name: ADDRESS value: /csi/csi.sock @@ -764,6 +794,10 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=aks-engine" - "--vmss-cache-ttl-seconds=-1" +{{- if IsKubernetesVersionGe "1.27.0"}} + - "--enable-traffic-manager=false" + - "--traffic-manager-port=7788" +{{- end}} ports: - containerPort: 29602 name: healthz @@ -833,6 +867,17 @@ spec: selector: matchLabels: app: csi-snapshot-controller +{{- if IsKubernetesVersionGe "1.27.0"}} + # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable + # in #504 the snapshot-controller will exit after around 7.5 seconds if it + # can't find the v1 CRDs so this value should be greater than that + minReadySeconds: 15 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate +{{- end}} template: metadata: labels: @@ -843,6 +888,11 @@ spec: kubernetes.io/os: linux kubernetes.io/role: master priorityClassName: system-cluster-critical +{{- if IsKubernetesVersionGe "1.27.0"}} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- end}} tolerations: - key: "node-role.kubernetes.io/master" operator: "Equal" @@ -954,8 +1004,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -976,15 +1032,18 @@ spec: jsonPath: .status.readyToUse name: ReadyToUse type: boolean - - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + - description: If a new snapshot needs to be created, this contains the name of + the source PVC from which this snapshot was (or will be) created. jsonPath: .spec.source.persistentVolumeClaimName name: SourcePVC type: string - - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + - description: If a snapshot already exists, this contains the name of the existing + VolumeSnapshotContent object representing the existing snapshot. jsonPath: .spec.source.volumeSnapshotContentName name: SourceSnapshotContent type: string - - description: Represents the minimum size of volume required to rehydrate from this snapshot. + - description: Represents the minimum size of volume required to rehydrate from + this snapshot. jsonPath: .status.restoreSize name: RestoreSize type: string @@ -992,11 +1051,16 @@ spec: jsonPath: .spec.volumeSnapshotClassName name: SnapshotClass type: string - - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage of + this object. jsonPath: .status.boundVolumeSnapshotContentName name: SnapshotContent type: string - - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + - description: Timestamp when the point-in-time snapshot was taken by the underlying + storage system. jsonPath: .status.creationTime name: CreationTime type: date @@ -1006,51 +1070,102 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: - description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. properties: source: - description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. properties: persistentVolumeClaimName: - description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object representing the volume from which + a snapshot should be created. This PVC is assumed to be in the + same namespace as the VolumeSnapshot object. This field should + be set if the snapshot does not exists, and needs to be created. + This field is immutable. type: string volumeSnapshotContentName: - description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + description: volumeSnapshotContentName specifies the name of a + pre-existing VolumeSnapshotContent object representing an existing + volume snapshot. This field should be set if the snapshot already + exists and only needs a representation in Kubernetes. This field + is immutable. type: string type: object oneOf: - required: ["persistentVolumeClaimName"] - required: ["volumeSnapshotContentName"] volumeSnapshotClassName: - description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be used. + A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one VolumeSnapshotClass + exist for a given CSI Driver and more than one have been marked + as default, CreateSnapshot will fail and generate an event. Empty + string is not allowed for this field.' type: string required: - source type: object status: - description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object. properties: boundVolumeSnapshotContentName: - description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent + object to which this VolumeSnapshot object intends to bind to. If + not specified, it indicates that the VolumeSnapshot object has not + been successfully bound to a VolumeSnapshotContent object yet. NOTE: + To avoid possible security issues, consumers must verify binding + between VolumeSnapshot and VolumeSnapshotContent objects is successful + (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object.' type: string creationTime: - description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the snapshot controller + with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may indicate + that the creation time of the snapshot is unknown. format: date-time type: string error: - description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. The snapshot controller will keep retrying when an error + occurs during the snapshot creation. Upon success, this error field + will be cleared. properties: message: - description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' type: string time: description: time is the timestamp when the error was encountered. @@ -1058,11 +1173,27 @@ spec: type: string type: object readyToUse: - description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + description: readyToUse indicates if the snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. type: boolean restoreSize: type: string - description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller with + the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the + "size_bytes" value returned from the CSI "ListSnapshots" gRPC call + if the driver supports it. When restoring a volume from this snapshot, + the size of the volume MUST NOT be smaller than the restoreSize + if it is specified, otherwise the restoration will fail. If not + specified, it indicates that the size is unknown. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -1151,7 +1282,7 @@ spec: format: date-time type: string error: - description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. properties: message: description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' @@ -1173,7 +1304,12 @@ spec: required: - spec type: object +{{- if IsKubernetesVersionGe "1.27.0"}} + served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true +{{- end}} storage: false subresources: status: {} @@ -1189,8 +1325,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -1211,7 +1353,8 @@ spec: - jsonPath: .driver name: Driver type: string - - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. jsonPath: .deletionPolicy name: DeletionPolicy type: string @@ -1221,27 +1364,42 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string deletionPolicy: - description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its bound + VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. + Required. enum: - Delete - Retain type: string driver: - description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string parameters: additionalProperties: type: string - description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. type: object required: - deletionPolicy @@ -1295,7 +1453,12 @@ spec: - deletionPolicy - driver type: object +{{- if IsKubernetesVersionGe "1.27.0"}} + served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true +{{- end}} storage: false subresources: {} status: @@ -1310,8 +1473,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -1334,11 +1503,14 @@ spec: jsonPath: .status.restoreSize name: RestoreSize type: integer - - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. jsonPath: .spec.deletionPolicy name: DeletionPolicy type: string - - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. jsonPath: .spec.driver name: Driver type: string @@ -1346,7 +1518,8 @@ spec: jsonPath: .spec.volumeSnapshotClassName name: VolumeSnapshotClass type: string - - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. jsonPath: .spec.volumeSnapshotRef.name name: VolumeSnapshot type: string @@ -1360,50 +1533,104 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: - description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. properties: deletionPolicy: - description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are deleted. For dynamically provisioned + snapshots, this field will automatically be filled in by the CSI + snapshotter sidecar with the "DeletionPolicy" field defined in the + corresponding VolumeSnapshotClass. For pre-existing snapshots, users + MUST specify this field when creating the VolumeSnapshotContent + object. Required. enum: - Delete - Retain type: string driver: - description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be + the same as the name returned by the CSI GetPluginName() call for + that driver. Required. type: string source: - description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + description: source specifies whether the snapshot is (or should be) + dynamically provisioned or already exists, and just requires a Kubernetes + object representation. This field is immutable after creation. Required. properties: snapshotHandle: - description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + description: snapshotHandle specifies the CSI "snapshot_id" of + a pre-existing snapshot on the underlying storage system for + which a Kubernetes object representation was (or should be) + created. This field is immutable. type: string volumeHandle: - description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. type: string type: object oneOf: - required: ["snapshotHandle"] - required: ["volumeHandle"] +{{- if IsKubernetesVersionGe "1.27.0"}} + sourceVolumeMode: + description: SourceVolumeMode is the mode of the volume whose snapshot + is taken. Can be either “Filesystem” or “Block”. If not specified, + it indicates the source volume's mode is unknown. This field is + immutable. This field is an alpha field. + type: string +{{- end}} volumeSnapshotClassName: - description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and as + such, should not be referenced post-snapshot creation. type: string volumeSnapshotRef: - description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after creation. + Required. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' type: string kind: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' @@ -1415,7 +1642,8 @@ spec: description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' type: string resourceVersion: - description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' type: string uid: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' @@ -1431,14 +1659,27 @@ spec: description: status represents the current information of a snapshot. properties: creationTime: - description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the CSI snapshotter + sidecar with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it indicates + the creation time is unknown. The format of this field is a Unix + nanoseconds time encoded as an int64. On Unix, the command `date + +%s%N` returns the current time in nanoseconds since 1970-01-01 + 00:00:00 UTC. format: int64 type: integer error: - description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. properties: message: - description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' type: string time: description: time is the timestamp when the error was encountered. @@ -1446,15 +1687,34 @@ spec: type: string type: object readyToUse: - description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. type: boolean restoreSize: - description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" value + returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. When restoring a volume from this snapshot, the size of the + volume MUST NOT be smaller than the restoreSize if it is specified, + otherwise the restoration will fail. If not specified, it indicates + that the size is unknown. format: int64 minimum: 0 type: integer snapshotHandle: - description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. type: string type: object required: @@ -1601,7 +1861,12 @@ spec: required: - spec type: object +{{- if IsKubernetesVersionGe "1.27.0"}} + served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true +{{- end}} storage: false subresources: status: {} diff --git a/pkg/api/k8s_versions.go b/pkg/api/k8s_versions.go index 31bd1609f..7c745ee3a 100644 --- a/pkg/api/k8s_versions.go +++ b/pkg/api/k8s_versions.go @@ -33,16 +33,6 @@ const ( azurePolicyImageReference string = "mcr.microsoft.com/azure-policy/policy-kubernetes-addon-prod:prod_20201023.1" gatekeeperImageReference string = "mcr.microsoft.com/oss/open-policy-agent/gatekeeper:v3.2.3" nodeProblemDetectorImageReference string = "registry.k8s.io/node-problem-detector/node-problem-detector:v0.8.4" - csiProvisionerImageReference string = "oss/kubernetes-csi/csi-provisioner:v3.3.0" - csiAttacherImageReference string = "oss/kubernetes-csi/csi-attacher:v4.0.0" - csiLivenessProbeImageReference string = "oss/kubernetes-csi/livenessprobe:v2.8.0" - csiLivenessProbeWindowsImageReference string = "oss/kubernetes-csi/livenessprobe:v2.8.0" - csiNodeDriverRegistrarImageReference string = "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2" - csiNodeDriverRegistrarWindowsImageReference string = "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2" - csiResizerImageReference string = "oss/kubernetes-csi/csi-resizer:v1.6.0" - csiSnapshotterImageReference string = "oss/kubernetes-csi/csi-snapshotter:v5.0.1" - csiSnapshotControllerImageReference string = "oss/kubernetes-csi/snapshot-controller:v5.0.1" - csiAzureDiskImageReference string = "oss/kubernetes-csi/azuredisk-csi:v1.26.5" csiAzureFileImageReference string = "oss/kubernetes-csi/azurefile-csi:v1.9.0" azureCloudControllerManagerImageReference string = "oss/kubernetes/azure-cloud-controller-manager:v1.1.1" azureCloudNodeManagerImageReference string = "oss/kubernetes/azure-cloud-node-manager:v1.1.1" @@ -113,73 +103,198 @@ func getDefaultImage(image, kubernetesImageBaseType string) string { // The map supports GCR or MCR image string flavors var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string{ common.KubernetesImageBaseTypeGCR: { + "1.27": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.5.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.10.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.10.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v6.2.2", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v6.2.2", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.8.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.28.3", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.6", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + }, "1.26": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.6", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.3.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.0.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v5.0.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v5.0.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.6.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.26.5", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.6", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.25": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.6", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.3.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.0.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v5.0.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v5.0.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.6.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.26.5", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.6", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.24": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.23": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.22": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.21": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.20": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.19": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.18": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.17": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.17.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.17.1", }, "1.16": { - common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", - common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", - common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.16.4", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "addon-resizer:1.8.7", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.5.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.16.4", }, "1.15": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.AddonResizerComponentName: "addon-resizer:1.8.7", common.MetricsServerAddonName: "metrics-server-amd64:v0.2.1", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", @@ -187,6 +302,11 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string common.CloudControllerManagerComponentName: "cloud-controller-manager-amd64", }, "1.14": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.AddonResizerComponentName: "addon-resizer:1.8.4", common.MetricsServerAddonName: "metrics-server-amd64:v0.2.1", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.0.2", @@ -194,6 +314,11 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string common.CloudControllerManagerComponentName: "cloud-controller-manager-amd64", }, "1.13": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.AddonResizerComponentName: "addon-resizer:1.8.4", common.MetricsServerAddonName: "metrics-server-amd64:v0.2.1", common.AddonManagerComponentName: "kube-addon-manager-amd64:v8.9.1", @@ -261,73 +386,198 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string }, }, common.KubernetesImageBaseTypeMCR: { + "1.27": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.5.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.10.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.10.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v6.2.2", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v6.2.2", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.8.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.28.3", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.6", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + }, "1.26": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.6", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.3.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.0.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v5.0.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v5.0.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.6.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.26.5", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.6", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.25": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.6", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.3.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.0.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.8.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v5.0.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v5.0.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.6.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.26.5", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.6", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.24": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.23": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.22": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.21": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.20": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.19": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.2", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.1", }, "1.18": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.22.0", }, "1.17": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", }, "1.16": { - common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", - common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", - common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.16.7", + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.5.0", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.16.7", }, "1.15": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", @@ -335,6 +585,11 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", }, "1.14": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.0.2", @@ -342,6 +597,11 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string common.CloudControllerManagerComponentName: "oss/kubernetes/cloud-controller-manager", }, "1.13": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.4", common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.2.1", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v8.9.1", @@ -573,16 +833,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -656,16 +916,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -739,16 +999,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -822,16 +1082,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -905,16 +1165,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -988,16 +1248,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1071,16 +1331,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1154,16 +1414,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1237,16 +1497,16 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSILivenessProbeWindowsContainerName: k8sComponent[common.CSILivenessProbeWindowsContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSINodeDriverRegistrarWindowsContainerName: k8sComponent[common.CSINodeDriverRegistrarWindowsContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1320,14 +1580,14 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSISnapshotterContainerName: k8sComponent[common.CSISnapshotterContainerName], + common.CSISnapshotControllerContainerName: k8sComponent[common.CSISnapshotControllerContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1399,12 +1659,12 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSIResizerContainerName: k8sComponent[common.CSIResizerContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1475,11 +1735,11 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1548,11 +1808,11 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -1619,11 +1879,11 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: k8sComponent[common.CSIProvisionerContainerName], + common.CSIAttacherContainerName: k8sComponent[common.CSIAttacherContainerName], + common.CSILivenessProbeContainerName: k8sComponent[common.CSILivenessProbeContainerName], + common.CSINodeDriverRegistrarContainerName: k8sComponent[common.CSINodeDriverRegistrarContainerName], + common.CSIAzureDiskContainerName: k8sComponent[common.CSIAzureDiskContainerName], common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, diff --git a/pkg/api/k8s_versions_test.go b/pkg/api/k8s_versions_test.go index a8ddcc24c..3e738b596 100644 --- a/pkg/api/k8s_versions_test.go +++ b/pkg/api/k8s_versions_test.go @@ -68,16 +68,16 @@ func TestGetK8sVersionComponents(t *testing.T) { common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSILivenessProbeWindowsContainerName: csiLivenessProbeWindowsImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSINodeDriverRegistrarWindowsContainerName: csiNodeDriverRegistrarWindowsImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -157,14 +157,14 @@ func TestGetK8sVersionComponents(t *testing.T) { common.AzurePolicyAddonName: azurePolicyImageReference, common.GatekeeperContainerName: gatekeeperImageReference, common.NodeProblemDetectorAddonName: nodeProblemDetectorImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSISnapshotterContainerName: csiSnapshotterImageReference, - common.CSISnapshotControllerContainerName: csiSnapshotControllerImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v4.2.1", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v4.2.1", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.KubeFlannelContainerName: kubeFlannelImageReference, "flannel" + common.FlannelInstallCNIContainerName: flannelInstallCNIImageReference, @@ -224,12 +224,12 @@ func TestGetK8sVersionComponents(t *testing.T) { common.SMBFlexVolumeAddonName: smbFlexVolumeImageReference, common.IPMASQAgentAddonName: getDefaultImage(common.IPMASQAgentAddonName, kubernetesImageBaseType), common.AzureNetworkPolicyAddonName: azureNPMContainerImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIResizerContainerName: csiResizerImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.3.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.CalicoTyphaComponentName: calicoTyphaImageReference, common.CalicoCNIComponentName: calicoCNIImageReference, @@ -307,11 +307,11 @@ func TestGetK8sVersionComponents(t *testing.T) { common.SMBFlexVolumeAddonName: smbFlexVolumeImageReference, common.IPMASQAgentAddonName: getDefaultImage(common.IPMASQAgentAddonName, kubernetesImageBaseType), common.AzureNetworkPolicyAddonName: azureNPMContainerImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.CalicoTyphaComponentName: calicoTyphaImageReference, common.CalicoCNIComponentName: calicoCNIImageReference, @@ -385,11 +385,11 @@ func TestGetK8sVersionComponents(t *testing.T) { common.SMBFlexVolumeAddonName: smbFlexVolumeImageReference, common.IPMASQAgentAddonName: getDefaultImage(common.IPMASQAgentAddonName, kubernetesImageBaseType), common.AzureNetworkPolicyAddonName: azureNPMContainerImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.CalicoTyphaComponentName: calicoTyphaImageReference, common.CalicoCNIComponentName: calicoCNIImageReference, @@ -462,11 +462,11 @@ func TestGetK8sVersionComponents(t *testing.T) { common.SMBFlexVolumeAddonName: smbFlexVolumeImageReference, common.IPMASQAgentAddonName: getDefaultImage(common.IPMASQAgentAddonName, kubernetesImageBaseType), common.AzureNetworkPolicyAddonName: azureNPMContainerImageReference, - common.CSIProvisionerContainerName: csiProvisionerImageReference, - common.CSIAttacherContainerName: csiAttacherImageReference, - common.CSILivenessProbeContainerName: csiLivenessProbeImageReference, - common.CSINodeDriverRegistrarContainerName: csiNodeDriverRegistrarImageReference, - common.CSIAzureDiskContainerName: csiAzureDiskImageReference, + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v3.0.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v3.3.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.5.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.10.0", common.CSIAzureFileContainerName: csiAzureFileImageReference, common.CalicoTyphaComponentName: calicoTyphaImageReference, common.CalicoCNIComponentName: calicoCNIImageReference, diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index ea36d6b87..a45813d1e 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -7628,6 +7628,11 @@ spec: nodeSelector: kubernetes.io/os: windows priorityClassName: system-node-critical +{{- if IsKubernetesVersionGe "1.27.0"}} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- end}} tolerations: - key: "node.kubernetes.io/os" operator: "Exists" @@ -7696,10 +7701,15 @@ spec: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(KUBE_NODE_NAME)" +{{- if not (IsKubernetesVersionGe "1.27.0")}} - "--metrics-address=0.0.0.0:29605" +{{- end}} - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" +{{- if IsKubernetesVersionGe "1.27.0"}} + - "--get-nodeid-from-imds=false" +{{- end}} ports: - containerPort: 29603 name: healthz @@ -7821,6 +7831,11 @@ spec: nodeSelector: kubernetes.io/os: linux priorityClassName: system-node-critical +{{- if IsKubernetesVersionGe "1.27.0"}} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- end}} tolerations: - operator: "Exists" containers: @@ -7883,6 +7898,9 @@ spec: - "--user-agent-suffix=aks-engine" - "--support-zone=false" - "--get-node-info-from-labels=true" +{{- if IsKubernetesVersionGe "1.27.0"}} + - "--get-nodeid-from-imds=false" +{{- end}} ports: - containerPort: 29603 name: healthz @@ -7998,6 +8016,11 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical +{{- if IsKubernetesVersionGe "1.27.0"}} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- end}} tolerations: - key: "node-role.kubernetes.io/master" operator: "Exists" @@ -8041,9 +8064,16 @@ spec: - "-timeout=1200s" - "-leader-election" - "--leader-election-namespace=kube-system" +{{- if IsKubernetesVersionGe "1.27.0"}} + - "-worker-threads=1000" + - "-kube-api-qps=200" + - "-kube-api-burst=400" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} - "-worker-threads=500" - "-kube-api-qps=50" - "-kube-api-burst=100" +{{- end}} env: - name: ADDRESS value: /csi/csi.sock @@ -8127,6 +8157,10 @@ spec: - "--metrics-address=0.0.0.0:29604" - "--user-agent-suffix=aks-engine" - "--vmss-cache-ttl-seconds=-1" +{{- if IsKubernetesVersionGe "1.27.0"}} + - "--enable-traffic-manager=false" + - "--traffic-manager-port=7788" +{{- end}} ports: - containerPort: 29602 name: healthz @@ -8196,6 +8230,17 @@ spec: selector: matchLabels: app: csi-snapshot-controller +{{- if IsKubernetesVersionGe "1.27.0"}} + # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable + # in #504 the snapshot-controller will exit after around 7.5 seconds if it + # can't find the v1 CRDs so this value should be greater than that + minReadySeconds: 15 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate +{{- end}} template: metadata: labels: @@ -8206,6 +8251,11 @@ spec: kubernetes.io/os: linux kubernetes.io/role: master priorityClassName: system-cluster-critical +{{- if IsKubernetesVersionGe "1.27.0"}} + securityContext: + seccompProfile: + type: RuntimeDefault +{{- end}} tolerations: - key: "node-role.kubernetes.io/master" operator: "Equal" @@ -8317,8 +8367,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -8339,15 +8395,18 @@ spec: jsonPath: .status.readyToUse name: ReadyToUse type: boolean - - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + - description: If a new snapshot needs to be created, this contains the name of + the source PVC from which this snapshot was (or will be) created. jsonPath: .spec.source.persistentVolumeClaimName name: SourcePVC type: string - - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + - description: If a snapshot already exists, this contains the name of the existing + VolumeSnapshotContent object representing the existing snapshot. jsonPath: .spec.source.volumeSnapshotContentName name: SourceSnapshotContent type: string - - description: Represents the minimum size of volume required to rehydrate from this snapshot. + - description: Represents the minimum size of volume required to rehydrate from + this snapshot. jsonPath: .status.restoreSize name: RestoreSize type: string @@ -8355,11 +8414,16 @@ spec: jsonPath: .spec.volumeSnapshotClassName name: SnapshotClass type: string - - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage of + this object. jsonPath: .status.boundVolumeSnapshotContentName name: SnapshotContent type: string - - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + - description: Timestamp when the point-in-time snapshot was taken by the underlying + storage system. jsonPath: .status.creationTime name: CreationTime type: date @@ -8369,51 +8433,102 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: - description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. properties: source: - description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. properties: persistentVolumeClaimName: - description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object representing the volume from which + a snapshot should be created. This PVC is assumed to be in the + same namespace as the VolumeSnapshot object. This field should + be set if the snapshot does not exists, and needs to be created. + This field is immutable. type: string volumeSnapshotContentName: - description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + description: volumeSnapshotContentName specifies the name of a + pre-existing VolumeSnapshotContent object representing an existing + volume snapshot. This field should be set if the snapshot already + exists and only needs a representation in Kubernetes. This field + is immutable. type: string type: object oneOf: - required: ["persistentVolumeClaimName"] - required: ["volumeSnapshotContentName"] volumeSnapshotClassName: - description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be used. + A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one VolumeSnapshotClass + exist for a given CSI Driver and more than one have been marked + as default, CreateSnapshot will fail and generate an event. Empty + string is not allowed for this field.' type: string required: - source type: object status: - description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object. properties: boundVolumeSnapshotContentName: - description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent + object to which this VolumeSnapshot object intends to bind to. If + not specified, it indicates that the VolumeSnapshot object has not + been successfully bound to a VolumeSnapshotContent object yet. NOTE: + To avoid possible security issues, consumers must verify binding + between VolumeSnapshot and VolumeSnapshotContent objects is successful + (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object.' type: string creationTime: - description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the snapshot controller + with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may indicate + that the creation time of the snapshot is unknown. format: date-time type: string error: - description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. The snapshot controller will keep retrying when an error + occurs during the snapshot creation. Upon success, this error field + will be cleared. properties: message: - description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' type: string time: description: time is the timestamp when the error was encountered. @@ -8421,11 +8536,27 @@ spec: type: string type: object readyToUse: - description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + description: readyToUse indicates if the snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. type: boolean restoreSize: type: string - description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller with + the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the + "size_bytes" value returned from the CSI "ListSnapshots" gRPC call + if the driver supports it. When restoring a volume from this snapshot, + the size of the volume MUST NOT be smaller than the restoreSize + if it is specified, otherwise the restoration will fail. If not + specified, it indicates that the size is unknown. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object @@ -8514,7 +8645,7 @@ spec: format: date-time type: string error: - description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. properties: message: description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' @@ -8536,7 +8667,12 @@ spec: required: - spec type: object +{{- if IsKubernetesVersionGe "1.27.0"}} + served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true +{{- end}} storage: false subresources: status: {} @@ -8552,8 +8688,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -8574,7 +8716,8 @@ spec: - jsonPath: .driver name: Driver type: string - - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. jsonPath: .deletionPolicy name: DeletionPolicy type: string @@ -8584,27 +8727,42 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string deletionPolicy: - description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its bound + VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. + Required. enum: - Delete - Retain type: string driver: - description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string parameters: additionalProperties: type: string - description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. type: object required: - deletionPolicy @@ -8658,7 +8816,12 @@ spec: - deletionPolicy - driver type: object +{{- if IsKubernetesVersionGe "1.27.0"}} + served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true +{{- end}} storage: false subresources: {} status: @@ -8673,8 +8836,14 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: +{{- if IsKubernetesVersionGe "1.27.0"}} + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} controller-gen.kubebuilder.io/version: v0.4.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" +{{- end}} labels: addonmanager.kubernetes.io/mode: Reconcile creationTimestamp: null @@ -8697,11 +8866,14 @@ spec: jsonPath: .status.restoreSize name: RestoreSize type: integer - - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. jsonPath: .spec.deletionPolicy name: DeletionPolicy type: string - - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. jsonPath: .spec.driver name: Driver type: string @@ -8709,7 +8881,8 @@ spec: jsonPath: .spec.volumeSnapshotClassName name: VolumeSnapshotClass type: string - - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. jsonPath: .spec.volumeSnapshotRef.name name: VolumeSnapshot type: string @@ -8723,50 +8896,104 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string spec: - description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. properties: deletionPolicy: - description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are deleted. For dynamically provisioned + snapshots, this field will automatically be filled in by the CSI + snapshotter sidecar with the "DeletionPolicy" field defined in the + corresponding VolumeSnapshotClass. For pre-existing snapshots, users + MUST specify this field when creating the VolumeSnapshotContent + object. Required. enum: - Delete - Retain type: string driver: - description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be + the same as the name returned by the CSI GetPluginName() call for + that driver. Required. type: string source: - description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + description: source specifies whether the snapshot is (or should be) + dynamically provisioned or already exists, and just requires a Kubernetes + object representation. This field is immutable after creation. Required. properties: snapshotHandle: - description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + description: snapshotHandle specifies the CSI "snapshot_id" of + a pre-existing snapshot on the underlying storage system for + which a Kubernetes object representation was (or should be) + created. This field is immutable. type: string volumeHandle: - description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. type: string type: object oneOf: - required: ["snapshotHandle"] - required: ["volumeHandle"] +{{- if IsKubernetesVersionGe "1.27.0"}} + sourceVolumeMode: + description: SourceVolumeMode is the mode of the volume whose snapshot + is taken. Can be either “Filesystem” or “Block”. If not specified, + it indicates the source volume's mode is unknown. This field is + immutable. This field is an alpha field. + type: string +{{- end}} volumeSnapshotClassName: - description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and as + such, should not be referenced post-snapshot creation. type: string volumeSnapshotRef: - description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after creation. + Required. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' type: string kind: description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' @@ -8778,7 +9005,8 @@ spec: description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' type: string resourceVersion: - description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' type: string uid: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' @@ -8794,14 +9022,27 @@ spec: description: status represents the current information of a snapshot. properties: creationTime: - description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command ` + "`" + `date +%s%N` + "`" + ` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the CSI snapshotter + sidecar with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it indicates + the creation time is unknown. The format of this field is a Unix + nanoseconds time encoded as an int64. On Unix, the command ` + "`" + `date + +%s%N` + "`" + ` returns the current time in nanoseconds since 1970-01-01 + 00:00:00 UTC. format: int64 type: integer error: - description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. properties: message: - description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' type: string time: description: time is the timestamp when the error was encountered. @@ -8809,15 +9050,34 @@ spec: type: string type: object readyToUse: - description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. type: boolean restoreSize: - description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" value + returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. When restoring a volume from this snapshot, the size of the + volume MUST NOT be smaller than the restoreSize if it is specified, + otherwise the restoration will fail. If not specified, it indicates + that the size is unknown. format: int64 minimum: 0 type: integer snapshotHandle: - description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. type: string type: object required: @@ -8964,7 +9224,12 @@ spec: required: - spec type: object +{{- if IsKubernetesVersionGe "1.27.0"}} + served: false +{{- end}} +{{- if not (IsKubernetesVersionGe "1.27.0")}} served: true +{{- end}} storage: false subresources: status: {} diff --git a/vhd/packer/configure-windows-vhd.ps1 b/vhd/packer/configure-windows-vhd.ps1 index 98e276b36..11764099b 100644 --- a/vhd/packer/configure-windows-vhd.ps1 +++ b/vhd/packer/configure-windows-vhd.ps1 @@ -56,10 +56,14 @@ function Get-ContainerImages { "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.10.0-windows-hp", "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.26.5", "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.26.5-windows-hp", + "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.28.3", + "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.28.3-windows-hp", "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0", "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.6.2", + "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.5.0", "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.8.0", + "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.10.0", "mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0") } '2004' { diff --git a/vhd/packer/install-dependencies.sh b/vhd/packer/install-dependencies.sh index a73d5eff6..d5b8f2cee 100644 --- a/vhd/packer/install-dependencies.sh +++ b/vhd/packer/install-dependencies.sh @@ -239,6 +239,7 @@ done AZUREDISK_CSI_VERSIONS=" 1.10.0 1.26.5 +1.28.3 " for AZUREDISK_CSI_VERSION in ${AZUREDISK_CSI_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v${AZUREDISK_CSI_VERSION}" @@ -256,6 +257,7 @@ for AZUREFILE_CSI_VERSION in ${AZUREFILE_CSI_VERSIONS}; do done CSI_ATTACHER_VERSIONS=" +4.3.0 4.0.0 3.3.0 " @@ -266,6 +268,7 @@ for CSI_ATTACHER_VERSION in ${CSI_ATTACHER_VERSIONS}; do done CSI_NODE_DRIVER_REGISTRAR_VERSIONS=" +2.8.0 2.6.2 2.4.0 " @@ -276,6 +279,7 @@ for CSI_NODE_DRIVER_REGISTRAR_VERSION in ${CSI_NODE_DRIVER_REGISTRAR_VERSIONS}; done CSI_PROVISIONER_VERSIONS=" +3.5.0 3.3.0 3.0.0 " @@ -286,6 +290,7 @@ for CSI_PROVISIONER_VERSION in ${CSI_PROVISIONER_VERSIONS}; do done LIVENESSPROBE_VERSIONS=" +2.10.0 2.8.0 2.5.0 " @@ -296,6 +301,7 @@ for LIVENESSPROBE_VERSION in ${LIVENESSPROBE_VERSIONS}; do done CSI_RESIZER_VERSIONS=" +1.8.0 1.6.0 1.3.0 " @@ -306,6 +312,7 @@ for CSI_RESIZER_VERSION in ${CSI_RESIZER_VERSIONS}; do done CSI_SNAPSHOTTER_VERSIONS=" +6.2.2 5.0.1 4.2.1 " @@ -316,6 +323,7 @@ for CSI_SNAPSHOTTER_VERSION in ${CSI_SNAPSHOTTER_VERSIONS}; do done SNAPSHOT_CONTROLLER_VERSIONS=" +6.2.2 5.0.1 4.2.1 "