Sync from upstream 1.8 to downstream 4.10

Documentation/ceph-upgrade.md

@@ -549,9 +549,9 @@ ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.5.1"
 ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0"
 ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0"
 ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0"
-ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
-ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
-CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
+ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.4.0"
+ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1"
+CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.3.0"
 ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.2.1"
 ```
 
@@ -574,9 +574,9 @@ kubectl --namespace rook-ceph get pod -o jsonpath='{range .items[*]}{range .spec
 k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
 k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
 k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
-k8s.gcr.io/sig-storage/csi-resizer:v1.3.0
-k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
+k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
+k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
 quay.io/cephcsi/cephcsi:v3.5.1
-quay.io/csiaddons/volumereplication-operator:v0.1.0
+quay.io/csiaddons/volumereplication-operator:v0.3.0
 quay.io/csiaddons/k8s-sidecar:v0.2.1
 ```

Documentation/helm-operator.md

@@ -138,14 +138,14 @@ The following tables lists the configurable parameters of the rook-operator char
 | `csi.rbdPluginUpdateStrategy`       | CSI Rbd plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.                                  | `OnDelete`                                                |
 | `csi.cephFSPluginUpdateStrategy`    | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.                               | `OnDelete`                                                |
 | `csi.registrar.image`               | Kubernetes CSI registrar image.                                                                                             | `k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0` |
-| `csi.resizer.image`                 | Kubernetes CSI resizer image.                                                                                               | `k8s.gcr.io/sig-storage/csi-resizer:v1.3.0`               |
+| `csi.resizer.image`                 | Kubernetes CSI resizer image.                                                                                               | `k8s.gcr.io/sig-storage/csi-resizer:v1.4.0`               |
 | `csi.provisioner.image`             | Kubernetes CSI provisioner image.                                                                                           | `k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0`           |
-| `csi.snapshotter.image`             | Kubernetes CSI snapshotter image.                                                                                           | `k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0`           |
+| `csi.snapshotter.image`             | Kubernetes CSI snapshotter image.                                                                                           | `k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1`           |
 | `csi.attacher.image`                | Kubernetes CSI Attacher image.                                                                                              | `k8s.gcr.io/sig-storage/csi-attacher:v3.4.0`              |
 | `csi.cephfsPodLabels`               | Labels to add to the CSI CephFS Pods.                                                                                       | <none>                                                    |
 | `csi.rbdPodLabels`                  | Labels to add to the CSI RBD Pods.                                                                                          | <none>                                                    |
 | `csi.volumeReplication.enabled`     | Enable Volume Replication.                                                                                                  | `false`                                                   |
-| `csi.volumeReplication.image`       | Volume Replication Controller image.                                                                                        | `quay.io/csiaddons/volumereplication-operator:v0.1.0`     |
+| `csi.volumeReplication.image`       | Volume Replication Controller image.                                                                                        | `quay.io/csiaddons/volumereplication-operator:v0.3.0`     |
 | `csi.csiAddons.enabled`     | Enable CSIAddons                                                                                                  | `false`                                                   |
 | `csi.csiAddons.image`       | CSIAddons Sidecar image.                                                                                        | `quay.io/csiaddons/k8s-sidecar:v0.2.1`     |
 | `admissionController.tolerations`   | Array of tolerations in YAML format which will be added to admission controller deployment.                                 | <none>                                                    |

deploy/charts/library/templates/_cluster-psp.tpl

@@ -61,4 +61,4 @@ subjects:
   - kind: ServiceAccount
     name: rook-ceph-cmd-reporter
     namespace: {{ .Release.Namespace }} # namespace:cluster
-{{- end -}}
+{{- end }}

deploy/charts/library/templates/_cluster-rolebinding.tpl

@@ -90,4 +90,4 @@ subjects:
   - kind: ServiceAccount
     name: rook-ceph-purge-osd
     namespace: {{ .Release.Namespace }} # namespace:cluster
-{{- end -}}
+{{- end }}

deploy/charts/library/templates/_recommended-labels.tpl

@@ -6,4 +6,4 @@ app.kubernetes.io/part-of: rook-ceph-operator
 app.kubernetes.io/managed-by: helm
 app.kubernetes.io/created-by: helm
 helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
-{{- end -}}
+{{- end }}

deploy/charts/library/templates/_suffix-cluster-namespace.tpl

@@ -14,5 +14,5 @@ If the cluster namespace is different from the operator namespace, we want to na
 {{- $clusterNamespace := .Release.Namespace -}}
 {{- if ne $clusterNamespace $operatorNamespace -}}
 {{ printf "-%s" $clusterNamespace }}
-{{- end -}}
-{{- end -}}
+{{- end }}
+{{- end }}

deploy/charts/rook-ceph-cluster/templates/_helpers.tpl

@@ -3,14 +3,14 @@ Define the clusterName as defaulting to the release namespace
 */}}
 {{- define "clusterName" -}}
 {{ .Values.clusterName | default .Release.Namespace }}
-{{- end -}}
+{{- end }}
 
 {{/*
 Return the target Kubernetes version.
 */}}
 {{- define "capabilities.kubeVersion" -}}
 {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}}
-{{- end -}}
+{{- end }}
 
 {{/*
 Return the appropriate apiVersion for ingress.
@@ -22,5 +22,5 @@ Return the appropriate apiVersion for ingress.
 {{- print "networking.k8s.io/v1beta1" -}}
 {{- else -}}
 {{- print "networking.k8s.io/v1" -}}
-{{- end -}}
-{{- end -}}
+{{- end }}
+{{- end }}

deploy/charts/rook-ceph-cluster/templates/rbac.yaml

@@ -14,7 +14,7 @@ clusterrolebindings
 {{- if .Values.pspEnable }}
 ---
 {{ include "library.cluster.psp.rolebindings" . }}
-{{- end -}}
+{{- end }}
 
 {{/*
 roles
@@ -38,4 +38,4 @@ rolebindings
 {{ include "library.cluster.monitoring.rolebindings" . }}
 {{- end }}
 
-{{- end -}}
+{{- end }}

deploy/charts/rook-ceph-cluster/templates/volumesnapshotclass.yaml

@@ -11,8 +11,15 @@ apiVersion: snapshot.storage.k8s.io/v1beta1
 kind: VolumeSnapshotClass
 metadata:
   name: {{ $filesystemvsc.name }}
+{{- if $filesystemvsc.labels }}
+  labels:
+{{ toYaml $filesystemvsc.labels | indent 4 }}
+{{- end }}
   annotations:
     snapshot.storage.kubernetes.io/is-default-class: "{{ if default false $filesystemvsc.isDefault }}true{{ else }}false{{ end }}"
+{{- if $filesystemvsc.annotations }}
+{{ toYaml $filesystemvsc.annotations | indent 4 }}
+{{- end }}
 driver: {{ .Values.operatorNamespace }}.cephfs.csi.ceph.com
 parameters:
   clusterID: {{ .Release.Namespace }}
@@ -34,8 +41,15 @@ apiVersion: snapshot.storage.k8s.io/v1beta1
 kind: VolumeSnapshotClass
 metadata:
   name: {{ $blockpoolvsc.name }}
+{{- if $blockpoolvsc.labels }}
+  labels:
+{{ toYaml $blockpoolvsc.labels | indent 4 }}
+{{- end }}
   annotations:
     snapshot.storage.kubernetes.io/is-default-class: "{{ if default false $blockpoolvsc.isDefault }}true{{ else }}false{{ end }}"
+{{- if $blockpoolvsc.annotations }}
+{{ toYaml $blockpoolvsc.annotations | indent 4 }}
+{{- end }}
 driver: {{ .Values.operatorNamespace }}.rbd.csi.ceph.com
 parameters:
   clusterID: {{ .Release.Namespace }}

deploy/charts/rook-ceph-cluster/values.yaml

@@ -403,6 +403,8 @@ cephFileSystemVolumeSnapshotClass:
   name: ceph-filesystem
   isDefault: true
   deletionPolicy: Delete
+  annotations: {}
+  labels: {}
   # see https://rook.io/docs/rook/latest/ceph-csi-snapshot.html#cephfs-snapshots for available configuration
   parameters: {}
 
@@ -411,6 +413,8 @@ cephBlockPoolsVolumeSnapshotClass:
   name: ceph-block
   isDefault: false
   deletionPolicy: Delete
+  annotations: {}
+  labels: {}
   # see https://rook.io/docs/rook/latest/ceph-csi-snapshot.html#rbd-snapshots for available configuration
   parameters: {}
 

deploy/charts/rook-ceph/templates/cluster-rbac.yaml

@@ -18,7 +18,7 @@ clusterrolebindings
 {{- if .Values.pspEnable }}
 ---
 {{ include "library.cluster.psp.rolebindings" . }}
-{{- end -}}
+{{- end }}
 
 {{/*
 roles

deploy/charts/rook-ceph/templates/clusterrole.yaml

@@ -425,19 +425,19 @@ rules:
     verbs: ["update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update"]
+    verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotclasses"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents/status"]
-    verbs: ["update"]
+    verbs: ["update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots/status"]
-    verbs: ["update"]
+    verbs: ["update", "patch"]
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -501,19 +501,19 @@ rules:
     verbs: ["list", "watch", "create", "update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update"]
+    verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotclasses"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents/status"]
-    verbs: ["update"]
+    verbs: ["update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots/status"]
-    verbs: ["update"]
+    verbs: ["update", "patch"]
   - apiGroups: [""]
     resources: ["persistentvolumeclaims/status"]
     verbs: ["update", "patch"]
@@ -535,4 +535,4 @@ rules:
   - apiGroups: [""]
     resources: ["serviceaccounts"]
     verbs: ["get"]
-{{- end -}}
+{{- end }}

deploy/charts/rook-ceph/templates/clusterrolebinding.yaml

@@ -99,4 +99,4 @@ roleRef:
   kind: ClusterRole
   name: rbd-external-provisioner-runner
   apiGroup: rbac.authorization.k8s.io
-{{- end -}}
+{{- end }}

deploy/charts/rook-ceph/templates/psp.yaml

@@ -168,4 +168,4 @@ subjects:
     name: rook-csi-rbd-provisioner-sa
     namespace: {{ .Release.Namespace }} # namespace:operator
 {{- end }}
-{{- end -}}
+{{- end }}

deploy/charts/rook-ceph/templates/rolebinding.yaml

@@ -47,7 +47,7 @@ roleRef:
   name: rbd-csi-nodeplugin
   apiGroup: rbac.authorization.k8s.io
 ---
-{{- end -}}
+{{- end }}
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -61,4 +61,4 @@ roleRef:
   kind: Role
   name: rbd-external-provisioner-cfg
   apiGroup: rbac.authorization.k8s.io
-{{- end -}}
+{{- end }}

deploy/charts/rook-ceph/values.yaml

@@ -282,11 +282,11 @@ csi:
   #provisioner:
     #image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
   #snapshotter:
-    #image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
+    #image: k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
   #attacher:
     #image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
   #resizer:
-    #image: k8s.gcr.io/sig-storage/csi-resizer:v1.3.0
+    #image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
   # Labels to add to the CSI CephFS Deployments and DaemonSets Pods.
   #cephfsPodLabels: "key1=value1,key2=value2"
   # Labels to add to the CSI RBD Deployments and DaemonSets Pods.
@@ -296,7 +296,7 @@ csi:
   # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring
   volumeReplication:
     enabled: false
-    #image: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
+    #image: "quay.io/csiaddons/volumereplication-operator:v0.3.0"
   # Enable the CSIAddons sidecar.
   csiAddons:
     enabled: false

deploy/examples/common.yaml

@@ -66,19 +66,19 @@ rules:
     verbs: ["update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update"]
+    verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotclasses"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents/status"]
-    verbs: ["update"]
+    verbs: ["update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots/status"]
-    verbs: ["update"]
+    verbs: ["update", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -160,19 +160,19 @@ rules:
     verbs: ["list", "watch", "create", "update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update"]
+    verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotclasses"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents/status"]
-    verbs: ["update"]
+    verbs: ["update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots/status"]
-    verbs: ["update"]
+    verbs: ["update", "patch"]
   - apiGroups: [""]
     resources: ["persistentvolumeclaims/status"]
     verbs: ["update", "patch"]

deploy/examples/images.txt

@@ -1,10 +1,10 @@
  k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
  k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0
  k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
- k8s.gcr.io/sig-storage/csi-resizer:v1.3.0
- k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
+ k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
+ k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
  quay.io/ceph/ceph:v16.2.7
  quay.io/cephcsi/cephcsi:v3.5.1
  quay.io/csiaddons/k8s-sidecar:v0.2.1
- quay.io/csiaddons/volumereplication-operator:v0.1.0
+ quay.io/csiaddons/volumereplication-operator:v0.3.0
  rook/ceph:v1.8.3

deploy/examples/operator-openshift.yaml

@@ -163,9 +163,9 @@ data:
   # these images to the desired release of the CSI driver.
   # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.5.1"
   # ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0"
-  # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
+  # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.4.0"
   # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0"
-  # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
+  # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1"
   # ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0"
 
   # (Optional) set user created priorityclassName for csi plugin pods.
@@ -417,7 +417,7 @@ data:
   CSI_ENABLE_VOLUME_REPLICATION: "false"
   # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15.
   ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15"
-  # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
+  # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.3.0"
   # Enable the csi addons sidecar.
   CSI_ENABLE_CSIADDONS: "false"
   # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.2.1"

deploy/examples/operator.yaml

@@ -81,9 +81,9 @@ data:
   # these images to the desired release of the CSI driver.
   # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.5.1"
   # ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0"
-  # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
+  # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.4.0"
   # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0"
-  # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
+  # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1"
   # ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0"
 
   # (Optional) set user created priorityclassName for csi plugin pods.
@@ -335,7 +335,7 @@ data:
   # Before enabling, ensure the Volume Replication CRDs are created.
   # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring
   CSI_ENABLE_VOLUME_REPLICATION: "false"
-  # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
+  # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.3.0"
   # Enable the csi addons sidecar.
   CSI_ENABLE_CSIADDONS: "false"
   # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.2.1"

deploy/examples/osd-env-override.yaml

@@ -0,0 +1,19 @@
+# ###############################################################################################################
+# The `rook-ceph-osd-env-override` ConfigMap is a development feature
+# that allows to inject arbitrary environment variables to OSD-related
+# containers created by the operator.
+# ###############################################################################################################
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rook-ceph-osd-env-override
+  namespace: rook-ceph
+data:
+  # Bypass the ASan's assertion that it is the very first loaded DSO.
+  # This is necessary for crimson-osd as it's currently built with
+  # the ASan sanitizer turned on which means the `libasan.so` must
+  # the be the very first loaded dynamic library. Unfortunately, this
+  # isn't fulfilled as the containers use `ld.preload`, so ASan was
+  # aborting the entire OSD.
+  ASAN_OPTIONS: verify_asan_link_order=0