From 00d7a31d39fc19d9d6e512f397278afd8e147463 Mon Sep 17 00:00:00 2001 From: JeffYang Date: Tue, 26 Sep 2023 11:41:48 +0000 Subject: [PATCH] Add finalizer to VolumeSnapshot when create PVC with it This patch implement a PVC watcher. When the watcher receive a PVC ADDED event and PVC with VolumeSnapshot kind dataSource, the VolumeSnapshot will be added a finalizer. And the finalizer will be removed when the watcher receive the PVC's DELETED event. NOTE: Asynchronously add finalizer by watcher seems like unsafety, might result in some potential issues. After the PR [1] merged and released, we need to refactor it and implement a synchronous way to add finalizer. [1] https://github.com/kubernetes-csi/external-provisioner/pull/1070 --- charts/cinder-csi-plugin/Chart.yaml | 2 +- .../controllerplugin-deployment.yaml | 1 + cmd/cinder-csi-plugin/main.go | 33 ++++- go.mod | 6 +- go.sum | 19 ++- .../cinder-csi-controllerplugin.yaml | 1 + pkg/csi/cinder/watcher.go | 106 ++++++++++++++++ pkg/csi/cinder/watcher_test.go | 114 ++++++++++++++++++ 8 files changed, 268 insertions(+), 14 deletions(-) create mode 100644 pkg/csi/cinder/watcher.go create mode 100644 pkg/csi/cinder/watcher_test.go diff --git a/charts/cinder-csi-plugin/Chart.yaml b/charts/cinder-csi-plugin/Chart.yaml index 5a74d5927c..a61cc625a3 100644 --- a/charts/cinder-csi-plugin/Chart.yaml +++ b/charts/cinder-csi-plugin/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: v1.28.0 description: Cinder CSI Chart for OpenStack name: openstack-cinder-csi -version: 2.29.0-alpha.2 +version: 2.29.0-alpha.3 home: https://github.com/kubernetes/cloud-provider-openstack icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png maintainers: diff --git a/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml b/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml index 774623134d..5cc5dcc75e 100644 --- a/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml +++ b/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml @@ -158,6 +158,7 @@ spec: - "--endpoint=$(CSI_ENDPOINT)" - "--cloud-config=$(CLOUD_CONFIG)" - "--cluster=$(CLUSTER_NAME)" + - "--enable-vs-deletion-protection" {{- if .Values.csi.plugin.httpEndpoint.enabled }} - "--http-endpoint=:{{ .Values.csi.plugin.httpEndpoint.port }}" {{- end }} diff --git a/cmd/cinder-csi-plugin/main.go b/cmd/cinder-csi-plugin/main.go index e59502a5cc..d803ca6860 100644 --- a/cmd/cinder-csi-plugin/main.go +++ b/cmd/cinder-csi-plugin/main.go @@ -21,6 +21,8 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "k8s.io/cloud-provider-openstack/pkg/csi/cinder" "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" "k8s.io/cloud-provider-openstack/pkg/util/metadata" @@ -31,11 +33,13 @@ import ( ) var ( - endpoint string - nodeID string - cloudConfig []string - cluster string - httpEndpoint string + endpoint string + nodeID string + cloudConfig []string + cluster string + httpEndpoint string + kubeconfig string + vsDeletionProtectionEnabled bool ) func main() { @@ -65,6 +69,7 @@ func main() { cmd.PersistentFlags().StringVar(&cluster, "cluster", "", "The identifier of the cluster that the plugin is running in.") cmd.PersistentFlags().StringVar(&httpEndpoint, "http-endpoint", "", "The TCP network address where the HTTP server for providing metrics for diagnostics, will listen (example: `:8080`). The default is empty string, which means the server is disabled.") + cmd.PersistentFlags().BoolVar(&vsDeletionProtectionEnabled, "enable-vs-deletion-protection", false, "If set, The VS will be added a finalizer while a PVC created base on it, ensure the VS couldn't be deleted before the PVC be deleted.") openstack.AddExtraFlags(pflag.CommandLine) code := cli.Run(cmd) @@ -86,6 +91,24 @@ func handle() { //Initialize Metadata metadata := metadata.GetMetadataProvider(cloud.GetMetadataOpts().SearchOrder) + var restConfig *rest.Config + if kubeconfig != "" { + restConfig, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + } else { + restConfig, err = rest.InClusterConfig() + } + if err != nil { + klog.Warning("Failed to init rest config: %v", err) + return + } + if vsDeletionProtectionEnabled { + err = cinder.StartContollerWatcher(restConfig) + if err != nil { + klog.Warningf("Failed to StartContollerWatcher: %v", err) + return + } + } + d.SetupDriver(cloud, mount, metadata) d.Run() } diff --git a/go.mod b/go.mod index 47798a2b06..7feb25bfd8 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/kubernetes-csi/csi-lib-utils v0.13.0 github.com/kubernetes-csi/csi-test/v5 v5.0.0 + github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.5.0 github.com/onsi/ginkgo/v2 v2.13.0 @@ -39,6 +40,7 @@ require ( k8s.io/kubernetes v1.29.0-alpha.2 k8s.io/mount-utils v0.29.0-alpha.2 k8s.io/utils v0.0.0-20230726121419-3b25d923346b + sigs.k8s.io/controller-runtime v0.16.3 software.sslmate.com/src/go-pkcs12 v0.2.0 ) @@ -58,6 +60,7 @@ require ( github.com/distribution/reference v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect @@ -121,9 +124,8 @@ require ( go.opentelemetry.io/otel/sdk v1.14.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.24.0 // indirect + go.uber.org/zap v1.25.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/oauth2 v0.8.0 // indirect diff --git a/go.sum b/go.sum index b528d6c446..4383c424e3 100644 --- a/go.sum +++ b/go.sum @@ -54,7 +54,7 @@ github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1/g github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -108,6 +108,8 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= @@ -127,7 +129,7 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -253,6 +255,7 @@ github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -274,6 +277,8 @@ github.com/kubernetes-csi/csi-lib-utils v0.13.0 h1:QrTdZVZbHlaSUBN9ReayBPnnF1N0e github.com/kubernetes-csi/csi-lib-utils v0.13.0/go.mod h1:JS9eDIZmSjx4F9o0bLTVK/qfhIIOifdjEfVXzxWapfE= github.com/kubernetes-csi/csi-test/v5 v5.0.0 h1:GJ0M+ppcKgWhafXH3B2Ssfw1Egzly9GlMx3JOQApekM= github.com/kubernetes-csi/csi-test/v5 v5.0.0/go.mod h1:jVEIqf8Nv1roo/4zhl/r6Tc68MAgRX/OQSQK0azTHyo= +github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0 h1:qS4r4ljINLWKJ9m9Ge3Q3sGZ/eIoDVDT2RhAdQFHb1k= +github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0/go.mod h1:oGXx2XTEzs9ikW2V6IC1dD8trgjRsS/Mvc2JRiC618Y= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -323,6 +328,7 @@ github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= @@ -423,13 +429,11 @@ go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+go go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -686,6 +690,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -877,6 +882,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y= +sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= +sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= diff --git a/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml b/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml index 8dedfc90d3..59367d5701 100644 --- a/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml +++ b/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml @@ -99,6 +99,7 @@ spec: - "--endpoint=$(CSI_ENDPOINT)" - "--cloud-config=$(CLOUD_CONFIG)" - "--cluster=$(CLUSTER_NAME)" + - "--enable-vs-deletion-protection" - "--v=1" env: - name: CSI_ENDPOINT diff --git a/pkg/csi/cinder/watcher.go b/pkg/csi/cinder/watcher.go new file mode 100644 index 0000000000..4c6326fffb --- /dev/null +++ b/pkg/csi/cinder/watcher.go @@ -0,0 +1,106 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cinder + +import ( + "fmt" + + snap "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + "golang.org/x/net/context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + annStorageProvisioner = "volume.kubernetes.io/storage-provisioner" + snapshotKind = "VolumeSnapshot" +) + +type controllerWatcher struct { + snapClient snap.Interface + watcher watch.Interface +} + +func (cw *controllerWatcher) run() { + klog.V(2).Info("Controller watcher started") + for event := range cw.watcher.ResultChan() { + obj := event.Object.DeepCopyObject() + pvc, ok := obj.(*corev1.PersistentVolumeClaim) + if !ok { + continue + } + if pvc.ObjectMeta.Annotations[annStorageProvisioner] != driverName { + continue + } + if pvc.Spec.DataSource == nil || pvc.Spec.DataSource.Kind != snapshotKind { + continue + } + snapshotObj, err := cw.snapClient.SnapshotV1().VolumeSnapshots(pvc.Namespace).Get(context.Background(), pvc.Spec.DataSource.Name, metav1.GetOptions{}) + if err != nil { + klog.ErrorS(err, "Error get VolumeSnapshot", "namespace", pvc.Namespace, "name", pvc.Spec.DataSource.Name) + continue + } + update := false + finalizer := fmt.Sprintf("%s/pvc-%s", driverName, pvc.UID) + // TODO(JeffYang): Asynchronously add finalizer seems like unsafety, might have some potential issues. + // Move it into controllerServer CreateVolume after the PR https://github.com/kubernetes-csi/external-provisioner/pull/1070 merged and released. + // We can synchronously add finalizer there + if event.Type == watch.Added { + klog.V(5).InfoS("Receive ADDED event try to add finalizer to VolumeSnapshot", "PersistentVolumeClaim", pvc.Name, "VolumeSnapshot", snapshotObj.Name) + update = controllerutil.AddFinalizer(snapshotObj, finalizer) + } + if event.Type == watch.Deleted { + klog.V(5).InfoS("Receive DELETED event try to remove finalizer from VolumeSnapshot", "PersistentVolumeClaim", pvc.Name, "VolumeSnapshot", snapshotObj.Name) + update = controllerutil.RemoveFinalizer(snapshotObj, finalizer) + } + if update { + _, err := cw.snapClient.SnapshotV1().VolumeSnapshots(pvc.Namespace).Update(context.Background(), snapshotObj, metav1.UpdateOptions{}) + if err != nil { + klog.ErrorS(err, "Error update VolumeSnapshot", "name", snapshotObj.Name, "finalizer", finalizer) + } + } + } +} + +func StartContollerWatcher(restConfig *rest.Config) error { + snapClient, err := snap.NewForConfig(restConfig) + if err != nil { + klog.ErrorS(err, "Error building snapshot clientset") + return err + } + + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + klog.ErrorS(err, "Error building kubernetes clientset") + return err + } + pvcClient := clientset.CoreV1().PersistentVolumeClaims(corev1.NamespaceAll) + watchInterface, err := pvcClient.Watch(context.Background(), metav1.ListOptions{}) + if err != nil { + klog.ErrorS(err, "Error starting watcher") + return err + } + + cw := controllerWatcher{snapClient: snapClient, watcher: watchInterface} + go cw.run() + return nil +} diff --git a/pkg/csi/cinder/watcher_test.go b/pkg/csi/cinder/watcher_test.go new file mode 100644 index 0000000000..973f1152d3 --- /dev/null +++ b/pkg/csi/cinder/watcher_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cinder + +import ( + "context" + "fmt" + "testing" + "time" + + snapapi "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + snapfake "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake" + uuid "github.com/pborman/uuid" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + clientfake "k8s.io/client-go/kubernetes/fake" +) + +func TestControllerWatcherRun(t *testing.T) { + // Init assert + assert := assert.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // Init fake client and watch interface + snapClient := snapfake.NewSimpleClientset() + clientset := clientfake.NewSimpleClientset() + pvcClient := clientset.CoreV1().PersistentVolumeClaims(corev1.NamespaceDefault) + watchInterface, err := pvcClient.Watch(ctx, metav1.ListOptions{}) + assert.Nil(err) + // Start watcher + ctrlWatcher := controllerWatcher{snapClient: snapClient, watcher: watchInterface} + go ctrlWatcher.run() + // Prepare necessary resources + originalPVCName := "original-pvc" + testVSName := "test-vs" + vs := snapapi.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{Name: testVSName}, + Spec: snapapi.VolumeSnapshotSpec{ + Source: snapapi.VolumeSnapshotSource{ + PersistentVolumeClaimName: &originalPVCName, + }, + }, + } + _, err = snapClient.SnapshotV1().VolumeSnapshots(corev1.NamespaceDefault).Create(ctx, &vs, metav1.CreateOptions{}) + assert.Nil(err) + storageAPIGroup := "snapshot.storage.k8s.io" + // Test PVC creation with VolumeSnapshot + testPVCUID := uuid.New() + testPVCName := "test-pvc" + _, err = pvcClient.Create(context.TODO(), &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: testPVCName, + UID: types.UID(testPVCUID), + Annotations: map[string]string{ + annStorageProvisioner: driverName, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + DataSource: &corev1.TypedLocalObjectReference{ + Name: testVSName, + Kind: "VolumeSnapshot", + APIGroup: &storageAPIGroup, + }, + }, + }, metav1.CreateOptions{}) + assert.Nil(err) + expectedFinalizer := fmt.Sprintf("%s/pvc-%s", driverName, testPVCUID) + err = wait.PollUntilContextTimeout(ctx, time.Second*1, time.Second*5, true, func(context.Context) (done bool, err error) { + vs, err := snapClient.SnapshotV1().VolumeSnapshots(corev1.NamespaceDefault).Get(ctx, testVSName, metav1.GetOptions{}) + assert.Nil(err) + f := vs.GetFinalizers() + for _, e := range f { + // If VolumeSnashot contain expectedFinalizer, test successful + if e == expectedFinalizer { + return true, nil + } + } + return false, nil + }) + assert.Nil(err) + // Test PVC deletion + err = pvcClient.Delete(context.TODO(), testPVCName, metav1.DeleteOptions{}) + assert.Nil(err) + err = wait.PollUntilContextTimeout(ctx, time.Second*1, time.Second*5, true, func(context.Context) (done bool, err error) { + vs, err := snapClient.SnapshotV1().VolumeSnapshots(corev1.NamespaceDefault).Get(ctx, testVSName, metav1.GetOptions{}) + assert.Nil(err) + f := vs.GetFinalizers() + for _, e := range f { + // If VolumeSnashot still contain expectedFinalizer, wait a scond and recheck until timeout + if e == expectedFinalizer { + return false, nil + } + } + return true, nil + }) + assert.Nil(err) +}