forked from kubevirt/containerized-data-importer
-
Notifications
You must be signed in to change notification settings - Fork 0
/
smart-clone-controller.go
388 lines (344 loc) · 13.1 KB
/
smart-clone-controller.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
package controller
import (
"fmt"
"time"
"reflect"
csisnapshotv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
coreinformers "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog"
cdiv1 "kubevirt.io/containerized-data-importer/pkg/apis/core/v1alpha1"
clientset "kubevirt.io/containerized-data-importer/pkg/client/clientset/versioned"
cdischeme "kubevirt.io/containerized-data-importer/pkg/client/clientset/versioned/scheme"
informers "kubevirt.io/containerized-data-importer/pkg/client/informers/externalversions/core/v1alpha1"
listers "kubevirt.io/containerized-data-importer/pkg/client/listers/core/v1alpha1"
"kubevirt.io/containerized-data-importer/pkg/common"
csiclientset "kubevirt.io/containerized-data-importer/pkg/snapshot-client/clientset/versioned"
snapshotsinformers "kubevirt.io/containerized-data-importer/pkg/snapshot-client/informers/externalversions/volumesnapshot/v1alpha1"
snapshotslisters "kubevirt.io/containerized-data-importer/pkg/snapshot-client/listers/volumesnapshot/v1alpha1"
)
const (
//AnnSmartCloneRequest sets our expected annotation for a CloneRequest
AnnSmartCloneRequest = "k8s.io/SmartCloneRequest"
)
// SmartCloneController represents the CDI SmartClone Controller
type SmartCloneController struct {
clientset kubernetes.Interface
cdiClientSet clientset.Interface
csiClientSet csiclientset.Interface
snapshotInformer cache.SharedIndexInformer
snapshotsLister snapshotslisters.VolumeSnapshotLister
pvcLister corelisters.PersistentVolumeClaimLister
dataVolumeLister listers.DataVolumeLister
dataVolumesSynced cache.InformerSynced
snapshotsSynced cache.InformerSynced
queue workqueue.RateLimitingInterface
recorder record.EventRecorder
}
// NewSmartCloneController sets up a Smart Clone Controller, and returns a pointer to
// to the newly created Controller
func NewSmartCloneController(client kubernetes.Interface,
cdiClientSet clientset.Interface,
csiClientSet csiclientset.Interface,
pvcInformer coreinformers.PersistentVolumeClaimInformer,
snapshotInformer snapshotsinformers.VolumeSnapshotInformer,
dataVolumeInformer informers.DataVolumeInformer) *SmartCloneController {
// Create event broadcaster
// Add smart-clone-controller types to the default Kubernetes Scheme so Events can be
// logged for smart-clone-controller types.
cdischeme.AddToScheme(scheme.Scheme)
klog.V(3).Info("Creating event broadcaster")
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(klog.V(2).Infof)
eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events("")})
recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName})
c := &SmartCloneController{
clientset: client,
cdiClientSet: cdiClientSet,
csiClientSet: csiClientSet,
pvcLister: pvcInformer.Lister(),
snapshotsLister: snapshotInformer.Lister(),
snapshotInformer: snapshotInformer.Informer(),
dataVolumeLister: dataVolumeInformer.Lister(),
dataVolumesSynced: dataVolumeInformer.Informer().HasSynced,
snapshotsSynced: snapshotInformer.Informer().HasSynced,
recorder: recorder,
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
}
// Set up an event handler for when VolumeSnapshot resources change
snapshotInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: c.enqueueItem,
UpdateFunc: func(old, new interface{}) {
c.enqueueItem(new)
},
DeleteFunc: c.enqueueItem,
})
// Set up an event handler for when PVC resources change
// handleObject function ensures we filter PVCs not created by this controller
pvcInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: c.enqueueItem,
UpdateFunc: func(old, new interface{}) {
c.enqueueItem(new)
},
DeleteFunc: c.enqueueItem,
})
return c
}
//ProcessNextItem ...
func (c *SmartCloneController) ProcessNextItem() bool {
key, shutdown := c.queue.Get()
if shutdown {
return false
}
defer c.queue.Done(key)
ns, name, err := cache.SplitMetaNamespaceKey(key.(string))
if err != nil {
runtime.HandleError(errors.Errorf("invalid resource key: %s", key))
return false
}
pvc, err := c.pvcLister.PersistentVolumeClaims(ns).Get(name)
if err != nil {
if k8serrors.IsNotFound(err) {
pvc = nil
} else {
// Error getting PVC - return
return false
}
}
snapshot, err := c.snapshotsLister.VolumeSnapshots(ns).Get(name)
if err != nil {
if k8serrors.IsNotFound(err) {
snapshot = nil
} else {
// Error getting Snapshot - return
return false
}
}
if pvc != nil {
if pvc.Status.Phase != corev1.ClaimBound {
// PVC isn't bound yet - return
return false
}
if pvc.ObjectMeta.Annotations[AnnSmartCloneRequest] == "true" {
snapshotName := pvc.Spec.DataSource.Name
snapshotToDelete, err := c.snapshotsLister.VolumeSnapshots(ns).Get(snapshotName)
if err != nil {
// Error getting Snapshot - return
return true
}
if snapshotToDelete != nil {
klog.V(3).Infof("ProcessNextItem snapshotName: %s", snapshotName)
err = c.csiClientSet.SnapshotV1alpha1().VolumeSnapshots(ns).Delete(snapshotName, &metav1.DeleteOptions{})
if err != nil {
klog.Errorf("error deleting snapshot for smart-clone %q: %v", key, err)
return true
}
klog.V(3).Infof("Snapshot deleted: %s", snapshotName)
dataVolume, err := c.dataVolumeLister.DataVolumes(snapshot.Namespace).Get(snapshot.Name)
if err != nil {
return true
}
// Update DV phase and emit PVC in progress event
c.updateSmartCloneStatusPhase(cdiv1.Succeeded, dataVolume, pvc)
}
}
} else if snapshot != nil {
err := c.syncSnapshot(key.(string))
if err != nil {
klog.Errorf("error processing snapshot %q: %v", key, err)
return true
}
}
return c.forgetKey(key, fmt.Sprintf("ProcessNextItem: processing pvc/snapshot %q completed", key))
}
// forget the passed-in key for this event and optionally log a message.
func (c *SmartCloneController) forgetKey(key interface{}, msg string) bool {
if len(msg) > 0 {
klog.V(3).Info(msg)
}
c.queue.Forget(key)
return true
}
func (c *SmartCloneController) syncSnapshot(key string) error {
snapshot, exists, err := c.snapshotFromKey(key)
if err != nil {
return err
} else if !exists {
return nil
}
_, ok := snapshot.Annotations[AnnSmartCloneRequest]
if !ok {
//ignoring snapshot, not created by DataVolume Controller
return nil
}
snapshotReadyToUse := snapshot.Status.ReadyToUse
klog.V(3).Infof("Snapshot \"%s/%s\" - ReadyToUse: %t", snapshot.Namespace, snapshot.Name, snapshotReadyToUse)
if !snapshotReadyToUse {
return nil
}
return c.processNextSnapshotItem(snapshot)
}
// The snapshot is ReadyToUse, then we can create the PVC and update DV status
func (c *SmartCloneController) processNextSnapshotItem(snapshot *csisnapshotv1.VolumeSnapshot) error {
dataVolume, err := c.dataVolumeLister.DataVolumes(snapshot.Namespace).Get(snapshot.Name)
if err != nil {
return err
}
// Update DV phase and emit PVC in progress event
c.updateSmartCloneStatusPhase(SmartClonePVCInProgress, dataVolume, nil)
newPvc := newPvcFromSnapshot(snapshot, dataVolume)
if newPvc == nil {
klog.Errorf("error creating new pvc from snapshot object")
return nil
}
_, err = c.clientset.CoreV1().PersistentVolumeClaims(snapshot.Namespace).Create(newPvc)
if err != nil {
return err
}
return nil
}
func (c *SmartCloneController) objFromKey(informer cache.SharedIndexInformer, key interface{}) (interface{}, bool, error) {
keyString, ok := key.(string)
if !ok {
return nil, false, errors.New("keys is not of type string")
}
obj, ok, err := informer.GetIndexer().GetByKey(keyString)
if err != nil {
return nil, false, errors.Wrap(err, "error getting interface obj from store")
}
if !ok {
return nil, false, nil
}
return obj, true, nil
}
// return a VolumeSnapshot pointer based on the passed-in work queue key.
func (c *SmartCloneController) snapshotFromKey(key interface{}) (*csisnapshotv1.VolumeSnapshot, bool, error) {
obj, exists, err := c.objFromKey(c.snapshotInformer, key)
if err != nil {
return nil, false, errors.Wrap(err, "could not get pvc object from key")
} else if !exists {
return nil, false, nil
}
snapshot, ok := obj.(*csisnapshotv1.VolumeSnapshot)
if !ok {
return nil, false, errors.New("Object not of type *v1.PersistentVolumeClaim")
}
return snapshot, true, nil
}
//Run is being called from cdi-controller (cmd)
func (c *SmartCloneController) Run(threadiness int, stopCh <-chan struct{}) error {
defer runtime.HandleCrash()
defer c.queue.ShutDown()
// Start the informer factories to begin populating the informer caches
klog.V(2).Info("Starting SmartCloneController controller")
// Wait for the caches to be synced before starting workers
klog.V(2).Info("Waiting for informer caches to sync")
if !cache.WaitForCacheSync(stopCh, c.snapshotsSynced, c.dataVolumesSynced) {
return errors.New("Timeout waiting for caches sync")
}
klog.V(2).Info("Starting worker")
for i := 0; i < threadiness; i++ {
go wait.Until(c.runSnapshotWorkers, time.Second, stopCh)
}
klog.V(2).Info("Started workers")
<-stopCh
klog.V(2).Info("Shutting down workers")
return nil
}
func (c *SmartCloneController) runSnapshotWorkers() {
for c.ProcessNextItem() {
// empty
}
}
// enqueueItem takes a VolumeSnapshot or PVC resource and converts it into a namespace/name
// string which is then put onto the work queue.
func (c *SmartCloneController) enqueueItem(obj interface{}) {
var key string
var err error
if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
runtime.HandleError(err)
return
}
c.queue.AddRateLimited(key)
}
func (c *SmartCloneController) updateSmartCloneStatusPhase(phase cdiv1.DataVolumePhase, dataVolume *cdiv1.DataVolume, newPVC *corev1.PersistentVolumeClaim) error {
var dataVolumeCopy = dataVolume.DeepCopy()
var event DataVolumeEvent
switch phase {
case cdiv1.SnapshotForSmartCloneInProgress:
dataVolumeCopy.Status.Phase = cdiv1.SnapshotForSmartCloneInProgress
event.eventType = corev1.EventTypeNormal
event.reason = SnapshotForSmartCloneInProgress
event.message = fmt.Sprintf(MessageSmartCloneInProgress, dataVolumeCopy.Spec.Source.PVC.Namespace, dataVolumeCopy.Spec.Source.PVC.Name)
case cdiv1.SmartClonePVCInProgress:
dataVolumeCopy.Status.Phase = cdiv1.SmartClonePVCInProgress
event.eventType = corev1.EventTypeNormal
event.reason = SmartClonePVCInProgress
event.message = fmt.Sprintf(MessageSmartClonePVCInProgress, dataVolumeCopy.Spec.Source.PVC.Namespace, dataVolumeCopy.Spec.Source.PVC.Name)
case cdiv1.Succeeded:
dataVolumeCopy.Status.Phase = cdiv1.Succeeded
event.eventType = corev1.EventTypeNormal
event.reason = CloneSucceeded
event.message = fmt.Sprintf(MessageCloneSucceeded, dataVolumeCopy.Spec.Source.PVC.Namespace, dataVolumeCopy.Spec.Source.PVC.Name, newPVC.Namespace, newPVC.Name)
}
return c.emitEvent(dataVolume, dataVolumeCopy, &event)
}
func (c *SmartCloneController) emitEvent(dataVolume *cdiv1.DataVolume, dataVolumeCopy *cdiv1.DataVolume, event *DataVolumeEvent) error {
// Only update the object if something actually changed in the status.
if !reflect.DeepEqual(dataVolume.Status, dataVolumeCopy.Status) {
_, err := c.cdiClientSet.CdiV1alpha1().DataVolumes(dataVolume.Namespace).Update(dataVolumeCopy)
// Emit the event only when the status change happens, not every time
if event.eventType != "" {
c.recorder.Event(dataVolume, event.eventType, event.reason, event.message)
}
return err
}
return nil
}
func newPvcFromSnapshot(snapshot *csisnapshotv1.VolumeSnapshot, dataVolume *cdiv1.DataVolume) *corev1.PersistentVolumeClaim {
labels := map[string]string{
"cdi-controller": snapshot.Name,
common.CDILabelKey: common.CDILabelValue,
common.CDIComponentLabel: common.SmartClonerCDILabel,
}
ownerRef := metav1.GetControllerOf(snapshot)
if ownerRef == nil {
return nil
}
annotations := make(map[string]string)
annotations[AnnSmartCloneRequest] = "true"
return &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: snapshot.Name,
Namespace: snapshot.Namespace,
Labels: labels,
Annotations: annotations,
OwnerReferences: []metav1.OwnerReference{*ownerRef},
},
Spec: corev1.PersistentVolumeClaimSpec{
DataSource: &corev1.TypedLocalObjectReference{
Name: snapshot.Name,
Kind: "VolumeSnapshot",
APIGroup: &csisnapshotv1.SchemeGroupVersion.Group,
},
VolumeMode: dataVolume.Spec.PVC.VolumeMode,
AccessModes: dataVolume.Spec.PVC.AccessModes,
StorageClassName: dataVolume.Spec.PVC.StorageClassName,
Resources: corev1.ResourceRequirements{
Requests: dataVolume.Spec.PVC.Resources.Requests,
},
},
}
}