Permalink
May 26, 2014
Nov 8, 2013
Nov 8, 2013
May 19, 2014
Oct 18, 2016
Oct 18, 2016
Dec 5, 2016
Oct 18, 2016
Nov 8, 2013
Oct 18, 2016
Nov 8, 2013
Nov 8, 2013
Nov 8, 2013
Nov 8, 2013
Jul 21, 2014
Nov 8, 2013
Jul 21, 2014
Nov 8, 2013
Oct 18, 2016
Jan 9, 2017
May 19, 2014
Feb 17, 2015
Jun 11, 2015
Oct 18, 2016
Dec 5, 2016
Nov 8, 2013
Oct 18, 2016
Nov 8, 2013
Nov 8, 2013
Nov 8, 2013
Nov 8, 2013
Oct 18, 2016
Oct 18, 2016
Nov 8, 2013
Nov 11, 2015
Oct 18, 2016
Oct 18, 2016
Oct 18, 2016
Nov 11, 2015
Oct 18, 2016
Dec 5, 2016
Mar 2, 2017
Dec 5, 2016
Dec 5, 2016
Jan 9, 2017
Dec 5, 2016
Oct 18, 2016
Jan 9, 2017
Oct 19, 2016
Dec 5, 2016
Nov 8, 2013
Oct 18, 2016
May 12, 2015
May 15, 2014
May 12, 2015
May 15, 2014
May 15, 2014
May 12, 2015
Oct 18, 2016
May 12, 2015
Oct 18, 2016
May 12, 2015
May 26, 2014
Oct 18, 2016
Nov 23, 2016
Oct 18, 2016
Jun 11, 2015
May 19, 2014
Nov 8, 2013
Nov 8, 2013
Nov 8, 2013
Nov 11, 2013
Nov 8, 2013
Nov 11, 2013
Nov 8, 2013
Nov 11, 2013
Sep 15, 2014
Dec 10, 2013
Nov 8, 2013
May 19, 2014
Nov 8, 2013
Nov 8, 2013
Nov 8, 2013
Dec 10, 2013
Nov 8, 2013
Jun 11, 2015
Feb 27, 2017
Jun 11, 2015
Feb 27, 2017
Jun 11, 2015
Nov 8, 2013
Nov 8, 2013
Nov 11, 2013
Nov 8, 2013
Nov 8, 2013
Nov 8, 2013
Nov 8, 2013
Oct 18, 2016
Nov 8, 2013
Feb 17, 2015
Jun 11, 2015
Feb 17, 2015
Feb 17, 2015
Jun 11, 2015
Jun 11, 2015
Jun 11, 2015
Jun 15, 2015
Jun 11, 2015
Jun 11, 2015
Newer
100644
711 lines (664 sloc)
22.6 KB
1
// Copyright 2014 Canonical Ltd.
2
// Licensed under the AGPLv3, see LICENCE file for details.
3
4
package state
5
6
import (
7
"fmt"
8
15
)
16
17
type cleanupKind string
18
19
const (
20
// SCHEMACHANGE: the names are expressive, the values not so much.
21
cleanupRelationSettings cleanupKind = "settings"
22
cleanupUnitsForDyingApplication cleanupKind = "units"
23
cleanupCharm cleanupKind = "charm"
24
cleanupDyingUnit cleanupKind = "dyingUnit"
25
cleanupRemovedUnit cleanupKind = "removedUnit"
27
cleanupDyingMachine cleanupKind = "dyingMachine"
28
cleanupForceDestroyedMachine cleanupKind = "machine"
29
cleanupAttachmentsForDyingStorage cleanupKind = "storageAttachments"
30
cleanupAttachmentsForDyingVolume cleanupKind = "volumeAttachments"
31
cleanupAttachmentsForDyingFilesystem cleanupKind = "filesystemAttachments"
32
cleanupModelsForDyingController cleanupKind = "models"
33
cleanupMachinesForDyingModel cleanupKind = "modelMachines"
34
cleanupVolumesForDyingModel cleanupKind = "modelVolumes"
35
cleanupFilesystemsForDyingModel cleanupKind = "modelFilesystems"
38
// cleanupDoc originally represented a set of documents that should be
39
// removed, but the Prefix field no longer means anything more than
40
// "what will be passed to the cleanup func".
41
type cleanupDoc struct {
42
DocID string `bson:"_id"`
43
Kind cleanupKind `bson:"kind"`
44
Prefix string `bson:"prefix"`
45
}
46
47
// newCleanupOp returns a txn.Op that creates a cleanup document with a unique
48
// id and the supplied kind and prefix.
50
doc := &cleanupDoc{
54
}
55
return txn.Op{
58
Insert: doc,
59
}
60
}
61
62
// NeedsCleanup returns true if documents previously marked for removal exist.
63
func (st *State) NeedsCleanup() (bool, error) {
64
cleanups, closer := st.getCollection(cleanupsC)
65
defer closer()
66
count, err := cleanups.Count()
67
if err != nil {
68
return false, err
69
}
70
return count > 0, nil
71
}
72
73
// Cleanup removes all documents that were previously marked for removal, if
74
// any such exist. It should be called periodically by at least one element
75
// of the system.
78
cleanups, closer := st.getCollection(cleanupsC)
79
defer closer()
80
iter := cleanups.Find(nil).Iter()
82
for iter.Next(&doc) {
83
var err error
84
logger.Debugf("running %q cleanup: %q", doc.Kind, doc.Prefix)
85
switch doc.Kind {
88
case cleanupCharm:
89
err = st.cleanupCharm(doc.Prefix)
90
case cleanupUnitsForDyingApplication:
96
case cleanupApplicationsForDyingModel:
97
err = st.cleanupApplicationsForDyingModel()
100
case cleanupForceDestroyedMachine:
101
err = st.cleanupForceDestroyedMachine(doc.Prefix)
102
case cleanupAttachmentsForDyingStorage:
103
err = st.cleanupAttachmentsForDyingStorage(doc.Prefix)
104
case cleanupAttachmentsForDyingVolume:
105
err = st.cleanupAttachmentsForDyingVolume(doc.Prefix)
106
case cleanupAttachmentsForDyingFilesystem:
107
err = st.cleanupAttachmentsForDyingFilesystem(doc.Prefix)
108
case cleanupModelsForDyingController:
109
err = st.cleanupModelsForDyingController()
110
case cleanupMachinesForDyingModel:
111
err = st.cleanupMachinesForDyingModel()
112
case cleanupVolumesForDyingModel:
113
err = st.cleanupVolumesForDyingModel()
114
case cleanupFilesystemsForDyingModel:
115
err = st.cleanupFilesystemsForDyingModel()
116
default:
117
handler, ok := cleanupHandlers[doc.Kind]
118
if !ok {
119
err = errors.Errorf("unknown cleanup kind %q", doc.Kind)
120
} else {
121
persist := st.newPersistence()
122
err = handler(st, persist, doc.Prefix)
123
}
124
}
125
if err != nil {
127
continue
128
}
129
ops := []txn.Op{{
132
Remove: true,
133
}}
134
if err := st.runTransaction(ops); err != nil {
136
}
137
}
138
return nil
139
}
140
141
// CleanupHandler is a function that state may call during cleanup
142
// to perform cleanup actions for some cleanup type.
143
type CleanupHandler func(st *State, persist Persistence, prefix string) error
144
145
var cleanupHandlers = map[cleanupKind]CleanupHandler{}
146
147
// RegisterCleanupHandler identifies the handler to use a given
148
// cleanup kind.
149
func RegisterCleanupHandler(kindStr string, handler CleanupHandler) error {
150
kind := cleanupKind(kindStr)
151
if _, ok := cleanupHandlers[kind]; ok {
152
return errors.NewAlreadyExists(nil, fmt.Sprintf("cleanup handler for %q already registered", kindStr))
153
}
154
cleanupHandlers[kind] = handler
155
return nil
156
}
157
159
change := relationSettingsCleanupChange{Prefix: st.docID(prefix)}
160
if err := Apply(st.database, change); err != nil {
161
return errors.Trace(err)
162
}
163
return nil
164
}
165
167
// they are not already Dying or Dead. It's expected to be used when a
168
// controller is destroyed.
169
func (st *State) cleanupModelsForDyingController() (err error) {
170
models, err := st.AllModels()
174
for _, model := range models {
175
if err := model.Destroy(); err != nil {
176
return errors.Trace(err)
182
// cleanupMachinesForDyingModel sets all non-manager machines to Dying,
183
// if they are not already Dying or Dead. It's expected to be used when
184
// a model is destroyed.
185
func (st *State) cleanupMachinesForDyingModel() (err error) {
186
// This won't miss machines, because a Dying model cannot have
187
// machines added to it. But we do have to remove the machines themselves
188
// via individual transactions, because they could be in any state at all.
189
machines, err := st.AllMachines()
190
if err != nil {
191
return errors.Trace(err)
192
}
193
for _, m := range machines {
194
if m.IsManager() {
195
continue
196
}
197
if _, isContainer := m.ParentId(); isContainer {
198
continue
199
}
200
manual, err := m.IsManual()
201
if err != nil {
204
destroy := m.ForceDestroy
205
if manual {
206
// Manually added machines should never be force-
207
// destroyed automatically. That should be a user-
208
// driven decision, since it may leak applications
209
// and resources on the machine. If something is
210
// stuck, then the user can still force-destroy
211
// the manual machines.
212
destroy = m.Destroy
213
}
214
if err := destroy(); err != nil {
221
// cleanupVolumesForDyingModel sets all persistent volumes to Dying,
222
// if they are not already Dying or Dead. It's expected to be used when
223
// a model is destroyed.
224
func (st *State) cleanupVolumesForDyingModel() (err error) {
225
volumes, err := st.AllVolumes()
226
if err != nil {
227
return errors.Trace(err)
228
}
229
for _, v := range volumes {
230
err := st.DestroyVolume(v.VolumeTag())
231
if errors.IsNotFound(err) {
232
continue
233
} else if IsContainsFilesystem(err) {
234
continue
235
} else if err != nil {
236
return errors.Trace(err)
237
}
238
}
239
return nil
240
}
241
242
// cleanupFilesystemsForDyingModel sets all persistent filesystems to
243
// Dying, if they are not already Dying or Dead. It's expected to be used
244
// when a model is destroyed.
245
func (st *State) cleanupFilesystemsForDyingModel() (err error) {
246
filesystems, err := st.AllFilesystems()
247
if err != nil {
248
return errors.Trace(err)
249
}
250
for _, fs := range filesystems {
251
err := st.DestroyFilesystem(fs.FilesystemTag())
255
return errors.Trace(err)
256
}
257
}
258
return nil
259
}
260
261
// cleanupApplicationsForDyingModel sets all applications to Dying, if they are
264
func (st *State) cleanupApplicationsForDyingModel() (err error) {
265
if err := st.removeRemoteApplicationsForDyingModel(); err != nil {
266
return err
267
}
268
return st.removeApplicationsForDyingModel()
269
}
270
271
func (st *State) removeApplicationsForDyingModel() (err error) {
272
// This won't miss applications, because a Dying model cannot have
273
// applications added to it. But we do have to remove the applications
274
// themselves via individual transactions, because they could be in any
275
// state at all.
282
for iter.Next(&application.doc) {
283
if err := application.Destroy(); err != nil {
284
return errors.Trace(err)
285
}
286
}
287
return nil
288
}
289
290
func (st *State) removeRemoteApplicationsForDyingModel() (err error) {
291
// This won't miss remote applications, because a Dying model cannot have
292
// applications added to it. But we do have to remove the applications themselves
293
// via individual transactions, because they could be in any state at all.
294
remoteApps, closer := st.getCollection(remoteApplicationsC)
295
defer closer()
296
remoteApp := RemoteApplication{st: st}
297
sel := bson.D{{"life", Alive}}
298
iter := remoteApps.Find(sel).Iter()
299
defer closeIter(iter, &err, "reading remote application document")
300
for iter.Next(&remoteApp.doc) {
301
if err := remoteApp.Destroy(); err != nil {
302
return errors.Trace(err)
311
func (st *State) cleanupUnitsForDyingApplication(applicationname string) (err error) {
312
// This won't miss units, because a Dying application cannot have units
313
// added to it. But we do have to remove the units themselves via
314
// individual transactions, because they could be in any state at all.
322
for iter.Next(&unit.doc) {
323
if err := unit.Destroy(); err != nil {
324
return err
325
}
326
}
327
return nil
328
}
329
330
// cleanupCharm is speculative: it can abort without error for many
331
// reasons, because it's triggered somewhat overenthusiastically for
332
// simplicity's sake.
333
func (st *State) cleanupCharm(charmURL string) error {
334
curl, err := charm.ParseURL(charmURL)
335
if err != nil {
336
return errors.Annotatef(err, "invalid charm URL %v", charmURL)
337
}
338
339
ch, err := st.Charm(curl)
340
if errors.IsNotFound(err) {
341
// Charm already removed.
342
return nil
343
} else if err != nil {
344
return errors.Annotate(err, "reading charm")
345
}
346
347
err = ch.Destroy()
348
switch errors.Cause(err) {
349
case nil:
350
case errCharmInUse:
351
// No cleanup necessary at this time.
352
return nil
353
default:
354
return errors.Annotate(err, "destroying charm")
355
}
356
357
if err := ch.Remove(); err != nil {
358
return errors.Trace(err)
359
}
360
return nil
361
}
362
363
// cleanupDyingUnit marks resources owned by the unit as dying, to ensure
364
// they are cleaned up as well.
365
func (st *State) cleanupDyingUnit(name string) error {
366
unit, err := st.Unit(name)
367
if errors.IsNotFound(err) {
368
return nil
369
} else if err != nil {
370
return err
371
}
372
// Mark the unit as departing from its joined relations, allowing
373
// related units to start converging to a state in which that unit
374
// is gone as quickly as possible.
376
if err != nil {
377
return err
378
}
379
for _, relation := range relations {
380
relationUnit, err := relation.Unit(unit)
381
if errors.IsNotFound(err) {
383
} else if err != nil {
384
return err
385
}
386
if err := relationUnit.PrepareLeaveScope(); err != nil {
387
return err
388
}
389
}
390
// Mark storage attachments as dying, so that they are detached
391
// and removed from state, allowing the unit to terminate.
392
return st.cleanupUnitStorageAttachments(unit.UnitTag(), false)
393
}
394
395
func (st *State) cleanupUnitStorageAttachments(unitTag names.UnitTag, remove bool) error {
396
storageAttachments, err := st.UnitStorageAttachments(unitTag)
397
if err != nil {
398
return err
399
}
400
for _, storageAttachment := range storageAttachments {
403
if errors.IsNotFound(err) {
404
continue
405
} else if err != nil {
406
return err
407
}
408
if !remove {
409
continue
410
}
411
err = st.RemoveStorageAttachment(storageTag, unitTag)
412
if errors.IsNotFound(err) {
413
continue
414
} else if err != nil {
415
return err
416
}
417
}
421
// cleanupRemovedUnit takes care of all the final cleanup required when
422
// a unit is removed.
426
return errors.Trace(err)
427
}
428
cancelled := ActionResults{
429
Status: ActionCancelled,
430
Message: "unit removed",
433
switch action.Status() {
434
case ActionCompleted, ActionCancelled, ActionFailed:
435
// nothing to do here
436
default:
437
if _, err = action.Finish(cancelled); err != nil {
438
return errors.Trace(err)
439
}
442
443
change := payloadCleanupChange{
444
Unit: unitId,
445
}
446
if err := Apply(st.database, change); err != nil {
447
return errors.Trace(err)
448
}
452
// cleanupDyingMachine marks resources owned by the machine as dying, to ensure
453
// they are cleaned up as well.
454
func (st *State) cleanupDyingMachine(machineId string) error {
455
machine, err := st.Machine(machineId)
456
if errors.IsNotFound(err) {
457
return nil
458
} else if err != nil {
459
return err
460
}
461
return cleanupDyingMachineResources(machine)
462
}
463
464
// cleanupForceDestroyedMachine systematically destroys and removes all entities
465
// that depend upon the supplied machine, and removes the machine from state. It's
466
// expected to be used in response to destroy-machine --force.
468
machine, err := st.Machine(machineId)
470
return nil
471
} else if err != nil {
472
return err
473
}
474
// In an ideal world, we'd call machine.Destroy() here, and thus prevent
475
// new dependencies being added while we clean up the ones we know about.
476
// But machine destruction is unsophisticated, and doesn't allow for
477
// destruction while dependencies exist; so we just have to deal with that
478
// possibility below.
479
if err := st.cleanupContainers(machine); err != nil {
480
return err
481
}
482
for _, unitName := range machine.doc.Principals {
483
if err := st.obliterateUnit(unitName); err != nil {
484
return err
485
}
486
}
490
// We need to refresh the machine at this point, because the local copy
491
// of the document will not reflect changes caused by the unit cleanups
492
// above, and may thus fail immediately.
496
return err
497
}
498
// TODO(fwereade): 2013-11-11 bug 1250104
499
// If this fails, it's *probably* due to a race in which new dependencies
500
// were added while we cleaned up the old ones. If the cleanup doesn't run
501
// again -- which it *probably* will anyway -- the issue can be resolved by
502
// force-destroying the machine again; that's better than adding layer
503
// upon layer of complication here.
504
if err := machine.EnsureDead(); err != nil {
505
return err
506
}
507
removePortsOps, err := machine.removePortsOps()
508
if err != nil {
509
return err
510
}
511
return st.runTransaction(removePortsOps)
512
513
// Note that we do *not* remove the machine entirely: we leave it for the
514
// provisioner to clean up, so that we don't end up with an unreferenced
515
// instance that would otherwise be ignored when in provisioner-safe-mode.
516
}
517
518
// cleanupContainers recursively calls cleanupForceDestroyedMachine on the supplied
520
func (st *State) cleanupContainers(machine *Machine) error {
521
containerIds, err := machine.Containers()
523
return nil
524
} else if err != nil {
525
return err
526
}
527
for _, containerId := range containerIds {
529
return err
530
}
533
return nil
534
} else if err != nil {
535
return err
536
}
537
if err := container.Remove(); err != nil {
538
return err
539
}
540
}
541
return nil
542
}
543
544
func cleanupDyingMachineResources(m *Machine) error {
545
volumeAttachments, err := m.st.MachineVolumeAttachments(m.MachineTag())
546
if err != nil {
547
return errors.Annotate(err, "getting machine volume attachments")
548
}
549
for _, va := range volumeAttachments {
550
if detachable, err := isDetachableVolumeTag(m.st, va.Volume()); err != nil {
551
return errors.Trace(err)
552
} else if !detachable {
553
// Non-detachable volumes will be removed along with the machine.
554
continue
555
}
556
if err := m.st.DetachVolume(va.Machine(), va.Volume()); err != nil {
557
if IsContainsFilesystem(err) {
558
// The volume will be destroyed when the
559
// contained filesystem is removed, whose
560
// destruction is initiated below.
561
continue
562
}
563
return errors.Trace(err)
564
}
565
}
566
filesystemAttachments, err := m.st.MachineFilesystemAttachments(m.MachineTag())
567
if err != nil {
568
return errors.Annotate(err, "getting machine filesystem attachments")
569
}
570
for _, fsa := range filesystemAttachments {
571
if detachable, err := isDetachableFilesystemTag(m.st, fsa.Filesystem()); err != nil {
572
return errors.Trace(err)
573
} else if !detachable {
574
// Non-detachable filesystems will be removed along with the machine.
575
continue
576
}
577
if err := m.st.DetachFilesystem(fsa.Machine(), fsa.Filesystem()); err != nil {
578
return errors.Trace(err)
579
}
580
}
581
return nil
582
}
583
584
// obliterateUnit removes a unit from state completely. It is not safe or
585
// sane to obliterate any unit in isolation; its only reasonable use is in
586
// the context of machine obliteration, in which we can be sure that unclean
587
// shutdown of units is not going to leave a machine in a difficult state.
588
func (st *State) obliterateUnit(unitName string) error {
589
unit, err := st.Unit(unitName)
591
return nil
592
} else if err != nil {
593
return err
594
}
595
// Unlike the machine, we *can* always destroy the unit, and (at least)
596
// prevent further dependencies being added. If we're really lucky, the
597
// unit will be removed immediately.
598
if err := unit.Destroy(); err != nil {
600
}
602
return nil
605
}
606
// Destroy and remove all storage attachments for the unit.
607
if err := st.cleanupUnitStorageAttachments(unit.UnitTag(), true); err != nil {
608
return errors.Annotatef(err, "cannot destroy storage for unit %q", unitName)
609
}
610
for _, subName := range unit.SubordinateNames() {
611
if err := st.obliterateUnit(subName); err != nil {
612
return err
613
}
614
}
615
if err := unit.EnsureDead(); err != nil {
616
return err
617
}
618
return unit.Remove()
619
}
620
621
// cleanupAttachmentsForDyingStorage sets all storage attachments related
622
// to the specified storage instance to Dying, if they are not already Dying
623
// or Dead. It's expected to be used when a storage instance is destroyed.
627
// This won't miss attachments, because a Dying storage instance cannot
628
// have attachments added to it. But we do have to remove the attachments
629
// themselves via individual transactions, because they could be in
630
// any state at all.
631
coll, closer := st.getCollection(storageAttachmentsC)
632
defer closer()
633
634
var doc storageAttachmentDoc
635
fields := bson.D{{"unitid", 1}}
641
return errors.Annotate(err, "destroying storage attachment")
642
}
643
}
644
return nil
645
}
646
647
// cleanupAttachmentsForDyingVolume sets all volume attachments related
648
// to the specified volume to Dying, if they are not already Dying or
649
// Dead. It's expected to be used when a volume is destroyed.
651
volumeTag := names.NewVolumeTag(volumeId)
652
653
// This won't miss attachments, because a Dying volume cannot have
654
// attachments added to it. But we do have to remove the attachments
655
// themselves via individual transactions, because they could be in
656
// any state at all.
657
coll, closer := st.getCollection(volumeAttachmentsC)
658
defer closer()
659
660
var doc volumeAttachmentDoc
661
fields := bson.D{{"machineid", 1}}
662
iter := coll.Find(bson.D{{"volumeid", volumeId}}).Select(fields).Iter()
664
for iter.Next(&doc) {
665
machineTag := names.NewMachineTag(doc.Machine)
666
if err := st.DetachVolume(machineTag, volumeTag); err != nil {
667
return errors.Annotate(err, "destroying volume attachment")
668
}
669
}
670
return nil
671
}
672
673
// cleanupAttachmentsForDyingFilesystem sets all filesystem attachments related
674
// to the specified filesystem to Dying, if they are not already Dying or
675
// Dead. It's expected to be used when a filesystem is destroyed.
676
func (st *State) cleanupAttachmentsForDyingFilesystem(filesystemId string) (err error) {
677
filesystemTag := names.NewFilesystemTag(filesystemId)
678
679
// This won't miss attachments, because a Dying filesystem cannot have
680
// attachments added to it. But we do have to remove the attachments
681
// themselves via individual transactions, because they could be in
682
// any state at all.
683
coll, closer := st.getCollection(filesystemAttachmentsC)
684
defer closer()
685
686
var doc filesystemAttachmentDoc
687
fields := bson.D{{"machineid", 1}}
688
iter := coll.Find(bson.D{{"filesystemid", filesystemId}}).Select(fields).Iter()
690
for iter.Next(&doc) {
691
machineTag := names.NewMachineTag(doc.Machine)
692
if err := st.DetachFilesystem(machineTag, filesystemTag); err != nil {
693
return errors.Annotate(err, "destroying filesystem attachment")
694
}
695
}
696
return nil
697
}