Implement interruption-free update and rollback

During an update, where a resource must be replaced in its entirety, create
the replacement resource before deleting the old resource.

Also, allow rollback to the previous version of the resource without
replacing it, where that is possible.

Fixes bug #1176142

Change-Id: Id89654bad297815bdbcc86f666367772889b5df4

heat/engine/resource.py

@@ -602,6 +602,7 @@ def _store_or_update(self, action, status, reason):
                 rs.update_and_save({'action': self.action,
                                     'status': self.status,
                                     'status_reason': reason,
+                                    'stack_id': self.stack.id,
                                     'nova_instance': self.resource_id})
 
                 self.stack.updated_time = datetime.utcnow()

heat/engine/update.py

@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from heat.db import api as db_api
+
 from heat.engine import resource
 from heat.engine import scheduler
 
@@ -51,6 +53,10 @@ def __call__(self):
         existing_deps = self.existing_stack.dependencies
         new_deps = self.new_stack.dependencies
 
+        cleanup_prev = scheduler.DependencyTaskGroup(
+            self.previous_stack.dependencies,
+            self._remove_backup_resource,
+            reverse=True)
         cleanup = scheduler.DependencyTaskGroup(existing_deps,
                                                 self._remove_old_resource,
                                                 reverse=True)
@@ -59,13 +65,32 @@ def __call__(self):
         update = scheduler.DependencyTaskGroup(new_deps,
                                                self._update_resource)
 
-        yield cleanup()
+        if not self.rollback:
+            yield cleanup_prev()
+
         yield create_new()
-        yield update()
+        try:
+            yield update()
+        finally:
+            prev_deps = self.previous_stack._get_dependencies(
+                self.previous_stack.resources.itervalues())
+            self.previous_stack.dependencies = prev_deps
+        yield cleanup()
+
+    @scheduler.wrappertask
+    def _remove_backup_resource(self, prev_res):
+        if prev_res.state not in ((prev_res.INIT, prev_res.COMPLETE),
+                                  (prev_res.DELETE, prev_res.COMPLETE)):
+            logger.debug("Deleting backup resource %s" % prev_res.name)
+            yield prev_res.destroy()
 
     @scheduler.wrappertask
     def _remove_old_resource(self, existing_res):
         res_name = existing_res.name
+
+        if res_name in self.previous_stack:
+            yield self._remove_backup_resource(self.previous_stack[res_name])
+
         if res_name not in self.new_stack:
             logger.debug("resource %s not found in updated stack"
                          % res_name + " definition, deleting")
@@ -78,14 +103,45 @@ def _create_new_resource(self, new_res):
         if res_name not in self.existing_stack:
             logger.debug("resource %s not found in current stack"
                          % res_name + " definition, adding")
-            new_res.stack = self.existing_stack
-            self.existing_stack[res_name] = new_res
-            yield new_res.create()
+            yield self._create_resource(new_res)
+
+    @staticmethod
+    def _exchange_stacks(existing_res, prev_res):
+        db_api.resource_exchange_stacks(existing_res.stack.context,
+                                        existing_res.id, prev_res.id)
+        existing_res.stack, prev_res.stack = prev_res.stack, existing_res.stack
+        existing_res.stack[existing_res.name] = existing_res
+        prev_res.stack[prev_res.name] = prev_res
 
     @scheduler.wrappertask
-    def _replace_resource(self, new_res):
+    def _create_resource(self, new_res):
         res_name = new_res.name
-        yield self.existing_stack[res_name].destroy()
+
+        # Clean up previous resource
+        if res_name in self.previous_stack:
+            prev_res = self.previous_stack[res_name]
+
+            if prev_res.state not in ((prev_res.INIT, prev_res.COMPLETE),
+                                      (prev_res.DELETE, prev_res.COMPLETE)):
+                # Swap in the backup resource if it is in a valid state,
+                # instead of creating a new resource
+                if prev_res.status == prev_res.COMPLETE:
+                    logger.debug("Swapping in backup Resource %s" % res_name)
+                    self._exchange_stacks(self.existing_stack[res_name],
+                                          prev_res)
+                    return
+
+                logger.debug("Deleting backup Resource %s" % res_name)
+                yield prev_res.destroy()
+
+        # Back up existing resource
+        if res_name in self.existing_stack:
+            logger.debug("Backing up existing Resource %s" % res_name)
+            existing_res = self.existing_stack[res_name]
+            existing_res.stack = self.previous_stack
+            self.previous_stack[res_name] = existing_res
+            existing_res.state_set(existing_res.UPDATE, existing_res.COMPLETE)
+
         new_res.stack = self.existing_stack
         self.existing_stack[res_name] = new_res
         yield new_res.create()
@@ -108,7 +164,7 @@ def _update_resource(self, new_res):
                 yield self.existing_stack[res_name].update(new_snippet,
                                                            existing_snippet)
             except resource.UpdateReplace:
-                yield self._replace_resource(new_res)
+                yield self._create_resource(new_res)
             else:
                 logger.info("Resource %s for stack %s updated" %
                             (res_name, self.existing_stack.name))

heat/tests/test_parser.py

@@ -1179,10 +1179,9 @@ def test_update_rollback(self):
         # key/property in update_allowed_keys/update_allowed_properties
 
         # patch in a dummy handle_create making the replace fail when creating
-        # the replacement rsrc, but succeed the second call (rollback)
+        # the replacement rsrc
         self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_create')
         generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
-        generic_rsrc.ResourceWithProps.handle_create().AndReturn(None)
         self.m.ReplayAll()
 
         self.stack.update(updated_stack)
@@ -1217,8 +1216,9 @@ def test_update_rollback_fail(self):
         # patch in a dummy handle_create making the replace fail when creating
         # the replacement rsrc, and again on the second call (rollback)
         self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_create')
+        self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_delete')
         generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
-        generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
+        generic_rsrc.ResourceWithProps.handle_delete().AndRaise(Exception)
         self.m.ReplayAll()
 
         self.stack.update(updated_stack)
@@ -1289,6 +1289,40 @@ def test_update_rollback_remove(self):
         # Unset here so delete() is not stubbed for stack.delete cleanup
         self.m.UnsetStubs()
 
+    @utils.stack_delete_after
+    def test_update_rollback_replace(self):
+        tmpl = {'Resources': {
+                'AResource': {'Type': 'ResourceWithPropsType',
+                              'Properties': {'Foo': 'foo'}}}}
+
+        self.stack = parser.Stack(self.ctx, 'update_test_stack',
+                                  template.Template(tmpl),
+                                  disable_rollback=False)
+        self.stack.store()
+        self.stack.create()
+        self.assertEqual(self.stack.state,
+                         (parser.Stack.CREATE, parser.Stack.COMPLETE))
+
+        tmpl2 = {'Resources': {'AResource': {'Type': 'ResourceWithPropsType',
+                                             'Properties': {'Foo': 'bar'}}}}
+
+        updated_stack = parser.Stack(self.ctx, 'updated_stack',
+                                     template.Template(tmpl2))
+
+        # patch in a dummy delete making the destroy fail
+        self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_delete')
+        generic_rsrc.ResourceWithProps.handle_delete().AndRaise(Exception)
+        generic_rsrc.ResourceWithProps.handle_delete().AndReturn(None)
+        generic_rsrc.ResourceWithProps.handle_delete().AndReturn(None)
+        self.m.ReplayAll()
+
+        self.stack.update(updated_stack)
+        self.assertEqual(self.stack.state,
+                         (parser.Stack.ROLLBACK, parser.Stack.COMPLETE))
+        self.m.VerifyAll()
+        # Unset here so delete() is not stubbed for stack.delete cleanup
+        self.m.UnsetStubs()
+
     @utils.stack_delete_after
     def test_update_replace_by_reference(self):
         '''
@@ -1386,16 +1420,12 @@ def test_update_by_reference_and_rollback_1(self):
         # resource.UpdateReplace because we've not specified the modified
         # key/property in update_allowed_keys/update_allowed_properties
 
-        generic_rsrc.ResourceWithProps.FnGetRefId().AndReturn(
+        generic_rsrc.ResourceWithProps.FnGetRefId().MultipleTimes().AndReturn(
             'AResource')
 
         # mock to make the replace fail when creating the replacement resource
         generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
 
-        generic_rsrc.ResourceWithProps.handle_create().AndReturn(None)
-        generic_rsrc.ResourceWithProps.FnGetRefId().MultipleTimes().AndReturn(
-            'AResource')
-
         self.m.ReplayAll()
 
         updated_stack = parser.Stack(self.ctx, 'updated_stack',
@@ -1464,12 +1494,6 @@ def handle_create(self):
         # replacement resource
         generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
 
-        # Calls to ResourceWithProps.handle_update will raise
-        # resource.UpdateReplace because we've not specified the modified
-        # key/property in update_allowed_keys/update_allowed_properties
-
-        generic_rsrc.ResourceWithProps.handle_create().AndReturn(None)
-
         self.m.ReplayAll()
 
         updated_stack = parser.Stack(self.ctx, 'updated_stack',
@@ -1480,7 +1504,7 @@ def handle_create(self):
                          (parser.Stack.ROLLBACK, parser.Stack.COMPLETE))
         self.assertEqual(self.stack['AResource'].properties['Foo'], 'abc')
         self.assertEqual(self.stack['BResource'].properties['Foo'],
-                         'AResource3')
+                         'AResource1')
 
         self.m.VerifyAll()