Shutdown VM if disk cannot be removed from running VM

Before VM was (always) shutdown only if IDE CD-ROM was removed.
Now VM will be shutdown if disk removal from running VM fails.

Fixes #249

Signed-off-by: Justin Cinkelj <justin.cinkelj@xlab.si>

Author: justinc1

plugins/module_utils/disk.py

@@ -198,9 +198,10 @@ def needs_reboot(self, action: str, desired_disk=None) -> bool:
         # Delete and change type.
         if desired_disk and action == "update" and self.type != desired_disk.type:
             return True
-        if (
-            action == "delete" and self.type == "ide_cdrom"
-        ):  # ide_cdrom can never be deleted when VM is running.
+        if action == "delete" and self.type == "ide_cdrom":
+            # ide_cdrom can never be deleted when VM is running.
+            # Also other disks types cannot be deleted when VM is running
+            # if HyperCore thinks disk is being "used".
             return True
         return False
 

plugins/module_utils/errors.py

@@ -7,6 +7,7 @@
 
 __metaclass__ = type
 
+import json
 from typing import Union
 from ansible.module_utils.urls import Request
 
@@ -111,3 +112,13 @@ class ScaleTimeoutError(ScaleComputingError):
     def __init__(self, data: Union[str, Exception]):
         self.message = f"Request timed out: {data}."
         super(ScaleTimeoutError, self).__init__(self.message)
+
+
+class TaskTagError(ScaleComputingError):
+    def __init__(self, task_tag: dict):
+        message = "There was a problem during this task execution."
+        message += f" Task details: {json.dumps(task_tag)}"
+        self.message = message
+        self.task_tag_state = task_tag["state"]
+        self.task_tag = task_tag
+        super().__init__(self.message)

plugins/module_utils/task_tag.py

@@ -9,7 +9,6 @@
 
 __metaclass__ = type
 
-import json
 from time import sleep
 
 from ..module_utils import errors
@@ -47,9 +46,7 @@ def wait_task(
                 "ERROR",
                 "UNINITIALIZED",
             ):  # TaskTag has finished unsucessfully or was never initialized, both are errors.
-                msg = "There was a problem during this task execution."
-                msg += f" Task details: {json.dumps(task_status)}"
-                raise errors.ScaleComputingError(msg)
+                raise errors.TaskTagError(task_status)
             if task_status.get("state", "") not in (
                 "RUNNING",
                 "QUEUED",

plugins/module_utils/vm.py

@@ -1091,6 +1091,10 @@ def _delete_not_used_disks(cls, module, rest_client, vm, changed, disk_key):
                 ):
                     to_delete = False
             if to_delete:
+                # HyperCore is sometimes able to delete disk on running VM,
+                # but sometimes we need to shutdown VM to remove disk.
+                # It is hard to know in advance if shutdown is required.
+                # We try to remove disk without shutdown, if delete fails, we shutdown VM and try again.
                 if existing_disk.needs_reboot("delete"):
                     vm.do_shutdown_steps(module, rest_client)
                 task_tag = rest_client.delete_record(
@@ -1099,7 +1103,33 @@ def _delete_not_used_disks(cls, module, rest_client, vm, changed, disk_key):
                     ),
                     module.check_mode,
                 )
-                TaskTag.wait_task(rest_client, task_tag, module.check_mode)
+                try:
+                    TaskTag.wait_task(rest_client, task_tag, module.check_mode)
+                except errors.TaskTagError as ex:
+                    # Delete failed, maybe because VM was running and disk was in use.
+                    # If VM is running, shutdown VM and retry delete.
+                    if ex.task_tag_state != "ERROR":
+                        raise
+                    # The particular "formattedMessage" is returned by HyperCore 9.2.17
+                    if (
+                        ex.task_tag["formattedMessage"]
+                        != "Unable to delete block device from VM '%@': Still in use"
+                    ):
+                        raise
+                    vm_fresh_data = rest_client.get_record(
+                        f"/rest/v1/VirDomain/{vm.uuid}", must_exist=True
+                    )
+                    if vm_fresh_data["state"] != "RUNNING":
+                        raise
+                    # shutdown and retry remove
+                    vm.do_shutdown_steps(module, rest_client)
+                    task_tag = rest_client.delete_record(
+                        "{0}/{1}".format(
+                            "/rest/v1/VirDomainBlockDevice", existing_disk.uuid
+                        ),
+                        module.check_mode,
+                    )
+                    TaskTag.wait_task(rest_client, task_tag, module.check_mode)
                 changed = True
         return changed
 

plugins/modules/vm.py

@@ -18,6 +18,13 @@
 description:
   - Use this module to create, update or delete a VM. When creating or
     updating a VM, setting the disks, network nics and boot order is possible.
+
+  - Module tries to remove disks from a running VM.
+    If disk cannot be removed from running VM,
+    then VM will be shutdown, disk will be removed, and VM is started back.
+  - VM has C(shutdown_timeout) time to respond to shutdown request.
+    If VM is not shutoff within I(shutdown_timeout),
+    then a force shutdown will be issued if C(force_reboot=True).
 version_added: 1.0.0
 extends_documentation_fragment:
   - scale_computing.hypercore.cluster_instance

plugins/modules/vm_disk.py

@@ -15,21 +15,28 @@
   - Tjaž Eržen (@tjazsch)
 short_description: Manage VM's disks
 description:
-  Use this module to add, delete or set disks to the VM.
-  The module can also remove all disks from a VM,
-  attach and/or detach ISO image to the VM by ISO's name,
-  detach ISO image from the VM by disk's disk slot,
-  or update the existing disks (disk size etc.).
-
-  For a given VM, a particular disk is selected by combination of (I(type), I(disk_slot)).
-  I(disk_slot) means slot on bus (IDE, virtio or SCSI bus).
-
-  Changing disk I(type) can change its I(disk_slot).
-  For example, VM has one IDE CD-ROM and one virtio_disk.
-  The disk will have C(type=virtio_disk) and C(disk_slot=0),
-  and CD-ROM will have C(type=ide_cdrom) and C(disk_slot=0).
-  Changing disk I(type) to C(ide_disk) will as place disk on IDE bus,
-  after the CD-ROM, and disk will get C(disk_slot=1).
+  - Use this module to add, delete or set disks to the VM.
+    The module can also remove all disks from a VM,
+    attach and/or detach ISO image to the VM by ISO's name,
+    detach ISO image from the VM by disk's disk slot,
+    or update the existing disks (disk size etc.).
+
+  - For a given VM, a particular disk is selected by combination of (I(type), I(disk_slot)).
+    I(disk_slot) means slot on bus (IDE, virtio or SCSI bus).
+
+  - Changing disk I(type) can change its I(disk_slot).
+    For example, VM has one IDE CD-ROM and one virtio_disk.
+    The disk will have C(type=virtio_disk) and C(disk_slot=0),
+    and CD-ROM will have C(type=ide_cdrom) and C(disk_slot=0).
+    Changing disk I(type) to C(ide_disk) will as place disk on IDE bus,
+    after the CD-ROM, and disk will get C(disk_slot=1).
+
+  - Module tries to remove disks from a running VM.
+    If disk cannot be removed from running VM,
+    then VM will be shutdown, disk will be removed, and VM is started back.
+  - VM has C(shutdown_timeout) time to respond to shutdown request.
+    If VM is not shutoff within I(shutdown_timeout),
+    then a force shutdown will be issued if C(force_reboot=True).
 
 version_added: 1.0.0
 extends_documentation_fragment: