[root@ol84-acs-mgnt-01 ~]# grep 'job-36272' /var/log/cloudstack/management/management-server.log
2023-05-03 14:27:33,404 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-103:ctx-019711bb job-36271 ctx-df7a4e9c) (logid:6b610288) Sync job-36272 execution on object VmWorkJobQueue.3052
2023-05-03 14:27:35,195 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (AsyncJobMgr-Heartbeat-1:ctx-957b8383) (logid:149ffcda) Schedule queued job-36272
2023-05-03 14:27:35,201 INFO  [o.a.c.f.j.i.AsyncJobMonitor] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:d2edc658) Add job-36272 into job monitoring
2023-05-03 14:27:35,207 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Executing AsyncJobVO: {id:36272, userId: 63, accountId: 60, instanceType: null, instanceId: null, cmd: com.cloud.vm.VmWorkMigrate, cmdInfo: rO0ABXNyABpjb20uY2xvdWQudm0uVm1Xb3JrTWlncmF0ZRdxQXtPtzYqAgAGSgAJc3JjSG9zdElkTAAJY2x1c3RlcklkdAAQTGphdmEvbGFuZy9Mb25nO0wABmhvc3RJZHEAfgABTAAFcG9kSWRxAH4AAUwAB3N0b3JhZ2V0AA9MamF2YS91dGlsL01hcDtMAAZ6b25lSWRxAH4AAXhyABNjb20uY2xvdWQudm0uVm1Xb3Jrn5m2VvAlZ2sCAARKAAlhY2NvdW50SWRKAAZ1c2VySWRKAAR2bUlkTAALaGFuZGxlck5hbWV0ABJMamF2YS9sYW5nL1N0cmluZzt4cAAAAAAAAAA8AAAAAAAAAD8AAAAAAAAL7HQAGVZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwAAAAAAAAAKHNyAA5qYXZhLmxhbmcuTG9uZzuL5JDMjyPfAgABSgAFdmFsdWV4cgAQamF2YS5sYW5nLk51bWJlcoaslR0LlOCLAgAAeHAAAAAAAAAACXNxAH4ABwAAAAAAAAAmc3EAfgAHAAAAAAAAAAZwc3EAfgAHAAAAAAAAAAU, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 108597816265214, completeMsid: null, lastUpdated: null, lastPolled: null, created: Wed May 03 14:27:33 MSK 2023, removed: null}
2023-05-03 14:27:35,207 DEBUG [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Run VM work job: com.cloud.vm.VmWorkMigrate for VM 3052, job origin: 36271
2023-05-03 14:27:35,209 DEBUG [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Execute VM work job: com.cloud.vm.VmWorkMigrate{"zoneId":5,"podId":6,"clusterId":9,"hostId":38,"srcHostId":40,"userId":63,"accountId":60,"vmId":3052,"handlerName":"VirtualMachineManagerImpl"}
2023-05-03 14:27:35,217 INFO  [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Migrating VM instance {"id":3052,"instanceName":"i-60-3052-VM","type":"User","uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11"} to Dest[Zone(Id)-Pod(Id)-Cluster(Id)-Host(Id)-Storage(Volume(Id|Type-->Pool(Id))] : Dest[Zone(5)-Pod(6)-Cluster(9)-Host(38)-Storage()]
2023-05-03 14:27:35,232 DEBUG [c.c.n.NetworkModelImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Service SecurityGroup is not supported in the network id=829
2023-05-03 14:27:35,250 DEBUG [c.c.n.NetworkModelImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Service SecurityGroup is not supported in the network id=829
2023-05-03 14:27:35,261 DEBUG [o.a.c.e.o.VolumeOrchestrator] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Preparing to migrate [1] volumes for VM [VM instance {"id":3052,"instanceName":"i-60-3052-VM","type":"User","uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11"}].
2023-05-03 14:27:35,314 DEBUG [c.c.h.g.VmwareVmImplementer] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Due to 'FALSE'(globalNestedVirtualisationEnabled) and 'FALSE'(globalNestedVPerVMEnabled) I'm adding a flag with value FALSE to the vm configuration for Nested Virtualisation.
2023-05-03 14:27:35,327 DEBUG [c.c.a.t.Request] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Seq 38-9188750614719369077: Sending  { Cmd , MgmtId: 108597816265214, via: 38(esx-sr184-u0910.aisrp.local), Ver: v1, Flags: 100111, [{"com.cloud.agent.api.PrepareForMigrationCommand":{"vm":{"id":"3052","name":"i-60-3052-VM","bootloader":"HVM","state":"Running","type":"User","cpus":"12","minSpeed":"1000","maxSpeed":"1000","minRam":"(12.00 GB) 12884901888","maxRam":"(12.00 GB) 12884901888","hostName":"for-transfer-3","arch":"x86_64","os":"CentOS 8","platformEmulator":"centos8_64Guest","bootArgs":"","enableHA":"true","limitCpuUse":"true","enableDynamicallyScaleVm":"true","params":{"cpuNumber":"12","memory":"12288","memoryOvercommitRatio":"1.0","Message.ReservedCapacityFreed.Flag":"false","nestedVirtualizationFlag":"false","vmware.reserve.cpu":"false","dataDiskController":"scsi","cpuOvercommitRatio":"3.0","rootdisksize":"50","vmware.reserve.mem":"false","nicAdapter":"Vmxnet3","rootDiskController":"scsi"},"uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11","enterHardwareSetup":"false","disks":[{"data":{"org.apache.cloudstack.storage.to.VolumeObjectTO":{"uuid":"f8d89f88-d681-42e9-997a-9f1d93dba76a","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"ad1a78a8-0d75-3b0f-973c-05f80c990691","name":"ds-sr183_185_os5500-ssd-1","id":"27","poolType":"PreSetup","host":"vcsa-cs.aisrp.local","path":"/dc1/ds-sr183_185_os5500-ssd-1","port":"0","url":"PreSetup://vcsa-cs.aisrp.local/dc1/ds-sr183_185_os5500-ssd-1/?ROLE=Primary&STOREUUID=ad1a78a8-0d75-3b0f-973c-05f80c990691","isManaged":"false","parentPoolType":"DatastoreCluster"}},"name":"ROOT-3052","size":"(50.00 GB) 53687091200","path":"i-60-3052-VM_2","volumeId":"3173","vmName":"i-60-3052-VM","accountId":"60","chainInfo":"{"diskDeviceBusName":"scsi0:0","diskChain":["[ds-sr183_185_os5500-ssd-1] i-60-3052-VM/i-60-3052-VM_2.vmdk"]}","format":"OVA","provisioningType":"THIN","poolId":"27","id":"3173","deviceId":"0","bytesReadRate":"(0 bytes) 0","bytesWriteRate":"(0 bytes) 0","iopsReadRate":"(0 bytes) 0","iopsWriteRate":"(0 bytes) 0","cacheMode":"NONE","hypervisorType":"VMware","directDownload":"false","deployAsIs":"false"}},"diskSeq":"0","path":"i-60-3052-VM_2","type":"ROOT","_details":{"storageHost":"vcsa-cs.aisrp.local","managed":"false","storagePort":"0","protocoltype":"DatastoreCluster","storage.pool.disk.wait":"60","volumeSize":"(50.00 GB) 53687091200"}},{"data":{"org.apache.cloudstack.storage.to.TemplateObjectTO":{"id":"0","format":"ISO","accountId":"0","hvm":"false","bootable":"false","directDownload":"false","deployAsIs":"false"}},"diskSeq":"3","type":"ISO"}],"nics":[{"deviceId":"0","networkRateMbps":"10000","defaultNic":"true","pxeDisable":"false","nicUuid":"2ef825ea-ed84-4897-8ea0-b0d1e0df3e90","details":{"ForgedTransmits":"true","internetProtocol":"IPv4","MacAddressChanges":"true","MacLearning":"false","PromiscuousMode":"false"},"dpdkEnabled":"false","uuid":"d4152505-c430-43b1-ba14-31ab9257c73b","ip":"10.1.2.99","netmask":"255.255.255.0","gateway":"10.1.2.1","mac":"02:00:7c:0b:00:6c","dns1":"93.125.22.110","dns2":"93.125.22.108","broadcastType":"Vlan","type":"Guest","isSecurityGroupEnabled":"false","name":"dsw-clusters_1-2,2100-2300,vmwaredvs"}],"configDriveLabel":"config-2","configDriveLocation":"SECONDARY","guestOsDetails":{},"extraConfig":{}},"rollback":"false","wait":"0","bypassHostMaintenance":"false"}}] }
2023-05-03 14:27:35,330 DEBUG [c.c.a.t.Request] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Seq 38-9188750614719369077: Executing:  { Cmd , MgmtId: 108597816265214, via: 38(esx-sr184-u0910.aisrp.local), Ver: v1, Flags: 100111, [{"com.cloud.agent.api.PrepareForMigrationCommand":{"vm":{"id":"3052","name":"i-60-3052-VM","bootloader":"HVM","state":"Running","type":"User","cpus":"12","minSpeed":"1000","maxSpeed":"1000","minRam":"(12.00 GB) 12884901888","maxRam":"(12.00 GB) 12884901888","hostName":"for-transfer-3","arch":"x86_64","os":"CentOS 8","platformEmulator":"centos8_64Guest","bootArgs":"","enableHA":"true","limitCpuUse":"true","enableDynamicallyScaleVm":"true","params":{"cpuNumber":"12","memory":"12288","memoryOvercommitRatio":"1.0","Message.ReservedCapacityFreed.Flag":"false","nestedVirtualizationFlag":"false","vmware.reserve.cpu":"false","dataDiskController":"scsi","cpuOvercommitRatio":"3.0","rootdisksize":"50","vmware.reserve.mem":"false","nicAdapter":"Vmxnet3","rootDiskController":"scsi"},"uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11","enterHardwareSetup":"false","disks":[{"data":{"org.apache.cloudstack.storage.to.VolumeObjectTO":{"uuid":"f8d89f88-d681-42e9-997a-9f1d93dba76a","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"ad1a78a8-0d75-3b0f-973c-05f80c990691","name":"ds-sr183_185_os5500-ssd-1","id":"27","poolType":"PreSetup","host":"vcsa-cs.aisrp.local","path":"/dc1/ds-sr183_185_os5500-ssd-1","port":"0","url":"PreSetup://vcsa-cs.aisrp.local/dc1/ds-sr183_185_os5500-ssd-1/?ROLE=Primary&STOREUUID=ad1a78a8-0d75-3b0f-973c-05f80c990691","isManaged":"false","parentPoolType":"DatastoreCluster"}},"name":"ROOT-3052","size":"(50.00 GB) 53687091200","path":"i-60-3052-VM_2","volumeId":"3173","vmName":"i-60-3052-VM","accountId":"60","chainInfo":"{"diskDeviceBusName":"scsi0:0","diskChain":["[ds-sr183_185_os5500-ssd-1] i-60-3052-VM/i-60-3052-VM_2.vmdk"]}","format":"OVA","provisioningType":"THIN","poolId":"27","id":"3173","deviceId":"0","bytesReadRate":"(0 bytes) 0","bytesWriteRate":"(0 bytes) 0","iopsReadRate":"(0 bytes) 0","iopsWriteRate":"(0 bytes) 0","cacheMode":"NONE","hypervisorType":"VMware","directDownload":"false","deployAsIs":"false"}},"diskSeq":"0","path":"i-60-3052-VM_2","type":"ROOT","_details":{"storageHost":"vcsa-cs.aisrp.local","managed":"false","storagePort":"0","protocoltype":"DatastoreCluster","storage.pool.disk.wait":"60","volumeSize":"(50.00 GB) 53687091200"}},{"data":{"org.apache.cloudstack.storage.to.TemplateObjectTO":{"id":"0","format":"ISO","accountId":"0","hvm":"false","bootable":"false","directDownload":"false","deployAsIs":"false"}},"diskSeq":"3","type":"ISO"}],"nics":[{"deviceId":"0","networkRateMbps":"10000","defaultNic":"true","pxeDisable":"false","nicUuid":"2ef825ea-ed84-4897-8ea0-b0d1e0df3e90","details":{"ForgedTransmits":"true","internetProtocol":"IPv4","MacAddressChanges":"true","MacLearning":"false","PromiscuousMode":"false"},"dpdkEnabled":"false","uuid":"d4152505-c430-43b1-ba14-31ab9257c73b","ip":"10.1.2.99","netmask":"255.255.255.0","gateway":"10.1.2.1","mac":"02:00:7c:0b:00:6c","dns1":"93.125.22.110","dns2":"93.125.22.108","broadcastType":"Vlan","type":"Guest","isSecurityGroupEnabled":"false","name":"dsw-clusters_1-2,2100-2300,vmwaredvs"}],"configDriveLabel":"config-2","configDriveLocation":"SECONDARY","guestOsDetails":{},"extraConfig":{}},"rollback":"false","wait":"0","bypassHostMaintenance":"false"}}] }
2023-05-03 14:27:35,368 INFO  [c.c.h.v.r.VmwareResource] (DirectAgent-62:ctx-94069856 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: PrepareForMigrationCommand) (logid:6b610288) Prepare network on vmwaredvs dsw-clusters_1-2 with name prefix: cloud.guest
2023-05-03 14:27:35,368 WARN  [c.c.h.v.r.VmwareResource] (DirectAgent-62:ctx-94069856 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: PrepareForMigrationCommand) (logid:6b610288) BroadcastType is not claimed as VLAN or PVLAN, but without vlan info in broadcast URI. Use vlan info from labeling: 2100-2300
2023-05-03 14:27:35,391 DEBUG [c.c.h.v.m.HypervisorHostHelper] (DirectAgent-62:ctx-94069856 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: PrepareForMigrationCommand) (logid:6b610288) Found distributed vSwitch: dsw-clusters_1-2 with product version: 6.6.0
2023-05-03 14:27:35,391 DEBUG [c.c.h.v.m.HypervisorHostHelper] (DirectAgent-62:ctx-94069856 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: PrepareForMigrationCommand) (logid:6b610288) Creating dvSwitch port vlan-trunk spec with range: 2100-2300
2023-05-03 14:27:35,397 INFO  [c.c.h.v.m.HypervisorHostHelper] (DirectAgent-62:ctx-94069856 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: PrepareForMigrationCommand) (logid:6b610288) Found Distributed Virtual Port group cloud.guest.2100-2300.10000.1-dsw-clusters_1-2
2023-05-03 14:27:35,417 DEBUG [c.c.h.v.m.HypervisorHostHelper] (DirectAgent-62:ctx-94069856 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: PrepareForMigrationCommand) (logid:6b610288) Checking if configuration of dvPortGroup [cloud.guest.2100-2300.10000.1-dsw-clusters_1-2] has changed.
2023-05-03 14:27:35,434 DEBUG [c.c.h.v.m.HypervisorHostHelper] (DirectAgent-62:ctx-94069856 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: PrepareForMigrationCommand) (logid:6b610288) Added custom field : cloud.gc.dvp
2023-05-03 14:27:35,438 DEBUG [c.c.s.StatsCollector] (DirectAgent-62:ctx-94069856 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: PrepareForMigrationCommand) (logid:6b610288) Verifying image storage [4]. Capacity: total=[10 TB], used=[137 GB], threshold=[89.99999761581421%].
2023-05-03 14:27:35,439 DEBUG [c.c.s.StatsCollector] (DirectAgent-62:ctx-94069856 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: PrepareForMigrationCommand) (logid:6b610288) Verifying image storage [4]. Capacity: total=[10 TB], used=[137 GB], threshold=[89.99999761581421%].
2023-05-03 14:27:35,463 DEBUG [c.c.a.t.Request] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Seq 38-9188750614719369077: Received:  { Ans: , MgmtId: 108597816265214, via: 38(esx-sr184-u0910.aisrp.local), Ver: v1, Flags: 110, { PrepareForMigrationAnswer } }
2023-05-03 14:27:35,479 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) VM instance {"id":3052,"instanceName":"i-60-3052-VM","type":"User","uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11"} state transited from [Running] to [Migrating] with event [MigrationRequested]. VM's original host: Host {"id":40,"name":"esx-sr183-u2324.aisrp.local","type":"Routing","uuid":"bbfeaced-6594-48b2-b4cb-80ad1d3370a2"}, new host: Host {"id":38,"name":"esx-sr184-u0910.aisrp.local","type":"Routing","uuid":"1adc2f7e-66b8-4bb0-a846-19334390c6f3"}, host before state transition: Host {"id":40,"name":"esx-sr183-u2324.aisrp.local","type":"Routing","uuid":"bbfeaced-6594-48b2-b4cb-80ad1d3370a2"}
2023-05-03 14:27:35,489 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Hosts's actual total CPU: 111720 and CPU after applying overprovisioning: 335160
2023-05-03 14:27:35,489 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) We are allocating VM, increasing the used capacity of this host:38
2023-05-03 14:27:35,489 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Current Used CPU: 13980 , Free CPU:321180 ,Requested CPU: 12000
2023-05-03 14:27:35,489 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Current Used RAM: (9.00 GB) 9663676416 , Free RAM:(246.61 GB) 264792096768 ,Requested RAM: (12.00 GB) 12884901888
2023-05-03 14:27:35,489 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) CPU STATS after allocation: for host: 38, old used: 13980, old reserved: 0, actual total: 111720, total with overprovisioning: 335160; new used:25980, reserved:0; requested cpu:12000,alloc_from_last:false
2023-05-03 14:27:35,489 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) RAM STATS after allocation: for host: 38, old used: (9.00 GB) 9663676416, old reserved: (0 bytes) 0, total: (255.61 GB) 274455773184; new used: (21.00 GB) 22548578304, reserved: (0 bytes) 0; requested mem: (12.00 GB) 12884901888,alloc_from_last:false
2023-05-03 14:27:35,491 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Host: 38 has cpu capability (cpu:56, speed:1995) to support requested CPU: 12 and requested speed: 1000
2023-05-03 14:27:35,491 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Checking if host: 38 has enough capacity for requested CPU: 12000 and requested RAM: (12.00 GB) 12884901888 , cpuOverprovisioningFactor: 3.0
2023-05-03 14:27:35,492 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Hosts's actual total CPU: 111720 and CPU after applying overprovisioning: 335160
2023-05-03 14:27:35,492 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) We need to allocate to the last host again, so checking if there is enough reserved capacity
2023-05-03 14:27:35,492 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Reserved CPU: 0 , Requested CPU: 12000
2023-05-03 14:27:35,492 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Reserved RAM: (0 bytes) 0 , Requested RAM: (12.00 GB) 12884901888
2023-05-03 14:27:35,492 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) STATS: Failed to alloc resource from host: 38 reservedCpu: 0, requested cpu: 12000, reservedMem: (0 bytes) 0, requested mem: (12.00 GB) 12884901888
2023-05-03 14:27:35,492 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Host does not have enough reserved CPU available, cannot allocate to this host.
2023-05-03 14:27:35,492 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Checking if host: 38 has enough capacity for requested CPU: 12000 and requested RAM: (12.00 GB) 12884901888 , cpuOverprovisioningFactor: 3.0
2023-05-03 14:27:35,493 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Hosts's actual total CPU: 111720 and CPU after applying overprovisioning: 335160
2023-05-03 14:27:35,493 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Free CPU: 321180 , Requested CPU: 12000
2023-05-03 14:27:35,493 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Free RAM: (246.61 GB) 264792096768 , Requested RAM: (12.00 GB) 12884901888
2023-05-03 14:27:35,493 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Host has enough CPU and RAM available
2023-05-03 14:27:35,493 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) STATS: Can alloc CPU from host: 38, used: 13980, reserved: 0, actual total: 111720, total with overprovisioning: 335160; requested cpu:12000,alloc_from_last_host?:false ,considerReservedCapacity?: true
2023-05-03 14:27:35,493 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) STATS: Can alloc MEM from host: 38, used: (9.00 GB) 9663676416, reserved: (0 bytes) 0, total: (255.61 GB) 274455773184; requested mem: (12.00 GB) 12884901888, alloc_from_last_host?: false , considerReservedCapacity?: true
2023-05-03 14:27:35,518 DEBUG [c.c.a.t.Request] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Seq 40-4836584524819205952: Sending  { Cmd , MgmtId: 108597816265214, via: 40(esx-sr183-u2324.aisrp.local), Ver: v1, Flags: 100111, [{"com.cloud.agent.api.MigrateCommand":{"vmName":"i-60-3052-VM","destIp":"10.69.107.23","migrateStorageManaged":"false","migrateNonSharedInc":"false","autoConvergence":"false","hostGuid":"HostSystem:host-334@vcsa-cs.aisrp.local","isWindows":"false","vmTO":{"id":"3052","name":"i-60-3052-VM","bootloader":"HVM","state":"Running","type":"User","cpus":"12","minSpeed":"1000","maxSpeed":"1000","minRam":"(12.00 GB) 12884901888","maxRam":"(12.00 GB) 12884901888","hostName":"for-transfer-3","arch":"x86_64","os":"CentOS 8","platformEmulator":"centos8_64Guest","bootArgs":"","enableHA":"true","limitCpuUse":"true","enableDynamicallyScaleVm":"true","params":{"cpuNumber":"12","memory":"12288","memoryOvercommitRatio":"1.0","Message.ReservedCapacityFreed.Flag":"false","nestedVirtualizationFlag":"false","vmware.reserve.cpu":"false","dataDiskController":"scsi","cpuOvercommitRatio":"3.0","rootdisksize":"50","vmware.reserve.mem":"false","nicAdapter":"Vmxnet3","rootDiskController":"scsi"},"uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11","enterHardwareSetup":"false","disks":[{"data":{"org.apache.cloudstack.storage.to.VolumeObjectTO":{"uuid":"f8d89f88-d681-42e9-997a-9f1d93dba76a","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"ad1a78a8-0d75-3b0f-973c-05f80c990691","name":"ds-sr183_185_os5500-ssd-1","id":"27","poolType":"PreSetup","host":"vcsa-cs.aisrp.local","path":"/dc1/ds-sr183_185_os5500-ssd-1","port":"0","url":"PreSetup://vcsa-cs.aisrp.local/dc1/ds-sr183_185_os5500-ssd-1/?ROLE=Primary&STOREUUID=ad1a78a8-0d75-3b0f-973c-05f80c990691","isManaged":"false","parentPoolType":"DatastoreCluster"}},"name":"ROOT-3052","size":"(50.00 GB) 53687091200","path":"i-60-3052-VM_2","volumeId":"3173","vmName":"i-60-3052-VM","accountId":"60","chainInfo":"{"diskDeviceBusName":"scsi0:0","diskChain":["[ds-sr183_185_os5500-ssd-1] i-60-3052-VM/i-60-3052-VM_2.vmdk"]}","format":"OVA","provisioningType":"THIN","poolId":"27","id":"3173","deviceId":"0","bytesReadRate":"(0 bytes) 0","bytesWriteRate":"(0 bytes) 0","iopsReadRate":"(0 bytes) 0","iopsWriteRate":"(0 bytes) 0","cacheMode":"NONE","hypervisorType":"VMware","directDownload":"false","deployAsIs":"false"}},"diskSeq":"0","path":"i-60-3052-VM_2","type":"ROOT","_details":{"storageHost":"vcsa-cs.aisrp.local","managed":"false","storagePort":"0","protocoltype":"DatastoreCluster","storage.pool.disk.wait":"60","volumeSize":"(50.00 GB) 53687091200"}},{"data":{"org.apache.cloudstack.storage.to.TemplateObjectTO":{"id":"0","format":"ISO","accountId":"0","hvm":"false","bootable":"false","directDownload":"false","deployAsIs":"false"}},"diskSeq":"3","type":"ISO"}],"nics":[{"deviceId":"0","networkRateMbps":"10000","defaultNic":"true","pxeDisable":"false","nicUuid":"2ef825ea-ed84-4897-8ea0-b0d1e0df3e90","details":{"ForgedTransmits":"true","internetProtocol":"IPv4","MacAddressChanges":"true","MacLearning":"false","PromiscuousMode":"false"},"dpdkEnabled":"false","uuid":"d4152505-c430-43b1-ba14-31ab9257c73b","ip":"10.1.2.99","netmask":"255.255.255.0","gateway":"10.1.2.1","mac":"02:00:7c:0b:00:6c","dns1":"93.125.22.110","dns2":"93.125.22.108","broadcastType":"Vlan","type":"Guest","isSecurityGroupEnabled":"false","name":"dsw-clusters_1-2,2100-2300,vmwaredvs"}],"configDriveLabel":"config-2","configDriveLocation":"SECONDARY","guestOsDetails":{},"extraConfig":{}},"executeInSequence":"true","migrateDiskInfoList":[],"dpdkInterfaceMapping":{},"vlanToPersistenceMap":{"2118":"true"},"wait":"0","bypassHostMaintenance":"false"}}] }
2023-05-03 14:27:35,521 DEBUG [c.c.a.t.Request] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Seq 40-4836584524819205952: Executing:  { Cmd , MgmtId: 108597816265214, via: 40(esx-sr183-u2324.aisrp.local), Ver: v1, Flags: 100111, [{"com.cloud.agent.api.MigrateCommand":{"vmName":"i-60-3052-VM","destIp":"10.69.107.23","migrateStorageManaged":"false","migrateNonSharedInc":"false","autoConvergence":"false","hostGuid":"HostSystem:host-334@vcsa-cs.aisrp.local","isWindows":"false","vmTO":{"id":"3052","name":"i-60-3052-VM","bootloader":"HVM","state":"Running","type":"User","cpus":"12","minSpeed":"1000","maxSpeed":"1000","minRam":"(12.00 GB) 12884901888","maxRam":"(12.00 GB) 12884901888","hostName":"for-transfer-3","arch":"x86_64","os":"CentOS 8","platformEmulator":"centos8_64Guest","bootArgs":"","enableHA":"true","limitCpuUse":"true","enableDynamicallyScaleVm":"true","params":{"cpuNumber":"12","memory":"12288","memoryOvercommitRatio":"1.0","Message.ReservedCapacityFreed.Flag":"false","nestedVirtualizationFlag":"false","vmware.reserve.cpu":"false","dataDiskController":"scsi","cpuOvercommitRatio":"3.0","rootdisksize":"50","vmware.reserve.mem":"false","nicAdapter":"Vmxnet3","rootDiskController":"scsi"},"uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11","enterHardwareSetup":"false","disks":[{"data":{"org.apache.cloudstack.storage.to.VolumeObjectTO":{"uuid":"f8d89f88-d681-42e9-997a-9f1d93dba76a","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"ad1a78a8-0d75-3b0f-973c-05f80c990691","name":"ds-sr183_185_os5500-ssd-1","id":"27","poolType":"PreSetup","host":"vcsa-cs.aisrp.local","path":"/dc1/ds-sr183_185_os5500-ssd-1","port":"0","url":"PreSetup://vcsa-cs.aisrp.local/dc1/ds-sr183_185_os5500-ssd-1/?ROLE=Primary&STOREUUID=ad1a78a8-0d75-3b0f-973c-05f80c990691","isManaged":"false","parentPoolType":"DatastoreCluster"}},"name":"ROOT-3052","size":"(50.00 GB) 53687091200","path":"i-60-3052-VM_2","volumeId":"3173","vmName":"i-60-3052-VM","accountId":"60","chainInfo":"{"diskDeviceBusName":"scsi0:0","diskChain":["[ds-sr183_185_os5500-ssd-1] i-60-3052-VM/i-60-3052-VM_2.vmdk"]}","format":"OVA","provisioningType":"THIN","poolId":"27","id":"3173","deviceId":"0","bytesReadRate":"(0 bytes) 0","bytesWriteRate":"(0 bytes) 0","iopsReadRate":"(0 bytes) 0","iopsWriteRate":"(0 bytes) 0","cacheMode":"NONE","hypervisorType":"VMware","directDownload":"false","deployAsIs":"false"}},"diskSeq":"0","path":"i-60-3052-VM_2","type":"ROOT","_details":{"storageHost":"vcsa-cs.aisrp.local","managed":"false","storagePort":"0","protocoltype":"DatastoreCluster","storage.pool.disk.wait":"60","volumeSize":"(50.00 GB) 53687091200"}},{"data":{"org.apache.cloudstack.storage.to.TemplateObjectTO":{"id":"0","format":"ISO","accountId":"0","hvm":"false","bootable":"false","directDownload":"false","deployAsIs":"false"}},"diskSeq":"3","type":"ISO"}],"nics":[{"deviceId":"0","networkRateMbps":"10000","defaultNic":"true","pxeDisable":"false","nicUuid":"2ef825ea-ed84-4897-8ea0-b0d1e0df3e90","details":{"ForgedTransmits":"true","internetProtocol":"IPv4","MacAddressChanges":"true","MacLearning":"false","PromiscuousMode":"false"},"dpdkEnabled":"false","uuid":"d4152505-c430-43b1-ba14-31ab9257c73b","ip":"10.1.2.99","netmask":"255.255.255.0","gateway":"10.1.2.1","mac":"02:00:7c:0b:00:6c","dns1":"93.125.22.110","dns2":"93.125.22.108","broadcastType":"Vlan","type":"Guest","isSecurityGroupEnabled":"false","name":"dsw-clusters_1-2,2100-2300,vmwaredvs"}],"configDriveLabel":"config-2","configDriveLocation":"SECONDARY","guestOsDetails":{},"extraConfig":{}},"executeInSequence":"true","migrateDiskInfoList":[],"dpdkInterfaceMapping":{},"vlanToPersistenceMap":{"2118":"true"},"wait":"0","bypassHostMaintenance":"false"}}] }
2023-05-03 14:27:35,614 INFO  [c.c.h.v.u.VmwareHelper] (DirectAgent-183:ctx-82de3964 esx-sr183-u2324.aisrp.local, job-36271/job-36272, cmd: MigrateCommand) (logid:6b610288) [ignored]failed to get message for exception: Unable to find management port group Management Network
2023-05-03 14:27:35,614 ERROR [c.c.h.v.r.VmwareResource] (DirectAgent-183:ctx-82de3964 esx-sr183-u2324.aisrp.local, job-36271/job-36272, cmd: MigrateCommand) (logid:6b610288) MigrateCommand failed due to [Exception: java.lang.Exception
2023-05-03 14:27:35,615 DEBUG [c.c.a.t.Request] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Seq 40-4836584524819205952: Received:  { Ans: , MgmtId: 108597816265214, via: 40(esx-sr183-u2324.aisrp.local), Ver: v1, Flags: 110, { MigrateAnswer } }
2023-05-03 14:27:35,615 INFO  [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Migration was unsuccessful.  Cleaning up: VM instance {"id":3052,"instanceName":"i-60-3052-VM","type":"User","uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11"}
2023-05-03 14:27:35,630 DEBUG [o.a.c.e.o.VolumeOrchestrator] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Releasing [1] volumes for VM [VM instance {"id":3052,"instanceName":"i-60-3052-VM","type":"User","uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11"}] from host [Host {"id":38,"name":"esx-sr184-u0910.aisrp.local","type":"Routing","uuid":"1adc2f7e-66b8-4bb0-a846-19334390c6f3"}].
2023-05-03 14:27:35,639 WARN  [c.c.a.AlertManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) alertType=[17] dataCenterId=[5] podId=[6] clusterId=[null] message=[Unable to migrate vm i-60-3052-VM from host esx-sr183-u2324.aisrp.local in zone zone2-vmware and pod pod-sr183_185].
2023-05-03 14:27:35,931 DEBUG [c.c.a.AlertManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Have already sent: 1 emails for alert type '17' -- skipping send email
2023-05-03 14:27:35,933 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) VM instance {"id":3052,"instanceName":"i-60-3052-VM","type":"User","uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11"} is a [User], returning null for control Nic IP.
2023-05-03 14:27:35,943 DEBUG [c.c.a.m.AgentAttache] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Seq 38-9188750614719369078: Routed from 108597816265214
2023-05-03 14:27:35,943 DEBUG [c.c.a.t.Request] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Seq 38-9188750614719369078: Sending  { Cmd , MgmtId: 108597816265214, via: 38(esx-sr184-u0910.aisrp.local), Ver: v1, Flags: 100111, [{"com.cloud.agent.api.StopCommand":{"isProxy":"false","checkBeforeCleanup":"false","forceStop":"false","vlanToPersistenceMap":{"2118":"true"},"volumesToDisconnect":[],"vmName":"i-60-3052-VM","executeInSequence":"true","wait":"0","bypassHostMaintenance":"false"}}] }
2023-05-03 14:27:35,943 DEBUG [c.c.a.t.Request] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Seq 38-9188750614719369078: Executing:  { Cmd , MgmtId: 108597816265214, via: 38(esx-sr184-u0910.aisrp.local), Ver: v1, Flags: 100111, [{"com.cloud.agent.api.StopCommand":{"isProxy":"false","checkBeforeCleanup":"false","forceStop":"false","vlanToPersistenceMap":{"2118":"true"},"volumesToDisconnect":[],"vmName":"i-60-3052-VM","executeInSequence":"true","wait":"0","bypassHostMaintenance":"false"}}] }
2023-05-03 14:27:35,952 DEBUG [c.c.h.v.m.HostMO] (DirectAgent-318:ctx-8709eaa5 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: StopCommand) (logid:6b610288) find VM i-60-3052-VM on host
2023-05-03 14:27:35,953 INFO  [c.c.h.v.m.HostMO] (DirectAgent-318:ctx-8709eaa5 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: StopCommand) (logid:6b610288) VM i-60-3052-VM not found in host cache
2023-05-03 14:27:35,953 DEBUG [c.c.h.v.m.HostMO] (DirectAgent-318:ctx-8709eaa5 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: StopCommand) (logid:6b610288) load VM cache on host
2023-05-03 14:27:35,955 DEBUG [c.c.n.NetworkModelImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Service SecurityGroup is not supported in the network id=829
2023-05-03 14:27:35,964 INFO  [c.c.h.v.r.VmwareResource] (DirectAgent-318:ctx-8709eaa5 esx-sr184-u0910.aisrp.local, job-36271/job-36272, cmd: StopCommand) (logid:6b610288) VM i-60-3052-VM is no longer on the expected host in vSphere
2023-05-03 14:27:35,990 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) VM instance {"id":3052,"instanceName":"i-60-3052-VM","type":"User","uuid":"38f58a9b-7e73-44ab-b228-5b49906a4a11"} state transited from [Migrating] to [Running] with event [OperationFailed]. VM's original host: Host {"id":40,"name":"esx-sr183-u2324.aisrp.local","type":"Routing","uuid":"bbfeaced-6594-48b2-b4cb-80ad1d3370a2"}, new host: Host {"id":40,"name":"esx-sr183-u2324.aisrp.local","type":"Routing","uuid":"bbfeaced-6594-48b2-b4cb-80ad1d3370a2"}, host before state transition: Host {"id":38,"name":"esx-sr184-u0910.aisrp.local","type":"Routing","uuid":"1adc2f7e-66b8-4bb0-a846-19334390c6f3"}
2023-05-03 14:27:35,998 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Hosts's actual total CPU: 111720 and CPU after applying overprovisioning: 335160
2023-05-03 14:27:35,999 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Hosts's actual total RAM: (255.61 GB) 274455769088 and RAM after applying overprovisioning: (255.61 GB) 274455773184
2023-05-03 14:27:35,999 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) release cpu from host: 38, old used: 25980,reserved: 0, actual total: 111720, total with overprovisioning: 335160; new used: 13980,reserved:0; movedfromreserved: false,moveToReserveredfalse
2023-05-03 14:27:35,999 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) release mem from host: 38, old used: (21.00 GB) 22548578304,reserved: (0 bytes) 0, total: (255.61 GB) 274455773184; new used: (9.00 GB) 9663676416,reserved:(0 bytes) 0; movedfromreserved: false,moveToReserveredfalse
2023-05-03 14:27:36,053 ERROR [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Invocation exception, caused by: com.cloud.utils.exception.CloudRuntimeException: MigrateCommand failed due to [Exception: java.lang.Exception
Message: Unable to find management port group Management Network
].
2023-05-03 14:27:36,053 INFO  [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272 ctx-30c0ef1d) (logid:6b610288) Rethrow exception com.cloud.utils.exception.CloudRuntimeException: MigrateCommand failed due to [Exception: java.lang.Exception
Message: Unable to find management port group Management Network
].
2023-05-03 14:27:36,053 DEBUG [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Done with run of VM work job: com.cloud.vm.VmWorkMigrate for VM 3052, job origin: 36271
2023-05-03 14:27:36,053 ERROR [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Unable to complete AsyncJobVO: {id:36272, userId: 63, accountId: 60, instanceType: null, instanceId: null, cmd: com.cloud.vm.VmWorkMigrate, cmdInfo: rO0ABXNyABpjb20uY2xvdWQudm0uVm1Xb3JrTWlncmF0ZRdxQXtPtzYqAgAGSgAJc3JjSG9zdElkTAAJY2x1c3RlcklkdAAQTGphdmEvbGFuZy9Mb25nO0wABmhvc3RJZHEAfgABTAAFcG9kSWRxAH4AAUwAB3N0b3JhZ2V0AA9MamF2YS91dGlsL01hcDtMAAZ6b25lSWRxAH4AAXhyABNjb20uY2xvdWQudm0uVm1Xb3Jrn5m2VvAlZ2sCAARKAAlhY2NvdW50SWRKAAZ1c2VySWRKAAR2bUlkTAALaGFuZGxlck5hbWV0ABJMamF2YS9sYW5nL1N0cmluZzt4cAAAAAAAAAA8AAAAAAAAAD8AAAAAAAAL7HQAGVZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwAAAAAAAAAKHNyAA5qYXZhLmxhbmcuTG9uZzuL5JDMjyPfAgABSgAFdmFsdWV4cgAQamF2YS5sYW5nLk51bWJlcoaslR0LlOCLAgAAeHAAAAAAAAAACXNxAH4ABwAAAAAAAAAmc3EAfgAHAAAAAAAAAAZwc3EAfgAHAAAAAAAAAAU, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 108597816265214, completeMsid: null, lastUpdated: null, lastPolled: null, created: Wed May 03 14:27:33 MSK 2023, removed: null}, job origin:36271
Message: Unable to find management port group Management Network
].
2023-05-03 14:27:36,053 DEBUG [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Done with run of VM work job: com.cloud.vm.VmWorkMigrate for VM 3052, job origin: 36271
2023-05-03 14:27:36,053 ERROR [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Unable to complete AsyncJobVO: {id:36272, userId: 63, accountId: 60, instanceType: null, instanceId: null, cmd: com.cloud.vm.VmWorkMigrate, cmdInfo: rO0ABXNyABpjb20uY2xvdWQudm0uVm1Xb3JrTWlncmF0ZRdxQXtPtzYqAgAGSgAJc3JjSG9zdElkTAAJY2x1c3RlcklkdAAQTGphdmEvbGFuZy9Mb25nO0wABmhvc3RJZHEAfgABTAAFcG9kSWRxAH4AAUwAB3N0b3JhZ2V0AA9MamF2YS91dGlsL01hcDtMAAZ6b25lSWRxAH4AAXhyABNjb20uY2xvdWQudm0uVm1Xb3Jrn5m2VvAlZ2sCAARKAAlhY2NvdW50SWRKAAZ1c2VySWRKAAR2bUlkTAALaGFuZGxlck5hbWV0ABJMamF2YS9sYW5nL1N0cmluZzt4cAAAAAAAAAA8AAAAAAAAAD8AAAAAAAAL7HQAGVZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwAAAAAAAAAKHNyAA5qYXZhLmxhbmcuTG9uZzuL5JDMjyPfAgABSgAFdmFsdWV4cgAQamF2YS5sYW5nLk51bWJlcoaslR0LlOCLAgAAeHAAAAAAAAAACXNxAH4ABwAAAAAAAAAmc3EAfgAHAAAAAAAAAAZwc3EAfgAHAAAAAAAAAAU, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 108597816265214, completeMsid: null, lastUpdated: null, lastPolled: null, created: Wed May 03 14:27:33 MSK 2023, removed: null}, job origin:36271
com.cloud.utils.exception.CloudRuntimeException: MigrateCommand failed due to [Exception: java.lang.Exception
Message: Unable to find management port group Management Network
].
        at com.cloud.vm.VirtualMachineManagerImpl.migrate(VirtualMachineManagerImpl.java:2734)
        at com.cloud.vm.VirtualMachineManagerImpl.orchestrateMigrate(VirtualMachineManagerImpl.java:2598)
        at com.cloud.vm.VirtualMachineManagerImpl.orchestrateMigrate(VirtualMachineManagerImpl.java:5378)
        at jdk.internal.reflect.GeneratedMethodAccessor2594.invoke(Unknown Source)
        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
        at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:5475)
        at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
        at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:620)
        at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:48)
        at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55)
        at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:102)
        at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52)
        at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:45)
        at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:568)
        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base/java.lang.Thread.run(Thread.java:829)
2023-05-03 14:27:36,061 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Complete async job-36272, jobStatus: FAILED, resultCode: 0, result: rO0ABXNyAC9jb20uY2xvdWQudXRpbHMuZXhjZXB0aW9uLkNsb3VkUnVudGltZUV4Y2VwdGlvbgAAAABWTU9yAwABSQALY3NFcnJvckNvZGV4cgAaamF2YS5sYW5nLlJ1bnRpbWVFeGNlcHRpb26eXwZHCjSD5QIAAHhyABNqYXZhLmxhbmcuRXhjZXB0aW9u0P0fPho7HMQCAAB4cgATamF2YS5sYW5nLlRocm93YWJsZdXGNSc5d7jLAwAETAAFY2F1c2V0ABVMamF2YS9sYW5nL1Rocm93YWJsZTtMAA1kZXRhaWxNZXNzYWdldAASTGphdmEvbGFuZy9TdHJpbmc7WwAKc3RhY2tUcmFjZXQAHltMamF2YS9sYW5nL1N0YWNrVHJhY2VFbGVtZW50O0wAFHN1cHByZXNzZWRFeGNlcHRpb25zdAAQTGphdmEvdXRpbC9MaXN0O3hwcQB-AAh0AIBNaWdyYXRlQ29tbWFuZCBmYWlsZWQgZHVlIHRvIFtFeGNlcHRpb246IGphdmEubGFuZy5FeGNlcHRpb24KTWVzc2FnZTogVW5hYmxlIHRvIGZpbmQgbWFuYWdlbWVudCBwb3J0IGdyb3VwIE1hbmFnZW1lbnQgTmV0d29yawpdLnVyAB5bTGphdmEubGFuZy5TdGFja1RyYWNlRWxlbWVudDsCRio8PP0iOQIAAHhwAAAAFXNyABtqYXZhLmxhbmcuU3RhY2tUcmFjZUVsZW1lbnRhCcWaJjbdhQIACEIABmZvcm1hdEkACmxpbmVOdW1iZXJMAA9jbGFzc0xvYWRlck5hbWVxAH4ABUwADmRlY2xhcmluZ0NsYXNzcQB-AAVMAAhmaWxlTmFtZXEAfgAFTAAKbWV0aG9kTmFtZXEAfgAFTAAKbW9kdWxlTmFtZXEAfgAFTAANbW9kdWxlVmVyc2lvbnEAfgAFeHABAAAKrnQAA2FwcHQAJmNvbS5jbG91ZC52bS5WaXJ0dWFsTWFjaGluZU1hbmFnZXJJbXBsdAAeVmlydHVhbE1hY2hpbmVNYW5hZ2VySW1wbC5qYXZhdAAHbWlncmF0ZXBwc3EAfgAMAQAACiZxAH4ADnEAfgAPcQB-ABB0ABJvcmNoZXN0cmF0ZU1pZ3JhdGVwcHNxAH4ADAEAABUCcQB-AA5xAH4AD3EAfgAQcQB-ABNwcHNxAH4ADAD_____cHQAMGpkay5pbnRlcm5hbC5yZWZsZWN0LkdlbmVyYXRlZE1ldGhvZEFjY2Vzc29yMjU5NHB0AAZpbnZva2VwcHNxAH4ADAIAAAArcHQAMWpkay5pbnRlcm5hbC5yZWZsZWN0LkRlbGVnYXRpbmdNZXRob2RBY2Nlc3NvckltcGx0ACFEZWxlZ2F0aW5nTWV0aG9kQWNjZXNzb3JJbXBsLmphdmFxAH4AF3QACWphdmEuYmFzZXQABzExLjAuMThzcQB-AAwCAAACNnB0ABhqYXZhLmxhbmcucmVmbGVjdC5NZXRob2R0AAtNZXRob2QuamF2YXEAfgAXcQB-ABtxAH4AHHNxAH4ADAEAAABrcQB-AA50ACJjb20uY2xvdWQudm0uVm1Xb3JrSm9iSGFuZGxlclByb3h5dAAaVm1Xb3JrSm9iSGFuZGxlclByb3h5LmphdmF0AA9oYW5kbGVWbVdvcmtKb2JwcHNxAH4ADAEAABVjcQB-AA5xAH4AD3EAfgAQcQB-ACNwcHNxAH4ADAEAAABmcQB-AA50ACBjb20uY2xvdWQudm0uVm1Xb3JrSm9iRGlzcGF0Y2hlcnQAGFZtV29ya0pvYkRpc3BhdGNoZXIuamF2YXQABnJ1bkpvYnBwc3EAfgAMAQAAAmxxAH4ADnQAP29yZy5hcGFjaGUuY2xvdWRzdGFjay5mcmFtZXdvcmsuam9icy5pbXBsLkFzeW5jSm9iTWFuYWdlckltcGwkNXQAGEFzeW5jSm9iTWFuYWdlckltcGwuamF2YXQADHJ1bkluQ29udGV4dHBwc3EAfgAMAQAAADBxAH4ADnQAPm9yZy5hcGFjaGUuY2xvdWRzdGFjay5tYW5hZ2VkLmNvbnRleHQuTWFuYWdlZENvbnRleHRSdW5uYWJsZSQxdAAbTWFuYWdlZENvbnRleHRSdW5uYWJsZS5qYXZhdAADcnVucHBzcQB-AAwBAAAAN3EAfgAOdABCb3JnLmFwYWNoZS5jbG91ZHN0YWNrLm1hbmFnZWQuY29udGV4dC5pbXBsLkRlZmF1bHRNYW5hZ2VkQ29udGV4dCQxdAAaRGVmYXVsdE1hbmFnZWRDb250ZXh0LmphdmF0AARjYWxscHBzcQB-AAwBAAAAZnEAfgAOdABAb3JnLmFwYWNoZS5jbG91ZHN0YWNrLm1hbmFnZWQuY29udGV4dC5pbXBsLkRlZmF1bHRNYW5hZ2VkQ29udGV4dHEAfgAzdAAPY2FsbFdpdGhDb250ZXh0cHBzcQB-AAwBAAAANHEAfgAOcQB-ADZxAH4AM3QADnJ1bldpdGhDb250ZXh0cHBzcQB-AAwBAAAALXEAfgAOdAA8b3JnLmFwYWNoZS5jbG91ZHN0YWNrLm1hbmFnZWQuY29udGV4dC5NYW5hZ2VkQ29udGV4dFJ1bm5hYmxlcQB-AC9xAH4AMHBwc3EAfgAMAQAAAjhxAH4ADnEAfgAqcQB-ACtxAH4AMHBwc3EAfgAMAgAAAgNwdAAuamF2YS51dGlsLmNvbmN1cnJlbnQuRXhlY3V0b3JzJFJ1bm5hYmxlQWRhcHRlcnQADkV4ZWN1dG9ycy5qYXZhcQB-ADRxAH4AG3EAfgAcc3EAfgAMAgAAAQhwdAAfamF2YS51dGlsLmNvbmN1cnJlbnQuRnV0dXJlVGFza3QAD0Z1dHVyZVRhc2suamF2YXEAfgAwcQB-ABtxAH4AHHNxAH4ADAIAAARocHQAJ2phdmEudXRpbC5jb25jdXJyZW50LlRocmVhZFBvb2xFeGVjdXRvcnQAF1RocmVhZFBvb2xFeGVjdXRvci5qYXZhdAAJcnVuV29ya2VycQB-ABtxAH4AHHNxAH4ADAIAAAJ0cHQALmphdmEudXRpbC5jb25jdXJyZW50LlRocmVhZFBvb2xFeGVjdXRvciRXb3JrZXJxAH4ARXEAfgAwcQB-ABtxAH4AHHNxAH4ADAIAAAM9cHQAEGphdmEubGFuZy5UaHJlYWR0AAtUaHJlYWQuamF2YXEAfgAwcQB-ABtxAH4AHHNyAB9qYXZhLnV0aWwuQ29sbGVjdGlvbnMkRW1wdHlMaXN0ergXtDynnt4CAAB4cHgAABCadwgAAAAAAAAAAHg
2023-05-03 14:27:36,063 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Publish async job-36272 complete on message bus
2023-05-03 14:27:36,063 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Wake up jobs related to job-36272
2023-05-03 14:27:36,063 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Update db status for job-36272
2023-05-03 14:27:36,066 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Wake up jobs joined with job-36272 and disjoin all subjobs created from job- 36272
2023-05-03 14:27:36,091 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Done executing com.cloud.vm.VmWorkMigrate for job-36272
2023-05-03 14:27:36,093 INFO  [o.a.c.f.j.i.AsyncJobMonitor] (Work-Job-Executor-80:ctx-4c12042d job-36271/job-36272) (logid:6b610288) Remove job-36272 from job monitoring